Skip to content

Commit 32fc399

Browse files
himsnghhimsngh
authored
feat(apiserver): implement APIService caBundle reconciliation (#1808)
* feat(apiserver): implement APIService caBundle reconciliation Signed-off-by: Himanshu Singh <himansh.singh3@gmail.com> * chore: resolve review comments Signed-off-by: Himanshu Singh <himansh.singh3@gmail.com> * chore: add vendor modules.txt changes Signed-off-by: Himanshu Singh <himansh.singh3@gmail.com> * update golangci-lint version to 2.11 Signed-off-by: Himanshu Singh <himansh.singh3@gmail.com> * go mod vendor changes Signed-off-by: Himanshu Singh <himansh.singh3@gmail.com> --------- Signed-off-by: Himanshu Singh <himansh.singh3@gmail.com>
1 parent 46eb34c commit 32fc399

15 files changed

Lines changed: 818 additions & 23 deletions

File tree

‎.github/workflows/golangci-lint.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,5 +28,5 @@ jobs:
2828
- name: golangci-lint
2929
uses: golangci/golangci-lint-action@v7
3030
with:
31-
version: v2.4
31+
version: v2.11
3232
args: -v

‎pkg/apiserver/apiserver.go‎

Lines changed: 54 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
package apiserver
55

66
import (
7+
"bytes"
78
"context"
89
"fmt"
910
"net"
@@ -35,6 +36,7 @@ import (
3536
"k8s.io/client-go/discovery"
3637
"k8s.io/client-go/kubernetes"
3738
"k8s.io/client-go/rest"
39+
"k8s.io/client-go/util/retry"
3840
apiregv1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1"
3941
aggregatorclient "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset"
4042
)
@@ -49,6 +51,8 @@ const (
4951
kappctrlSVCEnvKey = "KAPPCTRL_SYSTEM_SERVICE"
5052

5153
apiServiceName = "v1alpha1.data.packaging.carvel.dev"
54+
55+
apiServiceReconcileInterval = 30 * time.Second
5256
)
5357

5458
var (
@@ -107,7 +111,7 @@ func NewAPIServer(clientConfig *rest.Config, coreClient kubernetes.Interface, kc
107111
return nil, fmt.Errorf("building aggregation client: %v", err)
108112
}
109113

110-
config, err := newServerConfig(aggClient, opts)
114+
config, caContentProvider, err := newServerConfig(aggClient, opts)
111115
if err != nil {
112116
return nil, err
113117
}
@@ -117,6 +121,29 @@ func NewAPIServer(clientConfig *rest.Config, coreClient kubernetes.Interface, kc
117121
return nil, err
118122
}
119123

124+
// Register the PostStartHook to reconcile the CA Bundle
125+
if err := server.AddPostStartHook("apiservice-ca-reconciler", func(hookContext genericapiserver.PostStartHookContext) error {
126+
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
127+
defer cancel()
128+
if err := updateAPIService(ctx, opts.Logger, aggClient, caContentProvider); err != nil {
129+
opts.Logger.Error(err, "Initial APIService CA sync failed")
130+
return err
131+
}
132+
133+
// Background Reconciliation
134+
go wait.Until(func() {
135+
ctx, syncCancel := context.WithTimeout(context.Background(), 10*time.Second)
136+
defer syncCancel()
137+
if err := updateAPIService(ctx, opts.Logger, aggClient, caContentProvider); err != nil {
138+
opts.Logger.Error(err, "Background APIService CA reconciliation failed")
139+
}
140+
}, apiServiceReconcileInterval, hookContext.StopCh)
141+
142+
return nil
143+
}); err != nil {
144+
return nil, fmt.Errorf("error registering APIService CA reconciler hook: %v", err)
145+
}
146+
120147
packageMetadatasStorage := packagerest.NewPackageMetadataCRDREST(kcClient, coreClient, opts.GlobalNamespace)
121148
packageStorage := packagerest.NewPackageCRDREST(kcClient, coreClient, opts.GlobalNamespace, opts.Logger)
122149

@@ -168,7 +195,7 @@ func (as *APIServer) isReady() (bool, error) {
168195
return false, nil
169196
}
170197

171-
func newServerConfig(aggClient aggregatorclient.Interface, opts NewAPIServerOpts) (*genericapiserver.RecommendedConfig, error) {
198+
func newServerConfig(aggClient aggregatorclient.Interface, opts NewAPIServerOpts) (*genericapiserver.RecommendedConfig, *dynamiccertificates.DynamicFileCAContent, error) {
172199

173200
recommendedOptions := genericoptions.NewRecommendedOptions("", Codecs.LegacyCodec(v1alpha1.SchemeGroupVersion))
174201
recommendedOptions.Etcd = nil
@@ -180,26 +207,22 @@ func newServerConfig(aggClient aggregatorclient.Interface, opts NewAPIServerOpts
180207

181208
// ports below 1024 are probably the wrong port, see https://en.wikipedia.org/wiki/List_of_TCP_and_UDP_port_numbers#Well-known_ports
182209
if opts.BindPort < 1024 {
183-
return nil, fmt.Errorf("error initializing API Port to %v - try passing a port above 1023", opts.BindPort)
210+
return nil, nil, fmt.Errorf("error initializing API Port to %v - try passing a port above 1023", opts.BindPort)
184211
}
185212
recommendedOptions.SecureServing.BindPort = opts.BindPort
186213

187214
if err := recommendedOptions.SecureServing.MaybeDefaultWithSelfSignedCerts("kapp-controller", []string{apiServiceEndoint()}, []net.IP{net.ParseIP("127.0.0.1")}); err != nil {
188-
return nil, fmt.Errorf("error creating self-signed certificates: %v", err)
215+
return nil, nil, fmt.Errorf("error creating self-signed certificates: %v", err)
189216
}
190217

191218
caContentProvider, err := dynamiccertificates.NewDynamicCAContentFromFile("self-signed cert", recommendedOptions.SecureServing.ServerCert.CertKey.CertFile)
192219
if err != nil {
193-
return nil, fmt.Errorf("error reading self-signed CA certificate: %v", err)
194-
}
195-
196-
if err := updateAPIService(opts.Logger, aggClient, caContentProvider); err != nil {
197-
return nil, fmt.Errorf("error updating api service with generated certs: %v", err)
220+
return nil, nil, fmt.Errorf("error reading self-signed CA certificate: %v", err)
198221
}
199222

200223
serverVersion, err := getServerVersion(aggClient.Discovery())
201224
if err != nil {
202-
return nil, err
225+
return nil, nil, err
203226
}
204227

205228
// this feature gate is not enabled in k8s <1.29 as the
@@ -208,7 +231,7 @@ func newServerConfig(aggClient aggregatorclient.Interface, opts NewAPIServerOpts
208231
// so the best we can do for older k8s clusters is to allow it to be disabled.
209232
minSupportedVersionForAPF, err := semver.New("1.29.0")
210233
if err != nil {
211-
return nil, err
234+
return nil, nil, err
212235
}
213236
isServerVerLTminSupportedVer := serverVersion.LT(*minSupportedVersionForAPF)
214237
if !opts.EnableAPIPriorityAndFairness || isServerVerLTminSupportedVer {
@@ -223,7 +246,7 @@ func newServerConfig(aggClient aggregatorclient.Interface, opts NewAPIServerOpts
223246
// However, we will still run namespaceLifecycle, mutatingAdmissionWebhook and validatingAdmissionWebhooks.
224247
minSupportedVersionForValidatingAdmissionPolicy, err := semver.New("1.30.0")
225248
if err != nil {
226-
return nil, err
249+
return nil, nil, err
227250
}
228251
isServerVerLTminSupportedVer = serverVersion.LT(*minSupportedVersionForValidatingAdmissionPolicy)
229252
if isServerVerLTminSupportedVer {
@@ -248,10 +271,10 @@ func newServerConfig(aggClient aggregatorclient.Interface, opts NewAPIServerOpts
248271
serverConfig.OpenAPIConfig.Info.Version = "v1alpha1"
249272

250273
if err := recommendedOptions.ApplyTo(serverConfig); err != nil {
251-
return nil, err
274+
return nil, nil, err
252275
}
253276

254-
return serverConfig, nil
277+
return serverConfig, caContentProvider, nil
255278
}
256279

257280
func getServerVersion(discoveryClient discovery.DiscoveryInterface) (semver.Version, error) {
@@ -268,14 +291,23 @@ func getServerVersion(discoveryClient discovery.DiscoveryInterface) (semver.Vers
268291
return retv, nil
269292
}
270293

271-
func updateAPIService(logger logr.Logger, client aggregatorclient.Interface, caProvider dynamiccertificates.CAContentProvider) error {
272-
logger.Info("Syncing CA certificate with APIServices")
273-
apiService, err := client.ApiregistrationV1().APIServices().Get(context.TODO(), apiServiceName, metav1.GetOptions{})
274-
if err != nil {
275-
return fmt.Errorf("error getting APIService %s: %v", apiServiceName, err)
276-
}
277-
apiService.Spec.CABundle = caProvider.CurrentCABundleContent()
278-
if _, err := client.ApiregistrationV1().APIServices().Update(context.TODO(), apiService, metav1.UpdateOptions{}); err != nil {
294+
func updateAPIService(ctx context.Context, logger logr.Logger, client aggregatorclient.Interface, caProvider dynamiccertificates.CAContentProvider) error {
295+
if err := retry.RetryOnConflict(retry.DefaultRetry, func() error {
296+
apiService, err := client.ApiregistrationV1().APIServices().Get(ctx, apiServiceName, metav1.GetOptions{})
297+
if err != nil {
298+
return fmt.Errorf("error getting APIService %s: %v", apiServiceName, err)
299+
}
300+
301+
caBundle := caProvider.CurrentCABundleContent()
302+
if bytes.Equal(apiService.Spec.CABundle, caBundle) {
303+
return nil
304+
}
305+
306+
logger.Info("Syncing CA certificate with APIServices")
307+
apiService.Spec.CABundle = caBundle
308+
_, err = client.ApiregistrationV1().APIServices().Update(ctx, apiService, metav1.UpdateOptions{})
309+
return err
310+
}); err != nil {
279311
return fmt.Errorf("error updating kapp-controller CA cert of APIService %s: %v", apiServiceName, err)
280312
}
281313
return nil

‎pkg/apiserver/apiserver_test.go‎

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
// Copyright 2026 The Carvel Authors.
2+
// SPDX-License-Identifier: Apache-2.0
3+
4+
package apiserver
5+
6+
import (
7+
"context"
8+
"crypto/x509"
9+
"testing"
10+
11+
"github.com/go-logr/logr"
12+
"github.com/stretchr/testify/require"
13+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
14+
"k8s.io/apiserver/pkg/server/dynamiccertificates"
15+
clienttesting "k8s.io/client-go/testing"
16+
apiregv1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1"
17+
fakeaggregator "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/fake"
18+
)
19+
20+
// fakeCAProvider implements dynamiccertificates.CAContentProvider
21+
type fakeCAProvider struct {
22+
bundle []byte
23+
}
24+
25+
func (f *fakeCAProvider) Name() string { return "fake-ca-provider" }
26+
func (f *fakeCAProvider) CurrentCABundleContent() []byte { return f.bundle }
27+
func (f *fakeCAProvider) AddListener(_ dynamiccertificates.Listener) {}
28+
func (f *fakeCAProvider) VerifyOptions() (x509.VerifyOptions, bool) {
29+
return x509.VerifyOptions{}, false
30+
}
31+
32+
func Test_updateAPIService(t *testing.T) {
33+
logger := logr.Discard()
34+
35+
tests := []struct {
36+
name string
37+
existingBundle []byte
38+
newBundle []byte
39+
expectUpdate bool
40+
}{
41+
{
42+
name: "updates APIService when CA bundle is different",
43+
existingBundle: []byte("old-dead-pod-cert"),
44+
newBundle: []byte("new-active-pod-cert"),
45+
expectUpdate: true,
46+
},
47+
{
48+
name: "does nothing when CA bundle is identical",
49+
existingBundle: []byte("active-pod-cert"),
50+
newBundle: []byte("active-pod-cert"),
51+
expectUpdate: false,
52+
},
53+
}
54+
55+
for _, tc := range tests {
56+
t.Run(tc.name, func(t *testing.T) {
57+
apiSvc := &apiregv1.APIService{
58+
ObjectMeta: metav1.ObjectMeta{Name: apiServiceName},
59+
Spec: apiregv1.APIServiceSpec{
60+
CABundle: tc.existingBundle,
61+
},
62+
}
63+
fakeClient := fakeaggregator.NewSimpleClientset(apiSvc)
64+
65+
fakeProvider := &fakeCAProvider{bundle: tc.newBundle}
66+
67+
err := updateAPIService(context.TODO(), logger, fakeClient, fakeProvider)
68+
require.NoError(t, err)
69+
70+
actions := fakeClient.Actions()
71+
require.GreaterOrEqual(t, len(actions), 1, "expected at least a GET action")
72+
require.Equal(t, "get", actions[0].GetVerb())
73+
74+
var updateActionFound bool
75+
for _, action := range actions {
76+
if action.GetVerb() == "update" {
77+
updateActionFound = true
78+
updateAction, ok := action.(clienttesting.UpdateAction)
79+
if !ok {
80+
t.Fatalf("Expected UpdateAction, got %T", action)
81+
}
82+
updatedSvc := updateAction.GetObject().(*apiregv1.APIService)
83+
require.Equal(t, tc.newBundle, updatedSvc.Spec.CABundle)
84+
}
85+
}
86+
87+
if tc.expectUpdate {
88+
require.True(t, updateActionFound, "expected an UPDATE action to be executed, but none was found")
89+
} else {
90+
require.False(t, updateActionFound, "expected NO UPDATE action, but one was executed")
91+
}
92+
})
93+
}
94+
}

‎vendor/k8s.io/client-go/util/retry/OWNERS‎

Lines changed: 4 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎vendor/k8s.io/client-go/util/retry/util.go‎

Lines changed: 105 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)