Skip to content

Commit 8253d0b

Browse files
ChristianZaccariaChristianZaccaria
committed
test: add unit and envtest coverage for ConfigMap scoped cache
1 parent bee259d commit 8253d0b

3 files changed

Lines changed: 337 additions & 37 deletions

File tree

‎kagenti-operator/cmd/configmap_cache_test.go‎

Lines changed: 200 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,200 @@
1+
/*
2+
Copyright 2026.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
// Run with: make setup-envtest && go test -v -count=1 -timeout 120s ./cmd/ -run TestConfigMapCacheVisibility
18+
19+
package main
20+
21+
import (
22+
"context"
23+
"os"
24+
"path/filepath"
25+
"testing"
26+
27+
corev1 "k8s.io/api/core/v1"
28+
apierrors "k8s.io/apimachinery/pkg/api/errors"
29+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
30+
"k8s.io/apimachinery/pkg/types"
31+
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
32+
ctrl "sigs.k8s.io/controller-runtime"
33+
"sigs.k8s.io/controller-runtime/pkg/cache"
34+
"sigs.k8s.io/controller-runtime/pkg/client"
35+
"sigs.k8s.io/controller-runtime/pkg/envtest"
36+
logf "sigs.k8s.io/controller-runtime/pkg/log"
37+
"sigs.k8s.io/controller-runtime/pkg/log/zap"
38+
metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
39+
40+
"github.com/kagenti/operator/internal/controller"
41+
)
42+
43+
func TestConfigMapCacheVisibility(t *testing.T) {
44+
logf.SetLogger(zap.New(zap.WriteTo(os.Stderr), zap.UseDevMode(true)))
45+
46+
// 1. Start a local kube-apiserver + etcd via envtest.
47+
testEnv := &envtest.Environment{}
48+
if dir := firstEnvTestBinaryDir(); dir != "" {
49+
testEnv.BinaryAssetsDirectory = dir
50+
}
51+
52+
cfg, err := testEnv.Start()
53+
if err != nil {
54+
t.Fatalf("failed to start envtest: %v", err)
55+
}
56+
defer func() { _ = testEnv.Stop() }()
57+
58+
// 2. Create a direct (non-cached) client for seeding test data.
59+
directClient, err := client.New(cfg, client.Options{Scheme: clientgoscheme.Scheme})
60+
if err != nil {
61+
t.Fatalf("failed to create direct client: %v", err)
62+
}
63+
64+
ctx, cancel := context.WithCancel(context.Background())
65+
defer cancel()
66+
67+
const (
68+
spireNS = "zero-trust-workload-identity-manager"
69+
spireBundleName = "spire-bundle"
70+
)
71+
72+
// 3. Pre-create the namespaces the scoped cache will watch.
73+
for _, ns := range []string{controller.ClusterDefaultsNamespace, spireNS, "agent-team-a"} {
74+
nsObj := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: ns}}
75+
if err := directClient.Create(ctx, nsObj); err != nil {
76+
t.Fatalf("failed to create namespace %s: %v", ns, err)
77+
}
78+
}
79+
80+
// 4. Build the scoped cache config (function under test) and start a
81+
// manager whose ConfigMap informers are restricted to those selectors.
82+
cmCacheNamespaces := buildConfigMapCacheNamespaces(true, spireBundleName, spireNS)
83+
84+
mgr, err := ctrl.NewManager(cfg, ctrl.Options{
85+
Scheme: clientgoscheme.Scheme,
86+
LeaderElection: false,
87+
Metrics: metricsserver.Options{BindAddress: "0"},
88+
Cache: cache.Options{
89+
ByObject: map[client.Object]cache.ByObject{
90+
&corev1.ConfigMap{}: {Namespaces: cmCacheNamespaces},
91+
},
92+
},
93+
})
94+
if err != nil {
95+
t.Fatalf("failed to create manager: %v", err)
96+
}
97+
98+
go func() {
99+
if err := mgr.Start(ctx); err != nil {
100+
t.Errorf("manager exited with error: %v", err)
101+
}
102+
}()
103+
if !mgr.GetCache().WaitForCacheSync(ctx) {
104+
t.Fatal("cache never synced")
105+
}
106+
107+
// 5. Obtain the cached client — reads go through the scoped informer cache,
108+
// so only ConfigMaps matching our selectors will be visible.
109+
cachedClient := mgr.GetClient()
110+
111+
// 6. Seed ConfigMaps via the direct client (bypasses the cache) so we can
112+
// then verify which ones are visible through the cached client.
113+
spireBundleCM := &corev1.ConfigMap{
114+
ObjectMeta: metav1.ObjectMeta{
115+
Name: spireBundleName, Namespace: spireNS,
116+
},
117+
Data: map[string]string{"bundle.crt": "FAKE-BUNDLE"},
118+
}
119+
clusterDefaultsCM := &corev1.ConfigMap{
120+
ObjectMeta: metav1.ObjectMeta{
121+
Name: controller.ClusterDefaultsConfigMapName,
122+
Namespace: controller.ClusterDefaultsNamespace,
123+
Labels: map[string]string{"app.kubernetes.io/name": "kagenti-operator-chart"},
124+
},
125+
Data: map[string]string{"otel-endpoint": "collector:4317"},
126+
}
127+
nsDefaultsCM := &corev1.ConfigMap{
128+
ObjectMeta: metav1.ObjectMeta{
129+
Name: "team-a-defaults", Namespace: "agent-team-a",
130+
Labels: map[string]string{controller.LabelNamespaceDefaults: "true"},
131+
},
132+
Data: map[string]string{"sampling-rate": "0.5"},
133+
}
134+
unrelatedCM := &corev1.ConfigMap{
135+
ObjectMeta: metav1.ObjectMeta{
136+
Name: "app-config", Namespace: spireNS,
137+
},
138+
Data: map[string]string{"key": "value"},
139+
}
140+
141+
for _, cm := range []*corev1.ConfigMap{spireBundleCM, clusterDefaultsCM, nsDefaultsCM, unrelatedCM} {
142+
if err := directClient.Create(ctx, cm); err != nil {
143+
t.Fatalf("failed to create ConfigMap %s/%s: %v", cm.Namespace, cm.Name, err)
144+
}
145+
}
146+
147+
t.Run("SPIRE trust bundle visible through cache", func(t *testing.T) {
148+
got := &corev1.ConfigMap{}
149+
err := cachedClient.Get(ctx, types.NamespacedName{Name: spireBundleName, Namespace: spireNS}, got)
150+
if err != nil {
151+
t.Fatalf("expected SPIRE trust bundle to be visible, got: %v", err)
152+
}
153+
if got.Data["bundle.crt"] != "FAKE-BUNDLE" {
154+
t.Fatalf("unexpected data: %v", got.Data)
155+
}
156+
})
157+
158+
t.Run("cluster defaults visible through cache", func(t *testing.T) {
159+
got := &corev1.ConfigMap{}
160+
err := cachedClient.Get(ctx, types.NamespacedName{
161+
Name: controller.ClusterDefaultsConfigMapName, Namespace: controller.ClusterDefaultsNamespace,
162+
}, got)
163+
if err != nil {
164+
t.Fatalf("expected cluster defaults to be visible, got: %v", err)
165+
}
166+
})
167+
168+
t.Run("namespace defaults visible through cache", func(t *testing.T) {
169+
got := &corev1.ConfigMap{}
170+
err := cachedClient.Get(ctx, types.NamespacedName{Name: "team-a-defaults", Namespace: "agent-team-a"}, got)
171+
if err != nil {
172+
t.Fatalf("expected namespace defaults to be visible, got: %v", err)
173+
}
174+
})
175+
176+
t.Run("unrelated ConfigMap in SPIRE namespace is NOT visible", func(t *testing.T) {
177+
got := &corev1.ConfigMap{}
178+
err := cachedClient.Get(ctx, types.NamespacedName{Name: "app-config", Namespace: spireNS}, got)
179+
if err == nil {
180+
t.Fatal("expected unrelated ConfigMap to be filtered out by field selector, but Get succeeded")
181+
}
182+
if !apierrors.IsNotFound(err) {
183+
t.Fatalf("expected NotFound error, got: %v", err)
184+
}
185+
})
186+
}
187+
188+
func firstEnvTestBinaryDir() string {
189+
basePath := filepath.Join("..", "bin", "k8s")
190+
entries, err := os.ReadDir(basePath)
191+
if err != nil {
192+
return ""
193+
}
194+
for _, entry := range entries {
195+
if entry.IsDir() {
196+
return filepath.Join(basePath, entry.Name())
197+
}
198+
}
199+
return ""
200+
}

‎kagenti-operator/cmd/main.go‎

Lines changed: 44 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -254,42 +254,7 @@ func main() {
254254
})
255255
}
256256

257-
// Scope the ConfigMap informer to only kagenti-relevant ConfigMaps.
258-
// Without this, the controller would cache ALL ConfigMaps cluster-wide.
259-
//
260-
// Three types of ConfigMaps are relevant:
261-
// 1. Cluster-level defaults in kagenti-system:
262-
// - kagenti-platform-config (platform-wide sidecar config)
263-
// - kagenti-feature-gates (which AuthBridge components are enabled)
264-
// Both are deployed by the kagenti-operator Helm chart and share the
265-
// label app.kubernetes.io/name=kagenti-operator-chart.
266-
//
267-
// 2. Namespace-level defaults in agent namespaces:
268-
// ConfigMaps labeled kagenti.io/defaults=true, deployed by platform
269-
// engineers via Helm/Kustomize to override cluster defaults per namespace.
270-
//
271-
// 3. SPIRE trust bundle (when signature verification is enabled):
272-
// The trust bundle ConfigMap (e.g. spire-bundle) in its configured namespace,
273-
// selected by metadata.name via a field selector.
274-
cmCacheNamespaces := map[string]cache.Config{
275-
controller.ClusterDefaultsNamespace: {
276-
LabelSelector: labels.SelectorFromSet(map[string]string{
277-
"app.kubernetes.io/name": "kagenti-operator-chart",
278-
}),
279-
},
280-
cache.AllNamespaces: {
281-
LabelSelector: labels.SelectorFromSet(map[string]string{
282-
controller.LabelNamespaceDefaults: "true",
283-
}),
284-
},
285-
}
286-
if requireA2ASignature && spireTrustBundleConfigMapNS != "" {
287-
cmCacheNamespaces[spireTrustBundleConfigMapNS] = cache.Config{
288-
FieldSelector: fields.SelectorFromSet(fields.Set{
289-
"metadata.name": spireTrustBundleConfigMapName,
290-
}),
291-
}
292-
}
257+
cmCacheNamespaces := buildConfigMapCacheNamespaces(requireA2ASignature, spireTrustBundleConfigMapName, spireTrustBundleConfigMapNS)
293258

294259
mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
295260
Scheme: scheme,
@@ -514,3 +479,46 @@ func getNamespacesToWatch() map[string]cache.Config {
514479
}
515480
return namespaces
516481
}
482+
483+
// buildConfigMapCacheNamespaces returns the per-namespace cache selectors for
484+
// ConfigMaps. The scoped cache ensures only kagenti-relevant ConfigMaps are
485+
// watched instead of every ConfigMap cluster-wide.
486+
//
487+
// Three categories are included:
488+
// 1. Cluster-level defaults in kagenti-system (label selector).
489+
// 2. Namespace-level defaults in any namespace (label selector).
490+
// 3. SPIRE trust bundle (field selector on metadata.name), added only when
491+
// signature verification is enabled.
492+
func buildConfigMapCacheNamespaces(requireA2ASignature bool, spireTrustBundleConfigMapName, spireTrustBundleConfigMapNS string) map[string]cache.Config {
493+
namespaces := map[string]cache.Config{
494+
controller.ClusterDefaultsNamespace: {
495+
LabelSelector: labels.SelectorFromSet(map[string]string{
496+
"app.kubernetes.io/name": "kagenti-operator-chart",
497+
}),
498+
},
499+
cache.AllNamespaces: {
500+
LabelSelector: labels.SelectorFromSet(map[string]string{
501+
controller.LabelNamespaceDefaults: "true",
502+
}),
503+
},
504+
}
505+
if requireA2ASignature && spireTrustBundleConfigMapNS != "" {
506+
if _, collision := namespaces[spireTrustBundleConfigMapNS]; collision {
507+
setupLog.Error(
508+
errors.New("namespace collision: --spire-trust-bundle-configmap-namespace matches "+
509+
"the cluster defaults namespace"),
510+
"SPIRE trust bundle will not be cached; signature verification may fail. "+
511+
"Use a different namespace for the trust bundle ConfigMap",
512+
"trustBundleNamespace", spireTrustBundleConfigMapNS,
513+
"clusterDefaultsNamespace", controller.ClusterDefaultsNamespace,
514+
)
515+
} else {
516+
namespaces[spireTrustBundleConfigMapNS] = cache.Config{
517+
FieldSelector: fields.SelectorFromSet(fields.Set{
518+
"metadata.name": spireTrustBundleConfigMapName,
519+
}),
520+
}
521+
}
522+
}
523+
return namespaces
524+
}

‎kagenti-operator/cmd/main_test.go‎

Lines changed: 93 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,9 +14,17 @@ See the License for the specific language governing permissions and
1414
limitations under the License.
1515
*/
1616

17+
// Run with: go test -v -count=1 ./cmd/ -run TestBuildConfigMapCacheNamespaces
18+
1719
package main
1820

19-
import "testing"
21+
import (
22+
"testing"
23+
24+
"sigs.k8s.io/controller-runtime/pkg/cache"
25+
26+
"github.com/kagenti/operator/internal/controller"
27+
)
2028

2129
func TestAuthBridgeWebhooksEnabled(t *testing.T) {
2230
t.Run("enables when unset", func(t *testing.T) {
@@ -38,3 +46,87 @@ func TestAuthBridgeWebhooksEnabled(t *testing.T) {
3846
}
3947
})
4048
}
49+
50+
func TestBuildConfigMapCacheNamespaces(t *testing.T) {
51+
t.Run("base config always includes cluster defaults and namespace defaults", func(t *testing.T) {
52+
result := buildConfigMapCacheNamespaces(false, "", "")
53+
54+
if _, ok := result[controller.ClusterDefaultsNamespace]; !ok {
55+
t.Fatalf("expected entry for %s", controller.ClusterDefaultsNamespace)
56+
}
57+
if _, ok := result[cache.AllNamespaces]; !ok {
58+
t.Fatal("expected entry for AllNamespaces (namespace-level defaults)")
59+
}
60+
if len(result) != 2 {
61+
t.Fatalf("expected exactly 2 entries, got %d", len(result))
62+
}
63+
})
64+
65+
t.Run("adds SPIRE trust bundle namespace when signature verification enabled", func(t *testing.T) {
66+
const spireNS = "zero-trust-workload-identity-manager"
67+
result := buildConfigMapCacheNamespaces(true, "spire-bundle", spireNS)
68+
69+
spireCfg, ok := result[spireNS]
70+
if !ok {
71+
t.Fatalf("expected cache entry for %s namespace", spireNS)
72+
}
73+
if spireCfg.FieldSelector == nil {
74+
t.Fatal("expected FieldSelector on SPIRE cache entry")
75+
}
76+
if !spireCfg.FieldSelector.Matches(fieldSet("spire-bundle")) {
77+
t.Fatal("expected FieldSelector to match ConfigMap named spire-bundle")
78+
}
79+
if spireCfg.FieldSelector.Matches(fieldSet("other-configmap")) {
80+
t.Fatal("expected FieldSelector to NOT match other ConfigMap names")
81+
}
82+
if len(result) != 3 {
83+
t.Fatalf("expected 3 entries (cluster + namespace + SPIRE), got %d", len(result))
84+
}
85+
})
86+
87+
t.Run("does not add SPIRE entry when flag is false", func(t *testing.T) {
88+
const spireNS = "zero-trust-workload-identity-manager"
89+
result := buildConfigMapCacheNamespaces(false, "spire-bundle", spireNS)
90+
91+
if _, ok := result[spireNS]; ok {
92+
t.Fatal("expected no SPIRE entry when requireA2ASignature is false")
93+
}
94+
if len(result) != 2 {
95+
t.Fatalf("expected 2 entries, got %d", len(result))
96+
}
97+
})
98+
99+
t.Run("does not add SPIRE entry when namespace is empty", func(t *testing.T) {
100+
result := buildConfigMapCacheNamespaces(true, "spire-bundle", "")
101+
102+
if len(result) != 2 {
103+
t.Fatalf("expected 2 entries, got %d", len(result))
104+
}
105+
})
106+
107+
t.Run("namespace collision preserves existing label selector", func(t *testing.T) {
108+
result := buildConfigMapCacheNamespaces(true, "spire-bundle", controller.ClusterDefaultsNamespace)
109+
110+
cfg := result[controller.ClusterDefaultsNamespace]
111+
if cfg.LabelSelector == nil {
112+
t.Fatal("expected original LabelSelector to be preserved when namespaces collide")
113+
}
114+
if cfg.FieldSelector != nil {
115+
t.Fatal("expected SPIRE FieldSelector to NOT be added when namespaces collide")
116+
}
117+
if len(result) != 2 {
118+
t.Fatalf("expected 2 entries (no SPIRE entry added), got %d", len(result))
119+
}
120+
})
121+
}
122+
123+
// fieldSet is a minimal fields.Fields implementation for test assertions.
124+
type fieldSet string
125+
126+
func (f fieldSet) Has(field string) bool { return field == "metadata.name" }
127+
func (f fieldSet) Get(field string) string {
128+
if field == "metadata.name" {
129+
return string(f)
130+
}
131+
return ""
132+
}

0 commit comments

Comments
 (0)