diff --git a/staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/dynamic_cafile_content.go b/staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/dynamic_cafile_content.go index 2f21951349ffc..f1fd2001057da 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/dynamic_cafile_content.go +++ b/staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/dynamic_cafile_content.go @@ -35,7 +35,13 @@ import ( "k8s.io/klog/v2" ) -// FileRefreshDuration is exposed so that integration tests can crank up the reload speed. +// FileRefreshDuration is how often file-backed CA content is re-read even if +// fsnotify did not report a change. Integration tests may shorten this. +// +// fsnotify watches a specific inode. Atomic replace (rename), overlay, or +// bind-mount updates can leave the watch on a stale inode so no event is +// delivered. The poll is the safety net; it was removed when fsnotify was +// added in #104102 but FileRefreshDuration was left unused. var FileRefreshDuration = 1 * time.Minute // ControllerRunner is a generic interface for starting a controller @@ -164,6 +170,14 @@ func (c *DynamicFileCAContent) Run(ctx context.Context, workers int) { // doesn't matter what workers say, only start one. go wait.Until(c.runWorker, time.Second, ctx.Done()) + // Periodic reload in case fsnotify misses the write (new inode / bind-mount / + // atomic rename). This is the original FileRefreshDuration loop from before + // #104102; watchCAFile blocks for the life of a successful watch, so without + // this poll the in-memory client CA bundle can stay stale forever. + go wait.Until(func() { + c.queue.Add(workItemKey) + }, FileRefreshDuration, ctx.Done()) + // start the loop that watches the CA file until stopCh is closed. go wait.Until(func() { if err := c.watchCAFile(ctx.Done()); err != nil { diff --git a/staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/dynamic_cafile_content_linux_test.go b/staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/dynamic_cafile_content_linux_test.go new file mode 100644 index 0000000000000..fa21e029e6211 --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/dynamic_cafile_content_linux_test.go @@ -0,0 +1,93 @@ +//go:build linux + +/* +Copyright 2026 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dynamiccertificates + +import ( + "bytes" + "context" + "os" + "path/filepath" + "syscall" + "testing" + "time" +) + +// TestDynamicFileCAContentPollReloadsAfterBindMountHidesInode covers the miss +// class from atomic replace / overlay / bind-mount: fsnotify stays on the old +// inode while os.ReadFile of the path returns new bytes. Deleting the file is +// not a valid stand-in — loadCABundle errors and the workqueue retries, which +// can reload without FileRefreshDuration. +func TestDynamicFileCAContentPollReloadsAfterBindMountHidesInode(t *testing.T) { + orig := FileRefreshDuration + FileRefreshDuration = 50 * time.Millisecond + t.Cleanup(func() { FileRefreshDuration = orig }) + + dir := t.TempDir() + filename := filepath.Join(dir, "ca.crt") + shadow := filepath.Join(dir, "ca.shadow") + ca1 := mustCreateCA(t, "ca-one") + ca2 := mustCreateCA(t, "ca-two") + if err := os.WriteFile(filename, ca1, 0644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(shadow, ca1, 0644); err != nil { + t.Fatal(err) + } + + c, err := NewDynamicCAContentFromFile("test", filename) + if err != nil { + t.Fatal(err) + } + if got := c.CurrentCABundleContent(); !bytes.Equal(got, ca1) { + t.Fatalf("initial bundle mismatch") + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go c.Run(ctx, 1) + time.Sleep(150 * time.Millisecond) + + if err := syscall.Mount(shadow, filename, "", syscall.MS_BIND, ""); err != nil { + t.Skipf("bind mount not permitted in this environment: %v", err) + } + t.Cleanup(func() { + if err := syscall.Unmount(filename, syscall.MNT_DETACH); err != nil { + t.Errorf("unmount %s: %v", filename, err) + } + }) + + // Append on the new inode. The watch still holds the hidden original inode, + // so fsnotify should not fire; FileRefreshDuration must reload from the path. + f, err := os.OpenFile(shadow, os.O_APPEND|os.O_WRONLY, 0) + if err != nil { + t.Fatal(err) + } + if _, err := f.Write(ca2); err != nil { + t.Fatal(err) + } + if err := f.Close(); err != nil { + t.Fatal(err) + } + + want, err := os.ReadFile(filename) + if err != nil { + t.Fatal(err) + } + waitForCABundle(t, c, want, 2*time.Second) +} diff --git a/staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/dynamic_cafile_content_test.go b/staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/dynamic_cafile_content_test.go new file mode 100644 index 0000000000000..d0fe81910873a --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/dynamic_cafile_content_test.go @@ -0,0 +1,106 @@ +/* +Copyright 2026 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dynamiccertificates + +import ( + "bytes" + "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "math/big" + "os" + "path/filepath" + "testing" + "time" + + "k8s.io/apimachinery/pkg/util/wait" +) + +func mustCreateCA(t *testing.T, cn string) []byte { + t.Helper() + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatal(err) + } + tmpl := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: cn}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(time.Hour), + KeyUsage: x509.KeyUsageCertSign, + BasicConstraintsValid: true, + IsCA: true, + } + der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key) + if err != nil { + t.Fatal(err) + } + return pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}) +} + +func waitForCABundle(t *testing.T, c *DynamicFileCAContent, want []byte, timeout time.Duration) { + t.Helper() + deadline := time.Now().Add(timeout) + var last []byte + for time.Now().Before(deadline) { + last = c.CurrentCABundleContent() + if bytes.Equal(last, want) { + return + } + time.Sleep(20 * time.Millisecond) + } + t.Fatalf("timed out waiting for CA bundle update: got %d bytes, want %d bytes", len(last), len(want)) +} + +// TestFileRefreshPollReloadsWithoutFsnotify starts only the worker + FileRefreshDuration +// enqueue loop from Run (not watchCAFile). A rewrite of the CA file then cannot be +// attributed to inotify; the poll must pick it up. Run() must keep that loop. +func TestFileRefreshPollReloadsWithoutFsnotify(t *testing.T) { + orig := FileRefreshDuration + FileRefreshDuration = 50 * time.Millisecond + t.Cleanup(func() { FileRefreshDuration = orig }) + + dir := t.TempDir() + filename := filepath.Join(dir, "ca.crt") + ca1 := mustCreateCA(t, "ca-one") + ca2 := mustCreateCA(t, "ca-two") + if err := os.WriteFile(filename, ca1, 0644); err != nil { + t.Fatal(err) + } + + c, err := NewDynamicCAContentFromFile("test", filename) + if err != nil { + t.Fatal(err) + } + + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(func() { + cancel() + c.queue.ShutDown() + }) + go wait.Until(c.runWorker, time.Second, ctx.Done()) + go wait.Until(func() { c.queue.Add(workItemKey) }, FileRefreshDuration, ctx.Done()) + + if err := os.WriteFile(filename, ca2, 0644); err != nil { + t.Fatal(err) + } + waitForCABundle(t, c, ca2, 2*time.Second) +} diff --git a/staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/dynamic_serving_content.go b/staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/dynamic_serving_content.go index 268fe70b0d5fc..11b5e068fb290 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/dynamic_serving_content.go +++ b/staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/dynamic_serving_content.go @@ -138,6 +138,11 @@ func (c *DynamicCertKeyPairContent) Run(ctx context.Context, workers int) { // doesn't matter what workers say, only start one. go wait.Until(c.runWorker, time.Second, ctx.Done()) + // Periodic reload in case fsnotify misses the write. See DynamicFileCAContent.Run. + go wait.Until(func() { + c.queue.Add(workItemKey) + }, FileRefreshDuration, ctx.Done()) + // start the loop that watches the cert and key files until stopCh is closed. go wait.Until(func() { if err := c.watchCertKeyFile(ctx.Done()); err != nil {