From 53fcadc028d076169b9dcfadd7a76f3bf9ea0011 Mon Sep 17 00:00:00 2001 From: Brad Davidson Date: Tue, 25 Feb 2025 00:28:44 +0000 Subject: [PATCH] Serve HTTP bootstrap data from datastore before disk Fixes issue where CA rotation would fail on servers with join URL set due to using old data from disk on other server Signed-off-by: Brad Davidson --- pkg/bootstrap/bootstrap.go | 8 ------- pkg/cluster/bootstrap.go | 40 ++++++++++++++++++++++++++------- pkg/cluster/storage.go | 19 ++++++++++++++++ pkg/daemons/config/types.go | 2 +- pkg/server/handlers/handlers.go | 5 ++--- 5 files changed, 54 insertions(+), 20 deletions(-) diff --git a/pkg/bootstrap/bootstrap.go b/pkg/bootstrap/bootstrap.go index ebc63a2758..84eded104b 100644 --- a/pkg/bootstrap/bootstrap.go +++ b/pkg/bootstrap/bootstrap.go @@ -3,7 +3,6 @@ package bootstrap import ( "encoding/json" "io" - "net/http" "os" "path/filepath" "time" @@ -13,13 +12,6 @@ import ( "github.com/sirupsen/logrus" ) -func Handler(bootstrap *config.ControlRuntimeBootstrap) http.Handler { - return http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) { - rw.Header().Set("Content-Type", "application/json") - ReadFromDisk(rw, bootstrap) - }) -} - // ReadFromDisk reads the bootstrap data from the files on disk and // writes their content in JSON form to the given io.Writer. func ReadFromDisk(w io.Writer, bootstrap *config.ControlRuntimeBootstrap) error { diff --git a/pkg/cluster/bootstrap.go b/pkg/cluster/bootstrap.go index 7dd224693b..a9f255fc14 100644 --- a/pkg/cluster/bootstrap.go +++ b/pkg/cluster/bootstrap.go @@ -7,6 +7,7 @@ import ( "fmt" "io" "net" + "net/http" "os" "path/filepath" "reflect" @@ -81,7 +82,7 @@ func (c *Cluster) Bootstrap(ctx context.Context, clusterReset bool) error { func (c *Cluster) shouldBootstrapLoad(ctx context.Context) (bool, bool, error) { // Non-nil managedDB indicates that the database is either initialized, initializing, or joining if c.managedDB != nil { - c.config.Runtime.HTTPBootstrap = true + c.config.Runtime.HTTPBootstrap = c.serveBootstrap() isInitialized, err := c.managedDB.IsInitialized() if err != nil { @@ -387,11 +388,30 @@ func isNewerFile(path string, file bootstrap.File) (updated bool, newerOnDisk bo return true, false, nil } +// serveBootstrap sends bootstrap data to the client, a server that is joining the cluster and +// has only a server token, and cannot use CA certs/keys to access the datastore directly. +func (c *Cluster) serveBootstrap() http.Handler { + return http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) { + rw.Header().Set("Content-Type", "application/json") + // Try getting data from the datastore first. Token has already been validated by the request handler. + _, token, _ := req.BasicAuth() + data, err := c.getBootstrapData(req.Context(), token) + if err != nil { + // If we failed to read data from the datastore, just send data from disk. + logrus.Warnf("Failed to retrieve HTTP bootstrap data from datastore; falling back to disk for %s: %v", req.RemoteAddr, err) + bootstrap.ReadFromDisk(rw, &c.config.Runtime.ControlRuntimeBootstrap) + return + } + logrus.Infof("Serving HTTP bootstrap from datastore for %s", req.RemoteAddr) + rw.Write(data) + }) +} + // httpBootstrap retrieves bootstrap data (certs and keys, etc) from the remote server via HTTP // and loads it into the ControlRuntimeBootstrap struct. Unlike the storage bootstrap path, // this data does not need to be decrypted since it is generated on-demand by an existing server. func (c *Cluster) httpBootstrap(ctx context.Context) error { - content, err := c.clientAccessInfo.Get("/v1-" + version.Program + "/server-bootstrap") + content, err := c.clientAccessInfo.Get("/v1-"+version.Program+"/server-bootstrap", clientaccess.WithTimeout(15*time.Second)) if err != nil { return err } @@ -399,7 +419,8 @@ func (c *Cluster) httpBootstrap(ctx context.Context) error { return c.ReconcileBootstrapData(ctx, bytes.NewReader(content), &c.config.Runtime.ControlRuntimeBootstrap, true) } -func (c *Cluster) retrieveInitializedDBdata(ctx context.Context) (*bytes.Buffer, error) { +// readBootstrapFromDisk returns a buffer holding the JSON-serialized bootstrap data read from disk. +func (c *Cluster) readBootstrapFromDisk() (*bytes.Buffer, error) { var buf bytes.Buffer if err := bootstrap.ReadFromDisk(&buf, &c.config.Runtime.ControlRuntimeBootstrap); err != nil { return nil, err @@ -408,13 +429,16 @@ func (c *Cluster) retrieveInitializedDBdata(ctx context.Context) (*bytes.Buffer, return &buf, nil } -// bootstrap performs cluster bootstrapping, either via HTTP (for managed databases) or direct load from datastore. +// bootstrap retrieves cluster bootstrap data: CA certs and other common config. This uses HTTP +// for etcd (as this node does not yet have CA data available), and direct load from datastore +// when using kine. func (c *Cluster) bootstrap(ctx context.Context) error { c.joining = true - // bootstrap managed database via HTTPS - if c.config.Runtime.HTTPBootstrap { - // Assuming we should just compare on managed databases + if c.config.Runtime.HTTPBootstrap != nil { + // We can only compare config when we have a server URL that we are joining against - + // if loading directly from the datastore we do not have any way to get the config + // from another server for comparison. if err := c.compareConfig(); err != nil { return errors.Wrap(err, "failed to validate server configuration") } @@ -517,7 +541,7 @@ func (c *Cluster) reconcileEtcd(ctx context.Context) error { } } - data, err := c.retrieveInitializedDBdata(reconcileCtx) + data, err := c.readBootstrapFromDisk() if err != nil { return err } diff --git a/pkg/cluster/storage.go b/pkg/cluster/storage.go index b555ac9763..1b579c3b9a 100644 --- a/pkg/cluster/storage.go +++ b/pkg/cluster/storage.go @@ -246,6 +246,25 @@ func (c *Cluster) storageBootstrap(ctx context.Context) error { }) } +// getBootstrapData makes a single attempt to retrieve and decrypt bootstrap data from the datastore. +func (c *Cluster) getBootstrapData(ctx context.Context, token string) ([]byte, error) { + storageClient, err := client.New(c.config.Runtime.EtcdConfig) + if err != nil { + return nil, err + } + defer storageClient.Close() + + ctx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + + value, err := storageClient.Get(ctx, storageKey(token)) + if err != nil { + return nil, err + } + + return decrypt(token, value.Data) +} + // getBootstrapKeyFromStorage will list all keys that has prefix /bootstrap and will check for key that is // hashed with empty string and will check for any key that is hashed by different token than the one // passed to it, it will return error if it finds a key that is hashed with different token and will return diff --git a/pkg/daemons/config/types.go b/pkg/daemons/config/types.go index fe2c5a73e3..1f7c0a2a5f 100644 --- a/pkg/daemons/config/types.go +++ b/pkg/daemons/config/types.go @@ -312,7 +312,6 @@ type ControlRuntimeBootstrap struct { type ControlRuntime struct { ControlRuntimeBootstrap - HTTPBootstrap bool APIServerReady <-chan struct{} ContainerRuntimeReady <-chan struct{} ETCDReady <-chan struct{} @@ -342,6 +341,7 @@ type ControlRuntime struct { AgentToken string APIServer http.Handler Handler http.Handler + HTTPBootstrap http.Handler Tunnel http.Handler Authenticator authenticator.Request diff --git a/pkg/server/handlers/handlers.go b/pkg/server/handlers/handlers.go index 101554a22c..9ccb901d3d 100644 --- a/pkg/server/handlers/handlers.go +++ b/pkg/server/handlers/handlers.go @@ -14,7 +14,6 @@ import ( "time" "github.com/gorilla/mux" - "github.com/k3s-io/k3s/pkg/bootstrap" "github.com/k3s-io/k3s/pkg/cli/cmds" "github.com/k3s-io/k3s/pkg/daemons/config" "github.com/k3s-io/k3s/pkg/etcd" @@ -199,8 +198,8 @@ func Readyz(control *config.Control) http.Handler { } func Bootstrap(control *config.Control) http.Handler { - if control.Runtime.HTTPBootstrap { - return bootstrap.Handler(&control.Runtime.ControlRuntimeBootstrap) + if control.Runtime.HTTPBootstrap != nil { + return control.Runtime.HTTPBootstrap } return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) { logrus.Warnf("Received HTTP bootstrap request from %s, but embedded etcd is not enabled.", req.RemoteAddr)