diff --git a/pkg/bootstrap/bootstrap.go b/pkg/bootstrap/bootstrap.go index ebc63a2758..84eded104b 100644 --- a/pkg/bootstrap/bootstrap.go +++ b/pkg/bootstrap/bootstrap.go @@ -3,7 +3,6 @@ package bootstrap import ( "encoding/json" "io" - "net/http" "os" "path/filepath" "time" @@ -13,13 +12,6 @@ import ( "github.com/sirupsen/logrus" ) -func Handler(bootstrap *config.ControlRuntimeBootstrap) http.Handler { - return http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) { - rw.Header().Set("Content-Type", "application/json") - ReadFromDisk(rw, bootstrap) - }) -} - // ReadFromDisk reads the bootstrap data from the files on disk and // writes their content in JSON form to the given io.Writer. func ReadFromDisk(w io.Writer, bootstrap *config.ControlRuntimeBootstrap) error { diff --git a/pkg/cluster/bootstrap.go b/pkg/cluster/bootstrap.go index 7dd224693b..a9f255fc14 100644 --- a/pkg/cluster/bootstrap.go +++ b/pkg/cluster/bootstrap.go @@ -7,6 +7,7 @@ import ( "fmt" "io" "net" + "net/http" "os" "path/filepath" "reflect" @@ -81,7 +82,7 @@ func (c *Cluster) Bootstrap(ctx context.Context, clusterReset bool) error { func (c *Cluster) shouldBootstrapLoad(ctx context.Context) (bool, bool, error) { // Non-nil managedDB indicates that the database is either initialized, initializing, or joining if c.managedDB != nil { - c.config.Runtime.HTTPBootstrap = true + c.config.Runtime.HTTPBootstrap = c.serveBootstrap() isInitialized, err := c.managedDB.IsInitialized() if err != nil { @@ -387,11 +388,30 @@ func isNewerFile(path string, file bootstrap.File) (updated bool, newerOnDisk bo return true, false, nil } +// serveBootstrap sends bootstrap data to the client, a server that is joining the cluster and +// has only a server token, and cannot use CA certs/keys to access the datastore directly. +func (c *Cluster) serveBootstrap() http.Handler { + return http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) { + rw.Header().Set("Content-Type", "application/json") + // Try getting data from the datastore first. Token has already been validated by the request handler. + _, token, _ := req.BasicAuth() + data, err := c.getBootstrapData(req.Context(), token) + if err != nil { + // If we failed to read data from the datastore, just send data from disk. + logrus.Warnf("Failed to retrieve HTTP bootstrap data from datastore; falling back to disk for %s: %v", req.RemoteAddr, err) + bootstrap.ReadFromDisk(rw, &c.config.Runtime.ControlRuntimeBootstrap) + return + } + logrus.Infof("Serving HTTP bootstrap from datastore for %s", req.RemoteAddr) + rw.Write(data) + }) +} + // httpBootstrap retrieves bootstrap data (certs and keys, etc) from the remote server via HTTP // and loads it into the ControlRuntimeBootstrap struct. Unlike the storage bootstrap path, // this data does not need to be decrypted since it is generated on-demand by an existing server. func (c *Cluster) httpBootstrap(ctx context.Context) error { - content, err := c.clientAccessInfo.Get("/v1-" + version.Program + "/server-bootstrap") + content, err := c.clientAccessInfo.Get("/v1-"+version.Program+"/server-bootstrap", clientaccess.WithTimeout(15*time.Second)) if err != nil { return err } @@ -399,7 +419,8 @@ func (c *Cluster) httpBootstrap(ctx context.Context) error { return c.ReconcileBootstrapData(ctx, bytes.NewReader(content), &c.config.Runtime.ControlRuntimeBootstrap, true) } -func (c *Cluster) retrieveInitializedDBdata(ctx context.Context) (*bytes.Buffer, error) { +// readBootstrapFromDisk returns a buffer holding the JSON-serialized bootstrap data read from disk. +func (c *Cluster) readBootstrapFromDisk() (*bytes.Buffer, error) { var buf bytes.Buffer if err := bootstrap.ReadFromDisk(&buf, &c.config.Runtime.ControlRuntimeBootstrap); err != nil { return nil, err @@ -408,13 +429,16 @@ func (c *Cluster) retrieveInitializedDBdata(ctx context.Context) (*bytes.Buffer, return &buf, nil } -// bootstrap performs cluster bootstrapping, either via HTTP (for managed databases) or direct load from datastore. +// bootstrap retrieves cluster bootstrap data: CA certs and other common config. This uses HTTP +// for etcd (as this node does not yet have CA data available), and direct load from datastore +// when using kine. func (c *Cluster) bootstrap(ctx context.Context) error { c.joining = true - // bootstrap managed database via HTTPS - if c.config.Runtime.HTTPBootstrap { - // Assuming we should just compare on managed databases + if c.config.Runtime.HTTPBootstrap != nil { + // We can only compare config when we have a server URL that we are joining against - + // if loading directly from the datastore we do not have any way to get the config + // from another server for comparison. if err := c.compareConfig(); err != nil { return errors.Wrap(err, "failed to validate server configuration") } @@ -517,7 +541,7 @@ func (c *Cluster) reconcileEtcd(ctx context.Context) error { } } - data, err := c.retrieveInitializedDBdata(reconcileCtx) + data, err := c.readBootstrapFromDisk() if err != nil { return err } diff --git a/pkg/cluster/storage.go b/pkg/cluster/storage.go index b555ac9763..1b579c3b9a 100644 --- a/pkg/cluster/storage.go +++ b/pkg/cluster/storage.go @@ -246,6 +246,25 @@ func (c *Cluster) storageBootstrap(ctx context.Context) error { }) } +// getBootstrapData makes a single attempt to retrieve and decrypt bootstrap data from the datastore. +func (c *Cluster) getBootstrapData(ctx context.Context, token string) ([]byte, error) { + storageClient, err := client.New(c.config.Runtime.EtcdConfig) + if err != nil { + return nil, err + } + defer storageClient.Close() + + ctx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + + value, err := storageClient.Get(ctx, storageKey(token)) + if err != nil { + return nil, err + } + + return decrypt(token, value.Data) +} + // getBootstrapKeyFromStorage will list all keys that has prefix /bootstrap and will check for key that is // hashed with empty string and will check for any key that is hashed by different token than the one // passed to it, it will return error if it finds a key that is hashed with different token and will return diff --git a/pkg/daemons/config/types.go b/pkg/daemons/config/types.go index fe2c5a73e3..1f7c0a2a5f 100644 --- a/pkg/daemons/config/types.go +++ b/pkg/daemons/config/types.go @@ -312,7 +312,6 @@ type ControlRuntimeBootstrap struct { type ControlRuntime struct { ControlRuntimeBootstrap - HTTPBootstrap bool APIServerReady <-chan struct{} ContainerRuntimeReady <-chan struct{} ETCDReady <-chan struct{} @@ -342,6 +341,7 @@ type ControlRuntime struct { AgentToken string APIServer http.Handler Handler http.Handler + HTTPBootstrap http.Handler Tunnel http.Handler Authenticator authenticator.Request diff --git a/pkg/server/handlers/handlers.go b/pkg/server/handlers/handlers.go index 101554a22c..9ccb901d3d 100644 --- a/pkg/server/handlers/handlers.go +++ b/pkg/server/handlers/handlers.go @@ -14,7 +14,6 @@ import ( "time" "github.com/gorilla/mux" - "github.com/k3s-io/k3s/pkg/bootstrap" "github.com/k3s-io/k3s/pkg/cli/cmds" "github.com/k3s-io/k3s/pkg/daemons/config" "github.com/k3s-io/k3s/pkg/etcd" @@ -199,8 +198,8 @@ func Readyz(control *config.Control) http.Handler { } func Bootstrap(control *config.Control) http.Handler { - if control.Runtime.HTTPBootstrap { - return bootstrap.Handler(&control.Runtime.ControlRuntimeBootstrap) + if control.Runtime.HTTPBootstrap != nil { + return control.Runtime.HTTPBootstrap } return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) { logrus.Warnf("Received HTTP bootstrap request from %s, but embedded etcd is not enabled.", req.RemoteAddr)