diff --git a/.github/workflows/unitcoverage.yaml b/.github/workflows/unitcoverage.yaml index d90885dacd..659e1bdd0a 100644 --- a/.github/workflows/unitcoverage.yaml +++ b/.github/workflows/unitcoverage.yaml @@ -11,6 +11,8 @@ jobs: steps: - name: Install Go uses: actions/setup-go@v2 + with: + go-version: '1.16.5' - name: Checkout uses: actions/checkout@v2 with: @@ -19,6 +21,10 @@ jobs: run: | go test ./pkg/... -coverprofile coverage.out go tool cover -func coverage.out + - name: On Failure, Launch Debug Session + if: ${{ failure() }} + uses: mxschmitt/action-tmate@v3 + timeout-minutes: 2 - name: Upload Results To Codecov uses: codecov/codecov-action@v1 with: diff --git a/pkg/daemons/control/deps/deps.go b/pkg/daemons/control/deps/deps.go index b694afb796..acd385c88b 100644 --- a/pkg/daemons/control/deps/deps.go +++ b/pkg/daemons/control/deps/deps.go @@ -90,6 +90,67 @@ func KubeConfig(dest, url, caCert, clientCert, clientKey string) error { return kubeconfigTemplate.Execute(output, &data) } +// FillRuntimeCerts is responsible for filling out all the +// .crt and .key filenames for a ControlRuntime. +func FillRuntimeCerts(config *config.Control, runtime *config.ControlRuntime) { + runtime.ClientCA = filepath.Join(config.DataDir, "tls", "client-ca.crt") + runtime.ClientCAKey = filepath.Join(config.DataDir, "tls", "client-ca.key") + runtime.ServerCA = filepath.Join(config.DataDir, "tls", "server-ca.crt") + runtime.ServerCAKey = filepath.Join(config.DataDir, "tls", "server-ca.key") + runtime.RequestHeaderCA = filepath.Join(config.DataDir, "tls", "request-header-ca.crt") + runtime.RequestHeaderCAKey = filepath.Join(config.DataDir, "tls", "request-header-ca.key") + runtime.IPSECKey = filepath.Join(config.DataDir, "cred", "ipsec.psk") + + runtime.ServiceKey = filepath.Join(config.DataDir, "tls", "service.key") + runtime.PasswdFile = filepath.Join(config.DataDir, "cred", "passwd") + runtime.NodePasswdFile = filepath.Join(config.DataDir, "cred", "node-passwd") + + runtime.KubeConfigAdmin = filepath.Join(config.DataDir, "cred", "admin.kubeconfig") + runtime.KubeConfigController = filepath.Join(config.DataDir, "cred", "controller.kubeconfig") + runtime.KubeConfigScheduler = filepath.Join(config.DataDir, "cred", "scheduler.kubeconfig") + runtime.KubeConfigAPIServer = filepath.Join(config.DataDir, "cred", "api-server.kubeconfig") + runtime.KubeConfigCloudController = filepath.Join(config.DataDir, "cred", "cloud-controller.kubeconfig") + + runtime.ClientAdminCert = filepath.Join(config.DataDir, "tls", "client-admin.crt") + runtime.ClientAdminKey = filepath.Join(config.DataDir, "tls", "client-admin.key") + runtime.ClientControllerCert = filepath.Join(config.DataDir, "tls", "client-controller.crt") + runtime.ClientControllerKey = filepath.Join(config.DataDir, "tls", "client-controller.key") + runtime.ClientCloudControllerCert = filepath.Join(config.DataDir, "tls", "client-"+version.Program+"-cloud-controller.crt") + runtime.ClientCloudControllerKey = filepath.Join(config.DataDir, "tls", "client-"+version.Program+"-cloud-controller.key") + runtime.ClientSchedulerCert = filepath.Join(config.DataDir, "tls", "client-scheduler.crt") + runtime.ClientSchedulerKey = filepath.Join(config.DataDir, "tls", "client-scheduler.key") + runtime.ClientKubeAPICert = filepath.Join(config.DataDir, "tls", "client-kube-apiserver.crt") + runtime.ClientKubeAPIKey = filepath.Join(config.DataDir, "tls", "client-kube-apiserver.key") + runtime.ClientKubeProxyCert = filepath.Join(config.DataDir, "tls", "client-kube-proxy.crt") + runtime.ClientKubeProxyKey = filepath.Join(config.DataDir, "tls", "client-kube-proxy.key") + runtime.ClientK3sControllerCert = filepath.Join(config.DataDir, "tls", "client-"+version.Program+"-controller.crt") + runtime.ClientK3sControllerKey = filepath.Join(config.DataDir, "tls", "client-"+version.Program+"-controller.key") + + runtime.ServingKubeAPICert = filepath.Join(config.DataDir, "tls", "serving-kube-apiserver.crt") + runtime.ServingKubeAPIKey = filepath.Join(config.DataDir, "tls", "serving-kube-apiserver.key") + + runtime.ClientKubeletKey = filepath.Join(config.DataDir, "tls", "client-kubelet.key") + runtime.ServingKubeletKey = filepath.Join(config.DataDir, "tls", "serving-kubelet.key") + + runtime.ClientAuthProxyCert = filepath.Join(config.DataDir, "tls", "client-auth-proxy.crt") + runtime.ClientAuthProxyKey = filepath.Join(config.DataDir, "tls", "client-auth-proxy.key") + + runtime.ETCDServerCA = filepath.Join(config.DataDir, "tls", "etcd", "server-ca.crt") + runtime.ETCDServerCAKey = filepath.Join(config.DataDir, "tls", "etcd", "server-ca.key") + runtime.ETCDPeerCA = filepath.Join(config.DataDir, "tls", "etcd", "peer-ca.crt") + runtime.ETCDPeerCAKey = filepath.Join(config.DataDir, "tls", "etcd", "peer-ca.key") + runtime.ServerETCDCert = filepath.Join(config.DataDir, "tls", "etcd", "server-client.crt") + runtime.ServerETCDKey = filepath.Join(config.DataDir, "tls", "etcd", "server-client.key") + runtime.PeerServerClientETCDCert = filepath.Join(config.DataDir, "tls", "etcd", "peer-server-client.crt") + runtime.PeerServerClientETCDKey = filepath.Join(config.DataDir, "tls", "etcd", "peer-server-client.key") + runtime.ClientETCDCert = filepath.Join(config.DataDir, "tls", "etcd", "client.crt") + runtime.ClientETCDKey = filepath.Join(config.DataDir, "tls", "etcd", "client.key") + + if config.EncryptSecrets { + runtime.EncryptionConfig = filepath.Join(config.DataDir, "cred", "encryption-config.json") + } +} + // GenServerDeps is responsible for generating the cluster dependencies // needed to successfully bootstrap a cluster. func GenServerDeps(config *config.Control, runtime *config.ControlRuntime) error { diff --git a/pkg/daemons/control/server.go b/pkg/daemons/control/server.go index 7653555cbc..29844189cf 100644 --- a/pkg/daemons/control/server.go +++ b/pkg/daemons/control/server.go @@ -248,62 +248,7 @@ func prepare(ctx context.Context, config *config.Control, runtime *config.Contro os.MkdirAll(filepath.Join(config.DataDir, "tls"), 0700) os.MkdirAll(filepath.Join(config.DataDir, "cred"), 0700) - runtime.ClientCA = filepath.Join(config.DataDir, "tls", "client-ca.crt") - runtime.ClientCAKey = filepath.Join(config.DataDir, "tls", "client-ca.key") - runtime.ServerCA = filepath.Join(config.DataDir, "tls", "server-ca.crt") - runtime.ServerCAKey = filepath.Join(config.DataDir, "tls", "server-ca.key") - runtime.RequestHeaderCA = filepath.Join(config.DataDir, "tls", "request-header-ca.crt") - runtime.RequestHeaderCAKey = filepath.Join(config.DataDir, "tls", "request-header-ca.key") - runtime.IPSECKey = filepath.Join(config.DataDir, "cred", "ipsec.psk") - - runtime.ServiceKey = filepath.Join(config.DataDir, "tls", "service.key") - runtime.PasswdFile = filepath.Join(config.DataDir, "cred", "passwd") - runtime.NodePasswdFile = filepath.Join(config.DataDir, "cred", "node-passwd") - - runtime.KubeConfigAdmin = filepath.Join(config.DataDir, "cred", "admin.kubeconfig") - runtime.KubeConfigController = filepath.Join(config.DataDir, "cred", "controller.kubeconfig") - runtime.KubeConfigScheduler = filepath.Join(config.DataDir, "cred", "scheduler.kubeconfig") - runtime.KubeConfigAPIServer = filepath.Join(config.DataDir, "cred", "api-server.kubeconfig") - runtime.KubeConfigCloudController = filepath.Join(config.DataDir, "cred", "cloud-controller.kubeconfig") - - runtime.ClientAdminCert = filepath.Join(config.DataDir, "tls", "client-admin.crt") - runtime.ClientAdminKey = filepath.Join(config.DataDir, "tls", "client-admin.key") - runtime.ClientControllerCert = filepath.Join(config.DataDir, "tls", "client-controller.crt") - runtime.ClientControllerKey = filepath.Join(config.DataDir, "tls", "client-controller.key") - runtime.ClientCloudControllerCert = filepath.Join(config.DataDir, "tls", "client-"+version.Program+"-cloud-controller.crt") - runtime.ClientCloudControllerKey = filepath.Join(config.DataDir, "tls", "client-"+version.Program+"-cloud-controller.key") - runtime.ClientSchedulerCert = filepath.Join(config.DataDir, "tls", "client-scheduler.crt") - runtime.ClientSchedulerKey = filepath.Join(config.DataDir, "tls", "client-scheduler.key") - runtime.ClientKubeAPICert = filepath.Join(config.DataDir, "tls", "client-kube-apiserver.crt") - runtime.ClientKubeAPIKey = filepath.Join(config.DataDir, "tls", "client-kube-apiserver.key") - runtime.ClientKubeProxyCert = filepath.Join(config.DataDir, "tls", "client-kube-proxy.crt") - runtime.ClientKubeProxyKey = filepath.Join(config.DataDir, "tls", "client-kube-proxy.key") - runtime.ClientK3sControllerCert = filepath.Join(config.DataDir, "tls", "client-"+version.Program+"-controller.crt") - runtime.ClientK3sControllerKey = filepath.Join(config.DataDir, "tls", "client-"+version.Program+"-controller.key") - - runtime.ServingKubeAPICert = filepath.Join(config.DataDir, "tls", "serving-kube-apiserver.crt") - runtime.ServingKubeAPIKey = filepath.Join(config.DataDir, "tls", "serving-kube-apiserver.key") - - runtime.ClientKubeletKey = filepath.Join(config.DataDir, "tls", "client-kubelet.key") - runtime.ServingKubeletKey = filepath.Join(config.DataDir, "tls", "serving-kubelet.key") - - runtime.ClientAuthProxyCert = filepath.Join(config.DataDir, "tls", "client-auth-proxy.crt") - runtime.ClientAuthProxyKey = filepath.Join(config.DataDir, "tls", "client-auth-proxy.key") - - runtime.ETCDServerCA = filepath.Join(config.DataDir, "tls", "etcd", "server-ca.crt") - runtime.ETCDServerCAKey = filepath.Join(config.DataDir, "tls", "etcd", "server-ca.key") - runtime.ETCDPeerCA = filepath.Join(config.DataDir, "tls", "etcd", "peer-ca.crt") - runtime.ETCDPeerCAKey = filepath.Join(config.DataDir, "tls", "etcd", "peer-ca.key") - runtime.ServerETCDCert = filepath.Join(config.DataDir, "tls", "etcd", "server-client.crt") - runtime.ServerETCDKey = filepath.Join(config.DataDir, "tls", "etcd", "server-client.key") - runtime.PeerServerClientETCDCert = filepath.Join(config.DataDir, "tls", "etcd", "peer-server-client.crt") - runtime.PeerServerClientETCDKey = filepath.Join(config.DataDir, "tls", "etcd", "peer-server-client.key") - runtime.ClientETCDCert = filepath.Join(config.DataDir, "tls", "etcd", "client.crt") - runtime.ClientETCDKey = filepath.Join(config.DataDir, "tls", "etcd", "client.key") - - if config.EncryptSecrets { - runtime.EncryptionConfig = filepath.Join(config.DataDir, "cred", "encryption-config.json") - } + deps.FillRuntimeCerts(config, runtime) cluster := cluster.New(config) diff --git a/pkg/etcd/etcd_test.go b/pkg/etcd/etcd_test.go new file mode 100644 index 0000000000..5f9b03d45d --- /dev/null +++ b/pkg/etcd/etcd_test.go @@ -0,0 +1,299 @@ +package etcd + +import ( + "context" + "net" + "net/http" + "os" + "path/filepath" + "testing" + + "github.com/rancher/k3s/pkg/clientaccess" + "github.com/rancher/k3s/pkg/daemons/config" + "github.com/rancher/k3s/pkg/util/tests" + "github.com/robfig/cron/v3" + etcd "go.etcd.io/etcd/clientv3" +) + +func generateTestConfig() *config.Control { + _, clusterIPNet, _ := net.ParseCIDR("10.42.0.0/16") + _, serviceIPNet, _ := net.ParseCIDR("10.43.0.0/16") + + return &config.Control{ + HTTPSPort: 6443, + SupervisorPort: 6443, + AdvertisePort: 6443, + ClusterDomain: "cluster.local", + ClusterDNS: net.ParseIP("10.43.0.10"), + ClusterIPRange: clusterIPNet, + DataDir: "/tmp/k3s/", // Different than the default value + FlannelBackend: "vxlan", + EtcdSnapshotName: "etcd-snapshot", + EtcdSnapshotCron: "0 */12 * * *", + EtcdSnapshotRetention: 5, + EtcdS3Endpoint: "s3.amazonaws.com", + EtcdS3Region: "us-east-1", + SANs: []string{"127.0.0.1"}, + ServiceIPRange: serviceIPNet, + } +} + +func generateTestHandler() http.Handler { + return http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {}) +} + +func TestETCD_IsInitialized(t *testing.T) { + type args struct { + ctx context.Context + config *config.Control + } + tests := []struct { + name string + args args + setup func(*config.Control) error + teardown func(*config.Control) error + want bool + wantErr bool + }{ + { + name: "Directory exists", + args: args{ + ctx: context.TODO(), + config: generateTestConfig(), + }, + setup: func(cnf *config.Control) error { + if err := tests.GenerateDataDir(cnf); err != nil { + return err + } + return os.MkdirAll(walDir(cnf), 0700) + }, + teardown: func(cnf *config.Control) error { + tests.CleanupDataDir(cnf) + return os.Remove(walDir(cnf)) + }, + wantErr: false, + want: true, + }, + { + name: "Directory does not exist", + args: args{ + ctx: context.TODO(), + config: generateTestConfig(), + }, + setup: func(cnf *config.Control) error { + if err := tests.GenerateDataDir(cnf); err != nil { + return err + } + // We don't care if removal fails to find the dir + os.Remove(walDir(cnf)) + return nil + }, + teardown: func(cnf *config.Control) error { + tests.CleanupDataDir(cnf) + return nil + }, + wantErr: false, + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + e := NewETCD() + defer tt.teardown(tt.args.config) + if err := tt.setup(tt.args.config); err != nil { + t.Errorf("Prep for ETCD.IsInitialized() failed = %v", err) + return + } + got, err := e.IsInitialized(tt.args.ctx, tt.args.config) + if (err != nil) != tt.wantErr { + t.Errorf("ETCD.IsInitialized() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("ETCD.IsInitialized() = %+v\nWant = %+v", got, tt.want) + return + } + }) + } +} + +func TestETCD_Register(t *testing.T) { + type args struct { + ctx context.Context + config *config.Control + handler http.Handler + } + tests := []struct { + name string + args args + setup func(cnf *config.Control) error + teardown func(cnf *config.Control) error + wantErr bool + }{ + { + name: "Call Register with standard config", + args: args{ + ctx: context.TODO(), + config: generateTestConfig(), + handler: generateTestHandler(), + }, + setup: func(cnf *config.Control) error { + return tests.GenerateRuntime(cnf) + }, + teardown: func(cnf *config.Control) error { + tests.CleanupDataDir(cnf) + return nil + }, + }, + { + name: "Call Register with a tombstone file created", + args: args{ + ctx: context.TODO(), + config: generateTestConfig(), + handler: generateTestHandler(), + }, + setup: func(cnf *config.Control) error { + if err := tests.GenerateRuntime(cnf); err != nil { + return err + } + if err := os.MkdirAll(etcdDBDir(cnf), 0700); err != nil { + return err + } + tombstoneFile := filepath.Join(etcdDBDir(cnf), "tombstone") + if _, err := os.Create(tombstoneFile); err != nil { + return err + } + return nil + }, + teardown: func(cnf *config.Control) error { + tombstoneFile := filepath.Join(etcdDBDir(cnf), "tombstone") + os.Remove(tombstoneFile) + tests.CleanupDataDir(cnf) + return nil + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + e := NewETCD() + + defer tt.teardown(tt.args.config) + if err := tt.setup(tt.args.config); err != nil { + t.Errorf("Setup for ETCD.Register() failed = %v", err) + return + } + _, err := e.Register(tt.args.ctx, tt.args.config, tt.args.handler) + if (err != nil) != tt.wantErr { + t.Errorf("ETCD.Register() error = %v, wantErr %v", err, tt.wantErr) + return + } + }) + } +} + +func TestETCD_Start(t *testing.T) { + type fields struct { + client *etcd.Client + config *config.Control + name string + runtime *config.ControlRuntime + address string + cron *cron.Cron + s3 *S3 + } + type args struct { + ctx context.Context + clientAccessInfo *clientaccess.Info + } + tests := []struct { + name string + fields fields + args args + setup func(cnf *config.Control) error + teardown func(cnf *config.Control) error + wantErr bool + }{ + { + name: "Start etcd without clientAccessInfo and without snapshots", + fields: fields{ + config: generateTestConfig(), + address: "192.168.1.123", // Local IP address + }, + args: args{ + ctx: context.TODO(), + clientAccessInfo: nil, + }, + setup: func(cnf *config.Control) error { + cnf.EtcdDisableSnapshots = true + return tests.GenerateRuntime(cnf) + }, + teardown: func(cnf *config.Control) error { + tests.CleanupDataDir(cnf) + return nil + }, + }, + { + name: "Start etcd without clientAccessInfo on", + fields: fields{ + config: generateTestConfig(), + address: "192.168.1.123", // Local IP address + cron: cron.New(), + }, + args: args{ + ctx: context.TODO(), + clientAccessInfo: nil, + }, + setup: func(cnf *config.Control) error { + return tests.GenerateRuntime(cnf) + }, + teardown: func(cnf *config.Control) error { + tests.CleanupDataDir(cnf) + return nil + }, + }, + { + name: "Start etcd with an existing cluster", + fields: fields{ + config: generateTestConfig(), + address: "192.168.1.123", // Local IP address + cron: cron.New(), + }, + args: args{ + ctx: context.TODO(), + clientAccessInfo: nil, + }, + setup: func(cnf *config.Control) error { + if err := tests.GenerateRuntime(cnf); err != nil { + return err + } + return os.MkdirAll(walDir(cnf), 0700) + }, + teardown: func(cnf *config.Control) error { + tests.CleanupDataDir(cnf) + os.Remove(walDir(cnf)) + return nil + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + e := &ETCD{ + client: tt.fields.client, + config: tt.fields.config, + name: tt.fields.name, + runtime: tt.fields.runtime, + address: tt.fields.address, + cron: tt.fields.cron, + s3: tt.fields.s3, + } + defer tt.teardown(e.config) + if err := tt.setup(e.config); err != nil { + t.Errorf("Setup for ETCD.Start() failed = %v", err) + return + } + if err := e.Start(tt.args.ctx, tt.args.clientAccessInfo); (err != nil) != tt.wantErr { + t.Errorf("ETCD.Start() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} diff --git a/pkg/util/tests/runtime.go b/pkg/util/tests/runtime.go new file mode 100644 index 0000000000..7af97c9a2b --- /dev/null +++ b/pkg/util/tests/runtime.go @@ -0,0 +1,59 @@ +package tests + +import ( + "os" + "path/filepath" + + "github.com/rancher/k3s/pkg/daemons/config" + "github.com/rancher/k3s/pkg/daemons/control/deps" +) + +// GenerateDataDir creates a temporary directory at "/tmp/k3s//". +// The latest directory created with this function is soft linked to "/tmp/k3s/latest/". +// This allows tests to replicate the "/var/lib/rancher/k3s" directory structure. +func GenerateDataDir(cnf *config.Control) error { + if err := os.MkdirAll(cnf.DataDir, 0700); err != nil { + return err + } + testDir, err := os.MkdirTemp(cnf.DataDir, "*") + if err != nil { + return err + } + // Remove old symlink and add new one + os.Remove(filepath.Join(cnf.DataDir, "latest")) + if err = os.Symlink(testDir, filepath.Join(cnf.DataDir, "latest")); err != nil { + return err + } + cnf.DataDir = testDir + cnf.DataDir, err = filepath.Abs(cnf.DataDir) + if err != nil { + return err + } + return nil +} + +// CleanupDataDir removes the associated "/tmp/k3s/" +// directory. +func CleanupDataDir(cnf *config.Control) { + os.RemoveAll(cnf.DataDir) +} + +// GenerateRuntime creates a temporary data dir and configures +// config.ControlRuntime with all the appropriate certificate keys. +func GenerateRuntime(cnf *config.Control) error { + runtime := &config.ControlRuntime{} + if err := GenerateDataDir(cnf); err != nil { + return err + } + + os.MkdirAll(filepath.Join(cnf.DataDir, "tls"), 0700) + os.MkdirAll(filepath.Join(cnf.DataDir, "cred"), 0700) + + deps.FillRuntimeCerts(cnf, runtime) + + if err := deps.GenServerDeps(cnf, runtime); err != nil { + return err + } + cnf.Runtime = runtime + return nil +}