From 923529f49a9f4a8d9678e087735459ea875ac5d6 Mon Sep 17 00:00:00 2001 From: Yuhang Wei Date: Fri, 4 Aug 2023 10:17:24 +0800 Subject: [PATCH 04/17] KubeOS: add agent, proxy and operator ut add disk_image, server, config, proxy and operator unit testing fix the bug that agent allows configuring kv with nil key Signed-off-by: Yuhang Wei --- Makefile | 4 +- cmd/agent/server/config.go | 4 + cmd/agent/server/config_test.go | 130 +++++++-- cmd/agent/server/disk_image.go | 8 +- cmd/agent/server/disk_image_test.go | 262 ++++++++++++------ cmd/agent/server/server.go | 4 + cmd/agent/server/server_test.go | 86 +++++- cmd/agent/server/utils_test.go | 146 +++++++++- .../controllers/os_controller_test.go | 67 +++++ cmd/operator/controllers/suite_test.go | 2 +- cmd/proxy/controllers/os_controller.go | 6 + cmd/proxy/controllers/os_controller_test.go | 153 +++++++++- cmd/proxy/controllers/suite_test.go | 2 +- 13 files changed, 748 insertions(+), 126 deletions(-) diff --git a/Makefile b/Makefile index fbabda6..b5b6161 100644 --- a/Makefile +++ b/Makefile @@ -133,14 +133,14 @@ kustomize: $(call go-get-tool,$(KUSTOMIZE),sigs.k8s.io/kustomize/kustomize/v3@v3.8.7) ARCH := $(shell uname -m) -TEST_CMD := go test ./... -race -count=1 -timeout=300s -cover -gcflags=all=-l +TEST_CMD := go test ./... -race -count=1 -timeout=300s -cover -gcflags=all=-l -p 1 ifeq ($(ARCH), aarch64) TEST_CMD := ETCD_UNSUPPORTED_ARCH=arm64 $(TEST_CMD) endif .PHONY: test -test: manifests fmt vet envtest ## Run tests. +test: fmt vet envtest ## Run tests. KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" $(TEST_CMD) .PHONY: envtest diff --git a/cmd/agent/server/config.go b/cmd/agent/server/config.go index e7110f8..653f913 100644 --- a/cmd/agent/server/config.go +++ b/cmd/agent/server/config.go @@ -374,6 +374,10 @@ func handleUpdateKey(config []string, configInfo *agent.KeyInfo, isFound bool) s func handleAddKey(m map[string]*agent.KeyInfo, isOnlyKeyValid bool) []string { var configs []string for key, keyInfo := range m { + if key == "" { + logrus.Warnln("Failed to add nil key") + continue + } if keyInfo.Operation == "delete" { logrus.Warnf("Failed to delete inexistent key %s", key) continue diff --git a/cmd/agent/server/config_test.go b/cmd/agent/server/config_test.go index 903af87..2deb15f 100644 --- a/cmd/agent/server/config_test.go +++ b/cmd/agent/server/config_test.go @@ -67,6 +67,15 @@ func TestKernelSysctl_SetConfig(t *testing.T) { }, }}, }, + { + name: "nil value", + k: KernelSysctl{}, + args: args{config: &agent.SysConfig{ + Contents: map[string]*agent.KeyInfo{ + "d": {Value: ""}, + }, + }}, + }, } tmpDir := t.TempDir() patchGetProcPath := gomonkey.ApplyFuncReturn(getDefaultProcPath, tmpDir+"/") @@ -84,6 +93,7 @@ func TestKernelSysctl_SetConfig(t *testing.T) { func TestKerSysctlPersist_SetConfig(t *testing.T) { tmpDir := t.TempDir() persistPath := tmpDir + "/test-persist.conf" + comment := `# This file is managed by KubeOS for unit testing.` type args struct { config *agent.SysConfig } @@ -94,6 +104,7 @@ func TestKerSysctlPersist_SetConfig(t *testing.T) { want []string wantErr bool }{ + {name: "create file", args: args{config: &agent.SysConfig{ConfigPath: persistPath}}, want: []string{comment}, wantErr: false}, { name: "add configs", args: args{ @@ -103,12 +114,15 @@ func TestKerSysctlPersist_SetConfig(t *testing.T) { "a": {Value: "1"}, "b": {Value: "2"}, "c": {Value: ""}, + "": {Value: "4"}, + "e": {Value: "5"}, }, }, }, want: []string{ "a=1", "b=2", + "e=5", }, wantErr: false, }, @@ -126,6 +140,7 @@ func TestKerSysctlPersist_SetConfig(t *testing.T) { want: []string{ "a=2", "b=2", + "e=5", }, wantErr: false, }, @@ -137,11 +152,16 @@ func TestKerSysctlPersist_SetConfig(t *testing.T) { Contents: map[string]*agent.KeyInfo{ "a": {Value: "1", Operation: "delete"}, "b": {Value: "2", Operation: "delete"}, + "c": {Value: "3", Operation: "delete"}, + "e": {Value: "5", Operation: "remove"}, + "f": {Value: "6", Operation: "remove"}, }, }, }, want: []string{ "a=2", + "e=5", + "f=6", }, wantErr: false, }, @@ -152,14 +172,21 @@ func TestKerSysctlPersist_SetConfig(t *testing.T) { if err := k.SetConfig(tt.args.config); (err != nil) != tt.wantErr { t.Errorf("KerSysctlPersist.SetConfig() error = %v, wantErr %v", err, tt.wantErr) } + if tt.name == "create file" { + if err := os.WriteFile(persistPath, []byte(comment), 0644); err != nil { + t.Fatalf("failed to write file %s", persistPath) + } + } data, err := os.ReadFile(persistPath) if err != nil { t.Errorf("failed to read file %s", persistPath) } lines := strings.Split(string(data), "\n") - // remove the last empty line - lines = lines[:len(lines)-1] - sort.Strings(lines) + if tt.name != "create file" { + // remove the comment and the last empty line + lines = lines[1 : len(lines)-1] + sort.Strings(lines) + } if !reflect.DeepEqual(lines, tt.want) { t.Errorf("KerSysctlPersist file contents not equal, expect: %v, get: %v", tt.want, lines) } @@ -210,17 +237,36 @@ menuentry 'B' --class KubeOS --class gnu-linux --class gnu --class os --unrestri args: args{ config: &agent.SysConfig{ Contents: map[string]*agent.KeyInfo{ - "panic": {Value: "5"}, // update existent kv - "quiet": {Value: "", Operation: "delete"}, // delete existent key - "oops": {Value: ""}, // update existent kv with null value - "selinux": {Value: "1", Operation: "delete"}, // failed to delete inconsistent kv - "acpi": {Value: "off", Operation: "delete"}, // failed to delete inexistent kv - "debug": {}, // add key - "pci": {Value: "nomis"}, // add kv + "debug": {}, // add key + "pci": {Value: "nomis"}, // add kv + "quiet": {Value: "", Operation: "delete"}, // delete existent key + "panic": {Value: "5"}, // update existent kv + "nomodeset": {Operation: "update"}, // invalid operation, default to update existent key + "softlockup_panic": {Value: "0", Operation: "update"}, // invalid operation, default to update existent kv + "oops": {Value: ""}, // warning, skip, update existent kv with null value + "": {Value: "test"}, // warning, skip, failed to add kv with empty key + "selinux": {Value: "1", Operation: "delete"}, // failed to delete inconsistent kv + "acpi": {Value: "off", Operation: "delete"}, // failed to delete inexistent kv }, }, }, - pattern: `(?m)^\s+linux\s+\/boot\/vmlinuz\s+root=UUID=[0-1]\s+ro\s+rootfstype=ext4\s+nomodeset\s+oops=panic\s+softlockup_panic=1\s+nmi_watchdog=1\s+rd\.shell=0\s+selinux=0\s+crashkernel=256M\s+panic=5\s+(debug\spci=nomis|pci=nomis\sdebug)$`, + pattern: `(?m)^\s+linux\s+\/boot\/vmlinuz\s+root=UUID=[0-1]\s+ro\s+rootfstype=ext4\s+nomodeset\s+oops=panic\s+softlockup_panic=0\s+nmi_watchdog=1\s+rd\.shell=0\s+selinux=0\s+crashkernel=256M\s+panic=5\s+(debug\spci=nomis|pci=nomis\sdebug)$`, + wantErr: false, + }, + { + name: "delete and invalid operation", + g: GrubCmdline{isCurPartition: true}, + args: args{ + config: &agent.SysConfig{ + Contents: map[string]*agent.KeyInfo{ + "debug": {Operation: "delete"}, // delete key + "pci": {Value: "nomis", Operation: "delete"}, // delete kv + "debugpat": {Value: "", Operation: "add"}, // passed key, operation is invalid, default to add key + "audit": {Value: "1", Operation: "add"}, // passed kv, key is inexistent, operation is invalid, default to add kv + }, + }, + }, + pattern: `(?m)^\s+linux\s+\/boot\/vmlinuz\s+root=UUID=[0-1]\s+ro\s+rootfstype=ext4\s+nomodeset\s+oops=panic\s+softlockup_panic=0\s+nmi_watchdog=1\s+rd\.shell=0\s+selinux=0\s+crashkernel=256M\s+panic=5\s+(debugpat\saudit=1|audit=1\sdebugpat)$`, wantErr: false, }, { @@ -229,23 +275,27 @@ menuentry 'B' --class KubeOS --class gnu-linux --class gnu --class os --unrestri args: args{ config: &agent.SysConfig{ Contents: map[string]*agent.KeyInfo{ - "panic": {Value: "4"}, - "quiet": {Value: "", Operation: "delete"}, - "oops": {Value: ""}, // update existent kv with null value - "selinux": {Value: "1", Operation: "delete"}, - "acpi": {Value: "off", Operation: "delete"}, - "debug": {}, - "pci": {Value: "nomis"}, + "debug": {}, + "pci": {Value: "nomis"}, + "quiet": {Value: "", Operation: "delete"}, + "panic": {Value: "4"}, + "nomodeset": {Operation: "update"}, // invalid operation, default to update existent key + "softlockup_panic": {Value: "0", Operation: "update"}, // invalid operation, default to update existent kv + "oops": {Value: ""}, // update existent kv with null value + "": {Value: "test"}, // warning, skip, failed to add kv with empty key + "selinux": {Value: "1", Operation: "delete"}, + "acpi": {Value: "off", Operation: "delete"}, }, }, }, - pattern: `(?m)^\s+linux\s+\/boot\/vmlinuz\s+root=UUID=[0-1]\s+ro\s+rootfstype=ext4\s+nomodeset\s+oops=panic\s+softlockup_panic=1\s+nmi_watchdog=1\s+rd\.shell=0\s+selinux=0\s+crashkernel=256M\s+panic=4\s+(debug\spci=nomis|pci=nomis\sdebug)$`, + pattern: `(?m)^\s+linux\s+\/boot\/vmlinuz\s+root=UUID=[0-1]\s+ro\s+rootfstype=ext4\s+nomodeset\s+oops=panic\s+softlockup_panic=0\s+nmi_watchdog=1\s+rd\.shell=0\s+selinux=0\s+crashkernel=256M\s+panic=4\s+(debug\spci=nomis|pci=nomis\sdebug)$`, wantErr: false, }, } patchGetGrubPath := gomonkey.ApplyFuncReturn(getGrubCfgPath, grubCfgPath) defer patchGetGrubPath.Reset() patchGetConfigPartition := gomonkey.ApplyFuncSeq(getConfigPartition, []gomonkey.OutputCell{ + {Values: gomonkey.Params{false, nil}}, {Values: gomonkey.Params{false, nil}}, {Values: gomonkey.Params{true, nil}}, }) @@ -375,3 +425,45 @@ func Test_getGrubCfgPath(t *testing.T) { }) } } + +func Test_getConfigPartition(t *testing.T) { + type args struct { + isCurPartition bool + } + tests := []struct { + name string + args args + want bool + wantErr bool + }{ + { + name: "get current partition", + args: args{isCurPartition: true}, + want: false, + wantErr: false, + }, + { + name: "get next partition", + args: args{isCurPartition: false}, + want: true, + wantErr: false, + }, + } + patchRootfsDisks := gomonkey.ApplyFuncReturn(getRootfsDisks, "/dev/sda2", "/dev/sda3", nil) + defer patchRootfsDisks.Reset() + // assume now is partition A, want to swiching to partition B + patchGetNextPartition := gomonkey.ApplyFuncReturn(getNextPart, "/dev/sda3", "B", nil) + defer patchGetNextPartition.Reset() + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := getConfigPartition(tt.args.isCurPartition) + if (err != nil) != tt.wantErr { + t.Errorf("getConfigPartition() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("getConfigPartition() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/cmd/agent/server/disk_image.go b/cmd/agent/server/disk_image.go index c89003b..8bd6bf6 100644 --- a/cmd/agent/server/disk_image.go +++ b/cmd/agent/server/disk_image.go @@ -153,7 +153,7 @@ func loadCaCerts(caCert string) (*http.Client, error) { if err != nil { return &http.Client{}, err } - ca, err := ioutil.ReadFile(certPath + caCert) + ca, err := ioutil.ReadFile(getCertPath() + caCert) if err != nil { return &http.Client{}, fmt.Errorf("read the ca certificate error %s", err) } @@ -173,7 +173,7 @@ func loadClientCerts(caCert, clientCert, clientKey string) (*http.Client, error) if err != nil { return &http.Client{}, err } - ca, err := ioutil.ReadFile(certPath + caCert) + ca, err := ioutil.ReadFile(getCertPath() + caCert) if err != nil { return &http.Client{}, err } @@ -186,7 +186,7 @@ func loadClientCerts(caCert, clientCert, clientKey string) (*http.Client, error) if err != nil { return &http.Client{}, err } - cliCrt, err := tls.LoadX509KeyPair(certPath+clientCert, certPath+clientKey) + cliCrt, err := tls.LoadX509KeyPair(getCertPath()+clientCert, getCertPath()+clientKey) if err != nil { return &http.Client{}, err } @@ -206,7 +206,7 @@ func certExist(certFile string) error { if certFile == "" { return fmt.Errorf("please provide the certificate") } - _, err := os.Stat(certPath + certFile) + _, err := os.Stat(getCertPath() + certFile) if err != nil { if os.IsNotExist(err) { return fmt.Errorf("certificate is not exist %s ", err) diff --git a/cmd/agent/server/disk_image_test.go b/cmd/agent/server/disk_image_test.go index 3c82113..71c5de7 100644 --- a/cmd/agent/server/disk_image_test.go +++ b/cmd/agent/server/disk_image_test.go @@ -14,19 +14,35 @@ package server import ( - "crypto/tls" + "crypto/rand" + "crypto/rsa" + "crypto/sha256" "crypto/x509" + "crypto/x509/pkix" + "encoding/hex" + "encoding/pem" + "io" + "math/big" "net/http" "os" "reflect" + "strings" + "syscall" "testing" + "time" "github.com/agiledragon/gomonkey/v2" - pb "openeuler.org/KubeOS/cmd/agent/api" ) -func Testdownload(t *testing.T) { +func Test_download(t *testing.T) { + tmpDir := t.TempDir() + tmpFileForDownload := tmpDir + "/tmpFileForDownload" + tmpFile, err := os.Create(tmpFileForDownload) + if err != nil { + t.Errorf("open file error: %v", err) + } + defer tmpFile.Close() type args struct { req *pb.UpdateRequest } @@ -36,14 +52,35 @@ func Testdownload(t *testing.T) { want string wantErr bool }{ - {name: "errornil", args: args{&pb.UpdateRequest{Certs: &pb.CertsInfo{}}}, want: "", wantErr: true}, - {name: "normal", args: args{&pb.UpdateRequest{ImageUrl: "http://www.openeuler.org/zh/", FlagSafe: true, Certs: &pb.CertsInfo{}}}, want: "/persist/update.img", wantErr: false}, - {name: "errornodir", args: args{&pb.UpdateRequest{ImageUrl: "http://www.openeuler.org/zh/", FlagSafe: true, Certs: &pb.CertsInfo{}}}, want: "", wantErr: true}, + // {name: "errornil", args: args{&pb.UpdateRequest{Certs: &pb.CertsInfo{}}}, want: "", wantErr: true}, + // {name: "normal", args: args{&pb.UpdateRequest{ImageUrl: "http://www.openeuler.org/zh/", FlagSafe: true, Certs: &pb.CertsInfo{}}}, want: "/persist/update.img", wantErr: false}, + // {name: "errornodir", args: args{&pb.UpdateRequest{ImageUrl: "http://www.openeuler.org/zh/", FlagSafe: true, Certs: &pb.CertsInfo{}}}, want: "", wantErr: true}, + { + name: "normal", + args: args{ + req: &pb.UpdateRequest{ + ImageUrl: "http://www.openeuler.org/zh/", + }, + }, + want: tmpFileForDownload, + wantErr: false, + }, } + patchStatfs := gomonkey.ApplyFunc(syscall.Statfs, func(path string, stat *syscall.Statfs_t) error { + stat.Bfree = 3000 + stat.Bsize = 4096 + return nil + }) + defer patchStatfs.Reset() + patchGetImageUrl := gomonkey.ApplyFuncReturn(getImageURL, &http.Response{ + StatusCode: http.StatusOK, + ContentLength: 5, + Body: io.NopCloser(strings.NewReader("hello")), + }, nil) + defer patchGetImageUrl.Reset() + patchOSCreate := gomonkey.ApplyFuncReturn(os.Create, tmpFile, nil) + defer patchOSCreate.Reset() for _, tt := range tests { - if tt.name == "normal" { - os.Mkdir("/persist", os.ModePerm) - } t.Run(tt.name, func(t *testing.T) { got, err := download(tt.args.req) if (err != nil) != tt.wantErr { @@ -54,47 +91,42 @@ func Testdownload(t *testing.T) { t.Errorf("download() got = %v, want %v", got, tt.want) } }) - if tt.name == "normal" { - os.RemoveAll("/persist") - } } } -func TestcheckSumMatch(t *testing.T) { +func Test_checkSumMatch(t *testing.T) { + tmpDir := t.TempDir() + tmpFileForCheckSum := tmpDir + "/tmpFileForCheckSum" + err := os.WriteFile(tmpFileForCheckSum, []byte("hello"), 0644) + if err != nil { + t.Errorf("open file error: %v", err) + } type args struct { filePath string checkSum string } - ff, _ := os.Create("aa.txt") - ff.Chmod(os.ModePerm) tests := []struct { name string args args wantErr bool }{ - {name: "error", args: args{filePath: "aaa", checkSum: "aaa"}, wantErr: true}, - {name: "errordir", args: args{filePath: "/aaa", checkSum: "/aaa"}, wantErr: true}, - {name: "errortxt", args: args{filePath: "aa.txt", checkSum: "aa.txt"}, wantErr: true}, + { + name: "normal", + args: args{filePath: tmpFileForCheckSum, checkSum: calculateChecksum("hello")}, + wantErr: false, + }, + {name: "error", args: args{filePath: tmpFileForCheckSum, checkSum: "aaa"}, wantErr: true}, } for _, tt := range tests { - if tt.name == "errordir" { - os.Mkdir("/aaa", os.ModePerm) - } t.Run(tt.name, func(t *testing.T) { if err := checkSumMatch(tt.args.filePath, tt.args.checkSum); (err != nil) != tt.wantErr { t.Errorf("checkSumMatch() error = %v, wantErr %v", err, tt.wantErr) } }) - if tt.name == "errordir" { - os.RemoveAll("/aaa") - } } - defer os.Remove("aa.txt") - defer ff.Close() - } -func TestgetImageURL(t *testing.T) { +func Test_getImageURL(t *testing.T) { type args struct { req *pb.UpdateRequest } @@ -105,23 +137,29 @@ func TestgetImageURL(t *testing.T) { wantErr bool }{ {name: "httpNotSafe", args: args{req: &pb.UpdateRequest{ - ImageUrl: "http://www.openeuler.org/zh/", + ImageUrl: "http://www.openeuler.abc/zh/", FlagSafe: false, MTLS: false, Certs: &pb.CertsInfo{}, }}, want: &http.Response{}, wantErr: true}, - {name: "mTLSError", args: args{req: &pb.UpdateRequest{ - ImageUrl: "http://www.openeuler.org/zh/", + {name: "httpSuccess", args: args{req: &pb.UpdateRequest{ + ImageUrl: "http://www.openeuler.abc/zh/", + FlagSafe: true, + MTLS: false, + Certs: &pb.CertsInfo{}, + }}, want: &http.Response{StatusCode: http.StatusOK}, wantErr: false}, + {name: "mTLSGetSuccess", args: args{req: &pb.UpdateRequest{ + ImageUrl: "https://www.openeuler.abc/zh/", FlagSafe: true, MTLS: true, Certs: &pb.CertsInfo{}, - }}, want: &http.Response{}, wantErr: true}, - {name: "httpsError", args: args{req: &pb.UpdateRequest{ - ImageUrl: "https://www.openeuler.org/zh/", + }}, want: &http.Response{StatusCode: http.StatusOK}, wantErr: false}, + {name: "httpsGetSuccess", args: args{req: &pb.UpdateRequest{ + ImageUrl: "https://www.openeuler.abc/zh/", FlagSafe: true, MTLS: false, Certs: &pb.CertsInfo{}, - }}, want: &http.Response{}, wantErr: true}, + }}, want: &http.Response{StatusCode: http.StatusOK}, wantErr: false}, } patchLoadClientCerts := gomonkey.ApplyFunc(loadClientCerts, func(caCert, clientCert, clientKey string) (*http.Client, error) { return &http.Client{}, nil @@ -131,8 +169,20 @@ func TestgetImageURL(t *testing.T) { return &http.Client{}, nil }) defer patchLoadCaCerts.Reset() + patchGet := gomonkey.ApplyFunc(http.Get, func(url string) (resp *http.Response, err error) { + return &http.Response{StatusCode: http.StatusOK}, nil + }) + defer patchGet.Reset() + patchClientGet := gomonkey.ApplyMethod(reflect.TypeOf(&http.Client{}), "Get", func(_ *http.Client, url string) (resp *http.Response, err error) { + return &http.Response{StatusCode: http.StatusOK}, nil + }) + defer patchClientGet.Reset() for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + if tt.name == "httpSuccess" { + patchGet := gomonkey.ApplyFuncReturn(http.Get, &http.Response{StatusCode: http.StatusOK}, nil) + defer patchGet.Reset() + } got, err := getImageURL(tt.args.req) if (err != nil) != tt.wantErr { t.Errorf("getImageURL() error = %v, wantErr %v", err, tt.wantErr) @@ -145,20 +195,28 @@ func TestgetImageURL(t *testing.T) { } } -func TestloadCaCerts(t *testing.T) { +func Test_loadCaCerts(t *testing.T) { + tmpDir := t.TempDir() + caPath := tmpDir + "/fake.crt" + createFakeCertKey(caPath, "") type args struct { caCert string } tests := []struct { name string args args - want *http.Client wantErr bool }{ - {name: "noCaCertError", args: args{caCert: "bb.txt"}, want: &http.Client{}, wantErr: true}, + { + name: "normal", + args: args{ + caCert: caPath, + }, + wantErr: false, + }, } - os.MkdirAll(certPath, 0644) - defer os.RemoveAll(certPath) + patchGetCertPath := gomonkey.ApplyFuncReturn(getCertPath, "") + defer patchGetCertPath.Reset() for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { got, err := loadCaCerts(tt.args.caCert) @@ -166,48 +224,39 @@ func TestloadCaCerts(t *testing.T) { t.Errorf("loadCaCerts() error = %v, wantErr %v", err, tt.wantErr) return } - if !reflect.DeepEqual(got, tt.want) { - t.Errorf("loadCaCerts() = %v, want %v", got, tt.want) + if got == nil { + t.Errorf("loadCaCerts() = %v", got) } }) } } -func TestloadClientCerts(t *testing.T) { +func Test_loadClientCerts(t *testing.T) { + tmpDir := t.TempDir() + clientCertPath := tmpDir + "/fakeClientCert.crt" + clientKeyPath := tmpDir + "/fakeClientKey.crt" + createFakeCertKey(clientCertPath, clientKeyPath) type args struct { caCert string clientCert string clientKey string } - pool := &x509.CertPool{} tests := []struct { name string args args - want *http.Client wantErr bool }{ - {name: "noCaCertError", args: args{" dd.txt", "bb.txt", "cc.txt"}, want: &http.Client{}, wantErr: true}, - {name: "noClientCertError", args: args{"ca.crt", "bb.txt", "cc.txt"}, want: &http.Client{}, wantErr: true}, - {name: "noClientKeyError", args: args{"ca.crt", "client.crt", "cc.txt"}, want: &http.Client{}, wantErr: true}, - } - os.MkdirAll(certPath, 0644) - caFile, _ := os.Create(certPath + "ca.crt") - clientCertFile, _ := os.Create(certPath + "client.crt") - clientKeyFile, _ := os.Create(certPath + "client.key") - - patchNewCertPool := gomonkey.ApplyFunc(x509.NewCertPool, func() *x509.CertPool { - return pool - }) - defer patchNewCertPool.Reset() - patchAppendCertsFromPEM := gomonkey.ApplyMethod(reflect.TypeOf(pool), "AppendCertsFromPEM", func(_ *x509.CertPool, _ []byte) (ok bool) { - return true - }) - defer patchAppendCertsFromPEM.Reset() - patchLoadX509KeyPair := gomonkey.ApplyFunc(tls.LoadX509KeyPair, func(certFile string, keyFile string) (tls.Certificate, error) { - return tls.Certificate{}, nil - }) - defer patchLoadX509KeyPair.Reset() + { + name: "normal", + args: args{ + caCert: clientCertPath, clientCert: clientCertPath, clientKey: clientKeyPath, + }, + wantErr: false, + }, + } + patchGetCertPath := gomonkey.ApplyFuncReturn(getCertPath, "") + defer patchGetCertPath.Reset() for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { got, err := loadClientCerts(tt.args.caCert, tt.args.clientCert, tt.args.clientKey) @@ -215,19 +264,14 @@ func TestloadClientCerts(t *testing.T) { t.Errorf("loadClientCerts() error = %v, wantErr %v", err, tt.wantErr) return } - if !reflect.DeepEqual(got, tt.want) { - t.Errorf("loadClientCerts() got = %v, want %v", got, tt.want) + if got == nil { + t.Errorf("loadClientCerts() got = %v", got) } }) } - caFile.Close() - clientCertFile.Close() - clientKeyFile.Close() - defer os.RemoveAll("/etc/KubeOS") - } -func TestcertExist(t *testing.T) { +func Test_certExist(t *testing.T) { type args struct { certFile string } @@ -238,10 +282,7 @@ func TestcertExist(t *testing.T) { }{ {name: "fileEmpty", args: args{certFile: ""}, wantErr: true}, {name: "fileNotExist", args: args{certFile: "bb.txt"}, wantErr: true}, - {name: "normal", args: args{certFile: "aa.txt"}, wantErr: false}, } - os.MkdirAll(certPath, 0644) - ff, _ := os.Create(certPath + "aa.txt") for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { if err := certExist(tt.args.certFile); (err != nil) != tt.wantErr { @@ -249,6 +290,71 @@ func TestcertExist(t *testing.T) { } }) } - ff.Close() defer os.RemoveAll("/etc/KubeOS/") } + +func createFakeCertKey(certPath, keyPath string) { + privateKey, _ := rsa.GenerateKey(rand.Reader, 2048) + template := x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{ + CommonName: "Fake Client Certificate", + }, + NotBefore: time.Now(), + NotAfter: time.Now().AddDate(1, 0, 0), + KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, + BasicConstraintsValid: true, + } + certBytes, _ := x509.CreateCertificate(rand.Reader, &template, &template, &privateKey.PublicKey, privateKey) + certPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certBytes}) + keyPEM := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(privateKey)}) + os.WriteFile(certPath, certPEM, 0644) + if keyPath != "" { + os.WriteFile(keyPath, keyPEM, 0644) + } +} + +func calculateChecksum(data string) string { + hash := sha256.New() + hash.Write([]byte(data)) + return hex.EncodeToString(hash.Sum(nil)) +} + +func Test_diskHandler_getRootfsArchive(t *testing.T) { + type args struct { + req *pb.UpdateRequest + neededPath preparePath + } + tests := []struct { + name string + d diskHandler + args args + want string + wantErr bool + }{ + { + name: "normal", d: diskHandler{}, + args: args{req: &pb.UpdateRequest{ImageUrl: "http://www.openeuler.org/zh/"}, neededPath: preparePath{}}, + want: "/persist/update.img", + wantErr: false, + }, + } + patchDownload := gomonkey.ApplyFuncReturn(download, "/persist/update.img", nil) + defer patchDownload.Reset() + patchCheckSumMatch := gomonkey.ApplyFuncReturn(checkSumMatch, nil) + defer patchCheckSumMatch.Reset() + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + d := diskHandler{} + got, err := d.getRootfsArchive(tt.args.req, tt.args.neededPath) + if (err != nil) != tt.wantErr { + t.Errorf("diskHandler.getRootfsArchive() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("diskHandler.getRootfsArchive() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/cmd/agent/server/server.go b/cmd/agent/server/server.go index b41ebc4..8ac6ffd 100644 --- a/cmd/agent/server/server.go +++ b/cmd/agent/server/server.go @@ -171,3 +171,7 @@ func (s *Server) reboot() error { } return syscall.Reboot(syscall.LINUX_REBOOT_CMD_RESTART) } + +func getCertPath() string { + return certPath +} diff --git a/cmd/agent/server/server_test.go b/cmd/agent/server/server_test.go index 0aac36a..74e2ead 100644 --- a/cmd/agent/server/server_test.go +++ b/cmd/agent/server/server_test.go @@ -89,7 +89,21 @@ func TestServerUpdate(t *testing.T) { {name: "error", fields: fields{UnimplementedOSServer: pb.UnimplementedOSServer{}, disableReboot: true}, args: args{in0: context.Background(), req: &pb.UpdateRequest{Version: "test", Certs: &pb.CertsInfo{}}}, want: &pb.UpdateResponse{}, wantErr: true}, + {name: "success", fields: fields{UnimplementedOSServer: pb.UnimplementedOSServer{}, disableReboot: true}, + args: args{in0: context.Background(), req: &pb.UpdateRequest{Version: "test", Certs: &pb.CertsInfo{}, ImageType: "containerd"}}, + want: &pb.UpdateResponse{}, wantErr: false}, } + patchRootfsDisks := gomonkey.ApplyFuncReturn(getRootfsDisks, "/dev/sda2", "/dev/sda3", nil) + defer patchRootfsDisks.Reset() + // assume now is partition A, want to swiching to partition B + patchGetNextPartition := gomonkey.ApplyFuncReturn(getNextPart, "/dev/sda3", "B", nil) + defer patchGetNextPartition.Reset() + patchDownloadImage := gomonkey.ApplyPrivateMethod(conImageHandler{}, "downloadImage", func(_ conImageHandler, req *pb.UpdateRequest) (string, error) { + return "", nil + }) + defer patchDownloadImage.Reset() + patchInstall := gomonkey.ApplyFuncReturn(install, nil) + defer patchInstall.Reset() for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { s := &Server{ @@ -129,11 +143,26 @@ func TestServerRollback(t *testing.T) { {name: "error", fields: fields{UnimplementedOSServer: pb.UnimplementedOSServer{}, disableReboot: true}, args: args{in0: context.Background(), req: &pb.RollbackRequest{}}, want: &pb.RollbackResponse{}, wantErr: true}, + {name: "success", fields: fields{UnimplementedOSServer: pb.UnimplementedOSServer{}, disableReboot: true}, + args: args{in0: context.Background(), req: &pb.RollbackRequest{}}, + want: &pb.RollbackResponse{}, wantErr: false}, } - patchGetNextPart := gomonkey.ApplyFunc(getNextPart, func(partA string, partB string) (string, string, error) { - return "", "", fmt.Errorf("rollbak test error") + patchRootfsDisks := gomonkey.ApplyFuncReturn(getRootfsDisks, "/dev/sda2", "/dev/sda3", nil) + defer patchRootfsDisks.Reset() + // assume now is partition A, want to swiching to partition B + patchGetNextPartition := gomonkey.ApplyFuncSeq(getNextPart, []gomonkey.OutputCell{ + {Values: gomonkey.Params{"", "", fmt.Errorf("rollbak test error")}}, + {Values: gomonkey.Params{"/dev/sda3", "B", nil}}, }) - defer patchGetNextPart.Reset() + defer patchGetNextPartition.Reset() + patchDownloadImage := gomonkey.ApplyPrivateMethod(conImageHandler{}, "downloadImage", func(_ conImageHandler, req *pb.UpdateRequest) (string, error) { + return "", nil + }) + defer patchDownloadImage.Reset() + patchInstall := gomonkey.ApplyFuncReturn(install, nil) + defer patchInstall.Reset() + patchRunCommand := gomonkey.ApplyFuncReturn(runCommand, nil) + defer patchRunCommand.Reset() for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { s := &Server{ @@ -179,9 +208,9 @@ func TestServerupdate(t *testing.T) { }}, wantErr: true}, {name: "errordocker", args: args{&pb.UpdateRequest{ - DockerImage: "", - ImageType: "docker", - Certs: &pb.CertsInfo{}, + ContainerImage: "", + ImageType: "docker", + Certs: &pb.CertsInfo{}, }}, wantErr: true}, } @@ -264,3 +293,48 @@ func TestServerreboot(t *testing.T) { }) } } + +func TestServer_Configure(t *testing.T) { + type fields struct { + UnimplementedOSServer pb.UnimplementedOSServer + mutex Lock + disableReboot bool + } + type args struct { + in0 context.Context + req *pb.ConfigureRequest + } + tests := []struct { + name string + fields fields + args args + want *pb.ConfigureResponse + wantErr bool + }{ + // TODO: Add test cases. + { + name: "nil", + fields: fields{UnimplementedOSServer: pb.UnimplementedOSServer{}, disableReboot: true}, + args: args{in0: context.Background(), req: &pb.ConfigureRequest{}}, + want: &pb.ConfigureResponse{}, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := &Server{ + UnimplementedOSServer: tt.fields.UnimplementedOSServer, + mutex: tt.fields.mutex, + disableReboot: tt.fields.disableReboot, + } + got, err := s.Configure(tt.args.in0, tt.args.req) + if (err != nil) != tt.wantErr { + t.Errorf("Server.Configure() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("Server.Configure() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/cmd/agent/server/utils_test.go b/cmd/agent/server/utils_test.go index 8e7fd90..89b2c3b 100644 --- a/cmd/agent/server/utils_test.go +++ b/cmd/agent/server/utils_test.go @@ -14,13 +14,17 @@ package server import ( + "archive/tar" "os" "os/exec" - "strings" + "reflect" "testing" + "time" + + "github.com/agiledragon/gomonkey/v2" ) -func TestrunCommand(t *testing.T) { +func Test_runCommand(t *testing.T) { type args struct { name string args []string @@ -41,23 +45,21 @@ func TestrunCommand(t *testing.T) { } } -func Testinstall(t *testing.T) { +func Test_install(t *testing.T) { type args struct { imagePath string side string next string } - out, _ := exec.Command("bash", "-c", "df -h | grep '/$' | awk '{print $1}'").CombinedOutput() - mountPart := strings.TrimSpace(string(out)) tests := []struct { name string args args wantErr bool }{ - {name: "normal", args: args{imagePath: "aa.txt", side: mountPart, next: ""}, wantErr: false}, + {name: "normal", args: args{imagePath: "aa.txt", side: "/dev/sda3", next: "A"}, wantErr: false}, } - ff, _ := os.Create("aa.txt") - ff.Chmod(os.ModePerm) + patchRunCommand := gomonkey.ApplyFuncReturn(runCommand, nil) + defer patchRunCommand.Reset() for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { if err := install(tt.args.imagePath, tt.args.side, tt.args.next); (err != nil) != tt.wantErr { @@ -65,17 +67,13 @@ func Testinstall(t *testing.T) { } }) } - ff.Close() - defer os.Remove("aa.txt") } -func TestgetNextPart(t *testing.T) { +func Test_getNextPart(t *testing.T) { type args struct { partA string partB string } - out, _ := exec.Command("bash", "-c", "df -h | grep '/$' | awk '{print $1}'").CombinedOutput() - mountPart := strings.TrimSpace(string(out)) tests := []struct { name string args args @@ -83,8 +81,14 @@ func TestgetNextPart(t *testing.T) { want1 string wantErr bool }{ - {name: "normal", args: args{partA: mountPart, partB: "testB"}, want: "testB", want1: "B", wantErr: false}, + {name: "switch to sda3", args: args{partA: "/dev/sda2", partB: "/dev/sda3"}, want: "/dev/sda3", want1: "B", wantErr: false}, + {name: "switch to sda2", args: args{partA: "/dev/sda2", partB: "/dev/sda3"}, want: "/dev/sda2", want1: "A", wantErr: false}, } + patchExecCommand := gomonkey.ApplyMethodSeq(&exec.Cmd{}, "CombinedOutput", []gomonkey.OutputCell{ + {Values: gomonkey.Params{[]byte("/"), nil}}, + {Values: gomonkey.Params{[]byte(""), nil}}, + }) + defer patchExecCommand.Reset() for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { got, got1, err := getNextPart(tt.args.partA, tt.args.partB) @@ -101,3 +105,117 @@ func TestgetNextPart(t *testing.T) { }) } } + +func Test_prepareEnv(t *testing.T) { + mountPath := "/persist/KubeOS-Update/kubeos-update" + if err := os.MkdirAll(mountPath, 0644); err != nil { + t.Fatalf("mkdir err %v", err) + } + defer os.RemoveAll("/persist") + tests := []struct { + name string + want preparePath + wantErr bool + }{ + { + name: "success", + want: preparePath{ + updatePath: "/persist/KubeOS-Update", + mountPath: "/persist/KubeOS-Update/kubeos-update", + tarPath: "/persist/KubeOS-Update/os.tar", + imagePath: "/persist/update.img", + rootfsFile: "os.tar", + }, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := prepareEnv() + if (err != nil) != tt.wantErr { + t.Errorf("prepareEnv() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("prepareEnv() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_createOSImage(t *testing.T) { + mountPath := "/persist/KubeOS-Update/kubeos-update" + if err := os.MkdirAll(mountPath, 0644); err != nil { + t.Fatalf("mkdir err %v", err) + } + defer os.RemoveAll("/persist") + tarPath := "/persist/KubeOS-Update/os.tar" + path, err := createTmpTarFile(tarPath) + if path != tarPath && err != nil { + t.Fatalf("create temp zip file err %v", err) + } + type args struct { + neededPath preparePath + } + tests := []struct { + name string + args args + want string + wantErr bool + }{ + { + name: "normal", + args: args{ + neededPath: preparePath{ + updatePath: "/persist/KubeOS-Update", + mountPath: "/persist/KubeOS-Update/kubeos-update", + tarPath: "/persist/KubeOS-Update/os.tar", + imagePath: "/persist/update.img", + }, + }, + want: "/persist/update.img", + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := createOSImage(tt.args.neededPath) + if (err != nil) != tt.wantErr { + t.Errorf("createOSImage() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("createOSImage() = %v, want %v", got, tt.want) + } + }) + } +} + +func createTmpTarFile(tarPath string) (string, error) { + tempFile, err := os.Create(tarPath) + if err != nil { + return "", err + } + defer tempFile.Close() + + tarWriter := tar.NewWriter(tempFile) + fakeData := []byte("This is a fake file") + fakeFile := "fakefile.txt" + header := &tar.Header{ + Name: fakeFile, + Size: int64(len(fakeData)), + Mode: 0644, + ModTime: time.Now(), + } + + if err = tarWriter.WriteHeader(header); err != nil { + return "", err + } + if _, err := tarWriter.Write(fakeData); err != nil { + return "", err + } + if err := tarWriter.Flush(); err != nil { + return "", err + } + return tempFile.Name(), nil +} diff --git a/cmd/operator/controllers/os_controller_test.go b/cmd/operator/controllers/os_controller_test.go index a391005..98de6d0 100644 --- a/cmd/operator/controllers/os_controller_test.go +++ b/cmd/operator/controllers/os_controller_test.go @@ -14,16 +14,22 @@ package controllers import ( "context" + "testing" "time" + "github.com/agiledragon/gomonkey/v2" "github.com/google/uuid" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/util/workqueue" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/event" upgradev1 "openeuler.org/KubeOS/api/v1alpha1" "openeuler.org/KubeOS/pkg/values" @@ -506,3 +512,64 @@ var _ = Describe("OsController", func() { }) }) }) + +func TestOSReconciler_DeleteOSInstance(t *testing.T) { + type fields struct { + Scheme *runtime.Scheme + Client client.Client + } + kClient, _ := client.New(cfg, client.Options{Scheme: scheme.Scheme}) + type args struct { + e event.DeleteEvent + q workqueue.RateLimitingInterface + } + tests := []struct { + name string + fields fields + args args + }{ + { + name: "delete osinstance", + fields: fields{ + Scheme: nil, + Client: kClient, + }, + args: args{ + e: event.DeleteEvent{ + Object: &upgradev1.OSInstance{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-node1", + Namespace: "test", + }, + }, + }, + q: nil, + }, + }, + } + var patchList *gomonkey.Patches + var patchDelete *gomonkey.Patches + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + r := &OSReconciler{ + Scheme: tt.fields.Scheme, + Client: tt.fields.Client, + } + patchList = gomonkey.ApplyMethodFunc(r.Client, "List", func(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error { + list.(*upgradev1.OSInstanceList).Items = []upgradev1.OSInstance{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "test-node1", + Namespace: "test", + }, + }, + } + return nil + }) + patchDelete = gomonkey.ApplyMethodReturn(r.Client, "Delete", nil) + r.DeleteOSInstance(tt.args.e, tt.args.q) + }) + } + defer patchDelete.Reset() + defer patchList.Reset() +} diff --git a/cmd/operator/controllers/suite_test.go b/cmd/operator/controllers/suite_test.go index 889789e..aa6deea 100644 --- a/cmd/operator/controllers/suite_test.go +++ b/cmd/operator/controllers/suite_test.go @@ -51,7 +51,7 @@ var _ = BeforeSuite(func() { By("bootstrapping test environment") testEnv = &envtest.Environment{ - CRDDirectoryPaths: []string{filepath.Join("..", "..", "..", "config", "crd")}, + CRDDirectoryPaths: []string{filepath.Join("..", "..", "..", "docs", "example", "config", "crd")}, ErrorIfCRDPathMissing: true, } diff --git a/cmd/proxy/controllers/os_controller.go b/cmd/proxy/controllers/os_controller.go index b0b17e7..09d61c0 100644 --- a/cmd/proxy/controllers/os_controller.go +++ b/cmd/proxy/controllers/os_controller.go @@ -87,6 +87,9 @@ func (r *OSReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Re sameOSVersion := checkVersion(osCr.Spec.OSVersion, node.Status.NodeInfo.OSImage) if sameOSVersion { configOps, err := checkConfigVersion(osCr, osInstance, values.SysConfigName) + if err != nil { + return values.RequeueNow, err + } if configOps == values.Reassign { if err = r.refreshNode(ctx, &node, osInstance, osCr.Spec.SysConfigs.Version, values.SysConfigName); err != nil { return values.RequeueNow, err @@ -113,6 +116,9 @@ func (r *OSReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Re return values.RequeueNow, err } configOps, err := checkConfigVersion(osCr, osInstance, values.UpgradeConfigName) + if err != nil { + return values.RequeueNow, err + } if configOps == values.Reassign { if err = r.refreshNode(ctx, &node, osInstance, osCr.Spec.UpgradeConfigs.Version, values.UpgradeConfigName); err != nil { diff --git a/cmd/proxy/controllers/os_controller_test.go b/cmd/proxy/controllers/os_controller_test.go index e6cd5b7..d63f176 100644 --- a/cmd/proxy/controllers/os_controller_test.go +++ b/cmd/proxy/controllers/os_controller_test.go @@ -67,6 +67,148 @@ var _ = Describe("OsController", func() { testNamespace = existingNamespace.Name }) + Context("When we want to rollback", func() { + It("Should be able to rollback to previous version", func() { + ctx := context.Background() + + By("Creating a worker node") + node1Name = "test-node-" + uuid.New().String() + node1 := &v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: node1Name, + Namespace: testNamespace, + Labels: map[string]string{ + "beta.kubernetes.io/os": "linux", + values.LabelUpgrading: "", + }, + }, + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Node", + }, + Status: v1.NodeStatus{ + NodeInfo: v1.NodeSystemInfo{ + OSImage: "KubeOS v2", + }, + }, + } + err := k8sClient.Create(ctx, node1) + Expect(err).ToNot(HaveOccurred()) + existingNode := &v1.Node{} + Eventually(func() bool { + err := k8sClient.Get(context.Background(), + types.NamespacedName{Name: node1Name, Namespace: testNamespace}, existingNode) + return err == nil + }, timeout, interval).Should(BeTrue()) + reconciler.hostName = node1Name + + By("Creating the corresponding OSInstance") + OSIns := &upgradev1.OSInstance{ + TypeMeta: metav1.TypeMeta{ + Kind: "OSInstance", + APIVersion: "upgrade.openeuler.org/v1alpha1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: node1Name, + Namespace: testNamespace, + Labels: map[string]string{ + values.LabelOSinstance: node1Name, + }, + }, + Spec: upgradev1.OSInstanceSpec{ + NodeStatus: values.NodeStatusUpgrade.String(), + SysConfigs: upgradev1.SysConfigs{}, + UpgradeConfigs: upgradev1.SysConfigs{}, + }, + Status: upgradev1.OSInstanceStatus{}, + } + Expect(k8sClient.Create(ctx, OSIns)).Should(Succeed()) + + // Check that the corresponding OSIns CR has been created + osInsCRLookupKey := types.NamespacedName{Name: node1Name, Namespace: testNamespace} + createdOSIns := &upgradev1.OSInstance{} + Eventually(func() bool { + err := k8sClient.Get(ctx, osInsCRLookupKey, createdOSIns) + return err == nil + }, timeout, interval).Should(BeTrue()) + Expect(createdOSIns.Spec.NodeStatus).Should(Equal(values.NodeStatusUpgrade.String())) + + // stub r.Connection.RollbackSpec() + patchRollback := gomonkey.ApplyMethodReturn(reconciler.Connection, "RollbackSpec", nil) + defer patchRollback.Reset() + patchConfigure := gomonkey.ApplyMethodReturn(reconciler.Connection, "ConfigureSpec", nil) + defer patchConfigure.Reset() + + By("Creating a OS custom resource") + OS := &upgradev1.OS{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "upgrade.openeuler.org/v1alpha1", + Kind: "OS", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: OSName, + Namespace: testNamespace, + }, + Spec: upgradev1.OSSpec{ + OpsType: "rollback", + MaxUnavailable: 3, + OSVersion: "KubeOS v1", + FlagSafe: true, + MTLS: false, + EvictPodForce: true, + SysConfigs: upgradev1.SysConfigs{Configs: []upgradev1.SysConfig{}}, + UpgradeConfigs: upgradev1.SysConfigs{Configs: []upgradev1.SysConfig{}}, + }, + } + Expect(k8sClient.Create(ctx, OS)).Should(Succeed()) + + osCRLookupKey := types.NamespacedName{Name: OSName, Namespace: testNamespace} + createdOS := &upgradev1.OS{} + Eventually(func() bool { + err := k8sClient.Get(ctx, osCRLookupKey, createdOS) + return err == nil + }, timeout, interval).Should(BeTrue()) + Expect(createdOS.Spec.OSVersion).Should(Equal("KubeOS v1")) + Expect(createdOS.Spec.OpsType).Should(Equal("rollback")) + + By("Changing the nodeinfo OSImage to previous version, pretending the rollback success") + existingNode = &v1.Node{} + Eventually(func() bool { + err := k8sClient.Get(context.Background(), + types.NamespacedName{Name: node1Name, Namespace: testNamespace}, existingNode) + return err == nil + }, timeout, interval).Should(BeTrue()) + existingNode.Status.NodeInfo.OSImage = "KubeOS v1" + Expect(k8sClient.Status().Update(ctx, existingNode)).Should(Succeed()) + + By("Changing the OS Spec config to trigger reconcile") + createdOS = &upgradev1.OS{} + Eventually(func() bool { + err := k8sClient.Get(ctx, osCRLookupKey, createdOS) + return err == nil + }, timeout, interval).Should(BeTrue()) + createdOS.Spec.SysConfigs = upgradev1.SysConfigs{Version: "v1", Configs: []upgradev1.SysConfig{}} + Expect(k8sClient.Update(ctx, createdOS)).Should(Succeed()) + + time.Sleep(2 * time.Second) // sleep a while to make sure Reconcile finished + createdOSIns = &upgradev1.OSInstance{} + Eventually(func() bool { + err := k8sClient.Get(ctx, osInsCRLookupKey, createdOSIns) + return err == nil + }, timeout, interval).Should(BeTrue()) + // NodeStatus changes to idle then operator can reassign configs to this node + Expect(createdOSIns.Spec.NodeStatus).Should(Equal(values.NodeStatusIdle.String())) + existingNode = &v1.Node{} + Eventually(func() bool { + err := k8sClient.Get(context.Background(), + types.NamespacedName{Name: node1Name, Namespace: testNamespace}, existingNode) + return err == nil + }, timeout, interval).Should(BeTrue()) + _, ok := existingNode.Labels[values.LabelUpgrading] + Expect(ok).Should(Equal(false)) + }) + }) + Context("When we have a sysconfig whose version is different from current OSInstance config version", func() { It("Should configure the node", func() { ctx := context.Background() @@ -482,6 +624,9 @@ var _ = Describe("OsController", func() { ObjectMeta: metav1.ObjectMeta{ Name: node1Name, Namespace: testNamespace, + Labels: map[string]string{ + values.LabelOSinstance: node1Name, + }, }, Spec: upgradev1.OSInstanceSpec{ NodeStatus: values.NodeStatusConfig.String(), @@ -634,6 +779,9 @@ var _ = Describe("OsController", func() { ObjectMeta: metav1.ObjectMeta{ Name: node1Name, Namespace: testNamespace, + Labels: map[string]string{ + values.LabelOSinstance: node1Name, + }, }, Spec: upgradev1.OSInstanceSpec{ NodeStatus: values.NodeStatusUpgrade.String(), @@ -798,6 +946,9 @@ var _ = Describe("OsController", func() { ObjectMeta: metav1.ObjectMeta{ Name: node1Name, Namespace: testNamespace, + Labels: map[string]string{ + values.LabelOSinstance: node1Name, + }, }, Spec: upgradev1.OSInstanceSpec{ NodeStatus: values.NodeStatusUpgrade.String(), @@ -915,7 +1066,7 @@ var _ = Describe("OsController", func() { Expect(createdOS.Spec.OSVersion).Should(Equal("KubeOS v2")) By("Checking the OSInstance status config version failed to be updated") - time.Sleep(2 * time.Second) // sleep a while to make sure Reconcile finished + time.Sleep(1 * time.Second) // sleep a while to make sure Reconcile finished osInsCRLookupKey = types.NamespacedName{Name: node1Name, Namespace: testNamespace} createdOSIns = &upgradev1.OSInstance{} Eventually(func() bool { diff --git a/cmd/proxy/controllers/suite_test.go b/cmd/proxy/controllers/suite_test.go index a52c18f..00eebbf 100644 --- a/cmd/proxy/controllers/suite_test.go +++ b/cmd/proxy/controllers/suite_test.go @@ -53,7 +53,7 @@ var _ = BeforeSuite(func() { By("bootstrapping test environment") testEnv = &envtest.Environment{ - CRDDirectoryPaths: []string{filepath.Join("..", "..", "..", "config", "crd")}, + CRDDirectoryPaths: []string{filepath.Join("..", "..", "..", "docs", "example", "config", "crd")}, ErrorIfCRDPathMissing: true, } -- 2.39.0