From 64e36143591c014609fc517bcd6ee0f3ae1087fc Mon Sep 17 00:00:00 2001 From: Yuhang Wei Date: Thu, 3 Aug 2023 11:01:22 +0800 Subject: [PATCH 02/17] KubeOS: refactor assignUpgrade function the assignUpgrade function is refactored for maintainability replace "osinstance-node" to variable Signed-off-by: Yuhang Wei --- cmd/operator/controllers/os_controller.go | 73 ++++++----- .../controllers/os_controller_test.go | 121 +++++++++++++++++- cmd/proxy/controllers/os_controller.go | 2 +- cmd/proxy/controllers/os_controller_test.go | 9 ++ pkg/values/values.go | 6 +- 5 files changed, 176 insertions(+), 35 deletions(-) diff --git a/cmd/operator/controllers/os_controller.go b/cmd/operator/controllers/os_controller.go index f86a0b2..620739b 100644 --- a/cmd/operator/controllers/os_controller.go +++ b/cmd/operator/controllers/os_controller.go @@ -109,7 +109,7 @@ func (r *OSReconciler) SetupWithManager(mgr ctrl.Manager) error { func (r *OSReconciler) DeleteOSInstance(e event.DeleteEvent, q workqueue.RateLimitingInterface) { ctx := context.Background() hostname := e.Object.GetName() - labelSelector := labels.SelectorFromSet(labels.Set{"upgrade.openeuler.org/osinstance-node": hostname}) + labelSelector := labels.SelectorFromSet(labels.Set{values.LabelOSinstance: hostname}) osInstanceList := &upgradev1.OSInstanceList{} if err := r.List(ctx, osInstanceList, client.MatchingLabelsSelector{Selector: labelSelector}); err != nil { log.Error(err, "unable to list osInstances") @@ -157,12 +157,23 @@ func assignUpgrade(ctx context.Context, r common.ReadStatusWriter, os upgradev1. return false, err } - nodes, err := getNodes(ctx, r, limit+1, *requirement, *reqMaster) // one more to see if all node updated + nodes, err := getNodes(ctx, r, limit+1, *requirement, *reqMaster) // one more to see if all nodes updated if err != nil { return false, err } - var count = 0 + // Upgrade OS for selected nodes + count, err := upgradeNodes(ctx, r, &os, nodes, limit) + if err != nil { + return false, err + } + + return count >= limit, nil +} + +func upgradeNodes(ctx context.Context, r common.ReadStatusWriter, os *upgradev1.OS, + nodes []corev1.Node, limit int) (int, error) { + var count int for _, node := range nodes { if count >= limit { break @@ -170,43 +181,45 @@ func assignUpgrade(ctx context.Context, r common.ReadStatusWriter, os upgradev1. osVersionNode := node.Status.NodeInfo.OSImage if os.Spec.OSVersion != osVersionNode { var osInstance upgradev1.OSInstance - if err = r.Get(ctx, types.NamespacedName{Namespace: nameSpace, Name: node.Name}, &osInstance); err != nil { + if err := r.Get(ctx, types.NamespacedName{Namespace: os.GetObjectMeta().GetNamespace(), Name: node.Name}, &osInstance); err != nil { if err = client.IgnoreNotFound(err); err != nil { log.Error(err, "failed to get osInstance "+node.Name) - return false, err + return count, err } continue } + updateNodeAndOSins(ctx, r, os, &node, &osInstance) count++ - node.Labels[values.LabelUpgrading] = "" - expUpVersion := os.Spec.UpgradeConfigs.Version - osiUpVersion := osInstance.Spec.UpgradeConfigs.Version - if osiUpVersion != expUpVersion { - osInstance.Spec.UpgradeConfigs = os.Spec.UpgradeConfigs - } - expSysVersion := os.Spec.SysConfigs.Version - osiSysVersion := osInstance.Spec.SysConfigs.Version - if osiSysVersion != expSysVersion { - osInstance.Spec.SysConfigs = os.Spec.SysConfigs - for i, config := range osInstance.Spec.SysConfigs.Configs { - if config.Model == "grub.cmdline.current" { - osInstance.Spec.SysConfigs.Configs[i].Model = "grub.cmdline.next" - } - if config.Model == "grub.cmdline.next" { - osInstance.Spec.SysConfigs.Configs[i].Model = "grub.cmdline.current" - } - } - } - osInstance.Spec.NodeStatus = values.NodeStatusUpgrade.String() - if err = r.Update(ctx, &osInstance); err != nil { - log.Error(err, "unable to update", "osInstance", osInstance.Name) + } + } + return count, nil +} + +func updateNodeAndOSins(ctx context.Context, r common.ReadStatusWriter, os *upgradev1.OS, + node *corev1.Node, osInstance *upgradev1.OSInstance) { + if osInstance.Spec.UpgradeConfigs.Version != os.Spec.UpgradeConfigs.Version { + osInstance.Spec.UpgradeConfigs = os.Spec.UpgradeConfigs + } + if osInstance.Spec.SysConfigs.Version != os.Spec.SysConfigs.Version { + osInstance.Spec.SysConfigs = os.Spec.SysConfigs + // exchange "grub.cmdline.current" and "grub.cmdline.next" + for i, config := range osInstance.Spec.SysConfigs.Configs { + if config.Model == "grub.cmdline.current" { + osInstance.Spec.SysConfigs.Configs[i].Model = "grub.cmdline.next" } - if err = r.Update(ctx, &node); err != nil { - log.Error(err, "unable to label", "node", node.Name) + if config.Model == "grub.cmdline.next" { + osInstance.Spec.SysConfigs.Configs[i].Model = "grub.cmdline.current" } } } - return count >= limit, nil + osInstance.Spec.NodeStatus = values.NodeStatusUpgrade.String() + if err := r.Update(ctx, osInstance); err != nil { + log.Error(err, "unable to update", "osInstance", osInstance.Name) + } + node.Labels[values.LabelUpgrading] = "" + if err := r.Update(ctx, node); err != nil { + log.Error(err, "unable to label", "node", node.Name) + } } func assignConfig(ctx context.Context, r common.ReadStatusWriter, sysConfigs upgradev1.SysConfigs, diff --git a/cmd/operator/controllers/os_controller_test.go b/cmd/operator/controllers/os_controller_test.go index 30a773c..a391005 100644 --- a/cmd/operator/controllers/os_controller_test.go +++ b/cmd/operator/controllers/os_controller_test.go @@ -134,6 +134,9 @@ var _ = Describe("OsController", func() { ObjectMeta: metav1.ObjectMeta{ Name: node1Name, Namespace: testNamespace, + Labels: map[string]string{ + values.LabelOSinstance: node1Name, + }, }, Spec: upgradev1.OSInstanceSpec{ SysConfigs: upgradev1.SysConfigs{ @@ -189,7 +192,7 @@ var _ = Describe("OsController", func() { }, timeout, interval).Should(BeTrue()) Expect(createdOS.Spec.OSVersion).Should(Equal("KubeOS v1")) - time.Sleep(2 * time.Second) // sleep a while to make sure Reconcile finished + time.Sleep(1 * time.Second) // sleep a while to make sure Reconcile finished osInsCRLookupKey = types.NamespacedName{Name: node1Name, Namespace: testNamespace} createdOSIns = &upgradev1.OSInstance{} Eventually(func() bool { @@ -301,7 +304,7 @@ var _ = Describe("OsController", func() { }, timeout, interval).Should(BeTrue()) Expect(createdOS.Spec.OSVersion).Should(Equal("KubeOS v1")) - time.Sleep(2 * time.Second) // sleep a while to make sure Reconcile finished + time.Sleep(1 * time.Second) // sleep a while to make sure Reconcile finished configedOSIns := &upgradev1.OSInstance{} Eventually(func() bool { err := k8sClient.Get(ctx, osInsCRLookupKey, configedOSIns) @@ -388,4 +391,118 @@ var _ = Describe("OsController", func() { }) }) + Context("When we want to upgrade and do sysconfigs which contain grub.cmd.current and .next", func() { + It("Should exchange .current and .next", func() { + ctx := context.Background() + + // create Node + node1Name = "test-node-" + uuid.New().String() + node1 := &v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: node1Name, + Namespace: testNamespace, + Labels: map[string]string{ + "beta.kubernetes.io/os": "linux", + }, + }, + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Node", + }, + Status: v1.NodeStatus{ + NodeInfo: v1.NodeSystemInfo{ + OSImage: "KubeOS v1", + }, + }, + } + err := k8sClient.Create(ctx, node1) + Expect(err).ToNot(HaveOccurred()) + existingNode := &v1.Node{} + Eventually(func() bool { + err := k8sClient.Get(context.Background(), + types.NamespacedName{Name: node1Name, Namespace: testNamespace}, existingNode) + return err == nil + }, timeout, interval).Should(BeTrue()) + + // create OSInstance + OSIns := &upgradev1.OSInstance{ + TypeMeta: metav1.TypeMeta{ + Kind: "OSInstance", + APIVersion: "upgrade.openeuler.org/v1alpha1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: node1Name, + Namespace: testNamespace, + Labels: map[string]string{ + values.LabelOSinstance: node1Name, + }, + }, + Spec: upgradev1.OSInstanceSpec{ + SysConfigs: upgradev1.SysConfigs{ + Version: "v1", + Configs: []upgradev1.SysConfig{}, + }, + UpgradeConfigs: upgradev1.SysConfigs{Configs: []upgradev1.SysConfig{}}, + }, + } + Expect(k8sClient.Create(ctx, OSIns)).Should(Succeed()) + + // Check that the corresponding OSIns CR has been created + osInsCRLookupKey := types.NamespacedName{Name: node1Name, Namespace: testNamespace} + createdOSIns := &upgradev1.OSInstance{} + Eventually(func() bool { + err := k8sClient.Get(ctx, osInsCRLookupKey, createdOSIns) + return err == nil + }, timeout, interval).Should(BeTrue()) + Expect(createdOSIns.ObjectMeta.Name).Should(Equal(node1Name)) + + // create OS CR + OS := &upgradev1.OS{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "upgrade.openeuler.org/v1alpha1", + Kind: "OS", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: OSName, + Namespace: testNamespace, + }, + Spec: upgradev1.OSSpec{ + OpsType: "upgrade", + MaxUnavailable: 3, + OSVersion: "KubeOS v2", + FlagSafe: true, + MTLS: false, + EvictPodForce: true, + SysConfigs: upgradev1.SysConfigs{ + Version: "v2", + Configs: []upgradev1.SysConfig{ + {Model: "grub.cmdline.current", Contents: []upgradev1.Content{{Key: "a", Value: "1"}}}, + {Model: "grub.cmdline.next", Contents: []upgradev1.Content{{Key: "b", Value: "2"}}}, + }, + }, + UpgradeConfigs: upgradev1.SysConfigs{Configs: []upgradev1.SysConfig{}}, + }, + } + Expect(k8sClient.Create(ctx, OS)).Should(Succeed()) + + // Check that the corresponding OS CR has been created + osCRLookupKey := types.NamespacedName{Name: OSName, Namespace: testNamespace} + createdOS := &upgradev1.OS{} + Eventually(func() bool { + err := k8sClient.Get(ctx, osCRLookupKey, createdOS) + return err == nil + }, timeout, interval).Should(BeTrue()) + Expect(createdOS.Spec.OSVersion).Should(Equal("KubeOS v2")) + + time.Sleep(1 * time.Second) // sleep a while to make sure Reconcile finished + osInsCRLookupKey = types.NamespacedName{Name: node1Name, Namespace: testNamespace} + createdOSIns = &upgradev1.OSInstance{} + Eventually(func() bool { + err := k8sClient.Get(ctx, osInsCRLookupKey, createdOSIns) + return err == nil + }, timeout, interval).Should(BeTrue()) + Expect(createdOSIns.Spec.SysConfigs.Configs[0]).Should(Equal(upgradev1.SysConfig{Model: "grub.cmdline.next", Contents: []upgradev1.Content{{Key: "a", Value: "1"}}})) + Expect(createdOSIns.Spec.SysConfigs.Configs[1]).Should(Equal(upgradev1.SysConfig{Model: "grub.cmdline.current", Contents: []upgradev1.Content{{Key: "b", Value: "2"}}})) + }) + }) }) diff --git a/cmd/proxy/controllers/os_controller.go b/cmd/proxy/controllers/os_controller.go index a17afac..b0b17e7 100644 --- a/cmd/proxy/controllers/os_controller.go +++ b/cmd/proxy/controllers/os_controller.go @@ -261,7 +261,7 @@ func checkOsiExist(ctx context.Context, r common.ReadStatusWriter, nameSpace str Namespace: nameSpace, Name: nodeName, Labels: map[string]string{ - "upgrade.openeuler.org/osinstance-node": nodeName, + values.LabelOSinstance: nodeName, }, }, } diff --git a/cmd/proxy/controllers/os_controller_test.go b/cmd/proxy/controllers/os_controller_test.go index ff12f64..e6cd5b7 100644 --- a/cmd/proxy/controllers/os_controller_test.go +++ b/cmd/proxy/controllers/os_controller_test.go @@ -110,6 +110,9 @@ var _ = Describe("OsController", func() { ObjectMeta: metav1.ObjectMeta{ Name: node1Name, Namespace: testNamespace, + Labels: map[string]string{ + values.LabelOSinstance: node1Name, + }, }, Spec: upgradev1.OSInstanceSpec{ NodeStatus: values.NodeStatusConfig.String(), @@ -245,6 +248,9 @@ var _ = Describe("OsController", func() { ObjectMeta: metav1.ObjectMeta{ Name: node1Name, Namespace: testNamespace, + Labels: map[string]string{ + values.LabelOSinstance: node1Name, + }, }, Spec: upgradev1.OSInstanceSpec{ NodeStatus: values.NodeStatusUpgrade.String(), @@ -426,6 +432,9 @@ var _ = Describe("OsController", func() { return err == nil }, timeout, interval).Should(BeTrue()) Expect(createdOSIns.Spec.NodeStatus).Should(Equal(values.NodeStatusIdle.String())) + hostname, ok := createdOSIns.ObjectMeta.Labels[values.LabelOSinstance] + Expect(ok).Should(BeTrue()) + Expect(hostname).Should(Equal(node1Name)) }) }) diff --git a/pkg/values/values.go b/pkg/values/values.go index 30261bd..f488ae5 100644 --- a/pkg/values/values.go +++ b/pkg/values/values.go @@ -23,8 +23,10 @@ const ( // LabelUpgrading is the key of the upgrading label for nodes LabelUpgrading = "upgrade.openeuler.org/upgrading" // LabelMaster is the key of the master-node label for nodes - LabelMaster = "node-role.kubernetes.io/control-plane" - defaultPeriod = 15 * time.Second + LabelMaster = "node-role.kubernetes.io/control-plane" + // LabelOSinstance is used to select the osinstance with the nodeName by label + LabelOSinstance = "upgrade.openeuler.org/osinstance-node" + defaultPeriod = 15 * time.Second // OsiStatusName is param name of nodeStatus in osInstance OsiStatusName = "nodestatus" // UpgradeConfigName is param name of UpgradeConfig -- 2.39.0