From 77ed6fefe70edde63b01d797b76f389bc82bb1a0 Mon Sep 17 00:00:00 2001
From: Wei Gao <gaowei66@huawei.com>
Date: Mon, 9 Aug 2021 14:57:06 +0800
Subject: [PATCH 6/6] factory: add the template factory support for hypervisor
 type stratovirt.

Signed-off-by: Wei Gao <gaowei66@huawei.com>
---
 src/runtime/pkg/katautils/config.go           |  2 +-
 .../factory/template/template.go              | 21 +++--
 src/runtime/virtcontainers/kata_agent.go      |  7 +-
 src/runtime/virtcontainers/stratovirt.go      | 89 +++++++++++++++++--
 src/runtime/virtcontainers/vm.go              | 28 ++++--
 5 files changed, 125 insertions(+), 22 deletions(-)

diff --git a/src/runtime/pkg/katautils/config.go b/src/runtime/pkg/katautils/config.go
index 828c2a43..718677b4 100644
--- a/src/runtime/pkg/katautils/config.go
+++ b/src/runtime/pkg/katautils/config.go
@@ -1363,7 +1363,7 @@ func checkNetNsConfig(config oci.RuntimeConfig) error {
 // checkFactoryConfig ensures the VM factory configuration is valid.
 func checkFactoryConfig(config oci.RuntimeConfig) error {
 	if config.FactoryConfig.Template {
-		if config.HypervisorConfig.InitrdPath == "" {
+		if config.HypervisorConfig.InitrdPath == "" && (config.HypervisorType != vc.StratovirtHypervisor) {
 			return errors.New("Factory option enable_template requires an initrd image")
 		}
 	}
diff --git a/src/runtime/virtcontainers/factory/template/template.go b/src/runtime/virtcontainers/factory/template/template.go
index 66070126..02497097 100644
--- a/src/runtime/virtcontainers/factory/template/template.go
+++ b/src/runtime/virtcontainers/factory/template/template.go
@@ -96,11 +96,15 @@ func (t *template) prepareTemplateFiles() error {
 	if err != nil {
 		return err
 	}
-	flags := uintptr(syscall.MS_NOSUID | syscall.MS_NODEV)
-	opts := fmt.Sprintf("size=%dM", t.config.HypervisorConfig.MemorySize+templateDeviceStateSize)
-	if err = syscall.Mount("tmpfs", t.statePath, "tmpfs", flags, opts); err != nil {
-		t.close()
-		return err
+
+	// If use hypervisor stratovirt, no need to create template path with ramdisk.
+	if t.config.HypervisorType != vc.StratovirtHypervisor {
+		flags := uintptr(syscall.MS_NOSUID | syscall.MS_NODEV)
+		opts := fmt.Sprintf("size=%dM", t.config.HypervisorConfig.MemorySize+templateDeviceStateSize)
+		if err = syscall.Mount("tmpfs", t.statePath, "tmpfs", flags, opts); err != nil {
+			t.close()
+			return err
+		}
 	}
 	f, err := os.Create(t.statePath + "/memory")
 	if err != nil {
@@ -126,8 +130,11 @@ func (t *template) createTemplateVM(ctx context.Context) error {
 	}
 	defer vm.Stop(ctx)
 
-	if err = vm.Disconnect(ctx); err != nil {
-		return err
+	// Create template on hypervisor stratovirt, don't have connection with agent.
+	if config.HypervisorType != vc.StratovirtHypervisor {
+		if err = vm.Disconnect(ctx); err != nil {
+			return err
+		}
 	}
 
 	// Sleep a bit to let the agent grpc server clean up
diff --git a/src/runtime/virtcontainers/kata_agent.go b/src/runtime/virtcontainers/kata_agent.go
index 13d31658..bc882c70 100644
--- a/src/runtime/virtcontainers/kata_agent.go
+++ b/src/runtime/virtcontainers/kata_agent.go
@@ -1306,8 +1306,11 @@ func (k *kataAgent) buildContainerRootfs(ctx context.Context, sandbox *Sandbox,
 		// TODO: remove dependency on shared fs path. shared fs is just one kind of storage source.
 		// we should not always use shared fs path for all kinds of storage. Instead, all storage
 		// should be bind mounted to a tmpfs path for containers to use.
-		if err := os.MkdirAll(filepath.Join(getMountPath(c.sandbox.id), c.id, c.rootfsSuffix), DirMode); err != nil {
-			return nil, err
+		// If boot from template on stratovirt, no need to mkdir mount path.
+		if !((sandbox.config.HypervisorType == StratovirtHypervisor) && sandbox.config.HypervisorConfig.BootFromTemplate) {
+			if err := os.MkdirAll(filepath.Join(getMountPath(c.sandbox.id), c.id, c.rootfsSuffix), DirMode); err != nil {
+				return nil, err
+			}
 		}
 		return rootfs, nil
 	}
diff --git a/src/runtime/virtcontainers/stratovirt.go b/src/runtime/virtcontainers/stratovirt.go
index 47daa817..e9b2ba85 100644
--- a/src/runtime/virtcontainers/stratovirt.go
+++ b/src/runtime/virtcontainers/stratovirt.go
@@ -48,6 +48,7 @@ type stratovirt struct {
 	config          HypervisorConfig
 	rootfsPath      string
 	kernelPath      string
+	templatePath    string
 	pid             int
 	consolePath     string
 	socketPath      string
@@ -115,7 +116,7 @@ func (s *stratovirt) createSandbox(ctx context.Context, id string, networkNS Net
 
 	s.id = id
 	s.config = *hypervisorConfig
-	if s.config.OzonePath == "" {
+	if (s.config.OzonePath == "") || s.config.BootToBeTemplate {
 		s.useOzone = false
 		s.pidfile = filepath.Join(s.store.RunVMStoragePath(), s.id, "pid")
 		s.logfile = filepath.Join(s.store.RunVMStoragePath(), s.id, "/stratovirt.log")
@@ -129,6 +130,20 @@ func (s *stratovirt) createSandbox(ctx context.Context, id string, networkNS Net
 		s.socketPath = filepath.Join(s.ozoneRoot, apiSocket)
 		s.consolePath = filepath.Join(s.ozoneRoot, debugSocket)
 	}
+
+	if s.config.VMid != "" && s.useOzone {
+		// Make sure the symlinks do not exist
+		os.RemoveAll(s.ozoneRoot)
+		ozoneVmRoot := filepath.Join(ozoneBaseDir, s.config.VMid)
+		if err := os.Symlink(ozoneVmRoot, s.ozoneRoot); err != nil {
+			return err
+		}
+	}
+
+	if s.config.BootFromTemplate || s.config.BootToBeTemplate {
+		s.templatePath = strings.Replace(s.config.DevicesStatePath, "/state", "", -1)
+	}
+
 	s.netNSPath = networkNS.NetNsPath
 	s.qmpMonitorCh = qmpChannel{
 		ctx:  s.ctx,
@@ -221,6 +236,12 @@ func (s *stratovirt) createOzoneParams(params []string) ([]string, error) {
 		params = append(params, "-initrd", filepath.Base(s.rootfsPath))
 	}
 
+	// handle boot from template
+	if s.config.BootFromTemplate {
+		s.ozoneRes = append(s.ozoneRes, s.templatePath)
+		params = append(params, "-incoming", fmt.Sprintf("file:%s", filepath.Base(s.templatePath)))
+	}
+
 	// add devices to cmdline
 	for _, d := range s.devices {
 		switch v := d.dev.(type) {
@@ -266,6 +287,11 @@ func (s *stratovirt) createParams(params []string) ([]string, error) {
 		params = append(params, "-initrd", s.rootfsPath)
 	}
 
+	// handle boot from template
+	if s.config.BootFromTemplate {
+		params = append(params, "-incoming", fmt.Sprintf("file:%s", s.templatePath))
+	}
+
 	// add devices to cmdline
 	for _, d := range s.devices {
 		switch v := d.dev.(type) {
@@ -410,14 +436,55 @@ func (s *stratovirt) stopSandbox(ctx context.Context, force bool) error {
 }
 
 func (s *stratovirt) pauseSandbox(ctx context.Context) error {
-	return nil
+	span, _ := s.trace(ctx, "pauseSandbox")
+	defer span.End()
+
+	return s.togglePauseSandbox(ctx, true)
 }
 
 func (s *stratovirt) saveSandbox() error {
+	s.Logger().Info("save sandbox")
+
+	err := s.qmpSetup()
+	if err != nil {
+		return err
+	}
+
+	// BootToBeTemplate sets the VM to be a template that other VMs can can clone from.
+	// We would want to bypass shared memory when saving VM to local file through migrate.
+	if s.config.BootToBeTemplate {
+		err = s.qmpMonitorCh.qmp.ExecSetMigrateArguments(s.qmpMonitorCh.ctx, fmt.Sprintf("file:%s", s.templatePath))
+		if err != nil {
+			s.Logger().WithError(err).Error("exec migration")
+			return err
+		}
+	}
+
 	return nil
 }
 
 func (s *stratovirt) resumeSandbox(ctx context.Context) error {
+	span, _ := s.trace(ctx, "resumeSandbox")
+	defer span.End()
+
+	return s.togglePauseSandbox(ctx, false)
+}
+
+func (s *stratovirt) togglePauseSandbox(ctx context.Context, pause bool) error {
+	span, _ := s.trace(ctx, "togglePauseSandbox")
+	defer span.End()
+
+	err := s.qmpSetup()
+	if err != nil {
+		return err
+	}
+
+	if pause {
+		s.qmpMonitorCh.qmp.ExecuteStop(s.qmpMonitorCh.ctx)
+	} else {
+		s.qmpMonitorCh.qmp.ExecuteCont(s.qmpMonitorCh.ctx)
+	}
+
 	return nil
 }
 
@@ -734,11 +801,23 @@ func (s *stratovirt) updateOzoneRes(src string, add bool) (string, error) {
 }
 
 func (s *stratovirt) cleanOzoneRes() {
-	s.updateOzoneRes(s.rootfsPath, false)
-	s.updateOzoneRes(s.kernelPath, false)
+	// Umount all resource in ozoneRoot
+	if dir, err := ioutil.ReadDir(s.ozoneRoot); err == nil {
+		for _, file := range dir {
+			syscall.Unmount(filepath.Join(s.ozoneRoot, file.Name()), syscall.MNT_DETACH)
+		}
+	}
 
 	if err := os.RemoveAll(s.ozoneRoot); err != nil {
-		s.Logger().WithField("cleanupOzone failed", err).Error()
+		s.Logger().WithField("cleanup Ozone failed", err).Error()
+	}
+
+	// If have VMid, the VM is boot from template. ozoneVmRoot also need clean.
+	if s.config.VMid != "" {
+		ozoneVmRoot := filepath.Join(ozoneBaseDir, s.config.VMid)
+		if err := os.RemoveAll(ozoneVmRoot); err != nil {
+			s.Logger().WithField("cleanup Ozone failed", err).Error()
+		}
 	}
 }
 
diff --git a/src/runtime/virtcontainers/vm.go b/src/runtime/virtcontainers/vm.go
index e6f02b6e..c4f9df73 100644
--- a/src/runtime/virtcontainers/vm.go
+++ b/src/runtime/virtcontainers/vm.go
@@ -142,13 +142,19 @@ func NewVM(ctx context.Context, config VMConfig) (*VM, error) {
 	}()
 
 	// 4. check agent aliveness
-	// VMs booted from template are paused, do not check
-	if !config.HypervisorConfig.BootFromTemplate {
+	// On hypervisor StratoVirt, VMs booted from template are running, check agent
+	// On other hypervisors, VMs booted from template are paused, do not check
+	if config.HypervisorType == StratovirtHypervisor {
+		if !config.HypervisorConfig.BootToBeTemplate {
+			virtLog.WithField("vm", id).Info("check agent status")
+			err = agent.check(ctx)
+		}
+	} else if !config.HypervisorConfig.BootFromTemplate {
 		virtLog.WithField("vm", id).Info("check agent status")
 		err = agent.check(ctx)
-		if err != nil {
-			return nil, err
-		}
+	}
+	if err != nil {
+		return nil, err
 	}
 
 	return &VM{
@@ -329,9 +335,16 @@ func (v *VM) assignSandbox(s *Sandbox) error {
 	// - link 9pfs share path from sandbox dir (/run/kata-containers/shared/sandboxes/sbid/) to vm dir (/run/vc/vm/vmid/shared/)
 
 	vmSharePath := buildVMSharePath(v.id, v.store.RunVMStoragePath())
-	vmSockDir := filepath.Join(v.store.RunVMStoragePath(), v.id)
 	sbSharePath := getMountPath(s.id)
-	sbSockDir := filepath.Join(v.store.RunVMStoragePath(), s.id)
+	var vmSockDir string
+	var sbSockDir string
+	if v.hypervisor.hypervisorConfig().OzonePath != "" {
+		vmSockDir = filepath.Join(ozoneBaseDir, v.id)
+		sbSockDir = filepath.Join(ozoneBaseDir, s.id)
+	} else {
+		vmSockDir = filepath.Join(v.store.RunVMStoragePath(), v.id)
+		sbSockDir = filepath.Join(v.store.RunVMStoragePath(), s.id)
+	}
 
 	v.logger().WithFields(logrus.Fields{
 		"vmSharePath": vmSharePath,
@@ -359,6 +372,7 @@ func (v *VM) assignSandbox(s *Sandbox) error {
 
 	s.hypervisor = v.hypervisor
 	s.config.HypervisorConfig.VMid = v.id
+	s.config.HypervisorConfig.BootFromTemplate = true
 
 	return nil
 }
-- 
2.21.1 (Apple Git-122.3)

