From 3ffaa7ae39d1af1d7c5aae976f1d6017ad76dee9 Mon Sep 17 00:00:00 2001
From: haozi007 <liuhao27@huawei.com>
Date: Wed, 30 Nov 2022 17:33:16 +0800
Subject: [PATCH 55/65] retry call runtime ops

1. add retry macro;
2. retry call runtime cleanup ops;

Signed-off-by: haozi007 <liuhao27@huawei.com>
---
 src/daemon/modules/plugin/plugin.c            | 21 +++----------------
 .../modules/runtime/isula/isula_rt_ops.c      | 14 +++++++++++--
 src/utils/cutils/utils.c                      |  1 +
 src/utils/cutils/utils.h                      | 19 +++++++++++++++++
 4 files changed, 35 insertions(+), 20 deletions(-)

diff --git a/src/daemon/modules/plugin/plugin.c b/src/daemon/modules/plugin/plugin.c
index 725bca5b..18035518 100644
--- a/src/daemon/modules/plugin/plugin.c
+++ b/src/daemon/modules/plugin/plugin.c
@@ -71,7 +71,7 @@
 // suffix is '.sock'
 #define PLUGIN_SOCKET_FILE_SUFFIX_LEN 5
 
-#define PLUGIN_ACTIVATE_MAX_RETRY 3
+#define PLUGIN_ACTIVATE_MAX_RETRY 5
 
 #ifndef RestHttpHead
 #define RestHttpHead "http://localhost"
@@ -403,22 +403,6 @@ out:
     return -1;
 }
 
-static int pm_activate_plugin_with_retry(plugin_t *plugin, size_t retry)
-{
-    size_t i = 0;
-    int err = 0;
-
-    for (i = 0; i < retry; i++) {
-        err = pm_activate_plugin(plugin);
-        if (!err) {
-            return 0;
-        }
-        sleep((unsigned int)i + 1);
-    }
-
-    return err;
-}
-
 static void pm_rdlock(void)
 {
     int errcode;
@@ -500,7 +484,8 @@ static int pm_register_plugin(const char *name, const char *addr)
         ERROR("alloc plugin failed");
         goto failed;
     }
-    err = pm_activate_plugin_with_retry(plugin, PLUGIN_ACTIVATE_MAX_RETRY);
+
+    DO_RETYR_CALL(PLUGIN_ACTIVATE_MAX_RETRY, 1000000, err, pm_activate_plugin, plugin);
     if (err != 0) {
         ERROR("active plugin failed");
         goto failed;
diff --git a/src/daemon/modules/runtime/isula/isula_rt_ops.c b/src/daemon/modules/runtime/isula/isula_rt_ops.c
index c9667ee5..bfe7de08 100644
--- a/src/daemon/modules/runtime/isula/isula_rt_ops.c
+++ b/src/daemon/modules/runtime/isula/isula_rt_ops.c
@@ -946,6 +946,7 @@ int rt_isula_restart(const char *name, const char *runtime, const rt_restart_par
 int rt_isula_clean_resource(const char *id, const char *runtime, const rt_clean_params_t *params)
 {
     char workdir[PATH_MAX] = { 0 };
+    int nret;
 
     if (id == NULL || runtime == NULL || params == NULL) {
         ERROR("nullptr arguments not allowed");
@@ -966,8 +967,17 @@ int rt_isula_clean_resource(const char *id, const char *runtime, const rt_clean_
         shim_kill_force(workdir);
     }
 
-    (void)runtime_call_kill_force(workdir, runtime, id);
-    (void)runtime_call_delete_force(workdir, runtime, id);
+    // retry 10 count call runtime kill, every call sleep 1s
+    DO_RETYR_CALL(10, 1000000, nret, runtime_call_kill_force, workdir, runtime, id);
+    if (nret != 0) {
+        WARN("call runtime force kill failed");
+    }
+
+    // retry 10 count call runtime delete, every call sleep 1s
+    DO_RETYR_CALL(10, 1000000, nret, runtime_call_delete_force, workdir, runtime, id);
+    if (nret != 0) {
+        WARN("call runtime force delete failed");
+    }
 
     if (util_recursive_rmdir(workdir, 0) != 0) {
         ERROR("failed rmdir -r shim workdir");
diff --git a/src/utils/cutils/utils.c b/src/utils/cutils/utils.c
index 9f5deaf9..a154c52a 100644
--- a/src/utils/cutils/utils.c
+++ b/src/utils/cutils/utils.c
@@ -1216,6 +1216,7 @@ void util_usleep_nointerupt(unsigned long usec)
         request = remain;
     } while (ret == -1 && errno == EINTR);
 }
+
 int util_generate_random_str(char *id, size_t len)
 {
     int fd = -1;
diff --git a/src/utils/cutils/utils.h b/src/utils/cutils/utils.h
index 27cfc902..0a9535a1 100644
--- a/src/utils/cutils/utils.h
+++ b/src/utils/cutils/utils.h
@@ -381,6 +381,25 @@ defs_map_string_object * dup_map_string_empty_object(defs_map_string_object *src
 
 int convert_v2_runtime(const char *runtime, char *binary);
 
+/**
+ * retry_cnt: max count of call cb;
+ * interval_us: how many us to sleep, after call cb;
+ * cb: retry call function;
+ * return:
+ *  0 is cb successful at least once;
+ *  1 is all cb are failure;
+*/
+#define DO_RETYR_CALL(retry_cnt, interval_us, ret, cb, ...) do {    \
+        size_t i = 0;                                               \
+        for(; i < retry_cnt; i++) {                                 \
+            ret = cb(__VA_ARGS__);                                  \
+            if (ret == 0) {                                         \
+                break;                                              \
+            }                                                       \
+            util_usleep_nointerupt(interval_us);                    \
+        }                                                           \
+    } while(0)
+
 #ifdef __cplusplus
 }
 #endif
-- 
2.25.1

