packages/dpu-utilities
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

add compile option of epoll proxy thread and fix some problems

Browse Source 
Signed-off-by: liqiang <liqiang64@huawei.com>
This commit is contained in:
liqiang 2024-09-10 15:48:49 +08:00
parent 6e46b799ae
commit a8da3ca798
2 changed files with 375 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

370
0004-add-compile-option-of-EPOLL_PROXY-and-fix-some-probl.patch Normal file
View File

@ -0,0 +1,370 @@
From 77a98b652e40ed72dbf8f72be0a17bc7f3b3967b Mon Sep 17 00:00:00 2001
From: liqiang <liqiang64@huawei.com>
Date: Tue, 6 Aug 2024 19:52:06 +0800
Subject: [PATCH 3/3] add compile option of EPOLL_PROXY and fix some problem of
vsock connection
Signed-off-by: liqiang <liqiang64@huawei.com>
---
qtfs/qtfs/Makefile | 3 +++
qtfs/qtfs/qtfs-mod.c | 13 ++++++++++++-
qtfs/qtfs_common/conn.c | 10 ++++++++++
qtfs/qtfs_common/misc.c | 2 ++
qtfs/qtfs_common/socket.c | 8 +++++++-
qtfs/qtfs_common/user_engine.c | 12 +++++++++++-
qtfs/qtfs_server/Makefile | 4 ++++
qtfs/qtfs_server/qtfs-server.c | 12 ++++++++++++
8 files changed, 61 insertions(+), 3 deletions(-)
diff --git a/qtfs/qtfs/Makefile b/qtfs/qtfs/Makefile
index 9b3ebe5..a6a6dca 100644
--- a/qtfs/qtfs/Makefile
+++ b/qtfs/qtfs/Makefile
@@ -3,6 +3,9 @@ ccflags-y += -I$(src)/../ -I$(src)/../utils/ -I$(src)/../include/ -I$(src)/../ip
else
ccflags-y += -I$(src)/../ -I$(src)/../utils/ -I$(src)/../include/ -I$(src)/../ipc/ -I$(src) -DQTFS_CLIENT
endif
+ifdef EPOLL_PROXY
+ccflags-y += -DEPOLL_PROXY
+endif
KBUILD=/lib/modules/$(shell uname -r)/build/
COMM=../qtfs_common/
diff --git a/qtfs/qtfs/qtfs-mod.c b/qtfs/qtfs/qtfs-mod.c
index 0ce43fb..79256ad 100644
--- a/qtfs/qtfs/qtfs-mod.c
+++ b/qtfs/qtfs/qtfs-mod.c
@@ -33,8 +33,9 @@ static struct file_system_type qtfs_fs_type = {
};
MODULE_ALIAS_FS("qtfs");
struct kmem_cache *qtfs_inode_priv_cache;
+#ifdef EPOLL_PROXY
struct task_struct *g_qtfs_epoll_thread = NULL;
-
+#endif
/*
* 转发框架层：
* 1. 调用者先在pvar里预留框架头后，填好自己的私有发送数据。
@@ -124,6 +125,7 @@ retry:
return pvar->conn_ops->get_conn_msg_buf(pvar, QTFS_RECV);
}
+#ifdef EPOLL_PROXY
static int qtfs_epoll_thread(void *data)
{
struct qtfs_conn_var_s *pvar = NULL;
@@ -202,6 +204,7 @@ end:
g_qtfs_epoll_thread = NULL;
return 0;
}
+#endif
struct file_operations qtfs_misc_fops = {
.owner=THIS_MODULE,
@@ -236,12 +239,14 @@ static int __init qtfs_init(void)
goto conn_init_err;
}
qtfs_whitelist_initset();
+#ifdef EPOLL_PROXY
g_qtfs_epoll_thread = kthread_run(qtfs_epoll_thread, NULL, "qtfs_epoll");
if (IS_ERR_OR_NULL(g_qtfs_epoll_thread)) {
qtfs_err("qtfs epoll thread run failed.\n");
ret = QTFS_PTR_ERR(g_qtfs_epoll_thread);
goto epoll_thread_err;
}
+#endif
qtfs_diag_info = (struct qtinfo *)kmalloc(sizeof(struct qtinfo), GFP_KERNEL);
if (qtfs_diag_info == NULL) {
qtfs_err("kmalloc qtfs diag info failed.");
@@ -278,7 +283,9 @@ syscall_replace_err:
misc_register_err:
kfree(qtfs_diag_info);
diag_malloc_err:
+#ifdef EPOLL_PROXY
kthread_stop(g_qtfs_epoll_thread);
+#endif
epoll_thread_err:
qtfs_conn_param_fini();
conn_init_err:
@@ -293,18 +300,22 @@ static void __exit qtfs_exit(void)
int ret;
qtfs_mod_exiting = true;
+#ifdef EPOLL_PROXY
if (g_qtfs_epoll_thread) {
kthread_stop(g_qtfs_epoll_thread);
}
+#endif
qtfs_conn_param_fini();
qtfs_misc_destroy();
+#ifdef EPOLL_PROXY
if (qtfs_epoll_var != NULL) {
qtfs_epoll_cut_conn(qtfs_epoll_var);
qtfs_epoll_var->conn_ops->conn_var_fini(qtfs_epoll_var);
kfree(qtfs_epoll_var);
qtfs_epoll_var = NULL;
}
+#endif
qtfs_whitelist_exit();
kfree(qtfs_diag_info);
diff --git a/qtfs/qtfs_common/conn.c b/qtfs/qtfs_common/conn.c
index 5f080cd..fd5f654 100644
--- a/qtfs/qtfs_common/conn.c
+++ b/qtfs/qtfs_common/conn.c
@@ -42,7 +42,9 @@ static struct mutex g_param_mutex;
static struct mutex g_fifo_mutex;
int qtfs_mod_exiting = false;
struct qtfs_conn_var_s *qtfs_thread_var[QTFS_MAX_THREADS] = {NULL};
+#ifdef EPOLL_PROXY
struct qtfs_conn_var_s *qtfs_epoll_var = NULL;
+#endif
#ifdef QTFS_SERVER
struct qtfs_server_userp_s *qtfs_userps = NULL;
#endif
@@ -520,7 +522,11 @@ void *qtfs_conn_msg_buf(struct qtfs_conn_var_s *pvar, int dir)
static int qtfs_sm_connecting(struct qtfs_conn_var_s *pvar)
{
int ret = QTERROR;
+#ifdef QTFS_CLIENT
int retry = 3;
+#else
+ int retry = 1;
+#endif
while (qtfs_mod_exiting == false && retry-- > 0) {
ret = pvar->conn_ops->conn_new_connection(&pvar->conn_var, pvar->user_type);
if (ret == 0) {
@@ -873,6 +879,7 @@ retry:
return pvar;
}
+#ifdef EPOLL_PROXY
struct qtfs_conn_var_s *qtfs_epoll_establish_conn(void)
{
struct qtfs_conn_var_s *pvar = NULL;
@@ -920,6 +927,7 @@ struct qtfs_conn_var_s *qtfs_epoll_establish_conn(void)
qtfs_info("qtfs create new epoll param state:%s", QTCONN_CUR_STATE(pvar));
return pvar;
}
+#endif
void qtfs_conn_put_param(struct qtfs_conn_var_s *pvar)
{
@@ -944,6 +952,7 @@ void qtfs_conn_put_param(struct qtfs_conn_var_s *pvar)
mutex_unlock(&g_param_mutex);
}
+#ifdef EPOLL_PROXY
void qtfs_epoll_cut_conn(struct qtfs_conn_var_s *pvar)
{
int ret = 0;
@@ -958,6 +967,7 @@ void qtfs_epoll_cut_conn(struct qtfs_conn_var_s *pvar)
qtfs_err("qtfs epoll put param exit failed, ret:%d state:%s", ret, QTCONN_CUR_STATE(pvar));
}
}
+#endif
#ifdef QTFS_CLIENT
/* fifo的机制有所不同，每一个pvar对应唯一一个fifo的访问，生命周期贯穿
diff --git a/qtfs/qtfs_common/misc.c b/qtfs/qtfs_common/misc.c
index e1c3520..3adfb57 100644
--- a/qtfs/qtfs_common/misc.c
+++ b/qtfs/qtfs_common/misc.c
@@ -68,7 +68,9 @@ void qtfs_misc_flush_threadstate(void)
}
qtfs_diag_info->thread_state[i] = qtfs_thread_var[i]->state;
}
+#ifdef EPOLL_PROXY
qtfs_diag_info->epoll_state = (qtfs_epoll_var == NULL) ? -1 : qtfs_epoll_var->state;
+#endif
}
void qtfs_req_size(void)
diff --git a/qtfs/qtfs_common/socket.c b/qtfs/qtfs_common/socket.c
index f247be0..233a1ce 100644
--- a/qtfs/qtfs_common/socket.c
+++ b/qtfs/qtfs_common/socket.c
@@ -162,6 +162,11 @@ static int qtfs_conn_sock_server_accept(void *connvar, qtfs_conn_type_e type)
mutex_unlock(&qtfs_server_main_sock[type].lock);
return -EFAULT;
}
+ if (qtfs_server_thread_run == 0) {
+ qtfs_err("engine is exit now, return right now.");
+ mutex_unlock(&qtfs_server_main_sock[type].lock);
+ return -EINTR;
+ }
sock = qtfs_server_main_sock[type].sock;
if (sock == NULL) {
@@ -246,7 +251,7 @@ static int qtfs_conn_sock_init(void *connvar, qtfs_conn_type_e type)
}
qtfs_info("qtfs socket init sock OK, type:%d!", type);
- qtfs_sock_recvtimeo_set(sock, 0, 100000);
+ qtfs_sock_recvtimeo_set(sock, 1, 0);
qtfs_server_main_sock[type].sock = sock;
mutex_unlock(&qtfs_server_main_sock[type].lock);
return 0;
@@ -282,6 +287,7 @@ static int qtfs_conn_sock_client_connect(void *connvar, qtfs_conn_type_e type)
ret = sock->ops->connect(sock, (struct sockaddr *)&saddr, sizeof(saddr), 0);
if (ret < 0) {
+ qtfs_err("qtfs client connect failed, ret:%d", ret);
return ret;
}
#ifdef QTFS_TEST_MODE
diff --git a/qtfs/qtfs_common/user_engine.c b/qtfs/qtfs_common/user_engine.c
index a3edac2..a602980 100644
--- a/qtfs/qtfs_common/user_engine.c
+++ b/qtfs/qtfs_common/user_engine.c
@@ -177,8 +177,8 @@ static void *qtfs_engine_kthread(void *arg)
break;
}
if (sig_int_flag == 1) {
+ sig_int_flag = 0;
engine_out("qtfs engine recv SIGINT.");
-
if (qtfs_fd < 0) {
engine_err("qtfs engine signal int file:%s open failed, fd:%d.", QTFS_SERVER_FILE, qtfs_fd);
continue;
@@ -204,6 +204,7 @@ static void qtfs_signal_int(int signum)
return;
}
+#ifdef EPOLL_PROXY
static void *qtfs_engine_epoll_thread(void *data)
{
struct engine_arg *arg = (struct engine_arg *)data;
@@ -264,6 +265,7 @@ int qtfs_epoll_init(int fd)
return epfd;
}
+#endif
static void qtfs_whitelist_free_items(char **items, gsize count)
{
@@ -498,11 +500,13 @@ int main(int argc, char *argv[])
}
qtfs_fd = fd;
// init epoll
+#ifdef EPOLL_PROXY
int epfd = qtfs_epoll_init(fd);
if (epfd < 0) {
close(fd);
return -1;
}
+#endif
ret = ioctl(fd, QTFS_IOCTL_EXIT, 1);
if (ret == QTERROR) {
goto end;
@@ -539,7 +543,9 @@ int main(int argc, char *argv[])
arg[i].thread_idx = i;
(void)pthread_create(&texec[i], NULL, qtfs_engine_kthread, &arg[i]);
}
+#ifdef EPOLL_PROXY
(void)pthread_create(&tepoll, NULL, qtfs_engine_epoll_thread, &arg[0]);
+#endif
#ifdef QTFS_TEST_MODE
fifo_arg.addr = argv[3];
@@ -566,14 +572,18 @@ int main(int argc, char *argv[])
pthread_join(texec[i], NULL);
engine_out("qtfs engine join thread %d.", i);
}
+#ifdef EPOLL_PROXY
pthread_join(tepoll, NULL);
+#endif
pthread_join(tfifo, NULL);
engine_free:
qtfs_engine_userp_free(userp, thread_nums);
engine_out("qtfs engine join epoll thread.");
end:
(void)ioctl(fd, QTFS_IOCTL_EXIT, 0);
+#ifdef EPOLL_PROXY
close(epfd);
+#endif
close(fd);
engine_out("qtfs engine over.");
return ret;
diff --git a/qtfs/qtfs_server/Makefile b/qtfs/qtfs_server/Makefile
index 4608365..765567a 100644
--- a/qtfs/qtfs_server/Makefile
+++ b/qtfs/qtfs_server/Makefile
@@ -4,6 +4,10 @@ CFLAGS += -DUDS_TEST_MODE -DQTFS_TEST_MODE
else
ccflags-y += -I$(src)/../ -I$(src) -I$(src)/../ipc/ -I$(src)/../include/ -DQTFS_SERVER
endif
+ifdef EPOLL_PROXY
+ccflags-y += -DEPOLL_PROXY
+CFLAGS += -DEPOLL_PROXY
+endif
CFLAGS += -g -O2
CFLAGS += -fstack-protector-strong
diff --git a/qtfs/qtfs_server/qtfs-server.c b/qtfs/qtfs_server/qtfs-server.c
index f4e201c..48248ba 100644
--- a/qtfs/qtfs_server/qtfs-server.c
+++ b/qtfs/qtfs_server/qtfs-server.c
@@ -59,6 +59,7 @@ void qtfs_server_epoll_exit(void)
return;
}
+#ifdef EPOLL_PROXY
long qtfs_server_epoll_thread(struct qtfs_conn_var_s *pvar)
{
int n;
@@ -170,6 +171,7 @@ long qtfs_server_epoll_init(void)
return QTOK;
}
+#endif
int qtfs_server_fd_bitmap_init(void)
{
@@ -308,11 +310,16 @@ long qtfs_server_misc_ioctl(struct file *file, unsigned int cmd, unsigned long a
write_unlock(&qtfs_epoll_rwlock);
break;
case QTFS_IOCTL_EPOLL_THREAD_INIT:
+#ifdef EPOLL_PROXY
write_lock(&qtfs_epoll_rwlock);
ret = qtfs_server_epoll_init();
write_unlock(&qtfs_epoll_rwlock);
+#else
+ qtfs_warn("Qtfs not support epoll proxy, please compile with EPOLL_PROXY=1 to enable it.");
+#endif
break;
case QTFS_IOCTL_EPOLL_THREAD_RUN:
+#ifdef EPOLL_PROXY
write_lock(&qtfs_epoll_rwlock);
if (qtfs_epoll_var == NULL) {
qtfs_err("qtfs epoll thread run failed, var is invalid.");
@@ -326,6 +333,9 @@ long qtfs_server_misc_ioctl(struct file *file, unsigned int cmd, unsigned long a
qtfs_info("qtfs epoll thread exit.");
qtfs_epoll_cut_conn(qtfs_epoll_var);
}
+#else
+ qtfs_warn("Qtfs not support epoll proxy, please compile with EPOLL_PROXY=1 to enable it.");
+#endif
break;
case QTFS_IOCTL_EXIT:
if (arg != 0 && arg != 1) {
@@ -403,6 +413,7 @@ static void __exit qtfs_server_exit(void)
qtfs_conn_param_fini();
+#ifdef EPOLL_PROXY
if (qtfs_epoll_var != NULL) {
qtfs_epoll_cut_conn(qtfs_epoll_var);
qtfs_conn_fini(qtfs_epoll_var);
@@ -410,6 +421,7 @@ static void __exit qtfs_server_exit(void)
kfree(qtfs_epoll_var);
qtfs_epoll_var = NULL;
}
+#endif
if (qtfs_diag_info != NULL) {
kfree(qtfs_diag_info);
qtfs_diag_info = NULL;
--
2.37.1 (Apple Git-137.1)

6
dpu-utilities.spec
View File

@ -1,7 +1,7 @@
Name: dpu-utilities
Summary: openEuler dpu utilities
Version: 1.10
Release: 4
Release: 5
License: GPL-2.0
Source: https://gitee.com/openeuler/dpu-utilities/repository/archive/v%{version}.tar.gz
ExclusiveOS: linux
@ -16,6 +16,7 @@ BuildRequires: kernel-devel >= 5.10, gcc, make, json-c-devel, glib2-devel
Patch1: 0001-refactor-syscall-wrapper-for-aarch64-reduce-the-memo.patch
Patch2: 0002-fix-readdir-bug-in-devtmpfs.patch
Patch3: 0003-Change-the-maximum-number-of-qtfs-links-from-16-to-6.patch
Patch4: 0004-add-compile-option-of-EPOLL_PROXY-and-fix-some-probl.patch
%description
This package contains the software utilities on dpu.
@ -143,6 +144,9 @@ sed -i '/# product cut_conf/a\dpuos kiwi/minios/cfg_dpuos yes' /opt/imageT
sed -i '/<repository_rule>/a\dpuos 1 rpm-dir euler_base' /opt/imageTailor/repos/RepositoryRule.conf
%changelog
* Tue Sep 10 2024 liqiang <liqiang64@huawei.com> 1.10-5
- Add compile option for epoll proxy thread
* Tue Jul 23 2024 liqiang <liqiang64@huawei.com> 1.10-4
- Change the maximum number of qtfs links from 16 to 64