avoid frequent crash and restart

yxpandjay · Sep 9, 2020 · d12a9bb · d12a9bb
1 parent 4f3387c
commit d12a9bb
Show file tree

Hide file tree

Showing 2 changed files with 18 additions and 3 deletions.
diff --git a/base/hproc.h b/base/hproc.h
@@ -5,6 +5,8 @@
 
 typedef struct proc_ctx_s {
     pid_t           pid; // tid in Windows
+    time_t          start_time;
+    int             spawn_cnt;
     procedure_t     init;
     void*           init_userdata;
     procedure_t     proc;
@@ -28,6 +30,8 @@ static inline void hproc_run(proc_ctx_t* ctx) {
 #ifdef OS_UNIX
 // unix use multi-processes
 static inline int hproc_spawn(proc_ctx_t* ctx) {
+    ++ctx->spawn_cnt;
+    ctx->start_time = time(NULL);
     pid_t pid = fork();
     if (pid < 0) {
         perror("fork");
@@ -51,6 +55,8 @@ static void win_thread(void* userdata) {
     hproc_run(ctx);
 }
 static inline int hproc_spawn(proc_ctx_t* ctx) {
+    ++ctx->spawn_cnt;
+    ctx->start_time = time(NULL);
     HANDLE h = (HANDLE)_beginthread(win_thread, 0, ctx);
     if (h == NULL) {
         return -1;

diff --git a/utils/hmain.cpp b/utils/hmain.cpp
@@ -358,9 +358,18 @@ void signal_handler(int signo) {
         while ((pid = waitpid(-1, &status, WNOHANG)) > 0) {
             hlogw("proc stop/waiting, pid=%d status=%d", pid, status);
             for (int i = 0; i < g_main_ctx.worker_processes; ++i) {
-                if (g_main_ctx.proc_ctxs[i].pid == pid) {
-                    g_main_ctx.proc_ctxs[i].pid = -1;
-                    hproc_spawn(&g_main_ctx.proc_ctxs[i]);
+                proc_ctx_t* ctx = g_main_ctx.proc_ctxs + i;
+                if (ctx->pid == pid) {
+                    ctx->pid = -1;
+                    // NOTE: avoid frequent crash and restart
+                    time_t run_time = time(NULL) - ctx->start_time;
+                    if (ctx->spawn_cnt < 3 || run_time > 3600) {
+                        hproc_spawn(ctx);
+                    }
+                    else {
+                        hloge("proc crash, pid=%d spawn_cnt=%d run_time=%us",
+                                pid, ctx->spawn_cnt, (unsigned int)run_time);
+                    }
                     break;
                 }
             }