Project homepage Mailing List  Warmcat.com  API Docs  Github Mirror 
    npro  
 Modern all-safe Rust Network Protocol library supporting h1, h2, h3, ws, wt sans-IO and with socket IO + tls
git clone https://npro.rs/repo/npro
Author[]Andy Green <andy@warmcat.com> 2025-09-17 07:53 UTC
Committer[]Andy Green <andy@warmcat.com> 2025-09-17 10:12 UTC
Tree02d392439e8365d4291a64b1d287b6b7ce1ac93b   Raw Patch
 
builder: deletion process
builder: deletion process

Co-developed-by: Gemini 2.5 Pro
diff --git a/src/builder/b-private.h b/src/builder/b-private.h index ca06361..54608ca 100644 --- a/src/builder/b-private.h +++ b/src/builder/b-private.h @@ -131,6 +131,15 @@ struct sai_builder { ULARGE_INTEGER last_sys_user; #endif char stay; + + /* resource management */ + + uint64_t ram_limit_kib; + uint64_t ram_reserved_kib; + uint64_t disk_total_kib; + uint64_t disk_reserved_kib; + + int pipe_master_wr; }; struct jpargs { diff --git a/src/builder/b-sai.c b/src/builder/b-sai.c index 66365d4..f47cebe 100644 --- a/src/builder/b-sai.c +++ b/src/builder/b-sai.c @@ -37,19 +37,21 @@ #include <grp.h> #endif -#if defined(__linux__) +#if defined(__linux__) || defined(__APPLE__) #include <unistd.h> #endif #if defined(__APPLE__) #include <sys/stat.h> /* for mkdir() */ -#include <unistd.h> /* for chown() */ #endif #if defined(WIN32) #include <initguid.h> #include <KnownFolders.h> #include <Shlobj.h> +#include <processthreadsapi.h> +#include <handleapi.h> + #if !defined(PATH_MAX) #define PATH_MAX MAX_PATH @@ -61,6 +63,56 @@ int getpid(void) { return 0; } #include "b-private.h" +static int +sai_deletion_worker(const char *home_dir) +{ + char path[PATH_MAX], *p; + ssize_t n; + + lwsl_notice("%s: deletion worker started\n", __func__); + +#if defined(WIN32) + /* + * On Windows, stdin is not a pipe from the parent but a handle + * value passed on the commandline + */ + // detach from console... + FreeConsole(); +#endif + + while (1) { + n = read(0, path, sizeof(path) - 1); + + if (n <= 0) { + lwsl_notice("%s: pipe closed, exiting\n", __func__); + return 0; + } + + path[n] = '\0'; + p = path; + + /* sanitize: no .. or / or \ */ + while (*p) { + if (*p == '.' || *p == '/' || *p == '\\') { + lwsl_err("%s: invalid chars in delete path '%s'\n", + __func__, path); + p = NULL; + break; + } + p++; + } + if (!p) + continue; + + lws_snprintf(path, sizeof(path), "%s/jobs/%s", home_dir, path); + + if (lws_dir(path, NULL, lws_dir_rm_rf_cb)) + lwsl_err("%s: failed to delete %s\n", __func__, path); + } + + return 0; +} + /* * Periodically (eg, once per hour) we walk the jobs dir and find subdirs * that are older than a day. @@ -105,13 +157,22 @@ scan_jobs_dir_cb(const char *dirpath, void *user, struct lws_dir_entry *lde) /* older than 24h? */ if (((uint64_t)lws_now_secs() - (uint64_t)sb.st_mtime) > 24ull * 3600u) { - lwsl_notice("%s: removing old job dir %s\n", __func__, path); - lws_dir(path, NULL, lws_dir_rm_rf_cb); - - /* this may have taken a while... let's wait for next time - * for any others that need to go */ - - return 1; + lwsl_notice("%s: requesting removal of old job dir %s\n", + __func__, path); +#if !defined(WIN32) + if (write(builder.pipe_master_wr, lde->name, + LWS_POSIX_LENGTH_CAST(strlen(lde->name))) != (ssize_t)strlen(lde->name)) + lwsl_err("%s: failed to write to deletion worker\n", + __func__); +#else + { + DWORD written; + if (!WriteFile(builder.pipe_master_wr_win, lde->name, + (DWORD)strlen(lde->name), &written, NULL)) + lwsl_err("%s: failed to write to deletion worker\n", + __func__); + } +#endif } return 0; @@ -168,6 +229,7 @@ sul_cleanup_jobs_cb(lws_sorted_usec_list_t *sul) sul_cleanup_jobs_cb, 60 * LWS_US_PER_SEC); } + static const char *config_dir = "/etc/sai/builder"; static int interrupted; static lws_state_notify_link_t nl; @@ -361,7 +423,7 @@ saib_create_resproxy_listen_uds(struct lws_context *context, info.vhost_name = pv; pv += lws_snprintf(pv, sizeof(vhnames) - (size_t)(pv - vhnames), - "resproxy.%u.%d", getpid(), spm->index) + 1; + "resproxy.%u.%d", (unsigned int)getpid(), spm->index) + 1; info.options = LWS_SERVER_OPTION_ADOPT_APPLY_LISTEN_ACCEPT_CONFIG | LWS_SERVER_OPTION_UNIX_SOCK; @@ -781,6 +843,13 @@ int main(int argc, const char **argv) struct stat sb; const char *p; + if ((p = lws_cmdline_option(argc, argv, "--home"))) + /* + * This is the deletion worker process being spawned, it only + * needs to know the home dir to clean up inside + */ + return sai_deletion_worker(p); + if ((p = lws_cmdline_option(argc, argv, "-s"))) { ssize_t n = 0; @@ -888,6 +957,13 @@ int main(int argc, const char **argv) if ((p = lws_cmdline_option(argc, argv, "-d"))) logs = atoi(p); + if ((p = lws_cmdline_option(argc, argv, "--home"))) + /* + * This is the deletion worker process being spawned, it only + * needs to know the home dir to clean up inside + */ + return sai_deletion_worker(p); + if ((p = lws_cmdline_option(argc, argv, "-c"))) config_dir = p; @@ -950,6 +1026,46 @@ int main(int argc, const char **argv) return 1; } +#if !defined(WIN32) + { + int pfd[2]; + pid_t pid; + + if (pipe(pfd) == -1) { + lwsl_err("pipe() failed\n"); + return 1; + } + + pid = fork(); + if (pid == -1) { + lwsl_err("fork() failed\n"); + return 1; + } + + if (!pid) { + /* child: deletion worker */ + char home_arg[256]; + + lws_snprintf(home_arg, sizeof(home_arg), "--home=%s", + builder.home); + close(pfd[1]); /* wr */ + if (dup2(pfd[0], 0) < 0) + return 1; + close(pfd[0]); + + execlp(argv[0], argv[0], home_arg, "--delete-worker", (char *)NULL); + lwsl_err("execlp failed\n"); + return 1; + } + + /* parent */ + close(pfd[0]); /* rd */ + builder.pipe_master_wr = pfd[1]; + } +#else + /* TODO: windows worker process spawn */ +#endif + #if defined(__linux__) /* * At this point we're still root. So we should be able diff --git a/src/builder/b-task.c b/src/builder/b-task.c index 4dcc53e..a75e216 100644 --- a/src/builder/b-task.c +++ b/src/builder/b-task.c @@ -401,9 +401,21 @@ saib_task_destroy(struct sai_nspawn *ns) (ns->retcode & 0xff) == 0) { /* Task succeeded, so clean up the directory. */ - lwsl_notice("%s: task %s succeeded, removing job dir %s\n", + lwsl_notice("%s: task %s succeeded, requesting deletion of job dir %s\n", __func__, ns->task->uuid, ns->inp); - lws_dir(ns->inp, NULL, lws_dir_rm_rf_cb); + if (write(builder.pipe_master_wr, ns->task->uuid, + LWS_POSIX_LENGTH_CAST(strlen(ns->task->uuid))) != (ssize_t)strlen(ns->task->uuid)) + lwsl_err("%s: failed to write to deletion worker\n", + __func__); + } + + if (ns->task) { + builder.ram_reserved_kib -= ns->task->est_peak_mem_kib; + builder.disk_reserved_kib -= ns->task->est_disk_kib; + if (ns->spm) + lws_sul_schedule(builder.context, 0, + &ns->spm->sul_load_report, + saib_sul_load_report_cb, 1); } lws_dll2_remove(&ns->list);
Page fetched 0s ago, creation time: 4ms (vhost etag hits: 0%, cache hits: 0%)