Author: Andy Green Date: Thu Aug 28 19:54:34 2025 +0100 scheduler: Remove fixed instance concept completely Co-developed-by: Gemini 2.5 Pro diff --git a/README.md b/README.md index 0829e7c..ab255b1 100644 --- a/README.md +++ b/README.md @@ -232,8 +232,9 @@ trees concurrently inside the platform. Tests have to take care to disambiguate which instance they are running on, since the network namespace is shared between instances that are running in the same sai-builder process on the same platform. An environment var -`SAI_INSTANCE_IDX` is available inside the each build context set to 0, 1, etc -according to the builder instance. +`SAI_INSTANCE_IDX` is available inside the each build context set to 0, 1, 33 etc +according to the builder instance. Similar to how fds are allocated in C, the +lowest unused number is reused each time something new is spawned by Sai. For network related tests, `SAI_INSTANCE_IDX` should be referred to when choosing, eg, a test server port so it will not conflict with what other diff --git a/src/builder/b-artifacts.c b/src/builder/b-artifacts.c index 17e5ce2..f71de4b 100644 --- a/src/builder/b-artifacts.c +++ b/src/builder/b-artifacts.c @@ -157,6 +157,12 @@ saib_artifact_state(void *userobj, void *sh, lws_ss_constate_t state, ap->fd = -1; } unlink(ap->path); + + ap->ns->count_artifacts--; + if (!ap->ns->count_artifacts) { + lwsl_notice("%s: last artifact completed, destroying ns now\n", __func__); + saib_task_destroy(ap->ns); + } break; case LWSSSCS_CONNECTED: diff --git a/src/builder/b-comms.c b/src/builder/b-comms.c index 7b21996..62ccd72 100644 --- a/src/builder/b-comms.c +++ b/src/builder/b-comms.c @@ -253,6 +253,8 @@ send_logs: * * For that reason we remember the last dll2 who wrote logs, and start * looking for the next nspawn with pending logs after him next time. + * (The remembered dll2 is set to NULL when the ns it is inside is + * destroyed). * * That requires statefully rotating through... * @@ -367,10 +369,14 @@ send_logs: /* * He's in DONE state, and the draining he was waiting * for has now happened. + * + * Let's move on to UPLOADING_ARTIFACTS if any, this only + * happens after we sent all the related logs. */ - lwsl_notice("%s: drained and empty\n", __func__); + lwsl_notice("%s: logs cache drained and empty\n", __func__); ns->finished_when_logs_drained = 0; - saib_set_ns_state(ns, NSSTATE_UPLOADING_ARTIFACTS); + if (ns->state != NSSTATE_FAILED) + saib_set_ns_state(ns, NSSTATE_UPLOADING_ARTIFACTS); } break; @@ -407,6 +413,7 @@ cleanup_on_ss_destroy(struct lws_dll2 *d, void *user) lws_container_of(d, struct sai_nspawn, list); if (ns->spm == spm) { + lwsl_warn("%s: ns->spm %p, spm %p\n", __func__, ns->spm, spm); /* * This pss is about to go away, make sure the ns * can't reference it any more no matter what happens @@ -590,7 +597,7 @@ saib_m_state(void *userobj, void *sh, lws_ss_constate_t state, break; case LWSSSCS_CONNECTED: - lwsl_user("%s: CONNECTED: %p\n", __func__, spm->ss); + lwsl_ss_user(spm->ss, "CONNECTED"); spm->phase = PHASE_START_ATTACH; /* Initialize the load report SUL timer for this server connection */ lws_sul_cancel(&spm->sul_load_report); @@ -602,7 +609,7 @@ saib_m_state(void *userobj, void *sh, lws_ss_constate_t state, * clean up any ongoing spawns related to this connection */ - lwsl_user("%s: DISCONNECTED\n", __func__); + lwsl_ss_user(spm->ss, "DISCONNECTED"); lws_sul_cancel(&spm->sul_load_report); lws_dll2_foreach_safe(&builder.sai_plat_owner, spm, cleanup_on_ss_disconnect); diff --git a/src/builder/b-metrics.c b/src/builder/b-metrics.c index 170fcdc..5532cfb 100644 --- a/src/builder/b-metrics.c +++ b/src/builder/b-metrics.c @@ -68,6 +68,75 @@ saib_get_free_ram_kib(void) statex.dwLength = sizeof(statex); GlobalMemoryStatusEx(&statex); return (unsigned int)(statex.ullAvailPhys / 1024); +#elif defined(__APPLE__) + int mib[2]; + size_t len; + uint64_t total_mem; + + mib[0] = CTL_HW; + mib[1] = HW_MEMSIZE; + len = sizeof(total_mem); + sysctl(mib, 2, &total_mem, &len, NULL, 0); + + return (unsigned int)(total_mem / 1024); +#elif defined(_WIN32) + MEMORYSTATUSEX statex; + statex.dwLength = sizeof(statex); + GlobalMemoryStatusEx(&statex); + return (unsigned int)(statex.ullTotalPhys / 1024); +#else + return 0; +#endif +} + +unsigned int +saib_get_total_ram_kib(void) +{ +#if defined(__linux__) + char buf[256]; + FILE *f; + unsigned int total_kib = 0; + + f = fopen("/proc/meminfo", "r"); + if (!f) + return 0; + + while (fgets(buf, sizeof(buf), f)) { + if (sscanf(buf, "MemTotal: %u kB", &total_kib) == 1) + break; + } + + fclose(f); + return total_kib; +#else + return 0; +#endif +} + +unsigned int +saib_get_total_disk_kib(const char *path) +{ +#if defined(__linux__) + struct statvfs s; + + if (statvfs(path, &s)) + return 0; + + return (unsigned int)((uint64_t)s.f_blocks * s.f_frsize / 1024); +#elif defined(__APPLE__) + struct statfs s; + + if (statfs(path, &s)) + return 0; + + return (unsigned int)((uint64_t)s.f_blocks * (uint64_t)s.f_bsize / 1024); +#elif defined(_WIN32) + ULARGE_INTEGER total_bytes; + + if (!GetDiskFreeSpaceExA(path, NULL, &total_bytes, NULL)) + return 0; + + return (unsigned int)(total_bytes.QuadPart / 1024); #else return 0; #endif diff --git a/src/builder/b-nspawn.c b/src/builder/b-nspawn.c index 1c3a022..6935808 100644 --- a/src/builder/b-nspawn.c +++ b/src/builder/b-nspawn.c @@ -89,7 +89,7 @@ callback_sai_stdwsi(struct lws *wsi, enum lws_callback_reasons reason, switch (reason) { case LWS_CALLBACK_RAW_CLOSE_FILE: - lwsl_warn("%s: RAW_CLOSE_FILE at %llu, wsi %p: fd: %d, stdfd: %d\n", + lwsl_info("%s: RAW_CLOSE_FILE at %llu, wsi %p: fd: %d, stdfd: %d\n", __func__, (unsigned long long)lws_now_usecs(), wsi, lws_get_socket_fd(wsi), lws_spawn_get_stdfd(wsi)); @@ -102,7 +102,6 @@ callback_sai_stdwsi(struct lws *wsi, enum lws_callback_reasons reason, __func__); } - lwsl_wsi_err(wsi, "CLOSING: op %p, op->lsp %p", op, op ? op->lsp : NULL); if (op && op->lsp) { lws_spawn_stdwsi_closed(op->lsp, wsi); if (ns) @@ -150,6 +149,11 @@ callback_sai_stdwsi(struct lws *wsi, enum lws_callback_reasons reason, struct lws_protocols protocol_stdxxx = { "sai-stdxxx", callback_sai_stdwsi, 0, 0 }; +/* + * We are called when the process completed and has been reaped at + * lsp level, and we know that all the stdwsi related to the process + * are closed. + */ static void sai_lsp_reap_cb(void *opaque, const lws_spawn_resource_us_t *res, siginfo_t *si, @@ -249,7 +253,14 @@ sai_lsp_reap_cb(void *opaque, const lws_spawn_resource_us_t *res, siginfo_t *si, } if (op->spawn) { - sai_build_metric_t *m = malloc(sizeof(*m)); + sai_build_metric_t *m; + + if (!ns->spm) { + lwsl_err("%s: NULL ns->spm", __func__); + goto skip; + } + + m = malloc(sizeof(*m)); if (m) { char hash_input[8192]; @@ -288,6 +299,7 @@ sai_lsp_reap_cb(void *opaque, const lws_spawn_resource_us_t *res, siginfo_t *si, } } +skip: ns->current_step++; /* step succeeded, wait for next instruction */ @@ -296,9 +308,6 @@ sai_lsp_reap_cb(void *opaque, const lws_spawn_resource_us_t *res, siginfo_t *si, if (op->spawn) free(op->spawn); - saib_task_grace(ns); - saib_set_ns_state(ns, NSSTATE_DONE); - /* * add a final zero-length log with the retcode to the list of pending * logs @@ -310,6 +319,13 @@ sai_lsp_reap_cb(void *opaque, const lws_spawn_resource_us_t *res, siginfo_t *si, __func__, ns->chunk_cache.count, ns->spm ? ns->spm->logs_in_flight : -99); + /* + * saib_task_grace(ns) sets ns->finished_when_logs_drained + */ + + saib_task_grace(ns); + saib_set_ns_state(ns, NSSTATE_DONE); + if (ns) ns->op = NULL; free(op); @@ -320,6 +336,9 @@ fail: n = lws_snprintf(s, sizeof(s), "Build step %d FAILED\n", ns->clder.sai_plat_owner.head) { - struct sai_plat *sp = lws_container_of(mp, struct sai_plat, - sai_plat_list); - -#if defined(WIN32) - sp->windows = 1; -#endif - - /* - * ... for each nspawn on the platform... - */ - - lws_start_foreach_dll_safe(struct lws_dll2 *, np, np1, - sp->nspawn_owner.head) { - struct sai_nspawn *ns = - lws_container_of(np, struct sai_nspawn, list); - char *p; - int n; - - lws_strncpy(pur, sp->name, sizeof(pur)); - lws_filename_purify_inplace(pur); - p = pur; - while ((p = strchr(p, /))n-s_filename_purify_inplace(purp = strchr(p, /))n-s_filename_purifzdwf* cap, /))n-s_filename_purifzdwf* cap, /))n-s_filename_purifzdwf* cap, /))n-s_filename_purifzdwf* cap, /))n-s_filename_purifzdwf* cap, /))n-s_filename_purifzdwf* cap, /))n-s_filename_purifzdwf* cap, /))n-s_filename_purifzdwf* cap, /))n-s_filename_purifzdwf* cap, /))n-s_filename_purifzdwf* cap, /))n-s_filename_purifzdwf* cap, /))n-s_filename_purifzdwf* cap, /))n-s_filename_purifzdwf* cap, /))n-s_filename_purifzdwf* cap, /))n-s_filename_purifzdwf* cap, /))n-s_filename_purifzdwf* cap, /))n-s_filename_purifzdwf* cap, /))n-s_filename_purifzdwf* cap, /))n-s_filename_purifzdwf* cap, /))n-s_filename_purifzdwf* cap, /))n-s_filename_purifzdwf* cap, /))n-s_filename_purifzdwf* cap, /))n-s_filename_purifzdwf* cap, /))n-s_filename_purifzdwf* cap, /))n-s_filename_purifzdwf* cap, /))n-s_filename_purifzdwf* cap, /))n-s_filename_purifzdwf* cap, /))n-s_filename_purifzdwf* cap, /))n-s_filename_purifzdwf* cap, /))n-s_filename_purifzdwf* cap, /))n-s_filename_purifzdwf* cap, /))n-s_filename_purifzdwf* cap, /))n-s_filename_purifzdwf* cap, /))n-s_filename_purifzdwf* cap, /))n-s_filename_purifzdwf* cap, /))n-s_filename_purifzdwf* cap, /))n-s_filename_purifzdwf* cap, /))n-s_filename_purifzdwf* cap, /))n-s_filename_purifzdwf* cap, /))n-s_filename_purifzdwf* cap, /))n-s_filename_purifzdwf* cap, /))n-s_filename_purifzdwf* cap, /))n-s_filename_purifzdwf* cap, /))n-s_filename_purifzdwf* cap, /))n-s_filename_purifzdwf* cap, /))n-s_filename_purifzdwf* cap, /))n-s_filename_purifzdwf* cap, /))n-s_filename_purifzdwf* cap, /))n-s_filename_purifzdwf* cap, /))n-s_filename_purifzdwf* cap, /))n-s_filename_purifzdwf* cap, /))n-s_filename_purifzdwf* cap, /))n-s_filename_purifzdwf* cap, /))n-s_filename_purifzdwf* cap, /))n-s_filename_purifzdwf* cap, /))n-s_filename_purifzdwf* cap, /))n-s_filename_purifzdwf* cap, /))n-s_filename_purifzdwf* cap, /))n-s_filename_purifzdwf* cap, /))n-s_filename_purifzdwf* cap, /))n-s_filename_purifzdwf* cap, /))n-s_filename_purifzdwf* cap, /))n-s_filename_purifzdwf* cap, /))n-s_filename_purifzdwf* cap, /))n-s_filename_purifzdwf* cap, /))n-s_filename_purifzdwf* cap, /))n-s_f