Project homepage Mailing List  Warmcat.com  API Docs  Github Mirror 
    npro  
 Modern all-safe Rust Network Protocol library supporting h1, h2, h3, ws, wt sans-IO and with socket IO + tls
git clone https://npro.rs/repo/npro
 
root / READMEs / README-builder-env.md
Author[]Andy Green <andy@warmcat.com> 2025-09-14 09:31 UTC
Committer[]Andy Green <andy@warmcat.com> 2025-09-16 03:54 UTC
Tree307fb4a09ddedde58c1e0cc3fce39e892d804775   Raw Patch
 
builder: clean jobs
builder: clean jobs

Co-developed-by: Gemini 2.5 Pro
diff --git a/assets/sai.js b/assets/sai.js index d1b9363..2df3e43 100644 --- a/assets/sai.js +++ b/assets/sai.js @@ -883,7 +883,8 @@ function sai_event_render(o, now_ut, reset_all_icon) } s1 += "<div id=\"taskstate_" + t.uuid + "\" class=\"taskstate taskstate" + t.state + - "\" data-event-uuid=\"" + san(e.uuid) + "\" data-platform=\"" + san(t.platform) + "\">"; + "\" data-event-uuid=\"" + san(e.uuid) + "\" data-platform=\"" + san(t.platform) + + "\" data-rebuildable=\"" + t.rebuildable + "\">"; s1 += "<a href=\"/sai/index.html?task=" + t.uuid + "\">" + sai_plat_icon(t.platform, 0) + "</a>"; s1 += "</div>"; @@ -1936,8 +1937,7 @@ window.addEventListener("load", function() { } ]; - if (taskDiv.classList.contains("taskstate4") || - taskDiv.classList.contains("taskstate6")) + if (taskDiv.dataset.rebuildable === "1") menuItems.splice(1, 0, { label: "Rebuild last step", callback: () => { diff --git a/src/builder/b-nspawn.c b/src/builder/b-nspawn.c index c853640..f5ea985 100644 --- a/src/builder/b-nspawn.c +++ b/src/builder/b-nspawn.c @@ -308,8 +308,10 @@ skip: /* step succeeded, wait for next instruction */ lwsl_notice("%s: step succeeded\n", __func__); - if (op->spawn) + if (op->spawn) { free(op->spawn); + op->spawn = NULL; + } if (ns->spm) { ns->spm->phase = PHASE_START_ATTACH; diff --git a/src/builder/b-private.h b/src/builder/b-private.h index 6d14267..ca06361 100644 --- a/src/builder/b-private.h +++ b/src/builder/b-private.h @@ -97,6 +97,7 @@ struct sai_builder { lws_sorted_usec_list_t sul_idle; lws_sorted_usec_list_t sul_stay; + lws_sorted_usec_list_t sul_cleanup_jobs; const char *metrics_uri; const char *metrics_path; diff --git a/src/builder/b-sai.c b/src/builder/b-sai.c index aabdb46..66365d4 100644 --- a/src/builder/b-sai.c +++ b/src/builder/b-sai.c @@ -61,6 +61,113 @@ int getpid(void) { return 0; } #include "b-private.h" +/* + * Periodically (eg, once per hour) we walk the jobs dir and find subdirs + * that are older than a day. + * + * These represent failed jobs that were left for inspection, but should now + * be cleaned up. + * + * We are careful not to delete anything that is part of an ongoing job. + */ + +struct active_job_uuids { + lws_dll2_owner_t owner; +}; + +struct active_job_uuid { + lws_dll2_t list; + char uuid[65]; +}; + +static int +scan_jobs_dir_cb(const char *dirpath, void *user, struct lws_dir_entry *lde) +{ + struct active_job_uuids *active = (struct active_job_uuids *)user; + struct active_job_uuid *aj; + char path[512]; + struct stat sb; + + if (lde->type != LDOT_DIR || lde->name[0] == '.') + return 0; + + lws_start_foreach_dll(struct lws_dll2 *, p, active->owner.head) { + aj = lws_container_of(p, struct active_job_uuid, list); + if (!strcmp(aj->uuid, lde->name)) + /* it's an active job, leave it alone */ + return 0; + } lws_end_foreach_dll(p); + + lws_snprintf(path, sizeof(path), "%s/%s", dirpath, lde->name); + if (stat(path, &sb)) + return 0; + + /* older than 24h? */ + + if (((uint64_t)lws_now_secs() - (uint64_t)sb.st_mtime) > 24ull * 3600u) { + lwsl_notice("%s: removing old job dir %s\n", __func__, path); + lws_dir(path, NULL, lws_dir_rm_rf_cb); + + /* this may have taken a while... let's wait for next time + * for any others that need to go */ + + return 1; + } + + return 0; +} + +static void +sul_cleanup_jobs_cb(lws_sorted_usec_list_t *sul) +{ + struct sai_builder *b = lws_container_of(sul, struct sai_builder, + sul_cleanup_jobs); + struct active_job_uuids active; + struct lwsac *ac = NULL; + char path[256]; + + memset(&active, 0, sizeof(active)); + + /* + * We must not delete any active job directories, find out the uuids + * of any active jobs + */ + lws_start_foreach_dll_safe(struct lws_dll2 *, d, d1, + b->sai_plat_owner.head) { + struct sai_plat *sp = lws_container_of(d, + struct sai_plat, sai_plat_list); + lws_start_foreach_dll_safe(struct lws_dll2 *, d2, d3, + sp->nspawn_owner.head) { + struct sai_nspawn *ns = lws_container_of(d2, + struct sai_nspawn, list); + struct active_job_uuid *aj; + + if (!ns->task) + continue; + + aj = lwsac_use_zero(&ac, sizeof(*aj), 64); + if (!aj) + continue; + + lws_strncpy(aj->uuid, ns->task->uuid, sizeof(aj->uuid)); + lws_dll2_add_tail(&aj->list, &active.owner); + } lws_end_foreach_dll_safe(d2, d3); + } lws_end_foreach_dll_safe(d, d1); + + /* + * Now we have the active job uuids, scan the jobs dir and check + * for old, inactive job dirs to reap + */ + + lws_snprintf(path, sizeof(path), "%s/jobs", b->home); + lws_dir(path, &active, scan_jobs_dir_cb); + + lwsac_free(&ac); + + lws_sul_schedule(b->context, 0, &b->sul_cleanup_jobs, + sul_cleanup_jobs_cb, 60 * LWS_US_PER_SEC); +} + static const char *config_dir = "/etc/sai/builder"; static int interrupted; static lws_state_notify_link_t nl; @@ -475,6 +582,9 @@ app_system_state_nf(lws_state_manager_t *mgr, lws_state_notify_link_t *link, lws_sul_schedule(builder.context, 0, &builder.sul_stay, sul_stay_cb, 1000); + lws_sul_schedule(builder.context, 0, &builder.sul_cleanup_jobs, + sul_cleanup_jobs_cb, 3600 * LWS_US_PER_SEC); + break; } diff --git a/src/builder/b-task.c b/src/builder/b-task.c index 188a793..36197ab 100644 --- a/src/builder/b-task.c +++ b/src/builder/b-task.c @@ -395,7 +395,17 @@ saib_task_destroy(struct sai_nspawn *ns) * since they will touch it during their close handling. */ + if (ns->task && (ns->retcode & SAISPRF_EXIT) && + (ns->retcode & 0xff) == 0) { + /* Task succeeded, so clean up the directory. */ + + lwsl_notice("%s: task %s succeeded, removing job dir %s\n", + __func__, ns->task->uuid, ns->inp); + lws_dir(ns->inp, NULL, lws_dir_rm_rf_cb); + } + lws_dll2_remove(&ns->list); + lwsl_user("%s: free(ns) %p\n", __func__, (void *)ns); free(ns); } @@ -411,7 +421,7 @@ saib_sub_cleaner_cb(lws_sorted_usec_list_t *sul) lwsl_notice("%s: +++++++++++ killing child process\n", __func__); lws_spawn_piped_kill_child_process(ns->op->lsp); } else { - lwsl_err("%s: ========================== unable to kill child process (already dead?)\n", __func__); + lwsl_err("%s: ============= unable to kill child process -> destroying ns\n", __func__); saib_task_destroy(ns); } } diff --git a/src/common/include/private.h b/src/common/include/private.h index 89561ee..ac9ed4f 100644 --- a/src/common/include/private.h +++ b/src/common/include/private.h @@ -132,6 +132,8 @@ typedef struct { int parallel; char told_ongoing; + + char rebuildable; } sai_task_t; typedef struct sai_plat sai_plat_t; @@ -525,7 +527,7 @@ extern const lws_struct_map_t lsm_schema_map_ta[1], lsm_schema_map_plat_simple[1], lsm_event[10], - lsm_task[27], + lsm_task[28], lsm_log[7], lsm_artifact[8], lsm_plat_list[1], diff --git a/src/common/struct-metadata.c b/src/common/struct-metadata.c index e5a79cd..062e5a0 100644 --- a/src/common/struct-metadata.c +++ b/src/common/struct-metadata.c @@ -172,6 +172,7 @@ const lws_struct_map_t lsm_task[] = { LSM_UNSIGNED (sai_task_t, est_cpu_load_pct, "est_cpu_load_pct"), LSM_UNSIGNED (sai_task_t, est_disk_kib, "est_disk_kib"), LSM_SIGNED (sai_task_t, parallel, "parallel"), + LSM_SIGNED (sai_task_t, rebuildable, "rebuildable"), }; const lws_struct_map_t lsm_schema_json_map_task[] = { diff --git a/src/web/w-private.h b/src/web/w-private.h index f0da4cb..3bcbc5a 100644 --- a/src/web/w-private.h +++ b/src/web/w-private.h @@ -108,7 +108,7 @@ typedef struct saiw_scheduled { char task_uuid[65]; - const sai_task_t *one_task; /* only for browser */ + sai_task_t *one_task; /* only for browser */ const sai_event_t *one_event; lws_dll2_t *walk; diff --git a/src/web/w-ws-browser.c b/src/web/w-ws-browser.c index 44e3961..0cd8436 100644 --- a/src/web/w-ws-browser.c +++ b/src/web/w-ws-browser.c @@ -260,6 +260,7 @@ saiw_pss_schedule_taskinfo(struct pss *pss, const char *task_uuid, int logsub) char qu[192], esc[66], event_uuid[33], esc2[96]; sqlite3 *pdb = NULL; lws_dll2_owner_t o; + sai_task_t *pt; int n, m; if (!sch) @@ -304,7 +305,8 @@ saiw_pss_schedule_taskinfo(struct pss *pss, const char *task_uuid, int logsub) if (n < 0 || !o.head) goto bail; - sch->one_task = lws_container_of(o.head, sai_task_t, list); + pt = lws_container_of(o.head, sai_task_t, list); + sch->one_task = pt; lwsl_info("%s: browser ws asked for task hash: %s, plat %s\n", __func__, task_uuid, sch->one_task->platform); @@ -1044,6 +1046,9 @@ enum_tasks: t->art_up_nonce[0] = '\0'; t->art_down_nonce[0] = '\0'; + t->rebuildable = t->state == SAIES_FAIL && + (lws_now_secs() - (t->started + (t->duration / 1000000)) < 24 * 3600); + /* only one in it at a time */ t = lws_container_of(task_owner.head, sai_task_t, list); @@ -1205,6 +1210,9 @@ b_finish: task_reply.event = sch->one_event; task_reply.task = sch->one_task; + sch->one_task->rebuildable = sch->one_task->state == SAIES_FAIL && + (lws_now_secs() - (sch->one_task->started + + (sch->one_task->duration / 1000000)) < 24 * 3600); task_reply.auth_secs = (int)(pss->authorized ? pss->expiry_unix_time - lws_now_secs() : 0); task_reply.authorized = pss->authorized; lws_strncpy(task_reply.auth_user, pss->auth_user,
Page fetched 0s ago, creation time: 27ms (vhost etag hits: 0%, cache hits: 0%)