diff --git a/assets/sai.js b/assets/sai.js
index d1b9363..2df3e43 100644
--- a/assets/sai.js
+++ b/assets/sai.js
@@ -883,7 +883,8 @@ function sai_event_render(o, now_ut, reset_all_icon)
}
s1 += "<div id=\"taskstate_" + t.uuid + "\" class=\"taskstate taskstate" + t.state +
- "\" data-event-uuid=\"" + san(e.uuid) + "\" data-platform=\"" + san(t.platform) + "\">";
+ "\" data-event-uuid=\"" + san(e.uuid) + "\" data-platform=\"" + san(t.platform) +
+ "\" data-rebuildable=\"" + t.rebuildable + "\">";
s1 += "<a href=\"/sai/index.html?task=" + t.uuid + "\">" +
sai_plat_icon(t.platform, 0) + "</a>";
s1 += "</div>";
@@ -1936,8 +1937,7 @@ window.addEventListener("load", function() {
}
];
- if (taskDiv.classList.contains("taskstate4") ||
- taskDiv.classList.contains("taskstate6"))
+ if (taskDiv.dataset.rebuildable === "1")
menuItems.splice(1, 0, {
label: "Rebuild last step",
callback: () => {
diff --git a/src/builder/b-nspawn.c b/src/builder/b-nspawn.c
index c853640..f5ea985 100644
--- a/src/builder/b-nspawn.c
+++ b/src/builder/b-nspawn.c
@@ -308,8 +308,10 @@ skip:
/* step succeeded, wait for next instruction */
lwsl_notice("%s: step succeeded\n", __func__);
- if (op->spawn)
+ if (op->spawn) {
free(op->spawn);
+ op->spawn = NULL;
+ }
if (ns->spm) {
ns->spm->phase = PHASE_START_ATTACH;
diff --git a/src/builder/b-private.h b/src/builder/b-private.h
index 6d14267..ca06361 100644
--- a/src/builder/b-private.h
+++ b/src/builder/b-private.h
@@ -97,6 +97,7 @@ struct sai_builder {
lws_sorted_usec_list_t sul_idle;
lws_sorted_usec_list_t sul_stay;
+ lws_sorted_usec_list_t sul_cleanup_jobs;
const char *metrics_uri;
const char *metrics_path;
diff --git a/src/builder/b-sai.c b/src/builder/b-sai.c
index aabdb46..66365d4 100644
--- a/src/builder/b-sai.c
+++ b/src/builder/b-sai.c
@@ -61,6 +61,113 @@ int getpid(void) { return 0; }
#include "b-private.h"
+/*
+ * Periodically (eg, once per hour) we walk the jobs dir and find subdirs
+ * that are older than a day.
+ *
+ * These represent failed jobs that were left for inspection, but should now
+ * be cleaned up.
+ *
+ * We are careful not to delete anything that is part of an ongoing job.
+ */
+
+struct active_job_uuids {
+ lws_dll2_owner_t owner;
+};
+
+struct active_job_uuid {
+ lws_dll2_t list;
+ char uuid[65];
+};
+
+static int
+scan_jobs_dir_cb(const char *dirpath, void *user, struct lws_dir_entry *lde)
+{
+ struct active_job_uuids *active = (struct active_job_uuids *)user;
+ struct active_job_uuid *aj;
+ char path[512];
+ struct stat sb;
+
+ if (lde->type != LDOT_DIR || lde->name[0] == '.')
+ return 0;
+
+ lws_start_foreach_dll(struct lws_dll2 *, p, active->owner.head) {
+ aj = lws_container_of(p, struct active_job_uuid, list);
+ if (!strcmp(aj->uuid, lde->name))
+ /* it's an active job, leave it alone */
+ return 0;
+ } lws_end_foreach_dll(p);
+
+ lws_snprintf(path, sizeof(path), "%s/%s", dirpath, lde->name);
+ if (stat(path, &sb))
+ return 0;
+
+ /* older than 24h? */
+
+ if (((uint64_t)lws_now_secs() - (uint64_t)sb.st_mtime) > 24ull * 3600u) {
+ lwsl_notice("%s: removing old job dir %s\n", __func__, path);
+ lws_dir(path, NULL, lws_dir_rm_rf_cb);
+
+ /* this may have taken a while... let's wait for next time
+ * for any others that need to go */
+
+ return 1;
+ }
+
+ return 0;
+}
+
+static void
+sul_cleanup_jobs_cb(lws_sorted_usec_list_t *sul)
+{
+ struct sai_builder *b = lws_container_of(sul, struct sai_builder,
+ sul_cleanup_jobs);
+ struct active_job_uuids active;
+ struct lwsac *ac = NULL;
+ char path[256];
+
+ memset(&active, 0, sizeof(active));
+
+ /*
+ * We must not delete any active job directories, find out the uuids
+ * of any active jobs
+ */
+ lws_start_foreach_dll_safe(struct lws_dll2 *, d, d1,
+ b->sai_plat_owner.head) {
+ struct sai_plat *sp = lws_container_of(d,
+ struct sai_plat, sai_plat_list);
+ lws_start_foreach_dll_safe(struct lws_dll2 *, d2, d3,
+ sp->nspawn_owner.head) {
+ struct sai_nspawn *ns = lws_container_of(d2,
+ struct sai_nspawn, list);
+ struct active_job_uuid *aj;
+
+ if (!ns->task)
+ continue;
+
+ aj = lwsac_use_zero(&ac, sizeof(*aj), 64);
+ if (!aj)
+ continue;
+
+ lws_strncpy(aj->uuid, ns->task->uuid, sizeof(aj->uuid));
+ lws_dll2_add_tail(&aj->list, &active.owner);
+ } lws_end_foreach_dll_safe(d2, d3);
+ } lws_end_foreach_dll_safe(d, d1);
+
+ /*
+ * Now we have the active job uuids, scan the jobs dir and check
+ * for old, inactive job dirs to reap
+ */
+
+ lws_snprintf(path, sizeof(path), "%s/jobs", b->home);
+ lws_dir(path, &active, scan_jobs_dir_cb);
+
+ lwsac_free(&ac);
+
+ lws_sul_schedule(b->context, 0, &b->sul_cleanup_jobs,
+ sul_cleanup_jobs_cb, 60 * LWS_US_PER_SEC);
+}
+
static const char *config_dir = "/etc/sai/builder";
static int interrupted;
static lws_state_notify_link_t nl;
@@ -475,6 +582,9 @@ app_system_state_nf(lws_state_manager_t *mgr, lws_state_notify_link_t *link,
lws_sul_schedule(builder.context, 0, &builder.sul_stay,
sul_stay_cb, 1000);
+ lws_sul_schedule(builder.context, 0, &builder.sul_cleanup_jobs,
+ sul_cleanup_jobs_cb, 3600 * LWS_US_PER_SEC);
+
break;
}
diff --git a/src/builder/b-task.c b/src/builder/b-task.c
index 188a793..36197ab 100644
--- a/src/builder/b-task.c
+++ b/src/builder/b-task.c
@@ -395,7 +395,17 @@ saib_task_destroy(struct sai_nspawn *ns)
* since they will touch it during their close handling.
*/
+ if (ns->task && (ns->retcode & SAISPRF_EXIT) &&
+ (ns->retcode & 0xff) == 0) {
+ /* Task succeeded, so clean up the directory. */
+
+ lwsl_notice("%s: task %s succeeded, removing job dir %s\n",
+ __func__, ns->task->uuid, ns->inp);
+ lws_dir(ns->inp, NULL, lws_dir_rm_rf_cb);
+ }
+
lws_dll2_remove(&ns->list);
+ lwsl_user("%s: free(ns) %p\n", __func__, (void *)ns);
free(ns);
}
@@ -411,7 +421,7 @@ saib_sub_cleaner_cb(lws_sorted_usec_list_t *sul)
lwsl_notice("%s: +++++++++++ killing child process\n", __func__);
lws_spawn_piped_kill_child_process(ns->op->lsp);
} else {
- lwsl_err("%s: ========================== unable to kill child process (already dead?)\n", __func__);
+ lwsl_err("%s: ============= unable to kill child process -> destroying ns\n", __func__);
saib_task_destroy(ns);
}
}
diff --git a/src/common/include/private.h b/src/common/include/private.h
index 89561ee..ac9ed4f 100644
--- a/src/common/include/private.h
+++ b/src/common/include/private.h
@@ -132,6 +132,8 @@ typedef struct {
int parallel;
char told_ongoing;
+
+ char rebuildable;
} sai_task_t;
typedef struct sai_plat sai_plat_t;
@@ -525,7 +527,7 @@ extern const lws_struct_map_t
lsm_schema_map_ta[1],
lsm_schema_map_plat_simple[1],
lsm_event[10],
- lsm_task[27],
+ lsm_task[28],
lsm_log[7],
lsm_artifact[8],
lsm_plat_list[1],
diff --git a/src/common/struct-metadata.c b/src/common/struct-metadata.c
index e5a79cd..062e5a0 100644
--- a/src/common/struct-metadata.c
+++ b/src/common/struct-metadata.c
@@ -172,6 +172,7 @@ const lws_struct_map_t lsm_task[] = {
LSM_UNSIGNED (sai_task_t, est_cpu_load_pct, "est_cpu_load_pct"),
LSM_UNSIGNED (sai_task_t, est_disk_kib, "est_disk_kib"),
LSM_SIGNED (sai_task_t, parallel, "parallel"),
+ LSM_SIGNED (sai_task_t, rebuildable, "rebuildable"),
};
const lws_struct_map_t lsm_schema_json_map_task[] = {
diff --git a/src/web/w-private.h b/src/web/w-private.h
index f0da4cb..3bcbc5a 100644
--- a/src/web/w-private.h
+++ b/src/web/w-private.h
@@ -108,7 +108,7 @@ typedef struct saiw_scheduled {
char task_uuid[65];
- const sai_task_t *one_task; /* only for browser */
+ sai_task_t *one_task; /* only for browser */
const sai_event_t *one_event;
lws_dll2_t *walk;
diff --git a/src/web/w-ws-browser.c b/src/web/w-ws-browser.c
index 44e3961..0cd8436 100644
--- a/src/web/w-ws-browser.c
+++ b/src/web/w-ws-browser.c
@@ -260,6 +260,7 @@ saiw_pss_schedule_taskinfo(struct pss *pss, const char *task_uuid, int logsub)
char qu[192], esc[66], event_uuid[33], esc2[96];
sqlite3 *pdb = NULL;
lws_dll2_owner_t o;
+ sai_task_t *pt;
int n, m;
if (!sch)
@@ -304,7 +305,8 @@ saiw_pss_schedule_taskinfo(struct pss *pss, const char *task_uuid, int logsub)
if (n < 0 || !o.head)
goto bail;
- sch->one_task = lws_container_of(o.head, sai_task_t, list);
+ pt = lws_container_of(o.head, sai_task_t, list);
+ sch->one_task = pt;
lwsl_info("%s: browser ws asked for task hash: %s, plat %s\n",
__func__, task_uuid, sch->one_task->platform);
@@ -1044,6 +1046,9 @@ enum_tasks:
t->art_up_nonce[0] = '\0';
t->art_down_nonce[0] = '\0';
+ t->rebuildable = t->state == SAIES_FAIL &&
+ (lws_now_secs() - (t->started + (t->duration / 1000000)) < 24 * 3600);
+
/* only one in it at a time */
t = lws_container_of(task_owner.head, sai_task_t, list);
@@ -1205,6 +1210,9 @@ b_finish:
task_reply.event = sch->one_event;
task_reply.task = sch->one_task;
+ sch->one_task->rebuildable = sch->one_task->state == SAIES_FAIL &&
+ (lws_now_secs() - (sch->one_task->started +
+ (sch->one_task->duration / 1000000)) < 24 * 3600);
task_reply.auth_secs = (int)(pss->authorized ? pss->expiry_unix_time - lws_now_secs() : 0);
task_reply.authorized = pss->authorized;
lws_strncpy(task_reply.auth_user, pss->auth_user,