diff --git a/CMakeLists.txt b/CMakeLists.txt
index 2e0d0f9..73c9831 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -1,5 +1,8 @@
cmake_minimum_required(VERSION 3.10)
project(sai C)
+if (WIN32)
+ list(APPEND CMAKE_PREFIX_PATH "C:/Program Files (x86)/libwebsockets" "C:/Program Files/libwebsockets")
+endif()
find_package(libwebsockets CONFIG REQUIRED)
list(APPEND CMAKE_MODULE_PATH ${LWS_CMAKE_DIR})
include(CheckCSourceCompiles)
diff --git a/assets/sai.css b/assets/sai.css
index b060fba..2a8b6c9 100644
--- a/assets/sai.css
+++ b/assets/sai.css
@@ -1495,3 +1495,9 @@ canvas.power-graph {
.runs-time-cell {
white-space: nowrap;
}
+
+
+
+div.ibuil.vm-builder {
+ background: #e6ce8a;
+}
diff --git a/assets/sai.js b/assets/sai.js
index f057d2e..f0ccfb9 100644
--- a/assets/sai.js
+++ b/assets/sai.js
@@ -1590,6 +1590,9 @@ function createContextMenu(event, menuItems) {
function createBuilderDiv(plat) {
const platDiv = document.createElement("div");
platDiv.className = "ibuil bdr";
+ if (plat.name.startsWith("sai-vm-")) {
+ platDiv.className += " vm-builder";
+ }
if (!plat.online)
platDiv.className += " offline";
else {
@@ -1650,6 +1653,21 @@ function createBuilderDiv(plat) {
{ label: `<b>LWS:</b> ${plat.lws_hash}` },
];
+ if (authd && auth_is_admin && !plat.online) {
+ menuItems.push({
+ label: "<span style='color:#e74c3c; font-weight:bold;'>Delete Builder</span>",
+ callback: () => {
+ if (confirm("Are you sure you want to delete builder " + plat.name + "?")) {
+ const msg = {
+ schema: "com.warmcat.sai.builderdelete",
+ builder_name: plat.name
+ };
+ sai.send(JSON.stringify(msg));
+ }
+ }
+ });
+ }
+
platDiv.addEventListener("contextmenu", function(event) {
if (!authd)
return;
diff --git a/src/builder/b-nspawn.c b/src/builder/b-nspawn.c
index 2e5ce80..6ee11e7 100644
--- a/src/builder/b-nspawn.c
+++ b/src/builder/b-nspawn.c
@@ -330,35 +330,21 @@ skip:
lwsl_notice("%s: ns finished\n", __func__);
- saib_task_grace(ns);
- saib_set_ns_state(ns, NSSTATE_DONE);
- if (ns->state != NSSTATE_FAILED)
- saib_set_ns_state(ns, NSSTATE_UPLOADING_ARTIFACTS);
-
ns->reap_cb_called = 1;
if (ns)
ns->op = NULL;
+ saib_task_grace(ns);
+ saib_set_ns_state(ns, NSSTATE_DONE);
+ if (ns->state != NSSTATE_FAILED)
+ saib_set_ns_state(ns, NSSTATE_UPLOADING_ARTIFACTS);
+
if (!op->lsp || lws_spawn_get_stdwsi_open_count(op->lsp) == 0) {
lwsl_notice("%s: freeing op from reap_cb\n", __func__);
free(op);
}
- if (ns->task) {
- saib_queue_task_status_update(ns->sp, ns->spm, ns->task->uuid,
- (unsigned int)ns->retcode,
- SAI_TASK_REASON_DESTROYED);
- saib_reassess_idle_situation();
-
- builder.ram_reserved_kib -= ns->task->est_peak_mem_kib;
- builder.disk_reserved_kib -= ns->task->est_disk_kib;
- if (ns->spm)
- lws_sul_schedule(builder.context, 0,
- &ns->spm->sul_load_report,
- saib_sul_load_report_cb, 1);
- }
-
return;
fail:
@@ -371,20 +357,6 @@ fail:
saib_log_chunk_create(ns, NULL, 0, 2);
- if (ns->task) {
- saib_queue_task_status_update(ns->sp, ns->spm, ns->task->uuid,
- (unsigned int)ns->retcode,
- SAI_TASK_REASON_DESTROYED);
- saib_reassess_idle_situation();
-
- builder.ram_reserved_kib -= ns->task->est_peak_mem_kib;
- builder.disk_reserved_kib -= ns->task->est_disk_kib;
- if (ns->spm)
- lws_sul_schedule(builder.context, 0,
- &ns->spm->sul_load_report,
- saib_sul_load_report_cb, 1);
- }
-
if (op->spawn)
free(op->spawn);
diff --git a/src/builder/b-power.c b/src/builder/b-power.c
index 6ff8b54..d77bc1c 100644
--- a/src/builder/b-power.c
+++ b/src/builder/b-power.c
@@ -156,6 +156,8 @@ saib_reassess_idle_situation()
{
char in_use = 0;
+ lwsl_notice("%s: Assessing idle situation for %s (stay=%d)\n", __func__, builder.host, builder.stay);
+
if (builder.stay) {
/*
* We need to deal with finding we have been manually powered-on.
@@ -177,15 +179,21 @@ saib_reassess_idle_situation()
struct sai_plat *sp = lws_container_of(mp, struct sai_plat,
sai_plat_list);
+ lwsl_notice("%s: Checking plat %s (has %d nspawns)\n", __func__, sp->name, sp->nspawn_owner.count);
+
if (sp->nspawn_owner.head) {
lws_start_foreach_dll_safe(struct lws_dll2 *, d, d1,
sp->nspawn_owner.head) {
struct sai_nspawn *xns = lws_container_of(d,
struct sai_nspawn, list);
- if (xns->task)
- lwsl_info("%s: ongoing task: %s\n", __func__,
- xns->task->uuid);
+ if (xns->task) {
+ lwsl_info("%s: ongoing task: %s\n", __func__,
+ xns->task->uuid);
+ lwsl_notice("%s: Plat %s is busy with task %s\n", __func__, sp->name, xns->task->uuid);
+ } else {
+ lwsl_notice("%s: Plat %s is busy with an nspawn (no task uuid)\n", __func__, sp->name);
+ }
} lws_end_foreach_dll_safe(d, d1);
@@ -209,10 +217,13 @@ saib_reassess_idle_situation()
*/
if (lws_dll2_is_detached(&builder.sul_idle.list)) {
- lwsl_warn("%s: %s: no stay: starting idle grace time\n",
- __func__, builder.host);
+ int grace_secs = builder.event_affinity_active ? 1 : (int)(SAI_IDLE_GRACE_US / LWS_US_PER_SEC);
+ lwsl_notice("%s: %s: NO STAY and NO TASKS: starting %d sec idle grace time before auto-power-off\n",
+ __func__, builder.host, grace_secs);
lws_sul_schedule(builder.context, 0, &builder.sul_idle,
- sul_idle_cb, SAI_IDLE_GRACE_US);
+ sul_idle_cb, grace_secs * LWS_US_PER_SEC);
+ } else {
+ lwsl_notice("%s: %s: Idle grace time is ALREADY running\n", __func__, builder.host);
}
return 0;
@@ -238,6 +249,7 @@ saib_power_stay_rx(void *userobj, const uint8_t *buf, size_t len, int flags)
return 0;
builder.stay = *buf != '0';
+ lwsl_notice("%s: Received stay command: '%c' (stay=%d)\n", __func__, *buf, builder.stay);
saib_reassess_idle_situation();
@@ -308,10 +320,21 @@ saib_stay_init(void)
LWS_SS_USER_TYPEDEF
char payload[200];
size_t size;
- size_t pos;
+ size_t pos;
} saib_power_link_t;
static lws_ss_state_return_t
+saib_power_link_tx(void *userobj, lws_ss_tx_ordinal_t ord, uint8_t *buf,
+ size_t *len, int *flags)
+{
+ lwsl_notice("====== saib_power_link_tx called (GET request firing) ======\n");
+ *len = 0;
+ *flags = LWSSS_FLAG_SOM | LWSSS_FLAG_EOM;
+
+ return LWSSSSRET_OK;
+}
+
+static lws_ss_state_return_t
saib_power_link_rx(void *userobj, const uint8_t *buf, size_t len, int flags)
{
#if !defined(WIN32)
@@ -348,6 +371,7 @@ saib_power_link_rx(void *userobj, const uint8_t *buf, size_t len, int flags)
LWS_SS_INFO("sai_power", saib_power_link_t)
.rx = saib_power_link_rx,
+ .tx = saib_power_link_tx,
};
@@ -378,8 +402,14 @@ sul_do_suspend_cb(lws_sorted_usec_list_t *sul)
lwsl_notice("%s: resuming after suspend\n", __func__);
} else
lwsl_err("%s: failed to request suspend\n", __func__);
- } else
+ } else {
lwsl_err("%s: no suspender pipe\n", __func__);
+ if (builder.one_shot_active) {
+ lwsl_notice("%s: one-shot active and no suspender pipe, exiting cleanly\n", __func__);
+ interrupted = 1;
+ lws_cancel_service(builder.context);
+ }
+ }
#endif
}
@@ -424,12 +454,14 @@ sul_idle_cb(lws_sorted_usec_list_t *sul)
char path[256];
#if !defined(WIN32)
- if (builder.stay)
+ if (builder.stay) {
+ lwsl_err("====== SUL_IDLE_CB returning due to builder.stay ======\n");
return;
+ }
lwsl_notice("%s: idle period ended...\n", __func__);
- if (builder.power_off_type &&
+ if (!builder.one_shot_active && builder.power_off_type &&
!strcmp(builder.power_off_type, "suspend")) {
lwsl_notice("%s: starting suspend...\n", __func__);
@@ -468,8 +500,18 @@ sul_idle_cb(lws_sorted_usec_list_t *sul)
}
#endif
- if (!builder.url_sai_power)
+
+ lwsl_notice("%s: Idle grace period expired, initiating auto-power-off sequence\n", __func__);
+
+ if (!builder.url_sai_power) {
+ lwsl_warn("%s: no builder.url_sai_power set, cannot auto-power-off\n", __func__);
+ if (builder.one_shot_active) {
+ lwsl_notice("%s: one_shot_active (-O) is set and no url_sai_power. Exiting builder to allow VM to cleanly terminate.\n", __func__);
+ interrupted = 1;
+ lws_cancel_service(builder.context);
+ }
return;
+ }
/*
* We're planning to get ourselves turned off after we have shutdown
@@ -482,12 +524,16 @@ sul_idle_cb(lws_sorted_usec_list_t *sul)
snprintf(path, sizeof(path) - 1, "%s/auto-power-off/%s",
builder.url_sai_power, builder.host);
- lwsl_notice("%s: setting url metadata %s\n", __func__, path);
+ lwsl_notice("%s: requesting sai-power (or virt) to terminate us: %s\n", __func__, path);
r = lws_ss_set_metadata(builder.ss_power_off, "url", path, strlen(path));
if (r)
lwsl_err("%s: set_metadata said %d\n", __func__, (int)r);
+ r = lws_ss_client_connect(builder.ss_power_off);
+ if (r)
+ lwsl_ss_err(builder.ss_power_off, "Unable to connect ss_power_off (%d)", (int)r);
+
lws_ss_start_timeout(builder.ss_power_off, 3000); /* 3 sec */
if (lws_ss_request_tx(builder.ss_power_off))
@@ -530,11 +576,14 @@ saib_power_init(void)
lwsl_notice("%s: *** creating sai-power client ss...\n", __func__);
- if (lws_ss_create(builder.context, 0, &ssi_saib_power_client_t,
- NULL, &ss_power_client, NULL, NULL)) {
- lwsl_err("%s: *** failed to create sai-power client ss\n", __func__);
-
- return 1;
+ if (!builder.one_shot_active) {
+ if (lws_ss_create(builder.context, 0, &ssi_saib_power_client_t,
+ NULL, &ss_power_client, NULL, NULL)) {
+ lwsl_err("%s: *** failed to create sai-power client ss\n", __func__);
+ return 1;
+ }
+ } else {
+ lwsl_notice("%s: ephemeral VM, skipping sai_power_client websocket registration\n", __func__);
}
/*
@@ -544,15 +593,17 @@ saib_power_init(void)
*/
lws_ss_state_return_t r;
- lwsl_notice("%s: ****** starting sai-power-client link %s\n", __func__, builder.url_sai_power);
+ if (ss_power_client) {
+ lwsl_notice("%s: ****** starting sai-power-client link %s\n", __func__, builder.url_sai_power);
- if (lws_ss_set_metadata(ss_power_client, "url", builder.url_sai_power,
- strlen(builder.url_sai_power)))
- lwsl_warn("%s: unable to set url metadata\n", __func__);
+ if (lws_ss_set_metadata(ss_power_client, "url", builder.url_sai_power,
+ strlen(builder.url_sai_power)))
+ lwsl_warn("%s: unable to set url metadata\n", __func__);
- r = lws_ss_request_tx(ss_power_client);
- if (r)
- lwsl_notice("%s: initial tx request says %d\n", __func__, (int)r);
+ r = lws_ss_request_tx(ss_power_client);
+ if (r)
+ lwsl_notice("%s: initial tx request says %d\n", __func__, (int)r);
+ }
return 0;
}
diff --git a/src/builder/b-private.h b/src/builder/b-private.h
index 98f5375..9209248 100644
--- a/src/builder/b-private.h
+++ b/src/builder/b-private.h
@@ -77,7 +77,7 @@ struct saib_opaque_spawn {
};
#define SAI_LOAD_REPORT_US (1 * LWS_US_PER_SEC)
-#define SAI_IDLE_GRACE_US (builder.one_shot_active ? LWS_US_PER_SEC : (30 * LWS_US_PER_SEC))
+#define SAI_IDLE_GRACE_US (builder.one_shot_active ? (10 * LWS_US_PER_SEC) : (30 * LWS_US_PER_SEC))
#define SAI_STAY_POLL_US (20 * LWS_US_PER_SEC)
#define SAI_CLEANUP_JOBS_INTERVAL_US (60 * 60 * LWS_US_PER_SEC)
#define SAI_CLEANUP_JOB_DIR_MIN_AGE_SECS (24ull * 3600u)
@@ -326,6 +326,8 @@ suspender_destroy(void);
int
saib_reassess_idle_situation(void);
+extern int interrupted;
+
int
saib_app_run(int argc, const char **argv);
diff --git a/src/builder/b-sai.c b/src/builder/b-sai.c
index 03487a8..0c66feb 100644
--- a/src/builder/b-sai.c
+++ b/src/builder/b-sai.c
@@ -94,7 +94,7 @@ struct active_job_uuid {
};
static const char *config_dir = "/etc/sai/builder", *argv0;
-static int interrupted;
+int interrupted;
static lws_state_notify_link_t nl;
struct sai_builder builder;
@@ -444,7 +444,10 @@ app_system_state_nf(lws_state_manager_t *mgr, lws_state_notify_link_t *link,
/* let's sample the best possible free RAM + disk situation,
* we will derate it a bit when using it */
- builder.ram_limit_kib = saib_get_free_ram_kib();
+ if (builder.one_shot_active)
+ builder.ram_limit_kib = saib_get_total_ram_kib();
+ else
+ builder.ram_limit_kib = saib_get_free_ram_kib();
builder.disk_total_kib = saib_get_free_disk_kib(builder.home);
break;
@@ -517,7 +520,7 @@ crash_handler(int signum)
int
saib_app_run(int argc, const char **argv)
{
- int logs = LLL_USER | LLL_ERR | LLL_WARN | LLL_NOTICE;
+ int logs = 1039 | LLL_USER | LLL_ERR | LLL_WARN | LLL_NOTICE;
struct lws_context_creation_info info;
#if defined(WIN32)
char temp[256], stg_config_dir[256];
@@ -651,12 +654,9 @@ saib_app_run(int argc, const char **argv)
fw_id[--fw_n] = '\0';
if (fw_n > 0) {
- char compound[256];
- lws_snprintf(compound, sizeof(compound), "%s-%s", fw_id, builder.host ? builder.host : "");
-
- char *new_host = lwsac_use(&builder.conf_head, strlen(compound) + 1, 512);
+ char *new_host = lwsac_use(&builder.conf_head, strlen(fw_id) + 1, 512);
if (new_host) {
- strcpy(new_host, compound);
+ strcpy(new_host, fw_id);
builder.host = new_host;
lwsl_notice("%s: Applied dynamic fw_cfg builder identity: %s\n", __func__, builder.host);
}
@@ -676,12 +676,9 @@ saib_app_run(int argc, const char **argv)
if (RegQueryValueExA(hKey, "SystemSerialNumber", NULL, &dwType, (LPBYTE)fw_id, &dwSize) == ERROR_SUCCESS) {
if (!strncmp(fw_id, "sai_builder_id:", 15)) {
char *id = fw_id + 15;
- char compound[256];
- lws_snprintf(compound, sizeof(compound), "%s-%s", id, builder.host ? builder.host : "");
-
- char *new_host = lwsac_use(&builder.conf_head, strlen(compound) + 1, 512);
+ char *new_host = lwsac_use(&builder.conf_head, strlen(id) + 1, 512);
if (new_host) {
- strcpy(new_host, compound);
+ strcpy(new_host, id);
builder.host = new_host;
lwsl_notice("%s: Applied dynamic SMBIOS builder identity: %s\n", __func__, builder.host);
}
@@ -705,12 +702,9 @@ saib_app_run(int argc, const char **argv)
if (CFStringGetCString(serialNumberAsCFString, fw_id, sizeof(fw_id), kCFStringEncodingUTF8)) {
if (!strncmp(fw_id, "sai_builder_id:", 15)) {
char *id = fw_id + 15;
- char compound[256];
- lws_snprintf(compound, sizeof(compound), "%s-%s", id, builder.host ? builder.host : "");
-
- char *new_host = lwsac_use(&builder.conf_head, strlen(compound) + 1, 512);
+ char *new_host = lwsac_use(&builder.conf_head, strlen(id) + 1, 512);
if (new_host) {
- strcpy(new_host, compound);
+ strcpy(new_host, id);
builder.host = new_host;
lwsl_notice("%s: Applied dynamic SMBIOS builder identity: %s\n", __func__, builder.host);
}
diff --git a/src/builder/b-task.c b/src/builder/b-task.c
index c4e31d2..f966291 100644
--- a/src/builder/b-task.c
+++ b/src/builder/b-task.c
@@ -244,6 +244,7 @@ static void saib_start_artifact_upload(struct sai_nspawn *ns);
int
saib_set_ns_state(struct sai_nspawn *ns, int state)
{
+ struct sai_plat_server *spm = ns ? ns->spm : NULL;
ns->state = (uint8_t)state;
ns->state_changed = 1;
@@ -267,10 +268,10 @@ saib_set_ns_state(struct sai_nspawn *ns, int state)
break;
}
- if (!ns->spm || !ns->spm->ss)
+ if (!spm || !spm->ss)
return 0;
- return lws_ss_request_tx(ns->spm->ss) ? -1 : 0;
+ return lws_ss_request_tx(spm->ss) ? -1 : 0;
}
/*
@@ -383,10 +384,17 @@ saib_task_destroy(struct sai_nspawn *ns)
}
}
- if (ns->task && ns->task->ac_task_container) {
- /* contains the task object */
- lwsac_free(&ns->task->ac_task_container);
- ns->task = NULL;
+ if (ns->task) {
+ saib_queue_task_status_update(ns->sp, ns->spm, ns->task->uuid,
+ (unsigned int)ns->retcode,
+ SAI_TASK_REASON_DESTROYED);
+
+ builder.ram_reserved_kib -= ns->task->est_peak_mem_kib;
+ builder.disk_reserved_kib -= ns->task->est_disk_kib;
+ if (ns->spm)
+ lws_sul_schedule(builder.context, 0,
+ &ns->spm->sul_load_report,
+ saib_sul_load_report_cb, 1);
}
if (ns->script_path[0])
@@ -410,11 +418,12 @@ saib_task_destroy(struct sai_nspawn *ns)
*/
if (ns->task && (ns->retcode & SAISPRF_EXIT) &&
- (ns->retcode & 0xff) == 0) {
- /* Task succeeded, so clean up the directory. */
+ (ns->retcode & 0xff) == 0 &&
+ ns->task->build_step == ns->task->build_step_count - 1) {
+ /* Task succeeded completely, so clean up the directory. */
- lwsl_notice("%s: task %s succeeded, requesting deletion of job dir %s\n",
- __func__, ns->task->uuid, ns->inp);
+ lwsl_notice("%s: task %s succeeded (all %d steps), requesting deletion of job dir %s\n",
+ __func__, ns->task->uuid, ns->task->build_step_count, ns->inp);
#if defined(LWS_WITH_STUB)
if (builder.mgr_deletion) {
char json[256];
@@ -425,9 +434,18 @@ saib_task_destroy(struct sai_nspawn *ns)
#endif
}
+ if (ns->task && ns->task->ac_task_container) {
+ /* contains the task object */
+ lwsac_free(&ns->task->ac_task_container);
+ ns->task = NULL;
+ }
+
lws_dll2_remove(&ns->list);
lwsl_user("%s: free(ns) %p\n", __func__, (void *)ns);
free(ns);
+
+ saib_reassess_idle_situation();
+ lwsl_notice("====== saib_task_destroy completely finished ======\n");
}
static void
@@ -673,8 +691,8 @@ scan:
if (!ns->count_artifacts) {
lwsl_notice("%s: no artifacts, destroying ns now\n", __func__);
/* no artifacts to hang around for... nuke the ns now */
- lws_sul_schedule(builder.context, 0, &ns->sul_cleaner,
- saib_sub_cleaner_cb, 1);
+ lws_sul_cancel(&ns->sul_cleaner);
+ saib_task_destroy(ns);
} else
lwsl_notice("%s: created / waiting on %d artifact uploads\n",
__func__, ns->count_artifacts);
diff --git a/src/builder/b-ws-server.c b/src/builder/b-ws-server.c
index 63b9eab..fa52560 100644
--- a/src/builder/b-ws-server.c
+++ b/src/builder/b-ws-server.c
@@ -558,8 +558,8 @@ saib_m_state(void *userobj, void *sh, lws_ss_constate_t state,
const char *pq;
int n;
- // lwsl_user("%s: %s, ord 0x%x\n", __func__, lws_ss_state_name(state),
- // (unsigned int)ack);
+ lwsl_user("%s: %s, ord 0x%x\n", __func__, lws_ss_state_name(state),
+ (unsigned int)ack);
switch (state) {
@@ -604,6 +604,8 @@ saib_m_state(void *userobj, void *sh, lws_ss_constate_t state,
memcpy((char *)spm->name, pq, (unsigned int)n);
((char *)spm->name)[n] = '\0';
+ lwsl_user("%s: initial spm->name: '%s', url: '%s'\n", __func__, spm->name, spm->url);
+
{
char *p_name = (char *)spm->name;
char *p_found;
@@ -614,6 +616,8 @@ saib_m_state(void *userobj, void *sh, lws_ss_constate_t state,
*p_found = '_';
}
+ lwsl_user("%s: processed spm->name: '%s'\n", __func__, spm->name);
+
/* add us to the builder list of unique servers */
lws_dll2_add_head(&spm->list, &a->builder->sai_plat_server_owner);
@@ -622,6 +626,14 @@ saib_m_state(void *userobj, void *sh, lws_ss_constate_t state,
spm->refcount++;
lws_dll2_add_tail(&a->mref->list, &a->sai_plat->servers);
+ {
+ lws_ss_state_return_t r;
+ lwsl_user("%s: explicitly requesting tx to kickstart connection\n", __func__);
+ r = lws_ss_request_tx(spm->ss);
+ if (r)
+ lwsl_notice("%s: lws_ss_request_tx returned %d\n", __func__, (int)r);
+ }
+
break;
case LWSSSCS_DESTROYING:
diff --git a/src/common/include/private.h b/src/common/include/private.h
index cdf0ba9..094d194 100644
--- a/src/common/include/private.h
+++ b/src/common/include/private.h
@@ -342,6 +342,11 @@ typedef struct sai_platreset {
char platform[65];
} sai_browse_rx_platreset_t;
+typedef struct sai_builderdelete {
+ lws_dll2_t list;
+ char builder_name[96];
+} sai_browse_rx_builderdelete_t;
+
struct sai_event;
typedef struct sai_event {
@@ -763,6 +768,7 @@ typedef struct sai_platform_pending_task {
lws_dll2_t list;
char plat[64];
unsigned int pending;
+ unsigned int unmet;
} sai_platform_pending_task_t;
typedef struct sai_platform_pending_tasks {
@@ -838,7 +844,7 @@ extern const lws_struct_map_t
lsm_schema_sq3_map_watcher[1],
lsm_schema_json_map_watcher[1],
lsm_watcher_conf[1],
- lsm_pending_task[2],
+ lsm_pending_task[3],
lsm_pending_tasks[2],
lsm_schema_pending_tasks[1];
diff --git a/src/common/struct-metadata.c b/src/common/struct-metadata.c
index 242ead9..92e64bb 100644
--- a/src/common/struct-metadata.c
+++ b/src/common/struct-metadata.c
@@ -523,6 +523,7 @@ const lws_struct_map_t lsm_watcher_conf[] = {
const lws_struct_map_t lsm_pending_task[] = {
LSM_CARRAY (sai_platform_pending_task_t, plat, "plat"),
LSM_UNSIGNED (sai_platform_pending_task_t, pending, "pending"),
+ LSM_UNSIGNED (sai_platform_pending_task_t, unmet, "unmet"),
};
const lws_struct_map_t lsm_pending_tasks[] = {
diff --git a/src/power/p-sai.c b/src/power/p-sai.c
index def40ab..20ccde7 100644
--- a/src/power/p-sai.c
+++ b/src/power/p-sai.c
@@ -555,7 +555,7 @@ int main(int argc, const char **argv)
lws_start_foreach_dll_safe(struct lws_dll2 *, p, p1, temp_owner.head) {
sai_builder_registration_t *r = lws_container_of(p, sai_builder_registration_t, list);
- lwsl_notice("%s: Sticky registration found for PCON %s\n", __func__, r->power_controller_name);
+ // lwsl_notice("%s: Sticky registration found for PCON %s\n", __func__, r->power_controller_name);
saip_pcon_t *pc = saip_pcon_create(&power, r->power_controller_name);
if (pc) {
if (r->power_on_type[0])
diff --git a/src/power/p-utils.c b/src/power/p-utils.c
index e239343..9c253b7 100644
--- a/src/power/p-utils.c
+++ b/src/power/p-utils.c
@@ -55,7 +55,7 @@ saip_pcon_create(struct sai_power *power, const char *name)
lws_dll2_add_tail(&pc->list, &power->sai_pcon_owner);
- lwsl_notice("%s: Created dynamic PCON '%s'\n", __func__, name);
+ // lwsl_notice("%s: Created dynamic PCON '%s'\n", __func__, name);
return pc;
}
diff --git a/src/server/s-central.c b/src/server/s-central.c
index 2a08d86..e782e5e 100644
--- a/src/server/s-central.c
+++ b/src/server/s-central.c
@@ -73,7 +73,7 @@ sais_central_clean_abandoned(struct vhd *vhd)
*/
n = lws_struct_sq3_deserialize(vhd->server.pdb,
- " and (state != 3 and state != 4 and state != 5)",
+ " and (state != 3 and state != 4 and state != 5 and state != 7)",
NULL, lsm_schema_sq3_map_event, &o,
&ac, 0, 10);
if (n < 0 || !o.head)
@@ -117,22 +117,46 @@ sais_central_clean_abandoned(struct vhd *vhd)
sqlite3_stmt *sm;
lws_snprintf(s, sizeof(s),
- "SELECT uuid FROM tasks WHERE "
+ "SELECT uuid, started, builder_name FROM tasks WHERE "
"(state = %d OR state = %d) AND "
- "started != 0 AND started < %llu AND "
"run=(SELECT max(run) FROM tasks t2 WHERE t2.uuid = tasks.uuid)",
SAIES_PASSED_TO_BUILDER,
- SAIES_BEING_BUILT, (unsigned long long)
- (lws_now_secs() -
- (vhd->task_abandoned_timeout_mins * 60)));
+ SAIES_BEING_BUILT);
if (sqlite3_prepare_v2(pdb, s, -1, &sm, NULL) == SQLITE_OK) {
while (sqlite3_step(sm) == SQLITE_ROW) {
const unsigned char *task_uuid = sqlite3_column_text(sm, 0);
- if (task_uuid) {
- lwsl_notice("%s: resetting abandoned task %s\n",
- __func__, (const char *)task_uuid);
- sais_task_clear_build_and_logs(vhd, (const char *)task_uuid, 0);
+ unsigned long long started = (unsigned long long)sqlite3_column_int64(sm, 1);
+ const unsigned char *bname = sqlite3_column_text(sm, 2);
+ int orphaned_reason = 0;
+
+ if (started != 0 && started < (lws_now_secs() - (vhd->task_abandoned_timeout_mins * 60))) {
+ orphaned_reason = 1;
+ } else if (bname) {
+ /* Check if ANY connected builder has this task inflight */
+ if (!sais_is_task_inflight(vhd, NULL, (const char *)task_uuid, NULL)) {
+ orphaned_reason = 2;
+ }
+ } else {
+ orphaned_reason = 3;
+ }
+
+ if (orphaned_reason && task_uuid) {
+ const char *reason_str = "unknown";
+ if (orphaned_reason == 1)
+ reason_str = "timeout exceeded";
+ else if (orphaned_reason == 2)
+ reason_str = "not found in any builder's inflight list";
+ else if (orphaned_reason == 3)
+ reason_str = "bname is NULL";
+
+ lwsl_err("%s: DIAGNOSTIC: WOULD HAVE reset task %s (reason: %s, started: %llu, bname: %s)\n",
+ __func__, (const char *)task_uuid, reason_str,
+ started, bname ? (const char *)bname : "NULL");
+ /*
+ * Temporarily disabled to gather information on why tasks are being killed:
+ * sais_task_clear_build_and_logs(vhd, (const char *)task_uuid, 0);
+ */
}
}
sqlite3_finalize(sm);
diff --git a/src/server/s-comms.c b/src/server/s-comms.c
index 8ee231c..1113a61 100644
--- a/src/server/s-comms.c
+++ b/src/server/s-comms.c
@@ -68,7 +68,7 @@ sai_get_head_status(struct vhd *vhd, const char *projname)
sai_event_t *e;
int state;
- if (lws_struct_sq3_deserialize(vhd->server.pdb, NULL, "created ",
+ if (lws_struct_sq3_deserialize(vhd->server.pdb, " and state != 7", "created ",
lsm_schema_sq3_map_event, &o, &ac, 0, -1))
return -1;
diff --git a/src/server/s-notification.c b/src/server/s-notification.c
index 69e4973..6c56a45 100644
--- a/src/server/s-notification.c
+++ b/src/server/s-notification.c
@@ -518,10 +518,13 @@ next_plat: ;
*/
c = 2; /* git mirror and checkout */
- p = pl->build;
- while (*p)
- if (*p++ == '\n')
- c++;
+ if (pl->build[0]) {
+ c++;
+ p = pl->build;
+ while (*p)
+ if (*p++ == '\n')
+ c++;
+ }
pss->sn.t.build_step_count = c;
@@ -952,7 +955,7 @@ sai_notification_file_upload_cb(void *data, const char *name,
lwsl_notice("%s: m = %d\n", __func__, m);
- if (m != 1)
+ if (state != LWS_UFS_FINAL_CONTENT)
break;
lws_genhmac_destroy(&pss->hmac, result);
diff --git a/src/server/s-power.c b/src/server/s-power.c
index 9f0cdc2..30ebfa4 100644
--- a/src/server/s-power.c
+++ b/src/server/s-power.c
@@ -461,6 +461,7 @@ sais_power_tx(struct vhd *vhd, struct pss *pss, uint8_t *buf, size_t bl)
if (ptask) {
lws_strncpy(ptask->plat, pl->plat, sizeof(ptask->plat));
ptask->pending = (unsigned int)pl->pending_count;
+ ptask->unmet = (unsigned int)pl->unmet_count;
lws_dll2_add_tail(&ptask->list, &pt.tasks);
}
} lws_end_foreach_dll(px1);
diff --git a/src/server/s-private.h b/src/server/s-private.h
index 9e47cef..ce46d52 100644
--- a/src/server/s-private.h
+++ b/src/server/s-private.h
@@ -206,6 +206,7 @@ typedef struct sais_plat {
const char *plat;
char busy;
int pending_count;
+ int unmet_count;
} sais_plat_t;
struct vhd {
diff --git a/src/server/s-task-helpers.c b/src/server/s-task-helpers.c
index 2fa22aa..41d08f5 100644
--- a/src/server/s-task-helpers.c
+++ b/src/server/s-task-helpers.c
@@ -673,22 +673,22 @@ sais_task_rebuild_last_step(struct vhd *vhd, const char *task_uuid)
task = lws_container_of(o.head, sai_task_t, list);
- if (task->build_step > 0) {
- lws_snprintf(cmd, sizeof(cmd),
- "update tasks set build_step=%d where uuid='%s' and run=(select max(run) from tasks where uuid='%s')",
- task->build_step - 1, esc, esc);
-
- ret = sqlite3_exec(pdb, cmd, NULL, NULL, NULL);
- if (ret != SQLITE_OK) {
- sai_event_db_close(&vhd->sqlite3_cache, &pdb);
- lwsac_free(&ac);
- if (ret == SQLITE_BUSY)
- return SAI_DB_RESULT_BUSY;
-
- lwsl_err("%s: %s: %s: fail\n", __func__, cmd,
- sqlite3_errmsg(pdb));
- return SAI_DB_RESULT_ERROR;
- }
+ int new_step = task->build_step > 0 ? task->build_step - 1 : 0;
+
+ lws_snprintf(cmd, sizeof(cmd),
+ "update tasks set build_step=%d%s where uuid='%s' and run=(select max(run) from tasks where uuid='%s')",
+ new_step, new_step == 0 ? ",builder_name='',builder=''" : "", esc, esc);
+
+ ret = sqlite3_exec(pdb, cmd, NULL, NULL, NULL);
+ if (ret != SQLITE_OK) {
+ sai_event_db_close(&vhd->sqlite3_cache, &pdb);
+ lwsac_free(&ac);
+ if (ret == SQLITE_BUSY)
+ return SAI_DB_RESULT_BUSY;
+
+ lwsl_err("%s: %s: %s: fail\n", __func__, cmd,
+ sqlite3_errmsg(pdb));
+ return SAI_DB_RESULT_ERROR;
}
lwsac_free(&ac);
diff --git a/src/server/s-task.c b/src/server/s-task.c
index f8bda2c..25c12ce 100644
--- a/src/server/s-task.c
+++ b/src/server/s-task.c
@@ -155,7 +155,7 @@ sais_prune_inflight_list(struct vhd *vhd)
lws_start_foreach_dll_safe(struct lws_dll2 *, p1, p2, sp->inflight_owner.head) {
sai_uuid_list_t *u = lws_container_of(p1, sai_uuid_list_t, list);
- if (!u->started && (t - u->us_time_listed) > 3 * 1000 * 1000)
+ if (!u->started && (t - u->us_time_listed) > 30 * 1000 * 1000)
sais_inflight_entry_destroy(u);
} lws_end_foreach_dll_safe(p1, p2);
@@ -222,7 +222,7 @@ sais_task_pending(struct vhd *vhd, struct pss *pss, sai_plat_t *cb,
* SAIES_BEING_BUILT_HAS_FAILURES = 6,
* SAIES_DELETED = 7,
*/
- lws_snprintf(pf, sizeof(pf)," and (state != 3 and state != 4 and state != 5) and (created < %llu)",
+ lws_snprintf(pf, sizeof(pf)," and (state != 3 and state != 4 and state != 5 and state != 7) and (created < %llu)",
(unsigned long long)(lws_now_secs() - 10));
n = lws_struct_sq3_deserialize(vhd->server.pdb, pf, "created desc ",
@@ -256,9 +256,10 @@ sais_task_pending(struct vhd *vhd, struct pss *pss, sai_plat_t *cb,
* on this event
*/
- lws_snprintf(query, sizeof(query), "select count(state) from tasks where "
+ lws_snprintf(query, sizeof(query), "select count(state) from tasks t1 where "
"state IN(0,9) and platform='%s' and "
- "(builder_name IS NULL or builder_name IN('','%s'))",
+ "(builder_name IS NULL or builder_name IN('','%s'))"
+ " and run = (select max(run) from tasks t2 where t1.uuid = t2.uuid)",
esc_plat, esc_bname);
m = sqlite3_exec(pdb, query, sql3_get_integer_cb, &pending_count, NULL);
@@ -335,8 +336,9 @@ sais_task_pending(struct vhd *vhd, struct pss *pss, sai_plat_t *cb,
/* we are looking for failed tasks here */
lws_snprintf(query, sizeof(query),
- "select taskname from tasks where "
- "state = 4 and platform = ?");
+ "select taskname from tasks t1 where "
+ "state = 4 and platform = ?"
+ " and run = (select max(run) from tasks t2 where t1.uuid = t2.uuid)");
if (sqlite3_prepare_v2(prev_pdb, query, -1, &sm, NULL) == SQLITE_OK) {
const unsigned char *t;
@@ -391,7 +393,8 @@ sais_task_pending(struct vhd *vhd, struct pss *pss, sai_plat_t *cb,
lws_sql_purify(esc_taskname, fti->taskname, sizeof(esc_taskname));
lws_snprintf(pf, sizeof(pf),
" and state IN(0,9) and platform='%s' and taskname='%s' and "
- "(builder_name IS NULL or builder_name IN('','%s'))",
+ "(builder_name IS NULL or builder_name IN('','%s'))"
+ " and run = (select max(run) from tasks t2 where tasks.uuid = t2.uuid)",
esc_plat, esc_taskname, esc_bname);
lwsac_free(&pss->ac_alloc_task);
@@ -410,8 +413,8 @@ sais_task_pending(struct vhd *vhd, struct pss *pss, sai_plat_t *cb,
return NULL;
}
- lwsl_notice("%s: Prioritizing failed task for %s ('%s')\n",
- __func__, platform, fti->taskname);
+ // lwsl_notice("%s: Prioritizing failed task for %s ('%s')\n",
+ // __func__, platform, fti->taskname);
sai_event_db_close(&vhd->sqlite3_cache, &pdb);
lwsac_free(&ac);
@@ -430,7 +433,8 @@ next1: ;
lws_snprintf(pf, sizeof(pf),
" and state IN(0,9) and platform='%s' and "
- "(builder_name IS NULL or builder_name IN('','%s'))",
+ "(builder_name IS NULL or builder_name IN('','%s'))"
+ " and run = (select max(run) from tasks t2 where tasks.uuid = t2.uuid)",
esc_plat, esc_bname);
lwsac_free(&pss->ac_alloc_task);
@@ -482,7 +486,7 @@ bail:
*/
static int
-sais_find_or_add_pending_plat(struct vhd *vhd, const char *name, int count)
+sais_find_or_add_pending_plat(struct vhd *vhd, const char *name, int count, int unmet)
{
sais_plat_t *sp;
@@ -491,6 +495,7 @@ sais_find_or_add_pending_plat(struct vhd *vhd, const char *name, int count)
if (!strcmp(pl->plat, name)) {
pl->pending_count += count;
+ pl->unmet_count += unmet;
return 1;
}
@@ -503,6 +508,7 @@ sais_find_or_add_pending_plat(struct vhd *vhd, const char *name, int count)
sp->plat = (const char *)&sp[1]; /* start of overcommit */
memcpy(&sp[1], name, strlen(name) + 1);
sp->pending_count = count;
+ sp->unmet_count = unmet;
lws_dll2_add_tail(&sp->list, &vhd->pending_plats);
@@ -549,7 +555,7 @@ sais_platforms_with_tasks_pending(struct vhd *vhd)
* Collect a list of *events* (not tasks) that still have any open tasks
*/
- lws_snprintf(pf, sizeof(pf)," and (state != 3 and state != 5 and state != 7) and (created < %llu)",
+ lws_snprintf(pf, sizeof(pf)," and (state != 3 and state != 4 and state != 5 and state != 7) and (created < %llu)",
(unsigned long long)(lws_now_secs() - 10));
n = lws_struct_sq3_deserialize(vhd->server.pdb, pf, "created desc ",
@@ -574,9 +580,11 @@ sais_platforms_with_tasks_pending(struct vhd *vhd)
if (!sai_event_db_ensure_open(vhd->context, &vhd->sqlite3_cache,
vhd->sqlite3_path_lhs, e->uuid, 0, &pdb)) {
- if (sqlite3_prepare_v2(pdb, "select platform, count(*) "
- "from tasks where "
- "(state = 0 or state = 1 or state = 2) group by platform", -1, &sm,
+ if (sqlite3_prepare_v2(pdb, "select platform, count(*), "
+ "sum(case when state = 0 or state = 9 then 1 else 0 end) "
+ "from tasks t1 where "
+ "run = (select max(run) from tasks t2 where t1.uuid = t2.uuid) and "
+ "(state = 0 or state = 1 or state = 2 or state = 9) group by platform", -1, &sm,
NULL) != SQLITE_OK) {
lwsl_err("%s: Unable to %s\n",
__func__, sqlite3_errmsg(pdb));
@@ -589,7 +597,8 @@ sais_platforms_with_tasks_pending(struct vhd *vhd)
if (n == SQLITE_ROW)
sais_find_or_add_pending_plat(vhd,
(const char *)sqlite3_column_text(sm, 0),
- sqlite3_column_int(sm, 1));
+ sqlite3_column_int(sm, 1),
+ sqlite3_column_int(sm, 2));
} while (n == SQLITE_ROW);
sqlite3_reset(sm);
@@ -747,7 +756,7 @@ sais_activity_cb(lws_sorted_usec_list_t *sul)
goto next;
if (lws_struct_sq3_deserialize(pdb,
- " and (state = 1 or state = 2)",
+ " and (state = 1 or state = 2) and run = (select max(run) from tasks t2 where tasks.uuid = t2.uuid)",
NULL, lsm_schema_sq3_map_task, &o_tasks,
&ac_tasks, 0, 100) < 0 || !o_tasks.head)
goto next1;
diff --git a/src/server/s-ws-builder.c b/src/server/s-ws-builder.c
index 6131260..68700e8 100644
--- a/src/server/s-ws-builder.c
+++ b/src/server/s-ws-builder.c
@@ -453,7 +453,7 @@ sais_builder_disconnected(struct vhd *vhd, struct lws *wsi)
*/
n = lws_struct_sq3_deserialize(vhd->server.pdb,
- " and (state != 3 and state != 4 and state != 5)",
+ " and (state != 3 and state != 4 and state != 5 and state != 7)",
NULL, lsm_schema_sq3_map_event, &o, &ac, 0, 100);
if (n >= 0 && o.head) {
lws_start_foreach_dll(struct lws_dll2 *, pe, o.head) {
@@ -609,8 +609,7 @@ sais_process_rej(struct vhd *vhd, struct pss *pss,
break;
/* leave the uuid listed as inflight until step completed */
- if (sais_is_task_inflight(vhd, sp, rej->task_uuid, &ul)) {
- // lwsl_notice("%s: setting inflight started to 1 for %s\n", __func__, rej->task_uuid);
+ if (sais_is_task_inflight(vhd, NULL, rej->task_uuid, &ul)) {
ul->started = 1;
}
break;
@@ -624,6 +623,7 @@ sais_process_rej(struct vhd *vhd, struct pss *pss,
lwsl_notice("%s: SAI_TASK_REASON_BUSY: Set busy: %s\n",
__func__, rej->task_uuid);
do_remove_uuid = 1;
+ sais_set_task_state(vhd, rej->task_uuid, SAIES_WAITING, 0, 0);
sais_plat_busy(sp, 1);
break;
@@ -663,7 +663,7 @@ sais_process_rej(struct vhd *vhd, struct pss *pss,
}
if (do_remove_uuid &&
- sais_is_task_inflight(vhd, sp, rej->task_uuid, &ul)) {
+ sais_is_task_inflight(vhd, NULL, rej->task_uuid, &ul)) {
lwsl_notice("%s: ### Removing %s from inflight\n",
__func__, rej->task_uuid);
sais_inflight_entry_destroy(ul);
@@ -891,6 +891,7 @@ sais_ws_json_rx_builder(struct vhd *vhd, struct pss *pss, uint8_t *buf, size_t b
live_sp->online = 1;
live_sp->avail_mem_kib = (unsigned int)-1;
live_sp->avail_sto_kib = (unsigned int)-1;
+ sais_plat_busy(live_sp, 0);
} else {
/* New builder, create a deep-copied, malloc'd object */
size_t nlen = strlen(build->name) + 1;
diff --git a/src/server/s-ws-web.c b/src/server/s-ws-web.c
index 3045e10..5b3a6d7 100644
--- a/src/server/s-ws-web.c
+++ b/src/server/s-ws-web.c
@@ -67,6 +67,10 @@ static lws_struct_map_t lsm_browser_platreset[] = {
LSM_CARRAY (sai_browse_rx_platreset_t, platform, "platform"),
};
+static lws_struct_map_t lsm_browser_builderdelete[] = {
+ LSM_CARRAY (sai_browse_rx_builderdelete_t, builder_name, "builder_name"),
+};
+
static const lws_struct_map_t lsm_viewercount_members[] = {
LSM_UNSIGNED(sai_viewer_state_t, viewers, "count"),
};
@@ -105,7 +109,9 @@ static const lws_struct_map_t lsm_schema_json_map[] = {
LSM_SCHEMA (sai_browse_rx_evinfo_t, NULL, lsm_browser_taskreset,
/* shares struct */ "com.warmcat.sai.taskpause"),
LSM_SCHEMA (sai_browse_rx_evinfo_t, NULL, lsm_browser_taskreset,
- /* shares struct */ "com.warmcat.sai.taskresume")
+ /* shares struct */ "com.warmcat.sai.taskresume"),
+ LSM_SCHEMA (sai_browse_rx_builderdelete_t, NULL, lsm_browser_builderdelete,
+ "com.warmcat.sai.builderdelete")
};
enum {
@@ -122,6 +128,7 @@ enum {
SAIS_WS_WEBSRV_RX_TASKINFO,
SAIS_WS_WEBSRV_RX_TASKPAUSE,
SAIS_WS_WEBSRV_RX_TASKRESUME,
+ SAIS_WS_WEBSRV_RX_BUILDERDELETE,
};
static int
@@ -670,6 +677,32 @@ websrvss_ws_rx(void *userobj, const uint8_t *buf, size_t len, int flags)
lwsac_free(&a.ac);
break;
}
+ case SAIS_WS_WEBSRV_RX_BUILDERDELETE:
+ {
+ sai_browse_rx_builderdelete_t *bd = (sai_browse_rx_builderdelete_t *)a.dest;
+ char q[256], esc[96];
+
+ if (sais_validate_builder_name(bd->builder_name))
+ goto soft_error;
+
+ lwsl_notice("%s: builder delete received for %s\n", __func__, bd->builder_name);
+
+ lws_sql_purify(esc, bd->builder_name, sizeof(esc));
+
+ lws_snprintf(q, sizeof(q), "DELETE FROM builders WHERE name = '%s'", esc);
+ if (sai_sqlite3_statement(m->vhd->server.pdb, q, "delete builder"))
+ lwsl_err("%s: failed to delete builder %s\n", __func__, esc);
+
+ lws_snprintf(q, sizeof(q), "DELETE FROM pcon_builders WHERE builder_name = '%s'", esc);
+ if (sai_sqlite3_statement(m->vhd->server.pdb, q, "delete pcon builder"))
+ lwsl_err("%s: failed to delete pcon_builders %s\n", __func__, esc);
+
+ /* Force broadcast of updated builders list */
+ sais_list_builders(m->vhd);
+
+ lwsac_free(&a.ac);
+ break;
+ }
}
return 0;
diff --git a/src/virt/v-http-api.c b/src/virt/v-http-api.c
index 5a7ab55..f9cd831 100644
--- a/src/virt/v-http-api.c
+++ b/src/virt/v-http-api.c
@@ -43,17 +43,9 @@ callback_virt_http(struct lws *wsi, enum lws_callback_reasons reason,
} lws_end_foreach_dll(d);
if (found_vm) {
- if (virt.ops)
- virt.ops->destroy(&virt, found_vm);
-
- if (found_vm->plat->starting_vms > 0)
- found_vm->plat->starting_vms--;
-
- virt.running_vms--;
-
- lws_dll2_remove(&found_vm->list);
- lws_sul_cancel(&found_vm->sul_timeout);
- free(found_vm);
+ /* Delay destruction by 2s so sai-builder can cleanly flush its TCP FIN to sai-server */
+ lws_sul_schedule(virt.context, 0, &found_vm->sul_destroy,
+ saiv_vm_destroy_cb, 2 * LWS_US_PER_SEC);
}
lws_return_http_status(wsi, HTTP_STATUS_OK, NULL);
@@ -62,7 +54,7 @@ callback_virt_http(struct lws *wsi, enum lws_callback_reasons reason,
if (len > 6 && !strncmp(path, "/stay/", 6)) {
lws_strncpy(vm_id, path + 6, sizeof(vm_id));
- lwsl_notice("%s: Received stay for %s\n", __func__, vm_id);
+ lwsl_notice("%s: Received stay request for %s. Replying '0' (do not stay, proceed with auto-power-off grace period)\n", __func__, vm_id);
saiv_vm_t *found_vm = NULL;
lws_start_foreach_dll(struct lws_dll2 *, d, virt.plat_owner.head) {
@@ -85,8 +77,19 @@ callback_virt_http(struct lws *wsi, enum lws_callback_reasons reason,
}
/* We never return stay = true for ephemeral VMs */
+ uint8_t buf[LWS_PRE + 256], *p = buf + LWS_PRE, *end = p + 256;
+
+ if (lws_add_http_header_status(wsi, HTTP_STATUS_OK, &p, end)) return -1;
+ if (lws_add_http_header_by_token(wsi, WSI_TOKEN_HTTP_CONTENT_TYPE,
+ (unsigned char *)"text/plain", 10, &p, end)) return -1;
+ if (lws_add_http_header_content_length(wsi, 1, &p, end)) return -1;
+ if (lws_finalize_http_header(wsi, &p, end)) return -1;
+
+ if (lws_write(wsi, buf + LWS_PRE, (size_t)(p - (buf + LWS_PRE)), LWS_WRITE_HTTP_HEADERS) < 0)
+ return -1;
+
uint8_t stay_res = '0';
- if (lws_write(wsi, &stay_res, 1, LWS_WRITE_HTTP) != 1)
+ if (lws_write(wsi, &stay_res, 1, LWS_WRITE_HTTP_FINAL) != 1)
return -1;
return -1; /* hang up */
}
diff --git a/src/virt/v-libvirt.c b/src/virt/v-libvirt.c
index d31a844..86aefe7 100644
--- a/src/virt/v-libvirt.c
+++ b/src/virt/v-libvirt.c
@@ -144,6 +144,16 @@ ops_libvirt_spawn(struct sai_virt *virt, struct saiv_vm *vm)
capacity_unit, capacity_size,
vm->plat->base_image);
+ /* Ensure no stale volume exists */
+ char vol_name[128];
+ lws_snprintf(vol_name, sizeof(vol_name), "%s.qcow2", vm->name);
+ vol = virStorageVolLookupByName(pool, vol_name);
+ if (vol) {
+ lwsl_notice("Stale storage volume %s found, deleting...\n", vol_name);
+ virStorageVolDelete(vol, 0);
+ virStorageVolFree(vol);
+ }
+
vol = virStorageVolCreateXML(pool, vol_xml, 0);
if (!vol) {
lwsl_err("Failed to create libvirt storage volume for overlay\n");
diff --git a/src/virt/v-private.h b/src/virt/v-private.h
index 7b57f7b..da156f3 100644
--- a/src/virt/v-private.h
+++ b/src/virt/v-private.h
@@ -34,7 +34,6 @@ typedef struct saiv_plat {
char overlay_size[32];
int wait_magnification;
- int starting_vms;
lws_dll2_owner_t vm_owner;
} saiv_plat_t;
@@ -43,7 +42,9 @@ typedef struct saiv_vm {
lws_dll2_t list;
saiv_plat_t *plat;
char name[64];
+ int vm_index;
lws_sorted_usec_list_t sul_timeout;
+ lws_sorted_usec_list_t sul_destroy;
} saiv_vm_t;
/*
@@ -65,6 +66,9 @@ struct sai_virt {
const char *port; /* port we listen on */
char hostname[64];
+
+ struct lwsac *pending_tasks_ac;
+ struct sai_platform_pending_tasks *pending_tasks;
};
typedef struct saiv_server {
@@ -89,4 +93,13 @@ extern const struct lws_protocols virt_protocols[];
int saiv_config(struct sai_virt *virt, const char *d);
int saiv_config_global(struct sai_virt *virt, const char *filepath);
+void
+saiv_vm_timeout_cb(lws_sorted_usec_list_t *sul);
+
+void
+saiv_vm_destroy_cb(lws_sorted_usec_list_t *sul);
+
+void
+saiv_try_spawn(void);
+
#endif
diff --git a/src/virt/v-sai.c b/src/virt/v-sai.c
index 84a6322..460209d 100644
--- a/src/virt/v-sai.c
+++ b/src/virt/v-sai.c
@@ -137,6 +137,7 @@ int main(int argc, const char **argv)
free(p);
} lws_end_foreach_dll_safe(d, d1);
+ lwsac_free(&virt.pending_tasks_ac);
lws_context_destroy(virt.context);
return 0;
diff --git a/src/virt/v-ws-server.c b/src/virt/v-ws-server.c
index 347810e..ca184f2 100644
--- a/src/virt/v-ws-server.c
+++ b/src/virt/v-ws-server.c
@@ -30,7 +30,26 @@ saiv_server_tx(void *userobj, lws_ss_tx_ordinal_t ord, uint8_t *buf, size_t *len
return r;
}
-static void
+void
+saiv_vm_destroy_cb(lws_sorted_usec_list_t *sul)
+{
+ saiv_vm_t *vm = lws_container_of(sul, saiv_vm_t, sul_destroy);
+
+ lwsl_notice("%s: delayed destruction of %s executing\n", __func__, vm->name);
+
+ if (virt.ops)
+ virt.ops->destroy(&virt, vm);
+
+ virt.running_vms--;
+
+ lws_dll2_remove(&vm->list);
+ lws_sul_cancel(&vm->sul_timeout);
+ free(vm);
+
+ saiv_try_spawn();
+}
+
+void
saiv_vm_timeout_cb(lws_sorted_usec_list_t *sul)
{
saiv_vm_t *vm = lws_container_of(sul, saiv_vm_t, sul_timeout);
@@ -40,15 +59,130 @@ saiv_vm_timeout_cb(lws_sorted_usec_list_t *sul)
if (virt.ops)
virt.ops->destroy(&virt, vm);
- if (vm->plat->starting_vms > 0)
- vm->plat->starting_vms--;
-
virt.running_vms--;
lws_dll2_remove(&vm->list);
free(vm);
+
+ saiv_try_spawn();
+}
+
+void
+saiv_try_spawn(void)
+{
+ sai_platform_pending_tasks_t *pt = virt.pending_tasks;
+
+ if (!pt || !pt->tasks.head)
+ return;
+
+ while (virt.running_vms < virt.max_vms) {
+ int total_wheel_weight = 0;
+
+ /* Step 1: Count true demand */
+ lws_start_foreach_dll(struct lws_dll2 *, p, pt->tasks.head) {
+ sai_platform_pending_task_t *t = lws_container_of(p, sai_platform_pending_task_t, list);
+
+ saiv_plat_t *found_vp = NULL;
+ lws_start_foreach_dll(struct lws_dll2 *, d, virt.plat_owner.head) {
+ saiv_plat_t *vp = lws_container_of(d, saiv_plat_t, list);
+ const char *pname = vp->platform[0] ? vp->platform : vp->name;
+ if (!strcmp(pname, t->plat)) {
+ found_vp = vp;
+ break;
+ }
+ } lws_end_foreach_dll(d);
+
+ if (found_vp) {
+ int true_demand = (int)t->unmet - (int)found_vp->vm_owner.count;
+ if (true_demand > 0) {
+ /* Apply wait magnification factor */
+ total_wheel_weight += true_demand + found_vp->wait_magnification;
+ }
+ }
+ } lws_end_foreach_dll(p);
+
+ if (total_wheel_weight == 0)
+ break;
+
+ /* Step 2: Roll the dice */
+ uint32_t r;
+ lws_get_random(virt.context, &r, sizeof(r));
+ int target = (int)(r % (uint32_t)total_wheel_weight);
+
+ saiv_plat_t *winner = NULL;
+
+ lws_start_foreach_dll(struct lws_dll2 *, p, pt->tasks.head) {
+ sai_platform_pending_task_t *t = lws_container_of(p, sai_platform_pending_task_t, list);
+
+ saiv_plat_t *found_vp = NULL;
+ lws_start_foreach_dll(struct lws_dll2 *, d, virt.plat_owner.head) {
+ saiv_plat_t *vp = lws_container_of(d, saiv_plat_t, list);
+ const char *pname = vp->platform[0] ? vp->platform : vp->name;
+ if (!strcmp(pname, t->plat)) {
+ found_vp = vp;
+ break;
+ }
+ } lws_end_foreach_dll(d);
+
+ if (found_vp) {
+ int true_demand = (int)t->unmet - (int)found_vp->vm_owner.count;
+ if (true_demand > 0) {
+ target -= (true_demand + found_vp->wait_magnification);
+ if (target < 0) {
+ winner = found_vp;
+ break;
+ } else {
+ /* This platform wasn't picked, increase its wait magnification */
+ found_vp->wait_magnification++;
+ }
+ }
+ }
+ } lws_end_foreach_dll(p);
+
+ /* Step 3: Spawn the winner and reset its magnification */
+ if (winner && virt.ops) {
+ lwsl_notice("%s: Wheel picked platform %s (wait factor %d reset)\n",
+ __func__, winner->name, winner->wait_magnification);
+
+ saiv_vm_t *vm = malloc(sizeof(*vm));
+ if (vm) {
+ int i;
+ memset(vm, 0, sizeof(*vm));
+ vm->plat = winner;
+
+ /* Find lowest unused index */
+ for (i = 0; i < virt.max_vms; i++) {
+ int used = 0;
+ lws_start_foreach_dll(struct lws_dll2 *, d, winner->vm_owner.head) {
+ saiv_vm_t *v = lws_container_of(d, saiv_vm_t, list);
+ if (v->vm_index == i) {
+ used = 1;
+ break;
+ }
+ } lws_end_foreach_dll(d);
+ if (!used)
+ break;
+ }
+ vm->vm_index = i;
+
+ lws_snprintf(vm->name, sizeof(vm->name), "sai-vm-%s-%d", winner->name, vm->vm_index);
+ lws_dll2_add_tail(&vm->list, &winner->vm_owner);
+
+ virt.running_vms++;
+ winner->wait_magnification = 0;
+ virt.ops->spawn(&virt, vm);
+
+ /* Clean up if it never connects and terminates itself */
+ lws_sul_schedule(virt.context, 0, &vm->sul_timeout,
+ saiv_vm_timeout_cb, 5 * 60 * LWS_US_PER_SEC); /* 5 min */
+ }
+ } else {
+ break;
+ }
+ }
}
+
static lws_ss_state_return_t
saiv_server_rx(void *userobj, const uint8_t *buf, size_t len, int flags)
{
@@ -89,99 +223,20 @@ saiv_server_rx(void *userobj, const uint8_t *buf, size_t len, int flags)
stay->builder_name);
} else if (a.top_schema_index == 0) {
sai_platform_pending_tasks_t *pt = (sai_platform_pending_tasks_t *)a.dest;
- lwsl_notice("%s: Pending tasks for pcons: %s\n", __func__, pt->pcons);
-
- if (pt && pt->tasks.head) {
- while (virt.running_vms < virt.max_vms) {
- int total_wheel_weight = 0;
-
- /* Step 1: Count true demand */
- lws_start_foreach_dll(struct lws_dll2 *, p, pt->tasks.head) {
- sai_platform_pending_task_t *t = lws_container_of(p, sai_platform_pending_task_t, list);
-
- saiv_plat_t *found_vp = NULL;
- lws_start_foreach_dll(struct lws_dll2 *, d, virt.plat_owner.head) {
- saiv_plat_t *vp = lws_container_of(d, saiv_plat_t, list);
- const char *pname = vp->platform[0] ? vp->platform : vp->name;
- if (!strcmp(pname, t->plat)) {
- found_vp = vp;
- break;
- }
- } lws_end_foreach_dll(d);
-
- if (found_vp) {
- int true_demand = (int)t->pending - found_vp->starting_vms;
- if (true_demand > 0) {
- /* Apply wait magnification factor */
- total_wheel_weight += true_demand + found_vp->wait_magnification;
- }
- }
- } lws_end_foreach_dll(p);
-
- if (total_wheel_weight == 0)
- break;
-
- /* Step 2: Roll the dice */
- uint32_t r;
- lws_get_random(virt.context, &r, sizeof(r));
- int target = (int)(r % (uint32_t)total_wheel_weight);
-
- saiv_plat_t *winner = NULL;
+ // lwsl_notice("%s: Pending tasks for pcons: %s\n", __func__, pt->pcons);
- lws_start_foreach_dll(struct lws_dll2 *, p, pt->tasks.head) {
- sai_platform_pending_task_t *t = lws_container_of(p, sai_platform_pending_task_t, list);
+ if (virt.pending_tasks_ac)
+ lwsac_free(&virt.pending_tasks_ac);
- saiv_plat_t *found_vp = NULL;
- lws_start_foreach_dll(struct lws_dll2 *, d, virt.plat_owner.head) {
- saiv_plat_t *vp = lws_container_of(d, saiv_plat_t, list);
- const char *pname = vp->platform[0] ? vp->platform : vp->name;
- if (!strcmp(pname, t->plat)) {
- found_vp = vp;
- break;
- }
- } lws_end_foreach_dll(d);
-
- if (found_vp) {
- int true_demand = (int)t->pending - found_vp->starting_vms;
- if (true_demand > 0) {
- target -= (true_demand + found_vp->wait_magnification);
- if (target < 0) {
- winner = found_vp;
- break;
- } else {
- /* This platform wasn't picked, increase its wait magnification */
- found_vp->wait_magnification++;
- }
- }
- }
- } lws_end_foreach_dll(p);
+ virt.pending_tasks_ac = a.ac;
+ virt.pending_tasks = pt;
+ a.ac = NULL;
- /* Step 3: Spawn the winner and reset its magnification */
- if (winner && virt.ops) {
- lwsl_notice("%s: Wheel picked platform %s (wait factor %d reset)\n",
- __func__, winner->name, winner->wait_magnification);
-
- saiv_vm_t *vm = malloc(sizeof(*vm));
- if (vm) {
- memset(vm, 0, sizeof(*vm));
- vm->plat = winner;
- lws_snprintf(vm->name, sizeof(vm->name), "sai-vm-%s-%u", winner->name, (unsigned int)lws_now_usecs());
- lws_dll2_add_tail(&vm->list, &winner->vm_owner);
-
- winner->starting_vms++;
- virt.running_vms++;
- winner->wait_magnification = 0;
- virt.ops->spawn(&virt, vm);
-
- /* Clean up if it never connects and terminates itself */
- lws_sul_schedule(virt.context, 0, &vm->sul_timeout,
- saiv_vm_timeout_cb, 5 * 60 * LWS_US_PER_SEC); /* 5 min */
- }
- }
- }
+ saiv_try_spawn();
}
- lwsac_free(&a.ac);
+ if (a.ac)
+ lwsac_free(&a.ac);
return LWSSSSRET_OK;
}
diff --git a/src/web/w-ws-browser.c b/src/web/w-ws-browser.c
index bc3bff2..2be29fe 100644
--- a/src/web/w-ws-browser.c
+++ b/src/web/w-ws-browser.c
@@ -55,6 +55,10 @@ static lws_struct_map_t lsm_browser_platreset[] = {
LSM_CARRAY (sai_browse_rx_platreset_t, platform, "platform"),
};
+static lws_struct_map_t lsm_browser_builderdelete[] = {
+ LSM_CARRAY (sai_browse_rx_builderdelete_t, builder_name, "builder_name"),
+};
+
static lws_struct_map_t lsm_browser_taskinfo[] = {
LSM_CARRAY (sai_browse_rx_taskinfo_t, task_hash, "task_hash"),
LSM_UNSIGNED (sai_browse_rx_taskinfo_t, logs, "logs"),
@@ -96,6 +100,8 @@ static const lws_struct_map_t lsm_schema_json_map_bwsrx[] = {
/* shares struct */ "com.warmcat.sai.pcon_control"),
LSM_SCHEMA_DLL2 (sai_watcher_service_t, list, NULL, lsm_watcher_service,
"com.warmcat.sai.watcher_services"),
+ LSM_SCHEMA (sai_browse_rx_builderdelete_t, NULL, lsm_browser_builderdelete,
+ "com.warmcat.sai.builderdelete"),
};
enum {
@@ -111,6 +117,8 @@ enum {
SAIM_WS_BROWSER_RX_PLATRESET,
SAIM_WS_BROWSER_RX_STAY,
SAIM_WS_BROWSER_RX_PCON_CONTROL,
+ SAIM_WS_BROWSER_RX_WATCHER_SERVICES,
+ SAIM_WS_BROWSER_RX_BUILDERDELETE,
};
@@ -770,6 +778,15 @@ saiw_ws_json_rx_browser(struct vhd *vhd, struct pss *pss, uint8_t *buf,
*/
break;
+ case SAIM_WS_BROWSER_RX_BUILDERDELETE:
+ /*
+ * User is asking us to delete a builder
+ */
+ break;
+
+ case SAIM_WS_BROWSER_RX_WATCHER_SERVICES:
+ break;
+
default:
assert(0);
break;
![[]](/git/avatar/c50933ca2aa61e0fe2c43d46bb6b59cb_avatar){kind=avatar}
Andy Green <andy@warmcat.com> 2026-06-05 14:13 UTC