diff --git a/src/builder/b-nspawn.c b/src/builder/b-nspawn.c
index 52298f0..333874d 100644
--- a/src/builder/b-nspawn.c
+++ b/src/builder/b-nspawn.c
@@ -51,7 +51,22 @@ saib_log_chunk_create(struct sai_nspawn *ns, void *buf, size_t len, int channel)
if (!ns->task)
return 0;
-
+ {
+ unsigned int limit = ns->task->task_log_limit ? ns->task->task_log_limit : 30000;
+ ns->log_count++;
+
+ if (ns->log_count > limit) {
+ if (!ns->killed_for_spew) {
+ ns->killed_for_spew = 1;
+ if (ns->op && ns->op->lsp) {
+ const char *msg = ">saib> <=== Killed by Sai due to log spew limit exceeded\n";
+ saib_log_chunk_create(ns, (void *)msg, strlen(msg), 3);
+ lws_spawn_piped_kill_child_process(ns->op->lsp);
+ }
+ }
+ return 0;
+ }
+ }
n = lws_snprintf(lj + LWS_PRE, sizeof(lj) - LWS_PRE,
"{\"schema\":\"com-warmcat-sai-logs\","
"\"task_uuid\":\"%s\", \"timestamp\": %llu,"
@@ -234,7 +249,7 @@ sai_lsp_reap_cb(void *opaque, const lws_spawn_resource_us_t *res, siginfo_t *si,
goto fail;
}
- if (we_killed_him & 2) {
+ if ((we_killed_him & 2) || ns->killed_for_spew) {
lwsl_notice("%s: Process killed by Sai due to spew\n", __func__);
exit_code = -1;
ns->retcode = SAISPRF_TERMINATED;
diff --git a/src/builder/b-ws-server.c b/src/builder/b-ws-server.c
index 3a1acd8..20fba3d 100644
--- a/src/builder/b-ws-server.c
+++ b/src/builder/b-ws-server.c
@@ -373,26 +373,32 @@ saib_m_tx(void *userobj, lws_ss_tx_ordinal_t ord, uint8_t *buf, size_t *len,
if (*flags & LWSSS_FLAG_EOM)
spm->inside_msg = 0;
+ {
+ size_t hysteresis_limit = (LWS_BUFLIST_OOM_LIMIT - (256 * 1024)) / 10;
+ if (lws_buflist2_total_len(&spm->bl_to_srv) <= hysteresis_limit) {
+ /* buflist drained enough, unpause any backpressured stdwsi */
+ lws_start_foreach_dll_safe(struct lws_dll2 *, d, d1, builder.sai_plat_owner.head) {
+ sai_plat_t *sp = lws_container_of(d, sai_plat_t, sai_plat_list);
+ lws_start_foreach_dll_safe(struct lws_dll2 *, d2, d3, sp->nspawn_owner.head) {
+ struct sai_nspawn *ns = lws_container_of(d2, struct sai_nspawn, list);
+ if (ns->spm == spm) {
+ for (int i = 0; i < 3; i++) {
+ if (ns->stdwsi_paused[i] && ns->stdwsi[i]) {
+ lwsl_notice("%s: Unpausing ch %d (tot %zu)\n", __func__, i,
+ lws_buflist2_total_len(&spm->bl_to_srv));
+ ns->stdwsi_paused[i] = 0;
+ lws_rx_flow_control(ns->stdwsi[i], 1 | LWS_RXFLOW_REASON_USER_BOOL);
+ }
+ }
+ }
+ } lws_end_foreach_dll_safe(d2, d3);
+ } lws_end_foreach_dll_safe(d, d1);
+ }
+ }
+
if (spm->bl_to_srv.owner.head)
return lws_ss_request_tx(spm->ss);
- /* buflist is empty, unpause any backpressured stdwsi */
- lws_start_foreach_dll_safe(struct lws_dll2 *, d, d1, builder.sai_plat_owner.head) {
- sai_plat_t *sp = lws_container_of(d, sai_plat_t, sai_plat_list);
- lws_start_foreach_dll_safe(struct lws_dll2 *, d2, d3, sp->nspawn_owner.head) {
- struct sai_nspawn *ns = lws_container_of(d2, struct sai_nspawn, list);
- if (ns->spm == spm) {
- for (int i = 0; i < 3; i++) {
- if (ns->stdwsi_paused[i] && ns->stdwsi[i]) {
- lwsl_notice("%s: Unpausing ch %d\n", __func__, i);
- ns->stdwsi_paused[i] = 0;
- lws_rx_flow_control(ns->stdwsi[i], 1 | LWS_RXFLOW_REASON_USER_BOOL);
- }
- }
- }
- } lws_end_foreach_dll_safe(d2, d3);
- } lws_end_foreach_dll_safe(d, d1);
-
return LWSSSSRET_OK;
}
diff --git a/src/common/include/private.h b/src/common/include/private.h
index 4d76d91..19e461a 100644
--- a/src/common/include/private.h
+++ b/src/common/include/private.h
@@ -216,6 +216,7 @@ typedef struct {
unsigned int est_disk_kib;
unsigned int est_wallclock_ms;
unsigned int est_compute_ms;
+ unsigned int task_log_limit;
int parallel;
char told_ongoing;
@@ -251,6 +252,9 @@ struct sai_nspawn {
struct saib_opaque_spawn *op;
sai_task_t *task;
+ unsigned int log_count;
+ unsigned int killed_for_spew:1;
+
struct lws *stdwsi[3];
uint8_t stdwsi_paused[3];
@@ -812,7 +816,7 @@ extern const lws_struct_map_t
lsm_schema_map_ta[1],
lsm_schema_map_plat_simple[1],
lsm_event[12],
- lsm_task[31],
+ lsm_task[32],
lsm_log[8],
lsm_artifact[9],
lsm_plat_list[1],
diff --git a/src/common/struct-metadata.c b/src/common/struct-metadata.c
index 92e64bb..52f6229 100644
--- a/src/common/struct-metadata.c
+++ b/src/common/struct-metadata.c
@@ -176,6 +176,7 @@ const lws_struct_map_t lsm_task[] = {
LSM_UNSIGNED (sai_task_t, est_disk_kib, "est_disk_kib"),
LSM_UNSIGNED (sai_task_t, est_wallclock_ms, "est_wallclock_ms"),
LSM_UNSIGNED (sai_task_t, est_compute_ms, "est_compute_ms"),
+ LSM_UNSIGNED (sai_task_t, task_log_limit, "task_log_limit"),
LSM_SIGNED (sai_task_t, parallel, "parallel"),
LSM_SIGNED (sai_task_t, rebuildable, "rebuildable"),
LSM_SIGNED (sai_task_t, run, "run"),
diff --git a/src/server/s-notification.c b/src/server/s-notification.c
index 6c56a45..bc4dac4 100644
--- a/src/server/s-notification.c
+++ b/src/server/s-notification.c
@@ -75,6 +75,7 @@ static const char * const saifile_paths[] = {
"configurations.*.artifacts",
"configurations.*.cpack",
"configurations.*.branches",
+ "configurations.*.task_log_limit",
"configurations.*",
};
@@ -92,6 +93,7 @@ enum enum_saifile_paths {
LEJPNSAIF_CONFIGURATIONS_ARTIFACTS,
LEJPNSAIF_CONFIGURATIONS_CPACK,
LEJPNSAIF_CONFIGURATIONS_BRANCHES,
+ LEJPNSAIF_CONFIGURATIONS_TASK_LOG_LIMIT,
LEJPNSAIF_CONFIGURATIONS_NAME,
};
@@ -759,6 +761,10 @@ next_plat: ;
lws_strncpy(sn->t.branches, ctx->buf, sizeof(sn->t.branches));
break;
+ case LEJPNSAIF_CONFIGURATIONS_TASK_LOG_LIMIT:
+ sn->t.task_log_limit = (unsigned int)atoi(ctx->buf);
+ break;
+
case LEJPNSAIF_PLAT_BUILD:
case LEJPNSAIF_PLAT_BUILD_STAGE:
/*