diff --git a/AGENTS.md b/AGENTS.md
index 2251e1a..ebf8fd7 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -2,7 +2,8 @@
## Overview
-Please make high quality, not lazy implementation decisions.
+Please make high quality, not lazy, implementation decisions, bearing in mind
+maintainability.
Our work should follow the existing usage of apis in the project as much as possible.
@@ -16,6 +17,15 @@ We avoid casual linked-lists and use `lws_dll2_t`.
We consider using lwsac instead of discrete allocations, if the pattern of allocations will benefit from it.
+We consider using lws_struct to convert between sqlite storage <-> structs <-> JSON
+
+## Security
+
+Please bear in mind what parts of the system are secrets and look after the security of them.
+
+In particular, all web pieces are made available on the internet with a strict CSP. That means
+no inline styles or scripts. You can find the web pieces (JS, HTML, css) in ./assets/
+
## Build testing
Please don't worry about build-testing, just push patches when you are confident they are complete and have considered all affected code (ie, not half-assed) and ready and I will try them and report back with grounded information.
@@ -27,9 +37,9 @@ Sai dependencies:
- libwebsockets from main branch
- libssl-dev
- libsqlite3
- - Build lws: git init && git fetch https://github.com/warmcat/libwebsockets +main:main && cd libwebsockets && mkdir -p build && cd build && cmake .. -DLWS_UNIX_SOCK=1 -DLWS_WITH_STRUCT_JSON=1 -DLWS_WITH_JOSE=1 -DLWS_WITH_STRUCT_SQLITE3=1 -DLWS_WITH_GENCRYPTO=1 -DLWS_WITH_SPAWN=1 -DLWS_WITH_SECURE_STREAMS=1 && make && sudo make install && sudo ldconfig
+ - Build lws: `git init && git fetch https://github.com/warmcat/libwebsockets +main:main && cd libwebsockets && mkdir -p build && cd build && cmake .. -DLWS_UNIX_SOCK=1 -DLWS_WITH_STRUCT_JSON=1 -DLWS_WITH_JOSE=1 -DLWS_WITH_STRUCT_SQLITE3=1 -DLWS_WITH_GENCRYPTO=1 -DLWS_WITH_SPAWN=1 -DLWS_WITH_SECURE_STREAMS=1 && make && sudo make install && sudo ldconfig`
- sai (main branch shown)
- - Build sai: git init && git fetch https://github.com/warmcat/sai +main:m && cd libwebsockets && mkdir -p build && cd build && cmake .. && make && sudo make install && sudo ldconfig
+ - Build sai: `git init && git fetch https://github.com/warmcat/sai +main:m && cd libwebsockets && mkdir -p build && cd build && cmake .. && make && sudo make install && sudo ldconfig`
diff --git a/assets/sai.css b/assets/sai.css
index 14bc7cf..b846bb3 100644
--- a/assets/sai.css
+++ b/assets/sai.css
@@ -586,6 +586,7 @@ td.bn {
font-weight: normal;
font-size: 7pt;
padding: 1px;
+ position: relative;
}
td.bins {
@@ -604,7 +605,7 @@ table.nomar {
padding: 0px;
margin: -3px;
width: 100%;
- text-align:left;
+ text-align:center;
}
td.nomar {
@@ -641,6 +642,7 @@ div.ibuil {
user-select: none;
-webkit-user-select: none;
cursor: pointer;
+ width: 80px;
}
div.ibuilctr {
@@ -761,39 +763,6 @@ img.branch {
color: rgba(0, 0, 0, 0);
}
-.inst_box {
- display: inline-block;
- width: 12px;
- height: 12px;
- border: 1px solid #777;
- margin-left: 2px;
- vertical-align: middle;
-}
-.inst_idle { background-color: #d0d0d0; }
-.inst_run { background-color: #78c678; }
-.load_text { font-size: 9px; vertical-align: middle; }
-
-/* Container for all instance boxes for a given platform */
-.instload {
- display: flex;
- flex-wrap: wrap;
- gap: 2px;
- padding-top: 4px;
- padding-bottom: 4px;
- min-height: 22px; /* Ensure container doesn't collapse if empty */
-}
-
-/* The outer box for a single instance */
-.inst_box {
- width: 16px;
- height: 16px;
- border: 1px solid #555;
- box-sizing: border-box;
- background-color: #ddd; /* Dark background for the meter */
- position: relative; /* Needed for positioning the child bar */
- overflow: hidden; /* Hide any part of the bar that goes over 100% */
-}
-
/* The actual colored bar inside the box */
.inst_bar {
background-color: #00afff; /* A nice, high-contrast green */
@@ -804,18 +773,6 @@ img.branch {
transition: height 0.2s ease-out; /* Smoothly animate height changes */
}
-.inst_idle {
- border-color: #555;
- opacity: 0.3; /* Make idle instances semi-transparent */
- transition: opacity 0.3s; /* Smoothly fade in/out */
-}
-
-.inst_busy {
- border-color: #999;
- opacity: 1.0; /* Ensure busy instances are fully opaque */
- transition: opacity 0.3s;
-}
-
@keyframes flash-slow {
50% { opacity: 0.8; }
}
@@ -1025,3 +982,70 @@ body.overlay-active {
background-color: #ccc;
flex-shrink: 0;
}
+
+.step-progress {
+ display: flex;
+ align-items: center;
+}
+
+.step-block {
+ width: 8px;
+ height: 8px;
+ border: 1px solid #aaa;
+ margin-right: 2px;
+}
+
+.step-done {
+ background-color: #555;
+}
+
+.step-current {
+ background-color: #aaa;
+}
+
+.step-pending {
+ background-color: #fff;
+}
+
+.step-label {
+ font-size: 7pt;
+ margin-left: 4px;
+}
+
+.resource-bars {
+ display: flex;
+ flex-direction: column;
+ gap: 1px;
+ margin-top: 2px;
+}
+
+.res-bar {
+ height: 2px;
+ background-color: rgba(0, 0, 0, 0.2);
+ border: 1px solid #fff;
+ width: 100%;
+}
+
+.res-bar-inner {
+ height: 100%;
+ transition: width 0.2s ease-out;
+}
+
+.res-bar-cpu {
+ background-color: #ff4136; /* Red */
+}
+
+.res-bar-ram {
+ background-color: #0074d9; /* Blue */
+}
+
+.res-bar-disk {
+ background-color: #2ecc40; /* Green */
+}
+
+.res-bar.empty {
+ border: 1px solid #777;
+ height: 5px;
+ width: 24px;
+ margin-bottom: 1px;
+}
diff --git a/assets/sai.js b/assets/sai.js
index 777c73a..3d98622 100644
--- a/assets/sai.js
+++ b/assets/sai.js
@@ -295,11 +295,11 @@ var lang_ja = "{" +
"\"Committer\": \"コミッター\"," +
"\"Raw Patch\": \"生パッチ\"," +
"\"Page fetched %{pf} ago, creation time: %{ct}ms " +
- "(vhost etag hits: %{ve}%, cache hits: %{ch}%)\": " +
- "\"%{pf}間前に取得されたページ, 作成にかかった時間: %{ct}ms " +
- "(vhost etag キャッシュヒット: %{ve}%, キャッシュヒット: %{ch}%)\"," +
- "\"Created %{pf} ago, creation time: %{ct}ms \":" +
- "\"%{pf}間前に作成されました, 作成にかかった時間: %{ct}ms\"" +
+ "(vhost etag hits: %{ve}%, cache hits: %{ch}%)\": " +
+ "\"%{pf}間前に取得されたページ, 作成にかかった時間: %{ct}ms " +
+ "(vhost etag キャッシュヒット: %{ve}%, キャッシュヒット: %{ch}%)\"," +
+ "\"Created %{pf} ago, creation time: %{ct}ms \":\"" +
+ "%{pf}間前に作成されました, 作成にかかった時間: %{ct}ms\"" +
"}}";
var lang_zht = "{" +
@@ -341,11 +341,11 @@ var lang_zht = "{" +
"\"Committer\": \"提交者\"," +
"\"Raw Patch\": \"原始補丁\"," +
"\"Page fetched %{pf} ago, creation time: %{ct}ms " +
- "(vhost etag hits: %{ve}%, cache hits: %{ch}%)\": " +
- "\"頁面%{pf}前獲取, 創作時間: %{ct}ms " +
- "(vhost etag 緩存命中: %{ve}%, 緩存命中: %{ch}%)\"," +
- "\"Created %{pf} ago, creation time: %{ct}ms \":" +
- "\"%{pf}前創建, 創作時間: %{ct}ms \"" +
+ "(vhost etag hits: %{ve}%, cache hits: %{ch}%)\": " +
+ "\"頁面%{pf}前獲取, 創作時間: %{ct}ms " +
+ "(vhost etag 緩存命中: %{ve}%, 緩存命中: %{ch}%)\"," +
+ "\"Created %{pf} ago, creation time: %{ct}ms \":\"" +
+ "%{pf}前創建, 創作時間: %{ct}ms \"" +
"}}";
var lang_zhs = "{" +
@@ -396,7 +396,7 @@ var lang_zhs = "{" +
var logs = "", redpend = 0, gitohashi_integ = 0, authd = 0, exptimer, auth_user = "",
logAnsiState = {},
- ongoing_task_activities = {}, last_log_timestamp = 0;
+ ongoing_task_activities = {}, last_log_timestamp = 0, spreadsheet_data_cache = {}, loadreport_data_cache = {};
function update_task_activities() {
for (const uuid in ongoing_task_activities) {
@@ -508,6 +508,33 @@ function hsanitize(s)
}).replace(/\n/g, '\n');
}
+function renderSpreadsheet(tasks) {
+ if (!tasks || tasks.length === 0) {
+ return '';
+ }
+
+ tasks.sort((a, b) => b.started - a.started || a.task_name.localeCompare(b.task_name));
+
+ var now_ut = Math.round((new Date().getTime() / 1000));
+ let html = '<table class="spreadsheet">' +
+ '<thead><tr>' +
+ '<th>Task</th>' +
+ '<th>Build Step</th>' +
+ '<th>Started</th>' +
+ '</tr></thead><tbody>';
+
+ for (const task of tasks) {
+ html += '<tr>' +
+ `<td><a href="?task=${hsanitize(task.task_uuid)}">${hsanitize(task.task_name)}</a></td>` +
+ `<td>${hsanitize(task.build_step)} / ${hsanitize(task.total_steps)}</td>` +
+ `<td>${agify(now_ut, task.started)} ago</td>` +
+ '</tr>';
+ }
+
+ html += '</tbody></table>';
+ return html;
+}
+
var pos = 0, lli = 1, lines = "", times = "", locked = 1, tfirst = 0,
cont = [ 0, 0, 0, 0, 0];
@@ -988,7 +1015,6 @@ function createBuilderDiv(plat) {
platDiv.className = "ibuil bdr";
if (!plat.online)
platDiv.className += " offline";
- // Add a new class if the builder is powering up
if (plat.powering_up)
platDiv.className += " powering-up";
if (plat.powering_down)
@@ -1003,19 +1029,27 @@ function createBuilderDiv(plat) {
let plat_tc = plat_parts[2] || 'generic';
let innerHTML = `<table class="nomar"><tbody><tr><td class="bn">`;
- innerHTML += `<img class="ip1 zup" src="/sai/${plat_os}.svg" onerror="this.src='/sai/generic.svg';this.onerror=null;">`;
- innerHTML += `<img class="ip1 tread1" src="/sai/arch-${plat_arch}.svg" onerror="this.src='/sai/generic.svg';this.onerror=null;">`;
- innerHTML += `<img class="ip1 tread2" src="/sai/tc-${plat_tc}.svg" onerror="this.src='/sai/generic.svg';this.onerror=null;">`;
- innerHTML += `<br>${plat.peer_ip}`;
- innerHTML += `<div class="instload" id="instload-${plat.name}">` +
- `<div class="inst_box inst_idle" title="active steps: 0">` +
- `<div class="inst_text">0</div>` +
- `<div class="inst_bar"></div>` +
- `</div>` +
- `</div></td></tr></tbody></table>`;
+ innerHTML += `<img class="ip1 zup" data-sai-src="/sai/${plat_os}.svg">`;
+ innerHTML += `<img class="ip1 tread1" data-sai-src="/sai/arch-${plat_arch}.svg">`;
+ innerHTML += `<img class="ip1 tread2" data-sai-src="/sai/tc-${plat_tc}.svg">`;
+ innerHTML += `<div class="resource-bars">` +
+ `<div class="res-bar"><div class="res-bar-inner res-bar-cpu w-0"></div></div>` +
+ `<div class="res-bar"><div class="res-bar-inner res-bar-ram w-0"></div></div>` +
+ `<div class="res-bar"><div class="res-bar-inner res-bar-disk w-0"></div></div>` +
+ `</div>`;
+ innerHTML += `<br>${plat.peer_ip}</td></tr></tbody></table>`;
platDiv.innerHTML = innerHTML;
+ const images = platDiv.querySelectorAll('img[data-sai-src]');
+ images.forEach(img => {
+ img.onerror = () => {
+ img.src = '/sai/generic.svg';
+ img.onerror = null; // prevent infinite loops
+ };
+ img.src = img.getAttribute('data-sai-src');
+ });
+
const menuItems = [
{ label: `<b>SAI Hash:</b> ${plat.sai_hash}` },
{ label: `<b>LWS Hash:</b> ${plat.lws_hash}` },
@@ -1145,11 +1179,11 @@ function ws_open_sai()
console.log("tid " + tid);
sai.send("{\"schema\":" +
- "\"com.warmcat.sai.taskinfo\"," +
+ "\"com.warmcat.sai.taskinfo\"," +
"\"js_api_version\": " + SAI_JS_API_VERSION + "," +
- "\"logs\": 1," +
+ "\"logs\": 1," +
"\"last_log_ts\":" + last_log_timestamp + "," +
- "\"task_hash\":" +
+ "\"task_hash\":" +
JSON.stringify(tid) + "}");
return;
@@ -1165,7 +1199,7 @@ function ws_open_sai()
console.log("eid " + eid);
sai.send("{\"schema\":" +
- "\"com.warmcat.sai.eventinfo\"," +
+ "\"com.warmcat.sai.eventinfo\"," +
"\"js_api_version\": " + SAI_JS_API_VERSION + "," +
"\"event_hash\":" +
JSON.stringify(eid) + "}");
@@ -1178,7 +1212,7 @@ function ws_open_sai()
*/
sai.send("{\"schema\":" +
- "\"com.warmcat.sai.taskinfo\", \"js_api_version\": " + SAI_JS_API_VERSION + "}");
+ "\"com.warmcat.sai.taskinfo\", \"js_api_version\": " + SAI_JS_API_VERSION + "}");
};
sai.onmessage = function got_packet(msg) {
@@ -1252,9 +1286,10 @@ function ws_open_sai()
break;
}
+
// --- Reconciliation Logic ---
- buildersContainer.innerHTML = ""; // Clear for safety
+ buildersContainer.innerHTML = " "; // Clear for safety
const table = document.createElement("table");
table.className = "builders";
const tbody = document.createElement("tbody");
@@ -1279,6 +1314,21 @@ function ws_open_sai()
const tdSpreadsheet = document.createElement("td");
tdSpreadsheet.className = "spreadsheet-container";
tdSpreadsheet.id = "spreadsheet-" + plat.name;
+
+ let best_match_key = null;
+ for (const short_name in spreadsheet_data_cache) {
+ if (plat.name.startsWith(short_name)) {
+ if (!best_match_key || short_name.length > best_match_key.length) {
+ best_match_key = short_name;
+ }
+ }
+ }
+
+ if (best_match_key) {
+ tdSpreadsheet.innerHTML = renderSpreadsheet(spreadsheet_data_cache[best_match_key]);
+ aging();
+ }
+
tr.appendChild(tdSpreadsheet);
tbody.appendChild(tr);
@@ -1413,8 +1463,8 @@ function ws_open_sai()
addEventListener("click", function(e) {
console.log(e);
var rs= "{\"schema\":" +
- "\"com.warmcat.sai.eventreset\"," +
- "\"uuid\": " +
+ "\"com.warmcat.sai.eventreset\"," +
+ "\"uuid\": " +
JSON.stringify(san(e.srcElement.id.substring(11))) + "}";
console.log(rs);
@@ -1425,8 +1475,8 @@ function ws_open_sai()
addEventListener("click", function(e) {
console.log(e);
var rs= "{\"schema\":" +
- "\"com.warmcat.sai.eventdelete\"," +
- "\"uuid\": " +
+ "\"com.warmcat.sai.eventdelete\"," +
+ "\"uuid\": " +
JSON.stringify(san(e.srcElement.id.substring(10))) + "}";
console.log(rs);
@@ -1543,8 +1593,8 @@ function ws_open_sai()
document.getElementById("rebuild-" + san(jso.t.uuid)).
addEventListener("click", function(e) {
var rs= "{\"schema\":" +
- "\"com.warmcat.sai.taskreset\"," +
- "\"uuid\": " +
+ "\"com.warmcat.sai.taskreset\"," +
+ "\"uuid\": " +
JSON.stringify(san(e.srcElement.id.substring(8))) + "}";
console.log(rs);
@@ -1581,8 +1631,8 @@ function ws_open_sai()
document.getElementById("stop-" + san(jso.t.uuid)).
addEventListener("click", function(e) {
var rs= "{\"schema\":" +
- "\"com.warmcat.sai.taskcan\"," +
- "\"task_uuid\": " +
+ "\"com.warmcat.sai.taskcan\"," +
+ "\"task_uuid\": " +
JSON.stringify(san(e.srcElement.id.substring(5))) + "}";
console.log(rs);
sai.send(rs);
@@ -1593,68 +1643,64 @@ function ws_open_sai()
}
break;
- case "com.warmcat.sai.loadreport":
- const loadContainer = document.getElementById("instload-" + jso.builder_name);
- if (!loadContainer) {
- break;
- }
+ case "com.warmcat.sai.loadreport":
+ // Cache the whole report for subsequent builder redraws
+ loadreport_data_cache[jso.builder_name] = jso;
- const instanceDiv = loadContainer.querySelector(".inst_box");
- if (!instanceDiv) {
- break;
- }
+ const builderDiv = document.getElementById('binfo-' + jso.builder_name);
+ if (builderDiv) {
+ const cpuBar = builderDiv.querySelector(".res-bar-cpu");
+ const ramBar = builderDiv.querySelector(".res-bar-ram");
+ const diskBar = builderDiv.querySelector(".res-bar-disk");
- const textDiv = instanceDiv.querySelector(".inst_text");
- const barDiv = instanceDiv.querySelector(".inst_bar");
+ if (cpuBar) {
+ let cpu_percentage = (jso.cpu_percent / (jso.core_count * 1000)) * 100;
+ if (cpu_percentage > 100) cpu_percentage = 100;
+ if (cpu_percentage < 0) cpu_percentage = 0;
+ let width_class = `w-${Math.round(cpu_percentage / 5) * 5}`;
- if (textDiv) {
- textDiv.textContent = jso.active_steps;
- }
+ cpuBar.classList.forEach(c => { if (c.startsWith('w-')) cpuBar.classList.remove(c); });
+ cpuBar.classList.add(width_class);
+ }
+ if (ramBar) {
+ let ram_percentage = 0;
+ if (jso.initial_free_ram_kib > 0) {
+ ram_percentage = (jso.reserved_ram_kib / jso.initial_free_ram_kib) * 100;
+ }
+ if (ram_percentage > 100) ram_percentage = 100;
+ if (ram_percentage < 0) ram_percentage = 0;
+ let width_class = `w-${Math.round(ram_percentage / 5) * 5}`;
- instanceDiv.title = `Active steps: ${jso.active_steps}\n` +
- `CPU: ${(jso.cpu_percent / 10).toFixed(1)}%\n` +
- `Free RAM: ${humanize(jso.free_ram_kib * 1024)}B\n` +
- `Free Disk: ${humanize(jso.free_disk_kib * 1024)}B`;
-
- if (jso.active_steps > 0) {
- instanceDiv.classList.add("inst_busy");
- instanceDiv.classList.remove("inst_idle");
- } else {
- instanceDiv.classList.add("inst_idle");
- instanceDiv.classList.remove("inst_busy");
- }
+ ramBar.classList.forEach(c => { if (c.startsWith('w-')) ramBar.classList.remove(c); });
+ ramBar.classList.add(width_class);
+ }
+ if (diskBar) {
+ let disk_percentage = 0;
+ if (jso.initial_free_disk_kib > 0) {
+ disk_percentage = (jso.reserved_disk_kib / jso.initial_free_disk_kib) * 100;
+ }
+ if (disk_percentage > 100) disk_percentage = 100;
+ if (disk_percentage < 0) disk_percentage = 0;
+ let width_class = `w-${Math.round(disk_percentage / 5) * 5}`;
- if (barDiv) {
- let cpu_percentage = (jso.cpu_percent / (jso.core_count * 1000)) * 100;
- if (cpu_percentage > 100) cpu_percentage = 100;
- if (cpu_percentage < 0) cpu_percentage = 0;
- barDiv.style.height = `${cpu_percentage}%`;
- }
+ diskBar.classList.forEach(c => { if (c.startsWith('w-')) diskBar.classList.remove(c); });
+ diskBar.classList.add(width_class);
+ }
+ }
- const spreadsheetContainer = document.getElementById("spreadsheet-" + jso.builder_name);
- if (spreadsheetContainer) {
- let table = '<table><thead><tr>' +
- '<th>Task</th><th>Step</th><th>Started</th>' +
- '<th>Mem (KiB)</th><th>CPU (%)</th><th>Disk (KiB)</th>' +
- '</tr></thead><tbody>';
-
- if (jso.active_tasks) {
- jso.active_tasks.forEach(task => {
- table += `<tr>` +
- `<td>${hsanitize(task.task_name)}</td>` +
- `<td>${hsanitize(task.build_step)}</td>` +
- `<td>${agify(now_ut, task.started)}</td>` +
- `<td>${humanize(task.est_peak_mem_kib * 1024)}</td>` +
- `<td>${task.est_cpu_load_pct}</td>` +
- `<td>${humanize(task.est_disk_kib * 1024)}</td>` +
- `</tr>`;
- });
- }
+ // Part 2: Update the spreadsheet of active tasks for the builder
+ if (jso.active_tasks && jso.active_tasks.length > 0)
+ spreadsheet_data_cache[jso.builder_name] = jso.active_tasks;
+ else
+ delete spreadsheet_data_cache[jso.builder_name];
- table += '</tbody></table>';
- spreadsheetContainer.innerHTML = table;
- }
- break;
+ const spreadsheetContainer = document.getElementById('spreadsheet-' + jso.builder_name);
+ if (spreadsheetContainer) {
+ spreadsheetContainer.innerHTML = renderSpreadsheet(spreadsheet_data_cache[jso.builder_name]);
+ if (spreadsheet_data_cache[jso.builder_name])
+ aging();
+ }
+ break;
case "com-warmcat-sai-artifact":
console.log(jso);
@@ -1678,7 +1724,8 @@ function ws_open_sai()
var act = jso.activity[i];
ongoing_task_activities[act.uuid] = act.cat;
}
- }
+ } else
+ console.log("no spreadsheetContainer");
break;
case "com.warmcat.sai.unauthorized":
diff --git a/src/builder/b-comms.c b/src/builder/b-comms.c
index e8f8229..cd3542a 100644
--- a/src/builder/b-comms.c
+++ b/src/builder/b-comms.c
@@ -506,60 +506,122 @@ cleanup_on_ss_disconnect(struct lws_dll2 *d, void *user)
void
saib_sul_load_report_cb(struct lws_sorted_usec_list *sul)
{
- struct sai_plat_server *spm = lws_container_of(sul,
- struct sai_plat_server, sul_load_report);
- sai_load_report_t *lr = calloc(1, sizeof(*lr));
- struct sai_plat *sp = NULL;
- char somebody_not_idle = 0;
-
- if (!lr)
- return;
-
- /*
- * This builder process has one name, but may have multiple platforms,
- * each with multiple instances. For now, we report on the whole builder
- * under one name.
- */
- lws_strncpy(lr->builder_name, builder.host, sizeof(lr->builder_name));
- lr->core_count = saib_get_cpu_count();
- lr->free_ram_kib = saib_get_free_ram_kib();
- lr->free_disk_kib = saib_get_free_disk_kib(builder.home);
- lr->cpu_percent = (unsigned int)saib_get_system_cpu(&builder);
- lr->active_steps = 0;
- lws_dll2_owner_clear(&lr->active_tasks);
-
- lws_start_foreach_dll(struct lws_dll2 *, p, builder.sai_plat_owner.head) {
- sp = lws_container_of(p, sai_plat_t, sai_plat_list);
- lws_start_foreach_dll(struct lws_dll2 *, d, sp->nspawn_owner.head) {
- struct sai_nspawn *ns = lws_container_of(d,
- struct sai_nspawn, list);
- if (ns->state == NSSTATE_EXECUTING_STEPS && ns->task) {
- sai_active_task_info_t *ati = malloc(sizeof(*ati));
- if (ati) {
- memset(ati, 0, sizeof(*ati));
- lws_strncpy(ati->task_uuid, ns->task->uuid, sizeof(ati->task_uuid));
- lws_strncpy(ati->task_name, ns->task->taskname, sizeof(ati->task_name));
- ati->build_step = ns->current_step;
- ati->est_peak_mem_kib = ns->task->est_peak_mem_kib;
- ati->est_cpu_load_pct = ns->task->est_cpu_load_pct;
- ati->est_disk_kib = ns->task->est_disk_kib;
- ati->started = ns->task->started;
- lws_dll2_add_tail(&ati->list, &lr->active_tasks);
- lr->active_steps++;
- }
- somebody_not_idle = 1;
- }
- } lws_end_foreach_dll(d);
- } lws_end_foreach_dll(p);
-
- lws_dll2_add_tail(&lr->list, &spm->load_report_owner);
- if (lws_ss_request_tx(spm->ss))
- lwsl_debug("%s: request tx failed\n", __func__);
-
- if (somebody_not_idle)
+ struct sai_plat_server *spm = lws_container_of(sul,
+ struct sai_plat_server, sul_load_report);
+ char any_platform_on_this_spm_active = 0;
+
+ /*
+ * This builder process may have multiple platforms, each with
+ * multiple instances. We report on each platform separately so the
+ * UI can distinguish them.
+ *
+ * This SUL is per-server-connection. We iterate all platforms and
+ * for each, see if it's supposed to connect to this server.
+ *
+ * To avoid spamming idle reports, we only report if the platform
+ * is active for this server, OR if it was active the last time we
+ * checked (ie, it has just become idle, so we need to send one last
+ * report with no active tasks to clear the UI).
+ */
+
+ lws_start_foreach_dll(struct lws_dll2 *, p, builder.sai_plat_owner.head) {
+ struct sai_plat *sp = lws_container_of(p, sai_plat_t, sai_plat_list);
+ sai_plat_server_ref_t *ref = NULL;
+ sai_load_report_t *lr;
+ char is_active = 0;
+
+ /*
+ * Find the specific ref for this platform and this server
+ * connection (spm)
+ */
+ lws_start_foreach_dll(struct lws_dll2 *, s, sp->servers.head) {
+ sai_plat_server_ref_t *r = lws_container_of(s,
+ sai_plat_server_ref_t, list);
+ if (r->spm == spm) {
+ ref = r;
+ break;
+ }
+ } lws_end_foreach_dll(s);
+
+ if (!ref) /* This platform doesn't use this server connection */
+ continue;
+
+ /*
+ * Check for active tasks on this platform for this server conn
+ */
+ lws_start_foreach_dll(struct lws_dll2 *, d, sp->nspawn_owner.head) {
+ struct sai_nspawn *ns = lws_container_of(d,
+ struct sai_nspawn, list);
+ if (ns->spm == spm &&
+ ns->state == NSSTATE_EXECUTING_STEPS && ns->task) {
+ is_active = 1;
+ break;
+ }
+ } lws_end_foreach_dll(d);
+
+ if (is_active)
+ any_platform_on_this_spm_active = 1;
+
+ if (!is_active && !ref->was_active)
+ goto around;
+
+ /* This platform is active for this spm, or just became idle */
+
+ lr = calloc(1, sizeof(*lr));
+ if (!lr)
+ goto around;
+
+ lws_strncpy(lr->builder_name, sp->name, sizeof(lr->builder_name));
+ lr->core_count = saib_get_cpu_count();
+ lr->initial_free_ram_kib = saib_get_total_ram_kib();
+ lr->initial_free_disk_kib = saib_get_total_disk_kib(builder.home);
+ lr->reserved_ram_kib = 0;
+ lr->reserved_disk_kib = 0;
+ lr->cpu_percent = (unsigned int)saib_get_system_cpu(&builder);
+ lr->active_steps = 0;
+ lws_dll2_owner_clear(&lr->active_tasks);
+
+ if (is_active) {
+ lws_start_foreach_dll(struct lws_dll2 *, d, sp->nspawn_owner.head) {
+ struct sai_nspawn *ns = lws_container_of(d,
+ struct sai_nspawn, list);
+ if (ns->spm == spm &&
+ ns->state == NSSTATE_EXECUTING_STEPS && ns->task) {
+ sai_active_task_info_t *ati = malloc(sizeof(*ati));
+ if (ati) {
+ memset(ati, 0, sizeof(*ati));
+ lws_strncpy(ati->task_uuid, ns->task->uuid, sizeof(ati->task_uuid));
+ lws_strncpy(ati->task_name, ns->task->taskname, sizeof(ati->task_name));
+ ati->build_step = ns->current_step;
+ ati->total_steps = ns->build_step_count;
+ ati->est_peak_mem_kib = ns->task->est_peak_mem_kib;
+ ati->est_cpu_load_pct = ns->task->est_cpu_load_pct;
+ ati->est_disk_kib = ns->task->est_disk_kib;
+ ati->started = ns->task->started;
+ lws_dll2_add_tail(&ati->list, &lr->active_tasks);
+ lr->active_steps++;
+
+ lr->reserved_ram_kib += ns->task->est_peak_mem_kib;
+ lr->reserved_disk_kib += ns->task->est_disk_kib;
+ }
+ }
+ } lws_end_foreach_dll(d);
+ }
+
+ lws_dll2_add_tail(&lr->list, &spm->load_report_owner);
+ if (lws_ss_request_tx(spm->ss))
+ lwsl_debug("%s: request tx failed\n", __func__);
+
+ ref->was_active = is_active;
+
+around:
+ ;
+ } lws_end_foreach_dll(p);
+
+ if (any_platform_on_this_spm_active)
/* Reschedule the timer only if at least one active instance */
lws_sul_schedule(builder.context, 0, &spm->sul_load_report,
- saib_sul_load_report_cb, SAI_LOAD_REPORT_US);
+ saib_sul_load_report_cb, SAI_LOAD_REPORT_US);
}
static lws_ss_state_return_t
diff --git a/src/builder/b-deletion.c b/src/builder/b-deletion.c
index 5c26883..0690e4e 100644
--- a/src/builder/b-deletion.c
+++ b/src/builder/b-deletion.c
@@ -50,9 +50,6 @@
#if !defined(PATH_MAX)
#define PATH_MAX MAX_PATH
#endif
-
-int getpid(void) { return 0; }
-
#endif
#include "b-private.h"
diff --git a/src/builder/b-load.c b/src/builder/b-load.c
index bdf3a4e..c2325a5 100644
--- a/src/builder/b-load.c
+++ b/src/builder/b-load.c
@@ -211,7 +211,7 @@ int saib_get_system_cpu(struct sai_builder *b)
if (!GetSystemTimes((FILETIME *)&idle, (FILETIME *)&kernel, (FILETIME *)&user))
return -1;
- if (b->last_sys_total.QuadPart) {
+ if (b->last_sys_kernel.QuadPart || b->last_sys_user.QuadPart) {
ULONGLONG total_delta, idle_delta;
total_delta = (kernel.QuadPart - b->last_sys_kernel.QuadPart) +
@@ -220,8 +220,10 @@ int saib_get_system_cpu(struct sai_builder *b)
if (total_delta) {
n = (int)(((total_delta - idle_delta) * 1000) / total_delta);
- if (n > 1000)
- n = 1000;
+// if (n > 1000)
+// n = 1000;
+ if (n < 0)
+ n = 0;
ret = n;
}
}
diff --git a/src/builder/b-power.c b/src/builder/b-power.c
index a174e30..4d13efa 100644
--- a/src/builder/b-power.c
+++ b/src/builder/b-power.c
@@ -50,9 +50,6 @@
#if !defined(PATH_MAX)
#define PATH_MAX MAX_PATH
#endif
-
-int getpid(void) { return 0; }
-
#endif
#include "b-private.h"
diff --git a/src/builder/b-suspender.c b/src/builder/b-suspender.c
index 051df3c..eb9dce7 100644
--- a/src/builder/b-suspender.c
+++ b/src/builder/b-suspender.c
@@ -49,8 +49,6 @@
#if !defined(PATH_MAX)
#define PATH_MAX MAX_PATH
#endif
-
-int getpid(void) { return 0; }
#endif
#include "b-private.h"
diff --git a/src/builder/b-task.c b/src/builder/b-task.c
index 150c2c4..813a50d 100644
--- a/src/builder/b-task.c
+++ b/src/builder/b-task.c
@@ -255,6 +255,12 @@ saib_set_ns_state(struct sai_nspawn *ns, int state)
saib_log_chunk_create(ns, log, (unsigned int)n, 3);
+ if (state == NSSTATE_EXECUTING_STEPS && ns->spm &&
+ !ns->spm->sul_load_report.list.owner)
+ lws_sul_schedule(ns->builder->context, 0,
+ &ns->spm->sul_load_report,
+ saib_sul_load_report_cb, 1);
+
if (state == NSSTATE_UPLOADING_ARTIFACTS)
saib_start_artifact_upload(ns);
@@ -916,6 +922,7 @@ saib_ws_json_rx_builder(struct sai_plat_server *spm, const void *in, size_t len)
ns->task = task; /* we are owning this nspawn for the duration */
ns->current_step = task->build_step;
+ ns->build_step_count = task->build_step_count;
if (!ns->current_step) {
ns->spins = 0;
ns->user_cancel = 0;
@@ -1051,11 +1058,6 @@ saib_ws_json_rx_builder(struct sai_plat_server *spm, const void *in, size_t len)
lwsl_notice("%s: cancelling suspend grace time\n", __func__);
lws_sul_cancel(&ns->builder->sul_idle);
- /* if we weren't, we should report load on instances now we're busy */
- if (lws_dll2_is_detached(&spm->sul_load_report.list))
- lws_sul_schedule(ns->builder->context, 0, &spm->sul_load_report,
- saib_sul_load_report_cb, 1);
-
/*
* Let the mirror thread get on with things...
*
diff --git a/src/common/include/private.h b/src/common/include/private.h
index ee63fe2..9177117 100644
--- a/src/common/include/private.h
+++ b/src/common/include/private.h
@@ -68,6 +68,7 @@ typedef struct sai_active_task_info {
char task_uuid[65];
char task_name[96];
int build_step;
+ int total_steps;
unsigned int est_peak_mem_kib;
unsigned int est_cpu_load_pct;
unsigned int est_disk_kib;
@@ -78,8 +79,10 @@ typedef struct sai_load_report {
lws_dll2_t list; /* For queuing on sai_plat_server */
char builder_name[64];
int core_count;
- unsigned int free_ram_kib;
- unsigned int free_disk_kib;
+ unsigned int initial_free_ram_kib;
+ unsigned int reserved_ram_kib;
+ unsigned int initial_free_disk_kib;
+ unsigned int reserved_disk_kib;
unsigned int active_steps;
unsigned int cpu_percent;
lws_dll2_owner_t active_tasks;
@@ -136,6 +139,7 @@ typedef struct {
int state;
int uid;
int build_step;
+ int build_step_count;
/* estimations for builder resource consumption */
unsigned int est_peak_mem_kib;
@@ -413,6 +417,7 @@ struct sai_env {
typedef struct sai_plat_server_ref {
lws_dll2_t list;
sai_plat_server_t *spm;
+ char was_active;
} sai_plat_server_ref_t;
/*
@@ -538,7 +543,7 @@ extern const lws_struct_map_t
lsm_schema_map_ta[1],
lsm_schema_map_plat_simple[1],
lsm_event[10],
- lsm_task[28],
+ lsm_task[29],
lsm_log[7],
lsm_artifact[8],
lsm_plat_list[1],
@@ -553,7 +558,7 @@ extern const lws_struct_map_t
lsm_schema_rebuild[1],
lsm_schema_build_metric[1],
lsm_schema_sq3_map_build_metric[1],
- lsm_load_report_members[7]
+ lsm_load_report_members[9]
;
extern const lws_struct_map_t lsm_build_metric[12];
extern const lws_struct_map_t lsm_plat[8];
diff --git a/src/common/struct-metadata.c b/src/common/struct-metadata.c
index dbc8a9f..2ffe9b3 100644
--- a/src/common/struct-metadata.c
+++ b/src/common/struct-metadata.c
@@ -25,6 +25,7 @@ const lws_struct_map_t lsm_active_task_info[] = {
LSM_CARRAY (sai_active_task_info_t, task_uuid, "task_uuid"),
LSM_CARRAY (sai_active_task_info_t, task_name, "task_name"),
LSM_SIGNED (sai_active_task_info_t, build_step, "build_step"),
+ LSM_SIGNED (sai_active_task_info_t, total_steps, "total_steps"),
LSM_UNSIGNED (sai_active_task_info_t, est_peak_mem_kib, "est_peak_mem_kib"),
LSM_UNSIGNED (sai_active_task_info_t, est_cpu_load_pct, "est_cpu_load_pct"),
LSM_UNSIGNED (sai_active_task_info_t, est_disk_kib, "est_disk_kib"),
@@ -34,8 +35,10 @@ const lws_struct_map_t lsm_active_task_info[] = {
const lws_struct_map_t lsm_load_report_members[] = {
LSM_CARRAY (sai_load_report_t, builder_name, "builder_name"),
LSM_SIGNED (sai_load_report_t, core_count, "core_count"),
- LSM_UNSIGNED (sai_load_report_t, free_ram_kib, "free_ram_kib"),
- LSM_UNSIGNED (sai_load_report_t, free_disk_kib, "free_disk_kib"),
+ LSM_UNSIGNED (sai_load_report_t, initial_free_ram_kib, "initial_free_ram_kib"),
+ LSM_UNSIGNED (sai_load_report_t, reserved_ram_kib, "reserved_ram_kib"),
+ LSM_UNSIGNED (sai_load_report_t, initial_free_disk_kib, "initial_free_disk_kib"),
+ LSM_UNSIGNED (sai_load_report_t, reserved_disk_kib, "reserved_disk_kib"),
LSM_UNSIGNED (sai_load_report_t, active_steps, "active_steps"),
LSM_UNSIGNED (sai_load_report_t, cpu_percent, "cpu_percent"),
LSM_LIST (sai_load_report_t, active_tasks, sai_active_task_info_t, list,
@@ -178,6 +181,7 @@ const lws_struct_map_t lsm_task[] = {
LSM_STRING_PTR (sai_task_t, git_repo_url, "git_repo_url"),
LSM_CARRAY (sai_task_t, script, "script"),
LSM_SIGNED (sai_task_t, build_step, "build_step"),
+ LSM_SIGNED (sai_task_t, build_step_count, "build_step_count"),
LSM_UNSIGNED (sai_task_t, est_peak_mem_kib, "est_peak_mem_kib"),
LSM_UNSIGNED (sai_task_t, est_cpu_load_pct, "est_cpu_load_pct"),
LSM_UNSIGNED (sai_task_t, est_disk_kib, "est_disk_kib"),
diff --git a/src/server/s-notification.c b/src/server/s-notification.c
index cbe44b3..29df3cf 100644
--- a/src/server/s-notification.c
+++ b/src/server/s-notification.c
@@ -481,6 +481,9 @@ sai_saifile_lejp_cb(struct lejp_ctx *ctx, char reason)
}
if (match) {
+ const char *p;
+ int c;
+
/*
* For this platform, we want to create a task
* associated with this event. Tasks and logs
@@ -488,6 +491,14 @@ sai_saifile_lejp_cb(struct lejp_ctx *ctx, char reason)
* specific database file for scalability.
*/
+ c = 2; /* git mirror and checkout */
+ p = pl->build;
+ while (*p)
+ if (*p++ == '\n')
+ c++;
+
+ pss->sn.t.build_step_count = c;
+
lws_strexp_init(&sx, sn, exp_cmake, sn->t.build,
sizeof(sn->t.build));
@@ -1044,3 +1055,4 @@ saifile_bail:
return 0;
}
+
diff --git a/src/server/s-ws-builder.c b/src/server/s-ws-builder.c
index f5469a4..8d35e54 100644
--- a/src/server/s-ws-builder.c
+++ b/src/server/s-ws-builder.c
@@ -705,14 +705,14 @@ bail:
sai_load_report_t *lr = (sai_load_report_t *)pss->a.dest;
lwsl_notice("%s: @@@@@@@@@@@@@@@@@@ loadreport from %s: ram %uk, disk %uk\n",
- __func__, lr->builder_name, lr->free_ram_kib,
- lr->free_disk_kib);
+ __func__, lr->builder_name, lr->reserved_ram_kib,
+ lr->reserved_disk_kib);
ssize_t wr = write(2, buf, bl);
if (wr != (ssize_t)bl)
lwsl_notice("%s: write failed\n", __func__);
}
- // lwsl_wsi_user(pss->wsi, "SAIM_WSSCH_BUILDER_LOADREPORT broadcasting\n");
+ lwsl_wsi_user(pss->wsi, "SAIM_WSSCH_BUILDER_LOADREPORT broadcasting\n");
sais_websrv_broadcast(vhd->h_ss_websrv, (const char *)buf, bl);
break;
diff --git a/src/web/w-websrv.c b/src/web/w-websrv.c
index 0d32f3a..d83ee53 100644
--- a/src/web/w-websrv.c
+++ b/src/web/w-websrv.c
@@ -224,7 +224,8 @@ saiw_lp_rx(void *userobj, const uint8_t *buf, size_t len, int flags)
break;
case SAIS_WS_WEBSRV_RX_LOADREPORT:
- /* Forward the final fragment of the load report */
+ lwsl_notice("%s: ^^^^^^^^^^^^^^ SAIS_WS_WEBSRV_RX_LOADREPORT forwarding to browser\n", __func__);
+ lwsl_hexdump_notice(buf, len);
saiw_ws_broadcast_raw(vhd, buf, len - (unsigned int)n, 0,
lws_write_ws_flags(LWS_WRITE_TEXT, flags & LWSSS_FLAG_SOM, flags & LWSSS_FLAG_EOM));
break;
diff --git a/src/web/w-ws-browser.c b/src/web/w-ws-browser.c
index 1448ab2..28ea478 100644
--- a/src/web/w-ws-browser.c
+++ b/src/web/w-ws-browser.c
@@ -30,8 +30,6 @@
#include "w-private.h"
-extern const lws_struct_map_t lsm_load_report_members[7];
-
/*
* For decoding specific event data request from browser
*/
Log
Tree
Blame![[]](/git/avatar/c50933ca2aa61e0fe2c43d46bb6b59cb_avatar){kind=avatar}
Andy Green <andy@warmcat.com> 2025-09-24 03:28 UTC
Raw Patch