Project homepage Mailing List  Warmcat.com  API Docs  Github Mirror 
    npro  
 Modern all-safe Rust Network Protocol library supporting h1, h2, h3, ws, wt sans-IO and with socket IO + tls
git clone https://npro.rs/repo/npro
Fulltext search
 
root / assets / arch-x86-amd.svg
Author[]google-labs-jules[bot] <161369871+google-labs-jules[bot]@users.noreply.github.c...> 2025-10-25 16:46 UTC
Committer[]Andy Green <andy@warmcat.com> 2025-10-25 17:01 UTC
Tree6489aea40a3071908ca6932a2dca2ef6d6cd33f4   Raw Patch
 
Fix incorrect data in builder task reports 
Fix incorrect data in builder task reports

The `taskreport` messages sent by the builder contained several incorrect or missing values:
- `reserved_ram_kib` and `reserved_disk_kib` were always zero.
- `cpu_percent` was often reported as `255`.
- Per-task resource estimates were not included.

This was caused by a combination of issues:
1. The server was not fetching resource estimates from the `build_metrics` database when offering tasks to the builder.
2. The CPU load calculation functions in the builder returned -1 on error, which was then cast to a large unsigned integer.
3. The `est_cpu_load_pct` field was unused and unnecessary.

This commit addresses these issues by:
- Modifying the server to correctly fetch and send task resource estimates based on a SHA1 key.
- Updating the builder's CPU load calculation to return 0 on error, preventing incorrect values.
- Removing the unused `est_cpu_load_pct` field from the relevant data structures and serialization maps.

diff --git a/src/builder/b-comms.c b/src/builder/b-comms.c
index 371f863..cb71a99 100644
--- a/src/builder/b-comms.c
+++ b/src/builder/b-comms.c
@@ -330,7 +330,6 @@ saib_sul_load_report_cb(struct lws_sorted_usec_list *sul)
 						ati->build_step		= ns->current_step;
 						ati->total_steps	= ns->build_step_count;
 						ati->est_peak_mem_kib	= ns->task->est_peak_mem_kib;
-						ati->est_cpu_load_pct	= ns->task->est_cpu_load_pct;
 						ati->est_disk_kib	= ns->task->est_disk_kib;
 						ati->started		= ns->task->started;
 						lws_dll2_add_tail(&ati->list, &lr.active_tasks);
diff --git a/src/builder/b-load.c b/src/builder/b-load.c
index c2325a5..9df6d03 100644
--- a/src/builder/b-load.c
+++ b/src/builder/b-load.c
@@ -62,7 +62,7 @@ saib_get_cgroup_cpu(struct sai_nspawn *ns)
 	char path[256], buf[128];
 	lws_usec_t now;
 	FILE *f;
-	int n, ret = -1;
+	int n, ret = 0;
 
 	/*
 	 * On systemd systems, nspawn creates a scope unit for the container,
@@ -75,7 +75,7 @@ saib_get_cgroup_cpu(struct sai_nspawn *ns)
 
 	f = fopen(path, "r");
 	if (!f)
-		return -1; /* cgroup file not found, fall back to system load */
+		return 0; /* cgroup file not found, fall back to system load */
 
 	while (fgets(buf, sizeof(buf) - 1, f)) {
 		if (sscanf(buf, "usage_usec %llu",
@@ -86,7 +86,7 @@ saib_get_cgroup_cpu(struct sai_nspawn *ns)
 	fclose(f);
 
 	if (!usage_usec)
-		return -1;
+		return 0;
 
 	now = lws_now_usecs();
 
@@ -117,24 +117,24 @@ saib_get_system_cpu(struct sai_builder *b)
 {
 	unsigned long long user, nice, system, idle, iowait, irq, softirq, steal;
 	uint64_t total, idle_all, total_delta, idle_delta;
-	int n, ret = -1;
+	int n, ret = 0;
 	char buf[256];
 	FILE *f;
 
 	f = fopen("/proc/stat", "r");
 	if (!f)
-		return -1;
+		return 0;
 
 	if (!fgets(buf, sizeof(buf) -1, f)) {
 		fclose(f);
-		return -1;
+		return 0;
 	}
 	fclose(f);
 
 	n = sscanf(buf, "cpu  %llu %llu %llu %llu %llu %llu %llu %llu",
 		   &user, &nice, &system, &idle, &iowait, &irq, &softirq, &steal);
 	if (n < 4)
-		return -1;
+		return 0;
 
 	idle_all = idle + iowait;
 	total = user + nice + system + idle_all + irq + softirq + steal;
@@ -159,7 +159,7 @@ saib_get_system_cpu(struct sai_builder *b)
 #elif defined(__APPLE__)
 int saib_get_cgroup_cpu(struct sai_nspawn *ns)
 {
-	return -1; /* No cgroups on macOS */
+	return 0; /* No cgroups on macOS */
 }
 
 int saib_get_system_cpu(struct sai_builder *b)
@@ -168,11 +168,11 @@ int saib_get_system_cpu(struct sai_builder *b)
 	mach_msg_type_number_t count = HOST_CPU_LOAD_INFO_COUNT;
 	uint64_t total_ticks = 0, idle_ticks = 0;
 	uint64_t total_delta, idle_delta;
-	int n, ret = -1;
+	int n, ret = 0;
 
 	if (host_statistics(mach_host_self(), HOST_CPU_LOAD_INFO,
 			    (host_info_t)&cpuinfo, &count) != KERN_SUCCESS)
-		return -1;
+		return 0;
 
 	total_ticks = cpuinfo.cpu_ticks[CPU_STATE_USER] +
 		      cpuinfo.cpu_ticks[CPU_STATE_SYSTEM] +
@@ -200,16 +200,16 @@ int saib_get_system_cpu(struct sai_builder *b)
 #elif defined(WIN32)
 int saib_get_cgroup_cpu(struct sai_nspawn *ns)
 {
-	return -1; /* No cgroups on Windows */
+	return 0; /* No cgroups on Windows */
 }
 
 int saib_get_system_cpu(struct sai_builder *b)
 {
 	ULARGE_INTEGER idle, kernel, user;
-	int n, ret = -1;
+	int n, ret = 0;
 
 	if (!GetSystemTimes((FILETIME *)&idle, (FILETIME *)&kernel, (FILETIME *)&user))
-		return -1;
+		return 0;
 
 	if (b->last_sys_kernel.QuadPart || b->last_sys_user.QuadPart) {
 		ULONGLONG total_delta, idle_delta;
@@ -237,7 +237,7 @@ int saib_get_system_cpu(struct sai_builder *b)
 #else
 int saib_get_cgroup_cpu(struct sai_nspawn *ns)
 {
-	return -1; /* Not implemented on this platform */
+	return 0; /* Not implemented on this platform */
 }
 int saib_get_system_cpu(struct sai_builder *b)
 {
diff --git a/src/common/include/private.h b/src/common/include/private.h
index ca5f60c..5181b05 100644
--- a/src/common/include/private.h
+++ b/src/common/include/private.h
@@ -70,7 +70,6 @@ typedef struct sai_active_task_info {
 	int			build_step;
 	int			total_steps;
 	unsigned int		est_peak_mem_kib;
-	unsigned int		est_cpu_load_pct;
 	unsigned int		est_disk_kib;
 	uint64_t		started;
 } sai_active_task_info_t;
diff --git a/src/common/struct-metadata.c b/src/common/struct-metadata.c
index 9fb2d92..110bc1b 100644
--- a/src/common/struct-metadata.c
+++ b/src/common/struct-metadata.c
@@ -27,7 +27,6 @@ const lws_struct_map_t lsm_active_task_info[] = {
 	LSM_SIGNED	(sai_active_task_info_t, build_step, "build_step"),
 	LSM_SIGNED	(sai_active_task_info_t, total_steps, "total_steps"),
 	LSM_UNSIGNED	(sai_active_task_info_t, est_peak_mem_kib, "est_peak_mem_kib"),
-	LSM_UNSIGNED	(sai_active_task_info_t, est_cpu_load_pct, "est_cpu_load_pct"),
 	LSM_UNSIGNED	(sai_active_task_info_t, est_disk_kib, "est_disk_kib"),
 	LSM_UNSIGNED	(sai_active_task_info_t, started, "started"),
 };
diff --git a/src/server/s-task-helpers.c b/src/server/s-task-helpers.c
index 553d08a..a4776ea 100644
--- a/src/server/s-task-helpers.c
+++ b/src/server/s-task-helpers.c
@@ -27,11 +27,14 @@
 
 #include "s-private.h"
 
+
 void
 sais_get_task_metrics_estimates(struct vhd *vhd, sai_task_t *task)
 {
-	char query[256];
+        struct lws_genhash_ctx ctx;
+	char query[256], hex[65];
 	sqlite3_stmt *stmt;
+        uint8_t hash[32];
 
 	task->est_peak_mem_kib = 256 * 1024; /* 256MiB default */
 	task->est_cpu_load_pct = 10;
@@ -40,11 +43,24 @@ sais_get_task_metrics_estimates(struct vhd *vhd, sai_task_t *task)
 	if (!vhd->pdb_metrics)
 		return;
 
+	if (!task->repo_name || !task->platform[0])
+		return;
+
+        if (lws_genhash_init(&ctx, LWS_GENHASH_TYPE_SHA256) ||
+            lws_genhash_update(&ctx, (uint8_t *)task->repo_name, strlen(task->repo_name)) ||
+            lws_genhash_update(&ctx, (uint8_t *)task->platform, strlen(task->platform) ||
+            lws_genhash_update(&ctx, (uint8_t *)task->taskname, strlen(task->taskname)) ||
+            lws_genhash_destroy(&ctx, hash)))
+                lwsl_warn("%s: sha256 failed\n", __func__);
+
+        lws_hex_from_byte_array(hash, sizeof(hash) - 1, hex, sizeof(hex));
+	hex[64] = '\0';
+
 	lws_snprintf(query, sizeof(query),
 		     "SELECT AVG(peak_mem_rss), AVG(us_cpu_user), "
 		     "AVG(stg_bytes), AVG(wallclock_us) "
 		     "FROM build_metrics WHERE key = '%s'",
-		     task->taskname);
+		     hex);
 
 	if (sqlite3_prepare_v2(vhd->pdb_metrics, query, -1, &stmt, NULL) != SQLITE_OK)
 		return;
@@ -53,10 +69,12 @@ sais_get_task_metrics_estimates(struct vhd *vhd, sai_task_t *task)
 		uint64_t avg_us_cpu = (uint64_t)sqlite3_column_int64(stmt, 1);
 		uint64_t avg_wallclock = (uint64_t)sqlite3_column_int64(stmt, 3);
 
-		task->est_peak_mem_kib = (unsigned int)(sqlite3_column_int(stmt, 0) / 1024);
+		if (sqlite3_column_type(stmt, 0) != SQLITE_NULL)
+			task->est_peak_mem_kib = (unsigned int)(sqlite3_column_int(stmt, 0) / 1024);
 		if (avg_wallclock)
 			task->est_cpu_load_pct = (unsigned int)((avg_us_cpu * 100) / avg_wallclock);
-		task->est_disk_kib = (unsigned int)(sqlite3_column_int(stmt, 2) / 1024);
+		if (sqlite3_column_type(stmt, 2) != SQLITE_NULL)
+			task->est_disk_kib = (unsigned int)(sqlite3_column_int(stmt, 2) / 1024);
 	}
 
 	sqlite3_finalize(stmt);
@@ -323,4 +341,4 @@ sais_task_rebuild_last_step(struct vhd *vhd, const char *task_uuid)
 	lwsl_notice("%s: exiting OK\n", __func__);
 
 	return SAI_DB_RESULT_OK;
-}
\ No newline at end of file
+}
Page fetched 0s ago, creation time: 4ms (vhost etag hits: 0%, cache hits: 0%)