Author: Andy Green Date: Mon Aug 25 14:52:08 2025 +0100 feat: Add build step resource usage metrics Co-developed-by: Gemini 2.5 Pro diff --git a/src/builder/b-comms.c b/src/builder/b-comms.c index 12c1af2..c027424 100644 --- a/src/builder/b-comms.c +++ b/src/builder/b-comms.c @@ -72,6 +72,39 @@ saib_m_tx(void *userobj, lws_ss_tx_ordinal_t ord, uint8_t *buf, size_t *len, int n = 0; /* + * Any build metrics to process? + */ + + if (spm->build_metric_list.count) { + struct lws_dll2 *d = lws_dll2_get_head(&spm->build_metric_list); + sai_build_metric_t *m = + lws_container_of(d, sai_build_metric_t, list); + + lwsl_notice("%s: issuing build metric\n", __func__); + + js = lws_struct_json_serialize_create(lsm_schema_build_metric, + LWS_ARRAY_SIZE(lsm_schema_build_metric), 0, m); + if (!js) + return -1; + + n = (int)lws_struct_json_serialize(js, start, + lws_ptr_diff_size_t(end, start), &w); + lws_struct_json_serialize_destroy(&js); + + lwsl_hexdump_notice(start, w); + + n = (int)w; + + lws_dll2_remove(&m->list); + free(m); + + r = lws_ss_request_tx(spm->ss); + if (r) + return r; + goto sendify; + } + + /* * Any builder state updates / rejections to process? */ diff --git a/src/builder/b-nspawn.c b/src/builder/b-nspawn.c index 3b4d8d9..d262b24 100644 --- a/src/builder/b-nspawn.c +++ b/src/builder/b-nspawn.c @@ -244,11 +244,54 @@ sai_lsp_reap_cb(void *opaque, const lws_spawn_resource_us_t *res, siginfo_t *si, saib_log_chunk_create(ns, s, (size_t)n, 3); } + if (op->spawn) { + sai_build_metric_t *m = malloc(sizeof(*m)); + + if (m) { + char hash_input[8192]; + unsigned char hash[32]; + struct lws_genhash_ctx ctx; + int n; + + memset(m, 0, sizeof(*m)); + + lws_snprintf(hash_input, sizeof(hash_input), "%s%s%s%s", + ns->sp->name, op->spawn, + ns->project_name, ns->ref); + + if (lws_genhash_init(&ctx, LWS_GENHASH_TYPE_SHA256) || + lws_genhash_update(&ctx, hash_input, + strlen(hash_input)) || + lws_genhash_destroy(&ctx, hash)) + lwsl_warn("%s: sha256 failed\n", __func__); + else + for (n = 0; n < 32; n++) + lws_snprintf(m->key + (n * 2), 3, + "%02x", hash[n]); + + lws_strncpy(m->builder_name, ns->sp->name, sizeof(m->builder_name)); + lws_strncpy(m->project_name, ns->project_name, sizeof(m->project_name)); + lws_strncpy(m->ref, ns->ref, sizeof(m->ref)); + m->parallel = ns->parallel; + m->us_cpu_user = res->us_cpu_user; + m->us_cpu_sys = res->us_cpu_sys; + m->wallclock_us = (uint64_t)(lws_now_usecs() - op->start_time); + m->peak_mem_rss = res->peak_mem_rss; + m->stg_bytes = du.size_in_bytes; + + lws_dll2_add_tail(&m->list, &ns->spm->build_metric_list); + if (lws_ss_request_tx(ns->spm->ss)) + lwsl_warn("%s: lws_ss_request_tx failed\n", __func__); + } + } + ns->build_step++; if (ns->build_step < ns->build_step_count) { /* there are more steps, spawn the next one */ if (ns) ns->op = NULL; + if (op->spawn) + free(op->spawn); free(op); saib_spawn_step(ns); return; @@ -257,6 +300,9 @@ sai_lsp_reap_cb(void *opaque, const lws_spawn_resource_us_t *res, siginfo_t *si, /* all steps succeeded */ lwsl_notice("%s: all build steps succeeded\n", __func__); + if (op->spawn) + free(op->spawn); + saib_task_grace(ns); saib_set_ns_state(ns, NSSTATE_DONE); @@ -286,6 +332,9 @@ fail: saib_log_chunk_create(ns, NULL, 0, 2); + if (op->spawn) + free(op->spawn); + if (ns) ns->op = NULL; free(op); @@ -515,6 +564,7 @@ saib_spawn_step(struct sai_nspawn *ns) info.max_log_lines = 10000; info.timeout_us = 30 * 60 * LWS_US_PER_SEC; info.reap_cb = sai_lsp_reap_cb; + memset(&ns->res, 0, sizeof(ns->res)); info.res = &ns->res; #if defined(__linux__) info.cgroup_name_suffix = cgroup; @@ -528,6 +578,8 @@ saib_spawn_step(struct sai_nspawn *ns) op->ns = ns; ns->op = op; + op->spawn = strdup(one_step); + op->start_time = lws_now_usecs(); info.opaque = op; info.owner = &builder.lsp_owner; diff --git a/src/builder/b-private.h b/src/builder/b-private.h index 6de26bf..264b9f0 100644 --- a/src/builder/b-private.h +++ b/src/builder/b-private.h @@ -42,6 +42,8 @@ struct lws_spawn_piped; struct saib_opaque_spawn { struct sai_nspawn *ns; struct lws_spawn_piped *lsp; + char *spawn; + lws_usec_t start_time; }; #define SAI_LOAD_REPORT_US (1 * LWS_US_PER_SEC) diff --git a/src/common/include/private.h b/src/common/include/private.h index 7997ad2..b4afe7e 100644 --- a/src/common/include/private.h +++ b/src/common/include/private.h @@ -363,6 +363,7 @@ typedef struct sai_plat_server { lws_dll2_t list; lws_dll2_owner_t rejection_list; + lws_dll2_owner_t build_metric_list; lws_dll2_owner_t resource_req_list; /* sai_resource_msg_t */ lws_dll2_owner_t resource_pss_list; /* so we can find the cookie */ @@ -483,6 +484,35 @@ typedef struct sai_power_state { int powering_down; } sai_power_state_t; +typedef struct sai_build_metric { + lws_dll2_t list; + char key[65]; + char builder_name[96]; + char project_name[96]; + char ref[96]; + int parallel; + uint64_t us_cpu_user; + uint64_t us_cpu_sys; + uint64_t wallclock_us; + uint64_t peak_mem_rss; + uint64_t stg_bytes; +} sai_build_metric_t; + +typedef struct sai_build_metric_db { + lws_dll2_t list; /* for lws_struct */ + char key[65]; + uint64_t unixtime; + char builder_name[96]; + char project_name[96]; + char ref[96]; + int parallel; + uint64_t us_cpu_user; + uint64_t us_cpu_sys; + uint64_t wallclock_us; + uint64_t peak_mem_rss; + uint64_t stg_bytes; +} sai_build_metric_db_t; + extern const lws_struct_map_t lsm_schema_json_map_task[], lsm_schema_sq3_map_task[], @@ -506,8 +536,11 @@ extern const lws_struct_map_t lsm_resource[4], lsm_power_state[3], lsm_rebuild[1], - lsm_schema_rebuild[1] + lsm_schema_rebuild[1], + lsm_schema_build_metric[1], + lsm_schema_sq3_map_build_metric[1] ; +extern const lws_struct_map_t lsm_build_metric[10]; extern const lws_struct_map_t lsm_plat[8]; extern const lws_struct_map_t lsm_plat_for_json[12]; diff --git a/src/common/struct-metadata.c b/src/common/struct-metadata.c index 58c8dc5..21caec9 100644 --- a/src/common/struct-metadata.c +++ b/src/common/struct-metadata.c @@ -40,6 +40,41 @@ const lws_struct_map_t lsm_load_report_members[] = { NULL, lsm_platform_load, "platforms"), }; +const lws_struct_map_t lsm_build_metric[] = { + LSM_CARRAY (sai_build_metric_t, key, "key"), + LSM_CARRAY (sai_build_metric_t, builder_name, "builder_name"), + LSM_CARRAY (sai_build_metric_t, project_name, "project_name"), + LSM_CARRAY (sai_build_metric_t, ref, "ref"), + LSM_SIGNED (sai_build_metric_t, parallel, "parallel"), + LSM_UNSIGNED (sai_build_metric_t, us_cpu_user, "us_cpu_user"), + LSM_UNSIGNED (sai_build_metric_t, us_cpu_sys, "us_cpu_sys"), + LSM_UNSIGNED (sai_build_metric_t, wallclock_us, "wallclock_us"), + LSM_UNSIGNED (sai_build_metric_t, peak_mem_rss, "peak_mem_rss"), + LSM_UNSIGNED (sai_build_metric_t, stg_bytes, "stg_bytes"), +}; + +const lws_struct_map_t lsm_schema_build_metric[] = { + LSM_SCHEMA (sai_build_metric_t, NULL, lsm_build_metric, "com.warmcat.sai.build-metric") +}; + +const lws_struct_map_t lsm_sq3_build_metric[] = { + LSM_CARRAY (sai_build_metric_db_t, key, "key"), + LSM_UNSIGNED (sai_build_metric_db_t, unixtime, "unixtime"), + LSM_CARRAY (sai_build_metric_db_t, builder_name, "builder_name"), + LSM_CARRAY (sai_build_metric_db_t, project_name, "project_name"), + LSM_CARRAY (sai_build_metric_db_t, ref, "ref"), + LSM_SIGNED (sai_build_metric_db_t, parallel, "parallel"), + LSM_UNSIGNED (sai_build_metric_db_t, us_cpu_user, "us_cpu_user"), + LSM_UNSIGNED (sai_build_metric_db_t, us_cpu_sys, "us_cpu_sys"), + LSM_UNSIGNED (sai_build_metric_db_t, wallclock_us, "wallclock_us"), + LSM_UNSIGNED (sai_build_metric_db_t, peak_mem_rss, "peak_mem_rss"), + LSM_UNSIGNED (sai_build_metric_db_t, stg_bytes, "stg_bytes"), +}; + +const lws_struct_map_t lsm_schema_sq3_map_build_metric[] = { + LSM_SCHEMA_DLL2 (sai_build_metric_db_t, list, NULL, lsm_sq3_build_metric, "build_metrics"), +}; + const lws_struct_map_t lsm_plat[] = { /* !!! keep extern length in common/include/private.h in sync */ LSM_UNSIGNED (sai_plat_t, uid, "uid"), LSM_STRING_PTR (sai_plat_t, name, "name"), diff --git a/src/server/CMakeLists.txt b/src/server/CMakeLists.txt index 9b5eaba..774e0f7 100644 --- a/src/server/CMakeLists.txt +++ b/src/server/CMakeLists.txt @@ -12,6 +12,7 @@ set(SRCS s-central.c s-websrv.c s-resource.c + s-metrics-db.c ../common/c-utils.c ) diff --git a/src/server/s-metrics-db.c b/src/server/s-metrics-db.c new file mode 100644 index 0000000..685c32e --- /dev/null +++ b/src/server/s-metrics-db.c @@ -0,0 +1,163 @@ +/* + * Sai server metrics db + * + * Copyright (C) 2024 Andy Green + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation: + * version 2.1 of the License. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + * MA 02110-1301 USA + */ + +#include + +#include "s-private.h" +#include "s-metrics-db.h" + +static int +sais_metrics_db_prune(struct vhd *vhd, const char *key) +{ + sqlite3_stmt *stmt; + char sql[256]; + int rc, count = 0; + + if (!vhd->pdb_metrics) + return 0; + + lws_snprintf(sql, sizeof(sql), + "SELECT COUNT(*) FROM build_metrics WHERE key = ?;"); + + rc = sqlite3_prepare_v2(vhd->pdb_metrics, sql, -1, &stmt, 0); + if (rc != SQLITE_OK) { + lwsl_err("%s: failed to prepare statement: %s\n", __func__, + sqlite3_errmsg(vhd->pdb_metrics)); + return 1; + } + + sqlite3_bind_text(stmt, 1, key, -1, SQLITE_STATIC); + + if (sqlite3_step(stmt) == SQLITE_ROW) + count = sqlite3_column_int(stmt, 0); + + sqlite3_finalize(stmt); + + if (count <= 10) + return 0; + + lws_snprintf(sql, sizeof(sql), + "DELETE FROM build_metrics WHERE key = ? AND rowid IN " + "(SELECT rowid FROM build_metrics WHERE key = ? " + "ORDER BY unixtime ASC LIMIT %d);", count - 10); + + rc = sqlite3_prepare_v2(vhd->pdb_metrics, sql, -1, &stmt, 0); + if (rc != SQLITE_OK) { + lwsl_err("%s: failed to prepare statement: %s\n", __func__, + sqlite3_errmsg(vhd->pdb_metrics)); + return 1; + } + + sqlite3_bind_text(stmt, 1, key, -1, SQLITE_STATIC); + sqlite3_bind_text(stmt, 2, key, -1, SQLITE_STATIC); + + rc = sqlite3_step(stmt); + if (rc != SQLITE_DONE) { + lwsl_err("%s: failed to delete old metrics: %s\n", __func__, + sqlite3_errmsg(vhd->pdb_metrics)); + sqlite3_finalize(stmt); + return 1; + } + + sqlite3_finalize(stmt); + + return 0; +} + +int +sais_metrics_db_init(struct vhd *vhd) +{ + char db_path[PATH_MAX]; + int rc; + + if (vhd->pdb_metrics) + return 0; + + if (!vhd->sqlite3_path_lhs) + return 0; + + lws_snprintf(db_path, sizeof(db_path), "%s-build-metrics.sqlite3", + vhd->sqlite3_path_lhs); + + rc = sqlite3_open(db_path, &vhd->pdb_metrics); + if (rc != SQLITE_OK) { + lwsl_err("%s: cannot open database %s: %s\n", __func__, + db_path, sqlite3_errmsg(vhd->pdb_metrics)); + sqlite3_close(vhd->pdb_metrics); + vhd->pdb_metrics = NULL; + return 1; + } + + if (lws_struct_sq3_create_table(vhd->pdb_metrics, + lsm_schema_sq3_map_build_metric)) { + lwsl_err("%s: failed to create build_metrics table\n", __func__); + sqlite3_close(vhd->pdb_metrics); + vhd->pdb_metrics = NULL; + return 1; + } + + return 0; +} + +void +sais_metrics_db_close(void) +{ + /* This is managed by the vhd destruction */ +} + +int +sais_metrics_db_add(struct vhd *vhd, const struct sai_build_metric *m) +{ + sai_build_metric_db_t dbm; + lws_dll2_owner_t owner; + + if (!vhd->pdb_metrics) + return 0; + + memset(&dbm, 0, sizeof(dbm)); + + lws_strncpy(dbm.key, m->key, sizeof(dbm.key)); + dbm.unixtime = (uint64_t)time(NULL); + lws_strncpy(dbm.builder_name, m->builder_name, sizeof(dbm.builder_name)); + lws_strncpy(dbm.project_name, m->project_name, sizeof(dbm.project_name)); + lws_strncpy(dbm.ref, m->ref, sizeof(dbm.ref)); + dbm.parallel = m->parallel; + dbm.us_cpu_user = m->us_cpu_user; + dbm.us_cpu_sys = m->us_cpu_sys; + dbm.wallclock_us = m->wallclock_us; + dbm.peak_mem_rss = m->peak_mem_rss; + dbm.stg_bytes = m->stg_bytes; + + lws_dll2_owner_clear(&owner); + lws_dll2_add_tail(&dbm.list, &owner); + + if (lws_struct_sq3_serialize(vhd->pdb_metrics, + lsm_schema_sq3_map_build_metric, + &owner, 0)) { + lwsl_err("%s: failed to serialize build metric\n", __func__); + return 1; + } + + if (sais_metrics_db_prune(vhd, dbm.key)) + lwsl_warn("%s: pruning metrics failed\n", __func__); + + return 0; +} diff --git a/src/server/s-metrics-db.h b/src/server/s-metrics-db.h new file mode 100644 index 0000000..d7f8093 --- /dev/null +++ b/src/server/s-metrics-db.h @@ -0,0 +1,37 @@ +/* + * Sai server metrics db + * + * Copyright (C) 2024 Andy Green + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation: + * version 2.1 of the License. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + * MA 02110-1301 USA + */ + +#if !defined(__SAI_SERVER_METRICS_DB_H__) +#define __SAI_SERVER_METRICS_DB_H__ + +struct vhd; +struct sai_build_metric; + +int +sais_metrics_db_init(struct vhd *vhd); + +void +sais_metrics_db_close(void); + +int +sais_metrics_db_add(struct vhd *vhd, const struct sai_build_metric *m); + +#endif diff --git a/src/server/s-private.h b/src/server/s-private.h index d73368e..714bbb1 100644 --- a/src/server/s-private.h +++ b/src/server/s-private.h @@ -210,6 +210,7 @@ struct vhd { struct lwsac *ac_plats; const char *sqlite3_path_lhs; + sqlite3 *pdb_metrics; lws_dll2_owner_t sqlite3_cache; /* sais_sqlite_cache_t */ lws_dll2_owner_t tasklog_cache; diff --git a/src/server/s-ws-builder.c b/src/server/s-ws-builder.c index 29deb0e..1aa7411 100644 --- a/src/server/s-ws-builder.c +++ b/src/server/s-ws-builder.c @@ -28,6 +28,7 @@ #include #include "s-private.h" +#include "s-metrics-db.h" typedef struct { int count; @@ -95,6 +96,8 @@ static const lws_struct_map_t lsm_schema_map_ba[] = { "com.warmcat.sai.loadreport"), LSM_SCHEMA (sai_resource_t, NULL, lsm_resource, "com-warmcat-sai-resource"), + LSM_SCHEMA (sai_build_metric_t, NULL, lsm_build_metric, + "com.warmcat.sai.build-metric"), }; enum { @@ -104,6 +107,7 @@ enum { SAIM_WSSCH_BUILDER_ARTIFACT, SAIM_WSSCH_BUILDER_LOADREPORT, SAIM_WSSCH_BUILDER_RESOURCE_REQ, + SAIM_WSSCH_BUILDER_METRIC, }; static void @@ -484,6 +488,7 @@ int sais_ws_json_rx_builder(struct vhd *vhd, struct pss *pss, uint8_t *buf, size_t bl) { char event_uuid[33], s[128], esc[96]; + const sai_build_metric_t *metric; sai_resource_requisition_t *rr; sai_resource_wellknown_t *wk; struct lwsac *ac = NULL; @@ -497,6 +502,8 @@ sais_ws_json_rx_builder(struct vhd *vhd, struct pss *pss, uint8_t *buf, size_t b uint64_t rid; int n, m; + sais_metrics_db_init(vhd); + if (pss->bulk_binary_data) { lwsl_info("%s: bulk %d\n", __func__, (int)bl); m = (int)bl; @@ -1044,6 +1051,11 @@ bail: sais_resource_check_if_can_accept_queued(wk); break; + case SAIM_WSSCH_BUILDER_METRIC: + metric = (const sai_build_metric_t *)pss->a.dest; + sais_metrics_db_add(vhd, metric); + lwsac_free(&pss->a.ac); + break; } return 0;