diff --git a/.sai.json b/.sai.json
index 29336dc..056cae9 100644
--- a/.sai.json
+++ b/.sai.json
@@ -4,14 +4,14 @@
"platforms": {
"rocky9/aarch64-a72a55-rk3588/gcc": {
"build": [
- "mkdir build destdir;cd build; CCACHE_DISABLE=1 cmake .. ${cmake}",
+ "mkdir -p build destdir;cd build; CCACHE_DISABLE=1 cmake .. ${cmake}",
"cd build && make -j",
- "cd build && SAI_CPACK=\"-G RPM\" ${cpack}"
+ "cd build && SAI_CPACK=\"-G RPM\" ${cpack} ; echo \"cpack step finished with exit code $?\""
]
},
"netbsd-OSX-bigsur/x86_64-intel-i3/llvm": {
"build": [
- "mkdir build destdir; cd build; MACOSX_DEPLOYMENT_TARGET=10.14 ; cmake .. -DCMAKE_MAKE_PROGRAM=/usr/bin/make -DLWS_OPENSSL_INCLUDE_DIRS=/usr/local/opt/openssl@1.1/include -DLWS_OPENSSL_LIBRARIES=\"/usr/local/opt/openssl/lib/libssl.dylib;/usr/local/opt/openssl/lib/libcrypto.dylib\" ${cmake} ",
+ "mkdir -p build destdir; cd build; MACOSX_DEPLOYMENT_TARGET=10.14 ; cmake .. -DCMAKE_MAKE_PROGRAM=/usr/bin/make -DLWS_OPENSSL_INCLUDE_DIRS=/usr/local/opt/openssl@1.1/include -DLWS_OPENSSL_LIBRARIES=\"/usr/local/opt/openssl/lib/libssl.dylib;/usr/local/opt/openssl/lib/libcrypto.dylib\" ${cmake} ",
"cd build && make -j && make -j DESTDIR=../destdir install",
"cd build && ctest -j4 --output-on-failure",
"cd build && SAI_CPACK=\"-G ZIP\" ${cpack}"
diff --git a/src/builder/CMakeLists.txt b/src/builder/CMakeLists.txt
index f1259ee..1d24227 100644
--- a/src/builder/CMakeLists.txt
+++ b/src/builder/CMakeLists.txt
@@ -5,7 +5,6 @@ set(SRCS
b-sai.c
b-conf.c
b-comms.c
- b-mirror.c
b-nspawn.c
b-task.c
b-artifacts.c
@@ -80,5 +79,10 @@ if (requirements)
install(TARGETS "sai-builder"
RUNTIME DESTINATION "${BIN_DIR}" COMPONENT builder)
+ install(DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/scripts/"
+ DESTINATION "${BIN_DIR}"
+ COMPONENT builder
+ FILE_PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE)
+
endif(requirements)
include(CPack)
diff --git a/src/builder/b-comms.c b/src/builder/b-comms.c
index c027424..80116b6 100644
--- a/src/builder/b-comms.c
+++ b/src/builder/b-comms.c
@@ -72,6 +72,13 @@ saib_m_tx(void *userobj, lws_ss_tx_ordinal_t ord, uint8_t *buf, size_t *len,
int n = 0;
/*
+ * Are there some logs to dump?
+ */
+
+ if (spm->logs_in_flight)
+ goto send_logs; /* nothing to send */
+
+ /*
* Any build metrics to process?
*/
@@ -237,12 +244,9 @@ saib_m_tx(void *userobj, lws_ss_tx_ordinal_t ord, uint8_t *buf, size_t *len,
return LWSSSSRET_OK;
}
- /*
- * Are there some logs to dump?
- */
+ return 1;
- if (!spm->logs_in_flight)
- return 1; /* nothing to send */
+send_logs:
/*
* Yes somebody has some logs... since we handle all logs on any
@@ -313,23 +317,6 @@ saib_m_tx(void *userobj, lws_ss_tx_ordinal_t ord, uint8_t *buf, size_t *len,
if (spm != ns->spm)
continue;
- if (ns->state_changed) {
- ns->state_changed = 0;
-
- switch (ns->state) {
- case NSSTATE_CHECKEDOUT:
- saib_set_ns_state(ns, NSSTATE_BUILD);
- if (saib_spawn_build(ns)) {
- lwsl_err("%s: saib_spawn failed\n",
- __func__);
- saib_set_ns_state(ns, NSSTATE_FAILED);
- }
- break;
- default:
- break;
- }
- }
-
if (!ns->chunk_cache.count || !ns->chunk_cache.tail)
continue;
@@ -351,7 +338,7 @@ saib_m_tx(void *userobj, lws_ss_tx_ordinal_t ord, uint8_t *buf, size_t *len,
ns->task->uuid, (unsigned long long)lws_now_usecs(),
chunk->stdfd, (int)chunk->len);
- if (ns->finished_when_logs_drained && !ns->chunk_cache.count)
+ if (ns->finished_when_logs_drained && ns->chunk_cache.count == 1)
/*
* Let the last guy report the finished state
*/
@@ -382,13 +369,13 @@ saib_m_tx(void *userobj, lws_ss_tx_ordinal_t ord, uint8_t *buf, size_t *len,
spm->logs_in_flight);
if (ns->finished_when_logs_drained && !ns->chunk_cache.count) {
-
/*
* He's in DONE state, and the draining he was waiting
- * for has now happened
+ * for has now happened.
*/
lwsl_notice("%s: drained and empty\n", __func__);
- saib_task_destroy(ns);
+ ns->finished_when_logs_drained = 0;
+ saib_set_ns_state(ns, NSSTATE_UPLOADING_ARTIFACTS);
}
break;
@@ -518,7 +505,7 @@ saib_sul_load_report_cb(struct lws_sorted_usec_list *sul)
int load = -1;
if (il) {
- il->state = (ns->state == NSSTATE_BUILD);
+ il->state = (ns->state == NSSTATE_EXECUTING_STEPS);
if (!il->state) {
/* If the instance is idle, its load is 0, regardless of system load. */
diff --git a/src/builder/b-mirror.c b/src/builder/b-mirror.c
deleted file mode 100644
index 96b33c6..0000000
--- a/src/builder/b-mirror.c
+++ /dev/null
@@ -1,374 +0,0 @@
-/*
- * sai-builder com-warmcat-sai client protocol implementation
- *
- * Copyright (C) 2019 - 2021 Andy Green <andy@warmcat.com>
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation:
- * version 2.1 of the License.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301 USA
- */
-
-#include <libwebsockets.h>
-#include <string.h>
-#include <signal.h>
-#include <assert.h>
-#include <fcntl.h>
-
-#include "b-private.h"
-
-static const char * const git_helper_sh =
- "#!/bin/bash\n"
-#if defined(__APPLE__)
- "export PATH=/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin:/sbin:/usr/sbin\n"
-#else
- "export PATH=/usr/local/bin:$PATH\n"
-#endif
- "set -e\n"
- "echo \"git_helper_sh: starting\"\n"
- "OPERATION=$1\n"
- "shift\n"
- "if [ \"$OPERATION\" == \"mirror\" ]; then\n"
- " REMOTE_URL=$1\n"
- " REF=$2\n"
- " HASH=$3\n"
- " MIRROR_PATH=$4\n"
- " for i in $(seq 1 60); do\n"
- " if [ -d \"$MIRROR_PATH/.git\" ]; then\n"
- " if git -C \"$MIRROR_PATH\" rev-parse -q --verify \"ref-$HASH\" > /dev/null; then\n"
- " exit 0\n"
- " fi\n"
- " fi\n"
- " if mkdir \"$MIRROR_PATH.lock\" 2>/dev/null; then\n"
- " trap 'rm -rf \"$MIRROR_PATH.lock\"' EXIT\n"
- " if [ -d \"$MIRROR_PATH/.git\" ]; then\n"
- " if git -C \"$MIRROR_PATH\" rev-parse -q --verify \"ref-$HASH\" > /dev/null; then\n"
- " exit 0\n"
- " fi\n"
- " fi\n"
- " mkdir -p \"$MIRROR_PATH\"\n"
- " if [ ! -d \"$MIRROR_PATH/.git\" ]; then\n"
- " git init --bare \"$MIRROR_PATH\"\n"
- " fi\n"
- " REFSPEC=\"$REF:ref-$HASH\"\n"
- " git -C \"$MIRROR_PATH\" fetch \"$REMOTE_URL\" \"+$REFSPEC\"\n"
- " exit 0\n"
- " fi\n"
- " echo \"git mirror locked, waiting...\"\n"
- " sleep 1\n"
- " done\n"
- " exit 1\n"
- "elif [ \"$OPERATION\" == \"checkout\" ]; then\n"
- " MIRROR_PATH=$1\n"
- " BUILD_DIR=$2\n"
- " HASH=$3\n"
- " if [ ! -d \"$BUILD_DIR/.git\" ]; then\n"
- " rm -rf \"$BUILD_DIR\"\n"
- " mkdir -p \"$BUILD_DIR\"\n"
- " git -C \"$BUILD_DIR\" init\n"
- " fi\n"
- " if ! git -C \"$BUILD_DIR\" fetch \"$MIRROR_PATH\" \"ref-$HASH\"; then\n"
- " exit 2\n"
- " fi\n"
- " git -C \"$BUILD_DIR\" checkout -f \"$HASH\"\n"
- " git -C \"$BUILD_DIR\" clean -fdx\n"
- "else\n"
- " exit 1\n"
- "fi\n"
- "echo \">>> Git helper script finished.\"\n"
- "exit 0\n";
-
-#if defined(WIN32)
-static const char * const git_helper_bat =
- "@echo on\n"
- "setlocal EnableDelayedExpansion\n"
- "echo \"git_helper_bat: starting\"\n"
- "set \"OPERATION=%~1\"\n"
- "echo \"OPERATION: !OPERATION!\"\n"
- "if /i \"!OPERATION!\"==\"mirror\" (\n"
- " set \"REMOTE_URL=%~2\"\n"
- " set \"REF=%~3\"\n"
- " set \"HASH=%~4\"\n"
- " set \"MIRROR_PATH=%~5\"\n"
- " echo \"REMOTE_URL: !REMOTE_URL!\"\n"
- " echo \"REF: !REF!\"\n"
- " echo \"HASH: !HASH!\"\n"
- " echo \"MIRROR_PATH: !MIRROR_PATH!\"\n"
- " :lock_wait\n"
- " mkdir \"!MIRROR_PATH!.lock\" 2>nul\n"
- " if errorlevel 1 (\n"
- " echo \"git mirror locked, waiting...\"\n"
- " timeout /t 1 /nobreak > nul\n"
- " goto :lock_wait\n"
- " )\n"
- " if exist \"!MIRROR_PATH!\\.git\" (\n"
- " git -C \"!MIRROR_PATH!\" rev-parse -q --verify \"ref-!HASH!\" > nul 2> nul\n"
- " if exist !MIRROR_PATH!\\.git if not errorlevel 1 (\n"
- " rmdir \"!MIRROR_PATH!.lock\"\n"
- " exit /b 0\n"
- " )\n"
- " )\n"
- " if not exist \"!MIRROR_PATH!\\.\" (\n"
- " mkdir \"!MIRROR_PATH!\"\n"
- " )\n"
- " if not exist \"!MIRROR_PATH!\\.git\" (\n"
- " git init --bare \"!MIRROR_PATH!\"\n"
- " if errorlevel 1 (\n"
- " rmdir \"!MIRROR_PATH!.lock\"\n"
- " exit /b 1\n"
- " )\n"
- " )\n"
- " set \"REFSPEC=!REF!:ref-!HASH!\"\n"
- " echo \"REFSPEC: !REFSPEC!\"\n"
- " git -C \"!MIRROR_PATH!\" fetch \"!REMOTE_URL!\" \"!REFSPEC!\" 2>&1\n"
- " if !ERRORLEVEL! neq 0 (\n"
- " echo \"git fetch failed with errorlevel !ERRORLEVEL!\"\n"
- " rmdir \"!MIRROR_PATH!.lock\"\n"
- " exit /b 1\n"
- " )\n"
- " rmdir \"!MIRROR_PATH!.lock\"\n"
- " exit /b 0\n"
- ")\n"
- "if /i \"!OPERATION!\"==\"checkout\" (\n"
- " set \"MIRROR_PATH=%~2\"\n"
- " set \"BUILD_DIR=%~3\"\n"
- " set \"HASH=%~4\"\n"
- " echo \"MIRROR_PATH: !MIRROR_PATH!\"\n"
- " echo \"BUILD_DIR: !BUILD_DIR!\"\n"
- " echo \"HASH: !HASH!\"\n"
- " if not exist \"!BUILD_DIR!\\.git\" (\n"
- " if exist \"!BUILD_DIR!\\\" rmdir /s /q \"!BUILD_DIR!\"\n"
- " mkdir \"!BUILD_DIR!\"\n"
- " git -C \"!BUILD_DIR!\" init\n"
- " if errorlevel 1 exit /b 1\n"
- " )\n"
- " git -C \"!BUILD_DIR!\" fetch \"!MIRROR_PATH!\" \"ref-!HASH!\"\n"
- " if errorlevel 1 exit /b 2\n"
- " git -C \"!BUILD_DIR!\" checkout -f \"!HASH!\"\n"
- " if errorlevel 1 exit /b 1\n"
- " echo \">>> Git helper script finished.\"\n"
- " exit /b 0\n"
- ")\n"
- "exit /b 1\n";
-#endif
-
-static void sai_git_mirror_reap_cb(void *opaque, const struct lws_spawn_resource_us *res,
- siginfo_t *si, int we_killed_him);
-
-static void
-saib_start_mirror_cb(lws_sorted_usec_list_t *sul)
-{
- struct sai_nspawn *ns = lws_container_of(sul, struct sai_nspawn, sul_mirror);
-
- saib_start_mirror(ns);
-}
-
-static void
-sai_git_checkout_reap_cb(void *opaque, const struct lws_spawn_resource_us *res, siginfo_t *si,
- int we_killed_him)
-{
- struct saib_opaque_spawn *op = (struct saib_opaque_spawn *)opaque;
- struct sai_nspawn *ns = op->ns;
- int exit_code = -1;
-
- lwsl_notice("%s: mirror reap callback started\n", __func__);
-
- lwsl_warn("%s: reap at %llu: we_killed_him: %d\n",
- __func__, (unsigned long long)lws_now_usecs(),
- we_killed_him);
-
- if (we_killed_him)
- goto fail;
-
-#if !defined(WIN32)
- if (si->si_code == CLD_EXITED)
- exit_code = si->si_status;
-#else
- exit_code = si->retcode & 0xff;
-#endif
-
- if (exit_code == 0) {
- saib_set_ns_state(ns, NSSTATE_CHECKEDOUT);
- goto onward;
- }
-
- if (exit_code == 2) {
- if (ns->mirror_wait_budget-- < 0) {
- lwsl_err("%s: mirror wait budget exhausted\n", __func__);
- goto fail;
- }
- lws_sul_schedule(builder.context, 0, &ns->sul_mirror,
- saib_start_mirror_cb, LWS_US_PER_SEC);
- goto onward;
- }
-
-fail:
- saib_set_ns_state(ns, NSSTATE_FAILED);
-onward:
- ns->op = NULL;
- free(op);
-}
-
-static void
-sai_git_mirror_reap_cb(void *opaque, const struct lws_spawn_resource_us *res, siginfo_t *si,
- int we_killed_him)
-{
- struct saib_opaque_spawn *op = (struct saib_opaque_spawn *)opaque;
- struct sai_nspawn *ns = op->ns;
- int exit_code = -1;
-
- // lwsl_warn("%s: reap at %llu: we_killed_him: %d, si_code: %d, si_status: %d\n",
- // __func__, (unsigned long long)lws_now_usecs(),
- // we_killed_him, si->si_code, si->si_status);
-
- if (we_killed_him)
- goto fail;
-
-#if !defined(WIN32)
- if (si->si_code == CLD_EXITED)
- exit_code = si->si_status;
-#else
- exit_code = si->retcode & 0xff;
-#endif
-
- if (exit_code == 0) {
- /* mirror succeeded, now try checkout again */
- lwsl_notice("%s: mirror success, starting checkout\n", __func__);
- saib_start_checkout(ns);
- goto onward;
- } else
- lwsl_warn("%s: exit code 0x%x == failure\n", __func__, (unsigned int)exit_code);
-
-fail:
- saib_set_ns_state(ns, NSSTATE_FAILED);
-onward:
- ns->op = NULL;
- free(op);
-}
-
-static int
-saib_spawn_git_helper(struct sai_nspawn *ns, const char *operation)
-{
- struct lws_spawn_piped_info info;
- struct saib_opaque_spawn *op;
- char script_path[1024], inp[512];
- const char *pargs[9];
- const char **env = NULL;
-#if defined(__APPLE__)
- const char *env_array[2];
- char path_env[256];
-#endif
- ssize_t n;
- int fd, count = 0;
-
-#if defined(__APPLE__)
- lws_snprintf(path_env, sizeof(path_env),
- "PATH=/usr/local/bin:/opt/homebrew/bin:/usr/bin:/bin:/sbin:/usr/sbin");
- env_array[0] = path_env;
- env_array[1] = NULL;
- env = (const char **)env_array;
-#endif
-
-#if defined(WIN32)
- lws_snprintf(script_path, sizeof(script_path), "%s\\sai-git-helper-%d.bat",
- builder.home, ns->instance_idx);
-#else
- lws_snprintf(script_path, sizeof(script_path), "%s/sai-git-helper-%d.sh",
- builder.home, ns->instance_idx);
-#endif
-
- fd = open(script_path, O_CREAT | O_TRUNC | O_WRONLY, 0755);
- if (fd < 0)
- return -1;
-
-#if defined(WIN32)
- n = write(fd, git_helper_bat, strlen(git_helper_bat));
-#else
- n = write(fd, git_helper_sh, strlen(git_helper_sh));
-#endif
- close(fd);
-
- if (n < 0)
- return -1;
-
- pargs[count++] = script_path;
- pargs[count++] = operation;
-
- if (!strcmp(operation, "mirror")) {
- pargs[count++] = ns->git_repo_url;
- pargs[count++] = ns->ref;
- pargs[count++] = ns->hash;
- pargs[count++] = ns->path;
- } else { /* checkout */
- lws_strncpy(inp, ns->inp, sizeof(inp) - 1);
- if (inp[strlen(inp) - 1] == '\\')
- inp[strlen(inp) - 1] = '\0';
- pargs[count++] = ns->path;
- pargs[count++] = inp;
- pargs[count++] = ns->hash;
- }
- pargs[count++] = NULL;
-
- memset(&info, 0, sizeof(info));
- info.vh = builder.vhost;
- info.exec_array = pargs;
- info.env_array = env;
- info.protocol_name = "sai-stdxxx";
- info.timeout_us = 5 * 60 * LWS_US_PER_SEC;
-
- if (!strcmp(operation, "mirror"))
- info.reap_cb = sai_git_mirror_reap_cb;
- else
- info.reap_cb = sai_git_checkout_reap_cb;
-
- op = malloc(sizeof(*op));
- if (!op)
- return -1;
- memset(op, 0, sizeof(*op));
-
- op->ns = ns;
- ns->op = op;
-
- info.opaque = op;
- info.owner = &builder.lsp_owner;
- info.plsp = &op->lsp;
-
- // lwsl_warn("%s: spawning git-helper for %s at %llu\n", __func__,
- // operation, (unsigned long long)lws_now_usecs());
-
- if (lws_spawn_piped(&info) == NULL) {
- lwsl_err("%s: lws_spawn_piped for %s failed\n", __func__,
- operation);
- ns->op = NULL;
- /* op is attached to wsi and will be freed later */
- return -1;
- }
-
- if (!strcmp(operation, "mirror"))
- saib_set_ns_state(ns, NSSTATE_WAIT_REMOTE_MIRROR);
-
- return 0;
-}
-
-int
-saib_start_mirror(struct sai_nspawn *ns)
-{
- return saib_spawn_git_helper(ns, "mirror");
-}
-
-int
-saib_start_checkout(struct sai_nspawn *ns)
-{
- lwsl_notice("%s: starting checkout\n", __func__);
- return saib_spawn_git_helper(ns, "checkout");
-}
diff --git a/src/builder/b-nspawn.c b/src/builder/b-nspawn.c
index d262b24..6bd314d 100644
--- a/src/builder/b-nspawn.c
+++ b/src/builder/b-nspawn.c
@@ -239,7 +239,7 @@ sai_lsp_reap_cb(void *opaque, const lws_spawn_resource_us_t *res, siginfo_t *si,
n = lws_snprintf(s, sizeof(s),
">saib> Step %d: Total [ %s u / %s s, Mem: %sB, Stg: %sB ], Step [ %s u / %s s, Mem: %sB, Stg: %sB ]\n",
- ns->build_step + 1, h1, h2, h3, h4, h5, h6, h7, h8);
+ ns->current_step + 1, h1, h2, h3, h4, h5, h6, h7, h8);
saib_log_chunk_create(ns, s, (size_t)n, 3);
}
@@ -272,7 +272,6 @@ sai_lsp_reap_cb(void *opaque, const lws_spawn_resource_us_t *res, siginfo_t *si,
lws_strncpy(m->builder_name, ns->sp->name, sizeof(m->builder_name));
lws_strncpy(m->project_name, ns->project_name, sizeof(m->project_name));
lws_strncpy(m->ref, ns->ref, sizeof(m->ref));
- m->parallel = ns->parallel;
m->us_cpu_user = res->us_cpu_user;
m->us_cpu_sys = res->us_cpu_sys;
m->wallclock_us = (uint64_t)(lws_now_usecs() - op->start_time);
@@ -285,8 +284,8 @@ sai_lsp_reap_cb(void *opaque, const lws_spawn_resource_us_t *res, siginfo_t *si,
}
}
- ns->build_step++;
- if (ns->build_step < ns->build_step_count) {
+ ns->current_step++;
+ if (ns->current_step < ns->build_step_count) {
/* there are more steps, spawn the next one */
if (ns)
ns->op = NULL;
@@ -324,7 +323,7 @@ sai_lsp_reap_cb(void *opaque, const lws_spawn_resource_us_t *res, siginfo_t *si,
return;
fail:
- n = lws_snprintf(s, sizeof(s), "Build step %d FAILED", ns->build_step + 1);
+ n = lws_snprintf(s, sizeof(s), "Build step %d FAILED", ns->current_step + 1);
saib_log_chunk_create(ns, s, (size_t)n, 3);
saib_task_grace(ns);
@@ -352,7 +351,7 @@ static const char * const runscript_win_first =
"set HOME=%s\n"
"cd %s &&"
" rmdir /s /q build & "
- "%s"
+ "%s < NUL"
;
static const char * const runscript_win_next =
@@ -364,7 +363,7 @@ static const char * const runscript_win_next =
"set SAI_LOGPROXY_TTY1=%s\n"
"set HOME=%s\n"
"cd %s &&"
- "%s"
+ "%s < NUL"
;
#else
@@ -389,7 +388,7 @@ static const char * const runscript_first =
"set -e\n"
"cd %s/jobs/$SAI_OVN/$SAI_PROJECT\n"
"rm -rf build\n"
- "%s\n"
+ "%s < /dev/null\n"
"exit $?\n"
;
@@ -412,7 +411,7 @@ static const char * const runscript_next =
"export SAI_LOGPROXY_TTY1=%s\n"
"set -e\n"
"cd %s/jobs/$SAI_OVN/$SAI_PROJECT\n"
- "%s\n"
+ "%s < /dev/null\n"
"exit $?\n"
;
@@ -424,13 +423,13 @@ saib_spawn_step(struct sai_nspawn *ns);
int
saib_spawn_build(struct sai_nspawn *ns)
{
- const char *p = ns->task->build;
+ const char *p = ns->task->steps;
int n;
- ns->build_step = 0;
+ ns->current_step = 0;
ns->build_step_count = 0;
- lwsl_hexdump_err(ns->task->build, strlen(ns->task->build));
+ lwsl_hexdump_err(ns->task->steps, strlen(ns->task->steps));
while ((p = strchr(p, '\n'))) {
ns->build_step_count++;
@@ -482,12 +481,12 @@ saib_spawn_step(struct sai_nspawn *ns)
#endif
char one_step[4096];
- const char *p_build = ns->task->build, *q;
+ const char *p_build = ns->task->steps, *q;
int step = 0;
- // lwsl_hexdump_notice(ns->task->build, strlen(ns->task->build));
+ lwsl_hexdump_notice(ns->task->steps, strlen(ns->task->steps));
- while (step < ns->build_step && (p_build = strchr(p_build, '\n'))) {
+ while (step < ns->current_step && (p_build = strchr(p_build, '\n'))) {
p_build++;
step++;
}
@@ -523,18 +522,18 @@ saib_spawn_step(struct sai_nspawn *ns)
#if defined(WIN32)
n = lws_snprintf(st, sizeof(st),
- ns->build_step ? runscript_win_next : runscript_win_first,
+ ns->current_step ? runscript_win_next : runscript_win_first,
ns->instance_idx,
- ns->parallel ? ns->parallel : 1,
+ 1,
respath, ns->slp_control.sockpath,
ns->slp[0].sockpath, ns->slp[1].sockpath, builder.home,
ns->inp, one_step);
#else
n = lws_snprintf(st, sizeof(st),
- ns->build_step ? runscript_next : runscript_first,
+ ns->current_step ? runscript_next : runscript_first,
builder.home, ns->fsm.ovname,
ns->project_name, ns->ref, ns->instance_idx,
- ns->parallel ? ns->parallel : 1,
+ 1,
respath, ns->slp_control.sockpath,
ns->slp[0].sockpath, ns->slp[1].sockpath,
builder.home, one_step);
diff --git a/src/builder/b-private.h b/src/builder/b-private.h
index 264b9f0..d4ba8cb 100644
--- a/src/builder/b-private.h
+++ b/src/builder/b-private.h
@@ -69,17 +69,9 @@ struct saib_ws_pss;
enum {
NSSTATE_INIT,
NSSTATE_MOUNTING,
- NSSTATE_STARTING_MIRROR,
- /* speculatively see if we can already check out locally */
- NSSTATE_CHECKOUT_SPEC,
- /* we are stuck waiting for remote mirror thread */
- NSSTATE_WAIT_REMOTE_MIRROR,
- /* remote mirror completed, we can check out locally */
- NSSTATE_CHECKOUT,
- /* we have finished with threadpool / wait for remote */
- NSSTATE_CHECKEDOUT,
- NSSTATE_BUILD,
+ NSSTATE_EXECUTING_STEPS,
NSSTATE_DONE,
+ NSSTATE_UPLOADING_ARTIFACTS,
NSSTATE_FAILED,
};
@@ -240,5 +232,3 @@ saib_get_system_cpu(struct sai_builder *b);
int saib_get_cpu_count(void);
-int saib_start_mirror(struct sai_nspawn *ns);
-int saib_start_checkout(struct sai_nspawn *ns);
diff --git a/src/builder/b-sai.c b/src/builder/b-sai.c
index a04e7ac..9ca1b55 100644
--- a/src/builder/b-sai.c
+++ b/src/builder/b-sai.c
@@ -437,6 +437,10 @@ app_system_state_nf(lws_state_manager_t *mgr, lws_state_notify_link_t *link,
struct sai_plat *sp = lws_container_of(mp, struct sai_plat,
sai_plat_list);
+#if defined(WIN32)
+ sp->windows = 1;
+#endif
+
/*
* ... for each nspawn on the platform...
*/
diff --git a/src/builder/b-task.c b/src/builder/b-task.c
index ea09f4e..97c066a 100644
--- a/src/builder/b-task.c
+++ b/src/builder/b-task.c
@@ -57,16 +57,14 @@ enum {
static const char * const nsstates[] = {
"NSSTATE_INIT",
"NSSTATE_MOUNTING",
- "NSSTATE_STARTING_MIRROR",
- "NSSTATE_CHECKOUT_SPEC", /* initial speculative checkout */
- "NSSTATE_WAIT_REMOTE_MIRROR",
- "NSSTATE_CHECKOUT",
- "NSSTATE_CHECKEDOUT",
- "NSSTATE_BUILD",
+ "NSSTATE_EXECUTING_STEPS",
"NSSTATE_DONE",
+ "NSSTATE_UPLOADING_ARTIFACTS",
"NSSTATE_FAILED",
};
+static void saib_start_artifact_upload(struct sai_nspawn *ns);
+
int
saib_set_ns_state(struct sai_nspawn *ns, int state)
{
@@ -76,10 +74,16 @@ saib_set_ns_state(struct sai_nspawn *ns, int state)
ns->state = (uint8_t)state;
ns->state_changed = 1;
- n = lws_snprintf(log, sizeof(log), ">saib> %s\n", nsstates[state]);
+ if (state >= 0 && state < (int)LWS_ARRAY_SIZE(nsstates))
+ n = lws_snprintf(log, sizeof(log), ">saib> %s\n", nsstates[state]);
+ else
+ n = lws_snprintf(log, sizeof(log), ">saib> ILLEGAL_STATE %d\n", (int)state);
saib_log_chunk_create(ns, log, (unsigned int)n, 3);
+ if (state == NSSTATE_UPLOADING_ARTIFACTS)
+ saib_start_artifact_upload(ns);
+
if (state == NSSTATE_FAILED) {
ns->retcode = SAISPRF_EXIT | 254;
saib_task_grace(ns);
@@ -302,8 +306,8 @@ artifact_glob_cb(void *data, const char *path)
* of the task and reset the nspawn.
*/
-void
-saib_task_grace(struct sai_nspawn *ns)
+static void
+saib_start_artifact_upload(struct sai_nspawn *ns)
{
char filt[32], scandir[256];
struct lws_tokenize ts;
@@ -311,10 +315,6 @@ saib_task_grace(struct sai_nspawn *ns)
lws_dir_glob_t g;
int m;
- ns->finished_when_logs_drained = 1;
- lws_sul_schedule(builder.context, 0, &ns->sul_cleaner,
- saib_sub_cleaner_cb, 20 * LWS_USEC_PER_SEC);
-
if (!ns->spm)
return;
@@ -408,6 +408,14 @@ scan:
}
}
+void
+saib_task_grace(struct sai_nspawn *ns)
+{
+ ns->finished_when_logs_drained = 1;
+ lws_sul_schedule(builder.context, 0, &ns->sul_cleaner,
+ saib_sub_cleaner_cb, 20 * LWS_USEC_PER_SEC);
+}
+
static void
saib_sul_task_cancel(struct lws_sorted_usec_list *sul)
{
@@ -442,9 +450,9 @@ saib_ws_json_rx_builder(struct sai_plat_server *spm, const void *in, size_t len)
lws_struct_args_t a;
sai_cancel_t *can;
sai_rebuild_t *reb;
- char url[128], *p;
sai_task_t *task;
int n, m;
+ char *p;
/*
* use the schema name on the incoming JSON to decide what kind of
@@ -651,52 +659,13 @@ saib_ws_json_rx_builder(struct sai_plat_server *spm, const void *in, size_t len)
*/
ns->inp[n] = '\0';
- /* the git mirror is in the build host's /home/sai, NOT the
- * mounted overlayfs */
-
- m = lws_snprintf(ns->path, sizeof(ns->path),
- "%s%cgit-mirror", builder.home, csep);
-
- if (mkdir(ns->path, 0755) && errno != EEXIST)
- goto ebail;
-
- lws_strncpy(url, ns->git_repo_url, sizeof(url));
-
- /*
- * We want to convert the repo url to a unique subdir name
- * that's safe, like http___warmcat.com_repo_sai
- */
-
- lws_filename_purify_inplace(url);
- {
- char *q = url;
- while (*q) {
- if (*q == '/')
- *q = '_';
- if (*q == '.')
- *q = '_';
- q++;
- }
- }
- m += lws_snprintf(ns->path + m, sizeof(ns->path) - (unsigned int)m, "%c%s",
- csep, url);
- if (mkdir(ns->path, 0755) && errno != EEXIST) {
- lwsl_err("%s: unable to create %s\n", __func__,
- ns->path);
- goto bail;
- }
- lwsl_notice("%s: created %s\n", __func__, ns->path);
-
- /* manage the mirror dir using build host credentials */
-
- saib_set_ns_state(ns, NSSTATE_STARTING_MIRROR);
+ saib_set_ns_state(ns, NSSTATE_EXECUTING_STEPS);
ns->user_cancel = 0;
ns->spins = 0;
- ns->mirror_wait_budget = 120;
- if (saib_start_mirror(ns)) {
- lwsl_err("%s: saib_start_mirror failed\n", __func__);
+ if (saib_spawn_build(ns)) {
+ lwsl_err("%s: saib_spawn_build failed\n", __func__);
goto bail;
}
@@ -794,7 +763,7 @@ saib_ws_json_rx_builder(struct sai_plat_server *spm, const void *in, size_t len)
lws_start_foreach_dll(struct lws_dll2 *, d, sp->nspawn_owner.head) {
struct sai_nspawn *ns = lws_container_of(d, struct sai_nspawn, list);
- if (ns->state == NSSTATE_BUILD)
+ if (ns->state == NSSTATE_EXECUTING_STEPS)
any_busy = 1;
} lws_end_foreach_dll(d);
diff --git a/src/builder/scripts/git_helper.bat b/src/builder/scripts/git_helper.bat
new file mode 100644
index 0000000..ec2b3d8
--- /dev/null
+++ b/src/builder/scripts/git_helper.bat
@@ -0,0 +1,70 @@
+@echo on
+setlocal EnableDelayedExpansion
+echo "git_helper_bat: starting"
+set "OPERATION=%~1"
+echo "OPERATION: !OPERATION!"
+if /i "!OPERATION!"=="mirror" (
+ set "REMOTE_URL=%~2"
+ set "REF=%~3"
+ set "HASH=%~4"
+ set "MIRROR_PATH=%HOME%\\git-mirror\\%~5"
+ echo "REMOTE_URL: !REMOTE_URL!"
+ echo "REF: !REF!"
+ echo "HASH: !HASH!"
+ echo "MIRROR_PATH: !MIRROR_PATH!"
+ :lock_wait
+ mkdir "!MIRROR_PATH!.lock" 2>nul
+ if errorlevel 1 (
+ echo "git mirror locked, waiting..."
+ timeout /t 1 /nobreak > nul
+ goto :lock_wait
+ )
+ if exist "!MIRROR_PATH!\\.git" (
+ git -C "!MIRROR_PATH!" rev-parse -q --verify "ref-!HASH!" > nul 2> nul
+ if exist !MIRROR_PATH!\\.git if not errorlevel 1 (
+ rmdir "!MIRROR_PATH!.lock"
+ exit /b 0
+ )
+ )
+ if not exist "!MIRROR_PATH!\\." (
+ mkdir "!MIRROR_PATH!"
+ )
+ if not exist "!MIRROR_PATH!\\.git" (
+ git init --bare "!MIRROR_PATH!"
+ if errorlevel 1 (
+ rmdir "!MIRROR_PATH!.lock"
+ exit /b 1
+ )
+ )
+ set "REFSPEC=!REF!:ref-!HASH!"
+ echo "REFSPEC: !REFSPEC!"
+ git -C "!MIRROR_PATH!" fetch "!REMOTE_URL!" "!REFSPEC!" 2>&1
+ if !ERRORLEVEL! neq 0 (
+ echo "git fetch failed with errorlevel !ERRORLEVEL!"
+ rmdir "!MIRROR_PATH!.lock"
+ exit /b 1
+ )
+ rmdir "!MIRROR_PATH!.lock"
+ exit /b 0
+)
+if /i "!OPERATION!"=="checkout" (
+ set "MIRROR_PATH=%HOME%\\git-mirror\\%~2"
+ set "BUILD_DIR=%~3"
+ set "HASH=%~4"
+ echo "MIRROR_PATH: !MIRROR_PATH!"
+ echo "BUILD_DIR: !BUILD_DIR!"
+ echo "HASH: !HASH!"
+ if not exist "!BUILD_DIR!\\.git" (
+ if exist "!BUILD_DIR!\\" rmdir /s /q "!BUILD_DIR!"
+ mkdir "!BUILD_DIR!"
+ git -C "!BUILD_DIR!" init
+ if errorlevel 1 exit /b 1
+ )
+ git -C "!BUILD_DIR!" fetch "!MIRROR_PATH!" "ref-!HASH!"
+ if errorlevel 1 exit /b 2
+ git -C "!BUILD_DIR!" checkout -f "!HASH!"
+ if errorlevel 1 exit /b 1
+ echo ">>> Git helper script finished."
+ exit /b 0
+)
+exit /b 1
diff --git a/src/builder/scripts/git_helper.sh b/src/builder/scripts/git_helper.sh
new file mode 100644
index 0000000..99bd4e7
--- /dev/null
+++ b/src/builder/scripts/git_helper.sh
@@ -0,0 +1,55 @@
+#!/bin/bash
+export PATH=/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin:/sbin:/usr/sbin
+set -e
+echo "git_helper_sh: starting"
+OPERATION=$1
+shift
+if [ "$OPERATION" == "mirror" ]; then
+ REMOTE_URL=$1
+ REF=$2
+ HASH=$3
+ MIRROR_PATH="$HOME/git-mirror/$4"
+ for i in $(seq 1 60); do
+ if [ -d "$MIRROR_PATH/.git" ]; then
+ if git -C "$MIRROR_PATH" rev-parse -q --verify "ref-$HASH" > /dev/null; then
+ exit 0
+ fi
+ fi
+ if mkdir "$MIRROR_PATH.lock" 2>/dev/null; then
+ trap 'rm -rf "$MIRROR_PATH.lock"' EXIT
+ if [ -d "$MIRROR_PATH/.git" ]; then
+ if git -C "$MIRROR_PATH" rev-parse -q --verify "ref-$HASH" > /dev/null; then
+ exit 0
+ fi
+ fi
+ mkdir -p "$MIRROR_PATH"
+ if [ ! -d "$MIRROR_PATH/.git" ]; then
+ git init --bare "$MIRROR_PATH"
+ fi
+ REFSPEC="$REF:ref-$HASH"
+ git -C "$MIRROR_PATH" fetch "$REMOTE_URL" "+$REFSPEC"
+ exit 0
+ fi
+ echo "git mirror locked, waiting..."
+ sleep 1
+ done
+ exit 1
+elif [ "$OPERATION" == "checkout" ]; then
+ MIRROR_PATH="$HOME/git-mirror/$1"
+ BUILD_DIR=$2
+ HASH=$3
+ if [ ! -d "$BUILD_DIR/.git" ]; then
+ rm -rf "$BUILD_DIR"
+ mkdir -p "$BUILD_DIR"
+ git -C "$BUILD_DIR" init
+ fi
+ if ! git -C "$BUILD_DIR" fetch "$MIRROR_PATH" "ref-$HASH"; then
+ exit 2
+ fi
+ git -C "$BUILD_DIR" checkout -f "$HASH"
+ git -C "$BUILD_DIR" clean -fdx
+else
+ exit 1
+fi
+echo ">>> Git helper script finished."
+exit 0
diff --git a/src/common/include/private.h b/src/common/include/private.h
index b4afe7e..96ce6ed 100644
--- a/src/common/include/private.h
+++ b/src/common/include/private.h
@@ -116,6 +116,7 @@ typedef struct {
char uuid[65];
char builder_name[96];
char cpack[128];
+ char steps[4096];
struct lwsac *ac_task_container;
@@ -129,8 +130,6 @@ typedef struct {
uint64_t duration;
int state;
int uid;
- int build_step;
- int build_step_count;
char told_ongoing;
} sai_task_t;
@@ -194,15 +193,13 @@ struct sai_nspawn {
int retcode;
int instance_idx;
- int mirror_wait_budget;
- int parallel;
uint8_t spins;
uint8_t state; /* NSSTATE_ */
uint8_t stdcount;
uint8_t term_budget;
- int build_step;
+ int current_step;
int build_step_count;
uint8_t finished_when_logs_drained:1;
@@ -445,6 +442,8 @@ typedef struct sai_plat {
int instances;
int ongoing;
+ char windows;
+
int index; /* used to create unique build dir path */
} sai_plat_t;
@@ -490,7 +489,6 @@ typedef struct sai_build_metric {
char builder_name[96];
char project_name[96];
char ref[96];
- int parallel;
uint64_t us_cpu_user;
uint64_t us_cpu_sys;
uint64_t wallclock_us;
@@ -505,7 +503,6 @@ typedef struct sai_build_metric_db {
char builder_name[96];
char project_name[96];
char ref[96];
- int parallel;
uint64_t us_cpu_user;
uint64_t us_cpu_sys;
uint64_t wallclock_us;
@@ -519,12 +516,13 @@ extern const lws_struct_map_t
lsm_schema_sq3_map_event[],
lsm_schema_json_map_log[],
lsm_schema_sq3_map_log[],
+ lsm_schema_sq3_map_plat[1],
lsm_schema_json_map_artifact[1],
lsm_schema_sq3_map_artifact[1],
lsm_schema_map_ta[1],
lsm_schema_map_plat_simple[1],
- lsm_event[9],
- lsm_task[23],
+ lsm_event[10],
+ lsm_task[22],
lsm_log[7],
lsm_artifact[8],
lsm_plat_list[1],
@@ -540,9 +538,9 @@ extern const lws_struct_map_t
lsm_schema_build_metric[1],
lsm_schema_sq3_map_build_metric[1]
;
-extern const lws_struct_map_t lsm_build_metric[10];
-extern const lws_struct_map_t lsm_plat[8];
-extern const lws_struct_map_t lsm_plat_for_json[12];
+extern const lws_struct_map_t lsm_build_metric[9];
+extern const lws_struct_map_t lsm_plat[9];
+extern const lws_struct_map_t lsm_plat_for_json[13];
extern const lws_ss_info_t ssi_said_logproxy;
extern struct lws_ss_handle *ssh[3];
diff --git a/src/common/struct-metadata.c b/src/common/struct-metadata.c
index 21caec9..861b11f 100644
--- a/src/common/struct-metadata.c
+++ b/src/common/struct-metadata.c
@@ -45,7 +45,6 @@ const lws_struct_map_t lsm_build_metric[] = {
LSM_CARRAY (sai_build_metric_t, builder_name, "builder_name"),
LSM_CARRAY (sai_build_metric_t, project_name, "project_name"),
LSM_CARRAY (sai_build_metric_t, ref, "ref"),
- LSM_SIGNED (sai_build_metric_t, parallel, "parallel"),
LSM_UNSIGNED (sai_build_metric_t, us_cpu_user, "us_cpu_user"),
LSM_UNSIGNED (sai_build_metric_t, us_cpu_sys, "us_cpu_sys"),
LSM_UNSIGNED (sai_build_metric_t, wallclock_us, "wallclock_us"),
@@ -63,7 +62,6 @@ const lws_struct_map_t lsm_sq3_build_metric[] = {
LSM_CARRAY (sai_build_metric_db_t, builder_name, "builder_name"),
LSM_CARRAY (sai_build_metric_db_t, project_name, "project_name"),
LSM_CARRAY (sai_build_metric_db_t, ref, "ref"),
- LSM_SIGNED (sai_build_metric_db_t, parallel, "parallel"),
LSM_UNSIGNED (sai_build_metric_db_t, us_cpu_user, "us_cpu_user"),
LSM_UNSIGNED (sai_build_metric_db_t, us_cpu_sys, "us_cpu_sys"),
LSM_UNSIGNED (sai_build_metric_db_t, wallclock_us, "wallclock_us"),
@@ -84,6 +82,7 @@ const lws_struct_map_t lsm_plat[] = { /* !!! keep extern length in common/includ
LSM_CARRAY (sai_plat_t, peer_ip, "peer_ip"),
LSM_CARRAY (sai_plat_t, sai_hash, "sai_hash"),
LSM_CARRAY (sai_plat_t, lws_hash, "lws_hash"),
+ LSM_UNSIGNED (sai_plat_t, windows, "windows"),
};
// This is the map for serializing to JSON
@@ -100,6 +99,7 @@ const lws_struct_map_t lsm_plat_for_json[] = {
LSM_CARRAY(sai_plat_t, peer_ip, "peer_ip"),
LSM_CARRAY(sai_plat_t, sai_hash, "sai_hash"),
LSM_CARRAY(sai_plat_t, lws_hash, "lws_hash"),
+ LSM_UNSIGNED(sai_plat_t, windows, "windows"),
};
const lws_struct_map_t lsm_schema_map_plat_simple[] = {
@@ -124,6 +124,7 @@ const lws_struct_map_t lsm_schema_sq3_map_plat[] = {
const lws_struct_map_t lsm_event[] = {
LSM_UNSIGNED (sai_event_t, uid, "uid"),
LSM_CARRAY (sai_event_t, repo_name, "repo_name"),
+ LSM_CARRAY (sai_event_t, repo_fetchurl, "repo_fetchurl"),
LSM_CARRAY (sai_event_t, ref, "ref"),
LSM_CARRAY (sai_event_t, hash, "hash"),
LSM_CARRAY (sai_event_t, uuid, "uuid"),
@@ -171,8 +172,7 @@ const lws_struct_map_t lsm_task[] = {
LSM_STRING_PTR (sai_task_t, git_ref, "git_ref"),
LSM_STRING_PTR (sai_task_t, git_hash, "git_hash"),
LSM_STRING_PTR (sai_task_t, git_repo_url, "git_repo_url"),
- LSM_SIGNED (sai_task_t, build_step, "build_step"),
- LSM_SIGNED (sai_task_t, build_step_count, "build_step_count"),
+ LSM_CARRAY (sai_task_t, steps, "steps"),
};
const lws_struct_map_t lsm_schema_json_map_task[] = {
diff --git a/src/server/s-metrics-db.c b/src/server/s-metrics-db.c
index 685c32e..d338b96 100644
--- a/src/server/s-metrics-db.c
+++ b/src/server/s-metrics-db.c
@@ -139,7 +139,7 @@ sais_metrics_db_add(struct vhd *vhd, const struct sai_build_metric *m)
lws_strncpy(dbm.builder_name, m->builder_name, sizeof(dbm.builder_name));
lws_strncpy(dbm.project_name, m->project_name, sizeof(dbm.project_name));
lws_strncpy(dbm.ref, m->ref, sizeof(dbm.ref));
- dbm.parallel = m->parallel;
+ //dbm.parallel = m->parallel;
dbm.us_cpu_user = m->us_cpu_user;
dbm.us_cpu_sys = m->us_cpu_sys;
dbm.wallclock_us = m->wallclock_us;
diff --git a/src/server/s-notification.c b/src/server/s-notification.c
index f4c3930..0d6aeae 100644
--- a/src/server/s-notification.c
+++ b/src/server/s-notification.c
@@ -504,20 +504,6 @@ sai_saifile_lejp_cb(struct lejp_ctx *ctx, char reason)
return -1;
}
- pss->sn.t.build_step = 0;
- pss->sn.t.build_step_count = 0;
- if (pss->sn.t.build[0]) {
- const char *p = pss->sn.t.build;
- pss->sn.t.build_step_count = 1;
- while ((p = strchr(p, '\n'))) {
- pss->sn.t.build_step_count++;
- p++;
- }
- }
-
- lwsl_notice("Task %s: step_count %d\n",
- pss->sn.t.taskname,
- pss->sn.t.build_step_count);
/*
* Prepare a struct of the task object...
diff --git a/src/server/s-private.h b/src/server/s-private.h
index 714bbb1..374e969 100644
--- a/src/server/s-private.h
+++ b/src/server/s-private.h
@@ -229,9 +229,6 @@ struct vhd {
sais_t server;
};
-extern const lws_struct_map_t lsm_plat[];
-extern const lws_struct_map_t lsm_schema_sq3_map_plat[];
-
extern struct lws_context *
sai_lws_context_from_json(const char *config_dir,
struct lws_context_creation_info *info,
diff --git a/src/server/s-task.c b/src/server/s-task.c
index 982855e..f202d0d 100644
--- a/src/server/s-task.c
+++ b/src/server/s-task.c
@@ -773,24 +773,6 @@ sais_task_reset(struct vhd *vhd, const char *task_uuid)
sais_event_db_close(vhd, &pdb);
- if (sais_event_db_ensure_open(vhd, event_uuid, 0, &pdb)) {
- lwsl_err("%s: unable to open event-specific database for reset\n",
- __func__);
- return SAI_DB_RESULT_ERROR;
- }
- lws_snprintf(cmd, sizeof(cmd), "update tasks set build_step=0 where uuid='%s'",
- esc);
- ret = sqlite3_exec(pdb, cmd, NULL, NULL, NULL);
- if (ret != SQLITE_OK) {
- sais_event_db_close(vhd, &pdb);
- if (ret == SQLITE_BUSY)
- return SAI_DB_RESULT_BUSY;
- lwsl_err("%s: %s: %s: fail\n", __func__, cmd,
- sqlite3_errmsg(pdb));
- return SAI_DB_RESULT_ERROR;
- }
- sais_event_db_close(vhd, &pdb);
-
sais_set_task_state(vhd, NULL, NULL, task_uuid, SAIES_WAITING, 1, 1);
sais_task_cancel(vhd, task_uuid);
@@ -870,8 +852,53 @@ sais_allocate_task(struct vhd *vhd, struct pss *pss, sai_plat_t *cb,
task->repo_name = task->one_event->repo_name;
task->git_ref = task->one_event->ref;
task->git_hash = task->one_event->hash;
+ task->git_repo_url = task->one_event->repo_fetchurl;
task->ac_task_container = pss->a.ac;
+ if (!task->git_repo_url || !task->git_repo_url[0]) {
+ lwsl_err("%s: task %s has no repo_fetchurl, failing\n",
+ __func__, task->uuid);
+ sais_set_task_state(vhd, NULL, NULL, task->uuid, SAIES_FAIL, 0, 0);
+ free(task);
+ lwsac_free(&pss->ac_alloc_task);
+
+ return 1; /* try again for another task */
+ }
+
+ lwsl_notice("%s: windows: %d\n", __func__, cb->windows);
+
+ char url[128], mirror_path[256];
+
+ lws_strncpy(url, task->one_event->repo_fetchurl, sizeof(url));
+ lws_filename_purify_inplace(url);
+ char *q = url;
+ while (*q) {
+ if (*q == '/')
+ *q = '_';
+ if (*q == '.')
+ *q = '_';
+ q++;
+ }
+
+ lws_snprintf(mirror_path, sizeof(mirror_path), "%s", url);
+
+ if (cb->windows)
+ lws_snprintf(task->steps, sizeof(task->steps),
+ "git_helper.bat mirror \"%s\" %s %s %s\n"
+ "git_helper.bat checkout \"%s\" \"build/%s\" %s\n"
+ "%s",
+ task->git_repo_url, task->git_ref, task->git_hash, mirror_path,
+ mirror_path, task->repo_name, task->git_hash,
+ task->build);
+ else
+ lws_snprintf(task->steps, sizeof(task->steps),
+ "git_helper.sh mirror \"%s\" %s %s %s\n"
+ "git_helper.sh checkout \"%s\" \"build/%s\" %s\n"
+ "%s",
+ task->git_repo_url, task->git_ref, task->git_hash, mirror_path,
+ mirror_path, task->repo_name, task->git_hash,
+ task->build);
+
/*
* add to server's estimate of builder's ongoing tasks...
*/
diff --git a/src/server/s-ws-builder.c b/src/server/s-ws-builder.c
index 1aa7411..391e262 100644
--- a/src/server/s-ws-builder.c
+++ b/src/server/s-ws-builder.c
@@ -272,7 +272,7 @@ sais_log_to_db(struct vhd *vhd, sai_log_t *log)
if (log->channel == 3 && log->log) { /* control channel */
int step;
- if (sscanf(log->log, " Step %d OK:", &step) == 1) {
+ if (sscanf(log->log, " Step %d:", &step) == 1) {
char event_uuid[33];
sqlite3 *pdb = NULL;
char q[256], esc_uuid[129];
@@ -602,6 +602,7 @@ handle:
sizeof(live_cb->sai_hash));
lws_strncpy(live_cb->lws_hash, build->lws_hash,
sizeof(live_cb->lws_hash));
+ live_cb->windows = build->windows;
live_cb->online = 1;
} else {
/* New builder, create a deep-copied, malloc'd object */
@@ -623,6 +624,7 @@ handle:
sizeof(live_cb->sai_hash));
lws_strncpy(live_cb->lws_hash, build->lws_hash,
sizeof(live_cb->lws_hash));
+ live_cb->windows = build->windows;
live_cb->wsi = pss->wsi;
live_cb->online = 1;
lws_strncpy(live_cb->peer_ip, pss->peer_ip, sizeof(live_cb->peer_ip));
@@ -697,6 +699,9 @@ bail:
/*
* We have reached the end of the logs for this task
*/
+
+ sais_dump_logs_to_db(&vhd->sul_logcache);
+
lwsl_info("%s: log->finished says 0x%x, dur %lluus\n",
__func__, log->finished, (unsigned long long)(
log->timestamp - pss->first_log_timestamp));
![[]](/git/avatar/c50933ca2aa61e0fe2c43d46bb6b59cb_avatar){kind=avatar}
Andy Green <andy@warmcat.com> 2025-08-26 11:29 UTC