Project homepage Mailing List  Warmcat.com  API Docs  Github Mirror 
    npro  
 Modern all-safe Rust Network Protocol library supporting h1, h2, h3, ws, wt sans-IO and with socket IO + tls
git clone https://npro.rs/repo/npro
Fulltext search
 
root / src / builder / b-ws-server.c
Author[]Andy Green <andy@warmcat.com> 2026-10-03 19:36 UTC
Committer[]Andy Green <andy@warmcat.com> 2026-10-04 20:05 UTC
Tree96b5e44d633614f05493a9552555573cebcc1f88   Raw Patch
 
builder: git mirror repairs itself instead of failing every task 
builder: git mirror repairs itself instead of failing every task

A mirror ref left pointing at a missing object made every later fetch
into that builder's mirror die with "fatal: bad object", and the
fast path accepted the dangling ref as present, so every task on the
builder failed until someone deleted the mirror by hand.

Under the mirror lock, a failed fetch now drops refs whose object is
missing and retries.  If that still fails, fsck decides whether the
mirror or the remote is at fault: a remote outage just fails the task,
a damaged mirror is rebuilt alongside and swapped in.  The old mirror is
moved aside for an hour, since other tasks may still be checking out
from it, and the refs it can still serve are carried over so their
checkouts keep working.  Checkout retries briefly across the swap.

The fast path now needs the ref's object, not just the ref.

A killed lock holder used to wedge the mirror for every later task.  On
Unix the lock is a symlink naming the holder's pid, broken when that
process is gone; taking and breaking both happen inside a gate dir so a
breaker can't remove a lock a new holder just took.  Windows has no
cheap liveness check, so a lock older than an hour is broken there.
Waiters no longer give up after 60s while a live holder is still doing
a long first clone; the step timeout bounds them instead.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

diff --git a/README.md b/README.md
index 9d5211c..dd00400 100644
--- a/README.md
+++ b/README.md
@@ -442,6 +442,12 @@ Note: on redhat type distros like Fedora / Rocky, use `-gnobody`
  - /home/sai/
   - git-mirror/
    - `remote git url`_`project name` -- individual git mirrors
+   - `mirror`.lck (`.lock` on Windows) -- held by the one task updating the
+     mirror; a lock left by a killed task is broken (Windows: after an hour)
+   - `mirror`.gate -- briefly held while taking or breaking the lock
+   - `mirror`.new, `mirror`.old -- if a fetch fails and the mirror itself is
+     found damaged, it is rebuilt in .new and swapped in; .old is kept for an
+     hour for tasks still checking out from it
   - jobs/
    - `server hostname`-`platform name`-`instance index`/
     - `project_name`/  - checkouts and builds occur in here
diff --git a/src/builder/b-task.c b/src/builder/b-task.c
index 60b194d..e51ffae 100644
--- a/src/builder/b-task.c
+++ b/src/builder/b-task.c
@@ -32,8 +32,115 @@
 const char *git_helper_sh =
 	"#!/usr/bin/env bash\n"
 	"export PATH=/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin:/sbin:/usr/sbin\n"
-	"set -e\n"
 	"echo \"git_helper_sh: starting\"\n"
+	"\n"
+	"# The mirror is shared by every task on the builder, and many of them may be\n"
+	"# starting at once.  Only the holder of $LOCK changes it, so a damaged mirror is\n"
+	"# repaired by one task while the rest wait for it.  The lock is a symlink naming\n"
+	"# the holder's pid, so a holder that was killed can be detected and its lock\n"
+	"# broken, instead of wedging every later task on the builder.\n"
+	"\n"
+	"mirror_has_ref() {\n"
+	"    git -C \"$1\" cat-file -e \"refs/heads/ref-$HASH\" 2>/dev/null\n"
+	"}\n"
+	"\n"
+	"mirror_fetch() {\n"
+	"    git -C \"$1\" fetch -q \"$REMOTE_URL\" \"+$REF:refs/heads/ref-$HASH\"\n"
+	"}\n"
+	"\n"
+	"# Taking the lock and breaking a stale one both happen inside $GATE, so a\n"
+	"# breaker can't remove a lock a new holder took after it looked at the old one\n"
+	"\n"
+	"gate_take() {\n"
+	"    mkdir \"$GATE\" 2>/dev/null && return 0\n"
+	"    # a task killed inside the gate must not wedge us\n"
+	"    if [ -n \"$(find \"$GATE\" -maxdepth 0 -mmin +1 2>/dev/null)\" ]; then\n"
+	"        rmdir \"$GATE\" 2>/dev/null\n"
+	"    fi\n"
+	"\n"
+	"    return 1\n"
+	"}\n"
+	"\n"
+	"lock_take() {\n"
+	"    local r=1\n"
+	"\n"
+	"    gate_take || return 1\n"
+	"    ln -s \"$$\" \"$LOCK\" 2>/dev/null && r=0\n"
+	"    rmdir \"$GATE\"\n"
+	"\n"
+	"    return $r\n"
+	"}\n"
+	"\n"
+	"lock_release() {\n"
+	"    if [ \"$(readlink \"$LOCK\" 2>/dev/null)\" = \"$$\" ]; then\n"
+	"        rm -f \"$LOCK\"\n"
+	"    fi\n"
+	"}\n"
+	"\n"
+	"# True if $LOCK names a holder that is no longer running.  Outside $GATE this\n"
+	"# is only a hint, since the holder may release it and exit while we look.\n"
+	"\n"
+	"lock_stale() {\n"
+	"    STALE_PID=$(readlink \"$LOCK\" 2>/dev/null) || return 1\n"
+	"    case \"$STALE_PID\" in\n"
+	"    ''|*[!0-9]*) return 0 ;;\n"
+	"    esac\n"
+	"    kill -0 \"$STALE_PID\" 2>/dev/null || return 0\n"
+	"    # the pid may have been reused since the holder died, or be its zombie\n"
+	"    command -v ps >/dev/null 2>&1 || return 1\n"
+	"    ps -o args= -p \"$STALE_PID\" 2>/dev/null | grep -q git_helper && return 1\n"
+	"\n"
+	"    return 0\n"
+	"}\n"
+	"\n"
+	"lock_break() {\n"
+	"    gate_take || return 0\n"
+	"    # inside the gate the lock can be released, but not retaken\n"
+	"    if lock_stale && [ \"$(readlink \"$LOCK\" 2>/dev/null)\" = \"$STALE_PID\" ]; then\n"
+	"        echo \"git_helper_sh: breaking stale mirror lock of pid $STALE_PID\"\n"
+	"        rm -f \"$LOCK\"\n"
+	"    fi\n"
+	"    rmdir \"$GATE\"\n"
+	"\n"
+	"    return 0\n"
+	"}\n"
+	"\n"
+	"# Refs whose object is missing make every later fetch into the mirror fail.\n"
+	"# True if any were dropped.\n"
+	"\n"
+	"drop_broken_refs() {\n"
+	"    local oid ref dropped=1\n"
+	"\n"
+	"    while read -r oid ref; do\n"
+	"        if ! git -C \"$MIRROR_PATH\" cat-file -e \"$oid\" 2>/dev/null; then\n"
+	"            echo \"git_helper_sh: dropping $ref, its object is missing\"\n"
+	"            git -C \"$MIRROR_PATH\" update-ref -d \"$ref\" && dropped=0\n"
+	"        fi\n"
+	"    done < <(git -C \"$MIRROR_PATH\" for-each-ref --format='%(objectname) %(refname)' refs/heads/)\n"
+	"\n"
+	"    return $dropped\n"
+	"}\n"
+	"\n"
+	"# Fetch into a fresh mirror and swap it in.  Tasks already past this step may\n"
+	"# still be checking out from the old one, so it is moved aside and only removed\n"
+	"# once it is an hour old, and the refs it can still serve are carried over.\n"
+	"\n"
+	"mirror_rebuild() {\n"
+	"    echo \"git_helper_sh: mirror is damaged, rebuilding it\"\n"
+	"    git init -q --bare \"$NEW\" || return 1\n"
+	"    if ! mirror_fetch \"$NEW\"; then\n"
+	"        rm -rf \"$NEW\"\n"
+	"        return 1\n"
+	"    fi\n"
+	"    if ! git -C \"$NEW\" fetch -q \"$MIRROR_PATH\" \"refs/heads/*:refs/heads/*\" >/dev/null 2>&1; then\n"
+	"        echo \"git_helper_sh: not every old mirror ref could be carried over\"\n"
+	"    fi\n"
+	"    rm -rf \"$OLD\"\n"
+	"    mv \"$MIRROR_PATH\" \"$OLD\" || return 1\n"
+	"    touch \"$OLD\"\n"
+	"    mv \"$NEW\" \"$MIRROR_PATH\"\n"
+	"}\n"
+	"\n"
 	"OPERATION=$1\n"
 	"shift\n"
 	"if [ \"$OPERATION\" == \"mirror\" ]; then\n"
@@ -41,46 +148,72 @@ const char *git_helper_sh =
 	"    REF=$2\n"
 	"    HASH=$3\n"
 	"    MIRROR_PATH=\"$HOME/git-mirror/$4\"\n"
-	"    mkdir -p \"$HOME/git-mirror\"\n"
-	"    for i in $(seq 1 60); do\n"
-	"        if [ -d \"$MIRROR_PATH\" ]; then\n"
-	"            if git -C \"$MIRROR_PATH\" rev-parse -q --verify \"ref-$HASH\" > /dev/null 2>&1; then\n"
-	"                exit 0\n"
-	"            fi\n"
-	"        fi\n"
-	"        if mkdir \"$MIRROR_PATH.lock\" 2>/dev/null; then\n"
-	"            trap 'rm -rf \"$MIRROR_PATH.lock\"' EXIT\n"
-	"            if [ -d \"$MIRROR_PATH\" ]; then\n"
-	"                if git -C \"$MIRROR_PATH\" rev-parse -q --verify \"ref-$HASH\" > /dev/null 2>&1; then\n"
-	"                    exit 0\n"
-	"                fi\n"
-	"            fi\n"
-	"            mkdir -p \"$MIRROR_PATH\"\n"
-	"            if [ ! -f \"$MIRROR_PATH/HEAD\" ]; then\n"
-	"                git init --bare \"$MIRROR_PATH\"\n"
+	"    LOCK=\"$MIRROR_PATH.lck\"\n"
+	"    GATE=\"$MIRROR_PATH.gate\"\n"
+	"    NEW=\"$MIRROR_PATH.new\"\n"
+	"    OLD=\"$MIRROR_PATH.old\"\n"
+	"    mkdir -p \"$HOME/git-mirror\" || exit 1\n"
+	"    if mirror_has_ref \"$MIRROR_PATH\"; then\n"
+	"        exit 0\n"
+	"    fi\n"
+	"    WAITED=0\n"
+	"    while ! lock_take; do\n"
+	"        if lock_stale; then\n"
+	"            lock_break\n"
+	"        else\n"
+	"            if [ $((WAITED % 60)) -eq 0 ]; then\n"
+	"                echo \"git mirror locked by pid $(readlink \"$LOCK\" 2>/dev/null), waiting...\"\n"
 	"            fi\n"
-	"            REFSPEC=\"$REF:ref-$HASH\"\n"
-	"            git -C \"$MIRROR_PATH\" fetch -q \"$REMOTE_URL\" \"+$REFSPEC\"\n"
-	"            exit 0\n"
+	"            WAITED=$((WAITED + 1))\n"
 	"        fi\n"
-	"        echo \"git mirror locked, waiting...\"\n"
 	"        sleep 1\n"
 	"    done\n"
-	"    exit 1\n"
+	"    trap lock_release EXIT\n"
+	"    trap 'exit 1' HUP INT TERM\n"
+	"    # another task may have fetched it while we waited\n"
+	"    if mirror_has_ref \"$MIRROR_PATH\"; then\n"
+	"        exit 0\n"
+	"    fi\n"
+	"    if [ -n \"$(find \"$OLD\" -maxdepth 0 -mmin +60 2>/dev/null)\" ]; then\n"
+	"        rm -rf \"$OLD\"\n"
+	"    fi\n"
+	"    rm -rf \"$NEW\"\n"
+	"    if [ ! -f \"$MIRROR_PATH/HEAD\" ]; then\n"
+	"        rm -rf \"$MIRROR_PATH\"\n"
+	"        git init -q --bare \"$MIRROR_PATH\" || exit 1\n"
+	"    fi\n"
+	"    if mirror_fetch \"$MIRROR_PATH\"; then\n"
+	"        exit 0\n"
+	"    fi\n"
+	"    echo \"git_helper_sh: fetch failed, checking the mirror\"\n"
+	"    if drop_broken_refs && mirror_fetch \"$MIRROR_PATH\"; then\n"
+	"        exit 0\n"
+	"    fi\n"
+	"    if git -C \"$MIRROR_PATH\" fsck --connectivity-only --no-dangling >/dev/null 2>&1; then\n"
+	"        echo \"git_helper_sh: mirror is intact, the fetch from $REMOTE_URL is what fails\"\n"
+	"        exit 1\n"
+	"    fi\n"
+	"    mirror_rebuild || exit 1\n"
 	"elif [ \"$OPERATION\" == \"checkout\" ]; then\n"
 	"    MIRROR_PATH=\"$HOME/git-mirror/$1\"\n"
 	"    BUILD_DIR=$2\n"
 	"    HASH=$3\n"
 	"    if [ ! -d \"$BUILD_DIR/.git\" ]; then\n"
 	"        rm -rf \"$BUILD_DIR\"\n"
-	"        mkdir -p \"$BUILD_DIR\"\n"
-	"        git -C \"$BUILD_DIR\" init\n"
-	"    fi\n"
-	"    if ! git -C \"$BUILD_DIR\" fetch \"$MIRROR_PATH\" \"ref-$HASH\"; then\n"
-	"        exit 2\n"
+	"        mkdir -p \"$BUILD_DIR\" || exit 1\n"
+	"        git -C \"$BUILD_DIR\" init || exit 1\n"
 	"    fi\n"
-	"    git -C \"$BUILD_DIR\" checkout -f \"$HASH\"\n"
-	"    git -C \"$BUILD_DIR\" clean -fdx\n"
+	"    # the mirror may be mid-swap after a rebuild\n"
+	"    TRIES=0\n"
+	"    while ! git -C \"$BUILD_DIR\" fetch \"$MIRROR_PATH\" \"ref-$HASH\"; do\n"
+	"        TRIES=$((TRIES + 1))\n"
+	"        if [ $TRIES -ge 5 ]; then\n"
+	"            exit 2\n"
+	"        fi\n"
+	"        sleep 2\n"
+	"    done\n"
+	"    git -C \"$BUILD_DIR\" checkout -f \"$HASH\" || exit 1\n"
+	"    git -C \"$BUILD_DIR\" clean -fdx || exit 1\n"
 	"else\n"
 	"    exit 1\n"
 	"fi\n"
@@ -95,75 +228,210 @@ const char *git_helper_bat =
 	"echo \"git_helper_bat: starting\"\n"
 	"set \"OPERATION=%~1\"\n"
 	"echo \"OPERATION: !OPERATION!\"\n"
-	"if /i \"!OPERATION!\"==\"mirror\" (\n"
-	"    set \"REMOTE_URL=%~2\"\n"
-	"    set \"REF=%~3\"\n"
-	"    set \"HASH=%~4\"\n"
-	"    set \"MIRROR_PATH=%HOME%\\\\git-mirror\\\\%~5\"\n"
-	"    echo \"REMOTE_URL: !REMOTE_URL!\"\n"
-	"    echo \"REF: !REF!\"\n"
-	"    echo \"HASH: !HASH!\"\n"
-	"    echo \"MIRROR_PATH: !MIRROR_PATH!\"\n"
-	"    :lock_wait\n"
-	"    mkdir \"!MIRROR_PATH!.lock\"\n"
+	"if /i \"!OPERATION!\"==\"mirror\" goto :mirror\n"
+	"if /i \"!OPERATION!\"==\"checkout\" goto :checkout\n"
+	"exit /b 1\n"
+	"\n"
+	"rem The mirror is shared by every task on the builder, and many of them may be\n"
+	"rem starting at once.  Only the holder of the lock dir changes it, so a damaged\n"
+	"rem mirror is repaired by one task while the rest wait for it.  A holder that\n"
+	"rem was killed can't remove its lock, so a lock older than an hour is broken.\n"
+	"\n"
+	":mirror\n"
+	"set \"REMOTE_URL=%~2\"\n"
+	"set \"REF=%~3\"\n"
+	"set \"HASH=%~4\"\n"
+	"set \"MIRROR_PATH=%HOME%\\git-mirror\\%~5\"\n"
+	"set \"LOCK=!MIRROR_PATH!.lock\"\n"
+	"set \"GATE=!MIRROR_PATH!.gate\"\n"
+	"set \"NEW=!MIRROR_PATH!.new\"\n"
+	"set \"OLD=!MIRROR_PATH!.old\"\n"
+	"echo \"REMOTE_URL: !REMOTE_URL!\"\n"
+	"echo \"REF: !REF!\"\n"
+	"echo \"HASH: !HASH!\"\n"
+	"echo \"MIRROR_PATH: !MIRROR_PATH!\"\n"
+	"if not exist \"%HOME%\\git-mirror\\\" mkdir \"%HOME%\\git-mirror\"\n"
+	"call :mirror_has_ref \"!MIRROR_PATH!\"\n"
+	"if not errorlevel 1 exit /b 0\n"
+	"set \"WAITED=0\"\n"
+	":lock_wait\n"
+	"call :lock_take\n"
+	"if not errorlevel 1 goto :locked\n"
+	"set /a \"SAID=WAITED %% 60\"\n"
+	"if !SAID! equ 0 (\n"
+	"    echo \"git mirror locked, waiting...\"\n"
+	"    call :lock_break\n"
+	")\n"
+	"set /a \"WAITED+=1\"\n"
+	"ping -n 2 127.0.0.1 >nul\n"
+	"goto :lock_wait\n"
+	"\n"
+	":locked\n"
+	"rem another task may have fetched it while we waited\n"
+	"call :mirror_has_ref \"!MIRROR_PATH!\"\n"
+	"if not errorlevel 1 goto :unlock_ok\n"
+	"if exist \"!OLD!\\\" (\n"
+	"    call :older_than \"!OLD!\" 60\n"
+	"    if not errorlevel 1 call :rmtree \"!OLD!\"\n"
+	")\n"
+	"call :rmtree \"!NEW!\"\n"
+	"if not exist \"!MIRROR_PATH!\\HEAD\" (\n"
+	"    call :rmtree \"!MIRROR_PATH!\"\n"
+	"    git init -q --bare \"!MIRROR_PATH!\"\n"
+	"    if errorlevel 1 goto :unlock_fail\n"
+	")\n"
+	"call :mirror_fetch \"!MIRROR_PATH!\"\n"
+	"if not errorlevel 1 goto :unlock_ok\n"
+	"echo \"git_helper_bat: fetch failed, checking the mirror\"\n"
+	"call :drop_broken_refs\n"
+	"if not errorlevel 1 (\n"
+	"    call :mirror_fetch \"!MIRROR_PATH!\"\n"
+	"    if not errorlevel 1 goto :unlock_ok\n"
+	")\n"
+	"git -C \"!MIRROR_PATH!\" fsck --connectivity-only --no-dangling >nul 2>&1\n"
+	"if not errorlevel 1 (\n"
+	"    echo \"git_helper_bat: mirror is intact, the fetch from !REMOTE_URL! is what fails\"\n"
+	"    goto :unlock_fail\n"
+	")\n"
+	"call :mirror_rebuild\n"
+	"if errorlevel 1 goto :unlock_fail\n"
+	":unlock_ok\n"
+	"rmdir \"!LOCK!\" 2>nul\n"
+	"exit /b 0\n"
+	":unlock_fail\n"
+	"rmdir \"!LOCK!\" 2>nul\n"
+	"exit /b 1\n"
+	"\n"
+	":mirror_has_ref\n"
+	"git -C \"%~1\" cat-file -e \"refs/heads/ref-!HASH!\" >nul 2>&1\n"
+	"exit /b\n"
+	"\n"
+	":mirror_fetch\n"
+	"git -C \"%~1\" fetch -q \"!REMOTE_URL!\" \"+!REF!:refs/heads/ref-!HASH!\" 2>&1\n"
+	"exit /b\n"
+	"\n"
+	"rem Taking the lock and breaking a stale one both happen inside the gate dir,\n"
+	"rem so a breaker can't remove a lock a new holder took after it looked\n"
+	"\n"
+	":lock_take\n"
+	"mkdir \"!GATE!\" 2>nul || goto :lock_take_busy\n"
+	"set \"TOOK=1\"\n"
+	"mkdir \"!LOCK!\" 2>nul && set \"TOOK=0\"\n"
+	"rmdir \"!GATE!\"\n"
+	"exit /b !TOOK!\n"
+	":lock_take_busy\n"
+	"rem a task killed inside the gate must not wedge us\n"
+	"call :older_than \"!GATE!\" 1\n"
+	"if not errorlevel 1 rmdir \"!GATE!\" 2>nul\n"
+	"exit /b 1\n"
+	"\n"
+	":lock_break\n"
+	"if not exist \"!LOCK!\\\" exit /b 0\n"
+	"call :older_than \"!LOCK!\" 60\n"
+	"if errorlevel 1 exit /b 0\n"
+	"mkdir \"!GATE!\" 2>nul || exit /b 0\n"
+	"call :older_than \"!LOCK!\" 60\n"
+	"if not errorlevel 1 (\n"
+	"    echo \"git_helper_bat: breaking stale mirror lock\"\n"
+	"    rmdir \"!LOCK!\" 2>nul\n"
+	")\n"
+	"rmdir \"!GATE!\"\n"
+	"exit /b 0\n"
+	"\n"
+	"rem Refs whose object is missing make every later fetch into the mirror fail.\n"
+	"rem Succeeds if any were dropped.\n"
+	"\n"
+	":drop_broken_refs\n"
+	"set \"DROPPED=1\"\n"
+	"for /f \"tokens=1,2\" %%a in ('git -C \"%MIRROR_PATH%\" for-each-ref \"--format=%%(objectname) %%(refname)\" refs/heads/') do (\n"
+	"    git -C \"!MIRROR_PATH!\" cat-file -e %%a >nul 2>&1\n"
 	"    if errorlevel 1 (\n"
-	"        echo \"git mirror locked, waiting...\"\n"
-	"        ping -n 2 127.0.0.1 >nul\n"
-	"        goto :lock_wait\n"
-	"    )\n"
-	"    if exist \"!MIRROR_PATH!\\\\.\" (\n"
-	"        git -C \"!MIRROR_PATH!\" rev-parse -q --verify \"ref-!HASH!\" > nul 2> nul\n"
-	"        if not errorlevel 1 (\n"
-	"            rmdir \"!MIRROR_PATH!.lock\"\n"
-	"            exit /b 0\n"
-	"        )\n"
+	"        echo \"git_helper_bat: dropping %%b, its object is missing\"\n"
+	"        git -C \"!MIRROR_PATH!\" update-ref -d %%b && set \"DROPPED=0\"\n"
 	"    )\n"
-	"    if not exist \"!MIRROR_PATH!\\\\.\" (\n"
-	"    mkdir \"!MIRROR_PATH!\"\n"
-	"    )\n"
-	"    if not exist \"!MIRROR_PATH!\\\\HEAD\" (\n"
-	"        git init --bare \"!MIRROR_PATH!\"\n"
-	"        if errorlevel 1 (\n"
-	"            rmdir \"!MIRROR_PATH!.lock\"\n"
-	"            exit /b 1\n"
-	"        )\n"
-	"    )\n"
-	"    set \"REFSPEC=!REF!:ref-!HASH!\"\n"
-	"    echo \"REFSPEC: !REFSPEC!\"\n"
-	"    git -C \"!MIRROR_PATH!\" fetch -q \"!REMOTE_URL!\" \"!REFSPEC!\" 2>&1\n"
-	"    if !ERRORLEVEL! neq 0 (\n"
-	"        echo \"git fetch failed with errorlevel !ERRORLEVEL!\"\n"
-	"        rmdir \"!MIRROR_PATH!.lock\"\n"
-	"        exit /b 1\n"
-	"    )\n"
-	"    rmdir \"!MIRROR_PATH!.lock\"\n"
-	"    exit /b 0\n"
 	")\n"
-	"if /i \"!OPERATION!\"==\"checkout\" (\n"
-	"    set \"MIRROR_PATH=%HOME%\\\\git-mirror\\\\%~2\"\n"
-	"    set \"BUILD_DIR=%~3\"\n"
-	"    set \"HASH=%~4\"\n"
-	"    echo \"MIRROR_PATH: !MIRROR_PATH!\"\n"
-	"    echo \"BUILD_DIR: !BUILD_DIR!\"\n"
-	"    echo \"HASH: !HASH!\"\n"
-	"    if not exist \"!BUILD_DIR!\\\\.git\" (\n"
-	"        if exist \"!BUILD_DIR!\\\\\" (\n"
-	"            del /f /s /q \"!BUILD_DIR!\" > nul 2>&1\n"
-	"            rmdir /s /q \"!BUILD_DIR!\"\n"
-	"        )\n"
-	"        mkdir \"!BUILD_DIR!\"\n"
-	"        git -C \"!BUILD_DIR!\" init\n"
-	"        if errorlevel 1 exit /b 1\n"
-	"    )\n"
-	"    git -C \"!BUILD_DIR!\" fetch \"!MIRROR_PATH!\" \"ref-!HASH!\"\n"
-	"    if errorlevel 1 exit /b 2\n"
-	"    git -C \"!BUILD_DIR!\" checkout -f \"!HASH!\"\n"
+	"exit /b !DROPPED!\n"
+	"\n"
+	"rem Fetch into a fresh mirror and swap it in.  Tasks already past this step may\n"
+	"rem still be checking out from the old one, so it is moved aside and only\n"
+	"rem removed once it is an hour old, and the refs it can still serve are carried\n"
+	"rem over.  Windows refuses to move a dir something has open, so we may have to\n"
+	"rem wait for those checkouts.\n"
+	"\n"
+	":mirror_rebuild\n"
+	"echo \"git_helper_bat: mirror is damaged, rebuilding it\"\n"
+	"git init -q --bare \"!NEW!\"\n"
+	"if errorlevel 1 exit /b 1\n"
+	"call :mirror_fetch \"!NEW!\"\n"
+	"if errorlevel 1 (\n"
+	"    call :rmtree \"!NEW!\"\n"
+	"    exit /b 1\n"
+	")\n"
+	"git -C \"!NEW!\" fetch -q \"!MIRROR_PATH!\" \"refs/heads/*:refs/heads/*\" >nul 2>&1\n"
+	"if errorlevel 1 echo \"git_helper_bat: not every old mirror ref could be carried over\"\n"
+	"call :rmtree \"!OLD!\"\n"
+	"if exist \"!OLD!\\\" (\n"
+	"    echo \"git_helper_bat: the previous old mirror is still in use\"\n"
+	"    call :rmtree \"!NEW!\"\n"
+	"    exit /b 1\n"
+	")\n"
+	"set \"TRIES=0\"\n"
+	":rebuild_retire\n"
+	"move \"!MIRROR_PATH!\" \"!OLD!\" >nul 2>&1 && goto :rebuild_swap\n"
+	"set /a \"TRIES+=1\"\n"
+	"if !TRIES! geq 10 (\n"
+	"    echo \"git_helper_bat: the old mirror is busy, leaving it in place\"\n"
+	"    call :rmtree \"!NEW!\"\n"
+	"    exit /b 1\n"
+	")\n"
+	"ping -n 3 127.0.0.1 >nul\n"
+	"goto :rebuild_retire\n"
+	":rebuild_swap\n"
+	"rem its age now counts from being retired\n"
+	"type nul > \"!OLD!\\sai-retired\"\n"
+	"move \"!NEW!\" \"!MIRROR_PATH!\" >nul\n"
+	"exit /b\n"
+	"\n"
+	":older_than\n"
+	"set \"SAI_AGE_PATH=%~1\"\n"
+	"set \"SAI_AGE_MINS=%~2\"\n"
+	"powershell -NoProfile -NonInteractive -Command \"$ErrorActionPreference = 'Stop'; if (((Get-Date) - (Get-Item -LiteralPath $env:SAI_AGE_PATH).LastWriteTime).TotalMinutes -gt $env:SAI_AGE_MINS) { exit 0 }; exit 1\" >nul 2>&1\n"
+	"exit /b\n"
+	"\n"
+	":rmtree\n"
+	"if not exist \"%~1\\\" exit /b 0\n"
+	"del /f /s /q \"%~1\" >nul 2>&1\n"
+	"rmdir /s /q \"%~1\" 2>nul\n"
+	"exit /b 0\n"
+	"\n"
+	":checkout\n"
+	"set \"MIRROR_PATH=%HOME%\\git-mirror\\%~2\"\n"
+	"set \"BUILD_DIR=%~3\"\n"
+	"set \"HASH=%~4\"\n"
+	"echo \"MIRROR_PATH: !MIRROR_PATH!\"\n"
+	"echo \"BUILD_DIR: !BUILD_DIR!\"\n"
+	"echo \"HASH: !HASH!\"\n"
+	"if not exist \"!BUILD_DIR!\\.git\" (\n"
+	"    call :rmtree \"!BUILD_DIR!\"\n"
+	"    mkdir \"!BUILD_DIR!\"\n"
+	"    git -C \"!BUILD_DIR!\" init\n"
 	"    if errorlevel 1 exit /b 1\n"
-	"    git -C \"!BUILD_DIR!\" clean -fdx\n"
-	"    echo \">>> Git helper script finished.\"\n"
-	"    exit /b 0\n"
 	")\n"
-	"exit /b 1\n"
+	"rem the mirror may be mid-swap after a rebuild\n"
+	"set \"TRIES=0\"\n"
+	":checkout_fetch\n"
+	"git -C \"!BUILD_DIR!\" fetch \"!MIRROR_PATH!\" \"ref-!HASH!\"\n"
+	"if not errorlevel 1 goto :checkout_fetched\n"
+	"set /a \"TRIES+=1\"\n"
+	"if !TRIES! geq 5 exit /b 2\n"
+	"ping -n 3 127.0.0.1 >nul\n"
+	"goto :checkout_fetch\n"
+	":checkout_fetched\n"
+	"git -C \"!BUILD_DIR!\" checkout -f \"!HASH!\"\n"
+	"if errorlevel 1 exit /b 1\n"
+	"git -C \"!BUILD_DIR!\" clean -fdx\n"
+	"echo \">>> Git helper script finished.\"\n"
+	"exit /b 0\n"
 ;
Page fetched 0s ago, creation time: 17ms (vhost etag hits: 0%, cache hits: 0%)