diff --git a/.sai.json b/.sai.json
index e284804..eaefd4d 100644
--- a/.sai.json
+++ b/.sai.json
@@ -22,13 +22,13 @@
},
"configurations": {
- "master+builder": {
+ "server+builder": {
"cmake": "",
"cpack": "&& cpack -C DEBUG $SAI_CPACK",
"artifacts": "build/sai-*.deb, build/sai-*.rpm"
},
"builder-only": {
- "cmake": "-DSAI_MASTER=0 ",
+ "cmake": "-DSAI_SERVER=0 ",
"platforms": "windows-10/x86_64-amd/msvc, netbsd-OSX-catalina/x86_64-intel-i3/llvm",
"cpack": "&& cpack -C DEBUG $SAI_CPACK",
"artifacts": "build/sai-*.deb, build/sai-*.rpm, build/sai-*.7z"
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 19627ef..812536a 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -1,6 +1,9 @@
project(sai C)
cmake_minimum_required(VERSION 2.8)
+find_package(libwebsockets CONFIG REQUIRED)
+list(APPEND CMAKE_MODULE_PATH ${LWS_CMAKE_DIR})
include(CheckCSourceCompiles)
+include(LwsCheckRequirements)
set(CMAKE_SKIP_RPATH 1)
@@ -43,16 +46,16 @@ set(CPACK_DEB_COMPONENT_INSTALL 1)
set(CPACK_DEBIAN_FILE_NAME "DEB-DEFAULT")
set(CPACK_DEBIAN_PACKAGE_COMPONENT 1)
set(CPACK_DEBIAN_PACKAGE_SHLIBDEPS ON)
-set(CPACK_DEBIAN_MASTER_PACKAGE_NAME "sai-master")
+set(CPACK_DEBIAN_SERVER_PACKAGE_NAME "sai-server")
set(CPACK_DEBIAN_BUILDER_PACKAGE_NAME "sai-builder")
-option(SAI_MASTER "Build the master / server part" ON)
-option(SAI_BUILDER "Build the builder / client part" ON)
+option(SAI_SERVER "Build the server + web part" ON)
+option(SAI_BUILDER "Build the builder part" ON)
set(LWS_OPENSSL_LIBRARIES CACHE PATH "Path to the OpenSSL library")
set(LWS_OPENSSL_INCLUDE_DIRS CACHE PATH "Path to the OpenSSL include directory")
-if (NOT SAI_MASTER AND NOT SAI_BUILDER)
+if (NOT SAI_SERVER AND NOT SAI_BUILDER)
message(FATAL_ERROR "Have to build both or one or the other")
endif()
@@ -118,69 +121,10 @@ CHECK_C_SOURCE_COMPILES("#include <libwebsockets.h>\nint
0;\n#else\n fail;\n#endif\n return 0;\n}\n" HAS_LIBCAP)
-# If we are being built as part of lws, confirm current build config supports
-# reqconfig, else skip building ourselves.
-#
-# If we are being built externally, confirm installed lws was configured to
-# support reqconfig, else error out with a helpful message about the problem.
-#
-MACRO(require_lws_config reqconfig _val result)
-
- if (DEFINED ${reqconfig})
- if (${reqconfig})
- set (rq 1)
- else()
- set (rq 0)
- endif()
- else()
- set(rq 0)
- endif()
-
- if (${_val} EQUAL ${rq})
- set(SAME 1)
- else()
- set(SAME 0)
- endif()
-
- if (LWS_WITH_MINIMAL_EXAMPLES AND NOT ${SAME})
- if (${_val})
- message("${SAMP}: skipping as lws being built without ${reqconfig}")
- else()
- message("${SAMP}: skipping as lws built with ${reqconfig}")
- endif()
- set(${result} 0)
- else()
- if (LWS_WITH_MINIMAL_EXAMPLES)
- set(MET ${SAME})
- else()
- CHECK_C_SOURCE_COMPILES("#include <libwebsockets.h>\nint main(void) {\n#if defined(${reqconfig})\n return 0;\n#else\n fail;\n#endif\n return 0;\n}\n" HAS_${reqconfig})
- if (NOT DEFINED HAS_${reqconfig} OR NOT HAS_${reqconfig})
- set(HAS_${reqconfig} 0)
- else()
- set(HAS_${reqconfig} 1)
- endif()
- if ((HAS_${reqconfig} AND ${_val}) OR (NOT HAS_${reqconfig} AND NOT ${_val}))
- set(MET 1)
- else()
- set(MET 0)
- endif()
- endif()
- if (NOT MET)
- if (${_val})
- message(FATAL_ERROR "This project requires lws must have been configured with ${reqconfig}")
- else()
- message(FATAL_ERROR "Lws configuration of ${reqconfig} is incompatible with this project")
- endif()
- endif()
-
- endif()
-ENDMACRO()
-
-#set(CMAKE_REQUIRED_INCLUDES ${SAI_LWS_INC_PATH})
-
set(requirements 1)
require_lws_config(LWS_ROLE_H1 1 requirements)
require_lws_config(LWS_ROLE_WS 1 requirements)
+require_lws_config(LWS_WITH_JOSE 1 requirements)
if (requirements)
@@ -191,8 +135,9 @@ if (requirements)
set(CMAKE_C_FLAGS "-Wall -Wsign-compare -Wignored-qualifiers -Wtype-limits -Wuninitialized -Werror -Wundef ${CMAKE_C_FLAGS}" )
endif()
- if (SAI_MASTER)
- add_subdirectory(src/master)
+ if (SAI_SERVER)
+ add_subdirectory(src/server)
+ add_subdirectory(src/web)
endif()
if (SAI_BUILDER)
add_subdirectory(src/builder)
diff --git a/README.md b/README.md
index add3878..76e2aa4 100644
--- a/README.md
+++ b/README.md
@@ -10,11 +10,12 @@ run on an RPi3-class 'buddy' to flash and collect results (eg, over serial, or
gpio) from even smaller targets.
A self-assembling constellation of Sai Builder clients make their own
-connections to one or more Sai Masters, who then receive hook notifications,
+connections to one or more Sai Servers, who then receive hook notifications,
read JSON from the project describing what set of build variations and tests it
should run on which platforms, and distributes work concurrently over idle
-builders that have the required environment, and logs results at the master
-accessible by a web interface.
+builders that have the required environment. A parallel Sai-web server is
+available usually on :443 or via a proxy to provide a live web / websockets
+interface with synamic updates and realtime build logs.
@@ -24,17 +25,17 @@ accessible by a web interface.
platform they want to provide builds for (ie, inside a systemd-nspawn or VM).
- The `sai-builder` daemons maintain nailed-up outgoing client wss connections
- to a central `sai-master`, which manages the ad-hoc collection of builders
+ to a central `sai-server`, which manages the ad-hoc collection of builders
it finds, and provides a web UI (http/2 and wss2).
- When a git repo that wants sai tests is updated, a push hook performs a POST
- notiftying the Sai master of a new push event which creates an entry in an
- event sqlite3 database. (The master does not need to access the repo that
- sends the git notification). The hook sends the master information about the
+ notiftying the Sai server of a new push event which creates an entry in an
+ event sqlite3 database. (The server does not need to access the repo that
+ sends the git notification). The hook sends the server information about the
push and the revision of `.sai.json` from the new commit in the POST body...
- the master parses that JSON to fill an sqlite3 database with tasks and
+ the server parses that JSON to fill an sqlite3 database with tasks and
commandline options for the build on platforms mentioned in `.sai.json`.
Pushes to branches beginning with `_` are ignored by Sai, even if they have
a valid `.sai.json`; this allows casual sharing of trees via the same git
@@ -46,21 +47,21 @@ accessible by a web interface.
as a starting point for git dramatically reduces the time compared to a full
git fetch.
- - The master hands out waiting tasks on connected idle builders that offer the
+ - The server hands out waiting tasks on connected idle builders that offer the
requested platforms, which build them concurrently. The builders may be
inside a protected network along with the repos they connect to; both the
builders and the repo only make outgoing https or wss connections to the
- master.
+ server.
- On the builders, result channels (like stdout, stderr, eventually others like
- `/dev/acm0`) with logs and results are streamed back to the master over a wss
+ `/dev/acm0`) with logs and results are streamed back to the server over a wss
link as the build proceeds, and are stored in an event-specific sqlite3
database for scalability.
- Builders can run CTest or other tests after the build and collect the
results (CTest has the advantages it's lightweight and crossplatform).
- - The master makes human readable current and historical results available
+ - The server makes human readable current and historical results available
in realtime over https web interface
- One `sai-builder` daemon can be configured to offer multiple instances of
@@ -73,10 +74,10 @@ accessible by a web interface.
specified in a configuration file. One sai-jig instance can separately
manage external gpio sequencing for multiple test targets.
- - Largely the master is automatic, driven by git hook notifications over
+ - Largely the server is automatic, driven by git hook notifications over
HTTP and the UI is read-only. However there are some privileged UI operations
like deleting a whole event, or redoing whole events or individual tasks.
- For these, if the browser has an authentic JWT signed by the master, it can
+ For these, if the browser has an authentic JWT signed by the server, it can
see and operate these privileged controls.
## Build flow and support for embedded
@@ -165,7 +166,7 @@ or other scripts are able to directly write to the device.
The `sai-builder` instance opens three local listening Unix Domain Sockets for
every platform, these accept raw data which is turned into logs on the
-respective logging channel and passed up to the `sai-master` for storage and
+respective logging channel and passed up to the `sai-server` for storage and
display like the other logs.
The paths of these "log proxy" Unix Domain Sockets are exported as environment
@@ -225,7 +226,7 @@ builders may be doing in parallel.
## Builder git caching
-For each `<saimaster-project>`, the builder maintains a local git cache. This is
+For each `<saiserver-project>`, the builder maintains a local git cache. This is
updated once when the new ref appears and then the related tests check out a
fresh image of their ref from that each time. This is very fast after the first
update, because it doesn't even involve the network but fetching from the local
@@ -255,16 +256,29 @@ down `/home/sai` or `\Users\sai`.
- git-mirror/
- `remote git url`_`project name` -- individual git mirrors
- jobs/
- - `master hostname`-`platform name`-`instance index`/
+ - `server hostname`-`platform name`-`instance index`/
- `project_name`/ - checkouts and builds occur in here
## Build steps
-Building sai produces two different apps, `sai-master` and `sai-builder`.
-`sai-master` is a ws and http server that coordinates activities, and
-`sai-builder` is run on each machine that wants to offer build services to
-one or more masters. By default, both are built, but you can use a cmake
-option `-DSAI_MASTER=0` and `-DSAI_BUILDER=0` to disable one or the other.
+Building sai produces two different sets of apps and daemons by default, for
+running on a the server that coordinates the builds and for running on machines
+that offer the actual builds for particular platforms to one or more servers.
+
+You can use cmake options `-DSAI_MASTER=0` and `-DSAI_BUILDER=0` to disable one
+or the other.
+
+Server executables|Function
+---|---
+sai-server|The server that builders connect to
+sai-web|The server that browsers connect to
+
+Builder executables|Function
+---|---
+sai-builder|The daemon that connects to sai-server and runs builds
+sai-device|Helper that coordinates which builds wants and can use specific embedded hardware
+sai-expect|Helper run by embedded build flow to capture serial traffic and react to keywords
+sai-jig|Helper for embedded devices that lets another device control its buttons, reset etc as part of the embedded build flow
First you must build lws with appropriate options.
@@ -280,13 +294,13 @@ $ cd libwebsockets && mkdir build && cd build && \
$ make -j && sudo make -j install && sudo ldconfig
```
-The actual cmake options needed depends on if you are building sai-master and / or
+The actual cmake options needed depends on if you are building sai-server and / or
sai-builder.
Feature|lws options
---|---
either|`-DLWS_WITH_STRUCT_JSON=1` `-DLWS_WITH_SECURE_STREAMS=1`
-master|`-DLWS_UNIX_SOCK=1` `-DLWS_WITH_GENCRYPTO=1` `-DLWS_WITH_STRUCT_SQLITE3=1` `-DLWS_WITH_JOSE=1`
+server|`-DLWS_UNIX_SOCK=1` `-DLWS_WITH_GENCRYPTO=1` `-DLWS_WITH_STRUCT_SQLITE3=1` `-DLWS_WITH_JOSE=1`
builder + related|`-DLWS_WITH_SPAWN=1` `-DLWS_WITH_THREADPOOL=1`
Similarly the two daemons bring in different dependencies
@@ -294,9 +308,9 @@ Similarly the two daemons bring in different dependencies
Feature|dependency
---|---
either|libwebsockets
-master|libsqlite3
+server|libsqlite3
builder|libgit2 pthreads
-jig|libgpiod
+jig (linux only)|libgpiod
#### Unix / Linux
diff --git a/READMEs/README-auth.md b/READMEs/README-auth.md
index 42f4372..9cc8aaf 100644
--- a/READMEs/README-auth.md
+++ b/READMEs/README-auth.md
@@ -7,14 +7,14 @@ Sai uses signed JWTs
There's no UI at the moment for creating authorized users, everything except
event deletion and task restart works without authorization.
-## sai-master configuration for auth
+## sai-server configuration for auth
In the `vhosts|ws-protocols|com-warmcat-sai` section of the config JSON, the
following entries define the authorization operation
```
"jwt-auth-alg": "ES512",
- "jwt-auth-jwk-path": "/etc/sai/master/auth.jwk",
+ "jwt-auth-jwk-path": "/etc/sai/server/auth.jwk",
"jwt-iss": "com.warmcat",
"jwt-aud": "https://libwebsockets.org/sai",
```
@@ -34,20 +34,20 @@ Use this as below to create the server JWK used for signing and validation,
for ES512 algorithm in our case
```
-$ sudo ./bin/lws-crypto-jwk -t EC -b512 -vP-521 --alg ES512 > /etc/sai/master/auth.jwk
+$ sudo ./bin/lws-crypto-jwk -t EC -b512 -vP-521 --alg ES512 > /etc/sai/server/auth.jwk
```
## Defining an authorized user
-Sai-master creates a separate auth database and prepares the table schema in
-it on startup if not already existing. So you using sai-master at all already
+Sai-server creates a separate auth database and prepares the table schema in
+it on startup if not already existing. So you using sai-server at all already
did most of the work.
To create a user that can login via his browser and see and use the UI for the
additional actions, currently you can create the user by hand on the server:
```
-# sqlite3 /home/srv/sai/sai-master-auth.sqlite3
+# sqlite3 /home/srv/sai/sai-server-auth.sqlite3
SQLite version 3.32.3 2020-06-18 14:00:33
Enter ".help" for usage hints.
sqlite> .schema
diff --git a/READMEs/README-build-windows.md b/READMEs/README-build-windows.md
index e709a07..96b4135 100644
--- a/READMEs/README-build-windows.md
+++ b/READMEs/README-build-windows.md
@@ -15,7 +15,7 @@ also used in the next steps here.
For windows + sai-builder, the following lws cmake config is suitable
```
-> cmake .. -DLWS_HAVE_PTHREAD_H=1 -DLWS_EXT_PTHREAD_INCLUDE_DIR="C:\Program Files (x86)\pthreads\include" -DLWS_EXT_PTHREAD_LIBRARIES="C:\Program Files (x86)\pthreads\lib\x64\libpthreadGC2.a" -DLWS_WITH_MINIMAL_EXAMPLES=1 -DLWS_WITH_THREADPOOL=1 -DLWS_UNIX_SOCK=1 -DLWS_WITH_STRUCT_JSON=1 -DLWS_WITH_SPAWN=1 -DLWS_WITH_SECURE_STREAMS=1 -DLWS_WITH_DIR=1
+> cmake .. -DLWS_WITH_JOSE=1 -DLWS_HAVE_PTHREAD_H=1 -DLWS_EXT_PTHREAD_INCLUDE_DIR="C:\Program Files (x86)\pthreads\include" -DLWS_EXT_PTHREAD_LIBRARIES="C:\Program Files (x86)\pthreads\lib\x64\libpthreadGC2.a" -DLWS_WITH_MINIMAL_EXAMPLES=1 -DLWS_WITH_THREADPOOL=1 -DLWS_UNIX_SOCK=1 -DLWS_WITH_STRUCT_JSON=1 -DLWS_WITH_SPAWN=1 -DLWS_WITH_SECURE_STREAMS=1 -DLWS_WITH_DIR=1
```
## Building libgit2
diff --git a/READMEs/README-sai-jig.md b/READMEs/README-sai-jig.md
index 86f7a2c..8bc13f9 100644
--- a/READMEs/README-sai-jig.md
+++ b/READMEs/README-sai-jig.md
@@ -3,7 +3,7 @@
## Introduction
Sai-jig is a standalone daemon that normally doesn't run on the builder or
-master machines, but on a smaller helper close to embedded targets and
+server machines, but on a smaller helper close to embedded targets and
physically wired to, eg, the reset button or flash config GPIO on them.
The helper is typically an RPi or similar machine that has networking,
Linux and convenient GPIO.
diff --git a/READMEs/README-sai-json.md b/READMEs/README-sai-json.md
index 12a6686..bdddc25 100644
--- a/READMEs/README-sai-json.md
+++ b/READMEs/README-sai-json.md
@@ -4,7 +4,7 @@ Projects can inform Sai of what build and test variations they want to run
on push using a `.sai.json` file in the top level dir of the project.
The git push hook isolated this file from the pushed tree and passes it to
-the sai-master along with the rest of the push notification information in
+the sai-server along with the rest of the push notification information in
a signed JSON structure.
## Simple .sai.json file
@@ -47,7 +47,7 @@ platform builder with the arguments
In the top level `"platforms"` section you describe the different build
platforms you want to build and test on... the specified platforms must have
-builders that have connected themselves to the sai-master.
+builders that have connected themselves to the sai-server.
If you don't give a `"default": false` entry in the platform definition, the
platform is assumed to apply to all the configurations listed afterwards. If
@@ -92,5 +92,5 @@ or to disable all default platforms and just use the specified ones:
#### artifacts
You can also give a comma-separated list of build artifacts, these are
-arbitrary binary files which will be uploaded to sai-master and made available
+arbitrary binary files which will be uploaded to sai-server and made available
for download over https.
\ No newline at end of file
diff --git a/READMEs/README-systemd-nspawn.md b/READMEs/README-systemd-nspawn.md
index 739c7f1..b622746 100644
--- a/READMEs/README-systemd-nspawn.md
+++ b/READMEs/README-systemd-nspawn.md
@@ -1,7 +1,7 @@
# Setting up and administering systemd-nspawn containers
Sai's approach is to run `sai-builder` from inside the environment it will do
-builds from. It reaches out to the sai-master instance and git repos, and
+builds from. It reaches out to the sai-server instance and git repos, and
otherwise there is no ingress into the container or VM from outside.
So with `systemd-nspawn`, it means the containers should start on the host at
@@ -319,7 +319,7 @@ For redhat type distros, you probably need to add /usr/local/lib to the
```
Container # git clone https://libwebsockets.org/repo/libwebsockets
Container # cd libwebsockets && mkdir build && cd build && \
- cmake .. -DLWS_UNIX_SOCK=1 -DLWS_WITH_STRUCT_JSON=1 \
+ cmake .. -DLWS_WITH_JOSE=1 -DLWS_UNIX_SOCK=1 -DLWS_WITH_STRUCT_JSON=1 \
-DLWS_WITH_STRUCT_SQLITE3=1 -DLWS_WITH_GENCRYPTO=1 -DLWS_WITH_SPAWN=1 \
-DLWS_WITH_SECURE_STREAMS=1 -DLWS_WITH_THREADPOOL=1 -DLWS_WITH_JOSE=1
Container # make -j && make -j install && ldconfig
@@ -408,6 +408,8 @@ Make sai-builder run as a daemon, create the file `/Library/LaunchDaemons/com.wa
</array>
<key>RunAtLoad</key>
<true/>
+ <key>StandardErrorPath</key>
+ <string>/var/log/sai-builder.log</string>
</dict>
</plist>
```
diff --git a/READMEs/README-using-overlayfs.md b/READMEs/README-using-overlayfs.md
index 0c89448..630a07d 100644
--- a/READMEs/README-using-overlayfs.md
+++ b/READMEs/README-using-overlayfs.md
@@ -219,7 +219,7 @@ $ sudo git init
$ sudo git add *
$ sudo git commit -m "initial commit"
$ sudo git remote add origin ssh://git@example.com/linux-ubuntu-xenial-amd64+base
-$ sudo git push origin +master:master
+$ sudo git push origin +server:server
```
diff --git a/READMEs/sai-build-test-flow.png b/READMEs/sai-build-test-flow.png
index a3c439b..6ba4376 100644
Binary files a/READMEs/sai-build-test-flow.png and b/READMEs/sai-build-test-flow.png differ
diff --git a/READMEs/sai-embedded-test.png b/READMEs/sai-embedded-test.png
index 03e9c70..f4c534f 100644
Binary files a/READMEs/sai-embedded-test.png and b/READMEs/sai-embedded-test.png differ
diff --git a/READMEs/sai-overview.png b/READMEs/sai-overview.png
index 069460d..469adec 100644
Binary files a/READMEs/sai-overview.png and b/READMEs/sai-overview.png differ
diff --git a/assets/index.html b/assets/index.html
index 20f099f..e37102f 100644
--- a/assets/index.html
+++ b/assets/index.html
@@ -16,18 +16,18 @@
<div id="login" class="login">
<div id="creds" class="creds hide">
- <form class="login" action="login" method="post">
+ <form class="login">
<label for="lname">Name:</label>
<input class="crin" type="text" id="lname" name="lname" autocomplete="username"><br>
<label for="lpass">Password:</label>
<input class="crin" type="password" id="lpass" name="lpass" autocomplete="current-password"><br>
- <input class="crinb" type="submit" value="Login">
+ <input class="crinb" type="button" value="Login" id="login">
<input type="hidden" id="success_redir" name="success_redir" value="">
</form>
</div>
<div id="logout" class="creds hide">
<div id="remauth"></div>
- <input class="crin" type="submit" id="logout" value="Logout">
+ <input class="crin" type="button" id="logout" value="Logout">
</div>
</div>
diff --git a/assets/sai.css b/assets/sai.css
index 2f6f714..c8b3ab8 100644
--- a/assets/sai.css
+++ b/assets/sai.css
@@ -85,6 +85,16 @@ td.waiting {
border-radius:5px;
}
+td.waiting_gi {
+ vertical-align:top;
+ text-align:left;
+ color:#000000;
+ padding:0px;
+ font-size: 9pt;
+ font-weight: bold;
+ border-radius:2px;
+}
+
table.comp {
border-radius:7px;
}
@@ -148,6 +158,12 @@ td.jumble {
white-space: nowrap;
}
+img.saicon {
+ width: 24px;
+ height:24px;
+ padding-right:3px;
+}
+
span.group2 {
vertical-align:middle;
text-align:center;
@@ -183,6 +199,17 @@ span.nowrap {
div.taskinfo {
vertical-align: top;
+ position: fixed;
+ background: #fff;
+ z-index: 2002;
+ border-style: solid;
+ border-width: 16px 16px 0px 16px;
+ border-color: #fff;
+ margin-top: -16px;
+}
+
+table.scrollogs {
+ margin-top: 64px;
}
div.taskstate {
@@ -214,7 +241,23 @@ div.evr {
top: 16px;
left: 16px;
}
+div.evr_gi {
+ display: inline-block;
+ width: 16px;
+ height: 16px;
+}
+div.gi_popup {
+ box-shadow: 3px 5px 5px 0px #444;
+ left: 0px;
+ z-index: -2;
+ background: #ffe;
+ opacity: 0.0;
+ transition: opacity 1s ease;
+ display: block;
+ position: absolute;
+ margin-top: 35px
+}
.taskstate0 {
color:#000000;
@@ -274,42 +317,24 @@ img.bico {
}
.taskstate1 {
- background:#a0a0a0;
+ background: #d0d0d0;
color:#000000;
- opacity:0.75;
- animation-name: stretch;
- animation-duration: 2s;
- animation-timing-function: ease-out;
- animation-delay: 0s;
- animation-direction: alternate;
- animation-iteration-count: infinite;
- animation-fill-mode: none;
- animation-play-state: running;
+ opacity:1;
+ transition: background-color 500ms linear;
}
.taskstate2 {
background:#a0a0a0;
color:#000000;
- opacity:0.75;
- animation-name: stretch;
- animation-duration: 1s;
- animation-timing-function: ease-out;
- animation-delay: 0s;
- animation-direction: alternate;
- animation-iteration-count: infinite;
- animation-fill-mode: none;
- animation-play-state: running;
+ opacity:1;
+ transition: background-color 500ms linear;
}
.taskstate3 {
background:#01ff70;
color:#003000;
opacity:1;
- /*
- animation-name: fadeInOpacity;
- animation-iteration-count: 1;
- animation-timing-function: ease-in;
- animation-duration: 2s; */
+ transition: background-color 500ms linear;
}
@keyframes fadeInOpacity {
@@ -320,26 +345,32 @@ img.bico {
opacity: 1;
}
}
-}
+
.taskstate4 {
background:#ff851b;
color:#300000;
opacity:1;
+ transition: background-color 500ms linear;
}
.taskstate5 {
background:#808080;
color:#202020;
opacity:1;
+ transition: background-color 500ms linear;
}
.taskstate6 {
background:#df3030;
color:#300000;
opacity:1;
+ transition: background-color 500ms linear;
}
+div.stats_hidden {
+ display: none;
+}
div.dlogs {
vertical-align: top;
@@ -385,6 +416,10 @@ div.hide {
display: none;
}
+td.hide {
+ display: none;
+}
+
input.crin {
font-size: 7pt;
width: 60px;
@@ -483,6 +518,11 @@ td.e4 {
text-align: right;
}
+td.e6 {
+ font-weight: normal;
+ font-size: 6pt;
+ text-align: right;
+}
table.summary {
width: 100%;
diff --git a/assets/sai.js b/assets/sai.js
index 8900d1b..e8c2430 100644
--- a/assets/sai.js
+++ b/assets/sai.js
@@ -512,6 +512,9 @@ function aging()
}
}
+ if (next < 5)
+ next = 5;
+
/*
* We only need to come back when the age might have changed.
* Eg, if everything is counted in hours already, once per
@@ -527,7 +530,7 @@ function sai_plat_icon(plat, size)
s = plat.split('/');
if (s[0]) {
- console.log("plat " + plat + " plat[0] " + s[0]);
+ // console.log("plat " + plat + " plat[0] " + s[0]);
s1 = "<img class=\"ip" + size + " zup\" src=\"/sai/" + san(s[0]) +
".svg\">";
@@ -560,7 +563,7 @@ function sai_taskinfo_render(t, now_ut)
var now_ut = Math.round((new Date().getTime() / 1000));
var s = "";
- s = "<div class=\"taskinfo\"><table><tr class=\"nomar\"><td class=\"atop\"><table>" +
+ s = "<table><tr class=\"nomar\"><td class=\"atop\"><table>" +
sai_event_render(t, now_ut, 0) + "</table></td><td class=\"ti\">" +
"<span class=\"ti1\">" + sai_plat_icon(t.t.platform, 2) +
san(t.t.platform) + "</span> ";
@@ -589,16 +592,53 @@ function sai_taskinfo_render(t, now_ut)
s += "</td></tr>";
- s += "</td></tr></table></table></div>";
+ s += "</td></tr></table></table>";
return s;
}
+function summarize_build_situation(event_uuid)
+{
+ var good = 0, bad = 0, total = 0,
+ roo = document.getElementById("taskcont-" + event_uuid),
+ same;
+
+ if (!roo)
+ return "";
+
+ same = roo.querySelectorAll(".taskstate");
+ if (same)
+ total = same.length;
+ same = roo.querySelectorAll(".taskstate3");
+ if (same)
+ good = same.length;
+ same = roo.querySelectorAll(".taskstate4");
+ if (same)
+ bad = same.length;
+
+ if (good == total)
+ return "All " + good + " passed";
+
+ if (bad == total)
+ return "All " + bad + " failed";
+
+ if (!good && !bad)
+ return total + " pending";
+
+ if (!bad)
+ return good + " passed, " + (total - good) + " pending";
+
+ if (!good)
+ return bad + " failed, " + (total - bad) + " pending";
+
+ return good + " passed, " + bad + " failed, " + (total - good - bad) + " pending";
+}
+
function sai_event_summary_render(o, now_ut, reset_all_icon)
{
var s, q, ctn = "", wai, s1 = "", n, e = o.e;
- s = "<tr><td class=\"waiting\"><table class=\"comp";
+ s = "<table class=\"comp";
if (e.state == 3)
s += " comp_pass";
if (e.state == 4 || e.state == 6)
@@ -606,13 +646,18 @@ function sai_event_summary_render(o, now_ut, reset_all_icon)
s += "\"><tr><td class=\"jumble\"><a href=\"/sai/?event=" + san(e.uuid) +
"\"><img src=\"/sai/sai-event.svg\"";
+ if (gitohashi_integ)
+ s += " class=\"saicon\"";
if (e.state == 3 || e.state == 4)
s += " class=\"deemph\"";
s += ">";
+ var cl = "evr";
+ if (gitohashi_integ)
+ cl = "evr_gi";
if (e.state == 3)
- s += "<div class=\"evr\"><img src=\"/sai/passed.svg\"></div>";
+ s += "<div class=\"" + cl + "\"><img src=\"/sai/passed.svg\"></div>";
if (e.state == 4)
- s += "<div class=\"evr\"><img src=\"/sai/failed.svg\"></div>";
+ s += "<div class=\"" + cl + "\"><img src=\"/sai/failed.svg\"></div>";
s += "</a>";
if (reset_all_icon && !gitohashi_integ && authd) {
@@ -621,30 +666,38 @@ function sai_event_summary_render(o, now_ut, reset_all_icon)
s += "<img class=\"rebuild\" alt=\"delete event\" src=\"/sai/delete.png\" " +
"id=\"delete-ev-" + san(e.uuid) + "\">";
}
- s += "</td>" +
+ s += "</td>";
+
+ if (!gitohashi_integ) {
+ s +=
"<td><table class=\"nomar\">" +
- "<tr><td class=\"nomar\" colspan=2><span class=\"e1\">" +
- san(e.repo_name) +
+ "<tr><td class=\"nomar\" colspan=2>" +
+ "<span class=\"e1\">" + san(e.repo_name) +
"</span></td></tr><tr><td class=\"nomar\" colspan=2><span class=\"e2\">";
-
- if (e.ref.substr(0, 11) === "refs/heads/") {
- s += "<img class=\"branch\">" +
- san(e.ref.substr(11));
- } else
- if (e.ref.substr(0, 10) === "refs/tags/") {
- s += "<img class=\"tag\">" +
- san(e.ref.substr(10));
+
+ if (e.ref.substr(0, 11) === "refs/heads/") {
+ s += "<img class=\"branch\">" +
+ san(e.ref.substr(11));
} else
- s += san(e.ref);
-
- s += "</span></td></tr><tr><td class=\"nomar\"><span class=\"e3\">" +
- san(e.hash.substr(0, 8)) +
- "</span></td><td class=\"e4\" class=\"nomar\">" +
- agify(now_ut, e.created) + "</td></tr>";
- if (o.t.length > 1)
- s += "<tr><td class=\"nomar\"><span class=\"e3\">" + o.t.length + " builds</span></td><tr>";
- s += "</table>" +
- "</td></tr></table></td>";
+ if (e.ref.substr(0, 10) === "refs/tags/") {
+ s += "<img class=\"tag\">" +
+ san(e.ref.substr(10));
+ } else
+ s += san(e.ref);
+
+ s += "</span></td></tr><tr><td class=\"nomar e6\">" +
+ san(e.hash.substr(0, 8)) +
+ "</td><td class=\"e6 nomar\">" +
+ agify(now_ut, e.created) + "</td></tr>";
+ s += "<tr><td class=\"nomar e6\" colspan=\"2\" id=\"sumbs-" + e.uuid +"\">" + summarize_build_situation(e.uuid) + "</td><tr>";
+ s += "</table>" +
+ "</td>";
+ } else {
+ s +="<td><table><tr><td class=\"e6 nomar\">" + san(e.hash.substr(0, 8)) + " " + agify(now_ut, e.created);
+ s += "</td></tr><tr><td class=\"e6 nomar\" id=\"sumbs-" + e.uuid +"\">" + summarize_build_situation(e.uuid);
+ s += "</tr></table></td>";
+ }
+ s += "</tr></table>";
return s;
}
@@ -652,8 +705,11 @@ function sai_event_summary_render(o, now_ut, reset_all_icon)
function sai_event_render(o, now_ut, reset_all_icon)
{
var s, q, ctn = "", wai, s1 = "", n, e = o.e;
-
- s = sai_event_summary_render(o, now_ut, reset_all_icon);
+
+ s = "<tr><td class=\"waiting\"";
+ if (gitohashi_integ)
+ s += " id=\"gitohashi_sai_icon\"";
+ s += "><div id=\"esr-" + san(e.uuid) + "\"></div></td>";
if (o.t.length) {
var all_good, has_bad, all_done;
@@ -668,7 +724,11 @@ function sai_event_render(o, now_ut, reset_all_icon)
* SAIES_BEING_BUILT_HAS_FAILURES
*/
- s += "<td class=\"tasks\"><table><tr><td class=\"atop\">";
+ s += "<td class=\"tasks\" id=\"taskcont-" + san(e.uuid) + "\">";
+ if (gitohashi_integ)
+ s += "<div class=\"gi_popup\" id=\"gitohashi_sai_details\">";
+
+ s += "<table><tr><td class=\"atop\">";
for (q = 0; q < o.t.length; q++) {
var t = o.t[q];
@@ -683,6 +743,7 @@ function sai_event_render(o, now_ut, reset_all_icon)
*/
s += "<div class=\"";
+ /*
if (wai)
s += " awaiting";
if (all_good === 1) {
@@ -696,12 +757,12 @@ function sai_event_render(o, now_ut, reset_all_icon)
if (all_done === 0)
s += " ov_dunno";
}
+ */
s += " ib\"><table class=\"nomar\">" +
- "<tr><td class=\"keepline\">" + s1;
- s += "</td><td class=\"tn\">" +
- sai_stateful_taskname(st, ctn, 0) +
- "</td></tr></table></div>";
+ "<tr><td class=\"tn\">" + ctn +
+ "</td><td class=\"keepline\">" + s1 +
+ "</td></tr></table></div>";
s1 = "";
}
all_good = 1;
@@ -718,7 +779,7 @@ function sai_event_render(o, now_ut, reset_all_icon)
if (t.state === 4 || t.state == 6)
has_bad = 1;
- s1 += "<div class=\"taskstate taskstate" + t.state + "\">";
+ s1 += "<div id=\"taskstate_" + t.uuid + "\" class=\"taskstate taskstate" + t.state + "\">";
if (t.state === 0)
wai = 1;
@@ -732,6 +793,8 @@ function sai_event_render(o, now_ut, reset_all_icon)
var st = 0;
s += "<div class=\"";
+
+ /*
if (wai)
s += " awaiting";
if (all_good === 1) {
@@ -745,14 +808,17 @@ function sai_event_render(o, now_ut, reset_all_icon)
if (all_done === 0)
s += " ov_dunno";
}
+ */
s += " ib\"><table class=\"nomar\">" +
- "<tr><td class=\"keepline\">" + s1;
- s += "</td><td class=\"tn\">" +
- sai_stateful_taskname(st, ctn, 0) +
- "</td></tr></table></div>";
+ "<tr><td class=\"tn\">" + ctn +
+ "<td class=\"keepline\">" + s1 +
+ "</td></tr></table></div>";
}
- s += "</td></tr></table></td>";
+ s += "</td></tr></table>";
+ if (gitohashi_integ)
+ s += "</div>";
+ s += "</td>";
}
s += "</tr>";
@@ -802,7 +868,7 @@ function render_builders(jso)
host = sen.split('.')[0];
cia = host.split('-')[1];
- console.log("host " + host + ", cia " + cia);
+ // console.log("host " + host + ", cia " + cia);
if ((nc == conts.length && !cia) ||
(cia && cia === conts[nc])) {
@@ -834,6 +900,27 @@ function render_builders(jso)
return s;
}
+function refresh_state(task_uuid, task_state)
+{
+ var tsi = document.getElementById("taskstate_" + task_uuid);
+
+ if (tsi) {
+ tsi.classList.remove("taskstate1");
+ tsi.classList.remove("taskstate2");
+ tsi.classList.remove("taskstate3");
+ tsi.classList.remove("taskstate4");
+ tsi.classList.remove("taskstate5");
+ tsi.classList.remove("taskstate6");
+ tsi.classList.remove("taskstate7");
+ tsi.classList.add("taskstate" + task_state);
+ // console.log("refresh_state taskstate" + task_state);
+ }
+}
+
+function after_delete() {
+ location.reload();
+}
+
function ws_open_sai()
{
var s = "", q, qa, qi, q5, q5s;
@@ -862,6 +949,14 @@ function ws_open_sai()
gitohashi_integ = 1;
}
+ qi = q.indexOf("?task=");
+ if (qi != -1) {
+ /*
+ * it's a sai task details page
+ */
+ s += "/specific?task=" + q.substring(qi + 6);
+ }
+
var s1 = get_appropriate_ws_url() + "/sai/browse" + s;
// if (s1.split("?"))
// s1 = s1.split("?")[0];
@@ -919,15 +1014,15 @@ function ws_open_sai()
"\"com.warmcat.sai.taskinfo\"}");
};
- sai.onmessage =function got_packet(msg) {
+ sai.onmessage = function got_packet(msg) {
var u, ci, n;
var now_ut = Math.round((new Date().getTime() / 1000));
- console.log(msg.data);
+ // console.log(msg.data);
// if (msg.data.length < 10)
// return;
jso = JSON.parse(msg.data);
- console.log(jso.schema);
+ // console.log(jso.schema);
if (jso.alang) {
var a = jso.alang.split(","), n;
@@ -976,6 +1071,11 @@ function ws_open_sai()
}
if (jso.schema == "sai.warmcat.com.overview") {
+ /*
+ * Sent with an array of e[] to start, but also
+ * can send a single e[] if it just changed
+ * state
+ */
s = "<table>";
authd = jso.authorized;
@@ -1001,22 +1101,89 @@ function ws_open_sai()
san(auth_user) + " " + agify(now_ut, now_ut + jso.auth_secs);
}
}
+
+ /*
+ * Update existing?
+ */
- if (jso.overview.length)
- for (n = jso.overview.length - 1; n >= 0; n--)
- s += sai_event_render(jso.overview[n], now_ut, 1);
+ // console.log("jso.overview.length " + jso.overview.length);
- s = s + "</table>";
+ if (jso.overview.length == 1 &&
+ document.getElementById("esr-" + jso.overview[0].e.uuid)) {
+ /* this is just the summary box, not the tasks */
+ document.getElementById("esr-" + jso.overview[0].e.uuid).innerHTML =
+ sai_event_summary_render(jso.overview[0], now_ut, 1);
+
+ /* if the task status icons exist, update their state */
+
+ for (n = jso.overview[0].t.length - 1; n >= 0; n--)
+ refresh_state(jso.overview[0].t[n].uuid, jso.overview[0].t[n].state);
+
+ var sumbs = document.getElementById("sumbs-" + jso.overview[0].e.uuid);
+ if (sumbs)
+ sumbs.innerHTML = summarize_build_situation(jso.overview[0].e.uuid);
+
+ aging();
+ } else
+ {
+ /*
+ * display events wholesale
+ */
+ if (jso.overview.length) {
+ for (n = jso.overview.length - 1; n >= 0; n--)
+ s += sai_event_render(jso.overview[n], now_ut, 1);
- if (document.getElementById("sai_sticky"))
- document.getElementById("sai_sticky").innerHTML = s;
- aging();
+ s = s + "</table>";
+
+ if (document.getElementById("sai_sticky"))
+ document.getElementById("sai_sticky").innerHTML = s;
+
+ for (n = jso.overview.length - 1; n >= 0; n--) {
+ document.getElementById("esr-" + jso.overview[n].e.uuid).innerHTML =
+ sai_event_summary_render(jso.overview[n], now_ut, 1);
+ }
+
+ if (jso.overview[0] && jso.overview[0].e) {
+ var sumbs = document.getElementById("sumbs-" + jso.overview[0].e.uuid);
+ if (sumbs)
+ sumbs.innerHTML = summarize_build_situation(jso.overview[0].e.uuid);
+ }
+ aging();
+ }
+
+ if (gitohashi_integ && document.getElementById("gitohashi_sai_icon")) {
+ var integ_state = 0;
+ document.getElementById("gitohashi_sai_icon").addEventListener("mouseenter", function( event ) {
+ document.getElementById("gitohashi_sai_icon").style.zIndex = 1999;
+ document.getElementById("gitohashi_sai_details").style.zIndex = 2000;
+ document.getElementById("gitohashi_sai_details").style.opacity = 1.0;
+ integ_state = 1;
+ }, false);
+
+ document.getElementById("gitohashi_sai_details").addEventListener("mouseout", function( event ) {
+ var e = event.toElement || event.relatedTarget;
+ while (e && e.parentNode && e.parentNode != window) {
+ if (e.parentNode == this || e == this) {
+ if (e.preventDefault)
+ e.preventDefault();
+ return false;
+ }
+ e = e.parentNode;
+ }
+ document.getElementById("gitohashi_sai_details").style.opacity = 0.0;
+ document.getElementById("gitohashi_sai_details").style.zIndex = -1;
+ document.getElementById("gitohashi_sai_icon").style.zIndex = 2001;
+ }, true);
+
+ aging();
+ }
+ }
if (jso.overview.length)
for (n = jso.overview.length - 1; n >= 0; n--) {
if (document.getElementById("rebuild-ev-" + san(jso.overview[n].e.uuid)))
- document.getElementById("rebuild-ev-" + san(jso.overview[n].e.uuid)).
- addEventListener("click", function(e) {
+ document.getElementById("rebuild-ev-" + san(jso.overview[n].e.uuid)).
+ addEventListener("click", function(e) {
console.log(e);
var rs= "{\"schema\":" +
"\"com.warmcat.sai.eventreset\"," +
@@ -1028,7 +1195,7 @@ function ws_open_sai()
});
if (document.getElementById("delete-ev-" + san(jso.overview[n].e.uuid)))
document.getElementById("delete-ev-" + san(jso.overview[n].e.uuid)).
- addEventListener("click", function(e) {
+ addEventListener("click", function(e) {
console.log(e);
var rs= "{\"schema\":" +
"\"com.warmcat.sai.eventdelete\"," +
@@ -1037,6 +1204,7 @@ function ws_open_sai()
console.log(rs);
sai.send(rs);
+ setTimeout(after_delete, 750);
});
}
}
@@ -1066,57 +1234,118 @@ function ws_open_sai()
san(auth_user) + " " + agify(now_ut, now_ut + jso.auth_secs);
}
}
-
- s = sai_taskinfo_render(jso);
- if (document.getElementById("sai_sticky"))
- document.getElementById("sai_sticky").innerHTML = s;
+ /*
+ * We get told about changes to any task state,
+ * it's up to us to figure out if the page we
+ * showed should display the update and in what
+ * form.
+ *
+ * We make sure the div containing the task info
+ * has a special ID depending on if it's shown
+ * as a tuple or as extended info
+ *
+ * First see if it appears as a tuple, and if
+ * so, let's just update that
+ */
- s = "<table><td colspan=\"3\"><pre><table><tr>" +
- "<td class=\"atop\">" +
- "<div id=\"dlogsn\" class=\"dlogsn\">" + lines + "</div></td>" +
- "<td class=\"atop\">" +
- "<div id=\"dlogst\" class=\"dlogst\">" + times + "</div></td>" +
- "<td class=\"atop\"><div id=\"dlogs\" class=\"dlogs\">" +
- "<span id=\"logs\" class=\"nowrap\">" + logs +
- "</span>"+
- "</div></td></tr></table></pre>";
+ if (document.getElementById("taskstate_" + jso.t.uuid)) {
+ console.log("found taskstate_" + jso.t.uuid);
+ refresh_state(jso.t.uuid, jso.t.state);
+
+ var sumbs = document.getElementById("sumbs-" + jso.t.uuid.substring(0, 32));
+ if (sumbs)
+ sumbs.innerHTML = summarize_build_situation(jso.t.uuid.substring(0, 32));
+
+ } else
- if (document.getElementById("sai_overview"))
- document.getElementById("sai_overview").innerHTML = s;
-
- if (document.getElementById("rebuild-" + san(jso.t.uuid))) {
- document.getElementById("rebuild-" + san(jso.t.uuid)).
- addEventListener("click", function(e) {
- var rs= "{\"schema\":" +
- "\"com.warmcat.sai.taskreset\"," +
- "\"uuid\": " +
- JSON.stringify(san(e.srcElement.id.substring(8))) + "}";
-
- console.log(rs);
- sai.send(rs);
- document.getElementById("dlogsn").innerHTML = "";
- document.getElementById("dlogst").innerHTML = "";
- document.getElementById("logs").innerHTML = "";
- lines = times = logs = "";
- tfirst = 0;
- lli = 1;
- });
- }
+ /* update task summary if shown anywhere */
+
+ if (document.getElementById("taskinfo-" + jso.t.uuid)) {
+ console.log("FOUND taskinfo-" + jso.t.uuid);
+ document.getElementById("taskinfo-" + jso.t.uuid).innerHTML = sai_taskinfo_render(jso);
+ if (document.getElementById("esr-" + jso.e.uuid))
+ document.getElementById("esr-" + jso.e.uuid).innerHTML =
+ sai_event_summary_render(jso.e, now_ut, 1);
+ } else {
+
+ console.log("NO taskinfo- or taskstate_" + jso.t.uuid);
+
+ /*
+ * Last chance if we might be
+ * on a task-specific page, and
+ * want to show the task info
+ * at the top
+ */
+
+
+ const urlParams = new URLSearchParams(window.location.search);
+ const url_task_uuid = urlParams.get('task');
+
+ if (url_task_uuid === jso.t.uuid &&
+ document.getElementById("sai_sticky"))
+ document.getElementById("sai_sticky").innerHTML =
+ "<div class=\"taskinfo\" id=\"taskinfo-" +
+ san(jso.t.uuid) + "\">" +
+ sai_taskinfo_render(jso) +
+ "</div>";
+
- if (document.getElementById("stop-" + san(jso.t.uuid))) {
- document.getElementById("stop-" + san(jso.t.uuid)).
- addEventListener("click", function(e) {
- var rs= "{\"schema\":" +
- "\"com.warmcat.sai.taskcan\"," +
- "\"task_uuid\": " +
- JSON.stringify(san(e.srcElement.id.substring(5))) + "}";
- console.log(rs);
- sai.send(rs);
- });
+ s = "<table><td colspan=\"3\"><pre><table class=\"scrollogs\"><tr>" +
+ "<td class=\"atop\">" +
+ "<div id=\"dlogsn\" class=\"dlogsn\">" + lines + "</div></td>" +
+ "<td class=\"atop\">" +
+ "<div id=\"dlogst\" class=\"dlogst\">" + times + "</div></td>" +
+ "<td class=\"atop\"><div id=\"dlogs\" class=\"dlogs\">" +
+ "<span id=\"logs\" class=\"nowrap\">" + logs +
+ "</span>"+
+ "</div></td></tr></table></pre>";
+
+ if (document.getElementById("sai_overview")) {
+ document.getElementById("sai_overview").innerHTML = s;
+
+ if (document.getElementById("esr-" + jso.e.uuid))
+ document.getElementById("esr-" + jso.e.uuid).innerHTML =
+ sai_event_summary_render(jso, now_ut, 1);
+
+ var sumbs = document.getElementById("sumbs-" + jso.e.uuid);
+ if (sumbs)
+ sumbs.innerHTML = summarize_build_situation(jso.e.uuid);
+ }
+
+ if (document.getElementById("rebuild-" + san(jso.t.uuid))) {
+ document.getElementById("rebuild-" + san(jso.t.uuid)).
+ addEventListener("click", function(e) {
+ var rs= "{\"schema\":" +
+ "\"com.warmcat.sai.taskreset\"," +
+ "\"uuid\": " +
+ JSON.stringify(san(e.srcElement.id.substring(8))) + "}";
+
+ console.log(rs);
+ sai.send(rs);
+ document.getElementById("dlogsn").innerHTML = "";
+ document.getElementById("dlogst").innerHTML = "";
+ document.getElementById("logs").innerHTML = "";
+ lines = times = logs = "";
+ tfirst = 0;
+ lli = 1;
+ });
+ }
+
+ if (document.getElementById("stop-" + san(jso.t.uuid))) {
+ document.getElementById("stop-" + san(jso.t.uuid)).
+ addEventListener("click", function(e) {
+ var rs= "{\"schema\":" +
+ "\"com.warmcat.sai.taskcan\"," +
+ "\"task_uuid\": " +
+ JSON.stringify(san(e.srcElement.id.substring(5))) + "}";
+ console.log(rs);
+ sai.send(rs);
+ });
+ }
+
+ aging();
}
-
- aging();
}
@@ -1222,6 +1451,44 @@ function ws_open_sai()
}
}
+function post_login_form()
+{
+ var xhr = new XMLHttpRequest(), s ="", q = window.location.pathname;
+
+ s = "----boundo\x0d\x0acontent-disposition: form-data; name=\"lname\"\x0d\x0a\x0d\x0a" +
+ document.getElementById("lname").value +
+ "\x0d\x0a----boundo\x0d\x0acontent-disposition: form-data; name=\"lpass\"\x0d\x0a\x0d\x0a" +
+ document.getElementById("lpass").value +
+ "\x0d\x0a----boundo\x0d\x0acontent-disposition: form-data; name=\"success_redir\"\x0d\x0a\x0d\x0a" +
+ document.getElementById("success_redir").value +
+ "\x0d\x0a----boundo--";
+
+ if (q.length > 10 && q.substring(q.length - 10) == "index.html")
+ q = q.substring(0, q.length - 10);
+ xhr.open("POST", q + "login", true);
+ xhr.setRequestHeader( 'content-type', "multipart/form-data; boundary=--boundo");
+
+ console.log(s.length +" " + s);
+
+ xhr.onload = function (e) {
+ if (xhr.readyState === 4) {
+ if (xhr.status === 200 || xhr.status == 303) {
+ console.log(xhr.responseText);
+ location.reload();
+ } else {
+ console.error(xhr.statusText);
+ }
+ }
+ };
+ xhr.onerror = function (e) {
+ console.error(xhr.statusText);
+ };
+
+ xhr.send(s);
+
+ return false;
+}
+
/* stuff that has to be delayed until all the page assets are loaded */
window.addEventListener("load", function() {
@@ -1235,7 +1502,11 @@ window.addEventListener("load", function() {
window.location.href;
ws_open_sai();
aging();
-
+
+ if (document.getElementById("login")) {
+ document.getElementById("login").addEventListener("click", post_login_form);
+ document.getElementById("logout").addEventListener("click", post_login_form);
+ }
setInterval(function() {
diff --git a/etc-sai-EXAMPLE/builder/conf b/etc-sai-EXAMPLE/builder/conf
index b371b51..c6a9179 100644
--- a/etc-sai-EXAMPLE/builder/conf
+++ b/etc-sai-EXAMPLE/builder/conf
@@ -1,42 +1,21 @@
{
- #
- # where to find the nspawn contexts (scanned at runtime)
- #
- "systemd-nspawn": {
- "path": "/home/sai/sai-nspawn",
- "overlaypath": "/home/sai/sai-nspawn",
- "instances": 3,
- "timeout": 1800
- },
+ "perms": "sai:nobody",
+ "home": "/home/sai",
+ "host": "bionic-noi",
- #
- # physically connected targets, like microcontroller boards
- #
- "target": [
+ "platforms": [
+ {
+ "name": "linux-ubuntu-1804/x86_64-amd/gcc",
+ "instances": 3,
+ "servers": [ "wss://libwebsockets.org:4444/sai/builder" ]
+ },
{
- "name": "optee-hikey-arm64",
- "timeout": 3600,
- "consoles": [
- {
- "name": "dbg",
- "path": "/dev/by-id/usb-FTDI_FT232R_USB_UART_A50285BI-if00-port0",
- "baudrate": 115200,
- "mode": "n81"
- }
- ]
+ "name": "freertos-linkit/arm32-m4-mt7697-usi/gcc",
+ "instances": 1,
+ "env": [
+ { "SAI_CROSS_BASE": "/opt/linkit" }
+ ],
+ "servers": [ "wss://libwebsockets.org:4444/sai/builder" ]
}
- ],
-
- #
- # masters we will accept jobs from
- #
- "masters": [
- {
-# "url": "wss://warmcat.com/sai/builder",
- "url": "wss://localhost:443/builder",
- "selfsigned": 1,
- "priority": 1
- }
- ]
+ ]
}
-
diff --git a/etc-sai-EXAMPLE/master/conf b/etc-sai-EXAMPLE/master/conf
deleted file mode 100644
index 58d027a..0000000
--- a/etc-sai-EXAMPLE/master/conf
+++ /dev/null
@@ -1,14 +0,0 @@
-# these are the sai server global settings
-#
-# stuff related to each vhosts should go in one
-# file per vhost in ./conf.d/
-
-{
- "global": {
- "uid": "48", # after init, run as apache user
- "gid": "48", # after init, run as apache group
- "server-string": "sai",
- "init-ssl": "yes"
- }
-}
-
diff --git a/etc-sai-EXAMPLE/master/conf.d/localhost b/etc-sai-EXAMPLE/master/conf.d/localhost
deleted file mode 100644
index 7ccda44..0000000
--- a/etc-sai-EXAMPLE/master/conf.d/localhost
+++ /dev/null
@@ -1,127 +0,0 @@
-# you should create one file like this per vhost in /etc/sai/conf.d
-
-{
- "vhosts": [{
- # this should match the external hostname for the vhost,
- # like xyz.com
- "name": "localhost",
- # multiple vhosts can occupy the same port (SNI is used
- # to resolve)
- "port": "443",
- # required for avatar cache
- "enable-client-ssl": "on",
- # "ipv6": "on",
- "interface": "lo",
-
-# "ciphers": "DEFAULT",
-# "tls13-ciphers": "DEFAULT",
-
- # You should store your real certificate key somewhere safer,
- # eg on Fedora /etc/pki/tls/private, readable only by root. But
- # since the right place for this differs by distro and these
- # particular certs are not valuable, we get them from
- # /usr/local/share where they are installed.
-
- "host-ssl-key": "/usr/local/share/sai/sai-selfsigned-key.pem",
- "host-ssl-cert": "/usr/local/share/sai/sai-selfsigned-cert.pem",
-
- "mounts": [
- {
- # static assets like js, css, fonts
- "mountpoint": "/sai",
- "default": "index.html",
- "origin": "file:///usr/local/share/sai/assets",
- "cache-max-age": "7200",
- "cache-reuse": "1",
- "cache-revalidate": "0",
- "cache-intermediaries": "0",
- "extra-mimetypes": {
- ".zip": "application/zip",
- ".map": "application/json",
- ".ttf": "application/x-font-ttf"
- }
- },
- {
- # semi-static cached avatar icons
- "mountpoint": "/sai/avatar",
- "origin": "callback://avatar-proxy",
- "cache-max-age": "2400",
- "cache-reuse": "1",
- "cache-revalidate": "0",
- "cache-intermediaries": "0"
- }
- ],
-
- #
- # these headers, which will be sent on every transaction, make
- # recent browsers definitively ban any external script sources,
- # no matter what might manage to get injected later in the
- # page. It's a last line of defence against any successful XSS.
- #
-
- "headers": [{
- "content-security-policy": "default-src 'none'; img-src 'self' data: https://travis-ci.org https://api.travis-ci.org https://ci.appveyor.com https://scan.coverity.com; script-src 'self'; font-src 'self'; style-src 'self'; connect-src 'self'; frame-ancestors 'none'; base-uri 'none';",
- "x-content-type-options": "nosniff",
- "x-xss-protection": "1; mode=block",
- "x-frame-options": "deny",
- "referrer-policy": "no-referrer"
- }],
-
- "ws-protocols": [
-
- {"com-warmcat-sai": {
- # template HTML to use for this vhost. You'd normally
- # copy this to gitohashi-vhostname.html and modify it
- # to show the content, logos, links, fonts, css etc for
- # your vhost. It's not served directly but read from
- # the filesystem. Although it's cached in memory by
- # gitohashi, it checks for changes and reloads if
- # changed automatically.
- #
- # Putting logos etc as svg in css is highly recommended,
- # like fonts these can be transferred once with a loose
- # caching policy. So in practice they cost very little,
- # and allow the browser to compose the page without
- # delay.
- #
- "html-file": "/usr/local/share/gitohashi/templates/gitohashi-example.html",
- #
- # vpath required at start of links into this vhost's
- # gitohashi content for example if an external http
- # server is proxying us, and has been told to direct
- # URLs starting "/git" to us, this should be set to
- # "/git/" so URLs we generate referring to our own pages
- # can work.
- #
- "vpath": "/sai/",
- #
- # url mountpoint for the avatar cache
- #
- "avatar-url": "/sai/avatar/",
- #
- # libjsgit2 JSON cache... this
- # should not be directly served
- #
- "cache-base": "/var/cache/libjsongit2",
- #
- # restrict the JSON cache size
- # to 2GB
- #
- "cache-size": "2000000000",
- #
- # optional flags, b0 = 1 = blog mode
- #
- "flags": 0
- #"blog-repo-name": "myrepo"
- },
- "avatar-proxy": {
- "remote-base": "https://www.gravatar.com/avatar/",
- #
- # this dir is served via avatar-proxy
- #
- "cache-dir": "/var/cache/libjsongit2"
- }}
- ]
- }
- ]
-}
diff --git a/etc-sai-EXAMPLE/master/conf.d/unixskt b/etc-sai-EXAMPLE/master/conf.d/unixskt
deleted file mode 100644
index 2ab6da5..0000000
--- a/etc-sai-EXAMPLE/master/conf.d/unixskt
+++ /dev/null
@@ -1,145 +0,0 @@
-# you should create one file like this per vhost in /etc/sai/master/conf.d
-
-{
- "vhosts": [{
- # this has no special meaning but needs to be unique
- "name": "unixskt",
- "unix-socket": 1,
- # this should be owned by sai and allow group access from
- # whatever group your webserver runs under
- "unix-socket-perms": "sai:apache",
- # multiple vhosts can exist with different unix skt names
- "interface": "/var/run/sai",
- # required for avatar cache
- "enable-client-ssl": "on",
-
- "mounts": [
- {
- "mountpoint": "/sai",
- "default": "index.html",
- "origin": "file:///usr/local/share/sai/assets",
- "origin": "callback://com-warmcat-sai",
- "cache-max-age": "7200",
- "cache-reuse": "1",
- "cache-revalidate": "0",
- "cache-intermediaries": "0",
- "extra-mimetypes": {
- ".zip": "application/zip",
- ".map": "application/json",
- ".ttf": "application/x-font-ttf"
- }
- },
- {
- # semi-static cached avatar icons
- "mountpoint": "/sai/avatar",
- "origin": "callback://avatar-proxy",
- "cache-max-age": "2400",
- "cache-reuse": "1",
- "cache-revalidate": "0",
- "cache-intermediaries": "0"
- }
- ],
-
- #
- # these headers, which will be sent on every transaction, make
- # recent browsers definitively ban any external script sources,
- # no matter what might manage to get injected later in the
- # page. It's a last line of defence against any successful XSS.
- #
-
- "headers": [{
- "content-security-policy": "default-src 'none'; img-src 'self' data:; script-src 'self'; font-src 'self'; style-src 'self'; connect-src 'self'; frame-ancestors 'none'; base-uri 'none';",
- "x-content-type-options": "nosniff",
- "x-xss-protection": "1; mode=block",
- "referrer-policy": "no-referrer"
- }],
-
- "ws-protocols": [
- {"com-warmcat-sai": {
- # this is the lhs of the path to use, not a full filepath
- # the events go in ...-events.sqlite3 and the related
- # info for an event goes in its own database file in
- # ...-event-xxxx.sqlite3 where xxxx is the event uuid.
- #
- "database": "/srv/sai/sai-master",
- #
- # the push hook notification script creates a JSON
- # object and signs it with a secret... "notification-
- # key" should match the secret the notification hook
- # script has that we're willing to accept.
- #
- # It should be a 64-char hex-ascii representation of
- # a random 32-byte sequence, you can produce a strong
- # random key in the correct format like this
- #
- # dd if=/dev/random bs=32 count=1 | sha256sum | cut -d' ' -f1
- #
- "notification-key": "1234...5678",
-
- # auth jwk path
- # You can generate a suitable key like this
- #
- # lws-crypto-jwk -t EC -b512 -vP-521 --alg ES512 > mykey.jwk
- #
- "jwt-auth-alg": "ES512",
- "jwt-auth-jwk-path": "/etc/sai/master/auth.jwk",
-
- "jwt-iss": "com.warmcat",
- "jwt-aud": "https://libwebsockets.org/sai",
-
- # template HTML to use for this vhost. You'd normally
- # copy this to gitohashi-vhostname.html and modify it
- # to show the content, logos, links, fonts, css etc for
- # your vhost. It's not served directly but read from
- # the filesystem. Although it's cached in memory by
- # gitohashi, it checks for changes and reloads if
- # changed automatically.
- #
- # Putting logos etc as svg in css is highly recommended,
- # like fonts these can be transferred once with a loose
- # caching policy. So in practice they cost very little,
- # and allow the browser to compose the page without
- # delay.
- #
- "html-file": "/usr/local/share/gitohashi/templates/gitohashi-example.html",
- #
- # vpath required at start of links into this vhost's
- # gitohashi content for example if an external http
- # server is proxying us, and has been told to direct
- # URLs starting "/git" to us, this should be set to
- # "/git/" so URLs we generate referring to our own pages
- # can work.
- #
- "vpath": "/sai/",
- #
- # url mountpoint for the avatar cache
- #
- "avatar-url": "/sai/avatar/",
- #
- # libjsgit2 JSON cache... this
- # should not be directly served
- #
- "cache-base": "/var/cache/libjsongit2",
- #
- # restrict the JSON cache size
- # to 2GB
- #
- "cache-size": "2000000000",
- #
- # optional flags, b0 = 1 = blog mode
- #
- "flags": 0
- #"blog-repo-name": "myrepo"
- },
- "avatar-proxy": {
- "remote-base": "https://www.gravatar.com/avatar/",
- #
- # this dir is served via avatar-proxy
- #
- "cache-dir": "/var/cache/libjsongit2"
- }
- }
- ]
- }
- ]
-}
diff --git a/etc-sai-EXAMPLE/server/conf b/etc-sai-EXAMPLE/server/conf
new file mode 100644
index 0000000..58d027a
--- /dev/null
+++ b/etc-sai-EXAMPLE/server/conf
@@ -0,0 +1,14 @@
+# these are the sai server global settings
+#
+# stuff related to each vhosts should go in one
+# file per vhost in ./conf.d/
+
+{
+ "global": {
+ "uid": "48", # after init, run as apache user
+ "gid": "48", # after init, run as apache group
+ "server-string": "sai",
+ "init-ssl": "yes"
+ }
+}
+
diff --git a/etc-sai-EXAMPLE/server/conf.d/mydomain.com b/etc-sai-EXAMPLE/server/conf.d/mydomain.com
new file mode 100644
index 0000000..95c44ce
--- /dev/null
+++ b/etc-sai-EXAMPLE/server/conf.d/mydomain.com
@@ -0,0 +1,143 @@
+# you should create one file like this per vhost in /etc/sai/server/conf.d
+
+{
+ "vhosts": [{
+ "name": "mydomain.com",
+ "port": "4444",
+ "host-ssl-key": "/etc/letsencrypt/live/libwebsockets.org/privkey.pem",
+ "host-ssl-cert": "/etc/letsencrypt/live/libwebsockets.org/fullchain.pem",
+ "access-log": "/var/log/sai-server-access-log",
+ "disable-no-protocol-ws-upgrades": "on",
+ "enable-client-ssl": "on",
+ "allow-non-tls": "1",
+
+ "mounts": [
+ {
+ "mountpoint": "/sai",
+ "default": "index.html",
+ "origin": "file:///usr/local/share/sai/assets",
+ "origin": "callback://com-warmcat-sai",
+ "cache-max-age": "7200",
+ "cache-reuse": "1",
+ "cache-revalidate": "0",
+ "cache-intermediaries": "0",
+ "extra-mimetypes": {
+ ".zip": "application/zip",
+ ".map": "application/json",
+ ".ttf": "application/x-font-ttf"
+ }
+ },
+ {
+ # semi-static cached avatar icons
+ "mountpoint": "/sai/avatar",
+ "origin": "callback://avatar-proxy",
+ "cache-max-age": "2400",
+ "cache-reuse": "1",
+ "cache-revalidate": "0",
+ "cache-intermediaries": "0"
+ }
+ ],
+
+ #
+ # these headers, which will be sent on every transaction, make
+ # recent browsers definitively ban any external script sources,
+ # no matter what might manage to get injected later in the
+ # page. It's a last line of defence against any successful XSS.
+ #
+
+ "headers": [{
+ "content-security-policy": "default-src 'none'; img-src 'self' data:; script-src 'self'; font-src 'self'; style-src 'self'; connect-src 'self'; frame-ancestors 'none'; base-uri 'none';",
+ "x-content-type-options": "nosniff",
+ "x-xss-protection": "1; mode=block",
+ "referrer-policy": "no-referrer"
+ }],
+
+ "ws-protocols": [
+ {"com-warmcat-sai": {
+ # this is the lhs of the path to use, not a full filepath
+ # the events go in ...-events.sqlite3 and the related
+ # info for an event goes in its own database file in
+ # ...-event-xxxx.sqlite3 where xxxx is the event uuid.
+ #
+ "database": "/srv/sai/sai-master",
+ #
+ # the push hook notification script creates a JSON
+ # object and signs it with a secret... "notification-
+ # key" should match the secret the notification hook
+ # script has that we're willing to accept.
+ #
+ # It should be a 64-char hex-ascii representation of
+ # a random 32-byte sequence, you can produce a strong
+ # random key in the correct format like this
+ #
+ # dd if=/dev/random bs=32 count=1 | sha256sum | cut -d' ' -f1
+ #
+ "notification-key": "51b3ee2f06ef2a893cfe901972bd13065d7dbae4cf087b396ee38e7bf78f79a6",
+
+ # auth jwk path
+ # You can generate a suitable key like this
+ #
+ # lws-crypto-jwk -t EC -b512 -vP-521 --alg ES512 > mykey.jwk
+ #
+ "jwt-auth-alg": "ES512",
+ "jwt-auth-jwk-path": "/etc/sai/web/auth.jwk",
+
+ "jwt-iss": "com.warmcat",
+ "jwt-aud": "https://mydomain.com/sai",
+
+ # template HTML to use for this vhost. You'd normally
+ # copy this to gitohashi-vhostname.html and modify it
+ # to show the content, logos, links, fonts, css etc for
+ # your vhost. It's not served directly but read from
+ # the filesystem. Although it's cached in memory by
+ # gitohashi, it checks for changes and reloads if
+ # changed automatically.
+ #
+ # Putting logos etc as svg in css is highly recommended,
+ # like fonts these can be transferred once with a loose
+ # caching policy. So in practice they cost very little,
+ # and allow the browser to compose the page without
+ # delay.
+ #
+ "html-file": "/usr/local/share/gitohashi/templates/gitohashi-example.html",
+ #
+ # vpath required at start of links into this vhost's
+ # gitohashi content for example if an external http
+ # server is proxying us, and has been told to direct
+ # URLs starting "/git" to us, this should be set to
+ # "/git/" so URLs we generate referring to our own pages
+ # can work.
+ #
+ "vpath": "/sais/",
+ #
+ # url mountpoint for the avatar cache
+ #
+ "avatar-url": "/sais/avatar/",
+ #
+ # libjsgit2 JSON cache... this
+ # should not be directly served
+ #
+ "cache-base": "/var/cache/libjsongit2",
+ #
+ # restrict the JSON cache size
+ # to 2GB
+ #
+ "cache-size": "2000000000",
+ #
+ # optional flags, b0 = 1 = blog mode
+ #
+ "flags": 0
+ #"blog-repo-name": "myrepo"
+ },
+ "avatar-proxy": {
+ "remote-base": "https://www.gravatar.com/avatar/",
+ #
+ # this dir is served via avatar-proxy
+ #
+ "cache-dir": "/var/cache/libjsongit2"
+ }
+ }
+ ]
+ }
+ ]
+}
diff --git a/etc-sai-EXAMPLE/web/conf b/etc-sai-EXAMPLE/web/conf
new file mode 100644
index 0000000..673f250
--- /dev/null
+++ b/etc-sai-EXAMPLE/web/conf
@@ -0,0 +1,9 @@
+{
+ "global": {
+ "username": "sai", # sai
+ "groupname": "nobody", # nobody
+ "server-string": "sai",
+ "rlimit-nofile": "10000",
+ "init-ssl": "yes"
+ }
+}
diff --git a/etc-sai-EXAMPLE/web/conf.d/unixskt b/etc-sai-EXAMPLE/web/conf.d/unixskt
new file mode 100644
index 0000000..e14702e
--- /dev/null
+++ b/etc-sai-EXAMPLE/web/conf.d/unixskt
@@ -0,0 +1,133 @@
+# this specifies how the sai-web server operates
+
+{
+ "vhosts": [{
+ # this has no special meaning but needs to be unique
+ "name": "unixsktw",
+ "unix-socket": 1,
+ # this should be owned by sai and allow group access from
+ # whatever group your webserver runs under
+ "unix-socket-perms": "sai:apache",
+ # multiple vhosts can exist with different unix skt names
+ "interface": "/var/run/sai",
+ # required for avatar cache
+ "enable-client-ssl": "on",
+
+ "mounts": [
+ {
+ "mountpoint": "/sai",
+ "default": "index.html",
+ "origin": "file:///usr/local/share/sai/assets",
+ "origin": "callback://com-warmcat-sai",
+ "cache-max-age": "7200",
+ "cache-reuse": "1",
+ "cache-revalidate": "0",
+ "cache-intermediaries": "0",
+ "extra-mimetypes": {
+ ".zip": "application/zip",
+ ".map": "application/json",
+ ".ttf": "application/x-font-ttf"
+ }
+ },
+ {
+ # semi-static cached avatar icons
+ "mountpoint": "/sai/avatar",
+ "origin": "callback://avatar-proxy",
+ "cache-max-age": "2400",
+ "cache-reuse": "1",
+ "cache-revalidate": "0",
+ "cache-intermediaries": "0"
+ }
+ ],
+
+ #
+ # these headers, which will be sent on every transaction, make
+ # recent browsers definitively ban any external script sources,
+ # no matter what might manage to get injected later in the
+ # page. It's a last line of defence against any successful XSS.
+ #
+
+ "headers": [{
+ "content-security-policy": "default-src 'none'; img-src 'self' data:; script-src 'self'; font-src 'self'; style-src 'self'; connect-src 'self'; frame-ancestors 'none'; base-uri 'none';",
+ "x-content-type-options": "nosniff",
+ "x-xss-protection": "1; mode=block",
+ "referrer-policy": "no-referrer"
+ }],
+
+ "ws-protocols": [
+ {"com-warmcat-sai": {
+ # this is the lhs of the path to use, not a full filepath
+ # the events go in ...-events.sqlite3 and the related
+ # info for an event goes in its own database file in
+ # ...-event-xxxx.sqlite3 where xxxx is the event uuid.
+ #
+ "database": "/srv/sai/sai-master",
+
+
+ # auth jwk path
+ # You can generate a suitable key like this
+ #
+ # lws-crypto-jwk -t EC -b512 -vP-521 --alg ES512 > mykey.jwk
+ #
+ "jwt-auth-alg": "ES512",
+ "jwt-auth-jwk-path": "/etc/sai/web/auth.jwk",
+
+ "jwt-iss": "com.warmcat",
+ "jwt-aud": "https://mydomain.com/sai",
+
+ # template HTML to use for this vhost. You'd normally
+ # copy this to gitohashi-vhostname.html and modify it
+ # to show the content, logos, links, fonts, css etc for
+ # your vhost. It's not served directly but read from
+ # the filesystem. Although it's cached in memory by
+ # gitohashi, it checks for changes and reloads if
+ # changed automatically.
+ #
+ # Putting logos etc as svg in css is highly recommended,
+ # like fonts these can be transferred once with a loose
+ # caching policy. So in practice they cost very little,
+ # and allow the browser to compose the page without
+ # delay.
+ #
+ "html-file": "/usr/local/share/gitohashi/templates/gitohashi-example.html",
+ #
+ # vpath required at start of links into this vhost's
+ # gitohashi content for example if an external http
+ # server is proxying us, and has been told to direct
+ # URLs starting "/git" to us, this should be set to
+ # "/git/" so URLs we generate referring to our own pages
+ # can work.
+ #
+ "vpath": "/sai/",
+ #
+ # url mountpoint for the avatar cache
+ #
+ "avatar-url": "/sai/avatar/",
+ #
+ # libjsgit2 JSON cache... this
+ # should not be directly served
+ #
+ "cache-base": "/var/cache/libjsongit2",
+ #
+ # restrict the JSON cache size
+ # to 2GB
+ #
+ "cache-size": "2000000000",
+ #
+ # optional flags, b0 = 1 = blog mode
+ #
+ "flags": 0
+ #"blog-repo-name": "myrepo"
+ },
+ "avatar-proxy": {
+ "remote-base": "https://www.gravatar.com/avatar/",
+ #
+ # this dir is served via avatar-proxy
+ #
+ "cache-dir": "/var/cache/libjsongit2"
+ }
+ }
+ ]
+ }
+ ]
+}
diff --git a/scripts/builder-conf b/scripts/builder-conf
index ae69ffc..09bdc45 100644
--- a/scripts/builder-conf
+++ b/scripts/builder-conf
@@ -10,7 +10,7 @@
{
"name": "linux-debian-buster/x86_64-amd/gcc",
"instances": 1,
- "masters": [ "wss://warmcat.com/sai/builder" ]
+ "servers": [ "wss://warmcat.com:4444/sai/builder" ]
}
]
}
diff --git a/scripts/sai-master.service b/scripts/sai-master.service
deleted file mode 100644
index 041c738..0000000
--- a/scripts/sai-master.service
+++ /dev/null
@@ -1,13 +0,0 @@
-[Unit]
-Description=Sai Master
-After=syslog.target
-
-[Service]
-ExecStart=/usr/local/bin/sai-master
-ExecReload=/usr/bin/kill -HUP $MAINPID
-ExecStop=/usr/bin/killall sai-master
-
-[Install]
-WantedBy=multi-user.target
-
-
diff --git a/scripts/sai-server.service b/scripts/sai-server.service
new file mode 100644
index 0000000..6b7d949
--- /dev/null
+++ b/scripts/sai-server.service
@@ -0,0 +1,13 @@
+[Unit]
+Description=Sai Server
+After=syslog.target
+
+[Service]
+ExecStart=/usr/local/bin/sai-server
+ExecReload=/usr/bin/kill -HUP $MAINPID
+ExecStop=/usr/bin/killall sai-server
+
+[Install]
+WantedBy=multi-user.target
+
+
diff --git a/scripts/sai-web.service b/scripts/sai-web.service
new file mode 100644
index 0000000..8e14732
--- /dev/null
+++ b/scripts/sai-web.service
@@ -0,0 +1,13 @@
+[Unit]
+Description=Sai Web
+After=syslog.target
+
+[Service]
+ExecStart=/usr/local/bin/sai-web
+ExecReload=/usr/bin/kill -HUP $MAINPID
+ExecStop=/usr/bin/killall sai-web
+
+[Install]
+WantedBy=multi-user.target
+
+
diff --git a/src/builder/b-artifacts.c b/src/builder/b-artifacts.c
index a0f0bf4..08d6634 100644
--- a/src/builder/b-artifacts.c
+++ b/src/builder/b-artifacts.c
@@ -18,7 +18,7 @@
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
* MA 02110-1301 USA
*
- * This deals with POSTing artifacts to the related sai-master, using a secret
+ * This deals with POSTing artifacts to the related sai-server, using a secret
* that was passed to the builder as part of the task that was allocated
*/
diff --git a/src/builder/b-comms.c b/src/builder/b-comms.c
index c01ea4d..e6b04ba 100644
--- a/src/builder/b-comms.c
+++ b/src/builder/b-comms.c
@@ -30,7 +30,7 @@
static lws_ss_state_return_t
saib_m_rx(void *userobj, const uint8_t *buf, size_t len, int flags)
{
- struct sai_plat_master *spm = (struct sai_plat_master *)userobj;
+ struct sai_plat_server *spm = (struct sai_plat_server *)userobj;
//struct sai_plat *sp = (struct sai_plat *)spm->sai_plat;
lwsl_info("%s: len %d, flags: %d\n", __func__, (int)len, flags);
@@ -50,7 +50,7 @@ static int
tp_sync_check(struct lws_dll2 *d, void *user)
{
sai_plat_t *sp = lws_container_of(d, sai_plat_t, sai_plat_list);
- struct sai_plat_master *spm = (struct sai_plat_master *)user;
+ struct sai_plat_server *spm = (struct sai_plat_server *)user;
struct sai_nspawn *ns;
int n, soe;
void *vp;
@@ -66,7 +66,7 @@ tp_sync_check(struct lws_dll2 *d, void *user)
soe = ns->state;
/*
- * We can't deal with nspawns bound to a different master or
+ * We can't deal with nspawns bound to a different server or
* nspawns not with an active threadpool task
*/
@@ -199,7 +199,7 @@ next:
/*
* We cover requested tx for any instance of a platform that can takes tasks
- * from the same master... it means just by coming here, no particular
+ * from the same server... it means just by coming here, no particular
* platform / sai_plat is implied...
*/
@@ -207,7 +207,7 @@ static lws_ss_state_return_t
saib_m_tx(void *userobj, lws_ss_tx_ordinal_t ord, uint8_t *buf, size_t *len,
int *flags)
{
- struct sai_plat_master *spm = (struct sai_plat_master *)userobj;
+ struct sai_plat_server *spm = (struct sai_plat_server *)userobj;
uint8_t *start = buf, *end = buf + (*len) - 1, *p = start;
struct ws_capture_chunk *chunk;
struct sai_plat *sp = NULL;
@@ -219,7 +219,7 @@ saib_m_tx(void *userobj, lws_ss_tx_ordinal_t ord, uint8_t *buf, size_t *len,
/*
* We are the ss / wsi that any threadpool instances on any platform
- * with tasks for this master are trying to sync to. We need to handle
+ * with tasks for this server are trying to sync to. We need to handle
* and resume them all.
*/
@@ -265,7 +265,7 @@ saib_m_tx(void *userobj, lws_ss_tx_ordinal_t ord, uint8_t *buf, size_t *len,
default:
/*
- * Update master with platform status
+ * Update server with platform status
*/
js = lws_struct_json_serialize_create(lsm_schema_map_plat,
@@ -301,7 +301,7 @@ saib_m_tx(void *userobj, lws_ss_tx_ordinal_t ord, uint8_t *buf, size_t *len,
/*
* Yes somebody has some logs... since we handle all logs on any
- * platform doing tasks for the same master, we have to take care not
+ * platform doing tasks for the same server, we have to take care not
* to favour draining logs for any busy tasks over letting others
* getting a chance at the mic. If we just scan for guys with logs
* from the start of the list each time, we will never deal with guys
@@ -316,7 +316,7 @@ saib_m_tx(void *userobj, lws_ss_tx_ordinal_t ord, uint8_t *buf, size_t *len,
* = =
* spm->last_logging_platform : spm->last_logging_nspawn
*
- * ... filtered for nspawns associated with our spm / master SS link.
+ * ... filtered for nspawns associated with our spm / server SS link.
*
* The platforms and nspawns are allocated at conf-time statically.
*/
@@ -448,7 +448,7 @@ sendify:
static int
cleanup_on_ss_destroy(struct lws_dll2 *d, void *user)
{
- struct sai_plat_master *spm = (struct sai_plat_master *)user;
+ struct sai_plat_server *spm = (struct sai_plat_server *)user;
sai_plat_t *sp = lws_container_of(d, sai_plat_t, sai_plat_list);
lws_start_foreach_dll_safe(struct lws_dll2 *, d, d1,
@@ -471,7 +471,7 @@ cleanup_on_ss_destroy(struct lws_dll2 *d, void *user)
static int
cleanup_on_ss_disconnect(struct lws_dll2 *d, void *user)
{
- struct sai_plat_master *spm = (struct sai_plat_master *)user;
+ struct sai_plat_server *spm = (struct sai_plat_server *)user;
sai_plat_t *sp = lws_container_of(d, sai_plat_t, sai_plat_list);
struct ws_capture_chunk *cc;
@@ -512,7 +512,7 @@ static lws_ss_state_return_t
saib_m_state(void *userobj, void *sh, lws_ss_constate_t state,
lws_ss_tx_ordinal_t ack)
{
- struct sai_plat_master *spm = (struct sai_plat_master *)userobj;
+ struct sai_plat_server *spm = (struct sai_plat_server *)userobj;
lwsl_user("%s: %s, ord 0x%x\n", __func__, lws_ss_state_name(state),
(unsigned int)ack);
@@ -522,7 +522,7 @@ saib_m_state(void *userobj, void *sh, lws_ss_constate_t state,
/*
* If the logical SS itself is going down, every platform that
- * used us to connect to their master and has nspawns are also
+ * used us to connect to their server and has nspawns are also
* going down
*/
lws_dll2_foreach_safe(&builder.sai_plat_owner, spm,
@@ -561,11 +561,11 @@ saib_m_state(void *userobj, void *sh, lws_ss_constate_t state,
}
const lws_ss_info_t ssi_sai_builder = {
- .handle_offset = offsetof(struct sai_plat_master, ss),
- .opaque_user_data_offset = offsetof(struct sai_plat_master, opaque_data),
+ .handle_offset = offsetof(struct sai_plat_server, ss),
+ .opaque_user_data_offset = offsetof(struct sai_plat_server, opaque_data),
.rx = saib_m_rx,
.tx = saib_m_tx,
.state = saib_m_state,
- .user_alloc = sizeof(struct sai_plat_master),
+ .user_alloc = sizeof(struct sai_plat_server),
.streamtype = "sai_builder"
};
diff --git a/src/builder/b-conf.c b/src/builder/b-conf.c
index f037a5f..eac3467 100644
--- a/src/builder/b-conf.c
+++ b/src/builder/b-conf.c
@@ -49,7 +49,7 @@ static const char * const paths[] = {
"platforms[].instances",
"platforms[].env[].*",
"platforms[].env[]",
- "platforms[].masters",
+ "platforms[].servers",
"platforms[]",
};
@@ -59,7 +59,7 @@ enum enum_paths {
LEJPM_PLATFORMS_INSTANCES,
LEJPM_PLATFORMS_ENV_ITEM,
LEJPM_PLATFORMS_ENV,
- LEJPM_PLATFORMS_MASTERS,
+ LEJPM_PLATFORMS_SERVERS,
LEJPM_PLATFORMS,
};
@@ -71,7 +71,7 @@ struct jpargs {
struct sai_platform *pl;
- int next_master_index;
+ int next_server_index;
int next_plat_index;
};
@@ -79,8 +79,8 @@ static signed char
saib_conf_cb(struct lejp_ctx *ctx, char reason)
{
struct jpargs *a = (struct jpargs *)ctx->user;
- sai_plat_master_ref_t *mref;
- struct sai_plat_master *cm;
+ sai_plat_server_ref_t *mref;
+ struct sai_plat_server *cm;
struct lws_ss_handle *h;
const char **pp, *pq;
char temp[65];
@@ -175,41 +175,41 @@ saib_conf_cb(struct lejp_ctx *ctx, char reason)
a->sai_plat->index = a->next_plat_index++;
break;
- case LEJPM_PLATFORMS_MASTERS:
+ case LEJPM_PLATFORMS_SERVERS:
mref = lwsac_use_zero(&a->builder->conf_head, sizeof(*mref),
512);
/*
* The builder as a whole maintains only one SS connection to
- * each master. If there are multiple platforms supported by
- * the builder that want to accept tasks from the same master,
- * only the first platform creates the SS to the master, and
+ * each server. If there are multiple platforms supported by
+ * the builder that want to accept tasks from the same server,
+ * only the first platform creates the SS to the server, and
* the others just use that.
*
- * The sai_plat_master is instantiated as the ss userdata.
+ * The sai_plat_server is instantiated as the ss userdata.
*
* So let's see if we already have the connection created
- * for this master...
+ * for this server...
*/
lws_start_foreach_dll(struct lws_dll2 *, p,
- a->builder->sai_plat_master_owner.head) {
- cm = lws_container_of(p, sai_plat_master_t, list);
+ a->builder->sai_plat_server_owner.head) {
+ cm = lws_container_of(p, sai_plat_server_t, list);
if (!strncmp(ctx->buf, cm->url, ctx->npos)) {
/* we already have a logical connection... */
mref->spm = cm;
cm->refcount++;
lws_dll2_add_tail(&mref->list,
- &a->sai_plat->masters);
+ &a->sai_plat->servers);
return 0;
}
} lws_end_foreach_dll(p);
/*
- * This is the first plat that wants to talk to this master,
+ * This is the first plat that wants to talk to this server,
* we need to create the logical SS connection
*/
@@ -220,10 +220,10 @@ saib_conf_cb(struct lejp_ctx *ctx, char reason)
return -1;
}
- cm = lws_ss_to_user_object(h); /* the sai_plat_master object */
- cm->index = a->next_master_index++;
+ cm = lws_ss_to_user_object(h); /* the sai_plat_server object */
+ cm->index = a->next_server_index++;
- /* hook the ss up to the master url */
+ /* hook the ss up to the server url */
cm->url = lwsac_use(&a->builder->conf_head,
2 *(ctx->npos + 1), 512);
@@ -256,13 +256,13 @@ saib_conf_cb(struct lejp_ctx *ctx, char reason)
while (strchr(cm->name, '/'))
*strchr(cm->name, '/') = '_';
- /* add us to the builder list of unique masters */
- lws_dll2_add_head(&cm->list, &a->builder->sai_plat_master_owner);
+ /* add us to the builder list of unique servers */
+ lws_dll2_add_head(&cm->list, &a->builder->sai_plat_server_owner);
- /* add us to this platforms's list of masters it accepts */
+ /* add us to this platforms's list of servers it accepts */
mref->spm = cm;
cm->refcount++;
- lws_dll2_add_tail(&mref->list, &a->sai_plat->masters);
+ lws_dll2_add_tail(&mref->list, &a->sai_plat->servers);
lws_ss_client_connect(h);
diff --git a/src/builder/b-mirror.c b/src/builder/b-mirror.c
index 02c0df7..a9ba28d 100644
--- a/src/builder/b-mirror.c
+++ b/src/builder/b-mirror.c
@@ -415,7 +415,7 @@ saib_mirror_task(void *user, enum lws_threadpool_task_status s)
/*
* Leave in NSSTATE_CHECKOUT and come back to
* continue with checking out after we have
- * sync'd with comms to master
+ * sync'd with comms to server
*/
ns->state = NSSTATE_CHECKOUT;
diff --git a/src/builder/b-nspawn.c b/src/builder/b-nspawn.c
index e55542e..f755f41 100644
--- a/src/builder/b-nspawn.c
+++ b/src/builder/b-nspawn.c
@@ -238,7 +238,7 @@ saib_spawn(struct sai_nspawn *ns)
"/bin/ps",
NULL
};
- char *env[] = {
+ const char *env[] = {
"PATH=/usr/bin:/bin",
"LANG=en_US.UTF-8",
NULL
diff --git a/src/builder/b-private.h b/src/builder/b-private.h
index 7918199..ff5327d 100644
--- a/src/builder/b-private.h
+++ b/src/builder/b-private.h
@@ -105,13 +105,13 @@ struct sai_nspawn {
lws_sorted_usec_list_t sul_task_cancel;
sai_plat_t *sp; /* the sai_plat */
- struct sai_plat_master *spm; /* the sai plat / master with the ss / wsi */
+ struct sai_plat_server *spm; /* the sai plat / server with the ss / wsi */
size_t chunk_cache_size;
- const char *master_name; /* sai-master name who triggered this, eg, 'warmcat' */
+ const char *server_name; /* sai-server name who triggered this, eg, 'warmcat' */
const char *project_name; /* name of the git project, eg, 'libwebsockets' */
- const char *ref; /* remote refname, eg 'master' */
+ const char *ref; /* remote refname, eg 'server' */
const char *hash; /* remote hash */
const char *git_repo_url;
@@ -148,7 +148,7 @@ typedef struct sai_mirror_instance {
struct sai_builder {
lws_dll2_owner_t sai_plat_owner; /* list of platforms we offer */
- lws_dll2_owner_t sai_plat_master_owner; /* masters we connect to */
+ lws_dll2_owner_t sai_plat_server_owner; /* servers we connect to */
lws_dll2_owner_t devices_owner; /* sai_serial_t */
struct lwsac *conf_head;
@@ -195,7 +195,7 @@ int
saib_prepare_mount(struct sai_builder *b, struct sai_nspawn *ns);
int
-saib_ws_json_rx_builder(struct sai_plat_master *spm, const void *in, size_t len);
+saib_ws_json_rx_builder(struct sai_plat_server *spm, const void *in, size_t len);
int
saib_generate(struct sai_plat *sp, char *buf, int len);
@@ -216,7 +216,7 @@ struct ws_capture_chunk *
saib_log_chunk_create(struct sai_nspawn *ns, void *buf, size_t len, int channel);
int
-saib_queue_task_status_update(sai_plat_t *sp, struct sai_plat_master *spm,
+saib_queue_task_status_update(sai_plat_t *sp, struct sai_plat_server *spm,
const char *rej_task_uuid);
int
diff --git a/src/builder/b-sai.c b/src/builder/b-sai.c
index b3d0b3d..0d61db2 100644
--- a/src/builder/b-sai.c
+++ b/src/builder/b-sai.c
@@ -19,7 +19,7 @@
* MA 02110-1301 USA
*
* Sai-builder uses a secure streams template to make the client connections to
- * the masters listed in /etc/sai/builder/conf JSON config. The URL in the
+ * the servers listed in /etc/sai/builder/conf JSON config. The URL in the
* config is substituted for the endpoint URL at runtime.
*
* See b-comms.c for the secure stream template and callbacks for this.
@@ -75,7 +75,7 @@ static const char * const default_ss_policy =
"\"s\": ["
/*
- * The main connection to a master carrying events and logs
+ * The main connection to a server carrying events and logs
*/
"{\"sai_builder\": {"
"\"endpoint\":" "\"${url}\","
@@ -90,7 +90,7 @@ static const char * const default_ss_policy =
"]"
"}},"
/*
- * Ephemeral connections to the same master carrying artifact
+ * Ephemeral connections to the same server carrying artifact
* JSON + bulk data
*/
"{\"sai_artifact\": {"
@@ -177,7 +177,7 @@ app_system_state_nf(lws_state_manager_t *mgr, lws_state_notify_link_t *link,
break;
/*
- * The builder JSON conf listed masters we want to connect to,
+ * The builder JSON conf listed servers we want to connect to,
* let's collect the config, make a ss for each and add the
* saim into an lws_dll2 list owned by
* builder->builder->sai_plat_owner
@@ -456,12 +456,12 @@ int main(int argc, const char **argv)
bail:
- /* destroy the unique masters */
+ /* destroy the unique servers */
lws_start_foreach_dll_safe(struct lws_dll2 *, p, p1,
- builder.sai_plat_master_owner.head) {
- struct sai_plat_master *cm = lws_container_of(p,
- struct sai_plat_master, list);
+ builder.sai_plat_server_owner.head) {
+ struct sai_plat_server *cm = lws_container_of(p,
+ struct sai_plat_server, list);
lws_dll2_remove(&cm->list);
lws_ss_destroy(&cm->ss);
diff --git a/src/builder/b-task.c b/src/builder/b-task.c
index d20dcdd..1635f79 100644
--- a/src/builder/b-task.c
+++ b/src/builder/b-task.c
@@ -81,11 +81,11 @@ saib_set_ns_state(struct sai_nspawn *ns, int state)
}
/*
- * update all masters we're connected to about builder status / optional reject
+ * update all servers we're connected to about builder status / optional reject
*/
int
-saib_queue_task_status_update(sai_plat_t *sp, struct sai_plat_master *spm,
+saib_queue_task_status_update(sai_plat_t *sp, struct sai_plat_server *spm,
const char *rej_task_uuid)
{
struct sai_rejection *rej;
@@ -164,7 +164,7 @@ saib_task_destroy(struct sai_nspawn *ns)
ns->sp->ongoing--;
/*
- * Schedule informing all the masters we're connected to
+ * Schedule informing all the servers we're connected to
*/
saib_queue_task_status_update(ns->sp, ns->spm, NULL);
@@ -260,7 +260,7 @@ artifact_glob_cb(void *data, const char *path)
/*
* We need to set the metadata items for the post urlargs. spm->url is
* something like "wss://warmcat.com/sai/builder"... we will send JSON
- * on this connection first and that will be understood by the master
+ * on this connection first and that will be understood by the server
* as meaning the bulk data follows.
*/
@@ -405,7 +405,7 @@ saib_sul_task_cancel(struct lws_sorted_usec_list *sul)
}
int
-saib_ws_json_rx_builder(struct sai_plat_master *spm, const void *in, size_t len)
+saib_ws_json_rx_builder(struct sai_plat_server *spm, const void *in, size_t len)
{
struct lws_threadpool_create_args tca;
struct lws_threadpool_task_args tpa;
@@ -453,7 +453,7 @@ saib_ws_json_rx_builder(struct sai_plat_master *spm, const void *in, size_t len)
* Master is requesting that a platform adopt a task...
*
* Multiple platforms may be using this connection to a given
- * master so we have to disambiguate which platform he's
+ * server so we have to disambiguate which platform he's
* tasking first.
*/
@@ -483,9 +483,9 @@ saib_ws_json_rx_builder(struct sai_plat_master *spm, const void *in, size_t len)
/*
* Look for a spare nspawn...
*
- * We may connect to multiple masters and it's asynchronous
- * which master may have tasked us first, so it's not that
- * unusual to reject a task the master thought we could have
+ * We may connect to multiple servers and it's asynchronous
+ * which server may have tasked us first, so it's not that
+ * unusual to reject a task the server thought we could have
* taken
*/
@@ -508,7 +508,7 @@ saib_ws_json_rx_builder(struct sai_plat_master *spm, const void *in, size_t len)
/*
* Full up... reject the task and update every
- * master's model of our task load status
+ * server's model of our task load status
*/
lwsl_notice("%s: plat '%s': no idle nspawn (of %d), "
@@ -538,10 +538,10 @@ saib_ws_json_rx_builder(struct sai_plat_master *spm, const void *in, size_t len)
*p = '_';
/*
- * unique for remote master name ("warmcat"),
+ * unique for remote server name ("warmcat"),
* project name ("libwebsockets")
*/
- ns->master_name = spm->name;
+ ns->server_name = spm->name;
ns->project_name = task->repo_name;
if (!strncmp(task->git_ref, "refs/heads/", 11))
ns->ref = task->git_ref + 11;
@@ -574,7 +574,7 @@ saib_ws_json_rx_builder(struct sai_plat_master *spm, const void *in, size_t len)
lws_snprintf(ns->fsm.ovname, sizeof(ns->fsm.ovname), "%d-%d.%d",
spm->index, ns->sp->index, ns->instance_idx);
- lwsl_notice("%s: master %s\n", __func__, ns->master_name);
+ lwsl_notice("%s: server %s\n", __func__, ns->server_name);
lwsl_notice("%s: project %s\n", __func__, ns->project_name);
lwsl_notice("%s: ref %s\n", __func__, ns->ref);
lwsl_notice("%s: hash %s\n", __func__, ns->hash);
@@ -682,7 +682,7 @@ saib_ws_json_rx_builder(struct sai_plat_master *spm, const void *in, size_t len)
* after that needs to adjust sp->ongoing accordingly
*/
- lwsl_warn("%s: enqueued mirror thread\n", __func__);
+ lwsl_warn("%s: enqueued mirror thread, ns->tp_task %p\n", __func__, ns->tp_task);
lwsl_notice("%s: ongoing %d -> %d\n", __func__, sp->ongoing,
sp->ongoing + 1);
@@ -692,7 +692,7 @@ saib_ws_json_rx_builder(struct sai_plat_master *spm, const void *in, size_t len)
/*
* Let the mirror thread get on with things...
*
- * When we took on a task, we should inform any masters we're
+ * When we took on a task, we should inform any servers we're
* connected to about our change in task load status
*/
diff --git a/src/common/include/private.h b/src/common/include/private.h
index 09e1ff7..dfb6c43 100644
--- a/src/common/include/private.h
+++ b/src/common/include/private.h
@@ -18,7 +18,7 @@
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
* MA 02110-1301 USA
*
- * structs common to builder and master
+ * structs common to builder and server
*/
//#include <sai_config_private.h>
@@ -40,6 +40,7 @@ typedef enum {
SAIES_FAIL = 4,
SAIES_CANCELLED = 5,
SAIES_BEING_BUILT_HAS_FAILURES = 6,
+ SAIES_DELETED = 7,
} sai_event_state_t;
enum {
@@ -50,7 +51,7 @@ enum {
};
/*
- * Builder is indicating he can't take the task and master should free it up
+ * Builder is indicating he can't take the task and server should free it up
* and try another builder.
*/
@@ -96,7 +97,7 @@ typedef struct {
struct lwsac *ac_task_container;
- const char *master_name; /* used in offer */
+ const char *server_name; /* used in offer */
const char *repo_name; /* used in offer */
const char *git_ref; /* used in offer */
const char *git_hash; /* used in offer */
@@ -119,7 +120,7 @@ typedef struct sai_event {
char hash[65];
char uuid[65];
char source_ip[32];
- void *pdb; /* master only, sqlite3 */
+ void *pdb; /* server only, sqlite3 */
uint64_t created;
uint64_t last_updated;
sai_event_state_t state;
@@ -160,13 +161,13 @@ typedef struct {
struct sai_plat;
/*
- * One SS per unique master the builder connects to; one of these as the SS
+ * One SS per unique server the builder connects to; one of these as the SS
* userdata object
*
- * May be in use by multiple plats offered by same builder to same master
+ * May be in use by multiple plats offered by same builder to same server
*/
-typedef struct sai_plat_master {
+typedef struct sai_plat_server {
lws_dll2_t list;
lws_dll2_owner_t rejection_list;
@@ -187,7 +188,7 @@ typedef struct sai_plat_master {
int index; /* used to create unique build dir path */
uint16_t retries;
-} sai_plat_master_t;
+} sai_plat_server_t;
struct sai_env {
lws_dll2_t list;
@@ -196,26 +197,26 @@ struct sai_env {
const char *value;
};
-typedef struct sai_plat_master_ref {
+typedef struct sai_plat_server_ref {
lws_dll2_t list;
- sai_plat_master_t *spm;
-} sai_plat_master_ref_t;
+ sai_plat_server_t *spm;
+} sai_plat_server_ref_t;
/*
* One of these instantiated per platform instance
*
- * It lists a sai_plat_master per master / ss that can use the platform
+ * It lists a sai_plat_server per server / ss that can use the platform
*
* It contains an nspawn for each platform builder instance
*
- * It's also used as the object on master side that represents a builder /
+ * It's also used as the object on server side that represents a builder /
* platform instance and status.
*/
typedef struct sai_plat {
lws_dll2_t sai_plat_list;
- lws_dll2_owner_t masters; /* list of sai_plat_master_ref_t */
+ lws_dll2_owner_t servers; /* list of sai_plat_server_ref_t */
lws_dll2_owner_t chunk_cache;
const char *name;
@@ -225,7 +226,7 @@ typedef struct sai_plat {
lws_dll2_owner_t nspawn_owner;
struct lwsac *deserialization_ac;
- struct lws *wsi; /* master side only */
+ struct lws *wsi; /* server side only */
lws_dll2_owner_t env_head;
@@ -249,6 +250,17 @@ typedef enum {
SAILOGA_STARTED,
} sai_log_action_t;
+typedef struct sai_browse_rx_evinfo {
+ char event_hash[65];
+ int state;
+} sai_browse_rx_evinfo_t;
+
+typedef struct sai_browse_rx_taskinfo {
+ char task_hash[65];
+ unsigned int log_start;
+ uint8_t logs;
+} sai_browse_rx_taskinfo_t;
+
extern const lws_struct_map_t
lsm_schema_json_map_task[],
lsm_schema_sq3_map_task[],
diff --git a/src/common/struct-metadata.c b/src/common/struct-metadata.c
index 8bfbdde..67ff0ed 100644
--- a/src/common/struct-metadata.c
+++ b/src/common/struct-metadata.c
@@ -1,5 +1,5 @@
/*
- * Sai master - ./src/master/struct-metadata.c
+ * Sai server - ./src/server/struct-metadata.c
*
* Copyright (C) 2019 - 2020 Andy Green <andy@warmcat.com>
*
@@ -18,7 +18,7 @@
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
* MA 02110-1301 USA
*
- * lws_struct metadata for structs common to builder and master
+ * lws_struct metadata for structs common to builder and server
*/
static const lws_struct_map_t lsm_plat[] = {
@@ -87,7 +87,7 @@ const lws_struct_map_t lsm_task[] = {
LSM_CARRAY (sai_task_t, event_uuid, "event_uuid"),
LSM_CARRAY (sai_task_t, uuid, "uuid"),
LSM_CARRAY (sai_task_t, builder_name, "builder_name"),
- LSM_STRING_PTR (sai_task_t, master_name, "master_name"),
+ LSM_STRING_PTR (sai_task_t, server_name, "server_name"),
LSM_STRING_PTR (sai_task_t, repo_name, "repo_name"),
LSM_STRING_PTR (sai_task_t, git_ref, "git_ref"),
LSM_STRING_PTR (sai_task_t, git_hash, "git_hash"),
@@ -103,7 +103,7 @@ const lws_struct_map_t lsm_schema_sq3_map_task[] = {
LSM_SCHEMA_DLL2 (sai_task_t, list, NULL, lsm_task, "tasks"),
};
-/* builder -> master */
+/* builder -> server */
const lws_struct_map_t lsm_task_rej[] = {
LSM_CARRAY (sai_rejection_t, host_platform, "host_platform"),
@@ -117,7 +117,7 @@ const lws_struct_map_t lsm_schema_json_task_rej[] = {
"com.warmcat.sai.taskrej")
};
-/* master -> builder */
+/* server -> builder */
const lws_struct_map_t lsm_task_cancel[] = {
LSM_CARRAY (sai_cancel_t, task_uuid, "task_uuid"),
@@ -173,9 +173,9 @@ const lws_struct_map_t lsm_artifact[] = {
LSM_UNSIGNED (sai_artifact_t, uid, "uid"),
LSM_CARRAY (sai_artifact_t, task_uuid, "task_uuid"),
LSM_CARRAY (sai_artifact_t, blob_filename, "blob_filename"),
- /* created master-side, sent back with valid upload, checked at master */
+ /* created server-side, sent back with valid upload, checked at server */
LSM_CARRAY (sai_artifact_t, artifact_up_nonce, "artifact_up_nonce"),
- /* created master-side (not sent to builder), used in artifact links we generate */
+ /* created server-side (not sent to builder), used in artifact links we generate */
LSM_CARRAY (sai_artifact_t, artifact_down_nonce, "artifact_down_nonce"),
LSM_BLOB_PTR (sai_artifact_t, blob, "blob"),
LSM_UNSIGNED (sai_artifact_t, timestamp, "timestamp"),
diff --git a/src/device/d-sai.c b/src/device/d-sai.c
index 80d1b38..99237b3 100644
--- a/src/device/d-sai.c
+++ b/src/device/d-sai.c
@@ -52,6 +52,8 @@
#if defined(__linux__)
#include <sys/prctl.h>
#endif
+#include <sys/types.h>
+#include <sys/wait.h>
#include "d-private.h"
diff --git a/src/expect/e-sai.c b/src/expect/e-sai.c
index 56585a9..c90a796 100644
--- a/src/expect/e-sai.c
+++ b/src/expect/e-sai.c
@@ -25,7 +25,7 @@
*
* Sai-expect opportunistically opens ttys exported by sai-device in the
* environment, proxies logs to sai-builder (which forwards them to the sai-
- * master that is storing them with the task), and parses the received tty
+ * server that is storing them with the task), and parses the received tty
* data and emits strings as requested by its arguments.
*/
diff --git a/src/master/CMakeLists.txt b/src/master/CMakeLists.txt
deleted file mode 100644
index 5c41dc7..0000000
--- a/src/master/CMakeLists.txt
+++ /dev/null
@@ -1,141 +0,0 @@
-set(SUB "sai-master")
-
-set(CPACK_DEBIAN_MASTER_PACKAGE_NAME ${SUB})
-
-set(SRCS
- m-sai.c
- m-conf.c
- m-notification.c
- m-comms.c
- m-ws-builder.c
- m-task.c
- m-central.c
- m-artifact.c
- m-overview.c
- m-ws-browser.c
-)
-
-set(requirements 1)
-require_lws_config(LWS_WITH_STRUCT_SQLITE3 1 requirements)
-require_lws_config(LWS_WITH_SERVER 1 requirements)
-require_lws_config(LWS_WITH_GENCRYPTO 1 requirements)
-require_lws_config(LWS_WITH_UNIX_SOCK 1 requirements)
-
-if (requirements)
- add_executable(${SUB} ${SRCS})
- if (APPLE)
- set_property(TARGET sai-master PROPERTY MACOSX_RPATH YES)
- endif()
-
- #
- # sqlite3 paths (master)
- #
- if (SAI_MASTER)
- find_path( SQLITE3_INC_PATH NAMES "sqlite3.h")
- find_library(SQLITE3_LIB_PATH NAMES "sqlite3")
-
- if (SQLITE3_INC_PATH AND SQLITE3_LIB_PATH)
- include_directories(BEFORE "${SQLITE3_INC_PATH}")
- else()
- message(FATAL_ERROR " Unable to find sqlite3")
- endif()
-
- target_link_libraries(${SUB} ${SQLITE3_LIB_PATH})
- endif()
-
- include_directories(BEFORE "${SAI_LWS_INC_PATH}")
-
- CHECK_C_SOURCE_COMPILES("#include <libwebsockets.h>\nint
- main(void) {\n#if defined(LWS_HAVE_LIBCAP)\n return
- 0;\n#else\n fail;\n#endif\n return 0;\n}\n" HAS_LIBCAP)
- if (HAS_LIBCAP)
- find_library(CAP_LIB_PATH NAMES "cap")
- endif()
-
- target_link_libraries(${SUB} ${SAI_LWS_LIB_PATH})
-
- if (LWS_OPENSSL_LIBRARIES)
- target_link_libraries(${SUB} ${LWS_OPENSSL_LIBRARIES})
- endif()
-
- if (SAI_EXT_PTHREAD_LIBRARIES)
- target_link_libraries(${SUB} ${SAI_EXT_PTHREAD_LIBRARIES})
- endif()
-
- if (HAS_LIBCAP)
- target_link_libraries(${SUB} ${CAP_LIB_PATH})
- endif()
-
- if (MSVC)
- target_link_libraries(${SUB} ws2_32.lib userenv.lib psapi.lib iphlpapi.lib)
- endif()
-
- install(TARGETS ${SUB}
- RUNTIME DESTINATION "${BIN_DIR}" COMPONENT master)
- install(
- FILES ../../assets/index.html
- ../../assets/OpenSans-Light.ttf
- ../../assets/rebuild.png
- ../../assets/delete.png
- ../../assets/passed.svg
- ../../assets/failed.svg
- ../../assets/sai.css
- ../../assets/sai.js
- ../../assets/sai.svg
- ../../assets/sai-icon.svg
- ../../assets/builder.png
- ../../assets/favicon.ico
- ../../assets/strict-csp.svg
- ../../assets/lws-common.js
- ../../assets/gs/lwsgs.js
- ../../assets/gs/lwsgs.css
- ../../assets/arch-aarch64-a72-bcm2711-rpi4.svg
- ../../assets/arch-aarch64.svg
- ../../assets/arch-arm32-m4-mt7697-usi.svg
- ../../assets/arch-arm32.svg
- ../../assets/arch-freertos.svg
- ../../assets/arch-riscv64-virt.svg
- ../../assets/arch-riscv.svg
- ../../assets/arch-x86_64-amd.svg
- ../../assets/arch-x86_64-intel-i3.svg
- ../../assets/arch-x86_64-intel.svg
- ../../assets/arch-xl6-esp32.svg
- ../../assets/artifact.svg
- ../../assets/builder.svg
- ../../assets/decal-2.svg
- ../../assets/decal-3.svg
- ../../assets/decal-4.svg
- ../../assets/decal-5.svg
- ../../assets/decal-6.svg
- ../../assets/freertos-espidf.svg
- ../../assets/freertos-linkit.svg
- ../../assets/git.svg
- ../../assets/jsplease.svg
- ../../assets/linux-android.svg
- ../../assets/linux-centos-8.svg
- ../../assets/linux-fedora-32-riscv.svg
- ../../assets/linux-fedora-32.svg
- ../../assets/linux-gentoo.svg
- ../../assets/linux-ubuntu-1804.svg
- ../../assets/linux-ubuntu-2004.svg
- ../../assets/linux-ubuntu-xenial-amd64.svg
- ../../assets/netbsd-iOS.svg
- ../../assets/netbsd-OSX-catalina.svg
- ../../assets/sai-event.svg
- ../../assets/sai-icon.svg
- ../../assets/sai.svg
- ../../assets/stop.svg
- ../../assets/strict-csp.svg
- ../../assets/tc-gcc.svg
- ../../assets/tc-llvm.svg
- ../../assets/tc-mingw32.svg
- ../../assets/tc-mingw64.svg
- ../../assets/tc-msvc.svg
- ../../assets/ubuntu-focal-aarch64.svg
- ../../assets/virt-qemu.svg
- ../../assets/windows-10.svg
-
- DESTINATION "${DATA_DIR}/sai/assets"
- COMPONENT master)
-
-endif(requirements)
diff --git a/src/master/m-artifact.c b/src/master/m-artifact.c
deleted file mode 100644
index 527abc2..0000000
--- a/src/master/m-artifact.c
+++ /dev/null
@@ -1,144 +0,0 @@
-/*
- * Sai master
- *
- * Copyright (C) 2019 - 2020 Andy Green <andy@warmcat.com>
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation:
- * version 2.1 of the License.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301 USA
- */
-
-#include <libwebsockets.h>
-#include <string.h>
-#include <signal.h>
-#include <time.h>
-
-#include "m-private.h"
-
-int
-saim_get_blob(struct vhd *vhd, const char *url, sqlite3 **pdb,
- sqlite3_blob **blob, uint64_t *length)
-{
- char task_uuid[66], event_uuid[34], nonce[34], qu[200],
- esc[66], esc1[34];
- struct lwsac *ac = NULL;
- lws_dll2_owner_t o;
- sai_artifact_t *a;
- const char *p;
- uint64_t rid = 0;
- int n;
-
- /*
- * We get passed the RHS of a URL string like
- *
- * <task_uuid>/<down_nonce>/filename
- *
- * filename is not used for matching, but make sure the client saves it
- * using the name generated along with the link.
- *
- * Extract the pieces from the URL
- */
-
- n = 0;
- p = url;
- while (*p && *p != '/' && n < (int)sizeof(task_uuid) - 1)
- task_uuid[n++] = *p++;
-
- if (n != sizeof(task_uuid) - 2 || *p != '/') {
- lwsl_notice("%s: url layout 1: %d %s\n", __func__, n, url);
- return -1;
- }
-
- task_uuid[n] = '\0';
- p++; /* skip the / */
-
- n = 0;
- while (*p && *p != '/' && n < (int)sizeof(nonce) - 1)
- nonce[n++] = *p++;
-
- if (n != sizeof(nonce) - 2 || *p != '/') {
- lwsl_notice("%s: url layout 2\n", __func__);
- return -1;
- }
-
- sai_task_uuid_to_event_uuid(event_uuid, task_uuid);
-
- /* open the event-specific database object */
-
- if (saim_event_db_ensure_open(vhd, event_uuid, 0, pdb)) {
- lwsl_info("%s: unable to open event-specific database\n",
- __func__);
-
- return -1;
- }
-
- /*
- * Query the artifact he's asking for
- */
-
- lws_sql_purify(esc, task_uuid, sizeof(esc));
- lws_sql_purify(esc1, nonce, sizeof(esc1));
- lws_snprintf(qu, sizeof(qu),
- " and task_uuid='%s' and artifact_down_nonce='%s'",
- esc, esc1);
- n = lws_struct_sq3_deserialize(*pdb, qu, NULL,
- lsm_schema_sq3_map_artifact,
- &o, &ac, 0, 1);
- if (n < 0 || !o.head) {
- lwsl_err("%s: no result from %s\n", __func__, qu);
- goto fail;
- }
-
- a = (sai_artifact_t *)o.head;
-
- *length = a->len;
-
- /*
- * recover the rowid the blob api requires
- */
-
- lws_snprintf(qu, sizeof(qu), "select rowid from artifacts "
- "where artifact_down_nonce='%s'", esc1);
-
- if (sqlite3_exec(*pdb, qu, sai_sql3_get_uint64_cb, &rid, NULL) != SQLITE_OK) {
- lwsl_err("%s: %s: %s: fail\n", __func__, qu, sqlite3_errmsg(*pdb));
- goto fail;
- }
-
- /*
- * Get a read-only handle on the blob
- */
-
- if (sqlite3_blob_open(*pdb, "main", "artifacts", "blob", rid, 0, blob) != SQLITE_OK) {
- lwsl_err("%s: unable to open blob, rid %d\n", __func__, (int)rid);
- goto fail;
- }
-
- lwsac_free(&ac); /* drop the lwsac holding that result */
-
- /*
- * It was successful, *length was set, *pdb and *blob are live handles
- * to the db and the blob itself.
- */
-
- return 0;
-
-fail:
- lwsac_free(&ac);
- saim_event_db_close(vhd, pdb);
-
- lwsl_notice("%s: couldn't find blob %s\n", __func__, url);
-
- return -1;
-}
diff --git a/src/master/m-central.c b/src/master/m-central.c
deleted file mode 100644
index c967625..0000000
--- a/src/master/m-central.c
+++ /dev/null
@@ -1,159 +0,0 @@
-/*
- * Sai master
- *
- * Copyright (C) 2019 - 2020 Andy Green <andy@warmcat.com>
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation:
- * version 2.1 of the License.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301 USA
- *
- *
- * Central dispatcher for jobs from events that made it into the database. This
- * is done in an event-driven way in m-task.c, but management of it also has to
- * be done in the background for when there are no events coming,
- */
-
-#include <libwebsockets.h>
-#include <string.h>
-#include <signal.h>
-#include <time.h>
-
-#include "m-private.h"
-
-extern struct lws_context *context;
-
-static void
-saim_central_clean_abandoned(struct vhd *vhd)
-{
- saim_sqlite_cache_t *sc;
- struct lwsac *ac = NULL;
- lws_usec_t now = lws_now_usecs();
- lws_dll2_owner_t o;
- char s[160];
- int n, nzr;
-
- /*
- * Sqlite cache cleaning
- */
-
- nzr = 0;
- lws_start_foreach_dll_safe(struct lws_dll2 *, p, p1,
- vhd->sqlite3_cache.head) {
- sc = lws_container_of(p, saim_sqlite_cache_t, list);
-
- if (!sc->refcount &&
- (now - sc->idle_since) > (60 * LWS_USEC_PER_SEC)) {
- lwsl_notice("%s: delayed db pool clean %s\n", __func__,
- sc->uuid);
- lws_struct_sq3_close(&sc->pdb);
- lws_dll2_remove(&sc->list);
- free(sc);
- } else
- if (sc->refcount)
- nzr++;
-
- } lws_end_foreach_dll_safe(p, p1);
-
- if (vhd->sqlite3_cache.count)
- lwsl_notice("%s: db pool items: in-use: %d, total: %d\n",
- __func__, nzr, vhd->sqlite3_cache.count);
-
- /*
- * Collect the most recent <=10 events that still feel they're
- * incomplete and may be running something
- */
-
- n = lws_struct_sq3_deserialize(vhd->master.pdb,
- " and (state != 3 and state != 4 and state != 5)",
- NULL, lsm_schema_sq3_map_event, &o,
- &ac, 0, 10);
- if (n < 0 || !o.head)
- return;
-
- /*
- * For each of those events, look for tasks that have been running
- * "too long", eg, builder restarted or lost connection etc
- */
-
- lws_start_foreach_dll(struct lws_dll2 *, p, o.head) {
- sai_event_t *e = lws_container_of(p, sai_event_t, list);
- sqlite3 *pdb = NULL;
-
- if (!saim_event_db_ensure_open(vhd, e->uuid, 0, &pdb)) {
- /*
- * Such tasks should go into CANCELLED
- */
-
- lws_snprintf(s, sizeof(s),
- "update tasks set state=%d where "
- "(state=1 OR state=2) and started < %llu",
- SAIES_CANCELLED, (unsigned long long)
- (lws_now_secs() - (30 * 60)));
-
- if (sqlite3_exec(pdb, s, NULL, NULL, NULL) !=
- SQLITE_OK)
- lwsl_err("%s: %s: %s: fail\n", __func__, s,
- sqlite3_errmsg(pdb));
-
- saim_event_db_close(vhd, &pdb);
- }
-
- } lws_end_foreach_dll(p);
-
- lwsac_free(&ac);
-}
-
-void
-saim_central_cb(lws_sorted_usec_list_t *sul)
-{
- struct vhd *vhd = lws_container_of(sul, struct vhd, sul_central);
-
- /*
- * For each builder connected to us, see if it can handle a new task,
- * and if so, try to select one matching its supported platforms
- */
-
- lws_start_foreach_dll(struct lws_dll2 *, p,
- vhd->master.builder_owner.head) {
- sai_plat_t *cb = lws_container_of(p, sai_plat_t, sai_plat_list);
-
- lwsl_debug("%s: checking tasks %s %d %d %p\n", __func__,
- cb->name, cb->ongoing, cb->instances, cb->wsi);
-
- if (cb->wsi && lws_wsi_user(cb->wsi) &&
- cb->ongoing < cb->instances)
- saim_allocate_task(vhd,
- (struct pss *)lws_wsi_user(cb->wsi),
- cb, cb->platform);
-
- } lws_end_foreach_dll(p);
-
- /*
- * Need to globally check for abandoned tasks periodically
- */
-
- if (!vhd->last_check_abandoned_tasks ||
- lws_now_usecs() > (vhd->last_check_abandoned_tasks +
- (20 * LWS_USEC_PER_SEC))) {
-
- saim_central_clean_abandoned(vhd);
-
- vhd->last_check_abandoned_tasks = lws_now_usecs();
- }
-
- /* check again in 1s */
-
- lws_sul_schedule(context, 0, &vhd->sul_central, saim_central_cb,
- 1 * LWS_US_PER_SEC);
-}
diff --git a/src/master/m-comms.c b/src/master/m-comms.c
deleted file mode 100644
index de18157..0000000
--- a/src/master/m-comms.c
+++ /dev/null
@@ -1,1220 +0,0 @@
-/*
- * Sai master
- *
- * Copyright (C) 2019 - 2020 Andy Green <andy@warmcat.com>
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation:
- * version 2.1 of the License.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301 USA
- *
- * The same ws interface is connected-to by builders (on path /builder), and
- * provides the query transport for browsers (on path /browse).
- *
- * There's a single master slite3 database containing events, and a separate
- * sqlite3 database file for each event, it only contains tasks and logs for
- * the event and can be deleted when the event record associated with it is
- * deleted. This is to keep is scalable when there may be thousands of events
- * and related tasks and logs stored.
- */
-
-#include <libwebsockets.h>
-#include <string.h>
-#include <signal.h>
-#include <time.h>
-#include <stdio.h>
-#include <fcntl.h>
-
-#include "m-private.h"
-
-#include "../common/struct-metadata.c"
-
-typedef enum {
- SJS_CLONING,
- SJS_ASSIGNING,
- SJS_WAITING,
- SJS_DONE
-} sai_job_state_t;
-
-typedef struct sai_job {
- struct lws_dll2 jobs_list;
- char reponame[64];
- char ref[64];
- char head[64];
-
- time_t requested;
-
- sai_job_state_t state;
-
-} sai_job_t;
-
-const lws_struct_map_t lsm_schema_map_ta[] = {
- LSM_SCHEMA (sai_task_t, NULL, lsm_task, "com-warmcat-sai-ta"),
-};
-
-typedef struct sai_auth {
- lws_dll2_t list;
- char name[33];
- char passphrase[65];
- unsigned long since;
- unsigned long last_updated;
-} sai_auth_t;
-
-const lws_struct_map_t lsm_auth[] = {
- LSM_CARRAY (sai_auth_t, name, "name"),
- LSM_CARRAY (sai_auth_t, passphrase, "passphrase"),
- LSM_UNSIGNED (sai_auth_t, since, "since"),
- LSM_UNSIGNED (sai_auth_t, last_updated, "last_updated"),
-};
-
-const lws_struct_map_t lsm_schema_sq3_map_auth[] = {
- LSM_SCHEMA_DLL2 (sai_auth_t, list, NULL, lsm_auth, "auth"),
-};
-
-extern const lws_struct_map_t lsm_schema_sq3_map_event[];
-
-static int
-sai_destroy_builder(struct lws_dll2 *d, void *user)
-{
-// saib_t *b = lws_container_of(d, saib_t, c.builder_list);
-
- lws_dll2_remove(d);
-
- return 0;
-}
-
-static void
-saim_master_destroy(saim_t *master)
-{
- lws_dll2_foreach_safe(&master->builder_owner, NULL, sai_destroy_builder);
-
- lws_struct_sq3_close(&master->pdb);
-}
-
-/* len is typically 16 (event uuid is 32 chars + NUL)
- * But eg, task uuid is concatenated 32-char eventid and 32-char taskid
- */
-
-int
-sai_uuid16_create(struct lws_context *context, char *dest33)
-{
- return lws_hex_random(context, dest33, 33);
-}
-
-int
-sai_sqlite3_statement(sqlite3 *pdb, const char *cmd, const char *desc)
-{
- sqlite3_stmt *sm;
- int n;
-
- if (sqlite3_prepare_v2(pdb, cmd, -1, &sm, NULL) != SQLITE_OK) {
- lwsl_err("%s: Unable to %s: %s\n",
- __func__, desc, sqlite3_errmsg(pdb));
-
- return 1;
- }
-
- n = sqlite3_step(sm);
- sqlite3_reset(sm);
- sqlite3_finalize(sm);
- if (n != SQLITE_DONE) {
- lwsl_err("%s: %d: Unable to perform \"%s\": %s\n", __func__,
- sqlite3_extended_errcode(pdb), desc,
- sqlite3_errmsg(pdb));
- puts(cmd);
-
- return 1;
- }
-
- return 0;
-}
-
-int
-saim_event_db_ensure_open(struct vhd *vhd, const char *event_uuid,
- char create_if_needed, sqlite3 **ppdb)
-{
- char filepath[256], saf[33];
- saim_sqlite_cache_t *sc;
-
- if (*ppdb)
- return 0;
-
- /* do we have this guy cached? */
-
- lws_start_foreach_dll(struct lws_dll2 *, p, vhd->sqlite3_cache.head) {
- sc = lws_container_of(p, saim_sqlite_cache_t, list);
-
- if (!strcmp(event_uuid, sc->uuid)) {
- sc->refcount++;
- *ppdb = sc->pdb;
- return 0;
- }
-
- } lws_end_foreach_dll(p);
-
- /* ... nope, well, let's open and cache him then... */
-
- lws_strncpy(saf, event_uuid, sizeof(saf));
- lws_filename_purify_inplace(saf);
-
- lws_snprintf(filepath, sizeof(filepath), "%s-event-%s.sqlite3",
- vhd->sqlite3_path_lhs, saf);
-
- if (lws_struct_sq3_open(vhd->context, filepath, create_if_needed, ppdb)) {
- lwsl_err("%s: Unable to open db %s: %s\n", __func__,
- filepath, sqlite3_errmsg(*ppdb));
-
- return 1;
- }
-
- /* create / add to the schema for the tables we will have in here */
-
- if (lws_struct_sq3_create_table(*ppdb, lsm_schema_sq3_map_task))
- return 1;
-
- sai_sqlite3_statement(*ppdb, "PRAGMA journal_mode=WAL;", "set WAL");
-
- if (lws_struct_sq3_create_table(*ppdb, lsm_schema_sq3_map_log))
- return 1;
-
- if (lws_struct_sq3_create_table(*ppdb, lsm_schema_sq3_map_artifact))
- return 1;
-
- sc = malloc(sizeof(*sc));
- memset(sc, 0, sizeof(*sc));
- if (!sc) {
- lws_struct_sq3_close(ppdb);
- *ppdb = NULL;
- return 1;
- }
-
- lws_strncpy(sc->uuid, event_uuid, sizeof(sc->uuid));
- sc->refcount = 1;
- sc->pdb = *ppdb;
- lws_dll2_add_tail(&sc->list, &vhd->sqlite3_cache);
-
- return 0;
-}
-
-void
-saim_event_db_close(struct vhd *vhd, sqlite3 **ppdb)
-{
- saim_sqlite_cache_t *sc;
-
- if (!*ppdb)
- return;
-
- /* look for him in the cache */
-
- lws_start_foreach_dll(struct lws_dll2 *, p, vhd->sqlite3_cache.head) {
- sc = lws_container_of(p, saim_sqlite_cache_t, list);
-
- if (sc->pdb == *ppdb) {
- *ppdb = NULL;
- if (--sc->refcount) {
- lwsl_notice("%s: zero refcount to idle\n", __func__);
- /*
- * He's not currently in use then... don't
- * close him immediately, m-central.c has a
- * timer that closes and removes sqlite3
- * cache entries idle for longer than 60s
- */
- sc->idle_since = lws_now_usecs();
- }
-
- return;
- }
-
- } lws_end_foreach_dll(p);
-
- lws_struct_sq3_close(ppdb);
- *ppdb = NULL;
-}
-
-int
-saim_event_db_delete_database(struct vhd *vhd, const char *event_uuid)
-{
- char filepath[256], saf[33];
-
- lws_strncpy(saf, event_uuid, sizeof(saf));
- lws_filename_purify_inplace(saf);
-
- lws_snprintf(filepath, sizeof(filepath), "%s-event-%s.sqlite3",
- vhd->sqlite3_path_lhs, saf);
-
- return unlink(filepath);
-}
-
-
-#if 0
-static void
-saim_all_browser_on_writable(struct vhd *vhd)
-{
- lws_start_foreach_dll(struct lws_dll2 *, mp, vhd->browsers.head) {
- struct pss *pss = lws_container_of(mp, struct pss, same);
-
- lws_callback_on_writable(pss->wsi);
- } lws_end_foreach_dll(mp);
-}
-#endif
-
-typedef enum {
- SHMUT_NONE = -1,
- SHMUT_HOOK,
- SHMUT_BROWSE,
- SHMUT_STATUS,
- SHMUT_ARTIFACTS,
- SHMUT_LOGIN
-} sai_http_murl_t;
-
-static const char * const well_known[] = {
- "/update-hook",
- "/sai/browse",
- "/status",
- "/artifacts/", /* HTTP api for accessing build artifacts */
- "/login"
-};
-
-static const char *hmac_names[] = {
- "sai sha256=",
- "sai sha384=",
- "sai sha512="
-};
-
-void
-mark_pending(struct pss *pss, ws_state state)
-{
- pss->pending |= 1 << state;
- lws_callback_on_writable(pss->wsi);
-}
-
-static int
-browser_upd(struct lws_dll2 *d, void *user)
-{
- struct pss *pss = lws_container_of(d, struct pss, same);
- ws_state state = (ws_state)(intptr_t)user;
-
- mark_pending(pss, state);
-
- return 0;
-}
-
-int
-sai_get_head_status(struct vhd *vhd, const char *projname)
-{
- struct lwsac *ac = NULL;
- lws_dll2_owner_t o;
- sai_event_t *e;
- int state;
-
- if (lws_struct_sq3_deserialize(vhd->master.pdb, NULL, "created ",
- lsm_schema_sq3_map_event, &o, &ac, 0, -1))
- return -1;
-
- if (!o.head)
- return -1;
-
- e = lws_container_of(o.head, sai_event_t, list);
- state = e->state;
-
- lwsac_free(&ac);
-
- return state;
-}
-
-
-static int
-sai_login_cb(void *data, const char *name, const char *filename,
- char *buf, int len, enum lws_spa_fileupload_states state)
-{
- return 0;
-}
-
-static const char * const auth_param_names[] = {
- "lname",
- "lpass",
- "success_redir",
-};
-
-enum enum_param_names {
- EPN_LNAME,
- EPN_LPASS,
- EPN_SUCCESS_REDIR,
-};
-
-static int
-callback_ws(struct lws *wsi, enum lws_callback_reasons reason, void *user,
- void *in, size_t len)
-{
- struct vhd *vhd = (struct vhd *)lws_protocol_vh_priv_get(
- lws_get_vhost(wsi), lws_get_protocol(wsi));
- uint8_t buf[LWS_PRE + 8192], *start = &buf[LWS_PRE], *p = start,
- *end = &buf[sizeof(buf) - LWS_PRE - 1];
- struct pss *pss = (struct pss *)user;
- struct lws_jwt_sign_set_cookie ck;
- sai_http_murl_t mu = SHMUT_NONE;
- char projname[64];
- int n, resp, r;
- const char *cp;
- size_t cml;
-
- (void)end;
- (void)p;
-
- switch (reason) {
- case LWS_CALLBACK_PROTOCOL_INIT:
- vhd = lws_protocol_vh_priv_zalloc(lws_get_vhost(wsi),
- lws_get_protocol(wsi),
- sizeof(struct vhd));
- if (!vhd)
- return -1;
-
- vhd->context = lws_get_context(wsi);
- vhd->vhost = lws_get_vhost(wsi);
-
- if (lws_pvo_get_str(in, "notification-key",
- &vhd->notification_key)) {
- lwsl_err("%s: notification_key pvo required\n", __func__);
- return -1;
- }
-
- if (lws_pvo_get_str(in, "database", &vhd->sqlite3_path_lhs)) {
- lwsl_err("%s: database pvo required\n", __func__);
- return -1;
- }
-
- lws_snprintf((char *)buf, sizeof(buf), "%s-events.sqlite3",
- vhd->sqlite3_path_lhs);
-
- if (lws_struct_sq3_open(vhd->context, (char *)buf, 1,
- &vhd->master.pdb)) {
- lwsl_err("%s: Unable to open session db %s: %s\n",
- __func__, vhd->sqlite3_path_lhs, sqlite3_errmsg(
- vhd->master.pdb));
-
- return -1;
- }
-
- sai_sqlite3_statement(vhd->master.pdb,
- "PRAGMA journal_mode=WAL;", "set WAL");
-
- if (lws_struct_sq3_create_table(vhd->master.pdb,
- lsm_schema_sq3_map_event)) {
- lwsl_err("%s: unable to create event table\n", __func__);
- return -1;
- }
-
- /* auth database */
-
- lws_snprintf((char *)buf, sizeof(buf), "%s-auth.sqlite3",
- vhd->sqlite3_path_lhs);
-
- if (lws_struct_sq3_open(vhd->context, (char *)buf, 1,
- &vhd->master.pdb_auth)) {
- lwsl_err("%s: Unable to open auth db %s: %s\n",
- __func__, vhd->sqlite3_path_lhs, sqlite3_errmsg(
- vhd->master.pdb));
-
- return -1;
- }
-
- if (lws_struct_sq3_create_table(vhd->master.pdb_auth,
- lsm_schema_sq3_map_auth)) {
- lwsl_err("%s: unable to create auth table\n", __func__);
- return -1;
- }
-
- /*
- * jwt-iss
- */
-
- if (lws_pvo_get_str(in, "jwt-iss", &vhd->jwt_issuer)) {
- lwsl_err("%s: jwt-iss required\n", __func__);
- return -1;
- }
-
- /*
- * jwt-aud
- */
-
- if (lws_pvo_get_str(in, "jwt-aud", &vhd->jwt_audience)) {
- lwsl_err("%s: jwt-aud required\n", __func__);
- return -1;
- }
-
- /*
- * auth-alg
- */
-
- if (lws_pvo_get_str(in, "jwt-auth-alg", &cp)) {
- lwsl_err("%s: jwt-auth-alg required\n", __func__);
- return -1;
- }
-
- lws_strncpy(vhd->jwt_auth_alg, cp, sizeof(vhd->jwt_auth_alg));
-
- /*
- * auth-jwk-path
- */
-
- if (lws_pvo_get_str(in, "jwt-auth-jwk-path", &cp)) {
- lwsl_err("%s: jwt-auth-jwk-path required\n", __func__);
- return -1;
- }
-
- n = open(cp, LWS_O_RDONLY);
- if (!n) {
- lwsl_err("%s: can't open auth JWK %s\n", __func__, cp);
- return -1;
- }
- r = read(n, buf, sizeof(buf));
- close(n);
- if (r < 0) {
- lwsl_err("%s: can't read auth JWK %s\n", __func__, cp);
- return -1;
- }
-
- if (lws_jwk_import(&vhd->jwt_jwk_auth, NULL, NULL,
- (const char *)buf, r)) {
- lwsl_notice("%s: Failed to parse JWK key\n", __func__);
- return -1;
- }
-
- lwsl_notice("%s: Auth JWK type %d\n", __func__,
- vhd->jwt_jwk_auth.kty);
-
- lws_sul_schedule(vhd->context, 0, &vhd->sul_central,
- saim_central_cb, 500 * LWS_US_PER_MS);
-
- break;
-
- case LWS_CALLBACK_PROTOCOL_DESTROY:
- saim_master_destroy(&vhd->master);
- goto passthru;
-
- /*
- * receive http hook notifications
- */
-
- case LWS_CALLBACK_HTTP:
-
- resp = HTTP_STATUS_FORBIDDEN;
- pss->vhd = vhd;
-
- /*
- * What's the situation with a JWT cookie? Normal users won't
- * have any, but privileged users will have one, and we should
- * try to confirm it and set the pss auth level accordingly
- */
-
- memset(&ck, 0, sizeof(ck));
- ck.jwk = &vhd->jwt_jwk_auth;
- ck.alg = vhd->jwt_auth_alg;
- ck.iss = vhd->jwt_issuer;
- ck.aud = vhd->jwt_audience;
- ck.cookie_name = "__Host-sai_jwt";
-
- cml = sizeof(buf);
- if (!lws_jwt_get_http_cookie_validate_jwt(wsi, &ck,
- (char *)buf, &cml) &&
- ck.extra_json &&
- !lws_json_simple_strcmp(ck.extra_json, ck.extra_json_len,
- "\"authorized\":", "1")) {
- /* the token allows him to manage us */
- pss->authorized = 1;
- pss->expiry_unix_time = ck.expiry_unix_time;
- lws_strncpy(pss->auth_user, ck.sub,
- sizeof(pss->auth_user));
- } else
- lwsl_err("%s: cookie rejected\n", __func__);
-
- for (n = 0; n < (int)LWS_ARRAY_SIZE(well_known); n++)
- if (!strncmp((const char *)in, well_known[n],
- strlen(well_known[n]))) {
- mu = n;
- break;
- }
-
- pss->our_form = 0;
-
- lwsl_notice("%s: HTTP: xmu = %d\n", __func__, n);
-
- switch (mu) {
-
- case SHMUT_NONE:
- goto passthru;
-
- case SHMUT_HOOK:
- pss->our_form = 1;
- lwsl_notice("LWS_CALLBACK_HTTP: sees hook\n");
- return 0;
-
- case SHMUT_STATUS:
- /*
- * in is a string like /libwebsockets/status.svg
- */
- cp = ((const char *)in) + 7;
- while (*cp == '/')
- cp++;
- n = 0;
- while (*cp != '/' && *cp && (size_t)n < sizeof(projname) - 1)
- projname[n++] = *cp++;
- projname[n] = '\0';
-
- lwsl_notice("%s: status %s\n", __func__, projname);
-
- r = sai_get_head_status(vhd, projname);
- if (r < 2)
- r = 2;
- n = lws_snprintf(projname, sizeof(projname),
- "../decal-%d.svg", r);
-
- if (lws_http_redirect(wsi, 307,
- (unsigned char *)projname, n,
- &p, end) < 0)
- return -1;
-
- goto passthru;
-
- case SHMUT_LOGIN:
- pss->login_form = 1;
- lwsl_notice("LWS_CALLBACK_HTTP: sees login\n");
- return 0;
-
- case SHMUT_ARTIFACTS:
- /*
- * HTTP Bulk GET interface for artifact download
- *
- * /artifacts/<taskhash>/<down_nonce>/filename
- */
- lwsl_notice("%s: SHMUT_ARTIFACTS\n", __func__);
- pss->artifact_offset = 0;
- if (saim_get_blob(vhd, (const char *)in + 11,
- &pss->pdb_artifact,
- &pss->blob_artifact,
- &pss->artifact_length)) {
- lwsl_notice("%s: get_blob failed\n", __func__);
- resp = 404;
- goto http_resp;
- }
-
- /*
- * Well, it seems what he wanted exists..
- */
-
- if (lws_add_http_header_status(wsi, 200, &p, end))
- goto bail;
- if (lws_add_http_header_content_length(wsi,
- (unsigned long)pss->artifact_length,
- &p, end))
- goto bail;
-
- if (lws_add_http_header_by_token(wsi,
- WSI_TOKEN_HTTP_CONTENT_TYPE,
- (uint8_t *)"application/octet-stream",
- 24, &p, end))
- goto bail;
- if (lws_finalize_write_http_header(wsi, start, &p, end))
- goto bail;
-
- lwsl_notice("%s: started artifact transaction %d\n", __func__,
- (int)pss->artifact_length);
-
- lws_callback_on_writable(wsi);
- return 0;
-
- default:
- lwsl_notice("%s: DEFAULT!!!\n", __func__);
- return 0;
- }
-
- resp = HTTP_STATUS_OK;
-
- /* faillthru */
-
-http_resp:
- if (lws_add_http_header_status(wsi, resp, &p, end))
- goto bail;
- if (lws_add_http_header_content_length(wsi, 0, &p, end))
- goto bail;
- if (lws_finalize_write_http_header(wsi, start, &p, end))
- goto bail;
- goto try_to_reuse;
-
-
- case LWS_CALLBACK_HTTP_WRITEABLE:
-
- lwsl_notice("%s: HTTP_WRITEABLE\n", __func__);
-
- if (!pss || !pss->blob_artifact)
- break;
-
- n = lws_ptr_diff(end, start);
- if ((int)(pss->artifact_length - pss->artifact_offset) < n)
- n = (int)(pss->artifact_length - pss->artifact_offset);
-
- if (sqlite3_blob_read(pss->blob_artifact, start, n,
- pss->artifact_offset)) {
- lwsl_err("%s: blob read failed\n", __func__);
- return -1;
- }
-
- pss->artifact_offset += n;
-
- if (lws_write(wsi, start, n,
- pss->artifact_offset != pss->artifact_length ?
- LWS_WRITE_HTTP : LWS_WRITE_HTTP_FINAL) != n)
- return -1;
-
- if (pss->artifact_offset != pss->artifact_length)
- lws_callback_on_writable(wsi);
-
- break;
-
- /*
- * Notifcation POSTs
- */
-
- case LWS_CALLBACK_HTTP_BODY:
-
- if (pss->login_form) {
-
- if (!pss->spa) {
- pss->spa = lws_spa_create(wsi, auth_param_names,
- LWS_ARRAY_SIZE(auth_param_names),
- 1024, sai_login_cb, pss);
- if (!pss->spa) {
- lwsl_err("failed to create spa\n");
- return -1;
- }
- }
-
- goto spa_process;
-
- }
-
- if (!pss->our_form) {
- lwsl_notice("%s: not our form\n", __func__);
- goto passthru;
- }
-
- lwsl_user("LWS_CALLBACK_HTTP_BODY: %d\n", (int)len);
- /* create the POST argument parser if not already existing */
-
- if (!pss->spa) {
- pss->wsi = wsi;
- if (lws_hdr_copy(wsi, pss->notification_sig,
- sizeof(pss->notification_sig),
- WSI_TOKEN_HTTP_AUTHORIZATION) < 0) {
- lwsl_err("%s: failed to get signature hdr\n",
- __func__);
- return -1;
- }
-
- if (lws_hdr_copy(wsi, pss->sn.e.source_ip,
- sizeof(pss->sn.e.source_ip),
- WSI_TOKEN_X_FORWARDED_FOR) < 0)
- lws_get_peer_simple(wsi, pss->sn.e.source_ip,
- sizeof(pss->sn.e.source_ip));
-
- pss->spa = lws_spa_create(wsi, NULL, 0, 1024,
- sai_notification_file_upload_cb, pss);
- if (!pss->spa) {
- lwsl_err("failed to create spa\n");
- return -1;
- }
-
- /* find out the hmac used to sign it */
-
- pss->hmac_type = LWS_GENHMAC_TYPE_UNKNOWN;
- for (n = 0; n < (int)LWS_ARRAY_SIZE(hmac_names); n++)
- if (!strncmp(pss->notification_sig,
- hmac_names[n],
- strlen(hmac_names[n]))) {
- pss->hmac_type = n + 1;
- break;
- }
-
- if (pss->hmac_type == LWS_GENHMAC_TYPE_UNKNOWN) {
- lwsl_notice("%s: unknown sig hash type\n",
- __func__);
- return -1;
- }
-
- /* convert it to binary */
-
- n = lws_hex_to_byte_array(
- pss->notification_sig + strlen(hmac_names[n]),
- (uint8_t *)pss->notification_sig, 64);
-
- if (n != (int)lws_genhmac_size(pss->hmac_type)) {
- lwsl_notice("%s: notifcation hash bad length\n",
- __func__);
-
- return -1;
- }
- }
-
-spa_process:
-
- /* let it parse the POST data */
-
- if (!pss->spa_failed &&
- lws_spa_process(pss->spa, in, (int)len))
- /*
- * mark it as failed, and continue taking body until
- * completion, and return error there
- */
- pss->spa_failed = 1;
-
- break;
-
- case LWS_CALLBACK_HTTP_BODY_COMPLETION:
- lwsl_user("%s: LWS_CALLBACK_HTTP_BODY_COMPLETION: %d\n",
- __func__, (int)len);
-
- if (!pss->our_form && !pss->login_form) {
- lwsl_user("%s: no sai form\n", __func__);
- goto passthru;
- }
-
- /* inform the spa no more payload data coming */
- if (pss->spa)
- lws_spa_finalize(pss->spa);
-
- if (pss->login_form) {
- const char *un, *pw, *sr;
- lws_dll2_owner_t o;
- struct lwsac *ac = NULL;
-
- if (lws_add_http_header_status(wsi,
- HTTP_STATUS_SEE_OTHER, &p, end))
- goto clean_spa;
- if (lws_add_http_header_content_length(wsi, 0, &p, end))
- goto clean_spa;
-
- if (pss->spa_failed)
- goto final;
-
- un = lws_spa_get_string(pss->spa, EPN_LNAME);
- pw = lws_spa_get_string(pss->spa, EPN_LPASS);
- sr = lws_spa_get_string(pss->spa, EPN_SUCCESS_REDIR);
-
- if (!un || !pw || !sr) {
- pss->spa_failed = 1;
- goto final;
- }
-
- lwsl_notice("%s: login attempt %s %s %s\n",
- __func__, un, pw, sr);
-
- /*
- * Try to look up his credentials
- */
-
- lws_sql_purify((char *)buf + 512, un, 34);
- lws_sql_purify((char *)buf + 768, pw, 66);
- lws_snprintf((char *)buf + 256, 256,
- " and name='%s' and passphrase='%s'",
- (const char *)buf + 512,
- (const char *)buf + 768);
- lws_dll2_owner_clear(&o);
- n = lws_struct_sq3_deserialize(pss->vhd->master.pdb_auth,
- (const char *)buf + 256,
- NULL,
- lsm_schema_sq3_map_auth,
- &o, &ac, 0, 1);
- if (n < 0 || !o.head) {
- /* no results, failed */
- lwsl_notice("%s: login attempt %s failed %d\n",
- __func__, (const char *)buf, n);
- lwsac_free(&ac);
- pss->spa_failed = 1;
- goto final;
- }
-
- /* any result in o means a successful match */
-
- lwsac_free(&ac);
-
- /*
- * Produce a signed JWT allowing managing this Sai
- * instance for a short time, and redirect ourselves
- * back to the page we were on
- */
-
-
-
- lwsl_notice("%s: setting cookie\n", __func__);
- /* un is invalidated by destroying the spa */
- memset(&ck, 0, sizeof(ck));
- lws_strncpy(ck.sub, un, sizeof(ck.sub));
- ck.jwk = &vhd->jwt_jwk_auth;
- ck.alg = vhd->jwt_auth_alg;
- ck.iss = vhd->jwt_issuer;
- ck.aud = vhd->jwt_audience;
- ck.cookie_name = "sai_jwt";
- ck.extra_json = "\"authorized\": 1";
- ck.expiry_unix_time = 20 * 60;
-
- if (lws_jwt_sign_token_set_http_cookie(wsi, &ck, &p, end))
- goto clean_spa;
-
- /*
- * Auth succeeded, go to the page the form was on
- */
-
- if (lws_add_http_header_by_token(wsi,
- WSI_TOKEN_HTTP_LOCATION,
- (unsigned char *)sr,
- strlen((const char *)sr),
- &p, end)) {
- goto clean_spa;
- }
-
- if (pss->spa) {
- lws_spa_destroy(pss->spa);
- pss->spa = NULL;
- }
-
- if (lws_finalize_write_http_header(wsi, start, &p, end))
- goto bail;
-
- lwsl_notice("%s: setting cookie OK\n", __func__);
- lwsl_hexdump_notice(start, lws_ptr_diff(p, start));
- return 0;
-
-final:
- /*
- * Auth failed, go back to /
- */
- if (lws_add_http_header_by_token(wsi,
- WSI_TOKEN_HTTP_LOCATION,
- (unsigned char *)"/", 1,
- &p, end)) {
- goto clean_spa;
- }
- if (lws_finalize_write_http_header(wsi, start, &p, end))
- goto bail;
- return 0;
- }
-
- if (pss->spa) {
- lws_spa_destroy(pss->spa);
- pss->spa = NULL;
- }
-
- if (pss->spa_failed)
- lwsl_notice("%s: notification failed\n", __func__);
- else {
- lwsl_notice("%s: notification: %d %s %s %s\n", __func__,
- pss->sn.action, pss->sn.e.hash,
- pss->sn.e.ref, pss->sn.e.repo_name);
-
- /* update connected browsers */
-
- lws_dll2_foreach_safe(&vhd->browsers,
- (void *)WSS_PREPARE_OVERVIEW, browser_upd);
- }
-
- if (lws_return_http_status(wsi,
- pss->spa_failed ? HTTP_STATUS_FORBIDDEN :
- HTTP_STATUS_OK,
- NULL) < 0)
- return -1;
- break;
-
-clean_spa:
- if (pss->spa) {
- lws_spa_destroy(pss->spa);
- pss->spa = NULL;
- }
- pss->spa_failed = 1;
- goto final;
-
- /*
- * ws connections from builders and browsers
- */
-
- case LWS_CALLBACK_FILTER_PROTOCOL_CONNECTION:
- n = lws_hdr_copy(wsi, (char *)buf, sizeof(buf) - 1,
- WSI_TOKEN_GET_URI);
- if (!n)
- buf[0] = '\0';
- //lwsl_notice("%s: checking with lwsgs for ws conn: %s\n",
- // __func__, (const char *)buf);
-
- /*
- * Builders don't authenticate using sessions...
- */
-
- if (n >= 8 && !strncmp((const char *)buf + n - 8,
- "/builder", 8))
- return 0;
-
- return 0;
-#if 0
- /* but everything else does */
-
- car_args.max_len = LWSGS_AUTH_LOGGED_IN | LWSGS_AUTH_VERIFIED;
- car_args.final = 0;
- car_args.chunked = 1; /* ie, we are ws */
-
- in = &car_args;
- goto passthru;
-#endif
-
- case LWS_CALLBACK_ESTABLISHED:
-
- /*
- * What's the situation with a JWT cookie? Normal users won't
- * have any, but privileged users will have one, and we should
- * try to confirm it and set the pss auth level accordingly
- */
-
- memset(&ck, 0, sizeof(ck));
- ck.jwk = &vhd->jwt_jwk_auth;
- ck.alg = vhd->jwt_auth_alg;
- ck.iss = vhd->jwt_issuer;
- ck.aud = vhd->jwt_audience;
- ck.cookie_name = "__Host-sai_jwt";
-
- cml = sizeof(buf);
- if (!lws_jwt_get_http_cookie_validate_jwt(wsi, &ck,
- (char *)buf, &cml) &&
- ck.extra_json &&
- !lws_json_simple_strcmp(ck.extra_json, ck.extra_json_len,
- "\"authorized\":", "1")) {
- /* the token allows him to manage us */
- pss->authorized = 1;
- pss->expiry_unix_time = ck.expiry_unix_time;
- lws_strncpy(pss->auth_user, ck.sub,
- sizeof(pss->auth_user));
- } else
- lwsl_err("%s: cookie rejected\n", __func__);
- pss->wsi = wsi;
- pss->vhd = vhd;
- pss->alang[0] = '\0';
- lws_hdr_copy(wsi, pss->alang, sizeof(pss->alang),
- WSI_TOKEN_HTTP_ACCEPT_LANGUAGE);
-
- if (lws_hdr_total_length(wsi, WSI_TOKEN_GET_URI)) {
- if (lws_hdr_copy(wsi, (char *)start, 64,
- WSI_TOKEN_GET_URI) < 0)
- return -1;
- }
-#if defined(LWS_ROLE_H2)
- else
- if (lws_hdr_copy(wsi, (char *)start, 64,
- WSI_TOKEN_HTTP_COLON_PATH) < 0)
- return -1;
-#endif
-
- if (!memcmp((char *)start, "/sai", 4))
- start += 4;
-
- if (!strncmp((char *)start, "/browse/specific", 16)) {
- const char *spe;
-
- lwsl_info("%s: ESTABLISHED: browser (specific)\n", __func__);
- pss->type = SWT_BROWSE;
- pss->wsi = wsi;
- pss->specific_project[0] = '\0';
- spe = (const char *)start + 16;
- while(*spe == '/')
- spe++;
- n = 0;
- while(*spe && *spe != '/' &&
- (size_t)n < sizeof(pss->specific_project) - 2)
- pss->specific_project[n++] = *spe++;
-
- pss->specific_project[n] = '\0';
-
- strncpy(pss->specific, "refs/heads/",
- sizeof(pss->specific));
- if (lws_get_urlarg_by_name(wsi, "h", pss->specific + 9,
- sizeof(pss->specific) - 9)) {
- memcpy(pss->specific, "refs/heads/", 11);
- pss->specificity = SAIM_SPECIFIC_H;
- } else
- if (lws_get_urlarg_by_name(wsi, "id", (char *)buf,
- sizeof(buf))) {
- lws_strncpy(pss->specific, (const char *)buf + 3,
- sizeof(pss->specific));
- pss->specificity = SAIM_SPECIFIC_ID;
- } else {
- pss->specificity = SAIM_SPECIFIC_H;
- lws_strncpy(pss->specific,
- "refs/heads/master",
- sizeof(pss->specific));
- }
-
- //lwsl_notice("%s: spec %d, '%s'\n", __func__,
- // pss->specificity, pss->specific);
- break;
- }
-
- if (!strcmp((char *)start, "/browse")) {
- lwsl_info("%s: ESTABLISHED: browser\n", __func__);
- pss->type = SWT_BROWSE;
- pss->wsi = wsi;
- break;
- }
- if (!strcmp((char *)start, "/builder")) {
- lwsl_info("%s: ESTABLISHED: builder\n", __func__);
- pss->type = SWT_BUILDER;
- pss->wsi = wsi;
- /*
- * this adds our pss part, but not the logical builder
- * yet, until we get the ws rx
- */
- lws_dll2_add_head(&pss->same, &vhd->builders);
- break;
- }
-
- lwsl_err("%s: unknown URL '%s'\n", __func__, start);
-
- return -1;
-
- case LWS_CALLBACK_CLOSED:
- lwsac_free(&pss->query_ac);
- switch (pss->type) {
- case SWT_BROWSE:
- lwsl_info("%s: CLOSED browse conn\n", __func__);
- lws_dll2_remove(&pss->same);
- lws_dll2_remove(&pss->subs_list);
- lwsac_free(&pss->task_ac);
- lwsac_free(&pss->logs_ac);
- break;
-
- case SWT_BUILDER:
- lwsl_user("%s: CLOSED builder conn\n", __func__);
- /* remove pss from vhd->builders */
- lws_dll2_remove(&pss->same);
-
- /*
- * Destroy any the builder-tracking objects that
- * were using this departing connection
- */
-
- lws_start_foreach_dll_safe(struct lws_dll2 *, p, p1,
- vhd->master.builder_owner.head) {
- sai_plat_t *cb = lws_container_of(p, sai_plat_t,
- sai_plat_list);
-
- if (cb->wsi == wsi) {
- /* remove builder object itself from master list */
- cb->wsi = NULL;
- lws_dll2_remove(&cb->sai_plat_list);
- /*
- * free the deserialized builder object,
- * everything he pointed to was overallocated
- * when his deep copy was made
- */
- free(cb);
- }
-
- } lws_end_foreach_dll_safe(p, p1);
-
- if (pss->blob_artifact) {
- sqlite3_blob_close(pss->blob_artifact);
- pss->blob_artifact = NULL;
- }
-
- if (pss->pdb_artifact) {
- saim_event_db_close(pss->vhd, &pss->pdb_artifact);
- pss->pdb_artifact = NULL;
- }
-
- /* update the browsers about the builder removal */
- lws_dll2_foreach_safe(&vhd->browsers,
- (void *)WSS_PREPARE_BUILDER_SUMMARY, browser_upd);
- break;
- }
- break;
-
- case LWS_CALLBACK_RECEIVE:
-
- if (!pss->vhd)
- pss->vhd = vhd;
-
- switch (pss->type) {
- case SWT_BROWSE:
- // lwsl_user("SWT_BROWSE RX: %d\n", (int)len);
- /*
- * Browser UI sent us something on websockets
- */
- if (saim_ws_json_rx_browser(vhd, pss, in, len))
- return -1;
- break;
-
- case SWT_BUILDER:
- lwsl_info("SWT_BUILDER RX: %d\n", (int)len);
- /*
- * Builder sent us something on websockets
- */
- pss->wsi = wsi;
- if (saim_ws_json_rx_builder(vhd, pss, in, len))
- return -1;
- if (!pss->announced) {
- lws_dll2_foreach_safe(&vhd->browsers,
- (void *)WSS_PREPARE_BUILDER_SUMMARY,
- browser_upd);
- pss->announced = 1;
- }
- break;
- default:
- break;
- }
- break;
-
- case LWS_CALLBACK_SERVER_WRITEABLE:
- if (!vhd) {
- lwsl_notice("%s: no vhd\n", __func__);
- break;
- }
- switch (pss->type) {
- case SWT_BROWSE:
- return saim_ws_json_tx_browser(vhd, pss, buf, sizeof(buf));
- break;
-
- case SWT_BUILDER:
- return saim_ws_json_tx_builder(vhd, pss, buf, sizeof(buf));
- break;
- default:
- break;
- }
- break;
-
- default:
-passthru:
- // if (!pss || !vhd)
- break;
-
- // return vhd->gsp->callback(wsi, reason, pss->pss_gs, in, len);
- }
-
- return lws_callback_http_dummy(wsi, reason, user, in, len);
-
-bail:
- return 1;
-
-try_to_reuse:
- if (lws_http_transaction_completed(wsi))
- return -1;
-
- return 0;
-}
-
-const struct lws_protocols protocol_ws =
- { "com-warmcat-sai", callback_ws, sizeof(struct pss), 0 };
diff --git a/src/master/m-conf.c b/src/master/m-conf.c
deleted file mode 100644
index 498378f..0000000
--- a/src/master/m-conf.c
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Sai master src/master/conf.c
- *
- * Copyright (C) 2019 - 2020 Andy Green <andy@warmcat.com>
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation:
- * version 2.1 of the License.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301 USA
- */
-#include <libwebsockets.h>
-#include <string.h>
-#include <signal.h>
-#include <time.h>
-
-#define SAI_CONFIG_STRING_SIZE (16 * 1024)
-
-struct lws_context *
-sai_lws_context_from_json(const char *config_dir,
- struct lws_context_creation_info *info,
- const struct lws_protocols **pprotocols)
-{
- int cs_len = SAI_CONFIG_STRING_SIZE - 1;
- struct lws_context *context;
- char *cs, *config_strings;
-
- cs = config_strings = malloc(SAI_CONFIG_STRING_SIZE);
- if (!config_strings) {
- lwsl_err("Unable to allocate config strings heap\n");
-
- return NULL;
- }
-
- memset(info, 0, sizeof(*info));
-
- info->external_baggage_free_on_destroy = config_strings;
- info->pt_serv_buf_size = 8192;
- info->options = LWS_SERVER_OPTION_DO_SSL_GLOBAL_INIT |
- LWS_SERVER_OPTION_EXPLICIT_VHOSTS |
- LWS_SERVER_OPTION_HTTP_HEADERS_SECURITY_BEST_PRACTICES_ENFORCE |
- LWS_SERVER_OPTION_VALIDATE_UTF8;
-
- lwsl_notice("Using config dir: \"%s\"\n", config_dir);
-
- /*
- * first go through the config for creating the outer context
- */
- if (lwsws_get_config_globals(info, config_dir, &cs, &cs_len))
- goto init_failed;
-
- context = lws_create_context(info);
- if (context == NULL) {
- /* config_strings freed as 'external baggage' */
- return NULL;
- }
-
- info->pprotocols = pprotocols;
-
- if (lwsws_get_config_vhosts(context, info, config_dir, &cs, &cs_len)) {
- lwsl_err("%s: sai_lws_context_from_json failed\n", __func__);
- lws_context_destroy(context);
-
- return NULL;
- }
-
- return context;
-
-init_failed:
- free(config_strings);
-
- return NULL;
-}
diff --git a/src/master/m-notification.c b/src/master/m-notification.c
deleted file mode 100644
index 207e097..0000000
--- a/src/master/m-notification.c
+++ /dev/null
@@ -1,968 +0,0 @@
-/*
- * Sai master - src/master/notification.c
- *
- * Copyright (C) 2019 - 2020 Andy Green <andy@warmcat.com>
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation:
- * version 2.1 of the License.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301 USA
- *
- * The same ws interface is connected-to by builders (on path /builder), and
- * provides the query transport for browsers (on path /browse).
- */
-
-#include <libwebsockets.h>
-#include "m-private.h"
-
-/* starts from +1 of sai_notification_action_t */
-
-static const char *notifaction_action_names[] = {
- "repo-update"
-};
-
-static const char * const paths[] = {
- "schema",
- "action",
- "repository.name",
- "repository.fetchurl",
- "ref",
- "hash",
- "nonce",
- "saifile_len",
- "saifile",
-};
-
-enum enum_paths {
- LEJPN_SCHEMA,
- LEJPN_ACTION,
- LEJPN_REPOSITORY_NAME,
- LEJPN_REPOSITORY_FETCHURL,
- LEJPN_REF,
- LEJPN_HASH,
- LEJPN_NONCE,
- LEJPN_SAIFILE_LEN,
- LEJPN_SAIFILE,
-};
-
-/*
- * Saifile parser
- */
-
-static const char * const saifile_paths[] = {
- "schema",
- "platforms.*.build",
- "platforms.*.default",
- "platforms.*",
- "configurations.*.prep",
- "configurations.*.cmake",
- "configurations.*.deps",
- "configurations.*.platforms",
- "configurations.*.artifacts",
- "configurations.*.cpack",
- "configurations.*",
-};
-
-enum enum_saifile_paths {
- LEJPNSAIF_SCHEMA,
- LEJPNSAIF_PLAT_BUILD,
- LEJPNSAIF_PLAT_DEFAULT,
- LEJPNSAIF_PLAT_NAME,
- LEJPNSAIF_CONFIGURATIONS_PREP,
- LEJPNSAIF_CONFIGURATIONS_CMAKE,
- LEJPNSAIF_CONFIGURATIONS_DEPS,
- LEJPNSAIF_CONFIGURATIONS_PLATFORMS,
- LEJPNSAIF_CONFIGURATIONS_ARTIFACTS,
- LEJPNSAIF_CONFIGURATIONS_CPACK,
- LEJPNSAIF_CONFIGURATIONS_NAME,
-};
-
-/* do the string subst for ${cmake} etc */
-
-static int
-exp_cmake(void *priv, const char *name, char *out, size_t *pos, size_t olen,
- size_t *exp_ofs)
-{
- sai_notification_t *sn = (sai_notification_t *)priv;
- const char *replace = NULL;
- size_t replace_len, rem_out;
-
- if (!strcmp(name, "prep")) {
- replace = sn->t.prep;
- replace_len = strlen(sn->t.prep);
- goto expand;
- }
-
- if (!strcmp(name, "cmake")) {
- replace = sn->t.cmake;
- replace_len = strlen(sn->t.cmake);
- goto expand;
- }
-
- if (!strcmp(name, "cpack")) {
- replace = sn->t.cpack;
- replace_len = strlen(sn->t.cpack);
- goto expand;
- }
-
- return LSTRX_FATAL_NAME_UNKNOWN;
-
-expand:
- rem_out = olen - *pos; /* remaining rhs */
- replace_len -= *exp_ofs;
- if (replace_len < rem_out)
- rem_out = replace_len;
-
- memcpy(out + *pos, replace + (*exp_ofs), rem_out);
- *exp_ofs += rem_out;
- *pos += rem_out;
-
- if (rem_out == replace_len)
- return LSTRX_DONE;
-
- return LSTRX_FILLED_OUT;
-}
-
-static int
-arg_to_bool(const char *s)
-{
- static const char * const on[] = { "on", "yes", "true" };
- int n = atoi(s);
-
- if (n)
- return 1;
-
- for (n = 0; n < (int)LWS_ARRAY_SIZE(on); n++)
- if (!strcasecmp(s, on[n]))
- return 1;
-
- return 0;
-}
-
-static int
-sai_tuple_element_compare(const char *e1, const char *e2)
-{
- char *p;
-
- // lwsl_notice("%s: comp '%s' '%s'\n", __func__, e1, e2);
-
- if (strlen(e1) > strlen(e2))
- /* can't match if req is longer than plat */
- return -1;
-
- p = strchr(e2, '/');
- if (p && lws_ptr_diff(p, e2) < (int)strlen(e1))
- /* e2 contains a / section terminal inside e1 scope */
- return 1;
-
- return strncmp(e1, e2, strlen(e1));
-}
-
-static int
-sai_tuple_compare(const char *req, int req_len, const char *plat)
-{
- const char *pos = req;
- char e1[96];
- int n;
-
- do {
- n = 0;
- while (n < (int)sizeof(e1) - 1 && req_len && *pos != '/') {
- e1[n++] = *pos++;
- req_len--;
- }
- e1[n] = '\0';
- if (*pos == '/' && req_len) {
- req_len--;
- pos++;
- }
-
- if (n == sizeof(e1) - 1) {
- // lwsl_notice("%s: NOMATCH 3 '%s' '%s'\n", __func__, req, plat);
- /* element too long */
- return 1;
- }
-
- if (n && /* let it pass if empty match section, eg, linux//gcc */
- sai_tuple_element_compare(e1, plat)) {
- // lwsl_notice("%s: NOMATCH 2 '%s' '%s'\n", __func__, req, plat);
- /* section does not match */
- return 1;
- }
-
- while (*plat && *plat != '/')
- plat++;
- if (*plat == '/')
- plat++;
-
- if (!*plat && req_len) {
- // lwsl_notice("%s: NOMATCH 1 '%s' '%s'\n", __func__, req, plat);
- /* FAIL: req had more than we had */
- return 1;
- }
-
- if (!req_len) {
- // lwsl_notice("%s: MATCH '%s' '%s'\n", __func__, req, plat);
- /* MATCH: we matched at least as far as we had */
- return 0;
- }
-
- } while (1);
-
- return 1;
-}
-
-/*
- * We parse the saifile JSON
- */
-
-static signed char
-sai_saifile_lejp_cb(struct lejp_ctx *ctx, char reason)
-{
- struct pss *pss = (struct pss *)ctx->user;
- sai_notification_t *sn = (sai_notification_t *)&pss->sn;
- sqlite3 *pdb = NULL;
- size_t n;
-
- // lwsl_notice("%s: reason %d, %s\n", __func__, reason, ctx->path);
-
- if (reason == LEJPCB_COMPLETE) {
-
- if (!pss->dry)
- lwsl_notice("%s: saifile decode completed\n", __func__);
-
- return 0;
- }
-
- if (reason == LEJPCB_OBJECT_START &&
- ctx->path_match - 1 == LEJPNSAIF_CONFIGURATIONS_NAME) {
- /*
- * We're at the testname part of "testname" : { }
- */
- lws_strncpy(sn->t.taskname, &ctx->path[15],
- sizeof(sn->t.taskname));
- sn->t.prep[0] = '\0';
- sn->t.packages[0] = '\0';
- sn->t.cmake[0] = '\0';
- sn->t.artifacts[0] = '\0';
- sn->explicit_platforms[0] = '\0';
- return 0;
- }
-
- if (reason == LEJPCB_OBJECT_END &&
- ctx->path_match - 1 == LEJPNSAIF_CONFIGURATIONS_NAME &&
- sn->t.taskname[0]) {
- lws_dll2_owner_t owner;
-
- /*
- * We're at the testname part of "testname" : { }
- */
- if (pss->dry) {
- sn->t.taskname[0] = '\0';
-
- return 0;
- }
-
- /*
- * Iterate through each platform creating task entries for this
- * configuration customized for each platform... first time just
- * check the string subst for all of them is OK...
- */
-
- lwsl_notice("%s: Creating task entries for notification\n", __func__);
-
- lws_start_foreach_dll(struct lws_dll2 *, p,
- pss->platform_owner.head) {
- sai_platform_t *pl =
- lws_container_of(p, sai_platform_t, list);
- size_t used_in, used_out;
- struct lws_tokenize ts;
- int n, match = 0;
- lws_strexp_t sx;
-
- if (!pl->nondefault)
- /*
- * If it's a default platform, then match
- * by default
- */
- match = 1;
-
- if (sn->explicit_platforms[0]) {
- char not = 0;
-
- /*
- * If the configuration gives explicit
- * platforms, we have to filter the platform
- * against the comma-separated list.
- *
- * "none" - don't match any plats just because
- * they are default
- *
- * "not plat" - disallow specific plat "plat",
- * implies you're not using "none"
- *
- * "plat" - allow specific plat "plat"
- */
- memset(&ts, 0, sizeof(ts));
- ts.start = (char *)sn->explicit_platforms;
- ts.len = strlen(ts.start);
- ts.flags = LWS_TOKENIZE_F_DOT_NONTERM |
- LWS_TOKENIZE_F_SLASH_NONTERM |
- LWS_TOKENIZE_F_MINUS_NONTERM;
- do {
- ts.e = lws_tokenize(&ts);
- if (ts.e != LWS_TOKZE_TOKEN)
- continue;
-
- lwsl_notice("%s: check %.*s\n", __func__,
- (int)ts.token_len, ts.token);
-
- if (!strncmp(ts.token, "none",
- ts.token_len)) {
- not = match = 0;
- continue;
- }
- if (!strncmp(ts.token, "not",
- ts.token_len)) {
- not = 1;
- continue;
- }
- /*
- * We need to check with per-slash
- * minimal matching, ie, "linux"
- * matches "linux-ubuntu/x86_64/gcc"
- */
- if (!sai_tuple_compare(ts.token,
- ts.token_len, pl->name)) {
- match = !not;
- if (match)
- break;
- }
- not = 0;
- } while (ts.e > 0);
- }
-
- if (match) {
-
- /*
- * The platform build string (pl->build) is the
- * base, with optional entries like ${cmake}
- * that are filled in with the corresponding
- * info from the specific configuration's
- * "cmake" entry (sn->t.cmake)...
- *
- * sn->t.build becomes the string-substituted
- * copy that is serialized
- */
-
- lws_strexp_init(&sx, sn, exp_cmake, sn->t.build,
- sizeof(sn->t.build));
-
- n = lws_strexp_expand(&sx, pl->build,
- strlen(pl->build),
- &used_in, &used_out);
- if (n != LSTRX_DONE) {
- lwsl_notice("%s: strsubst fail n=%d %s %s\n",
- __func__, n, pl->build,
- sn->t.cmake);
- return -1;
- }
- }
- } lws_end_foreach_dll(p);
-
- /*
- * ...now we know the string handling will go well, create the
- * event database file and create task entries in there for this
- * configuration's tasks for each platform
- */
-
- if (saim_event_db_ensure_open(pss->vhd, pss->sn.e.uuid, 1, &pdb)) {
- lwsl_err("%s: unable to open event-specific db\n", __func__);
- return -1;
- }
-
- lws_start_foreach_dll(struct lws_dll2 *, p,
- pss->platform_owner.head) {
- sai_platform_t *pl =
- lws_container_of(p, sai_platform_t, list);
- size_t used_in, used_out;
- struct lws_tokenize ts;
- int n, match = 0;
- lws_strexp_t sx;
-
- if (!pl->nondefault)
- /*
- * If it's a nondefault platform, then match
- * by default
- */
- match = 1;
-
- if (sn->explicit_platforms[0]) {
- char not = 0;
-
- /*
- * If the configuration gives explicit
- * platforms, we have to filter the platform
- * against the comma-separated list.
- *
- * "none" - don't match any plats just because
- * they are default
- *
- * "not plat" - disallow specific plat "plat",
- * implies you're not using "none"
- *
- * "plat" - allow specific plat "plat"
- */
-
- memset(&ts, 0, sizeof(ts));
- ts.start = (char *)sn->explicit_platforms;
- ts.len = strlen(ts.start);
- ts.flags = LWS_TOKENIZE_F_DOT_NONTERM |
- LWS_TOKENIZE_F_SLASH_NONTERM |
- LWS_TOKENIZE_F_MINUS_NONTERM;
-
- do {
- ts.e = lws_tokenize(&ts);
- if (ts.e != LWS_TOKZE_TOKEN)
- continue;
-
- lwsl_notice("%s: check %.*s\n", __func__,
- (int)ts.token_len, ts.token);
-
- if (!strncmp(ts.token, "none",
- ts.token_len)) {
- not = match = 0;
- continue;
- }
- if (!strncmp(ts.token, "not",
- ts.token_len)) {
- not = 1;
- continue;
- }
- if (!sai_tuple_compare(ts.token,
- ts.token_len, pl->name)) {
- match = !not;
- if (match)
- break;
- }
- not = 0;
- } while (ts.e > 0);
- }
-
- if (match) {
- /*
- * For this platform, we want to create a task
- * associated with this event. Tasks and logs
- * associated with an event go in an event-
- * specific database file for scalability.
- */
-
- lws_strexp_init(&sx, sn, exp_cmake, sn->t.build,
- sizeof(sn->t.build));
-
- n = lws_strexp_expand(&sx, pl->build,
- strlen(pl->build),
- &used_in, &used_out);
- if (n != LSTRX_DONE) {
- lwsl_notice("%s: strsubst failed %s %s\n",
- __func__, pl->build,
- sn->t.cmake);
- saim_event_db_close(pss->vhd, &pdb);
- return -1;
- }
-
- /*
- * Prepare a struct of the task object...
- * task uuid is the event uuid and another
- * random 32 chars, so you can always recover
- * the related event uuid from the task uuid
- */
-
- memcpy(pss->sn.t.uuid, pss->sn.e.uuid, 32);
- sai_uuid16_create(lws_get_context(pss->wsi),
- pss->sn.t.uuid + 32);
- strcpy(pss->sn.t.event_uuid, pss->sn.e.uuid);
- pss->sn.t.uid = pss->sn.event_task_index++;
-
- /*
- * This is basically a secret that anything
- * trying to upload an artifact for the task
- * must provide to authenticate.
- */
- sai_uuid16_create(lws_get_context(pss->wsi),
- pss->sn.t.art_up_nonce);
- /*
- * An unrelated secret that anything
- * trying to download an artifact for the task
- * must provide to identify it.
- */
- sai_uuid16_create(lws_get_context(pss->wsi),
- pss->sn.t.art_down_nonce);
-
- pss->sn.t.git_repo_url =
- pss->sn.e.repo_fetchurl;
- pss->sn.e.last_updated =
- (unsigned long long)lws_now_secs();
- pss->sn.e.state = 0;
- lws_strncpy(pss->sn.t.platform, pl->name,
- sizeof(pss->sn.t.platform));
-
- memset(&pss->sn.t.list, 0, sizeof(pss->sn.t.list));
- lws_dll2_owner_clear(&owner);
- lws_dll2_add_head(&pss->sn.t.list, &owner);
-
- /*
- * Create the task in event-specific database
- */
-
- lws_struct_sq3_serialize(pdb,
- lsm_schema_sq3_map_task,
- &owner, pss->sn.t.uid);
- }
-
- } lws_end_foreach_dll(p);
-
- saim_event_db_close(pss->vhd, &pdb);
-
- lwsl_notice("%s: New test '%s', '%s', '%s'\n", __func__,
- sn->t.taskname, sn->t.cmake, sn->t.packages);
-
- sn->t.taskname[0] = '\0';
- return 0;
- }
-
- if (reason == LEJPCB_OBJECT_START &&
- ctx->path_match - 1 == LEJPNSAIF_PLAT_NAME) {
- /*
- * We're at the platformname part of "platformname" : { }
- */
- lws_strncpy(sn->platname, &ctx->path[10], sizeof(sn->platname));
- sn->t.prep[0] = '\0';
- sn->t.cmake[0] = '\0';
- sn->t.cpack[0] = '\0';
- sn->platbuild[0] = '\0';
- sn->nondefault = 0;
- return 0;
- }
-
- if (reason == LEJPCB_OBJECT_END &&
- ctx->path_match - 1 == LEJPNSAIF_PLAT_NAME &&
- sn->platname[0] && sn->platbuild[0]) {
- sai_platform_t *pl;
- uint8_t *plb;
- size_t pnl;
-
- if (pss->dry) {
- sn->t.platform[0] = '\0';
- return 0;
- }
-
- /*
- * We create a platform object with space for its strings after
- */
-
- pnl = strlen(sn->platname);
- pl = (sai_platform_t *)malloc(sizeof(*pl) + pnl +
- strlen(sn->platbuild) + 2);
- if (!pl)
- return -1;
-
- memset(pl, 0, sizeof(*pl));
- plb = (uint8_t *)&pl[1];
-
- /*
- * Platforms are malloc'd up and added to pss
- * .platform_owner
- */
-
- pl->name = (const char *)plb;
- memcpy(plb, sn->platname, pnl + 1);
- plb += pnl + 1;
-
- pl->build = (const char *)plb;
- memcpy(plb, sn->platbuild, strlen(sn->platbuild) + 1);
-
- pl->nondefault = sn->nondefault;
-
- lws_dll2_add_head(&pl->list, &pss->platform_owner);
-
- lwsl_notice("%s: New platform '%s', build '%s', notdefault %d\n",
- __func__, pl->name, pl->build, pl->nondefault);
-
- sn->platbuild[0] = '\0';
- return 0;
- }
-
- /* we only match on the prepared path strings */
- if (!(reason & LEJP_FLAG_CB_IS_VALUE) || !ctx->path_match)
- return 0;
-
-// if (reason != LEJPCB_VAL_STR_END)
-// return 0;
-
- /* only the end part of the string, where we know the length */
-
- switch (ctx->path_match - 1) {
- case LEJPNSAIF_SCHEMA:
- sn->event_task_index = 0;
- return 0;
-
- case LEJPNSAIF_CONFIGURATIONS_PREP:
- /* the additional cmake options for this test configuration */
- n = strlen(sn->t.prep);
- if (n < sizeof(sn->t.prep) - 2)
- lws_strnncpy(sn->t.prep + n, ctx->buf, ctx->npos,
- sizeof(sn->t.prep) - n);
- break;
-
- case LEJPNSAIF_CONFIGURATIONS_CMAKE:
- /* the additional cmake options for this test configuration */
- n = strlen(sn->t.cmake);
- if (n < sizeof(sn->t.cmake) - 2)
- lws_strnncpy(sn->t.cmake + n, ctx->buf, ctx->npos,
- sizeof(sn->t.cmake) - n);
- break;
-
- case LEJPNSAIF_CONFIGURATIONS_DEPS:
- /* the necessary dependent package strings */
- n = strlen(sn->t.packages);
- if (n < sizeof(sn->t.packages) - 2) {
- if (n)
- sn->t.packages[n++] = ',';
- lws_strncpy(sn->t.packages + n, ctx->buf,
- sizeof(sn->t.packages) - n);
- }
- break;
-
- case LEJPNSAIF_CONFIGURATIONS_PLATFORMS:
- /* the necessary dependent package strings */
- lws_strncpy(sn->explicit_platforms, ctx->buf,
- sizeof(sn->explicit_platforms));
- break;
-
- case LEJPNSAIF_CONFIGURATIONS_ARTIFACTS:
- lws_strncpy(sn->t.artifacts, ctx->buf, sizeof(sn->t.artifacts));
- break;
-
- case LEJPNSAIF_CONFIGURATIONS_CPACK:
- lws_strncpy(sn->t.cpack, ctx->buf, sizeof(sn->t.cpack));
- break;
-
- case LEJPNSAIF_PLAT_BUILD:
- /*
- * The overall build script for this platform
- * is appended into the temp sn.platbuild
- */
- n = strlen(sn->platbuild);
- if (n < sizeof(sn->platbuild) - 2)
- lws_strnncpy(sn->platbuild + n, ctx->buf, ctx->npos,
- sizeof(sn->platbuild) - n);
- break;
-
- case LEJPNSAIF_PLAT_DEFAULT:
- sn->nondefault = !arg_to_bool(ctx->buf);
- break;
-
- default:
- return 0;
- }
-
- return 0;
-}
-
-static signed char
-sai_notification_lejp_cb(struct lejp_ctx *ctx, char reason)
-{
- struct pss *pss = (struct pss *)ctx->user;
- sai_notification_t *sn = (sai_notification_t *)&pss->sn;
- size_t ile, ole;
- int n;
-
- /* we only match on the prepared path strings */
- if (!(reason & LEJP_FLAG_CB_IS_VALUE) || !ctx->path_match)
- return 0;
-
-// if (reason != LEJPCB_VAL_STR_END)
-// return 0;
-
- /* only the end part of the string, where we know the length */
-
- switch (ctx->path_match - 1) {
- case LEJPN_SCHEMA:
- if (strcmp(ctx->buf, "com-warmcat-sai-notification")) {
- lwsl_err("%s: unknown schema '%s'\n", __func__, ctx->buf);
- return -1;
- }
- return 0;
-
- case LEJPN_ACTION:
- for (n = 0; n < (int)LWS_ARRAY_SIZE(notifaction_action_names);n++)
- if (!strcmp(ctx->buf, notifaction_action_names[n])) {
- sn->action = n + 1;
-
- return 0;
- }
- lwsl_notice("%s: unknown action '%s' ignored\n",
- __func__, ctx->buf);
- return -1;
-
- case LEJPN_REPOSITORY_NAME:
- lws_strncpy(sn->e.repo_name, ctx->buf, sizeof(sn->e.repo_name));
- break;
-
- case LEJPN_REPOSITORY_FETCHURL:
- lws_strncpy(sn->e.repo_fetchurl, ctx->buf, sizeof(sn->e.repo_fetchurl));
- break;
-
- case LEJPN_REF:
- lws_strncpy(sn->e.ref, ctx->buf, sizeof(sn->e.ref));
- break;
-
- case LEJPN_HASH:
- lws_strncpy(sn->e.hash, ctx->buf, sizeof(sn->e.hash));
- break;
-
- case LEJPN_NONCE:
- break;
-
- case LEJPN_SAIFILE_LEN:
- sn->saifile_in_len = atoi(ctx->buf);
- /* only accept sane base64 size */
- if (sn->saifile_in_len < 8 || sn->saifile_in_len > 65536) {
- lwsl_err("%s: bad saifile_len %u\n", __func__,
- (unsigned int)sn->saifile_in_len);
-
- return -1;
- }
- sn->saifile_out_len = (sn->saifile_in_len * 3) / 4 + 4;
- sn->saifile = malloc(sn->saifile_out_len);
- if (!sn->saifile) {
- lwsl_err("%s: OOM\n", __func__);
- return -1;
- }
- sn->saifile_in_seen = sn->saifile_out_pos = 0;
- lws_b64_decode_state_init(&sn->b64);
- break;
-
- case LEJPN_SAIFILE:
- /*
- * Base64 encoding of the commit's .sai.json file contents... we
- * have to treat this with caution since it can be anything,
- * including unparsable JSON.
- *
- * Let's base64-decode it and collect it into an buffer first,
- * before sn test parse and then sn second parse to extract the
- * tasks
- */
-
- sn->saifile_in_seen += ctx->npos;
- if (sn->saifile_in_seen > sn->saifile_in_len) {
- lwsl_err("%s: SAIFILE: too large in %u vs %u\n", __func__,
- (unsigned int)sn->saifile_in_seen,
- (unsigned int)sn->saifile_in_len);
- return -1;
- }
- ile = ctx->npos;
- ole = sn->saifile_out_len - sn->saifile_out_pos;
- lws_b64_decode_stateful(&sn->b64, ctx->buf, &ile,
- (uint8_t *)sn->saifile + sn->saifile_out_pos, &ole,
- sn->saifile_in_seen == sn->saifile_in_len);
-
- sn->saifile_out_pos += ole;
- break;
-
- default:
- return 0;
- }
-
- return 0;
-}
-
-int
-sai_notification_file_upload_cb(void *data, const char *name,
- const char *filename, char *buf, int len,
- enum lws_spa_fileupload_states state)
-{
- struct pss *pss = (struct pss *)data;
- struct lejp_ctx saictx;
- lws_dll2_owner_t owner;
- uint8_t result[64];
- int m;
-
- switch (state) {
- case LWS_UFS_OPEN:
- lwsl_notice("%s: LWS_UFS_OPEN\n", __func__);
- lejp_construct(&pss->ctx, sai_notification_lejp_cb,
- pss, paths, LWS_ARRAY_SIZE(paths));
-
- if (lws_genhmac_init(&pss->hmac, pss->hmac_type,
- (uint8_t *)pss->vhd->notification_key,
- strlen(pss->vhd->notification_key))) {
- lwsl_err("%s: failed to init hmac\n", __func__);
-
- return -1;
- }
- break;
- case LWS_UFS_FINAL_CONTENT:
- case LWS_UFS_CONTENT:
- lwsl_notice("%s: LWS_UFS_[]CONTENT: %p %p, \n", __func__, pss, buf);
- if (len && lws_genhmac_update(&pss->hmac, buf, len))
- return -1;
-
- printf("%.*s", (int)len, buf);
-
- m = lejp_parse(&pss->ctx, (uint8_t *)buf, len);
- if (m < 0 && m != LEJP_CONTINUE) {
- lwsl_notice("%s: notif JSON decode failed '%s' (%d)\n",
- __func__, lejp_error_to_string(m), m);
- return m;
- }
-
- lwsl_notice("%s: m = %d\n", __func__, m);
-
- if (m != 1)
- break;
-
- lws_genhmac_destroy(&pss->hmac, result);
-
- if (memcmp(result, pss->notification_sig,
- lws_genhmac_size(pss->hmac_type))) {
- lwsl_err("%s: hmac mismatch\n", __func__);
-
- return -1;
- }
- lwsl_notice("%s: hmac OK\n", __func__);
-
- /*
- * We have the notification metadata JSON parsed into pss->sn.e,
- * eg, pss->sn->e.hash ... since it's common to, eg, push a tree
- * in a branch and then later tag the same commit, we don't want
- * to pointlessly repeat CI for the same tree multiple times,
- * and need to basically dedupe.
- */
-
- {
- uint64_t rid = 0;
- char qu[192];
-
- lws_snprintf(qu, sizeof(qu), "select rowid from events "
- "where hash='%s'",
- pss->sn.e.hash);
-
- if (sqlite3_exec(pss->vhd->master.pdb, qu,
- sai_sql3_get_uint64_cb,
- &rid, NULL) == SQLITE_OK && rid) {
- /* it already exists */
- lwsl_notice("%s: ignoring notification as "
- "tree hash event exists\n",
- __func__);
-
- return 0;
- }
- }
-
- if (!pss->sn.saifile)
- return -1;
-
- /*
- * We processed the notification JSON, but we only decoded the
- * base64'd copy of the .sai.json so far... now's the time we
- * want to process that and break it down into tasks.
- *
- * We don't trust it since it's controlled by the guy who pushed
- * the commit, there can be anything at all in there. We made
- * sure he can't attack us until now by base64-ing it at the
- * server hook, so he's just dumb payload.
- *
- * Let's try to parse it in two passes, first without acting on
- * the content to confirm it's going to succeed...
- */
-
- sai_uuid16_create(lws_get_context(pss->wsi), pss->sn.e.uuid);
- if (saim_event_db_ensure_open(pss->vhd, pss->sn.e.uuid, 1,
- (sqlite3 **)&pss->sn.e.pdb)) {
- lwsl_err("%s: unable to open event-specific database\n",
- __func__);
-
- return -1;
- }
-
- pss->dry = 1;
- lejp_construct(&saictx, sai_saifile_lejp_cb, pss, saifile_paths,
- LWS_ARRAY_SIZE(saifile_paths));
- m = lejp_parse(&saictx, (uint8_t *)pss->sn.saifile,
- pss->sn.saifile_out_pos);
- saim_event_db_close(pss->vhd, (sqlite3 **)&pss->sn.e.pdb);
- if (m < 0) {
- lwsl_notice("%s: saifile JSON decode failed '%s' (%d)\n",
- __func__, lejp_error_to_string(m), m);
- return m;
- }
-
- /* ... then add the 32-char event object in the database ... */
-
- pss->sn.e.created = (unsigned long long)lws_now_secs();
- pss->sn.e.state = SAIES_WAITING;
-
- memset(&pss->sn.e.list, 0, sizeof(pss->sn.e.list));
- lws_dll2_owner_clear(&owner);
- lws_dll2_add_head(&pss->sn.e.list, &owner);
-
- /*
- * This is our new event going into the event database...
- */
-
- lws_struct_sq3_serialize(pss->vhd->master.pdb,
- lsm_schema_sq3_map_event, &owner, 0);
-
- /*
- * ... process the saifile JSON again creating tasks for each
- * entry in the saifile, for each platform, against that event
- * object...
- */
-
- pss->dry = 0;
- lejp_construct(&saictx, sai_saifile_lejp_cb, pss, saifile_paths,
- LWS_ARRAY_SIZE(saifile_paths));
- m = lejp_parse(&saictx, (uint8_t *)pss->sn.saifile,
- pss->sn.saifile_out_pos);
- if (m < 0) {
- lwsl_notice("%s: saifile JSON decode failed '%s' (%d)\n",
- __func__, lejp_error_to_string(m), m);
- return m;
- }
-
- free(pss->sn.saifile);
- pss->sn.saifile = NULL;
-
- lwsl_notice("%s: notification inserted into db\n", __func__);
-
- /*
- * Reassess now if there's a builder we can match to a pending task
- */
-
- lws_sul_schedule(pss->vhd->context, 0, &pss->vhd->sul_central,
- saim_central_cb, 1);
-
- return 0;
-
-
- case LWS_UFS_CLOSE:
- // lwsl_info("%s: LWS_UFS_CLOSE\n", __func__);
- lejp_destruct(&pss->ctx);
- break;
- }
-
- return 0;
-}
-
diff --git a/src/master/m-overview.c b/src/master/m-overview.c
deleted file mode 100644
index e650c19..0000000
--- a/src/master/m-overview.c
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Sai master - src/master/overview.c
- *
- * Copyright (C) 2019 Andy Green <andy@warmcat.com>
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation:
- * version 2.1 of the License.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301 USA
- *
- * The same ws interface is connected-to by builders (on path /builder), and
- * provides the query transport for browsers (on path /browse).
- */
-
-#include <libwebsockets.h>
-#include "m-private.h"
-
-
diff --git a/src/master/m-private.h b/src/master/m-private.h
deleted file mode 100644
index b2f15c4..0000000
--- a/src/master/m-private.h
+++ /dev/null
@@ -1,296 +0,0 @@
-/*
- * Sai master definitions src/master/private.h
- *
- * Copyright (C) 2019 Andy Green <andy@warmcat.com>
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation:
- * version 2.1 of the License.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301 USA
- */
-
-#include "../common/include/private.h"
-#include <sqlite3.h>
-#include <sys/stat.h>
-
-struct sai_plat;
-
-typedef struct sai_platm {
- struct lws_dll2_owner builder_owner;
- struct lws_dll2_owner subs_owner;
-
- sqlite3 *pdb;
- sqlite3 *pdb_auth;
-} saim_t;
-
-typedef struct sai_platform {
- struct lws_dll2 list;
-
- const char *name;
- const char *build;
-
- uint8_t nondefault;
-
- /* build and name over-allocated here */
-} sai_platform_t;
-
-typedef enum {
- SAIN_ACTION_INVALID,
- SAIN_ACTION_REPO_UPDATED
-} sai_notification_action_t;
-
-typedef struct {
-
- sai_event_t e;
- sai_task_t t;
-
- char platbuild[4096];
- char platname[96];
- char explicit_platforms[2048];
-
- int event_task_index;
-
- struct lws_b64state b64;
- char *saifile;
- uint64_t when;
- size_t saifile_in_len;
- size_t saifile_out_len;
- size_t saifile_out_pos;
- size_t saifile_in_seen;
- sai_notification_action_t action;
-
- uint8_t nondefault;
-} sai_notification_t;
-
-typedef enum {
- SWT_BUILDER,
- SWT_BROWSE
-} ws_type;
-
-typedef enum {
- WSS_IDLE,
- WSS_PREPARE_OVERVIEW,
- WSS_SEND_OVERVIEW,
- WSS_PREPARE_BUILDER_SUMMARY,
- WSS_SEND_BUILDER_SUMMARY,
-
- WSS_PREPARE_TASKINFO,
- WSS_SEND_ARTIFACT_INFO,
-
- WSS_PREPARE_EVENTINFO,
- WSS_SEND_EVENTINFO,
-} ws_state;
-
-typedef struct sai_builder {
- saim_t c;
-} saib_t;
-
-struct vhd;
-
-enum {
- SAIM_NOT_SPECIFIC,
- SAIM_SPECIFIC_H,
- SAIM_SPECIFIC_ID
-};
-
-struct pss {
- struct vhd *vhd;
- struct lws *wsi;
-
- struct lws_spa *spa;
- struct lejp_ctx ctx;
- sai_notification_t sn;
- struct lws_dll2 same; /* owner: vhd.builders */
-
- struct lws_dll2 subs_list;
-
- uint64_t sub_timestamp;
- char sub_task_uuid[65];
- char specific[65];
- char specific_project[96];
- char auth_user[33];
-
- sqlite3 *pdb_artifact;
- sqlite3_blob *blob_artifact;
-
- lws_dll2_owner_t platform_owner; /* sai_platform_t builder offers */
- lws_dll2_owner_t task_cancel_owner; /* sai_platform_t builder offers */
- lws_dll2_owner_t aft_owner; /* for statefully spooling artifact info */
- lws_struct_args_t a;
-
- union {
- sai_plat_t *b;
- sai_plat_owner_t *o;
- } u;
- const char *master_name;
-
- struct lwsac *query_ac;
- struct lwsac *task_ac; /* tasks for an event */
- struct lwsac *logs_ac;
- lws_dll2_owner_t issue_task_owner; /* list of sai_task_t */
- const sai_task_t *one_task; /* only for browser */
- const sai_event_t *one_event;
- lws_dll2_owner_t query_owner;
- lws_dll2_t *walk;
-
- int task_index;
- int log_cache_index;
- int log_cache_size;
- int authorized;
- int specificity;
- unsigned long expiry_unix_time;
-
- /* notification hmac information */
- char notification_sig[128];
- char alang[128];
- struct lws_genhmac_ctx hmac;
- enum lws_genhmac_types hmac_type;
- char our_form;
- char login_form;
-
- uint64_t first_log_timestamp;
- uint64_t artifact_offset;
- uint64_t artifact_length;
-
- ws_type type;
- ws_state send_state;
-
- uint32_t pending; /* bitmap of things that need sending */
-
- unsigned int spa_failed:1;
- unsigned int subsequent:1; /* for individual JSON */
- unsigned int dry:1;
- unsigned int query_already_done:1;
- unsigned int frag:1;
- unsigned int mark_started:1;
- unsigned int wants_event_updates:1;
- unsigned int announced:1;
- unsigned int bulk_binary_data:1;
-
- uint8_t ovstate; /* SOS_ substate when doing overview */
-};
-
-typedef struct saim_sqlite_cache {
- lws_dll2_t list;
- char uuid[65];
- sqlite3 *pdb;
- lws_usec_t idle_since;
- int refcount;
-} saim_sqlite_cache_t;
-
-struct vhd {
- struct lws_context *context;
- struct lws_vhost *vhost;
-
- /* pss lists */
- struct lws_dll2_owner browsers;
- struct lws_dll2_owner builders;
-
- /* our keys */
- struct lws_jwk jwt_jwk_auth;
- char jwt_auth_alg[16];
- const char *jwt_issuer;
- const char *jwt_audience;
-
- const char *sqlite3_path_lhs;
-
- lws_dll2_owner_t sqlite3_cache; /* saim_sqlite_cache_t */
- lws_dll2_owner_t tasklog_cache;
- lws_sorted_usec_list_t sul_logcache;
- lws_sorted_usec_list_t sul_central; /* background task allocation sul */
-
- lws_usec_t last_check_abandoned_tasks;
-
- const char *notification_key;
-
- saim_t master;
-};
-
-extern struct lws_context *
-sai_lws_context_from_json(const char *config_dir,
- struct lws_context_creation_info *info,
- const struct lws_protocols **pprotocols);
-extern const struct lws_protocols protocol_ws;
-
-int
-sai_notification_file_upload_cb(void *data, const char *name,
- const char *filename, char *buf, int len,
- enum lws_spa_fileupload_states state);
-
-int
-sai_sqlite3_statement(sqlite3 *pdb, const char *cmd, const char *desc);
-
-int
-sai_uuid16_create(struct lws_context *context, char *dest33);
-
-int
-saim_event_db_ensure_open(struct vhd *vhd, const char *event_uuid, char can_create, sqlite3 **ppdb);
-
-void
-saim_event_db_close(struct vhd *vhd, sqlite3 **ppdb);
-
-int
-saim_event_db_delete_database(struct vhd *vhd, const char *event_uuid);
-
-int
-sai_sq3_event_lookup(sqlite3 *pdb, uint64_t start, lws_struct_args_cb cb, void *ca);
-
-int
-sai_sql3_get_uint64_cb(void *user, int cols, char **values, char **name);
-
-int
-saim_ws_json_tx_browser(struct vhd *vhd, struct pss *pss, uint8_t *buf, size_t bl);
-
-int
-saim_ws_json_rx_builder(struct vhd *vhd, struct pss *pss, uint8_t *buf, size_t bl);
-
-int
-lws_struct_map_set(const lws_struct_map_t *map, char *u);
-
-int
-saim_ws_json_rx_browser(struct vhd *vhd, struct pss *pss,
- uint8_t *buf, size_t bl);
-
-void
-sai_task_uuid_to_event_uuid(char *event_uuid33, const char *task_uuid65);
-
-int
-saim_ws_json_tx_builder(struct vhd *vhd, struct pss *pss, uint8_t *buf, size_t bl);
-
-void
-mark_pending(struct pss *pss, ws_state state);
-
-int
-saim_subs_request_writeable(struct vhd *vhd, const char *task_uuid);
-
-void
-saim_central_cb(lws_sorted_usec_list_t *sul);
-
-int
-saim_task_reset(struct vhd *vhd, const char *task_uuid);
-
-int
-saim_task_cancel(struct vhd *vhd, const char *task_uuid);
-
-int
-saim_allocate_task(struct vhd *vhd, struct pss *pss, sai_plat_t *cb,
- const char *cns_name);
-
-int
-saim_set_task_state(struct vhd *vhd, const char *builder_name,
- const char *builder_uuid, const char *task_uuid, int state,
- uint64_t started, uint64_t duration);
-
-int
-saim_get_blob(struct vhd *vhd, const char *url, sqlite3 **pdb,
- sqlite3_blob **blob, uint64_t *length);
diff --git a/src/master/m-sai.c b/src/master/m-sai.c
deleted file mode 100644
index 5e753d6..0000000
--- a/src/master/m-sai.c
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Sai master
- *
- * Copyright (C) 2019 - 2020 Andy Green <andy@warmcat.com>
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation:
- * version 2.1 of the License.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301 USA
- */
-
-#include <libwebsockets.h>
-#include <string.h>
-#include <signal.h>
-#include <time.h>
-
-#include "m-private.h"
-
-static int interrupted;
-struct lws_context *context;
-
-static const struct lws_protocols
- *pprotocols[] = { &protocol_ws, NULL };
-
-static void sigint_handler(int sig)
-{
- interrupted = 1;
-}
-
-int main(int argc, const char **argv)
-{
- int logs = LLL_USER | LLL_ERR | LLL_WARN | LLL_NOTICE;
- const char *p, *conf = "/etc/sai/master";
- struct lws_context_creation_info info;
-
- signal(SIGINT, sigint_handler);
-
- if ((p = lws_cmdline_option(argc, argv, "-d")))
- logs = atoi(p);
-
- lws_set_log_level(logs, NULL);
- lwsl_user("Sai Master - Copyright (C) 2019-2020 Andy Green <andy@warmcat.com>\n");
-
- if ((p = lws_cmdline_option(argc, argv, "-c")))
- conf = p;
-
- context = sai_lws_context_from_json(conf, &info, pprotocols);
- if (!context) {
- lwsl_err("lws init failed\n");
- return 1;
- }
-
- while (!lws_service(context, 0) && !interrupted)
- ;
-
- lws_context_destroy(context);
-
- return 0;
-}
diff --git a/src/master/m-task.c b/src/master/m-task.c
deleted file mode 100644
index 76ff2ae..0000000
--- a/src/master/m-task.c
+++ /dev/null
@@ -1,467 +0,0 @@
-/*
- * Sai master
- *
- * Copyright (C) 2019 - 2020 Andy Green <andy@warmcat.com>
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation:
- * version 2.1 of the License.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301 USA
- */
-
-#include <libwebsockets.h>
-#include <string.h>
-#include <signal.h>
-#include <time.h>
-#include <assert.h>
-
-#include "m-private.h"
-
-static int
-browser_upd(struct lws_dll2 *d, void *user)
-{
- struct pss *pss = lws_container_of(d, struct pss, same);
- ws_state state = (ws_state)(intptr_t)user;
-
- mark_pending(pss, state);
-
- return 0;
-}
-
-static int
-sql3_get_integer_cb(void *user, int cols, char **values, char **name)
-{
- unsigned int *pui = (unsigned int *)user;
-
- lwsl_warn("%s: values[0] '%s'\n", __func__, values[0]);
- *pui = atoi(values[0]);
-
- return 0;
-}
-
-void
-sai_task_uuid_to_event_uuid(char *event_uuid33, const char *task_uuid65)
-{
- memcpy(event_uuid33, task_uuid65, 32);
- event_uuid33[32] = '\0';
-}
-
-int
-saim_set_task_state(struct vhd *vhd, const char *builder_name,
- const char *builder_uuid, const char *task_uuid, int state,
- uint64_t started, uint64_t duration)
-{
- char update[384], esc[96], esc1[96], esc2[96], esc3[32], esc4[32],
- event_uuid[33];
- unsigned int count = 0, count_good = 0, count_bad = 0;
- sai_event_state_t oes, sta;
- struct lwsac *ac = NULL;
- sai_event_t *e = NULL;
- lws_dll2_owner_t o;
- int n;
-
- /*
- * Extract the event uuid from the task uuid
- */
-
- sai_task_uuid_to_event_uuid(event_uuid, task_uuid);
-
- /*
- * Look up the task's event in the event database...
- */
-
- lws_dll2_owner_clear(&o);
- lws_sql_purify(esc1, event_uuid, sizeof(esc1));
- lws_snprintf(esc2, sizeof(esc2), " and uuid='%s'", esc1);
- n = lws_struct_sq3_deserialize(vhd->master.pdb, esc2, NULL,
- lsm_schema_sq3_map_event, &o, &ac, 0, 1);
- if (n < 0 || !o.head) {
- lwsl_err("%s: failed to get task_uuid %s\n", __func__, esc1);
- goto bail;
- }
-
- e = lws_container_of(o.head, sai_event_t, list);
- oes = e->state;
-
- /*
- * Open the event-specific database on the temporary event object
- */
-
- if (saim_event_db_ensure_open(vhd, event_uuid, 0, (sqlite3 **)&e->pdb)) {
- lwsl_err("%s: unable to open event-specific database\n",
- __func__);
-
- return -1;
- }
-
- if (builder_name)
- lws_sql_purify(esc, builder_name, sizeof(esc));
- else
- esc[0] = '\0';
-
- if (builder_uuid)
- lws_sql_purify(esc1, builder_uuid, sizeof(esc1));
- else
- esc1[0] = '\0';
- lws_sql_purify(esc2, task_uuid, sizeof(esc2));
-
- esc3[0] = esc4[0] = '\0';
-
- if (started) {
- if (started == 1)
- started = 0;
- lws_snprintf(esc3, sizeof(esc3), ",started=%llu",
- (unsigned long long)started);
- }
- if (duration) {
- if (duration == 1)
- duration = 0;
- lws_snprintf(esc4, sizeof(esc4), ",duration=%llu",
- (unsigned long long)duration);
- }
-
- /*
- * Update the task by uuid, in the event-specific database
- */
-
- lws_snprintf(update, sizeof(update),
- "update tasks set state=%d%s%s%s%s%s%s%s%s where uuid='%s'",
- state, builder_uuid ? ",builder='": "",
- builder_uuid ? esc1 : "",
- builder_uuid ? "'" : "",
- builder_name ? ",builder_name='" : "",
- builder_name ? esc : "",
- builder_name ? "'" : "",
- esc3, esc4, esc2);
-
- if (sqlite3_exec((sqlite3 *)e->pdb, update, NULL, NULL, NULL) != SQLITE_OK) {
- lwsl_err("%s: %s: %s: fail\n", __func__, update,
- sqlite3_errmsg(vhd->master.pdb));
- goto bail;
- }
-
- /*
- * So, how many tasks for this event?
- */
-
- if (sqlite3_exec((sqlite3 *)e->pdb, "select count(state) from tasks",
- sql3_get_integer_cb, &count, NULL) != SQLITE_OK) {
- lwsl_err("%s: %s: %s: fail\n", __func__, update,
- sqlite3_errmsg(vhd->master.pdb));
- goto bail;
- }
-
- /*
- * ... how many completed well?
- */
-
- if (sqlite3_exec((sqlite3 *)e->pdb, "select count(state) from tasks where state == 3",
- sql3_get_integer_cb, &count_good, NULL) != SQLITE_OK) {
- lwsl_err("%s: %s: %s: fail\n", __func__, update,
- sqlite3_errmsg(vhd->master.pdb));
- goto bail;
- }
-
- /*
- * ... how many failed?
- */
-
- if (sqlite3_exec((sqlite3 *)e->pdb, "select count(state) from tasks where state == 4",
- sql3_get_integer_cb, &count_bad, NULL) != SQLITE_OK) {
- lwsl_err("%s: %s: %s: fail\n", __func__, update,
- sqlite3_errmsg(vhd->master.pdb));
- goto bail;
- }
-
- /*
- * Decide how to set the event state based on that
- */
-
- lwsl_warn("%s: ev %s, count %u, count_good %u, count_bad %u\n",
- __func__, event_uuid, count, count_good, count_bad);
-
- sta = SAIES_BEING_BUILT;
-
- if (count) {
- if (count == count_good)
- sta = SAIES_SUCCESS;
- else
- if (count == count_bad)
- sta = SAIES_FAIL;
- else
- if (count_bad)
- sta = SAIES_BEING_BUILT_HAS_FAILURES;
- }
-
- if (sta != oes) {
- lwsl_notice("%s: event state changed\n", __func__);
- }
-
- /*
- * Update the event
- */
-
- lws_sql_purify(esc1, event_uuid, sizeof(esc1));
- lws_snprintf(update, sizeof(update),
- "update events set state=%d where uuid='%s'", sta, esc1);
-
- if (sqlite3_exec(vhd->master.pdb, update, NULL, NULL, NULL) != SQLITE_OK) {
- lwsl_err("%s: %s: %s: fail\n", __func__, update,
- sqlite3_errmsg(vhd->master.pdb));
- goto bail;
- }
-
- saim_event_db_close(vhd, (sqlite3 **)&e->pdb);
-
- /*
- * We need to schedule sending an event info update to anyone
- * subscribing to the task...
- */
-
- lws_dll2_foreach_safe(&vhd->browsers,
- (void *)WSS_PREPARE_OVERVIEW, browser_upd);
-
- lwsac_free(&ac);
-
- return 0;
-
-bail:
- if (e)
- saim_event_db_close(vhd, (sqlite3 **)&e->pdb);
- lwsac_free(&ac);
-
- return 1;
-}
-
-/*
- * Find the most recent task that still needs doing for platform, on any event
- *
- * If any, the task pointed-to lives inside *pac, along with its strings etc
- */
-
-static const sai_task_t *
-saim_task_pending(struct vhd *vhd, struct lwsac **pac, const char *platform)
-{
- struct lwsac *ac = NULL;
- char esc[96], pf[96];
- lws_dll2_owner_t o;
- int n;
-
- lws_sql_purify(esc, platform, sizeof(esc));
-
- assert(platform);
- assert(pac);
-
- /*
- * Collect a list of events that still have any open tasks
- *
- * We don't put this list in the pac since we can dispose of it in this
- * scope whether we find something or not
- */
-
- n = lws_struct_sq3_deserialize(vhd->master.pdb,
- " and (state != 3 and state != 4 and state != 5)",
- "created desc ", lsm_schema_sq3_map_event, &o,
- &ac, 0, 10);
- if (n < 0 || !o.head) {
- // lwsl_notice("%s: all events complete\n", __func__);
- /* error, or there are no events that aren't complete */
- goto bail;
- }
-
- /*
- * Iterate through the events looking at his event-specific database
- * for tasks on the specified platform...
- */
-
- lws_start_foreach_dll(struct lws_dll2 *, p, o.head) {
- sai_event_t *e = lws_container_of(p, sai_event_t, list);
- lws_dll2_owner_t ot;
- sqlite3 *pdb = NULL;
-
- if (!saim_event_db_ensure_open(vhd, e->uuid, 0, &pdb)) {
- lws_snprintf(pf, sizeof(pf),
- " and state=0 and platform='%s'", esc);
- n = lws_struct_sq3_deserialize(pdb, pf, NULL,
- lsm_schema_sq3_map_task,
- &ot, pac, 0, 1);
- saim_event_db_close(vhd, &pdb);
- if (ot.count) {
-
- lwsl_notice("%s: found task for %s\n", __func__, esc);
-
- lwsac_free(&ac);
-
- return lws_container_of(ot.head,
- sai_task_t, list);
- }
- }
-
- } lws_end_foreach_dll(p);
-
- lwsl_debug("%s: no free tasks matching '%s'\n", __func__, esc);
-
-bail:
- lwsac_free(&ac);
-
- return NULL;
-}
-
-int
-saim_task_cancel(struct vhd *vhd, const char *task_uuid)
-{
- sai_cancel_t *can;
-
- /*
- * For every pss that we have from builders...
- */
- lws_start_foreach_dll(struct lws_dll2 *, p, vhd->builders.head) {
- struct pss *pss = lws_container_of(p, struct pss, same);
-
-
- /*
- * ... queue the task cancel message
- */
- can = malloc(sizeof *can);
- if (!can)
- return -1;
- memset(can, 0, sizeof(*can));
-
- strncpy(can->task_uuid, task_uuid, sizeof(can->task_uuid));
-
- lws_dll2_add_tail(&can->list, &pss->task_cancel_owner);
-
- lws_callback_on_writable(pss->wsi);
-
- } lws_end_foreach_dll(p);
-
- return 0;
-}
-
-/*
- * Keep the task record itself, but remove all logs and artifacts related to
- * it and reset the task state back to WAITING.
- */
-
-int
-saim_task_reset(struct vhd *vhd, const char *task_uuid)
-{
- char esc[96], cmd[256], event_uuid[33];
- sqlite3 *pdb = NULL;
-
- if (!task_uuid[0])
- return 0;
-
- lwsl_notice("%s: received request to reset task %s\n", __func__, task_uuid);
-
- sai_task_uuid_to_event_uuid(event_uuid, task_uuid);
-
- if (saim_event_db_ensure_open(vhd, event_uuid, 0, &pdb)) {
- lwsl_err("%s: unable to open event-specific database\n",
- __func__);
-
- return -1;
- }
-
- lws_sql_purify(esc, task_uuid, sizeof(esc));
- lws_snprintf(cmd, sizeof(cmd), "delete from logs where task_uuid='%s'",
- esc);
-
- if (sqlite3_exec(pdb, cmd, NULL, NULL, NULL) != SQLITE_OK) {
- saim_event_db_close(vhd, &pdb);
- lwsl_err("%s: %s: %s: fail\n", __func__, cmd,
- sqlite3_errmsg(vhd->master.pdb));
- return 1;
- }
- lws_snprintf(cmd, sizeof(cmd), "delete from artifacts where task_uuid='%s'",
- esc);
-
- sqlite3_exec(pdb, cmd, NULL, NULL, NULL);
- saim_event_db_close(vhd, &pdb);
-
- saim_set_task_state(vhd, NULL, NULL, task_uuid, SAIES_WAITING, 1, 1);
-
- saim_task_cancel(vhd, task_uuid);
-
- /*
- * Reassess now if there's a builder we can match to a pending task
- */
-
- lws_sul_schedule(vhd->context, 0, &vhd->sul_central, saim_central_cb, 1);
-
- return 0;
-}
-
-/*
- * Look for any task on any event that needs building on platform_name
- */
-
-int
-saim_allocate_task(struct vhd *vhd, struct pss *pss, sai_plat_t *cb,
- const char *platform_name)
-{
- char esc1[96], esc2[96];
- lws_dll2_owner_t o;
- sai_task_t *task;
- int n;
-
- if (cb->ongoing >= cb->instances)
- return 1;
-
- /*
- * Look for a task for this platform, on any event that needs building
- */
-
- task = (sai_task_t *)saim_task_pending(vhd, &pss->a.ac, platform_name);
-
- if (!task)
- return 1;
-
- lwsl_notice("%s: %s: task found %s\n", __func__, platform_name, cb->name);
-
- /* yes, we will offer it to him */
-
- if (saim_set_task_state(vhd, cb->name, cb->name, task->uuid,
- SAIES_PASSED_TO_BUILDER, lws_now_secs(), 0))
- return -1;
-
- /* advance the task state first time we get logs */
- pss->mark_started = 1;
-
- /* let's get ahold of his event as well */
-
- lws_sql_purify(esc1, task->event_uuid, sizeof(esc1));
- lws_snprintf(esc2, sizeof(esc2), " and uuid='%s'", esc1);
- n = lws_struct_sq3_deserialize(vhd->master.pdb, esc2, NULL,
- lsm_schema_sq3_map_event, &o, &pss->a.ac,
- 0, 1);
- if (n < 0 || !o.head)
- return -1;
-
- task->one_event = lws_container_of(o.head, sai_event_t, list);
-
- task->master_name = pss->master_name;
- task->repo_name = task->one_event->repo_name;
- task->git_ref = task->one_event->ref;
- task->git_hash = task->one_event->hash;
-
- /*
- * add to master's estimate of builder's ongoing tasks...
- */
- cb->ongoing++;
-
- lws_dll2_add_tail(&task->pending_assign_list, &pss->issue_task_owner);
- lws_callback_on_writable(pss->wsi);
-
- return 0;
-}
diff --git a/src/master/m-ws-browser.c b/src/master/m-ws-browser.c
deleted file mode 100644
index 58f0e3f..0000000
--- a/src/master/m-ws-browser.c
+++ /dev/null
@@ -1,1139 +0,0 @@
-/*
- * Sai master - ./src/master/m-ws-browser.c
- *
- * Copyright (C) 2019 - 2020 Andy Green <andy@warmcat.com>
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation:
- * version 2.1 of the License.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301 USA
- *
- * These are ws rx and tx handlers related to browser ws connections, on
- * /broswe URLs.
- */
-
-#include <libwebsockets.h>
-#include <string.h>
-#include <signal.h>
-#include <assert.h>
-#include <time.h>
-
-#include "m-private.h"
-
-/*
- * For decoding specific event data request from browser
- */
-
-
-typedef struct sai_browse_rx_evinfo {
- char event_hash[65];
-} sai_browse_rx_evinfo_t;
-
-static lws_struct_map_t lsm_browser_evinfo[] = {
- LSM_CARRAY (sai_browse_rx_evinfo_t, event_hash, "event_hash"),
-};
-
-
-/*
- * For decoding specific task data request from browser
- */
-
-typedef struct sai_browse_rx_taskinfo {
- char task_hash[65];
- unsigned int log_start;
- uint8_t logs;
-} sai_browse_rx_taskinfo_t;
-
-static lws_struct_map_t lsm_browser_taskinfo[] = {
- LSM_CARRAY (sai_browse_rx_taskinfo_t, task_hash, "task_hash"),
- LSM_UNSIGNED (sai_browse_rx_taskinfo_t, logs, "logs"),
-};
-
-static lws_struct_map_t lsm_browser_taskreset[] = {
- LSM_CARRAY (sai_browse_rx_evinfo_t, event_hash, "uuid"),
-
-};
-
-/*
- * Schema list so lws_struct can pick the right object to create based on the
- * incoming schema name
- */
-
-static const lws_struct_map_t lsm_schema_json_map_bwsrx[] = {
- LSM_SCHEMA (sai_browse_rx_taskinfo_t, NULL, lsm_browser_taskinfo,
- "com.warmcat.sai.taskinfo"),
- LSM_SCHEMA (sai_browse_rx_evinfo_t, NULL, lsm_browser_evinfo,
- "com.warmcat.sai.eventinfo"),
- LSM_SCHEMA (sai_browse_rx_evinfo_t, NULL, lsm_browser_taskreset,
- /* shares struct */ "com.warmcat.sai.taskreset"),
- LSM_SCHEMA (sai_browse_rx_evinfo_t, NULL, lsm_browser_taskreset,
- /* shares struct */ "com.warmcat.sai.eventreset"),
- LSM_SCHEMA (sai_browse_rx_evinfo_t, NULL, lsm_browser_taskreset,
- /* shares struct */ "com.warmcat.sai.eventdelete"),
- LSM_SCHEMA (sai_cancel_t, NULL, lsm_task_cancel,
- "com.warmcat.sai.taskcan"),
-};
-
-enum {
- SAIM_WS_BROWSER_RX_TASKINFO,
- SAIM_WS_BROWSER_RX_EVENTINFO,
- SAIM_WS_BROWSER_RX_TASKRESET,
- SAIM_WS_BROWSER_RX_EVENTRESET,
- SAIM_WS_BROWSER_RX_EVENTDELETE,
- SAIM_WS_BROWSER_RX_TASKCANCEL
-};
-
-
-/*
- * For issuing combined task and event data back to browser
- */
-
-typedef struct sai_browse_taskreply {
- const sai_event_t *event;
- const sai_task_t *task;
- char auth_user[33];
- int authorized;
- int auth_secs;
-} sai_browse_taskreply_t;
-
-static lws_struct_map_t lsm_taskreply[] = {
- LSM_CHILD_PTR (sai_browse_taskreply_t, event, sai_event_t, NULL,
- lsm_event, "e"),
- LSM_CHILD_PTR (sai_browse_taskreply_t, task, sai_task_t, NULL,
- lsm_task, "t"),
- LSM_CARRAY (sai_browse_taskreply_t, auth_user, "auth_user"),
- LSM_UNSIGNED (sai_browse_taskreply_t, authorized, "authorized"),
- LSM_UNSIGNED (sai_browse_taskreply_t, auth_secs, "auth_secs"),
-};
-
-const lws_struct_map_t lsm_schema_json_map_taskreply[] = {
- LSM_SCHEMA (sai_browse_taskreply_t, NULL, lsm_taskreply,
- "com.warmcat.sai.taskinfo"),
-};
-
-enum sai_overview_state {
- SOS_EVENT,
- SOS_TASKS,
-};
-
-/* 1 == authorized */
-
-static int
-saim_conn_auth(struct pss *pss)
-{
- if (!pss->authorized)
- return 0;
- if (pss->expiry_unix_time < (unsigned long)lws_now_secs())
- return 0;
-
- return 1;
-}
-
-/*
- * Ask for writeable cb on all browser connections subscribed to a particular
- * task (so we can send them some more logs)
- */
-
-int
-saim_subs_request_writeable(struct vhd *vhd, const char *task_uuid)
-{
- lws_start_foreach_dll(struct lws_dll2 *, p,
- vhd->master.subs_owner.head) {
- struct pss *pss = lws_container_of(p, struct pss, subs_list);
-
- if (!strcmp(pss->sub_task_uuid, task_uuid))
- lws_callback_on_writable(pss->wsi);
-
- } lws_end_foreach_dll(p);
-
- return 0;
-}
-
-static int
-saim_pss_schedule_eventinfo(struct pss *pss, const char *event_uuid)
-{
- char qu[80], esc[66];
- int n;
-
- /*
- * Just collect the event struct into pss->query_owner to dump
- */
-
- lws_sql_purify(esc, event_uuid, sizeof(esc));
- lws_snprintf(qu, sizeof(qu), " and uuid='%s'", esc);
- n = lws_struct_sq3_deserialize(pss->vhd->master.pdb, qu, NULL,
- lsm_schema_sq3_map_event,
- &pss->query_owner, &pss->query_ac, 0, 1);
- if (n < 0 || !pss->query_owner.head)
- return 1;
-
- pss->query_already_done = 1;
-
- mark_pending(pss, WSS_PREPARE_OVERVIEW);
- mark_pending(pss, WSS_PREPARE_BUILDER_SUMMARY);
-
- return 0;
-}
-
-static int
-saim_pss_schedule_taskinfo(struct pss *pss, const char *task_uuid, int logsub)
-{
- char qu[80], esc[66], event_uuid[33];
- sqlite3 *pdb = NULL;
- lws_dll2_owner_t o;
- int n;
-
- sai_task_uuid_to_event_uuid(event_uuid, task_uuid);
-
- /* open the event-specific database object */
-
- if (saim_event_db_ensure_open(pss->vhd, event_uuid, 0, &pdb))
- /* no longer exists, nothing to do */
- return 0;
-
- /*
- * get the related task object into its own ac... there might
- * be a lot of related data, so we hold the ac in the pss for
- * as long as needed to send it out
- */
-
- lws_sql_purify(esc, task_uuid, sizeof(esc));
- lws_snprintf(qu, sizeof(qu), " and uuid='%s'", esc);
- n = lws_struct_sq3_deserialize(pdb, qu, NULL, lsm_schema_sq3_map_task,
- &o, &pss->query_ac, 0, 1);
- saim_event_db_close(pss->vhd, &pdb);
- lwsl_info("%s: n %d, o.head %p\n", __func__, n, o.head);
- if (n < 0 || !o.head)
- return 1;
-
- pss->one_task = lws_container_of(o.head, sai_task_t, list);
-
- lwsl_info("%s: browser ws asked for task hash: %s, plat %s\n",
- __func__, task_uuid, pss->one_task->platform);
-
- /* let the pss take over the task info ac and schedule sending */
-
- lws_dll2_remove((struct lws_dll2 *)&pss->one_task->list);
-
- /*
- * let's also get the event object the task relates to into
- * its own event struct
- */
-
- lws_sql_purify(esc, event_uuid, sizeof(esc));
- lws_snprintf(qu, sizeof(qu), " and uuid='%s'", esc);
- n = lws_struct_sq3_deserialize(pss->vhd->master.pdb, qu, NULL,
- lsm_schema_sq3_map_event, &o,
- &pss->query_ac, 0, 1);
- if (n < 0 || !o.head)
- return 1;
-
- /* does he want to subscribe to logs? */
- if (logsub) {
- strcpy(pss->sub_task_uuid, pss->one_task->uuid);
- lws_dll2_add_head(&pss->subs_list,
- &pss->vhd->master.subs_owner);
- pss->sub_timestamp = 0; /* where we got up to */
- lws_callback_on_writable(pss->wsi);
- lwsl_info("%s: subscribed to logs for %s\n", __func__,
- pss->sub_task_uuid);
- }
-
- pss->one_event = lws_container_of(o.head, sai_event_t, list);
-
- mark_pending(pss, WSS_PREPARE_TASKINFO);
- mark_pending(pss, WSS_PREPARE_BUILDER_SUMMARY);
-
- return 0;
-}
-
-/*
- * We need to schedule re-sending out task and event state to anyone subscribed
- * to the task that changed or its associated event
- */
-
-int
-saim_subs_task_state_change(struct vhd *vhd, const char *task_uuid)
-{
- lws_start_foreach_dll(struct lws_dll2 *, p,
- vhd->master.subs_owner.head) {
- struct pss *pss = lws_container_of(p, struct pss, subs_list);
-
- if (!strcmp(pss->sub_task_uuid, task_uuid))
- saim_pss_schedule_taskinfo(pss, task_uuid, 0);
-
- } lws_end_foreach_dll(p);
-
- return 0;
-}
-
-
-/*
- * browser has sent us a request for either overview, or data on a specific
- * task
- */
-
-int
-saim_ws_json_rx_browser(struct vhd *vhd, struct pss *pss, uint8_t *buf,
- size_t bl)
-{
- sai_browse_rx_taskinfo_t *ti;
- char qu[128], esc[96], *err;
- sai_browse_rx_evinfo_t *ei;
- sqlite3 *pdb = NULL;
- lws_struct_args_t a;
- lws_dll2_owner_t o;
- sai_cancel_t *can;
- int m, ret = -1;
-
- memset(&a, 0, sizeof(a));
- a.map_st[0] = lsm_schema_json_map_bwsrx;
- a.map_entries_st[0] = LWS_ARRAY_SIZE(lsm_schema_json_map_bwsrx);
- a.map_entries_st[1] = LWS_ARRAY_SIZE(lsm_schema_json_map_bwsrx);
- a.ac_block_size = 128;
-
- lws_struct_json_init_parse(&pss->ctx, NULL, &a);
- m = lejp_parse(&pss->ctx, (uint8_t *)buf, bl);
- if (m < 0 || !a.dest) {
- lwsl_hexdump_notice(buf, bl);
- lwsl_notice("%s: notification JSON decode failed '%s'\n",
- __func__, lejp_error_to_string(m));
- return m;
- }
-
- /*
- * Which object we ended up with depends on the schema that came in...
- * a.top_schema_index is the index in lsm_schema_json_map_bwsrx it
- * matched on
- */
-
- switch (a.top_schema_index) {
-
- case SAIM_WS_BROWSER_RX_TASKINFO:
- ti = (sai_browse_rx_taskinfo_t *)a.dest;
-
- lwsl_info("%s: schema index %d, task hash %s\n", __func__,
- a.top_schema_index, ti->task_hash);
-
- if (!ti->task_hash[0]) {
- /*
- * he's asking for the overview schema
- */
- lws_dll2_add_head(&pss->same, &vhd->browsers);
- mark_pending(pss, WSS_PREPARE_OVERVIEW);
- mark_pending(pss, WSS_PREPARE_BUILDER_SUMMARY);
- ret = 0;
- goto bail;
- }
-
- /*
- * get the related task object into its own ac... there might
- * be a lot of related data, so we hold the ac in the pss for
- * as long as needed to send it out
- */
-
- if (saim_pss_schedule_taskinfo(pss, ti->task_hash, !!ti->logs))
- goto soft_error;
-
- return 0;
-
- case SAIM_WS_BROWSER_RX_EVENTINFO:
-
- ei = (sai_browse_rx_evinfo_t *)a.dest;
-
- if (saim_pss_schedule_eventinfo(pss, ei->event_hash))
- goto soft_error;
-
- lwsac_free(&a.ac);
-
- return 0;
-
- case SAIM_WS_BROWSER_RX_TASKRESET:
-
- if (!saim_conn_auth(pss))
- goto soft_error;
-
- /*
- * User is asking us to reset / rebuild this task
- */
-
- ei = (sai_browse_rx_evinfo_t *)a.dest;
-
- if (saim_task_reset(vhd, ei->event_hash))
- goto soft_error;
-
- if (saim_pss_schedule_taskinfo(pss, ei->event_hash, 0))
- goto soft_error;
-
- lwsac_free(&a.ac);
-
- return 0;
-
- case SAIM_WS_BROWSER_RX_EVENTRESET:
-
- if (!saim_conn_auth(pss))
- goto soft_error;
-
- /*
- * User is asking us to reset / rebuild every task in the event
- */
-
- ei = (sai_browse_rx_evinfo_t *)a.dest;
-
- lwsl_notice("%s: received request to reset event %s\n",
- __func__, ei->event_hash);
-
- /* open the event-specific database object */
-
- if (saim_event_db_ensure_open(pss->vhd, ei->event_hash, 0, &pdb)) {
- lwsl_err("%s: unable to open event-specific database\n",
- __func__);
- /*
- * hanging up isn't a good way to deal with browser
- * tabs left open with a live connection to a
- * now-deleted task... the page will reconnect endlessly
- */
- goto soft_error;
- }
-
- /*
- * Retreive all the related structs into a dll2 list
- */
-
- lws_sql_purify(esc, ei->event_hash, sizeof(esc));
-
- if (lws_struct_sq3_deserialize(pdb, NULL, NULL,
- lsm_schema_sq3_map_task,
- &o, &a.ac, 0, 999) >= 0) {
-
- sqlite3_exec(vhd->master.pdb, "BEGIN TRANSACTION",
- NULL, NULL, &err);
-
- /*
- * Walk the results list resetting all the tasks
- */
-
- lws_start_foreach_dll(struct lws_dll2 *, p, o.head) {
- sai_task_t *t = lws_container_of(p, sai_task_t,
- list);
-
- saim_task_reset(vhd, t->uuid);
-
- } lws_end_foreach_dll(p);
-
- sqlite3_exec(vhd->master.pdb, "END TRANSACTION",
- NULL, NULL, &err);
- }
-
- saim_event_db_close(vhd, &pdb);
- lwsac_free(&a.ac);
-
- return 0;
-
- case SAIM_WS_BROWSER_RX_EVENTDELETE:
- /*
- * User is asking us to delete the whole event
- */
-
- if (!saim_conn_auth(pss))
- goto soft_error;
-
- ei = (sai_browse_rx_evinfo_t *)a.dest;
-
- lwsl_notice("%s: received request to delete event %s\n",
- __func__, ei->event_hash);
-
- /* open the event-specific database object */
-
- if (saim_event_db_ensure_open(pss->vhd, ei->event_hash, 0, &pdb)) {
- lwsl_notice("%s: unable to open event-specific database\n",
- __func__);
- /*
- * hanging up isn't a good way to deal with browser
- * tabs left open with a live connection to a
- * now-deleted task... the page will reconnect endlessly
- */
-
- goto soft_error;
- }
-
- /*
- * Retreive all the related structs into a dll2 list
- */
-
- lws_sql_purify(esc, ei->event_hash, sizeof(esc));
-
- if (lws_struct_sq3_deserialize(pdb, NULL, NULL,
- lsm_schema_sq3_map_task,
- &o, &a.ac, 0, 999) >= 0) {
-
- sqlite3_exec(vhd->master.pdb, "BEGIN TRANSACTION",
- NULL, NULL, &err);
-
- /*
- * Walk the results list cancelling all the tasks
- * that look like they might be ongoing
- */
-
- lws_start_foreach_dll(struct lws_dll2 *, p, o.head) {
- sai_task_t *t = lws_container_of(p, sai_task_t,
- list);
-
- if (t->state != SAIES_WAITING &&
- t->state != SAIES_SUCCESS &&
- t->state != SAIES_FAIL &&
- t->state != SAIES_CANCELLED)
- saim_task_cancel(vhd, t->uuid);
-
- } lws_end_foreach_dll(p);
-
- sqlite3_exec(vhd->master.pdb, "END TRANSACTION",
- NULL, NULL, &err);
- }
-
- saim_event_db_close(vhd, &pdb);
-
- /* delete the event iself */
-
- lws_sql_purify(esc, ei->event_hash, sizeof(esc));
- lws_snprintf(qu, sizeof(qu), "delete from events where uuid='%s'",
- esc);
- sqlite3_exec(vhd->master.pdb, qu, NULL, NULL, &err);
-
- /* remove the event-specific database */
-
- saim_event_db_delete_database(vhd, ei->event_hash);
-
- ret = 0;
- lwsac_free(&a.ac);
- mark_pending(pss, WSS_PREPARE_OVERVIEW);
- mark_pending(pss, WSS_PREPARE_BUILDER_SUMMARY);
-
- goto bail;;
-
- case SAIM_WS_BROWSER_RX_TASKCANCEL:
-
- if (!saim_conn_auth(pss))
- goto soft_error;
-
- /*
- * Browser is informing us of task's STOP button clicked, we
- * need to inform any builder that might be building it
- */
- can = (sai_cancel_t *)a.dest;
-
- lwsl_notice("%s: received request to cancel task %s\n",
- __func__, can->task_uuid);
-
- saim_task_cancel(vhd, can->task_uuid);
- break;
-
- default:
- assert(0);
- break;
- }
-
- ret = 0;
-
-bail:
- lwsac_free(&a.ac);
-
- return ret;
-
-soft_error:
- lwsac_free(&a.ac);
-
- return 0;
-}
-
-
-/*
- * We're sending something on a browser ws connection. Returning nonzero from
- * here drops the connection, necessary if we fail partway through a message
- * but undesirable if a browser tab will keep reconnecting and asking for the
- * same, no-longer-existant thing.
- */
-
-int
-saim_ws_json_tx_browser(struct vhd *vhd, struct pss *pss, uint8_t *buf, size_t bl)
-{
- uint8_t *start = buf + LWS_PRE, *p = start, *end = p + bl - LWS_PRE - 1;
- int n, flags = LWS_WRITE_TEXT, first = 0, iu;
- sai_browse_taskreply_t task_reply;
- lws_dll2_owner_t task_owner;
- lws_struct_serialize_t *js;
- char esc[256], esc1[33], filt[128];
- char event_uuid[33];
- sqlite3 *pdb = NULL;
- sai_event_t *e;
- sai_task_t *t;
- char any;
- size_t w;
-
-again:
-
- start = buf + LWS_PRE;
- p = start;
- end = p + bl - LWS_PRE - 1;
- flags = LWS_WRITE_TEXT;
- first = 0;
-
- // lwsl_notice("%s: send_state %d, pss %p, wsi %p\n", __func__,
- // pss->send_state, pss, pss->wsi);
-
- switch (pss->send_state) {
- case WSS_IDLE:
-
- /*
- * Anything from a task log he's subscribed to?
- */
-
- if (pss->subs_list.owner) {
-
- lws_snprintf(esc, sizeof(esc),
- "and task_uuid='%s' and timestamp > %llu",
- pss->sub_task_uuid,
- (unsigned long long)pss->sub_timestamp);
-
- /*
- * For efficiency, let's try to grab the next 100 at
- * once from sqlite and work our way through sending
- * them
- */
-
- if (pss->log_cache_index == pss->log_cache_size) {
- int sr;
-
- sai_task_uuid_to_event_uuid(event_uuid,
- pss->sub_task_uuid);
-
- lws_dll2_owner_clear(&task_owner);
- lwsac_free(&pss->logs_ac);
-
- lwsl_info("%s: collecting logs %s\n",
- __func__, esc);
-
- if (saim_event_db_ensure_open(vhd, event_uuid, 0,
- &pdb)) {
- lwsl_notice("%s: unable to open event-specific database\n",
- __func__);
-
- return 0;
- }
-
- sr = lws_struct_sq3_deserialize(pdb, esc,
- "uid,timestamp ",
- lsm_schema_sq3_map_log,
- &pss->query_owner,
- &pss->logs_ac, 0, 100);
-
- saim_event_db_close(vhd, &pdb);
-
- if (sr) {
-
- lwsl_err("%s: subs failed\n", __func__);
-
- return 0;
- }
-
- pss->log_cache_index = 0;
- pss->log_cache_size = pss->query_owner.count;
- }
-
- if (pss->log_cache_index < pss->log_cache_size) {
- sai_log_t *log = lws_container_of(
- pss->query_owner.head,
- sai_log_t, list);
-
- lws_dll2_remove(&log->list);
- pss->log_cache_index++;
-
- /*
- * Turn it back into JSON so we can give it to
- * the browser
- */
-
- js = lws_struct_json_serialize_create(
- lsm_schema_json_map_log, 1, 0, log);
- if (!js) {
- lwsl_notice("%s: json ser fail\n", __func__);
- return 0;
- }
-
- n = lws_struct_json_serialize(js, p, end - p, &w);
- lws_struct_json_serialize_destroy(&js);
- if (n == LSJS_RESULT_ERROR) {
- lwsl_notice("%s: json ser error\n", __func__);
- return 0;
- }
-
- p += w;
- first = 1;
- pss->walk = NULL;
-
- /*
- * Record that this was the most recent log we
- * saw so far
- */
- pss->sub_timestamp = log->timestamp;
- goto send_it;
- }
- }
-
- if (!pss->pending)
- return 0;
-
- for (n = 0; n < 31; n++) {
- if (pss->pending & (1 << n)) {
- pss->pending &= ~(1 << n);
- pss->send_state = n;
- goto again;
- }
- }
- return 0;
-
- case WSS_PREPARE_OVERVIEW:
-
- filt[0] = '\0';
- if (pss->specific_project[0]) {
- lws_sql_purify(esc, pss->specific_project, sizeof(esc) - 1);
- lws_snprintf(filt, sizeof(filt), " and repo_name=\"%s\"", esc);
- }
- pss->wants_event_updates = 1;
- if (!pss->query_already_done &&
- lws_struct_sq3_deserialize(vhd->master.pdb,
- filt[0] ? filt : NULL, "created ",
- lsm_schema_sq3_map_event, &pss->query_owner,
- &pss->query_ac, 0, -8)) {
- lwsl_notice("%s: OVERVIEW 2 failed\n", __func__);
-
- pss->send_state = WSS_IDLE;
- lwsac_free(&pss->task_ac);
- lwsac_free(&pss->query_ac);
-
- return 0;
- }
-
- pss->query_already_done = 0;
-
- /*
- * we get zero or more sai_event_t laid out in pss->query_ac,
- * and listed in pss->query_owner
- */
-
- lwsl_debug("%s: WSS_PREPARE_OVERVIEW: %d results %p\n",
- __func__, pss->query_owner.count, pss->query_ac);
-
- p += lws_snprintf((char *)p, end - p,
- "{\"schema\":\"sai.warmcat.com.overview\","
- " \"alang\":\"%s\","
- " \"authorized\": %d,"
- " \"auth_secs\": %ld,"
- " \"auth_user\": \"%s\","
- "\"overview\":[",
- lws_json_purify(esc, pss->alang, sizeof(esc) - 1, &iu),
- pss->authorized, pss->expiry_unix_time - lws_now_secs(),
- lws_json_purify(esc1, pss->auth_user, sizeof(esc1) - 1, &iu)
- );
-
- /*
- * "authorized" here is used to decide whether to render the
- * additional controls clientside. The events the controls
- * cause if used are separately checked for coming from an
- * authorized pss when they are received.
- */
-
- if (pss->specificity)
- pss->walk = lws_dll2_get_head(&pss->query_owner);
- else
- pss->walk = lws_dll2_get_tail(&pss->query_owner);
- pss->subsequent = 0;
- first = 1;
-
- pss->send_state = WSS_SEND_OVERVIEW;
-
- if (!pss->query_owner.count)
- goto so_finish;
-
- /* fallthru */
-
- case WSS_SEND_OVERVIEW:
-
- if (pss->ovstate == SOS_TASKS)
- goto enum_tasks;
-
- any = 0;
- while (end - p > 2048 && pss->walk &&
- pss->send_state == WSS_SEND_OVERVIEW) {
-
- e = lws_container_of(pss->walk, sai_event_t, list);
-
- if (pss->specificity) {
- lwsl_notice("%s: Specificity: e->hash: %s, e->ref: '%s', pss->specific: '%s'\n",
- __func__, e->hash, e->ref, pss->specific);
-
- if (!strcmp(pss->specific, "refs/heads/master") && !strcmp(e->ref, "refs/heads/main")) {
- lwsl_notice("master->main\n");
- any = 1;
- } else {
-
- if (strcmp(e->hash, pss->specific) &&
- strcmp(e->ref, pss->specific)) {
- pss->walk = pss->walk->next;
- continue;
- }
- //lwsl_notice("%s: match\n", __func__);
- any = 1;
- }
- }
-
- js = lws_struct_json_serialize_create(
- lsm_schema_json_map_event,
- LWS_ARRAY_SIZE(lsm_schema_json_map_event), 0, e);
- if (!js) {
- lwsl_err("%s: json ser fail\n", __func__);
- return 1;
- }
- if (pss->subsequent)
- *p++ = ',';
- pss->subsequent = 1;
-
- p += lws_snprintf((char *)p, end - p, "{\"e\":");
-
- n = lws_struct_json_serialize(js, p, end - p, &w);
- lws_struct_json_serialize_destroy(&js);
- switch (n) {
- case LSJS_RESULT_ERROR:
- pss->send_state = WSS_IDLE;
- lwsl_err("%s: json ser error\n", __func__);
- return 1;
-
- case LSJS_RESULT_FINISH:
- case LSJS_RESULT_CONTINUE:
- p += w;
- pss->ovstate = SOS_TASKS;
- pss->task_index = 0;
- p += lws_snprintf((char *)p, end - p, ", \"t\":[");
- goto enum_tasks;
- }
- }
- if (!any) {
- pss->send_state = WSS_IDLE;
- lwsac_free(&pss->task_ac);
- lwsac_free(&pss->query_ac);
-
- return 0;
- }
- break;
-
-enum_tasks:
- /*
- * Enumerate the tasks associated with this event... we will
- * come back here as often as needed to dump all the tasks
- */
-
- e = lws_container_of(pss->walk, sai_event_t, list);
-
- do {
-
- if (saim_event_db_ensure_open(vhd, e->uuid, 0, &pdb)) {
- lwsl_err("%s: unable to open event-specific database\n",
- __func__);
-
- break;
- }
-
- lws_dll2_owner_clear(&task_owner);
- if (lws_struct_sq3_deserialize(pdb, NULL, NULL,
- lsm_schema_sq3_map_task, &task_owner,
- &pss->task_ac, pss->task_index, 1)) {
- lwsl_err("%s: OVERVIEW 1 failed\n", __func__);
- lwsac_free(&pss->task_ac);
- saim_event_db_close(vhd, &pdb);
-
- break;
- }
- saim_event_db_close(vhd, &pdb);
-
- if (!task_owner.count)
- break;
-
- if (pss->task_index)
- *p++ = ',';
-
- /*
- * We don't want to send everyone the artifact nonces...
- * the up nonce is a key for uploading artifacts on to
- * this task, it should only be stored in the master db
- * and sent to the builder to use.
- *
- * The down nonce is used in generated links, but still
- * you should have to acquire such a link via whatever
- * auth rather than be able to cook them up yourself
- * from knowing the task uuid.
- */
-
- t = (sai_task_t *)task_owner.head;
- t->art_up_nonce[0] = '\0';
- t->art_down_nonce[0] = '\0';
-
- /* only one in it at a time */
- t = lws_container_of(task_owner.head, sai_task_t, list);
-
- js = lws_struct_json_serialize_create(
- lsm_schema_json_map_task,
- LWS_ARRAY_SIZE(lsm_schema_json_map_task), 0, t);
-
- n = lws_struct_json_serialize(js, p, end - p, &w);
- lws_struct_json_serialize_destroy(&js);
- lwsac_free(&pss->task_ac);
- p += w;
-
- pss->task_index++;
- } while ((end - p > 2048) && task_owner.count);
-
- if (task_owner.count)
- /* may be more left to do */
- break;
-
- /* none left to do, go back up a level */
-
- p += lws_snprintf((char *)p, end - p, "]}");
-
- pss->ovstate = SOS_EVENT;
- if (pss->specificity)
- pss->walk = pss->walk->next;
- else
- pss->walk = pss->walk->prev;
- if (!pss->walk || pss->specificity) {
- while (pss->walk)
- pss->walk = pss->walk->next;
- goto so_finish;
- }
- break;
-
-so_finish:
- p += lws_snprintf((char *)p, end - p, "]}");
- pss->send_state = WSS_IDLE;
- lwsac_free(&pss->task_ac);
- lwsac_free(&pss->query_ac);
- break;
-
- case WSS_PREPARE_BUILDER_SUMMARY:
- p += lws_snprintf((char *)p, end - p,
- "{\"schema\":\"com.warmcat.sai.builders\","
- " \"alang\":\"%s\","
- " \"authorized\":%d,"
- " \"auth_secs\":%ld,"
- " \"auth_user\": \"%s\","
- " \"builders\":[",
- lws_sql_purify(esc, pss->alang, sizeof(esc) - 1),
- pss->authorized, pss->expiry_unix_time - lws_now_secs(),
- lws_json_purify(esc1, pss->auth_user, sizeof(esc1) - 1, &iu));
-
- pss->walk = lws_dll2_get_head(&vhd->master.builder_owner);
- pss->subsequent = 0;
- pss->send_state = WSS_SEND_BUILDER_SUMMARY;
- first = 1;
-
- /* fallthru */
-
- case WSS_SEND_BUILDER_SUMMARY:
- if (!pss->walk)
- goto b_finish;
-
- /*
- * We're going to send the browser some JSON about all the
- * builders / platforms we feel are connected to us
- */
-
- while (end - p > 512 && pss->walk &&
- pss->send_state == WSS_SEND_BUILDER_SUMMARY) {
-
- sai_plat_t *b = lws_container_of(pss->walk, sai_plat_t,
- sai_plat_list);
-
- js = lws_struct_json_serialize_create(
- lsm_schema_map_plat_simple,
- LWS_ARRAY_SIZE(lsm_schema_map_plat_simple),
- 0, b);
- if (!js)
- return 1;
- if (pss->subsequent)
- *p++ = ',';
- pss->subsequent = 1;
-
- switch (lws_struct_json_serialize(js, p, end - p, &w)) {
- case LSJS_RESULT_ERROR:
- lws_struct_json_serialize_destroy(&js);
- pss->send_state = WSS_IDLE;
- return 1;
- case LSJS_RESULT_FINISH:
- p += w;
- lws_struct_json_serialize_destroy(&js);
- pss->walk = pss->walk->next;
- if (!pss->walk)
- goto b_finish;
- break;
-
- case LSJS_RESULT_CONTINUE:
- p += w;
- lws_struct_json_serialize_destroy(&js);
- pss->walk = pss->walk->next;
- if (!pss->walk)
- goto b_finish;
- break;
- }
- }
- break;
-b_finish:
- p += lws_snprintf((char *)p, end - p, "]}");
- pss->send_state = WSS_IDLE;
-// lwsac_free(&pss->query_ac);
- lwsac_free(&pss->task_ac);
- break;
-
-
- case WSS_PREPARE_TASKINFO:
- /*
- * We're sending a browser the specific task info that he
- * asked for.
- *
- * We already got the task struct out of the db in .one_task
- * (all in .query_ac)
- */
-
- task_reply.event = pss->one_event;
- task_reply.task = pss->one_task;
- task_reply.auth_secs = pss->expiry_unix_time - lws_now_secs();
- task_reply.authorized = pss->authorized;
- lws_strncpy(task_reply.auth_user, pss->auth_user,
- sizeof(task_reply.auth_user));
-
- js = lws_struct_json_serialize_create(lsm_schema_json_map_taskreply,
- LWS_ARRAY_SIZE(lsm_schema_json_map_taskreply),
- 0, &task_reply);
- if (!js)
- return 1;
-
- n = lws_struct_json_serialize(js, p, end - p, &w);
- lws_struct_json_serialize_destroy(&js);
-
- /*
- * Let's also try to fetch any artifacts into pss->aft_owner...
- * no db or no artifacts can also be a normal situation...
- */
-
- if (pss->one_task) {
-
- sai_task_uuid_to_event_uuid(event_uuid, pss->one_task->uuid);
-
- lwsl_notice("%s: ---------------- event uuid '%s'\n", __func__,
- event_uuid);
-
- lws_dll2_owner_clear(&pss->aft_owner);
- if (!saim_event_db_ensure_open(vhd, event_uuid, 0, &pdb)) {
-
- lws_snprintf(filt, sizeof(filt), " and (task_uuid == '%s')",
- pss->one_task->uuid);
-
- lwsl_notice("%s: ---------------- %s\n", __func__, filt);
-
- if (lws_struct_sq3_deserialize(pdb, filt, NULL,
- lsm_schema_sq3_map_artifact,
- &pss->aft_owner,
- &pss->task_ac, 0, 10)) {
- lwsl_err("%s: get afcts failed\n", __func__);
- }
- saim_event_db_close(vhd, &pdb);
- }
- }
-
- first = 1;
- pss->walk = NULL;
- pss->send_state = WSS_SEND_ARTIFACT_INFO;
- if (!pss->aft_owner.head) {
- lwsl_notice("%s: ---------------- no artifacts\n", __func__);
- /* there's no artifact stuff to do */
- pss->send_state = WSS_IDLE;
- lwsac_free(&pss->query_ac);
- }
- pss->one_task = NULL;
- if (n == LSJS_RESULT_ERROR) {
- lwsl_notice("%s: taskinfo: error generating json\n", __func__);
- return 1;
- }
- p += w;
- if (!lws_ptr_diff(p, start)) {
- lwsl_notice("%s: taskinfo: empty json\n", __func__);
- return 0;
- }
- break;
-
- case WSS_SEND_ARTIFACT_INFO:
- if (pss->aft_owner.head) {
- sai_artifact_t *aft = (sai_artifact_t *)pss->aft_owner.head;
-
- lws_dll2_remove(&aft->list);
-
- /* we don't want to disclose this to browsers */
- aft->artifact_up_nonce[0] = '\0';
-
- js = lws_struct_json_serialize_create(lsm_schema_json_map_artifact,
- LWS_ARRAY_SIZE(lsm_schema_json_map_artifact),
- 0, aft);
- if (!js) {
- lwsl_err("%s ----------------- failed to render artifact json\n", __func__);
- return 1;
- }
-
- n = lws_struct_json_serialize(js, p, end - p, &w);
- lws_struct_json_serialize_destroy(&js);
- if (n == LSJS_RESULT_ERROR) {
- lwsl_notice("%s: taskinfo: ---------- error generating json\n", __func__);
- return 1;
- }
- first = 1;
- p += w;
- lwsl_warn("%s: --------------------- %.*s\n", __func__, (int)w, start);
- }
-
- if (!pss->aft_owner.head) {
- pss->send_state = WSS_IDLE;
- lwsac_free(&pss->query_ac);
- }
- break;
-
- default:
- lwsl_err("%s: pss state %d\n", __func__, pss->send_state);
- return 0;
- }
-
-send_it:
- flags = lws_write_ws_flags(LWS_WRITE_TEXT, first, !pss->walk);
-
- if (lws_write(pss->wsi, start, p - start, flags) < 0)
- return -1;
-
- /*
- * We get a bad ratio of reads to write when the builder spams us
- * with rx... we have to try to clear as much as we can in one go.
- */
-
- if (!lws_send_pipe_choked(pss->wsi))
- goto again;
-
- lws_callback_on_writable(pss->wsi);
-
- return 0;
-}
diff --git a/src/master/m-ws-builder.c b/src/master/m-ws-builder.c
deleted file mode 100644
index 0b717d0..0000000
--- a/src/master/m-ws-builder.c
+++ /dev/null
@@ -1,728 +0,0 @@
-/*
- * Sai master - ./src/master/ws-json-rx.c
- *
- * Copyright (C) 2019 - 2020 Andy Green <andy@warmcat.com>
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation:
- * version 2.1 of the License.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301 USA
- *
- * These are ws rx and tx handlers related to builder ws connections, on
- * /builder
- */
-
-#include <libwebsockets.h>
-#include <string.h>
-#include <signal.h>
-#include <time.h>
-
-#include "m-private.h"
-
-enum sai_overview_state {
- SOS_EVENT,
- SOS_TASKS,
-};
-
-typedef struct saim_logcache_pertask {
- lws_dll2_t list; /* vhd->tasklog_cache is the owner */
- char uuid[65];
- lws_dll2_owner_t cache; /* sai_log_t */
-} saim_logcache_pertask_t;
-
-/*
- * The Schema that may be sent to us by a builder
- *
- * Artifacts are sent on secondary SS connections so they don't block ongoing
- * log delivery etc. The JSON is immediately followed by binary data to the
- * length told in the JSON.
- */
-
-static const lws_struct_map_t lsm_schema_map_ba[] = {
- LSM_SCHEMA_DLL2 (sai_plat_owner_t, plat_owner, NULL, lsm_plat_list,
- "com-warmcat-sai-ba"),
- LSM_SCHEMA (sai_log_t, NULL, lsm_log,
- "com-warmcat-sai-logs"),
- LSM_SCHEMA (sai_event_t, NULL, lsm_task_rej,
- "com.warmcat.sai.taskrej"),
- LSM_SCHEMA (sai_artifact_t, NULL, lsm_artifact,
- "com-warmcat-sai-artifact"),
-};
-
-enum {
- SAIM_WSSCH_BUILDER_PLATS,
- SAIM_WSSCH_BUILDER_LOGS,
- SAIM_WSSCH_BUILDER_TASKREJ,
- SAIM_WSSCH_BUILDER_ARTIFACT
-};
-
-static void
-saim_dump_logs_to_db(lws_sorted_usec_list_t *sul)
-{
- struct vhd *vhd = lws_container_of(sul, struct vhd, sul_logcache);
- saim_logcache_pertask_t *lcpt;
- char event_uuid[33];
- sqlite3 *pdb = NULL;
- sai_log_t *hlog;
- char *err;
-
- /*
- * for each task that acquired logs in the interval
- */
-
- lws_start_foreach_dll_safe(struct lws_dll2 *, p, p1,
- vhd->tasklog_cache.head) {
- lcpt = lws_container_of(p, saim_logcache_pertask_t, list);
-
- sai_task_uuid_to_event_uuid(event_uuid, lcpt->uuid);
-
- if (!saim_event_db_ensure_open(vhd, event_uuid, 0, &pdb)) {
-
- /*
- * Empty the task-specific log cache into the event-
- * specific db for the task in one go, this is much
- * more efficient
- */
-
- sqlite3_exec(pdb, "BEGIN TRANSACTION", NULL, NULL, &err);
-
- lws_struct_sq3_serialize(pdb, lsm_schema_sq3_map_log,
- &lcpt->cache, 0);
-
- sqlite3_exec(pdb, "END TRANSACTION", NULL, NULL, &err);
- saim_event_db_close(vhd, &pdb);
-
- } else
- lwsl_err("%s: unable to open event-specific database\n",
- __func__);
-
- /*
- * Destroy the logs in the task cache and the task cache
- */
-
- lws_start_foreach_dll_safe(struct lws_dll2 *, pq, pq1,
- lcpt->cache.head) {
- hlog = lws_container_of(pq, sai_log_t, list);
- lws_dll2_remove(&hlog->list);
- free(hlog);
- } lws_end_foreach_dll_safe(pq, pq1);
-
- /*
- * Inform anybody who's looking at this task's logs that
- * something changed
- */
- saim_subs_request_writeable(vhd, lcpt->uuid);
-
- /*
- * Destroy the whole task-specific cache, it will regenerate
- * if more logs come for it
- */
-
- lws_dll2_remove(&lcpt->list);
- free(lcpt);
-
- } lws_end_foreach_dll_safe(p, p1);
-
-}
-
-/*
- * We're going to stash these logs on a per-task list, and deal with them
- * inside a single trasaction per task efficiently on a timer.
- */
-
-static void
-saim_log_to_db(struct vhd *vhd, sai_log_t *log)
-{
- saim_logcache_pertask_t *lcpt = NULL;
- sai_log_t *hlog;
-
- /*
- * find the pertask if one exists
- */
-
- lws_start_foreach_dll(struct lws_dll2 *, p, vhd->tasklog_cache.head) {
- lcpt = lws_container_of(p, saim_logcache_pertask_t, list);
-
- if (!strcmp(lcpt->uuid, log->task_uuid))
- break;
- lcpt = NULL;
-
- } lws_end_foreach_dll(p);
-
- if (!lcpt) {
- /*
- * Create a pertask and add it to the vhd list of them
- */
- lcpt = malloc(sizeof(*lcpt));
- memset(lcpt, 0, sizeof(*lcpt));
- lws_strncpy(lcpt->uuid, log->task_uuid, sizeof(lcpt->uuid));
- lws_dll2_add_tail(&lcpt->list, &vhd->tasklog_cache);
- }
-
- hlog = malloc(sizeof(*hlog) + log->len + strlen(log->log) + 1);
- if (!hlog)
- return;
-
- *hlog = *log;
- memset(&hlog->list, 0, sizeof(hlog->list));
- memcpy(&hlog[1], log->log, strlen(log->log) + 1);
- hlog->log = (char *)&hlog[1];
-
- /*
- * add our log copy to the task-specific cache
- */
-
- lws_dll2_add_tail(&hlog->list, &lcpt->cache);
-
- if (!vhd->sul_logcache.list.owner)
- /* if not already scheduled, schedule it for 250ms */
-
- lws_sul_schedule(vhd->context, 0, &vhd->sul_logcache,
- saim_dump_logs_to_db, 250 * LWS_US_PER_MS);
-}
-
-static sai_plat_t *
-saim_builder_from_uuid(struct vhd *vhd, const char *hostname)
-{
- lws_start_foreach_dll(struct lws_dll2 *, p,
- vhd->master.builder_owner.head) {
- sai_plat_t *cb = lws_container_of(p, sai_plat_t,
- sai_plat_list);
-
- if (!strcmp(hostname, cb->name))
- return cb;
-
- } lws_end_foreach_dll(p);
-
- return NULL;
-}
-
-int
-sai_sql3_get_uint64_cb(void *user, int cols, char **values, char **name)
-{
- uint64_t *pui = (uint64_t *)user;
-
- *pui = (uint64_t)atoll(values[0]);
-
- return 0;
-}
-
-/*
- * Master received a communication from a builder
- */
-
-int
-saim_ws_json_rx_builder(struct vhd *vhd, struct pss *pss, uint8_t *buf, size_t bl)
-{
- char event_uuid[33], s[128], esc[96];
- struct lwsac *ac = NULL;
- sai_plat_t *build, *cb;
- sai_rejection_t *rej;
- lws_dll2_owner_t o;
- sai_artifact_t *ap;
- sai_task_t *task;
- sai_log_t *log;
- uint64_t rid;
- int n, m;
-
- /*
- * use the schema name on the incoming JSON to decide what kind of
- * structure to instantiate
- *
- * We may have:
- *
- * - just received a fragment of the whole JSON
- *
- * - received the JSON and be handling appeneded blob data
- */
-
- if (pss->bulk_binary_data)
- goto handle;
-
- if (!pss->frag) {
- memset(&pss->a, 0, sizeof(pss->a));
- pss->a.map_st[0] = lsm_schema_map_ba;
- pss->a.map_entries_st[0] = LWS_ARRAY_SIZE(lsm_schema_map_ba);
- pss->a.ac_block_size = 4096;
-
- lws_struct_json_init_parse(&pss->ctx, NULL, &pss->a);
- } else
- pss->frag = 0;
-
- m = lejp_parse(&pss->ctx, (uint8_t *)buf, bl);
-
- /*
- * returns negative, or unused amount... for us, we either had a
- * (negative) error, had LEJP_CONTINUE, or if 0/positive, finished
- */
- if (m < 0 && m != LEJP_CONTINUE) {
- /* an explicit error */
- lwsl_hexdump_err(buf, bl);
- lwsl_err("%s: rx JSON decode failed '%s', %d, %s, %s, %d\n",
- __func__, lejp_error_to_string(m), m,
- pss->ctx.path, pss->ctx.buf, pss->ctx.npos);
- return 1;
- }
-
- if (m == LEJP_CONTINUE) {
- pss->frag = 1;
- return 0;
- }
-
- if (!pss->a.dest) {
- lwsl_err("%s: json decode didn't make an object\n", __func__);
- return 1;
- }
-
-handle:
- switch (pss->a.top_schema_index) {
- case SAIM_WSSCH_BUILDER_PLATS:
-
- lwsl_hexdump_notice(buf, bl);
-
- /*
- * builder is sending us an array of platforms it provides us
- */
-
- pss->u.o = (sai_plat_owner_t *)pss->a.dest;
-
- lwsl_notice("%s: seen platform list: count %d\n", __func__,
- pss->u.o->plat_owner.count);
-
- lws_start_foreach_dll(struct lws_dll2 *, pb,
- pss->u.o->plat_owner.head) {
- build = lws_container_of(pb, sai_plat_t, sai_plat_list);
-
- lwsl_notice("%s: seeing plat %s\n", __func__, build->name);
-
- /*
- * ... so is this one a new guy?
- */
-
- cb = saim_builder_from_uuid(vhd, build->name);
- if (!cb) {
- char *cp;
-
- /*
- * We need to make a persistent, deep, copy of
- * the (from JSON) builder object representing
- * this client.
- *
- * "platform" is eg "linux-ubuntu-bionic-arm64"
- * and "name" is "hostname.<platform>".
- */
-
- if (!build->name || !build->platform) {
- lwsl_err("%s: missing build '%s'/hostname '%s'\n",
- __func__,
- build->name ? build->name : "null",
- build->platform ? build->platform : "null");
- return -1;
- }
-
- cb = malloc(sizeof(*cb) +
- strlen(build->name) + 1 +
- strlen(build->platform) + 1);
-
- memset(cb, 0, sizeof(*cb));
- cp = (char *)&cb[1];
-
- memcpy(cp, build->name, strlen(build->name) + 1);
- cb->name = cp;
- cp += strlen(build->name) + 1;
-
- memcpy(cp, build->platform, strlen(build->platform) + 1);
- cb->platform = cp;
- cp += strlen(build->platform) + 1;
-
- cb->ongoing = build->ongoing;
- cb->instances = build->instances;
-
- cb->wsi = pss->wsi;
-
- /* Then attach the copy to the master in the vhd
- */
- lws_dll2_add_tail(&cb->sai_plat_list,
- &vhd->master.builder_owner);
- }
-
- /*
- * Even if he's not new, we should use his updated info about
- * builder load
- */
- cb->ongoing = build->ongoing;
- cb->instances = build->instances;
-
- lwsl_notice("%s: builder %s reports load %d/%d\n",
- __func__, cb->name, cb->ongoing,
- cb->instances);
-
- } lws_end_foreach_dll(pb);
-
- lwsac_free(&pss->a.ac);
-
- /*
- * look if we should offer the builder a task, given the
- * platforms he's offering
- */
-
- if (saim_allocate_task(vhd, pss, cb, cb->platform) < 0)
- goto bail;
-
- break;
-
-bail:
- lwsac_free(&pss->a.ac);
- return -1;
-
- case SAIM_WSSCH_BUILDER_LOGS:
- /*
- * builder is sending us info about task logs
- */
-
- log = (sai_log_t *)pss->a.dest;
- saim_log_to_db(vhd, log);
-
- if (pss->mark_started) {
- pss->mark_started = 0;
- pss->first_log_timestamp = log->timestamp;
- if (saim_set_task_state(vhd, NULL, NULL, log->task_uuid,
- SAIES_BEING_BUILT, 0, 0))
- goto bail;
- }
-
- if (log->finished) {
- /*
- * We have reached the end of the logs for this task
- */
- lwsl_info("%s: log->finished says 0x%x, dur %lluus\n",
- __func__, log->finished, (unsigned long long)(
- log->timestamp - pss->first_log_timestamp));
- if (log->finished & SAISPRF_EXIT) {
- if ((log->finished & 0xff) == 0)
- n = SAIES_SUCCESS;
- else
- n = SAIES_FAIL;
- } else
- if (log->finished & 8192)
- n = SAIES_CANCELLED;
- else
- n = SAIES_FAIL;
-
- if (saim_set_task_state(vhd, NULL, NULL, log->task_uuid,
- n, 0, log->timestamp -
- pss->first_log_timestamp))
- goto bail;
- }
-
- lwsac_free(&pss->a.ac);
-
- break;
-
- case SAIM_WSSCH_BUILDER_TASKREJ:
-
- /*
- * builder is updating us about his status, and may be
- * rejecting a task we tried to give him
- */
-
- rej = (sai_rejection_t *)pss->a.dest;
-
- cb = saim_builder_from_uuid(vhd, rej->host_platform);
- if (!cb) {
- lwsl_info("%s: unknown builder %s rejecting\n",
- __func__, rej->host_platform);
- break;
- }
-
- /* update our info about builder state with reality */
-
- cb->ongoing = rej->ongoing;
- cb->instances = rej->limit;
-
- lwsl_notice("%s: builder %s reports load %d/%d (rej %s)\n",
- __func__, cb->name, cb->ongoing, cb->instances,
- rej->task_uuid[0] ? rej->task_uuid : "none");
-
- if (rej->task_uuid[0])
- saim_task_reset(vhd, rej->task_uuid);
-
- break;
-
- case SAIM_WSSCH_BUILDER_ARTIFACT:
- /*
- * We get sent a JSON object immediately followed by binary
- * data for the artifact.
- *
- * We place the binary data as a blob in the sql record in the
- * artifact table.
- */
-
- lwsl_debug("%s: SAIM_WSSCH_BUILDER_ARTIFACT\n", __func__);
-
- if (!pss->bulk_binary_data) {
-
- ap = (sai_artifact_t *)pss->a.dest;
-
- sai_task_uuid_to_event_uuid(event_uuid, ap->task_uuid);
-
- /*
- * Open the event-specific database object... the
- * handle is closed when the stream closes for whatever
- * reason.
- */
-
- if (saim_event_db_ensure_open(pss->vhd, event_uuid, 0,
- &pss->pdb_artifact)) {
- lwsl_err("%s: unable to open event-specific "
- "database\n", __func__);
-
- return -1;
- }
-
- /*
- * Retreive the task object
- */
-
- lws_sql_purify(esc, ap->task_uuid, sizeof(esc));
- lws_snprintf(s, sizeof(s)," and uuid == \"%s\"", esc);
- n = lws_struct_sq3_deserialize(pss->pdb_artifact, s,
- NULL, lsm_schema_sq3_map_task,
- &o, &ac, 0, 1);
- if (n < 0 || !o.head) {
- saim_event_db_close(vhd, &pss->pdb_artifact);
- lwsl_notice("%s: no task of that id\n", __func__);
- return -1;
- }
-
- task = (sai_task_t *)o.head;
- n = strcmp(task->art_up_nonce, ap->artifact_up_nonce);
-
- if (n) {
- lwsl_err("%s: artifact nonce mismatch\n",
- __func__);
- goto afail;
- }
-
- /*
- * The task the sender is sending us an artifact for
- * exists. The sender knows the random upload nonce
- * for that task's artifacts.
- *
- * Create a random download nonce unrelated to the
- * random upload nonce (so knowing the download one
- * won't let you upload anything).
- *
- * Create the artifact's entry in the event-specific
- * database
- */
-
- sai_uuid16_create(pss->vhd->context,
- ap->artifact_down_nonce);
-
- lws_dll2_owner_clear(&o);
- lws_dll2_add_head(&ap->list, &o);
-
- /*
- * Create the task in event-specific database
- */
-
- if (lws_struct_sq3_serialize(pss->pdb_artifact,
- lsm_schema_sq3_map_artifact,
- &o, ap->uid)) {
- lwsl_err("%s: failed artifact struct insert\n",
- __func__);
-
- goto afail;
- }
-
- /*
- * recover the rowid
- */
-
- lws_snprintf(s, sizeof(s),
- "select rowid from artifacts "
- "where timestamp=%llu",
- (unsigned long long)ap->timestamp);
-
- if (sqlite3_exec((sqlite3 *)pss->pdb_artifact, s,
- sai_sql3_get_uint64_cb, &rid, NULL) !=
- SQLITE_OK) {
- lwsl_err("%s: %s: %s: fail\n", __func__, s,
- sqlite3_errmsg(pss->pdb_artifact));
- goto afail;
- }
-
- /*
- * Set the blob size on associated row
- */
-
- lws_snprintf(s, sizeof(s),
- "update artifacts set blob=zeroblob(%llu) "
- "where rowid=%llu",
- (unsigned long long)ap->len,
- (unsigned long long)rid);
-
- if (sqlite3_exec((sqlite3 *)pss->pdb_artifact, s,
- NULL, NULL, NULL) !=
- SQLITE_OK) {
- lwsl_err("%s: %s: %s: fail\n", __func__, s,
- sqlite3_errmsg(pss->pdb_artifact));
- goto afail;
- }
-
- /*
- * Open a blob on the associated row... the blob handle
- * is closed when this stream closes for whatever
- * reason.
- */
-
- if (sqlite3_blob_open(pss->pdb_artifact, "main",
- "artifacts", "blob", rid, 1,
- &pss->blob_artifact) != SQLITE_OK) {
- lwsl_err("%s: unable to open blob\n", __func__);
- goto afail;
- }
-
- /*
- * First time around, m == number of bytes let in buf
- * after JSON, (bl - m) offset
- */
- pss->bulk_binary_data = 1;
- pss->artifact_length = ap->len;
- } else
- m = bl;
-
- if (m) {
- lwsl_notice("%s: blob write +%d, ofs %llu / %llu, len %d\n",
- __func__, (int)(bl - m),
- (unsigned long long)pss->artifact_offset,
- (unsigned long long)pss->artifact_length, m);
- if (sqlite3_blob_write(pss->blob_artifact,
- (uint8_t *)buf + (bl - m), (int)m,
- pss->artifact_offset)) {
- lwsl_err("%s: writing blob failed\n", __func__);
- goto afail;
- }
-
- lws_set_timeout(pss->wsi, PENDING_TIMEOUT_HTTP_CONTENT, 5);
- pss->artifact_offset += (int)m;
- }
-
- if (pss->artifact_offset == pss->artifact_length) {
- lwsl_notice("%s: blob upload finished\n", __func__);
-
- goto afail;
- }
-
- }
-
- return 0;
-
-afail:
- lwsac_free(&ac);
- saim_event_db_close(vhd, &pss->pdb_artifact);
-
- return -1;
-}
-
-/*
- * We're sending something on a builder ws connection
- */
-
-int
-saim_ws_json_tx_builder(struct vhd *vhd, struct pss *pss, uint8_t *buf,
- size_t bl)
-{
- uint8_t *start = buf + LWS_PRE, *p = start, *end = p + bl - LWS_PRE - 1;
- int n, flags = LWS_WRITE_TEXT, first = 0;
- lws_struct_serialize_t *js;
- sai_task_t *task;
- size_t w;
-
- if (pss->task_cancel_owner.head) {
- /*
- * Pending cancel message to send
- */
- sai_cancel_t *c = lws_container_of(pss->task_cancel_owner.head,
- sai_cancel_t, list);
-
- js = lws_struct_json_serialize_create(lsm_schema_json_map_can,
- LWS_ARRAY_SIZE(lsm_schema_json_map_can), 0, c);
- if (!js)
- return 1;
-
- n = lws_struct_json_serialize(js, p, end - p, &w);
- lws_struct_json_serialize_destroy(&js);
-
- lws_dll2_remove(&c->list);
- free(c);
-
- first = 1;
- pss->walk = NULL;
-
- goto send_json;
- }
-
- if (!pss->issue_task_owner.count)
- return 0; /* nothing to send */
-
- /*
- * We're sending a browser the specific task info that he
- * asked for.
- *
- * We already got the task struct out of the db in .one_task
- * (all in .query_ac)
- */
-
- task = lws_container_of(pss->issue_task_owner.head, sai_task_t,
- pending_assign_list);
- lws_dll2_remove(&task->pending_assign_list);
-
- js = lws_struct_json_serialize_create(lsm_schema_map_ta,
- LWS_ARRAY_SIZE(lsm_schema_map_ta),
- 0, task);
- if (!js)
- return 1;
-
- n = lws_struct_json_serialize(js, p, end - p, &w);
- lws_struct_json_serialize_destroy(&js);
-
- first = 1;
- pss->walk = NULL;
-
- //lwsac_free(&pss->query_ac);
-
-send_json:
- p += w;
- if (n == LSJS_RESULT_ERROR) {
- lwsl_notice("%s: taskinfo: error generating json\n",
- __func__);
- return 1;
- }
- if (!lws_ptr_diff(p, start)) {
- lwsl_notice("%s: taskinfo: empty json\n", __func__);
- return 0;
- }
-
- flags = lws_write_ws_flags(LWS_WRITE_TEXT, first, !pss->walk);
-
- lwsl_hexdump_notice(start, p - start);
-
- if (lws_write(pss->wsi, start, p - start, flags) < 0)
- return -1;
-
- lws_callback_on_writable(pss->wsi);
-
- return 0;
-}
diff --git a/src/server/CMakeLists.txt b/src/server/CMakeLists.txt
new file mode 100644
index 0000000..b256075
--- /dev/null
+++ b/src/server/CMakeLists.txt
@@ -0,0 +1,73 @@
+set(SUB "sai-server")
+
+set(CPACK_DEBIAN_SERVER_PACKAGE_NAME ${SUB})
+
+set(SRCS
+ s-sai.c
+ s-conf.c
+ s-notification.c
+ s-comms.c
+ s-ws-builder.c
+ s-task.c
+ s-central.c
+ s-websrv.c
+)
+
+set(requirements 1)
+require_lws_config(LWS_WITH_STRUCT_SQLITE3 1 requirements)
+require_lws_config(LWS_WITH_SERVER 1 requirements)
+require_lws_config(LWS_WITH_GENCRYPTO 1 requirements)
+require_lws_config(LWS_WITH_UNIX_SOCK 1 requirements)
+
+if (requirements)
+ add_executable(${SUB} ${SRCS})
+ if (APPLE)
+ set_property(TARGET sai-server PROPERTY MACOSX_RPATH YES)
+ endif()
+
+ #
+ # sqlite3 paths (server)
+ #
+
+ find_path( SQLITE3_INC_PATH NAMES "sqlite3.h")
+ find_library(SQLITE3_LIB_PATH NAMES "sqlite3")
+
+ if (SQLITE3_INC_PATH AND SQLITE3_LIB_PATH)
+ include_directories(BEFORE "${SQLITE3_INC_PATH}")
+ else()
+ message(FATAL_ERROR " Unable to find sqlite3")
+ endif()
+
+ target_link_libraries(${SUB} websockets ${SQLITE3_LIB_PATH})
+
+ include_directories(BEFORE "${SAI_LWS_INC_PATH}")
+
+ CHECK_C_SOURCE_COMPILES("#include <libwebsockets.h>\nint
+ main(void) {\n#if defined(LWS_HAVE_LIBCAP)\n return
+ 0;\n#else\n fail;\n#endif\n return 0;\n}\n" HAS_LIBCAP)
+ if (HAS_LIBCAP)
+ find_library(CAP_LIB_PATH NAMES "cap")
+ endif()
+
+ target_link_libraries(${SUB} ${SAI_LWS_LIB_PATH})
+
+ if (LWS_OPENSSL_LIBRARIES)
+ target_link_libraries(${SUB} ${LWS_OPENSSL_LIBRARIES})
+ endif()
+
+ if (SAI_EXT_PTHREAD_LIBRARIES)
+ target_link_libraries(${SUB} ${SAI_EXT_PTHREAD_LIBRARIES})
+ endif()
+
+ if (HAS_LIBCAP)
+ target_link_libraries(${SUB} ${CAP_LIB_PATH})
+ endif()
+
+ if (MSVC)
+ target_link_libraries(${SUB} ws2_32.lib userenv.lib psapi.lib iphlpapi.lib)
+ endif()
+
+ install(TARGETS ${SUB} RUNTIME DESTINATION "${BIN_DIR}" COMPONENT server)
+
+endif(requirements)
+
diff --git a/src/server/s-central.c b/src/server/s-central.c
new file mode 100644
index 0000000..b04355e
--- /dev/null
+++ b/src/server/s-central.c
@@ -0,0 +1,168 @@
+/*
+ * Sai server
+ *
+ * Copyright (C) 2019 - 2020 Andy Green <andy@warmcat.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation:
+ * version 2.1 of the License.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301 USA
+ *
+ *
+ * Central dispatcher for jobs from events that made it into the database. This
+ * is done in an event-driven way in m-task.c, but management of it also has to
+ * be done in the background for when there are no events coming,
+ */
+
+#include <libwebsockets.h>
+#include <string.h>
+#include <signal.h>
+#include <time.h>
+
+#include "s-private.h"
+
+extern struct lws_context *context;
+
+static void
+sais_central_clean_abandoned(struct vhd *vhd)
+{
+ sais_sqlite_cache_t *sc;
+ struct lwsac *ac = NULL;
+ lws_usec_t now = lws_now_usecs();
+ lws_dll2_owner_t o;
+ char s[160];
+ int n, nzr;
+
+ /*
+ * Sqlite cache cleaning
+ */
+
+ nzr = 0;
+ lws_start_foreach_dll_safe(struct lws_dll2 *, p, p1,
+ vhd->sqlite3_cache.head) {
+ sc = lws_container_of(p, sais_sqlite_cache_t, list);
+
+ if (!sc->refcount &&
+ (now - sc->idle_since) > (60 * LWS_USEC_PER_SEC)) {
+ lwsl_notice("%s: delayed db pool clean %s\n", __func__,
+ sc->uuid);
+ lws_struct_sq3_close(&sc->pdb);
+ lws_dll2_remove(&sc->list);
+ free(sc);
+ } else
+ if (sc->refcount)
+ nzr++;
+
+ } lws_end_foreach_dll_safe(p, p1);
+
+ if (vhd->sqlite3_cache.count)
+ lwsl_notice("%s: db pool items: in-use: %d, total: %d\n",
+ __func__, nzr, vhd->sqlite3_cache.count);
+
+ /*
+ * Collect the most recent <=10 events that still feel they're
+ * incomplete and may be running something
+ */
+
+ n = lws_struct_sq3_deserialize(vhd->server.pdb,
+ " and (state != 3 and state != 4 and state != 5)",
+ NULL, lsm_schema_sq3_map_event, &o,
+ &ac, 0, 10);
+ if (n < 0 || !o.head)
+ return;
+
+ /*
+ * For each of those events, look for tasks that have been running
+ * "too long", eg, builder restarted or lost connection etc
+ */
+
+ lws_start_foreach_dll(struct lws_dll2 *, p, o.head) {
+ sai_event_t *e = lws_container_of(p, sai_event_t, list);
+ sqlite3 *pdb = NULL;
+
+ if (!sais_event_db_ensure_open(vhd, e->uuid, 0, &pdb)) {
+ char *err = NULL;
+
+ /*
+ * Such tasks should go into CANCELLED
+ */
+
+ lws_snprintf(s, sizeof(s),
+ "update tasks set state=%d where "
+ "(state=1 OR state=2) and started < %llu",
+ SAIES_CANCELLED, (unsigned long long)
+ (lws_now_secs() - (30 * 60)));
+
+ if (sqlite3_exec(pdb, s, NULL, NULL, &err) !=
+ SQLITE_OK) {
+ lwsl_err("%s: %s: %s: fail\n", __func__, s,
+ sqlite3_errmsg(pdb));
+ if (err)
+ sqlite3_free(err);
+ }
+
+ sais_event_db_close(vhd, &pdb);
+ }
+
+ } lws_end_foreach_dll(p);
+
+ lwsac_free(&ac);
+}
+
+void
+sais_central_cb(lws_sorted_usec_list_t *sul)
+{
+ struct vhd *vhd = lws_container_of(sul, struct vhd, sul_central);
+
+ /*
+ * For each builder connected to us, see if it can handle a new task,
+ * and if so, try to select one matching its supported platforms
+ */
+
+ lws_start_foreach_dll(struct lws_dll2 *, p,
+ vhd->server.builder_owner.head) {
+ sai_plat_t *cb = lws_container_of(p, sai_plat_t, sai_plat_list);
+
+ lwsl_debug("%s: checking tasks %s %d %d %p\n", __func__,
+ cb->name, cb->ongoing, cb->instances, cb->wsi);
+
+ if (cb->wsi && lws_wsi_user(cb->wsi) &&
+ cb->ongoing < cb->instances)
+ /*
+ * try to bind outstanding task to specific builder
+ * instance
+ */
+ sais_allocate_task(vhd,
+ (struct pss *)lws_wsi_user(cb->wsi),
+ cb, cb->platform);
+
+ } lws_end_foreach_dll(p);
+
+ /*
+ * Need to globally check for abandoned tasks periodically
+ */
+
+ if (!vhd->last_check_abandoned_tasks ||
+ lws_now_usecs() > (vhd->last_check_abandoned_tasks +
+ (20 * LWS_USEC_PER_SEC))) {
+
+ sais_central_clean_abandoned(vhd);
+
+ vhd->last_check_abandoned_tasks = lws_now_usecs();
+ }
+
+ /* check again in 1s */
+
+ lws_sul_schedule(context, 0, &vhd->sul_central, sais_central_cb,
+ 1 * LWS_US_PER_SEC);
+}
diff --git a/src/server/s-comms.c b/src/server/s-comms.c
new file mode 100644
index 0000000..c2937c7
--- /dev/null
+++ b/src/server/s-comms.c
@@ -0,0 +1,834 @@
+/*
+ * Sai server
+ *
+ * Copyright (C) 2019 - 2020 Andy Green <andy@warmcat.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation:
+ * version 2.1 of the License.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301 USA
+ *
+ * The same ws interface is connected-to by builders (on path /builder), and
+ * provides the query transport for browsers (on path /browse).
+ *
+ * There's a single server slite3 database containing events, and a separate
+ * sqlite3 database file for each event, it only contains tasks and logs for
+ * the event and can be deleted when the event record associated with it is
+ * deleted. This is to keep is scalable when there may be thousands of events
+ * and related tasks and logs stored.
+ */
+
+#include <libwebsockets.h>
+#include <string.h>
+#include <signal.h>
+#include <time.h>
+#include <stdio.h>
+#include <fcntl.h>
+
+#include "s-private.h"
+
+#include "../common/struct-metadata.c"
+
+typedef enum {
+ SJS_CLONING,
+ SJS_ASSIGNING,
+ SJS_WAITING,
+ SJS_DONE
+} sai_job_state_t;
+
+typedef struct sai_job {
+ struct lws_dll2 jobs_list;
+ char reponame[64];
+ char ref[64];
+ char head[64];
+
+ time_t requested;
+
+ sai_job_state_t state;
+
+} sai_job_t;
+
+const lws_struct_map_t lsm_schema_map_ta[] = {
+ LSM_SCHEMA (sai_task_t, NULL, lsm_task, "com-warmcat-sai-ta"),
+};
+
+extern const lws_struct_map_t lsm_schema_sq3_map_event[];
+extern const lws_ss_info_t ssi_server;
+
+/* len is typically 16 (event uuid is 32 chars + NUL)
+ * But eg, task uuid is concatenated 32-char eventid and 32-char taskid
+ */
+
+int
+sai_uuid16_create(struct lws_context *context, char *dest33)
+{
+ return lws_hex_random(context, dest33, 33);
+}
+
+int
+sai_sqlite3_statement(sqlite3 *pdb, const char *cmd, const char *desc)
+{
+ sqlite3_stmt *sm;
+ int n;
+
+ if (sqlite3_prepare_v2(pdb, cmd, -1, &sm, NULL) != SQLITE_OK) {
+ lwsl_err("%s: Unable to %s: %s\n",
+ __func__, desc, sqlite3_errmsg(pdb));
+
+ return 1;
+ }
+
+ n = sqlite3_step(sm);
+ sqlite3_reset(sm);
+ sqlite3_finalize(sm);
+ if (n != SQLITE_DONE) {
+ n = sqlite3_extended_errcode(pdb);
+ if (!n) {
+ lwsl_info("%s: failed '%s'\n", __func__, cmd);
+ return 0;
+ }
+
+ lwsl_err("%s: %d: Unable to perform \"%s\": %s\n", __func__,
+ n, desc, sqlite3_errmsg(pdb));
+ puts(cmd);
+
+ return 1;
+ }
+
+ return 0;
+}
+
+int
+sais_event_db_ensure_open(struct vhd *vhd, const char *event_uuid,
+ char create_if_needed, sqlite3 **ppdb)
+{
+ char filepath[256], saf[33];
+ sais_sqlite_cache_t *sc;
+
+ if (*ppdb)
+ return 0;
+
+ /* do we have this guy cached? */
+
+ lws_start_foreach_dll(struct lws_dll2 *, p, vhd->sqlite3_cache.head) {
+ sc = lws_container_of(p, sais_sqlite_cache_t, list);
+
+ if (!strcmp(event_uuid, sc->uuid)) {
+ sc->refcount++;
+ *ppdb = sc->pdb;
+ return 0;
+ }
+
+ } lws_end_foreach_dll(p);
+
+ /* ... nope, well, let's open and cache him then... */
+
+ lws_strncpy(saf, event_uuid, sizeof(saf));
+ lws_filename_purify_inplace(saf);
+
+ lws_snprintf(filepath, sizeof(filepath), "%s-event-%s.sqlite3",
+ vhd->sqlite3_path_lhs, saf);
+
+ if (lws_struct_sq3_open(vhd->context, filepath, create_if_needed, ppdb)) {
+ lwsl_err("%s: Unable to open db %s: %s\n", __func__,
+ filepath, sqlite3_errmsg(*ppdb));
+
+ return 1;
+ }
+
+ /* create / add to the schema for the tables we will have in here */
+
+ if (lws_struct_sq3_create_table(*ppdb, lsm_schema_sq3_map_task))
+ return 1;
+
+ sai_sqlite3_statement(*ppdb, "PRAGMA journal_mode=WAL;", "set WAL");
+
+ if (lws_struct_sq3_create_table(*ppdb, lsm_schema_sq3_map_log))
+ return 1;
+
+ if (lws_struct_sq3_create_table(*ppdb, lsm_schema_sq3_map_artifact))
+ return 1;
+
+ sc = malloc(sizeof(*sc));
+ memset(sc, 0, sizeof(*sc));
+ if (!sc) {
+ lws_struct_sq3_close(ppdb);
+ *ppdb = NULL;
+ return 1;
+ }
+
+ lws_strncpy(sc->uuid, event_uuid, sizeof(sc->uuid));
+ sc->refcount = 1;
+ sc->pdb = *ppdb;
+ lws_dll2_add_tail(&sc->list, &vhd->sqlite3_cache);
+
+ return 0;
+}
+
+void
+sais_event_db_close(struct vhd *vhd, sqlite3 **ppdb)
+{
+ sais_sqlite_cache_t *sc;
+
+ if (!*ppdb)
+ return;
+
+ /* look for him in the cache */
+
+ lws_start_foreach_dll(struct lws_dll2 *, p, vhd->sqlite3_cache.head) {
+ sc = lws_container_of(p, sais_sqlite_cache_t, list);
+
+ if (sc->pdb == *ppdb) {
+ *ppdb = NULL;
+ if (--sc->refcount) {
+ lwsl_notice("%s: zero refcount to idle\n",
+ __func__);
+ /*
+ * He's not currently in use then... don't
+ * close him immediately, s-central.c has a
+ * timer that closes and removes sqlite3
+ * cache entries idle for longer than 60s
+ */
+ sc->idle_since = lws_now_usecs();
+ }
+
+ return;
+ }
+
+ } lws_end_foreach_dll(p);
+
+ lws_struct_sq3_close(ppdb);
+ *ppdb = NULL;
+}
+
+int
+sais_event_db_close_all_now(struct vhd *vhd)
+{
+ sais_sqlite_cache_t *sc;
+
+ lws_start_foreach_dll_safe(struct lws_dll2 *, p, p1,
+ vhd->sqlite3_cache.head) {
+ sc = lws_container_of(p, sais_sqlite_cache_t, list);
+
+ lws_struct_sq3_close(&sc->pdb);
+ lws_dll2_remove(&sc->list);
+ free(sc);
+
+ } lws_end_foreach_dll_safe(p, p1);
+
+ return 0;
+}
+
+int
+sais_event_db_delete_database(struct vhd *vhd, const char *event_uuid)
+{
+ char filepath[256], saf[33], r = 0;
+
+ lws_strncpy(saf, event_uuid, sizeof(saf));
+ lws_filename_purify_inplace(saf);
+
+ lws_snprintf(filepath, sizeof(filepath), "%s-event-%s.sqlite3",
+ vhd->sqlite3_path_lhs, saf);
+
+ r = unlink(filepath);
+
+ lws_snprintf(filepath, sizeof(filepath), "%s-event-%s.sqlite3-wal",
+ vhd->sqlite3_path_lhs, saf);
+
+ r |= unlink(filepath);
+
+ lws_snprintf(filepath, sizeof(filepath), "%s-event-%s.sqlite3-shm",
+ vhd->sqlite3_path_lhs, saf);
+
+ return r | unlink(filepath);
+}
+
+
+#if 0
+static void
+sais_all_browser_on_writable(struct vhd *vhd)
+{
+ lws_start_foreach_dll(struct lws_dll2 *, mp, vhd->browsers.head) {
+ struct pss *pss = lws_container_of(mp, struct pss, same);
+
+ lws_callback_on_writable(pss->wsi);
+ } lws_end_foreach_dll(mp);
+}
+#endif
+
+static int
+sai_destroy_builder(struct lws_dll2 *d, void *user)
+{
+// saib_t *b = lws_container_of(d, saib_t, c.builder_list);
+
+ lws_dll2_remove(d);
+
+ return 0;
+}
+
+static void
+sais_server_destroy(struct vhd *vhd, sais_t *server)
+{
+ lws_dll2_foreach_safe(&server->builder_owner, NULL, sai_destroy_builder);
+
+ sais_event_db_close_all_now(vhd);
+
+ lws_struct_sq3_close(&server->pdb);
+}
+
+typedef enum {
+ SHMUT_NONE = -1,
+ SHMUT_HOOK,
+ SHMUT_BROWSE,
+ SHMUT_STATUS,
+ SHMUT_ARTIFACTS,
+ SHMUT_LOGIN
+} sai_http_murl_t;
+
+static const char * const well_known[] = {
+ "/update-hook",
+ "/sai/browse",
+ "/status",
+ "/artifacts/", /* HTTP api for accessing build artifacts */
+ "/login"
+};
+
+static const char *hmac_names[] = {
+ "sai sha256=",
+ "sai sha384=",
+ "sai sha512="
+};
+
+int
+sai_get_head_status(struct vhd *vhd, const char *projname)
+{
+ struct lwsac *ac = NULL;
+ lws_dll2_owner_t o;
+ sai_event_t *e;
+ int state;
+
+ if (lws_struct_sq3_deserialize(vhd->server.pdb, NULL, "created ",
+ lsm_schema_sq3_map_event, &o, &ac, 0, -1))
+ return -1;
+
+ if (!o.head)
+ return -1;
+
+ e = lws_container_of(o.head, sai_event_t, list);
+ state = e->state;
+
+ lwsac_free(&ac);
+
+ return state;
+}
+
+
+static int
+sai_login_cb(void *data, const char *name, const char *filename,
+ char *buf, int len, enum lws_spa_fileupload_states state)
+{
+ return 0;
+}
+
+static const char * const auth_param_names[] = {
+ "lname",
+ "lpass",
+ "success_redir",
+};
+
+enum enum_param_names {
+ EPN_LNAME,
+ EPN_LPASS,
+ EPN_SUCCESS_REDIR,
+};
+
+static int
+callback_ws(struct lws *wsi, enum lws_callback_reasons reason, void *user,
+ void *in, size_t len)
+{
+ struct vhd *vhd = (struct vhd *)lws_protocol_vh_priv_get(
+ lws_get_vhost(wsi), lws_get_protocol(wsi));
+ uint8_t buf[LWS_PRE + 8192], *start = &buf[LWS_PRE], *p = start,
+ *end = &buf[sizeof(buf) - LWS_PRE - 1];
+ struct pss *pss = (struct pss *)user;
+ sai_http_murl_t mu = SHMUT_NONE;
+ char projname[64];
+ int n, resp, r;
+ const char *cp;
+
+ (void)end;
+ (void)p;
+
+ switch (reason) {
+ case LWS_CALLBACK_PROTOCOL_INIT:
+ vhd = lws_protocol_vh_priv_zalloc(lws_get_vhost(wsi),
+ lws_get_protocol(wsi),
+ sizeof(struct vhd));
+ if (!vhd)
+ return -1;
+
+ vhd->context = lws_get_context(wsi);
+ vhd->vhost = lws_get_vhost(wsi);
+
+ if (lws_pvo_get_str(in, "notification-key",
+ &vhd->notification_key)) {
+ lwsl_err("%s: notification_key pvo required\n", __func__);
+ return -1;
+ }
+
+ if (lws_pvo_get_str(in, "database", &vhd->sqlite3_path_lhs)) {
+ lwsl_err("%s: database pvo required\n", __func__);
+ return -1;
+ }
+
+ lws_snprintf((char *)buf, sizeof(buf), "%s-events.sqlite3",
+ vhd->sqlite3_path_lhs);
+
+ if (lws_struct_sq3_open(vhd->context, (char *)buf, 1,
+ &vhd->server.pdb)) {
+ lwsl_err("%s: Unable to open session db %s: %s\n",
+ __func__, vhd->sqlite3_path_lhs, sqlite3_errmsg(
+ vhd->server.pdb));
+
+ return -1;
+ }
+
+ sai_sqlite3_statement(vhd->server.pdb,
+ "PRAGMA journal_mode=WAL;", "set WAL");
+
+ if (lws_struct_sq3_create_table(vhd->server.pdb,
+ lsm_schema_sq3_map_event)) {
+ lwsl_err("%s: unable to create event table\n", __func__);
+ return -1;
+ }
+
+ lwsl_notice("%s: creating server stream\n", __func__);
+
+ if (lws_ss_create(vhd->context, 0, &ssi_server, vhd,
+ &vhd->h_ss_websrv, NULL, NULL)) {
+ lwsl_err("%s: failed to create secure stream\n",
+ __func__);
+ return -1;
+ }
+
+ lws_sul_schedule(vhd->context, 0, &vhd->sul_central,
+ sais_central_cb, 500 * LWS_US_PER_MS);
+
+ break;
+
+ case LWS_CALLBACK_PROTOCOL_DESTROY:
+ sais_server_destroy(vhd, &vhd->server);
+ goto passthru;
+
+ /*
+ * receive http hook notifications
+ */
+
+ case LWS_CALLBACK_HTTP:
+
+ resp = HTTP_STATUS_FORBIDDEN;
+ pss->vhd = vhd;
+
+ for (n = 0; n < (int)LWS_ARRAY_SIZE(well_known); n++)
+ if (!strncmp((const char *)in, well_known[n],
+ strlen(well_known[n]))) {
+ mu = n;
+ break;
+ }
+
+ pss->our_form = 0;
+
+ // lwsl_notice("%s: HTTP: xmu = %d\n", __func__, n);
+
+ switch (mu) {
+
+ case SHMUT_NONE:
+ goto passthru;
+
+ case SHMUT_HOOK:
+ pss->our_form = 1;
+ lwsl_notice("LWS_CALLBACK_HTTP: sees hook\n");
+ return 0;
+
+ case SHMUT_STATUS:
+ /*
+ * in is a string like /libwebsockets/status.svg
+ */
+ cp = ((const char *)in) + 7;
+ while (*cp == '/')
+ cp++;
+ n = 0;
+ while (*cp != '/' && *cp && (size_t)n < sizeof(projname) - 1)
+ projname[n++] = *cp++;
+ projname[n] = '\0';
+
+ // lwsl_notice("%s: status %s\n", __func__, projname);
+
+ r = sai_get_head_status(vhd, projname);
+ if (r < 2)
+ r = 2;
+ n = lws_snprintf(projname, sizeof(projname),
+ "../decal-%d.svg", r);
+
+ if (lws_http_redirect(wsi, 307,
+ (unsigned char *)projname, n,
+ &p, end) < 0)
+ return -1;
+
+ goto passthru;
+
+ default:
+ lwsl_notice("%s: DEFAULT!!!\n", __func__);
+ return 0;
+ }
+
+ resp = HTTP_STATUS_OK;
+
+ if (lws_add_http_header_status(wsi, resp, &p, end))
+ goto bail;
+ if (lws_add_http_header_content_length(wsi, 0, &p, end))
+ goto bail;
+ if (lws_finalize_write_http_header(wsi, start, &p, end))
+ goto bail;
+ goto try_to_reuse;
+
+
+ case LWS_CALLBACK_HTTP_WRITEABLE:
+
+ lwsl_notice("%s: HTTP_WRITEABLE\n", __func__);
+
+ if (!pss || !pss->blob_artifact)
+ break;
+
+ n = lws_ptr_diff(end, start);
+ if ((int)(pss->artifact_length - pss->artifact_offset) < n)
+ n = (int)(pss->artifact_length - pss->artifact_offset);
+
+ if (sqlite3_blob_read(pss->blob_artifact, start, n,
+ pss->artifact_offset)) {
+ lwsl_err("%s: blob read failed\n", __func__);
+ return -1;
+ }
+
+ pss->artifact_offset += n;
+
+ if (lws_write(wsi, start, n,
+ pss->artifact_offset != pss->artifact_length ?
+ LWS_WRITE_HTTP : LWS_WRITE_HTTP_FINAL) != n)
+ return -1;
+
+ if (pss->artifact_offset != pss->artifact_length)
+ lws_callback_on_writable(wsi);
+
+ break;
+
+ /*
+ * Notifcation POSTs
+ */
+
+ case LWS_CALLBACK_HTTP_BODY:
+
+ if (pss->login_form) {
+
+ if (!pss->spa) {
+ pss->spa = lws_spa_create(wsi, auth_param_names,
+ LWS_ARRAY_SIZE(auth_param_names),
+ 1024, sai_login_cb, pss);
+ if (!pss->spa) {
+ lwsl_err("failed to create spa\n");
+ return -1;
+ }
+ }
+
+ goto spa_process;
+
+ }
+
+ if (!pss->our_form) {
+ lwsl_notice("%s: not our form\n", __func__);
+ goto passthru;
+ }
+
+ lwsl_user("LWS_CALLBACK_HTTP_BODY: %d\n", (int)len);
+ /* create the POST argument parser if not already existing */
+
+ if (!pss->spa) {
+ pss->wsi = wsi;
+ if (lws_hdr_copy(wsi, pss->notification_sig,
+ sizeof(pss->notification_sig),
+ WSI_TOKEN_HTTP_AUTHORIZATION) < 0) {
+ lwsl_err("%s: failed to get signature hdr\n",
+ __func__);
+ return -1;
+ }
+
+ if (lws_hdr_copy(wsi, pss->sn.e.source_ip,
+ sizeof(pss->sn.e.source_ip),
+ WSI_TOKEN_X_FORWARDED_FOR) < 0)
+ lws_get_peer_simple(wsi, pss->sn.e.source_ip,
+ sizeof(pss->sn.e.source_ip));
+
+ pss->spa = lws_spa_create(wsi, NULL, 0, 1024,
+ sai_notification_file_upload_cb, pss);
+ if (!pss->spa) {
+ lwsl_err("failed to create spa\n");
+ return -1;
+ }
+
+ /* find out the hmac used to sign it */
+
+ pss->hmac_type = LWS_GENHMAC_TYPE_UNKNOWN;
+ for (n = 0; n < (int)LWS_ARRAY_SIZE(hmac_names); n++)
+ if (!strncmp(pss->notification_sig,
+ hmac_names[n],
+ strlen(hmac_names[n]))) {
+ pss->hmac_type = n + 1;
+ break;
+ }
+
+ if (pss->hmac_type == LWS_GENHMAC_TYPE_UNKNOWN) {
+ lwsl_notice("%s: unknown sig hash type\n",
+ __func__);
+ return -1;
+ }
+
+ /* convert it to binary */
+
+ n = lws_hex_to_byte_array(
+ pss->notification_sig + strlen(hmac_names[n]),
+ (uint8_t *)pss->notification_sig, 64);
+
+ if (n != (int)lws_genhmac_size(pss->hmac_type)) {
+ lwsl_notice("%s: notifcation hash bad length\n",
+ __func__);
+
+ return -1;
+ }
+ }
+
+spa_process:
+
+ /* let it parse the POST data */
+
+ if (!pss->spa_failed &&
+ lws_spa_process(pss->spa, in, (int)len))
+ /*
+ * mark it as failed, and continue taking body until
+ * completion, and return error there
+ */
+ pss->spa_failed = 1;
+
+ break;
+
+ case LWS_CALLBACK_HTTP_BODY_COMPLETION:
+ lwsl_user("%s: LWS_CALLBACK_HTTP_BODY_COMPLETION: %d\n",
+ __func__, (int)len);
+
+ if (!pss->our_form && !pss->login_form) {
+ lwsl_user("%s: no sai form\n", __func__);
+ goto passthru;
+ }
+
+ if (pss->spa) {
+ lws_spa_finalize(pss->spa);
+ lws_spa_destroy(pss->spa);
+ pss->spa = NULL;
+ }
+
+ if (pss->spa_failed)
+ lwsl_notice("%s: notification failed\n", __func__);
+ else {
+ lwsl_notice("%s: notification: %d %s %s %s\n", __func__,
+ pss->sn.action, pss->sn.e.hash,
+ pss->sn.e.ref, pss->sn.e.repo_name);
+
+ /*
+ * Inform sai-webs about notification processing, so
+ * they can update connected browsers to show the new
+ * event
+ */
+
+ sais_websrv_broadcast(vhd->h_ss_websrv,
+ "{\"schema\":\"sai-overview\"}", 25);
+ }
+
+ if (lws_return_http_status(wsi,
+ pss->spa_failed ? HTTP_STATUS_FORBIDDEN :
+ HTTP_STATUS_OK,
+ NULL) < 0)
+ return -1;
+ return 0;
+
+ /*
+ * ws connections from builders and browsers
+ */
+
+ case LWS_CALLBACK_FILTER_PROTOCOL_CONNECTION:
+ n = lws_hdr_copy(wsi, (char *)buf, sizeof(buf) - 1,
+ WSI_TOKEN_GET_URI);
+ if (!n)
+ buf[0] = '\0';
+ //lwsl_notice("%s: checking with lwsgs for ws conn: %s\n",
+ // __func__, (const char *)buf);
+
+ /*
+ * Builders don't authenticate using sessions...
+ */
+
+ if (n >= 8 && !strncmp((const char *)buf + n - 8,
+ "/builder", 8))
+ return 0;
+
+ return 0;
+
+ case LWS_CALLBACK_ESTABLISHED:
+ pss->wsi = wsi;
+ pss->vhd = vhd;
+ if (!vhd)
+ return -1;
+ pss->alang[0] = '\0';
+ lws_hdr_copy(wsi, pss->alang, sizeof(pss->alang),
+ WSI_TOKEN_HTTP_ACCEPT_LANGUAGE);
+
+ if (lws_hdr_total_length(wsi, WSI_TOKEN_GET_URI)) {
+ if (lws_hdr_copy(wsi, (char *)start, 64,
+ WSI_TOKEN_GET_URI) < 0)
+ return -1;
+ }
+#if defined(LWS_ROLE_H2)
+ else
+ if (lws_hdr_copy(wsi, (char *)start, 64,
+ WSI_TOKEN_HTTP_COLON_PATH) < 0)
+ return -1;
+#endif
+
+ if (!memcmp((char *)start, "/sai", 4))
+ start += 4;
+
+ if (!strcmp((char *)start, "/builder")) {
+ lwsl_info("%s: ESTABLISHED: builder\n", __func__);
+ pss->wsi = wsi;
+ /*
+ * this adds our pss part, but not the logical builder
+ * yet, until we get the ws rx
+ */
+ lws_dll2_add_head(&pss->same, &vhd->builders);
+ break;
+ }
+
+ lwsl_err("%s: unknown URL '%s'\n", __func__, start);
+
+ return -1;
+
+ case LWS_CALLBACK_CLOSED:
+ lwsac_free(&pss->query_ac);
+
+ lwsl_user("%s: CLOSED builder conn\n", __func__);
+ /* remove pss from vhd->builders */
+ lws_dll2_remove(&pss->same);
+
+ /*
+ * Destroy any the builder-tracking objects that
+ * were using this departing connection
+ */
+
+ lws_start_foreach_dll_safe(struct lws_dll2 *, p, p1,
+ vhd->server.builder_owner.head) {
+ sai_plat_t *cb = lws_container_of(p, sai_plat_t,
+ sai_plat_list);
+
+ if (cb->wsi == wsi) {
+ /* remove builder object itself from server list */
+ cb->wsi = NULL;
+ lws_dll2_remove(&cb->sai_plat_list);
+ /*
+ * free the deserialized builder object,
+ * everything he pointed to was overallocated
+ * when his deep copy was made
+ */
+ free(cb);
+ }
+
+ } lws_end_foreach_dll_safe(p, p1);
+
+ if (pss->blob_artifact) {
+ sqlite3_blob_close(pss->blob_artifact);
+ pss->blob_artifact = NULL;
+ }
+
+ if (pss->pdb_artifact) {
+ sais_event_db_close(pss->vhd, &pss->pdb_artifact);
+ pss->pdb_artifact = NULL;
+ }
+
+ /*
+ * Update the sai-webs about the builder removal, so they
+ * can update their connected browsers
+ */
+ sais_list_builders(vhd);
+ break;
+
+ case LWS_CALLBACK_RECEIVE:
+
+ if (!pss->vhd)
+ pss->vhd = vhd;
+
+ lwsl_info("SWT_BUILDER RX: %d\n", (int)len);
+ /*
+ * Builder sent us something on websockets
+ */
+ pss->wsi = wsi;
+ if (sais_ws_json_rx_builder(vhd, pss, in, len))
+ return -1;
+ if (!pss->announced) {
+
+ /*
+ * Update the sai-webs about the builder removal, so they
+ * can update their connected browsers
+ */
+ sais_list_builders(vhd);
+
+ pss->announced = 1;
+ }
+ break;
+
+ case LWS_CALLBACK_SERVER_WRITEABLE:
+ if (!vhd) {
+ lwsl_notice("%s: no vhd\n", __func__);
+ break;
+ }
+
+ return sais_ws_json_tx_builder(vhd, pss, buf, sizeof(buf));
+
+ default:
+passthru:
+ // if (!pss || !vhd)
+ break;
+
+ // return vhd->gsp->callback(wsi, reason, pss->pss_gs, in, len);
+ }
+
+ return lws_callback_http_dummy(wsi, reason, user, in, len);
+
+bail:
+ return 1;
+
+try_to_reuse:
+ if (lws_http_transaction_completed(wsi))
+ return -1;
+
+ return 0;
+}
+
+const struct lws_protocols protocol_ws =
+ { "com-warmcat-sai", callback_ws, sizeof(struct pss), 0 };
diff --git a/src/server/s-conf.c b/src/server/s-conf.c
new file mode 100644
index 0000000..c03a236
--- /dev/null
+++ b/src/server/s-conf.c
@@ -0,0 +1,86 @@
+/*
+ * Sai server src/server/conf.c
+ *
+ * Copyright (C) 2019 - 2020 Andy Green <andy@warmcat.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation:
+ * version 2.1 of the License.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301 USA
+ */
+#include <libwebsockets.h>
+#include <string.h>
+#include <signal.h>
+#include <time.h>
+
+#define SAI_CONFIG_STRING_SIZE (16 * 1024)
+
+extern struct lws_context *context;
+
+struct lws_context *
+sai_lws_context_from_json(const char *config_dir,
+ struct lws_context_creation_info *info,
+ const struct lws_protocols **pprotocols,
+ const char *jpol)
+{
+ int cs_len = SAI_CONFIG_STRING_SIZE - 1;
+ struct lws_context *context;
+ char *cs, *config_strings;
+
+ cs = config_strings = malloc(SAI_CONFIG_STRING_SIZE);
+ if (!config_strings) {
+ lwsl_err("Unable to allocate config strings heap\n");
+
+ return NULL;
+ }
+
+ memset(info, 0, sizeof(*info));
+
+ info->external_baggage_free_on_destroy = config_strings;
+ info->pt_serv_buf_size = 8192;
+ info->options = LWS_SERVER_OPTION_DO_SSL_GLOBAL_INIT |
+ LWS_SERVER_OPTION_EXPLICIT_VHOSTS |
+ LWS_SERVER_OPTION_HTTP_HEADERS_SECURITY_BEST_PRACTICES_ENFORCE |
+ LWS_SERVER_OPTION_VALIDATE_UTF8;
+ info->pss_policies_json = jpol;
+
+ lwsl_notice("Using config dir: \"%s\"\n", config_dir);
+
+ /*
+ * first go through the config for creating the outer context
+ */
+ if (lwsws_get_config_globals(info, config_dir, &cs, &cs_len))
+ goto init_failed;
+
+ context = lws_create_context(info);
+ if (context == NULL) {
+ /* config_strings freed as 'external baggage' */
+ return NULL;
+ }
+
+ info->pprotocols = pprotocols;
+
+ if (lwsws_get_config_vhosts(context, info, config_dir, &cs, &cs_len)) {
+ lwsl_err("%s: sai_lws_context_from_json failed\n", __func__);
+ lws_context_destroy(context);
+
+ return NULL;
+ }
+
+ return context;
+
+init_failed:
+ free(config_strings);
+
+ return NULL;
+}
diff --git a/src/server/s-notification.c b/src/server/s-notification.c
new file mode 100644
index 0000000..315be97
--- /dev/null
+++ b/src/server/s-notification.c
@@ -0,0 +1,988 @@
+/*
+ * Sai server - src/server/notification.c
+ *
+ * Copyright (C) 2019 - 2020 Andy Green <andy@warmcat.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation:
+ * version 2.1 of the License.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301 USA
+ *
+ * The same ws interface is connected-to by builders (on path /builder), and
+ * provides the query transport for browsers (on path /browse).
+ */
+
+#include <libwebsockets.h>
+#include "s-private.h"
+
+/* starts from +1 of sai_notification_action_t */
+
+static const char *notifaction_action_names[] = {
+ "repo-update"
+};
+
+static const char * const paths[] = {
+ "schema",
+ "action",
+ "repository.name",
+ "repository.fetchurl",
+ "ref",
+ "hash",
+ "nonce",
+ "saifile_len",
+ "saifile",
+};
+
+enum enum_paths {
+ LEJPN_SCHEMA,
+ LEJPN_ACTION,
+ LEJPN_REPOSITORY_NAME,
+ LEJPN_REPOSITORY_FETCHURL,
+ LEJPN_REF,
+ LEJPN_HASH,
+ LEJPN_NONCE,
+ LEJPN_SAIFILE_LEN,
+ LEJPN_SAIFILE,
+};
+
+/*
+ * Saifile parser
+ */
+
+static const char * const saifile_paths[] = {
+ "schema",
+ "platforms.*.build",
+ "platforms.*.default",
+ "platforms.*",
+ "configurations.*.prep",
+ "configurations.*.cmake",
+ "configurations.*.deps",
+ "configurations.*.platforms",
+ "configurations.*.artifacts",
+ "configurations.*.cpack",
+ "configurations.*",
+};
+
+enum enum_saifile_paths {
+ LEJPNSAIF_SCHEMA,
+ LEJPNSAIF_PLAT_BUILD,
+ LEJPNSAIF_PLAT_DEFAULT,
+ LEJPNSAIF_PLAT_NAME,
+ LEJPNSAIF_CONFIGURATIONS_PREP,
+ LEJPNSAIF_CONFIGURATIONS_CMAKE,
+ LEJPNSAIF_CONFIGURATIONS_DEPS,
+ LEJPNSAIF_CONFIGURATIONS_PLATFORMS,
+ LEJPNSAIF_CONFIGURATIONS_ARTIFACTS,
+ LEJPNSAIF_CONFIGURATIONS_CPACK,
+ LEJPNSAIF_CONFIGURATIONS_NAME,
+};
+
+/* do the string subst for ${cmake} etc */
+
+static int
+exp_cmake(void *priv, const char *name, char *out, size_t *pos, size_t olen,
+ size_t *exp_ofs)
+{
+ sai_notification_t *sn = (sai_notification_t *)priv;
+ const char *replace = NULL;
+ size_t replace_len, rem_out;
+
+ if (!strcmp(name, "prep")) {
+ replace = sn->t.prep;
+ replace_len = strlen(sn->t.prep);
+ goto expand;
+ }
+
+ if (!strcmp(name, "cmake")) {
+ replace = sn->t.cmake;
+ replace_len = strlen(sn->t.cmake);
+ goto expand;
+ }
+
+ if (!strcmp(name, "cpack")) {
+ replace = sn->t.cpack;
+ replace_len = strlen(sn->t.cpack);
+ goto expand;
+ }
+
+ return LSTRX_FATAL_NAME_UNKNOWN;
+
+expand:
+ rem_out = olen - *pos; /* remaining rhs */
+ replace_len -= *exp_ofs;
+ if (replace_len < rem_out)
+ rem_out = replace_len;
+
+ memcpy(out + *pos, replace + (*exp_ofs), rem_out);
+ *exp_ofs += rem_out;
+ *pos += rem_out;
+
+ if (rem_out == replace_len)
+ return LSTRX_DONE;
+
+ return LSTRX_FILLED_OUT;
+}
+
+static int
+arg_to_bool(const char *s)
+{
+ static const char * const on[] = { "on", "yes", "true" };
+ int n = atoi(s);
+
+ if (n)
+ return 1;
+
+ for (n = 0; n < (int)LWS_ARRAY_SIZE(on); n++)
+ if (!strcasecmp(s, on[n]))
+ return 1;
+
+ return 0;
+}
+
+static int
+sai_tuple_element_compare(const char *e1, const char *e2)
+{
+ char *p;
+
+ // lwsl_notice("%s: comp '%s' '%s'\n", __func__, e1, e2);
+
+ if (strlen(e1) > strlen(e2))
+ /* can't match if req is longer than plat */
+ return -1;
+
+ p = strchr(e2, '/');
+ if (p && lws_ptr_diff(p, e2) < (int)strlen(e1))
+ /* e2 contains a / section terminal inside e1 scope */
+ return 1;
+
+ return strncmp(e1, e2, strlen(e1));
+}
+
+static int
+sai_tuple_compare(const char *req, int req_len, const char *plat)
+{
+ const char *pos = req;
+ char e1[96];
+ int n;
+
+ do {
+ n = 0;
+ while (n < (int)sizeof(e1) - 1 && req_len && *pos != '/') {
+ e1[n++] = *pos++;
+ req_len--;
+ }
+ e1[n] = '\0';
+ if (*pos == '/' && req_len) {
+ req_len--;
+ pos++;
+ }
+
+ if (n == sizeof(e1) - 1) {
+ // lwsl_notice("%s: NOMATCH 3 '%s' '%s'\n", __func__, req, plat);
+ /* element too long */
+ return 1;
+ }
+
+ if (n && /* let it pass if empty match section, eg, linux//gcc */
+ sai_tuple_element_compare(e1, plat)) {
+ // lwsl_notice("%s: NOMATCH 2 '%s' '%s'\n", __func__, req, plat);
+ /* section does not match */
+ return 1;
+ }
+
+ while (*plat && *plat != '/')
+ plat++;
+ if (*plat == '/')
+ plat++;
+
+ if (!*plat && req_len) {
+ // lwsl_notice("%s: NOMATCH 1 '%s' '%s'\n", __func__, req, plat);
+ /* FAIL: req had more than we had */
+ return 1;
+ }
+
+ if (!req_len) {
+ // lwsl_notice("%s: MATCH '%s' '%s'\n", __func__, req, plat);
+ /* MATCH: we matched at least as far as we had */
+ return 0;
+ }
+
+ } while (1);
+
+ return 1;
+}
+
+/*
+ * We parse the saifile JSON
+ */
+
+static signed char
+sai_saifile_lejp_cb(struct lejp_ctx *ctx, char reason)
+{
+ struct pss *pss = (struct pss *)ctx->user;
+ sai_notification_t *sn = (sai_notification_t *)&pss->sn;
+ sqlite3 *pdb = NULL;
+ size_t n;
+
+ // lwsl_notice("%s: reason %d, %s\n", __func__, reason, ctx->path);
+
+ if (reason == LEJPCB_COMPLETE) {
+
+ if (!pss->dry)
+ lwsl_notice("%s: saifile decode completed\n", __func__);
+
+ return 0;
+ }
+
+ if (reason == LEJPCB_OBJECT_START &&
+ ctx->path_match - 1 == LEJPNSAIF_CONFIGURATIONS_NAME) {
+ /*
+ * We're at the testname part of "testname" : { }
+ */
+ lws_strncpy(sn->t.taskname, &ctx->path[15],
+ sizeof(sn->t.taskname));
+ sn->t.prep[0] = '\0';
+ sn->t.packages[0] = '\0';
+ sn->t.cmake[0] = '\0';
+ sn->t.artifacts[0] = '\0';
+ sn->explicit_platforms[0] = '\0';
+ return 0;
+ }
+
+ if (reason == LEJPCB_OBJECT_END &&
+ ctx->path_match - 1 == LEJPNSAIF_CONFIGURATIONS_NAME &&
+ sn->t.taskname[0]) {
+ lws_dll2_owner_t owner;
+ char *err;
+
+ /*
+ * We're at the testname part of "testname" : { }
+ */
+ if (pss->dry) {
+ sn->t.taskname[0] = '\0';
+
+ return 0;
+ }
+
+ /*
+ * Iterate through each platform creating task entries for this
+ * configuration customized for each platform... first time just
+ * check the string subst for all of them is OK...
+ */
+
+ lwsl_notice("%s: Creating task entries for notification\n", __func__);
+
+ lws_start_foreach_dll(struct lws_dll2 *, p,
+ pss->platform_owner.head) {
+ sai_platform_t *pl =
+ lws_container_of(p, sai_platform_t, list);
+ size_t used_in, used_out;
+ struct lws_tokenize ts;
+ int n, match = 0;
+ lws_strexp_t sx;
+
+ if (!pl->nondefault)
+ /*
+ * If it's a default platform, then match
+ * by default
+ */
+ match = 1;
+
+ if (sn->explicit_platforms[0]) {
+ char not = 0;
+
+ /*
+ * If the configuration gives explicit
+ * platforms, we have to filter the platform
+ * against the comma-separated list.
+ *
+ * "none" - don't match any plats just because
+ * they are default
+ *
+ * "not plat" - disallow specific plat "plat",
+ * implies you're not using "none"
+ *
+ * "plat" - allow specific plat "plat"
+ */
+ memset(&ts, 0, sizeof(ts));
+ ts.start = (char *)sn->explicit_platforms;
+ ts.len = strlen(ts.start);
+ ts.flags = LWS_TOKENIZE_F_DOT_NONTERM |
+ LWS_TOKENIZE_F_SLASH_NONTERM |
+ LWS_TOKENIZE_F_MINUS_NONTERM;
+ do {
+ ts.e = lws_tokenize(&ts);
+ if (ts.e != LWS_TOKZE_TOKEN)
+ continue;
+
+ // lwsl_notice("%s: check %.*s\n", __func__,
+ // (int)ts.token_len, ts.token);
+
+ if (!strncmp(ts.token, "none",
+ ts.token_len)) {
+ not = match = 0;
+ continue;
+ }
+ if (!strncmp(ts.token, "not",
+ ts.token_len)) {
+ not = 1;
+ continue;
+ }
+ /*
+ * We need to check with per-slash
+ * minimal matching, ie, "linux"
+ * matches "linux-ubuntu/x86_64/gcc"
+ */
+ if (!sai_tuple_compare(ts.token,
+ ts.token_len, pl->name)) {
+ match = !not;
+ if (match)
+ break;
+ }
+ not = 0;
+ } while (ts.e > 0);
+ }
+
+ if (match) {
+
+ /*
+ * The platform build string (pl->build) is the
+ * base, with optional entries like ${cmake}
+ * that are filled in with the corresponding
+ * info from the specific configuration's
+ * "cmake" entry (sn->t.cmake)...
+ *
+ * sn->t.build becomes the string-substituted
+ * copy that is serialized
+ */
+
+ lws_strexp_init(&sx, sn, exp_cmake, sn->t.build,
+ sizeof(sn->t.build));
+
+ n = lws_strexp_expand(&sx, pl->build,
+ strlen(pl->build),
+ &used_in, &used_out);
+ if (n != LSTRX_DONE) {
+ lwsl_notice("%s: strsubst fail n=%d %s %s\n",
+ __func__, n, pl->build,
+ sn->t.cmake);
+ return -1;
+ }
+ }
+ } lws_end_foreach_dll(p);
+
+ /*
+ * ...now we know the string handling will go well, create the
+ * event database file and create task entries in there for this
+ * configuration's tasks for each platform
+ */
+
+ if (sais_event_db_ensure_open(pss->vhd, pss->sn.e.uuid, 1, &pdb)) {
+ lwsl_err("%s: unable to open event-specific db\n", __func__);
+ return -1;
+ }
+
+ sqlite3_exec(pdb, "BEGIN TRANSACTION", NULL, NULL, &err);
+ if (err)
+ sqlite3_free(err);
+
+ lws_start_foreach_dll(struct lws_dll2 *, p,
+ pss->platform_owner.head) {
+ sai_platform_t *pl =
+ lws_container_of(p, sai_platform_t, list);
+ size_t used_in, used_out;
+ struct lws_tokenize ts;
+ int n, match = 0;
+ lws_strexp_t sx;
+
+ if (!pl->nondefault)
+ /*
+ * If it's a nondefault platform, then match
+ * by default
+ */
+ match = 1;
+
+ if (sn->explicit_platforms[0]) {
+ char not = 0;
+
+ /*
+ * If the configuration gives explicit
+ * platforms, we have to filter the platform
+ * against the comma-separated list.
+ *
+ * "none" - don't match any plats just because
+ * they are default
+ *
+ * "not plat" - disallow specific plat "plat",
+ * implies you're not using "none"
+ *
+ * "plat" - allow specific plat "plat"
+ */
+
+ memset(&ts, 0, sizeof(ts));
+ ts.start = (char *)sn->explicit_platforms;
+ ts.len = strlen(ts.start);
+ ts.flags = LWS_TOKENIZE_F_DOT_NONTERM |
+ LWS_TOKENIZE_F_SLASH_NONTERM |
+ LWS_TOKENIZE_F_MINUS_NONTERM;
+
+ do {
+ ts.e = lws_tokenize(&ts);
+ if (ts.e != LWS_TOKZE_TOKEN)
+ continue;
+
+ // lwsl_notice("%s: check %.*s\n", __func__,
+ // (int)ts.token_len, ts.token);
+
+ if (!strncmp(ts.token, "none",
+ ts.token_len)) {
+ not = match = 0;
+ continue;
+ }
+ if (!strncmp(ts.token, "not",
+ ts.token_len)) {
+ not = 1;
+ continue;
+ }
+ if (!sai_tuple_compare(ts.token,
+ ts.token_len, pl->name)) {
+ match = !not;
+ if (match)
+ break;
+ }
+ not = 0;
+ } while (ts.e > 0);
+ }
+
+ if (match) {
+ /*
+ * For this platform, we want to create a task
+ * associated with this event. Tasks and logs
+ * associated with an event go in an event-
+ * specific database file for scalability.
+ */
+
+ lws_strexp_init(&sx, sn, exp_cmake, sn->t.build,
+ sizeof(sn->t.build));
+
+ n = lws_strexp_expand(&sx, pl->build,
+ strlen(pl->build),
+ &used_in, &used_out);
+ if (n != LSTRX_DONE) {
+ lwsl_notice("%s: strsubst failed %s %s\n",
+ __func__, pl->build,
+ sn->t.cmake);
+ sqlite3_exec(pdb, "END TRANSACTION", NULL, NULL, &err);
+ if (err)
+ sqlite3_free(err);
+ sais_event_db_close(pss->vhd, &pdb);
+ return -1;
+ }
+
+ /*
+ * Prepare a struct of the task object...
+ * task uuid is the event uuid and another
+ * random 32 chars, so you can always recover
+ * the related event uuid from the task uuid
+ */
+
+ memcpy(pss->sn.t.uuid, pss->sn.e.uuid, 32);
+ sai_uuid16_create(lws_get_context(pss->wsi),
+ pss->sn.t.uuid + 32);
+ strcpy(pss->sn.t.event_uuid, pss->sn.e.uuid);
+ pss->sn.t.uid = pss->sn.event_task_index++;
+
+ /*
+ * This is basically a secret that anything
+ * trying to upload an artifact for the task
+ * must provide to authenticate.
+ */
+ sai_uuid16_create(lws_get_context(pss->wsi),
+ pss->sn.t.art_up_nonce);
+ /*
+ * An unrelated secret that anything
+ * trying to download an artifact for the task
+ * must provide to identify it.
+ */
+ sai_uuid16_create(lws_get_context(pss->wsi),
+ pss->sn.t.art_down_nonce);
+
+ pss->sn.t.git_repo_url =
+ pss->sn.e.repo_fetchurl;
+ pss->sn.e.last_updated =
+ (unsigned long long)lws_now_secs();
+ pss->sn.e.state = 0;
+ lws_strncpy(pss->sn.t.platform, pl->name,
+ sizeof(pss->sn.t.platform));
+
+ memset(&pss->sn.t.list, 0, sizeof(pss->sn.t.list));
+ lws_dll2_owner_clear(&owner);
+ lws_dll2_add_head(&pss->sn.t.list, &owner);
+
+ /*
+ * Create the task in event-specific database
+ */
+
+ lws_struct_sq3_serialize(pdb,
+ lsm_schema_sq3_map_task,
+ &owner, pss->sn.t.uid);
+ }
+
+ } lws_end_foreach_dll(p);
+
+ sqlite3_exec(pdb, "END TRANSACTION", NULL, NULL, &err);
+ if (err)
+ sqlite3_free(err);
+
+ sais_event_db_close(pss->vhd, &pdb);
+
+// lwsl_notice("%s: New test '%s', '%s', '%s'\n", __func__,
+// sn->t.taskname, sn->t.cmake, sn->t.packages);
+
+ sn->t.taskname[0] = '\0';
+ return 0;
+ }
+
+ if (reason == LEJPCB_OBJECT_START &&
+ ctx->path_match - 1 == LEJPNSAIF_PLAT_NAME) {
+ /*
+ * We're at the platformname part of "platformname" : { }
+ */
+ lws_strncpy(sn->platname, &ctx->path[10], sizeof(sn->platname));
+ sn->t.prep[0] = '\0';
+ sn->t.cmake[0] = '\0';
+ sn->t.cpack[0] = '\0';
+ sn->platbuild[0] = '\0';
+ sn->nondefault = 0;
+ return 0;
+ }
+
+ if (reason == LEJPCB_OBJECT_END &&
+ ctx->path_match - 1 == LEJPNSAIF_PLAT_NAME &&
+ sn->platname[0] && sn->platbuild[0]) {
+ sai_platform_t *pl;
+ uint8_t *plb;
+ size_t pnl;
+
+ if (pss->dry) {
+ sn->t.platform[0] = '\0';
+ return 0;
+ }
+
+ /*
+ * We create a platform object with space for its strings after
+ */
+
+ pnl = strlen(sn->platname);
+ pl = (sai_platform_t *)malloc(sizeof(*pl) + pnl +
+ strlen(sn->platbuild) + 2);
+ if (!pl)
+ return -1;
+
+ memset(pl, 0, sizeof(*pl));
+ plb = (uint8_t *)&pl[1];
+
+ /*
+ * Platforms are malloc'd up and added to pss
+ * .platform_owner
+ */
+
+ pl->name = (const char *)plb;
+ memcpy(plb, sn->platname, pnl + 1);
+ plb += pnl + 1;
+
+ pl->build = (const char *)plb;
+ memcpy(plb, sn->platbuild, strlen(sn->platbuild) + 1);
+
+ pl->nondefault = sn->nondefault;
+
+ lws_dll2_add_head(&pl->list, &pss->platform_owner);
+
+// lwsl_notice("%s: New platform '%s', build '%s', notdefault %d\n",
+// __func__, pl->name, pl->build, pl->nondefault);
+
+ sn->platbuild[0] = '\0';
+ return 0;
+ }
+
+ /* we only match on the prepared path strings */
+ if (!(reason & LEJP_FLAG_CB_IS_VALUE) || !ctx->path_match)
+ return 0;
+
+// if (reason != LEJPCB_VAL_STR_END)
+// return 0;
+
+ /* only the end part of the string, where we know the length */
+
+ switch (ctx->path_match - 1) {
+ case LEJPNSAIF_SCHEMA:
+ sn->event_task_index = 0;
+ return 0;
+
+ case LEJPNSAIF_CONFIGURATIONS_PREP:
+ /* the additional cmake options for this test configuration */
+ n = strlen(sn->t.prep);
+ if (n < sizeof(sn->t.prep) - 2)
+ lws_strnncpy(sn->t.prep + n, ctx->buf, ctx->npos,
+ sizeof(sn->t.prep) - n);
+ break;
+
+ case LEJPNSAIF_CONFIGURATIONS_CMAKE:
+ /* the additional cmake options for this test configuration */
+ n = strlen(sn->t.cmake);
+ if (n < sizeof(sn->t.cmake) - 2)
+ lws_strnncpy(sn->t.cmake + n, ctx->buf, ctx->npos,
+ sizeof(sn->t.cmake) - n);
+ break;
+
+ case LEJPNSAIF_CONFIGURATIONS_DEPS:
+ /* the necessary dependent package strings */
+ n = strlen(sn->t.packages);
+ if (n < sizeof(sn->t.packages) - 2) {
+ if (n)
+ sn->t.packages[n++] = ',';
+ lws_strncpy(sn->t.packages + n, ctx->buf,
+ sizeof(sn->t.packages) - n);
+ }
+ break;
+
+ case LEJPNSAIF_CONFIGURATIONS_PLATFORMS:
+ /* the necessary dependent package strings */
+ lws_strncpy(sn->explicit_platforms, ctx->buf,
+ sizeof(sn->explicit_platforms));
+ break;
+
+ case LEJPNSAIF_CONFIGURATIONS_ARTIFACTS:
+ lws_strncpy(sn->t.artifacts, ctx->buf, sizeof(sn->t.artifacts));
+ break;
+
+ case LEJPNSAIF_CONFIGURATIONS_CPACK:
+ lws_strncpy(sn->t.cpack, ctx->buf, sizeof(sn->t.cpack));
+ break;
+
+ case LEJPNSAIF_PLAT_BUILD:
+ /*
+ * The overall build script for this platform
+ * is appended into the temp sn.platbuild
+ */
+ n = strlen(sn->platbuild);
+ if (n < sizeof(sn->platbuild) - 2)
+ lws_strnncpy(sn->platbuild + n, ctx->buf, ctx->npos,
+ sizeof(sn->platbuild) - n);
+ break;
+
+ case LEJPNSAIF_PLAT_DEFAULT:
+ sn->nondefault = !arg_to_bool(ctx->buf);
+ break;
+
+ default:
+ return 0;
+ }
+
+ return 0;
+}
+
+static signed char
+sai_notification_lejp_cb(struct lejp_ctx *ctx, char reason)
+{
+ struct pss *pss = (struct pss *)ctx->user;
+ sai_notification_t *sn = (sai_notification_t *)&pss->sn;
+ size_t ile, ole;
+ int n;
+
+ /* we only match on the prepared path strings */
+ if (!(reason & LEJP_FLAG_CB_IS_VALUE) || !ctx->path_match)
+ return 0;
+
+// if (reason != LEJPCB_VAL_STR_END)
+// return 0;
+
+ /* only the end part of the string, where we know the length */
+
+ switch (ctx->path_match - 1) {
+ case LEJPN_SCHEMA:
+ if (strcmp(ctx->buf, "com-warmcat-sai-notification")) {
+ lwsl_err("%s: unknown schema '%s'\n", __func__, ctx->buf);
+ return -1;
+ }
+ return 0;
+
+ case LEJPN_ACTION:
+ for (n = 0; n < (int)LWS_ARRAY_SIZE(notifaction_action_names);n++)
+ if (!strcmp(ctx->buf, notifaction_action_names[n])) {
+ sn->action = n + 1;
+
+ return 0;
+ }
+ lwsl_notice("%s: unknown action '%s' ignored\n",
+ __func__, ctx->buf);
+ return -1;
+
+ case LEJPN_REPOSITORY_NAME:
+ lws_strncpy(sn->e.repo_name, ctx->buf, sizeof(sn->e.repo_name));
+ break;
+
+ case LEJPN_REPOSITORY_FETCHURL:
+ lws_strncpy(sn->e.repo_fetchurl, ctx->buf, sizeof(sn->e.repo_fetchurl));
+ break;
+
+ case LEJPN_REF:
+ lws_strncpy(sn->e.ref, ctx->buf, sizeof(sn->e.ref));
+ break;
+
+ case LEJPN_HASH:
+ lws_strncpy(sn->e.hash, ctx->buf, sizeof(sn->e.hash));
+ break;
+
+ case LEJPN_NONCE:
+ break;
+
+ case LEJPN_SAIFILE_LEN:
+ sn->saifile_in_len = atoi(ctx->buf);
+ /* only accept sane base64 size */
+ if (sn->saifile_in_len < 8 || sn->saifile_in_len > 65536) {
+ lwsl_err("%s: bad saifile_len %u\n", __func__,
+ (unsigned int)sn->saifile_in_len);
+
+ return -1;
+ }
+ sn->saifile_out_len = (sn->saifile_in_len * 3) / 4 + 4;
+ sn->saifile = malloc(sn->saifile_out_len);
+ if (!sn->saifile) {
+ lwsl_err("%s: OOM\n", __func__);
+
+ return -1;
+ }
+ /*
+ * Caller must take responsibility for sn->saifile allocation
+ */
+ sn->saifile_in_seen = sn->saifile_out_pos = 0;
+ lws_b64_decode_state_init(&sn->b64);
+ break;
+
+ case LEJPN_SAIFILE:
+ /*
+ * Base64 encoding of the commit's .sai.json file contents... we
+ * have to treat this with caution since it can be anything,
+ * including unparsable JSON.
+ *
+ * Let's base64-decode it and collect it into an buffer first,
+ * before sn test parse and then sn second parse to extract the
+ * tasks
+ */
+
+ sn->saifile_in_seen += ctx->npos;
+ if (sn->saifile_in_seen > sn->saifile_in_len) {
+ lwsl_err("%s: SAIFILE: too large in %u vs %u\n", __func__,
+ (unsigned int)sn->saifile_in_seen,
+ (unsigned int)sn->saifile_in_len);
+ return -1;
+ }
+ ile = ctx->npos;
+ ole = sn->saifile_out_len - sn->saifile_out_pos;
+ lws_b64_decode_stateful(&sn->b64, ctx->buf, &ile,
+ (uint8_t *)sn->saifile + sn->saifile_out_pos, &ole,
+ sn->saifile_in_seen == sn->saifile_in_len);
+
+ sn->saifile_out_pos += ole;
+ break;
+
+ default:
+ return 0;
+ }
+
+ return 0;
+}
+
+int
+sai_notification_file_upload_cb(void *data, const char *name,
+ const char *filename, char *buf, int len,
+ enum lws_spa_fileupload_states state)
+{
+ struct pss *pss = (struct pss *)data;
+ struct lejp_ctx saictx;
+ lws_dll2_owner_t owner;
+ uint8_t result[64];
+ int m;
+
+ switch (state) {
+ case LWS_UFS_OPEN:
+ lwsl_notice("%s: LWS_UFS_OPEN\n", __func__);
+ lejp_construct(&pss->ctx, sai_notification_lejp_cb,
+ pss, paths, LWS_ARRAY_SIZE(paths));
+
+ if (lws_genhmac_init(&pss->hmac, pss->hmac_type,
+ (uint8_t *)pss->vhd->notification_key,
+ strlen(pss->vhd->notification_key))) {
+ lwsl_err("%s: failed to init hmac\n", __func__);
+
+ return -1;
+ }
+ break;
+ case LWS_UFS_FINAL_CONTENT:
+ case LWS_UFS_CONTENT:
+ lwsl_notice("%s: LWS_UFS_[]CONTENT: %p %p, \n", __func__, pss, buf);
+ if (len && lws_genhmac_update(&pss->hmac, buf, len))
+ return -1;
+
+ printf("%.*s", (int)len, buf);
+
+ m = lejp_parse(&pss->ctx, (uint8_t *)buf, len);
+ if (m < 0 && m != LEJP_CONTINUE) {
+ lwsl_notice("%s: notif JSON decode failed '%s' (%d)\n",
+ __func__, lejp_error_to_string(m), m);
+ return m;
+ }
+
+ lwsl_notice("%s: m = %d\n", __func__, m);
+
+ if (m != 1)
+ break;
+
+ lws_genhmac_destroy(&pss->hmac, result);
+
+ if (memcmp(result, pss->notification_sig,
+ lws_genhmac_size(pss->hmac_type))) {
+ lwsl_err("%s: hmac mismatch\n", __func__);
+
+ return -1;
+ }
+ lwsl_notice("%s: hmac OK\n", __func__);
+
+ /*
+ * We have the notification metadata JSON parsed into pss->sn.e,
+ * eg, pss->sn->e.hash ... since it's common to, eg, push a tree
+ * in a branch and then later tag the same commit, we don't want
+ * to pointlessly repeat CI for the same tree multiple times,
+ * and need to basically dedupe.
+ */
+
+ {
+ uint64_t rid = 0;
+ char qu[192];
+
+ lws_snprintf(qu, sizeof(qu), "select rowid from events "
+ "where hash='%s'",
+ pss->sn.e.hash);
+
+ if (sqlite3_exec(pss->vhd->server.pdb, qu,
+ sai_sql3_get_uint64_cb,
+ &rid, NULL) == SQLITE_OK && rid) {
+ /* it already exists */
+ lwsl_notice("%s: ignoring notification as "
+ "tree hash event exists\n",
+ __func__);
+
+ return 0;
+ }
+ }
+
+ if (!pss->sn.saifile)
+ return -1;
+
+ /*
+ * We processed the notification JSON, but we only decoded the
+ * base64'd copy of the .sai.json so far... now's the time we
+ * want to process that and break it down into tasks.
+ *
+ * We don't trust it since it's controlled by the guy who pushed
+ * the commit, there can be anything at all in there. We made
+ * sure he can't attack us until now by base64-ing it at the
+ * server hook, so he's just dumb payload.
+ *
+ * Let's try to parse it in two passes, first without acting on
+ * the content to confirm it's going to succeed...
+ */
+
+ sai_uuid16_create(lws_get_context(pss->wsi), pss->sn.e.uuid);
+ if (sais_event_db_ensure_open(pss->vhd, pss->sn.e.uuid, 1,
+ (sqlite3 **)&pss->sn.e.pdb)) {
+ lwsl_err("%s: unable to open event-specific database\n",
+ __func__);
+
+ goto saifile_bail;
+ }
+
+ pss->dry = 1;
+ lejp_construct(&saictx, sai_saifile_lejp_cb, pss, saifile_paths,
+ LWS_ARRAY_SIZE(saifile_paths));
+ m = lejp_parse(&saictx, (uint8_t *)pss->sn.saifile,
+ pss->sn.saifile_out_pos);
+ sais_event_db_close(pss->vhd, (sqlite3 **)&pss->sn.e.pdb);
+ if (m < 0) {
+ lwsl_notice("%s: saifile JSON decode failed '%s' (%d)\n",
+ __func__, lejp_error_to_string(m), m);
+ goto saifile_bail;
+ }
+
+ /* ... then add the 32-char event object in the database ... */
+
+ pss->sn.e.created = (unsigned long long)lws_now_secs();
+ pss->sn.e.state = SAIES_WAITING;
+
+ memset(&pss->sn.e.list, 0, sizeof(pss->sn.e.list));
+ lws_dll2_owner_clear(&owner);
+ lws_dll2_add_head(&pss->sn.e.list, &owner);
+
+ /*
+ * This is our new event going into the event database...
+ */
+
+ lws_struct_sq3_serialize(pss->vhd->server.pdb,
+ lsm_schema_sq3_map_event, &owner, 0);
+
+ /*
+ * ... process the saifile JSON again creating tasks for each
+ * entry in the saifile, for each platform, against that event
+ * object...
+ */
+
+ pss->dry = 0;
+ lejp_construct(&saictx, sai_saifile_lejp_cb, pss, saifile_paths,
+ LWS_ARRAY_SIZE(saifile_paths));
+ m = lejp_parse(&saictx, (uint8_t *)pss->sn.saifile,
+ pss->sn.saifile_out_pos);
+ free(pss->sn.saifile);
+ pss->sn.saifile = NULL;
+ if (m < 0) {
+ lwsl_notice("%s: saifile JSON decode failed '%s' (%d)\n",
+ __func__, lejp_error_to_string(m), m);
+ return m;
+ }
+
+ lwsl_notice("%s: notification inserted into db\n", __func__);
+
+ /*
+ * Reassess now if there's a builder we can match to a pending task
+ */
+
+ lws_sul_schedule(pss->vhd->context, 0, &pss->vhd->sul_central,
+ sais_central_cb, 1);
+
+ return 0;
+
+saifile_bail:
+ free(pss->sn.saifile);
+ pss->sn.saifile = NULL;
+
+ return -1;
+
+ case LWS_UFS_CLOSE:
+ // lwsl_info("%s: LWS_UFS_CLOSE\n", __func__);
+ lejp_destruct(&pss->ctx);
+ break;
+ }
+
+ return 0;
+}
+
diff --git a/src/server/s-private.h b/src/server/s-private.h
new file mode 100644
index 0000000..2337450
--- /dev/null
+++ b/src/server/s-private.h
@@ -0,0 +1,285 @@
+/*
+ * Sai server definitions src/server/private.h
+ *
+ * Copyright (C) 2019 - 2020 Andy Green <andy@warmcat.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation:
+ * version 2.1 of the License.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301 USA
+ */
+
+#include "../common/include/private.h"
+#include <sqlite3.h>
+#include <sys/stat.h>
+
+#define SAI_EVENTID_LEN 32
+#define SAI_TASKID_LEN 64
+
+struct sai_plat;
+
+typedef struct sai_platm {
+ struct lws_dll2_owner builder_owner;
+ struct lws_dll2_owner subs_owner;
+
+ sqlite3 *pdb;
+ sqlite3 *pdb_auth;
+} sais_t;
+
+typedef struct sai_platform {
+ struct lws_dll2 list;
+
+ const char *name;
+ const char *build;
+
+ uint8_t nondefault;
+
+ /* build and name over-allocated here */
+} sai_platform_t;
+
+typedef enum {
+ SAIN_ACTION_INVALID,
+ SAIN_ACTION_REPO_UPDATED
+} sai_notification_action_t;
+
+typedef struct {
+
+ sai_event_t e;
+ sai_task_t t;
+
+ char platbuild[4096];
+ char platname[96];
+ char explicit_platforms[2048];
+
+ int event_task_index;
+
+ struct lws_b64state b64;
+ char *saifile;
+ uint64_t when;
+ size_t saifile_in_len;
+ size_t saifile_out_len;
+ size_t saifile_out_pos;
+ size_t saifile_in_seen;
+ sai_notification_action_t action;
+
+ uint8_t nondefault;
+} sai_notification_t;
+
+typedef enum {
+ WSS_IDLE,
+ WSS_PREPARE_OVERVIEW,
+ WSS_SEND_OVERVIEW,
+ WSS_PREPARE_BUILDER_SUMMARY,
+ WSS_SEND_BUILDER_SUMMARY,
+
+ WSS_PREPARE_TASKINFO,
+ WSS_SEND_ARTIFACT_INFO,
+
+ WSS_PREPARE_EVENTINFO,
+ WSS_SEND_EVENTINFO,
+} ws_state;
+
+typedef struct sai_builder {
+ sais_t c;
+} saib_t;
+
+struct vhd;
+
+struct pss {
+ struct vhd *vhd;
+ struct lws *wsi;
+
+ struct lws_spa *spa;
+ struct lejp_ctx ctx;
+ sai_notification_t sn;
+ struct lws_dll2 same; /* owner: vhd.builders */
+
+ struct lws_dll2 subs_list;
+
+ uint64_t sub_timestamp;
+ char sub_task_uuid[65];
+ char specific[65];
+ char specific_project[96];
+ char auth_user[33];
+
+ sqlite3 *pdb_artifact;
+ sqlite3_blob *blob_artifact;
+
+ lws_dll2_owner_t platform_owner; /* sai_platform_t builder offers */
+ lws_dll2_owner_t task_cancel_owner; /* sai_platform_t builder offers */
+ lws_dll2_owner_t aft_owner; /* for statefully spooling artifact info */
+ lws_struct_args_t a;
+
+ union {
+ sai_plat_t *b;
+ sai_plat_owner_t *o;
+ } u;
+ const char *server_name;
+
+ struct lwsac *query_ac;
+ struct lwsac *logs_ac;
+ lws_dll2_owner_t issue_task_owner; /* list of sai_task_t */
+ const sai_task_t *one_task; /* only for browser */
+ const sai_event_t *one_event;
+ lws_dll2_owner_t query_owner;
+ lws_dll2_t *walk;
+
+ int task_index;
+ int log_cache_index;
+ int log_cache_size;
+ int authorized;
+ int specificity;
+ unsigned long expiry_unix_time;
+
+ /* notification hmac information */
+ char notification_sig[128];
+ char alang[128];
+ struct lws_genhmac_ctx hmac;
+ enum lws_genhmac_types hmac_type;
+ char our_form;
+ char login_form;
+
+ uint64_t first_log_timestamp;
+ uint64_t artifact_offset;
+ uint64_t artifact_length;
+
+ ws_state send_state;
+
+ unsigned int spa_failed:1;
+ unsigned int subsequent:1; /* for individual JSON */
+ unsigned int dry:1;
+ unsigned int frag:1;
+ unsigned int mark_started:1;
+ unsigned int wants_event_updates:1;
+ unsigned int announced:1;
+ unsigned int bulk_binary_data:1;
+
+ uint8_t ovstate; /* SOS_ substate when doing overview */
+};
+
+typedef struct sais_sqlite_cache {
+ lws_dll2_t list;
+ char uuid[65];
+ sqlite3 *pdb;
+ lws_usec_t idle_since;
+ int refcount;
+} sais_sqlite_cache_t;
+
+struct vhd {
+ struct lws_context *context;
+ struct lws_vhost *vhost;
+
+ struct lws_ss_handle *h_ss_websrv; /* server */
+
+ char json_builders[8192];
+
+ /* pss lists */
+ struct lws_dll2_owner builders;
+
+ const char *sqlite3_path_lhs;
+
+ lws_dll2_owner_t sqlite3_cache; /* sais_sqlite_cache_t */
+ lws_dll2_owner_t tasklog_cache;
+ lws_sorted_usec_list_t sul_logcache;
+ lws_sorted_usec_list_t sul_central; /* background task allocation sul */
+
+ lws_usec_t last_check_abandoned_tasks;
+
+ const char *notification_key;
+
+ sais_t server;
+};
+
+extern struct lws_context *
+sai_lws_context_from_json(const char *config_dir,
+ struct lws_context_creation_info *info,
+ const struct lws_protocols **pprotocols,
+ const char *jpol);
+extern const struct lws_protocols protocol_ws;
+
+int
+sai_notification_file_upload_cb(void *data, const char *name,
+ const char *filename, char *buf, int len,
+ enum lws_spa_fileupload_states state);
+
+int
+sai_sqlite3_statement(sqlite3 *pdb, const char *cmd, const char *desc);
+
+int
+sai_uuid16_create(struct lws_context *context, char *dest33);
+
+int
+sais_event_db_ensure_open(struct vhd *vhd, const char *event_uuid, char can_create, sqlite3 **ppdb);
+
+void
+sais_event_db_close(struct vhd *vhd, sqlite3 **ppdb);
+
+int
+sais_event_db_delete_database(struct vhd *vhd, const char *event_uuid);
+
+int
+sais_event_db_close_all_now(struct vhd *vhd);
+
+int
+sai_sq3_event_lookup(sqlite3 *pdb, uint64_t start, lws_struct_args_cb cb, void *ca);
+
+int
+sai_sql3_get_uint64_cb(void *user, int cols, char **values, char **name);
+
+int
+saiw_ws_json_tx_browser(struct vhd *vhd, struct pss *pss, uint8_t *buf, size_t bl);
+
+int
+sais_ws_json_rx_builder(struct vhd *vhd, struct pss *pss, uint8_t *buf, size_t bl);
+
+int
+sais_list_builders(struct vhd *vhd);
+
+void
+sais_eventchange(struct lws_ss_handle *h, const char *event_uuid, int state);
+
+void
+sais_taskchange(struct lws_ss_handle *h, const char *task_uuid, int state);
+
+int
+lws_struct_map_set(const lws_struct_map_t *map, char *u);
+
+void
+sai_task_uuid_to_event_uuid(char *event_uuid33, const char *task_uuid65);
+
+int
+sais_ws_json_tx_builder(struct vhd *vhd, struct pss *pss, uint8_t *buf, size_t bl);
+
+int
+sais_subs_request_writeable(struct vhd *vhd, const char *task_uuid);
+
+void
+sais_central_cb(lws_sorted_usec_list_t *sul);
+
+int
+sais_task_reset(struct vhd *vhd, const char *task_uuid);
+
+int
+sais_task_cancel(struct vhd *vhd, const char *task_uuid);
+
+int
+sais_allocate_task(struct vhd *vhd, struct pss *pss, sai_plat_t *cb,
+ const char *cns_name);
+
+int
+sais_set_task_state(struct vhd *vhd, const char *builder_name,
+ const char *builder_uuid, const char *task_uuid, int state,
+ uint64_t started, uint64_t duration);
+
+void
+sais_websrv_broadcast(struct lws_ss_handle *hsrv, const char *str, size_t len);
diff --git a/src/server/s-sai.c b/src/server/s-sai.c
new file mode 100644
index 0000000..df6ca96
--- /dev/null
+++ b/src/server/s-sai.c
@@ -0,0 +1,93 @@
+/*
+ * Sai server
+ *
+ * Copyright (C) 2019 - 2020 Andy Green <andy@warmcat.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation:
+ * version 2.1 of the License.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301 USA
+ */
+
+#include <libwebsockets.h>
+#include <string.h>
+#include <signal.h>
+#include <time.h>
+
+#include "s-private.h"
+
+static int interrupted;
+struct lws_context *context;
+
+/*
+ * sai-server also has a unix domain socket serving ws, that the sai-web part(s)
+ * connect to as client(s).
+ *
+ * The pass up authenticated browser task and event redo requests, and receive
+ * information about updates to tasks and events that they might have clients
+ * that are watching.
+ */
+
+
+static const char * const default_ss_policy =
+ "{"
+ "\"s\": ["
+ /* uds link between web and server pieces */
+ "{\"websrv\": {"
+ "\"server\":" "true,"
+ "\"endpoint\":" "\"+@com.warmcat.sai-websrv\","
+ "\"protocol\":" "\"ws\""
+ "}}"
+ "]"
+ "}"
+;
+
+static const struct lws_protocols
+ *pprotocols[] = { &protocol_ws, NULL };
+
+static void sigint_handler(int sig)
+{
+ interrupted = 1;
+}
+
+int main(int argc, const char **argv)
+{
+ int logs = LLL_USER | LLL_ERR | LLL_WARN | LLL_NOTICE;
+ const char *p, *conf = "/etc/sai/server";
+ struct lws_context_creation_info info;
+
+ signal(SIGINT, sigint_handler);
+
+ if ((p = lws_cmdline_option(argc, argv, "-d")))
+ logs = atoi(p);
+
+ lws_set_log_level(logs, NULL);
+ lwsl_user("Sai Server - Copyright (C) 2019-2020 Andy Green <andy@warmcat.com>\n");
+
+ if ((p = lws_cmdline_option(argc, argv, "-c")))
+ conf = p;
+
+ context = sai_lws_context_from_json(conf, &info, pprotocols,
+ default_ss_policy);
+ if (!context) {
+ lwsl_err("lws init failed\n");
+ return 1;
+ }
+
+ while (!lws_service(context, 0) && !interrupted)
+ ;
+
+ lws_context_destroy(context);
+
+ return 0;
+}
diff --git a/src/server/s-task.c b/src/server/s-task.c
new file mode 100644
index 0000000..66e26d3
--- /dev/null
+++ b/src/server/s-task.c
@@ -0,0 +1,494 @@
+/*
+ * Sai server
+ *
+ * Copyright (C) 2019 - 2020 Andy Green <andy@warmcat.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation:
+ * version 2.1 of the License.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301 USA
+ */
+
+#include <libwebsockets.h>
+#include <string.h>
+#include <signal.h>
+#include <time.h>
+#include <assert.h>
+
+#include "s-private.h"
+
+
+static int
+sql3_get_integer_cb(void *user, int cols, char **values, char **name)
+{
+ unsigned int *pui = (unsigned int *)user;
+
+ // lwsl_warn("%s: values[0] '%s'\n", __func__, values[0]);
+ *pui = atoi(values[0]);
+
+ return 0;
+}
+
+void
+sai_task_uuid_to_event_uuid(char *event_uuid33, const char *task_uuid65)
+{
+ memcpy(event_uuid33, task_uuid65, 32);
+ event_uuid33[32] = '\0';
+}
+
+int
+sais_set_task_state(struct vhd *vhd, const char *builder_name,
+ const char *builder_uuid, const char *task_uuid, int state,
+ uint64_t started, uint64_t duration)
+{
+ char update[384], esc[96], esc1[96], esc2[96], esc3[32], esc4[32],
+ event_uuid[33];
+ unsigned int count = 0, count_good = 0, count_bad = 0;
+ sai_event_state_t oes, sta;
+ struct lwsac *ac = NULL;
+ sai_event_t *e = NULL;
+ lws_dll2_owner_t o;
+ int n, task_ostate;
+
+ /*
+ * Extract the event uuid from the task uuid
+ */
+
+ sai_task_uuid_to_event_uuid(event_uuid, task_uuid);
+
+ /*
+ * Look up the task's event in the event database...
+ */
+
+ lws_dll2_owner_clear(&o);
+ lws_sql_purify(esc1, event_uuid, sizeof(esc1));
+ lws_snprintf(esc2, sizeof(esc2), " and uuid='%s'", esc1);
+ n = lws_struct_sq3_deserialize(vhd->server.pdb, esc2, NULL,
+ lsm_schema_sq3_map_event, &o, &ac, 0, 1);
+ if (n < 0 || !o.head) {
+ lwsl_err("%s: failed to get task_uuid %s\n", __func__, esc1);
+ goto bail;
+ }
+
+ e = lws_container_of(o.head, sai_event_t, list);
+ oes = e->state;
+
+ /*
+ * Open the event-specific database on the temporary event object
+ */
+
+ if (sais_event_db_ensure_open(vhd, event_uuid, 0, (sqlite3 **)&e->pdb)) {
+ lwsl_err("%s: unable to open event-specific database\n",
+ __func__);
+
+ return -1;
+ }
+
+ if (builder_name)
+ lws_sql_purify(esc, builder_name, sizeof(esc));
+ else
+ esc[0] = '\0';
+
+ if (builder_uuid)
+ lws_sql_purify(esc1, builder_uuid, sizeof(esc1));
+ else
+ esc1[0] = '\0';
+ lws_sql_purify(esc2, task_uuid, sizeof(esc2));
+
+ esc3[0] = esc4[0] = '\0';
+
+ /*
+ * grab the current state of it for seeing if it changed
+ */
+ lws_snprintf(update, sizeof(update),
+ "select state from tasks where uuid='%s'", esc2);
+ if (sqlite3_exec((sqlite3 *)e->pdb, update,
+ sql3_get_integer_cb, &task_ostate, NULL) != SQLITE_OK) {
+ lwsl_err("%s: %s: %s: fail\n", __func__, update,
+ sqlite3_errmsg(vhd->server.pdb));
+ goto bail;
+ }
+
+ if (started) {
+ if (started == 1)
+ started = 0;
+ lws_snprintf(esc3, sizeof(esc3), ",started=%llu",
+ (unsigned long long)started);
+ }
+ if (duration) {
+ if (duration == 1)
+ duration = 0;
+ lws_snprintf(esc4, sizeof(esc4), ",duration=%llu",
+ (unsigned long long)duration);
+ }
+
+ /*
+ * Update the task by uuid, in the event-specific database
+ */
+
+ lws_snprintf(update, sizeof(update),
+ "update tasks set state=%d%s%s%s%s%s%s%s%s where uuid='%s'",
+ state, builder_uuid ? ",builder='": "",
+ builder_uuid ? esc1 : "",
+ builder_uuid ? "'" : "",
+ builder_name ? ",builder_name='" : "",
+ builder_name ? esc : "",
+ builder_name ? "'" : "",
+ esc3, esc4, esc2);
+
+ if (sqlite3_exec((sqlite3 *)e->pdb, update, NULL, NULL, NULL) != SQLITE_OK) {
+ lwsl_err("%s: %s: %s: fail\n", __func__, update,
+ sqlite3_errmsg(vhd->server.pdb));
+ goto bail;
+ }
+
+ /*
+ * We tell interested parties about logs separately. So there's only
+ * something to tell about change to task state if he literally changed
+ * the state
+ */
+
+ if (state != task_ostate) {
+
+ lwsl_notice("%s: seen task %s %d -> %d\n", __func__,
+ task_uuid, task_ostate, state);
+
+ sais_taskchange(vhd->h_ss_websrv, task_uuid, state);
+
+ /*
+ * So, how many tasks for this event?
+ */
+
+ if (sqlite3_exec((sqlite3 *)e->pdb, "select count(state) from tasks",
+ sql3_get_integer_cb, &count, NULL) != SQLITE_OK) {
+ lwsl_err("%s: %s: %s: fail\n", __func__, update,
+ sqlite3_errmsg(vhd->server.pdb));
+ goto bail;
+ }
+
+ /*
+ * ... how many completed well?
+ */
+
+ if (sqlite3_exec((sqlite3 *)e->pdb, "select count(state) from tasks where state == 3",
+ sql3_get_integer_cb, &count_good, NULL) != SQLITE_OK) {
+ lwsl_err("%s: %s: %s: fail\n", __func__, update,
+ sqlite3_errmsg(vhd->server.pdb));
+ goto bail;
+ }
+
+ /*
+ * ... how many failed?
+ */
+
+ if (sqlite3_exec((sqlite3 *)e->pdb, "select count(state) from tasks where state == 4",
+ sql3_get_integer_cb, &count_bad, NULL) != SQLITE_OK) {
+ lwsl_err("%s: %s: %s: fail\n", __func__, update,
+ sqlite3_errmsg(vhd->server.pdb));
+ goto bail;
+ }
+
+ /*
+ * Decide how to set the event state based on that
+ */
+
+ lwsl_notice("%s: ev %s, count %u, count_good %u, count_bad %u\n",
+ __func__, event_uuid, count, count_good, count_bad);
+
+ sta = SAIES_BEING_BUILT;
+
+ if (count) {
+ if (count == count_good)
+ sta = SAIES_SUCCESS;
+ else
+ if (count == count_bad)
+ sta = SAIES_FAIL;
+ else
+ if (count_bad)
+ sta = SAIES_BEING_BUILT_HAS_FAILURES;
+ }
+
+ if (sta != oes) {
+ lwsl_notice("%s: event state changed\n", __func__);
+
+ /*
+ * Update the event
+ */
+
+ lws_sql_purify(esc1, event_uuid, sizeof(esc1));
+ lws_snprintf(update, sizeof(update),
+ "update events set state=%d where uuid='%s'", sta, esc1);
+
+ if (sqlite3_exec(vhd->server.pdb, update, NULL, NULL, NULL) != SQLITE_OK) {
+ lwsl_err("%s: %s: %s: fail\n", __func__, update,
+ sqlite3_errmsg(vhd->server.pdb));
+ goto bail;
+ }
+
+ sais_eventchange(vhd->h_ss_websrv, event_uuid, sta);
+ }
+ }
+
+ sais_event_db_close(vhd, (sqlite3 **)&e->pdb);
+ lwsac_free(&ac);
+
+ return 0;
+
+bail:
+ if (e)
+ sais_event_db_close(vhd, (sqlite3 **)&e->pdb);
+ lwsac_free(&ac);
+
+ return 1;
+}
+
+/*
+ * Find the most recent task that still needs doing for platform, on any event
+ *
+ * If any, the task pointed-to lives inside *pac, along with its strings etc
+ * and is the responsibility of the caller to free
+ */
+
+static const sai_task_t *
+sais_task_pending(struct vhd *vhd, struct lwsac **pac, const char *platform)
+{
+ struct lwsac *ac = NULL;
+ char esc[96], pf[96];
+ lws_dll2_owner_t o;
+ int n;
+
+ lws_sql_purify(esc, platform, sizeof(esc));
+
+ assert(platform);
+ assert(pac);
+
+ /*
+ * Collect a list of events that still have any open tasks
+ *
+ * We don't put this list in the pac since we can dispose of it in this
+ * scope whether we find something or not
+ */
+
+ n = lws_struct_sq3_deserialize(vhd->server.pdb,
+ " and (state != 3 and state != 4 and state != 5)",
+ "created desc ", lsm_schema_sq3_map_event, &o,
+ &ac, 0, 10);
+ if (n < 0 || !o.head) {
+ // lwsl_notice("%s: all events complete\n", __func__);
+ /* error, or there are no events that aren't complete */
+ goto bail;
+ }
+
+ /*
+ * Iterate through the events looking at his event-specific database
+ * for tasks on the specified platform...
+ */
+
+ lws_start_foreach_dll(struct lws_dll2 *, p, o.head) {
+ sai_event_t *e = lws_container_of(p, sai_event_t, list);
+ lws_dll2_owner_t ot;
+ sqlite3 *pdb = NULL;
+
+ if (!sais_event_db_ensure_open(vhd, e->uuid, 0, &pdb)) {
+ lws_snprintf(pf, sizeof(pf),
+ " and state=0 and platform='%s'", esc);
+ n = lws_struct_sq3_deserialize(pdb, pf, NULL,
+ lsm_schema_sq3_map_task,
+ &ot, pac, 0, 1);
+ sais_event_db_close(vhd, &pdb);
+ if (ot.count) {
+
+ lwsl_notice("%s: found task for %s\n",
+ __func__, esc);
+
+ lwsac_free(&ac);
+
+ return lws_container_of(ot.head,
+ sai_task_t, list);
+ }
+ }
+
+ } lws_end_foreach_dll(p);
+
+ lwsl_debug("%s: no free tasks matching '%s'\n", __func__, esc);
+
+bail:
+ lwsac_free(&ac);
+
+ return NULL;
+}
+
+int
+sais_task_cancel(struct vhd *vhd, const char *task_uuid)
+{
+ sai_cancel_t *can;
+
+ /*
+ * For every pss that we have from builders...
+ */
+ lws_start_foreach_dll(struct lws_dll2 *, p, vhd->builders.head) {
+ struct pss *pss = lws_container_of(p, struct pss, same);
+
+
+ /*
+ * ... queue the task cancel message
+ */
+ can = malloc(sizeof *can);
+ if (!can)
+ return -1;
+ memset(can, 0, sizeof(*can));
+
+ strncpy(can->task_uuid, task_uuid, sizeof(can->task_uuid));
+
+ lws_dll2_add_tail(&can->list, &pss->task_cancel_owner);
+
+ lws_callback_on_writable(pss->wsi);
+
+ } lws_end_foreach_dll(p);
+
+ sais_taskchange(vhd->h_ss_websrv, task_uuid, SAIES_CANCELLED);
+
+ return 0;
+}
+
+/*
+ * Keep the task record itself, but remove all logs and artifacts related to
+ * it and reset the task state back to WAITING.
+ */
+
+int
+sais_task_reset(struct vhd *vhd, const char *task_uuid)
+{
+ char esc[96], cmd[256], event_uuid[33];
+ sqlite3 *pdb = NULL;
+
+ if (!task_uuid[0])
+ return 0;
+
+ lwsl_notice("%s: received request to reset task %s\n", __func__, task_uuid);
+
+ sai_task_uuid_to_event_uuid(event_uuid, task_uuid);
+
+ if (sais_event_db_ensure_open(vhd, event_uuid, 0, &pdb)) {
+ lwsl_err("%s: unable to open event-specific database\n",
+ __func__);
+
+ return -1;
+ }
+
+ lws_sql_purify(esc, task_uuid, sizeof(esc));
+ lws_snprintf(cmd, sizeof(cmd), "delete from logs where task_uuid='%s'",
+ esc);
+
+ if (sqlite3_exec(pdb, cmd, NULL, NULL, NULL) != SQLITE_OK) {
+ sais_event_db_close(vhd, &pdb);
+ lwsl_err("%s: %s: %s: fail\n", __func__, cmd,
+ sqlite3_errmsg(vhd->server.pdb));
+ return 1;
+ }
+ lws_snprintf(cmd, sizeof(cmd), "delete from artifacts where task_uuid='%s'",
+ esc);
+
+ sqlite3_exec(pdb, cmd, NULL, NULL, NULL);
+ sais_event_db_close(vhd, &pdb);
+
+ sais_set_task_state(vhd, NULL, NULL, task_uuid, SAIES_WAITING, 1, 1);
+
+ sais_task_cancel(vhd, task_uuid);
+
+ /*
+ * Reassess now if there's a builder we can match to a pending task
+ */
+
+ lws_sul_schedule(vhd->context, 0, &vhd->sul_central, sais_central_cb, 1);
+
+ return 0;
+}
+
+/*
+ * Look for any task on any event that needs building on platform_name, if found
+ * the caller must take responsibility to free pss->a.ac
+ */
+
+int
+sais_allocate_task(struct vhd *vhd, struct pss *pss, sai_plat_t *cb,
+ const char *platform_name)
+{
+ char esc1[96], esc2[96];
+ lws_dll2_owner_t o;
+ sai_task_t *task;
+ int n;
+
+ if (cb->ongoing >= cb->instances)
+ return 1;
+
+ /*
+ * Look for a task for this platform, on any event that needs building
+ */
+
+ task = (sai_task_t *)sais_task_pending(vhd, &pss->a.ac, platform_name);
+ if (!task)
+ return 1;
+
+ lwsl_notice("%s: %s: task found %s\n", __func__, platform_name, cb->name);
+
+ /* yes, we will offer it to him */
+
+ if (sais_set_task_state(vhd, cb->name, cb->name, task->uuid,
+ SAIES_PASSED_TO_BUILDER, lws_now_secs(), 0))
+ goto bail;
+
+ /* advance the task state first time we get logs */
+ pss->mark_started = 1;
+
+ /* let's get ahold of his event as well */
+
+ lws_sql_purify(esc1, task->event_uuid, sizeof(esc1));
+ lws_snprintf(esc2, sizeof(esc2), " and uuid='%s'", esc1);
+ n = lws_struct_sq3_deserialize(vhd->server.pdb, esc2, NULL,
+ lsm_schema_sq3_map_event, &o,
+ &pss->a.ac, 0, 1);
+ if (n < 0 || !o.head)
+ goto bail;
+
+ task->one_event = lws_container_of(o.head, sai_event_t, list);
+
+ task->server_name = pss->server_name;
+ task->repo_name = task->one_event->repo_name;
+ task->git_ref = task->one_event->ref;
+ task->git_hash = task->one_event->hash;
+ task->ac_task_container = pss->a.ac;
+
+ /*
+ * add to server's estimate of builder's ongoing tasks...
+ */
+ cb->ongoing++;
+
+ lws_dll2_add_tail(&task->pending_assign_list, &pss->issue_task_owner);
+ lws_callback_on_writable(pss->wsi);
+
+ pss->a.ac = NULL;
+
+ /*
+ * We are going to leave here with a live pss->a.ac (pointed into by
+ * task->one_event) that the caller has to take responsibility to
+ * clean up pss->a.ac
+ */
+
+ return 0;
+
+bail:
+ lwsac_free(&pss->a.ac);
+
+ return -1;
+}
diff --git a/src/server/s-websrv.c b/src/server/s-websrv.c
new file mode 100644
index 0000000..1425648
--- /dev/null
+++ b/src/server/s-websrv.c
@@ -0,0 +1,514 @@
+/*
+ * Sai server
+ *
+ * Copyright (C) 2019 - 2020 Andy Green <andy@warmcat.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation:
+ * version 2.1 of the License.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301 USA
+ *
+ *
+ * This is a ws server over a unix domain socket made available by sai-server
+ * and connected to by sai-web instances running on the same box.
+ *
+ * The server notifies the sai-web instances of event and task changes (just
+ * that a particular event or task changed) and builder list updates (the
+ * whole current builder list JSON each time).
+ *
+ * Sai-web instances can send requests to restart or delete tasks and whole
+ * events made by authenticated clients.
+ *
+ * Since this is on a local UDS protected by user:group, there's no tls or auth
+ * on this link itself.
+ */
+
+#include <libwebsockets.h>
+#include <string.h>
+#include <signal.h>
+#include <time.h>
+
+#include "s-private.h"
+
+typedef struct websrvss_srv {
+ struct lws_ss_handle *ss;
+ struct vhd *vhd;
+ /* ... application specific state ... */
+
+ struct lejp_ctx ctx;
+ struct lws_buflist *bltx;
+
+} websrvss_srv_t;
+
+static lws_struct_map_t lsm_browser_taskreset[] = {
+ LSM_CARRAY (sai_browse_rx_evinfo_t, event_hash, "uuid"),
+};
+
+static const lws_struct_map_t lsm_schema_json_map[] = {
+ LSM_SCHEMA (sai_browse_rx_evinfo_t, NULL, lsm_browser_taskreset,
+ /* shares struct */ "com.warmcat.sai.taskreset"),
+ LSM_SCHEMA (sai_browse_rx_evinfo_t, NULL, lsm_browser_taskreset,
+ /* shares struct */ "com.warmcat.sai.eventreset"),
+ LSM_SCHEMA (sai_browse_rx_evinfo_t, NULL, lsm_browser_taskreset,
+ /* shares struct */ "com.warmcat.sai.eventdelete"),
+ LSM_SCHEMA (sai_cancel_t, NULL, lsm_task_cancel,
+ "com.warmcat.sai.taskcan"),
+};
+
+enum {
+ SAIS_WS_WEBSRV_RX_TASKRESET,
+ SAIS_WS_WEBSRV_RX_EVENTRESET,
+ SAIS_WS_WEBSRV_RX_EVENTDELETE,
+ SAIS_WS_WEBSRV_RX_TASKCANCEL
+};
+
+int
+sais_validate_id(const char *id, int reqlen)
+{
+ const char *idin = id;
+ int n = reqlen;
+
+ while (*id && n--) {
+ if (!((*id >= '0' && *id <= '9') ||
+ (*id >= 'a' && *id <= 'z') ||
+ (*id >= 'A' && *id <= 'Z')))
+ goto reject;
+ id++;
+ }
+
+ if (!n && !*id)
+ return 0;
+reject:
+
+ lwsl_notice("%s: Invalid ID (%d) '%s'\n", __func__, reqlen, idin);
+
+ return 1;
+}
+
+int
+sais_websrv_queue_tx(struct lws_ss_handle *h, void *buf, size_t len)
+{
+ websrvss_srv_t *m = (websrvss_srv_t *)lws_ss_to_user_object(h);
+ int n;
+
+ n = lws_buflist_append_segment(&m->bltx, buf, len);
+
+ lwsl_notice("%s: appened h %p: %d\n", __func__, h, n);
+ lws_ss_request_tx(h);
+
+ return n < 0;
+}
+
+typedef struct {
+ const uint8_t *buf;
+ size_t len;
+} sais_websrv_broadcast_t;
+
+static void
+_sais_websrv_broadcast(struct lws_ss_handle *h, void *arg)
+{
+ websrvss_srv_t *m = (websrvss_srv_t *)lws_ss_to_user_object(h);
+ sais_websrv_broadcast_t *a = (sais_websrv_broadcast_t *)arg;
+
+ if (lws_buflist_append_segment(&m->bltx, a->buf, a->len) >= 0)
+ lws_ss_request_tx(h);
+ else
+ lwsl_warn("%s: buflist append fail\n", __func__);
+}
+
+void
+sais_websrv_broadcast(struct lws_ss_handle *hsrv, const char *str, size_t len)
+{
+ sais_websrv_broadcast_t a;
+
+ a.buf = (const uint8_t *)str;
+ a.len = len;
+
+ lws_ss_server_foreach_client(hsrv, _sais_websrv_broadcast, &a);
+}
+
+
+int
+sais_list_builders(struct vhd *vhd)
+{
+ lws_dll2_t *walk = lws_dll2_get_head(&vhd->server.builder_owner);
+ char *p = vhd->json_builders, *end = p + sizeof(vhd->json_builders),
+ subsequent = 0;
+ lws_struct_serialize_t *js;
+ sai_plat_t *b;
+ size_t w;
+ int n;
+
+ p += lws_snprintf((char *)p, end - p, "{\"schema\":\"sai-builders\","
+ "\"platforms\":[");
+
+ while (end - p > 512 && walk) {
+
+ b = lws_container_of(walk, sai_plat_t, sai_plat_list);
+
+ js = lws_struct_json_serialize_create(
+ lsm_schema_map_plat_simple,
+ LWS_ARRAY_SIZE(lsm_schema_map_plat_simple),
+ 0, b);
+ if (!js)
+ return 1;
+ if (subsequent)
+ *p++ = ',';
+ subsequent = 1;
+
+ n = lws_struct_json_serialize(js, (unsigned char *)p,
+ lws_ptr_diff(end, p), &w);
+ p += w;
+ lws_struct_json_serialize_destroy(&js);
+
+ if (n == LSJS_RESULT_ERROR)
+ return 1;
+
+ walk = walk->next;
+ if (!walk) {
+ p += lws_snprintf((char *)p, end - p, "]}");
+
+ sais_websrv_broadcast(vhd->h_ss_websrv,
+ vhd->json_builders,
+ lws_ptr_diff(p, vhd->json_builders));
+
+ return 0;
+ }
+ }
+
+ return 1;
+}
+
+struct sais_arg {
+ const char *uid;
+ int state;
+};
+
+static void
+_sais_taskchange(struct lws_ss_handle *h, void *_arg)
+{
+ websrvss_srv_t *m = (websrvss_srv_t *)lws_ss_to_user_object(h);
+ struct sais_arg *arg = (struct sais_arg *)_arg;
+ char tc[128];
+ int n;
+
+ n = lws_snprintf(tc, sizeof(tc), "{\"schema\":\"sai-taskchange\", "
+ "\"event_hash\":\"%s\", \"state\":%d}",
+ arg->uid, arg->state);
+
+ if (lws_buflist_append_segment(&m->bltx, (uint8_t *)tc, n) >= 0)
+ lws_ss_request_tx(h);
+ else
+ lwsl_warn("%s: buflist append failed\n", __func__);
+}
+
+void
+sais_taskchange(struct lws_ss_handle *hsrv, const char *task_uuid, int state)
+{
+ struct sais_arg arg = { task_uuid, state };
+
+ lws_ss_server_foreach_client(hsrv, _sais_taskchange, (void *)&arg);
+}
+
+void
+_sais_eventchange(struct lws_ss_handle *h, void *_arg)
+{
+ websrvss_srv_t *m = (websrvss_srv_t *)lws_ss_to_user_object(h);
+ struct sais_arg *arg = (struct sais_arg *)_arg;
+ char tc[128];
+ int n;
+
+ n = lws_snprintf(tc, sizeof(tc), "{\"schema\":\"sai-eventchange\", "
+ "\"event_hash\":\"%s\", \"state\":%d}",
+ arg->uid, arg->state);
+
+ if (lws_buflist_append_segment(&m->bltx, (uint8_t *)tc, n) >= 0)
+ lws_ss_request_tx(h);
+ else
+ lwsl_warn("%s: buflist append failed\n", __func__);
+}
+
+void
+sais_eventchange(struct lws_ss_handle *hsrv, const char *event_uuid, int state)
+{
+ struct sais_arg arg = { event_uuid, state };
+
+ lws_ss_server_foreach_client(hsrv, _sais_eventchange, (void *)&arg);
+}
+
+
+static lws_ss_state_return_t
+websrvss_ws_rx(void *userobj, const uint8_t *buf, size_t len, int flags)
+{
+ websrvss_srv_t *m = (websrvss_srv_t *)userobj;
+ char qu[128], esc[96], *err = NULL;
+ sai_browse_rx_evinfo_t *ei;
+ sqlite3 *pdb = NULL;
+ lws_struct_args_t a;
+ lws_dll2_owner_t o;
+ int n;
+
+ lwsl_user("%s: len %d, flags: %d\n", __func__, (int)len, flags);
+ lwsl_hexdump_info(buf, len);
+
+ memset(&a, 0, sizeof(a));
+ a.map_st[0] = lsm_schema_json_map;
+ a.map_entries_st[0] = LWS_ARRAY_SIZE(lsm_schema_json_map);
+ a.map_entries_st[1] = LWS_ARRAY_SIZE(lsm_schema_json_map);
+ a.ac_block_size = 128;
+
+ lws_struct_json_init_parse(&m->ctx, NULL, &a);
+ n = lejp_parse(&m->ctx, (uint8_t *)buf, len);
+ if (n < 0 || !a.dest) {
+ lwsl_hexdump_notice(buf, len);
+ lwsl_notice("%s: notification JSON decode failed '%s'\n",
+ __func__, lejp_error_to_string(n));
+ return LWSSSSRET_DISCONNECT_ME;
+ }
+
+ lwsl_notice("%s: schema idx %d\n", __func__, a.top_schema_index);
+
+ switch (a.top_schema_index) {
+ case SAIS_WS_WEBSRV_RX_TASKRESET:
+
+ ei = (sai_browse_rx_evinfo_t *)a.dest;
+ if (sais_validate_id(ei->event_hash, SAI_TASKID_LEN))
+ goto soft_error;
+
+ sais_task_reset(m->vhd, ei->event_hash);
+ break;
+
+ case SAIS_WS_WEBSRV_RX_EVENTRESET:
+
+ ei = (sai_browse_rx_evinfo_t *)a.dest;
+ if (sais_validate_id(ei->event_hash, SAI_EVENTID_LEN))
+ goto soft_error;
+
+ /* open the event-specific database object */
+
+ if (sais_event_db_ensure_open(m->vhd, ei->event_hash, 0, &pdb)) {
+ lwsl_err("%s: unable to open event-specific database\n",
+ __func__);
+ /*
+ * hanging up isn't a good way to deal with browser
+ * tabs left open with a live connection to a
+ * now-deleted task... the page will reconnect endlessly
+ */
+ goto soft_error;
+ }
+
+ /*
+ * Retreive all the related structs into a dll2 list
+ */
+
+ lws_sql_purify(esc, ei->event_hash, sizeof(esc));
+
+ if (lws_struct_sq3_deserialize(pdb, NULL, NULL,
+ lsm_schema_sq3_map_task,
+ &o, &a.ac, 0, 999) >= 0) {
+
+ sqlite3_exec(pdb, "BEGIN TRANSACTION",
+ NULL, NULL, &err);
+
+ /*
+ * Walk the results list resetting all the tasks
+ */
+
+ lws_start_foreach_dll(struct lws_dll2 *, p, o.head) {
+ sai_task_t *t = lws_container_of(p, sai_task_t,
+ list);
+
+ sais_task_reset(m->vhd, t->uuid);
+
+ } lws_end_foreach_dll(p);
+
+ sqlite3_exec(pdb, "END TRANSACTION",
+ NULL, NULL, &err);
+ }
+
+ sais_event_db_close(m->vhd, &pdb);
+ lwsac_free(&a.ac);
+
+ return 0;
+
+ case SAIS_WS_WEBSRV_RX_EVENTDELETE:
+
+ ei = (sai_browse_rx_evinfo_t *)a.dest;
+ if (sais_validate_id(ei->event_hash, SAI_EVENTID_LEN))
+ goto soft_error;
+
+ lwsl_notice("%s: eventdelete %s\n", __func__, ei->event_hash);
+
+ /* open the event-specific database object */
+
+ if (sais_event_db_ensure_open(m->vhd, ei->event_hash, 0, &pdb)) {
+ lwsl_notice("%s: unable to open event-specific database\n",
+ __func__);
+ /*
+ * hanging up isn't a good way to deal with browser
+ * tabs left open with a live connection to a
+ * now-deleted task... the page will reconnect endlessly
+ */
+
+ // goto soft_error;
+ } else {
+
+ /*
+ * Retreive all the related structs into a dll2 list
+ */
+
+ lws_sql_purify(esc, ei->event_hash, sizeof(esc));
+
+ if (lws_struct_sq3_deserialize(pdb, NULL, NULL,
+ lsm_schema_sq3_map_task,
+ &o, &a.ac, 0, 999) >= 0) {
+
+ sqlite3_exec(pdb, "BEGIN TRANSACTION",
+ NULL, NULL, &err);
+
+ /*
+ * Walk the results list cancelling all the tasks
+ * that look like they might be ongoing
+ */
+
+ lws_start_foreach_dll(struct lws_dll2 *, p, o.head) {
+ sai_task_t *t = lws_container_of(p, sai_task_t,
+ list);
+
+ if (t->state != SAIES_WAITING &&
+ t->state != SAIES_SUCCESS &&
+ t->state != SAIES_FAIL &&
+ t->state != SAIES_CANCELLED)
+ sais_task_cancel(m->vhd, t->uuid);
+
+ } lws_end_foreach_dll(p);
+
+ sqlite3_exec(pdb, "END TRANSACTION",
+ NULL, NULL, &err);
+ }
+
+ sais_event_db_close(m->vhd, &pdb);
+ }
+
+ /* delete the event iself */
+
+ lws_sql_purify(esc, ei->event_hash, sizeof(esc));
+ lws_snprintf(qu, sizeof(qu), "delete from events where uuid='%s'",
+ esc);
+ sqlite3_exec(m->vhd->server.pdb, qu, NULL, NULL, &err);
+ if (err) {
+ lwsl_err("%s: evdel uuid %s, sq3 err %s\n", __func__,
+ esc, err);
+ sqlite3_free(err);
+ }
+
+ /* remove the event-specific database */
+
+ sais_event_db_delete_database(m->vhd, ei->event_hash);
+
+ lwsac_free(&a.ac);
+ sais_eventchange(m->vhd->h_ss_websrv, ei->event_hash,
+ SAIES_DELETED);
+ sais_websrv_broadcast(m->vhd->h_ss_websrv,
+ "{\"schema\":\"sai-overview\"}", 25);
+
+ return 0;
+
+ case SAIS_WS_WEBSRV_RX_TASKCANCEL:
+
+
+ ei = (sai_browse_rx_evinfo_t *)a.dest;
+ if (sais_validate_id(ei->event_hash, SAI_TASKID_LEN))
+ goto soft_error;
+
+ sais_task_cancel(m->vhd, ei->event_hash);
+
+ break;
+ }
+
+ return 0;
+
+soft_error:
+ lwsl_warn("%s: soft error\n", __func__);
+
+ return 0;
+}
+
+static lws_ss_state_return_t
+websrvss_ws_tx(void *userobj, lws_ss_tx_ordinal_t ord, uint8_t *buf,
+ size_t *len, int *flags)
+{
+ websrvss_srv_t *m = (websrvss_srv_t *)userobj;
+ char som, eom;
+ int used;
+
+ if (!m->bltx)
+ return LWSSSSRET_TX_DONT_SEND;
+
+ used = lws_buflist_fragment_use(&m->bltx, buf, *len, &som, &eom);
+ if (!used)
+ return LWSSSSRET_TX_DONT_SEND;
+
+ *flags = (som ? LWSSS_FLAG_SOM : 0) | (eom ? LWSSS_FLAG_EOM : 0);
+ *len = (size_t)used;
+
+ if (m->bltx)
+ lws_ss_request_tx(m->ss);
+
+ return 0;
+}
+
+static lws_ss_state_return_t
+websrvss_srv_state(void *userobj, void *sh, lws_ss_constate_t state,
+ lws_ss_tx_ordinal_t ack)
+{
+ websrvss_srv_t *m = (websrvss_srv_t *)userobj;
+
+ lwsl_user("%s: %p %s, ord 0x%x\n", __func__, m->ss,
+ lws_ss_state_name(state), (unsigned int)ack);
+
+ switch (state) {
+ case LWSSSCS_DISCONNECTED:
+ lws_buflist_destroy_all_segments(&m->bltx);
+ break;
+ case LWSSSCS_CREATING:
+ lws_ss_request_tx(m->ss);
+ break;
+ case LWSSSCS_CONNECTED:
+ sais_list_builders(m->vhd);
+ break;
+ case LWSSSCS_ALL_RETRIES_FAILED:
+ break;
+
+ case LWSSSCS_SERVER_TXN:
+ break;
+
+ case LWSSSCS_SERVER_UPGRADE:
+ break;
+
+ default:
+ break;
+ }
+
+ return 0;
+}
+
+const lws_ss_info_t ssi_server = {
+ .handle_offset = offsetof(websrvss_srv_t, ss),
+ .opaque_user_data_offset = offsetof(websrvss_srv_t, vhd),
+ .streamtype = "websrv",
+ .rx = websrvss_ws_rx,
+ .tx = websrvss_ws_tx,
+ .state = websrvss_srv_state,
+ .user_alloc = sizeof(websrvss_srv_t),
+};
diff --git a/src/server/s-ws-builder.c b/src/server/s-ws-builder.c
new file mode 100644
index 0000000..44936db
--- /dev/null
+++ b/src/server/s-ws-builder.c
@@ -0,0 +1,750 @@
+/*
+ * Sai server - ./src/server/ws-json-rx.c
+ *
+ * Copyright (C) 2019 - 2020 Andy Green <andy@warmcat.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation:
+ * version 2.1 of the License.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301 USA
+ *
+ * These are ws rx and tx handlers related to builder ws connections, on
+ * /builder
+ */
+
+#include <libwebsockets.h>
+#include <string.h>
+#include <signal.h>
+#include <time.h>
+
+#include "s-private.h"
+
+enum sai_overview_state {
+ SOS_EVENT,
+ SOS_TASKS,
+};
+
+typedef struct sais_logcache_pertask {
+ lws_dll2_t list; /* vhd->tasklog_cache is the owner */
+ char uuid[65];
+ lws_dll2_owner_t cache; /* sai_log_t */
+} sais_logcache_pertask_t;
+
+/*
+ * The Schema that may be sent to us by a builder
+ *
+ * Artifacts are sent on secondary SS connections so they don't block ongoing
+ * log delivery etc. The JSON is immediately followed by binary data to the
+ * length told in the JSON.
+ */
+
+static const lws_struct_map_t lsm_schema_map_ba[] = {
+ LSM_SCHEMA_DLL2 (sai_plat_owner_t, plat_owner, NULL, lsm_plat_list,
+ "com-warmcat-sai-ba"),
+ LSM_SCHEMA (sai_log_t, NULL, lsm_log,
+ "com-warmcat-sai-logs"),
+ LSM_SCHEMA (sai_event_t, NULL, lsm_task_rej,
+ "com.warmcat.sai.taskrej"),
+ LSM_SCHEMA (sai_artifact_t, NULL, lsm_artifact,
+ "com-warmcat-sai-artifact"),
+};
+
+enum {
+ SAIM_WSSCH_BUILDER_PLATS,
+ SAIM_WSSCH_BUILDER_LOGS,
+ SAIM_WSSCH_BUILDER_TASKREJ,
+ SAIM_WSSCH_BUILDER_ARTIFACT
+};
+
+static void
+sais_dump_logs_to_db(lws_sorted_usec_list_t *sul)
+{
+ struct vhd *vhd = lws_container_of(sul, struct vhd, sul_logcache);
+ sais_logcache_pertask_t *lcpt;
+ char event_uuid[33], sw[192];
+ sqlite3 *pdb = NULL;
+ sai_log_t *hlog;
+ char *err;
+ int n;
+
+ /*
+ * for each task that acquired logs in the interval
+ */
+
+ lws_start_foreach_dll_safe(struct lws_dll2 *, p, p1,
+ vhd->tasklog_cache.head) {
+ lcpt = lws_container_of(p, sais_logcache_pertask_t, list);
+
+ sai_task_uuid_to_event_uuid(event_uuid, lcpt->uuid);
+
+ if (!sais_event_db_ensure_open(vhd, event_uuid, 0, &pdb)) {
+
+ /*
+ * Empty the task-specific log cache into the event-
+ * specific db for the task in one go, this is much
+ * more efficient
+ */
+
+ sqlite3_exec(pdb, "BEGIN TRANSACTION", NULL, NULL, &err);
+ if (err)
+ sqlite3_free(err);
+
+ lws_struct_sq3_serialize(pdb, lsm_schema_sq3_map_log,
+ &lcpt->cache, 0);
+
+ sqlite3_exec(pdb, "END TRANSACTION", NULL, NULL, &err);
+ if (err)
+ sqlite3_free(err);
+ sais_event_db_close(vhd, &pdb);
+
+ } else
+ lwsl_err("%s: unable to open event-specific database\n",
+ __func__);
+
+ /*
+ * Destroy the logs in the task cache and the task cache
+ */
+
+ lws_start_foreach_dll_safe(struct lws_dll2 *, pq, pq1,
+ lcpt->cache.head) {
+ hlog = lws_container_of(pq, sai_log_t, list);
+ lws_dll2_remove(&hlog->list);
+ free(hlog);
+ } lws_end_foreach_dll_safe(pq, pq1);
+
+ /*
+ * Inform anybody who's looking at this task's logs that
+ * something changed (event_hash is actually the task hash)
+ */
+
+ n = lws_snprintf(sw, sizeof(sw), "{\"schema\":\"sai-tasklogs\","
+ "\"event_hash\":\"%s\"}", lcpt->uuid);
+ sais_websrv_broadcast(vhd->h_ss_websrv, sw, n);
+
+ /*
+ * Destroy the whole task-specific cache, it will regenerate
+ * if more logs come for it
+ */
+
+ lws_dll2_remove(&lcpt->list);
+ free(lcpt);
+
+ } lws_end_foreach_dll_safe(p, p1);
+
+}
+
+/*
+ * We're going to stash these logs on a per-task list, and deal with them
+ * inside a single trasaction per task efficiently on a timer.
+ */
+
+static void
+sais_log_to_db(struct vhd *vhd, sai_log_t *log)
+{
+ sais_logcache_pertask_t *lcpt = NULL;
+ sai_log_t *hlog;
+
+ /*
+ * find the pertask if one exists
+ */
+
+ lws_start_foreach_dll(struct lws_dll2 *, p, vhd->tasklog_cache.head) {
+ lcpt = lws_container_of(p, sais_logcache_pertask_t, list);
+
+ if (!strcmp(lcpt->uuid, log->task_uuid))
+ break;
+ lcpt = NULL;
+
+ } lws_end_foreach_dll(p);
+
+ if (!lcpt) {
+ /*
+ * Create a pertask and add it to the vhd list of them
+ */
+ lcpt = malloc(sizeof(*lcpt));
+ memset(lcpt, 0, sizeof(*lcpt));
+ lws_strncpy(lcpt->uuid, log->task_uuid, sizeof(lcpt->uuid));
+ lws_dll2_add_tail(&lcpt->list, &vhd->tasklog_cache);
+ }
+
+ hlog = malloc(sizeof(*hlog) + log->len + strlen(log->log) + 1);
+ if (!hlog)
+ return;
+
+ *hlog = *log;
+ memset(&hlog->list, 0, sizeof(hlog->list));
+ memcpy(&hlog[1], log->log, strlen(log->log) + 1);
+ hlog->log = (char *)&hlog[1];
+
+ /*
+ * add our log copy to the task-specific cache
+ */
+
+ lws_dll2_add_tail(&hlog->list, &lcpt->cache);
+
+ if (!vhd->sul_logcache.list.owner)
+ /* if not already scheduled, schedule it for 250ms */
+
+ lws_sul_schedule(vhd->context, 0, &vhd->sul_logcache,
+ sais_dump_logs_to_db, 250 * LWS_US_PER_MS);
+}
+
+static sai_plat_t *
+sais_builder_from_uuid(struct vhd *vhd, const char *hostname)
+{
+ lws_start_foreach_dll(struct lws_dll2 *, p,
+ vhd->server.builder_owner.head) {
+ sai_plat_t *cb = lws_container_of(p, sai_plat_t,
+ sai_plat_list);
+
+ if (!strcmp(hostname, cb->name))
+ return cb;
+
+ } lws_end_foreach_dll(p);
+
+ return NULL;
+}
+
+int
+sai_sql3_get_uint64_cb(void *user, int cols, char **values, char **name)
+{
+ uint64_t *pui = (uint64_t *)user;
+
+ *pui = (uint64_t)atoll(values[0]);
+
+ return 0;
+}
+
+/*
+ * Master received a communication from a builder
+ */
+
+int
+sais_ws_json_rx_builder(struct vhd *vhd, struct pss *pss, uint8_t *buf, size_t bl)
+{
+ char event_uuid[33], s[128], esc[96];
+ struct lwsac *ac = NULL;
+ sai_plat_t *build, *cb;
+ sai_rejection_t *rej;
+ lws_dll2_owner_t o;
+ sai_artifact_t *ap;
+ sai_task_t *task;
+ sai_log_t *log;
+ uint64_t rid;
+ int n, m;
+
+ /*
+ * use the schema name on the incoming JSON to decide what kind of
+ * structure to instantiate
+ *
+ * We may have:
+ *
+ * - just received a fragment of the whole JSON
+ *
+ * - received the JSON and be handling appeneded blob data
+ */
+
+ if (pss->bulk_binary_data)
+ goto handle;
+
+ if (!pss->frag) {
+ memset(&pss->a, 0, sizeof(pss->a));
+ pss->a.map_st[0] = lsm_schema_map_ba;
+ pss->a.map_entries_st[0] = LWS_ARRAY_SIZE(lsm_schema_map_ba);
+ pss->a.ac_block_size = 4096;
+
+ lws_struct_json_init_parse(&pss->ctx, NULL, &pss->a);
+ } else
+ pss->frag = 0;
+
+ m = lejp_parse(&pss->ctx, (uint8_t *)buf, bl);
+
+ /*
+ * returns negative, or unused amount... for us, we either had a
+ * (negative) error, had LEJP_CONTINUE, or if 0/positive, finished
+ */
+ if (m < 0 && m != LEJP_CONTINUE) {
+ /* an explicit error */
+ lwsl_hexdump_err(buf, bl);
+ lwsl_err("%s: rx JSON decode failed '%s', %d, %s, %s, %d\n",
+ __func__, lejp_error_to_string(m), m,
+ pss->ctx.path, pss->ctx.buf, pss->ctx.npos);
+ lwsac_free(&pss->a.ac);
+ return 1;
+ }
+
+ if (m == LEJP_CONTINUE) {
+ pss->frag = 1;
+ return 0;
+ }
+
+ if (!pss->a.dest) {
+ lwsac_free(&pss->a.ac);
+ lwsl_err("%s: json decode didn't make an object\n", __func__);
+ return 1;
+ }
+
+handle:
+ switch (pss->a.top_schema_index) {
+ case SAIM_WSSCH_BUILDER_PLATS:
+
+ // lwsl_hexdump_notice(buf, bl);
+
+ /*
+ * builder is sending us an array of platforms it provides us
+ */
+
+ pss->u.o = (sai_plat_owner_t *)pss->a.dest;
+
+ lwsl_notice("%s: seen platform list: count %d\n", __func__,
+ pss->u.o->plat_owner.count);
+
+ lws_start_foreach_dll(struct lws_dll2 *, pb,
+ pss->u.o->plat_owner.head) {
+ build = lws_container_of(pb, sai_plat_t, sai_plat_list);
+
+ lwsl_notice("%s: seeing plat %s\n", __func__, build->name);
+
+ /*
+ * ... so is this one a new guy?
+ */
+
+ cb = sais_builder_from_uuid(vhd, build->name);
+ if (!cb) {
+ char *cp;
+
+ /*
+ * We need to make a persistent, deep, copy of
+ * the (from JSON) builder object representing
+ * this client.
+ *
+ * "platform" is eg "linux-ubuntu-bionic-arm64"
+ * and "name" is "hostname.<platform>".
+ */
+
+ if (!build->name || !build->platform) {
+ lwsl_err("%s: missing build '%s'/hostname '%s'\n",
+ __func__,
+ build->name ? build->name : "null",
+ build->platform ? build->platform : "null");
+ return -1;
+ }
+
+ cb = malloc(sizeof(*cb) +
+ strlen(build->name) + 1 +
+ strlen(build->platform) + 1);
+
+ memset(cb, 0, sizeof(*cb));
+ cp = (char *)&cb[1];
+
+ memcpy(cp, build->name, strlen(build->name) + 1);
+ cb->name = cp;
+ cp += strlen(build->name) + 1;
+
+ memcpy(cp, build->platform, strlen(build->platform) + 1);
+ cb->platform = cp;
+ cp += strlen(build->platform) + 1;
+
+ cb->ongoing = build->ongoing;
+ cb->instances = build->instances;
+
+ cb->wsi = pss->wsi;
+
+ /* Then attach the copy to the server in the vhd
+ */
+ lws_dll2_add_tail(&cb->sai_plat_list,
+ &vhd->server.builder_owner);
+ }
+
+ /*
+ * Even if he's not new, we should use his updated info about
+ * builder load
+ */
+ cb->ongoing = build->ongoing;
+ cb->instances = build->instances;
+
+ lwsl_notice("%s: builder %s reports load %d/%d\n",
+ __func__, cb->name, cb->ongoing,
+ cb->instances);
+
+ } lws_end_foreach_dll(pb);
+
+ lwsac_free(&pss->a.ac);
+
+
+ /*
+ * look if we should offer the builder a task, given the
+ * platforms he's offering
+ */
+
+ if (sais_allocate_task(vhd, pss, cb, cb->platform) < 0)
+ goto bail;
+
+ /*
+ * If we did allocate a task in pss->a.ac, responsibility of
+ * callback_on_writable handler to empty it
+ */
+
+ break;
+
+bail:
+ lwsac_free(&pss->a.ac);
+ return -1;
+
+ case SAIM_WSSCH_BUILDER_LOGS:
+ /*
+ * builder is sending us info about task logs
+ */
+
+ log = (sai_log_t *)pss->a.dest;
+ sais_log_to_db(vhd, log);
+
+ if (pss->mark_started) {
+ pss->mark_started = 0;
+ pss->first_log_timestamp = log->timestamp;
+ if (sais_set_task_state(vhd, NULL, NULL, log->task_uuid,
+ SAIES_BEING_BUILT, 0, 0))
+ goto bail;
+ }
+
+ if (log->finished) {
+ /*
+ * We have reached the end of the logs for this task
+ */
+ lwsl_info("%s: log->finished says 0x%x, dur %lluus\n",
+ __func__, log->finished, (unsigned long long)(
+ log->timestamp - pss->first_log_timestamp));
+ if (log->finished & SAISPRF_EXIT) {
+ if ((log->finished & 0xff) == 0)
+ n = SAIES_SUCCESS;
+ else
+ n = SAIES_FAIL;
+ } else
+ if (log->finished & 8192)
+ n = SAIES_CANCELLED;
+ else
+ n = SAIES_FAIL;
+
+ if (sais_set_task_state(vhd, NULL, NULL, log->task_uuid,
+ n, 0, log->timestamp -
+ pss->first_log_timestamp))
+ goto bail;
+ }
+
+ lwsac_free(&pss->a.ac);
+
+ break;
+
+ case SAIM_WSSCH_BUILDER_TASKREJ:
+
+ /*
+ * builder is updating us about his status, and may be
+ * rejecting a task we tried to give him
+ */
+
+ rej = (sai_rejection_t *)pss->a.dest;
+
+ cb = sais_builder_from_uuid(vhd, rej->host_platform);
+ if (!cb) {
+ lwsl_info("%s: unknown builder %s rejecting\n",
+ __func__, rej->host_platform);
+ lwsac_free(&pss->a.ac);
+ break;
+ }
+
+ /* update our info about builder state with reality */
+
+ cb->ongoing = rej->ongoing;
+ cb->instances = rej->limit;
+
+ lwsl_notice("%s: builder %s reports load %d/%d (rej %s)\n",
+ __func__, cb->name, cb->ongoing, cb->instances,
+ rej->task_uuid[0] ? rej->task_uuid : "none");
+
+ if (rej->task_uuid[0])
+ sais_task_reset(vhd, rej->task_uuid);
+
+ lwsac_free(&pss->a.ac);
+ break;
+
+ case SAIM_WSSCH_BUILDER_ARTIFACT:
+ /*
+ * We get sent a JSON object immediately followed by binary
+ * data for the artifact.
+ *
+ * We place the binary data as a blob in the sql record in the
+ * artifact table.
+ */
+
+ lwsl_debug("%s: SAIM_WSSCH_BUILDER_ARTIFACT\n", __func__);
+
+ if (!pss->bulk_binary_data) {
+
+ ap = (sai_artifact_t *)pss->a.dest;
+
+ sai_task_uuid_to_event_uuid(event_uuid, ap->task_uuid);
+
+ /*
+ * Open the event-specific database object... the
+ * handle is closed when the stream closes for whatever
+ * reason.
+ */
+
+ if (sais_event_db_ensure_open(pss->vhd, event_uuid, 0,
+ &pss->pdb_artifact)) {
+ lwsl_err("%s: unable to open event-specific "
+ "database\n", __func__);
+
+ lwsac_free(&pss->a.ac);
+ return -1;
+ }
+
+ /*
+ * Retreive the task object
+ */
+
+ lws_sql_purify(esc, ap->task_uuid, sizeof(esc));
+ lws_snprintf(s, sizeof(s)," and uuid == \"%s\"", esc);
+ n = lws_struct_sq3_deserialize(pss->pdb_artifact, s,
+ NULL, lsm_schema_sq3_map_task,
+ &o, &ac, 0, 1);
+ if (n < 0 || !o.head) {
+ sais_event_db_close(vhd, &pss->pdb_artifact);
+ lwsl_notice("%s: no task of that id\n", __func__);
+ lwsac_free(&pss->a.ac);
+ return -1;
+ }
+
+ task = (sai_task_t *)o.head;
+ n = strcmp(task->art_up_nonce, ap->artifact_up_nonce);
+
+ if (n) {
+ lwsl_err("%s: artifact nonce mismatch\n",
+ __func__);
+ goto afail;
+ }
+
+ /*
+ * The task the sender is sending us an artifact for
+ * exists. The sender knows the random upload nonce
+ * for that task's artifacts.
+ *
+ * Create a random download nonce unrelated to the
+ * random upload nonce (so knowing the download one
+ * won't let you upload anything).
+ *
+ * Create the artifact's entry in the event-specific
+ * database
+ */
+
+ sai_uuid16_create(pss->vhd->context,
+ ap->artifact_down_nonce);
+
+ lws_dll2_owner_clear(&o);
+ lws_dll2_add_head(&ap->list, &o);
+
+ /*
+ * Create the task in event-specific database
+ */
+
+ if (lws_struct_sq3_serialize(pss->pdb_artifact,
+ lsm_schema_sq3_map_artifact,
+ &o, ap->uid)) {
+ lwsl_err("%s: failed artifact struct insert\n",
+ __func__);
+
+ goto afail;
+ }
+
+ /*
+ * recover the rowid
+ */
+
+ lws_snprintf(s, sizeof(s),
+ "select rowid from artifacts "
+ "where timestamp=%llu",
+ (unsigned long long)ap->timestamp);
+
+ if (sqlite3_exec((sqlite3 *)pss->pdb_artifact, s,
+ sai_sql3_get_uint64_cb, &rid, NULL) !=
+ SQLITE_OK) {
+ lwsl_err("%s: %s: %s: fail\n", __func__, s,
+ sqlite3_errmsg(pss->pdb_artifact));
+ goto afail;
+ }
+
+ /*
+ * Set the blob size on associated row
+ */
+
+ lws_snprintf(s, sizeof(s),
+ "update artifacts set blob=zeroblob(%llu) "
+ "where rowid=%llu",
+ (unsigned long long)ap->len,
+ (unsigned long long)rid);
+
+ if (sqlite3_exec((sqlite3 *)pss->pdb_artifact, s,
+ NULL, NULL, NULL) != SQLITE_OK) {
+ lwsl_err("%s: %s: %s: fail\n", __func__, s,
+ sqlite3_errmsg(pss->pdb_artifact));
+ goto afail;
+ }
+
+ /*
+ * Open a blob on the associated row... the blob handle
+ * is closed when this stream closes for whatever
+ * reason.
+ */
+
+ if (sqlite3_blob_open(pss->pdb_artifact, "main",
+ "artifacts", "blob", rid, 1,
+ &pss->blob_artifact) != SQLITE_OK) {
+ lwsl_err("%s: unable to open blob\n", __func__);
+ goto afail;
+ }
+
+ /*
+ * First time around, m == number of bytes let in buf
+ * after JSON, (bl - m) offset
+ */
+ pss->bulk_binary_data = 1;
+ pss->artifact_length = ap->len;
+ } else
+ m = bl;
+
+ if (m) {
+ lwsl_notice("%s: blob write +%d, ofs %llu / %llu, len %d\n",
+ __func__, (int)(bl - m),
+ (unsigned long long)pss->artifact_offset,
+ (unsigned long long)pss->artifact_length, m);
+ if (sqlite3_blob_write(pss->blob_artifact,
+ (uint8_t *)buf + (bl - m), (int)m,
+ pss->artifact_offset)) {
+ lwsl_err("%s: writing blob failed\n", __func__);
+ goto afail;
+ }
+
+ lws_set_timeout(pss->wsi, PENDING_TIMEOUT_HTTP_CONTENT, 5);
+ pss->artifact_offset += (int)m;
+ }
+
+ if (pss->artifact_offset == pss->artifact_length) {
+ lwsl_notice("%s: blob upload finished\n", __func__);
+
+ goto afail;
+ }
+
+ }
+
+ return 0;
+
+afail:
+ lwsac_free(&ac);
+ lwsac_free(&pss->a.ac);
+ sais_event_db_close(vhd, &pss->pdb_artifact);
+
+ return -1;
+}
+
+/*
+ * We're sending something on a builder ws connection
+ */
+
+int
+sais_ws_json_tx_builder(struct vhd *vhd, struct pss *pss, uint8_t *buf,
+ size_t bl)
+{
+ uint8_t *start = buf + LWS_PRE, *p = start, *end = p + bl - LWS_PRE - 1;
+ int n, flags = LWS_WRITE_TEXT, first = 0;
+ lws_struct_serialize_t *js;
+ sai_task_t *task;
+ size_t w;
+
+ if (pss->task_cancel_owner.head) {
+ /*
+ * Pending cancel message to send
+ */
+ sai_cancel_t *c = lws_container_of(pss->task_cancel_owner.head,
+ sai_cancel_t, list);
+
+ js = lws_struct_json_serialize_create(lsm_schema_json_map_can,
+ LWS_ARRAY_SIZE(lsm_schema_json_map_can), 0, c);
+ if (!js)
+ return 1;
+
+ n = lws_struct_json_serialize(js, p, end - p, &w);
+ lws_struct_json_serialize_destroy(&js);
+
+ lws_dll2_remove(&c->list);
+ free(c);
+
+ first = 1;
+ pss->walk = NULL;
+
+ goto send_json;
+ }
+
+ if (!pss->issue_task_owner.count)
+ return 0; /* nothing to send */
+
+ /*
+ * We're sending a builder specific task info that has been bound to the
+ * builder.
+ *
+ * We already got the task struct out of the db in .one_event
+ * (all in .ac)
+ */
+
+ task = lws_container_of(pss->issue_task_owner.head, sai_task_t,
+ pending_assign_list);
+ lws_dll2_remove(&task->pending_assign_list);
+
+ js = lws_struct_json_serialize_create(lsm_schema_map_ta,
+ LWS_ARRAY_SIZE(lsm_schema_map_ta),
+ 0, task);
+ if (!js)
+ return 1;
+
+ n = lws_struct_json_serialize(js, p, end - p, &w);
+ lws_struct_json_serialize_destroy(&js);
+ pss->one_event = NULL;
+ lwsac_free(&task->ac_task_container);
+
+ first = 1;
+ pss->walk = NULL;
+
+ //lwsac_free(&pss->query_ac);
+
+send_json:
+ p += w;
+ if (n == LSJS_RESULT_ERROR) {
+ lwsl_notice("%s: taskinfo: error generating json\n",
+ __func__);
+ return 1;
+ }
+ if (!lws_ptr_diff(p, start)) {
+ lwsl_notice("%s: taskinfo: empty json\n", __func__);
+ return 0;
+ }
+
+ flags = lws_write_ws_flags(LWS_WRITE_TEXT, first, !pss->walk);
+
+ // lwsl_hexdump_notice(start, p - start);
+
+ if (lws_write(pss->wsi, start, p - start, flags) < 0)
+ return -1;
+
+ lws_callback_on_writable(pss->wsi);
+
+ return 0;
+}
diff --git a/src/web/CMakeLists.txt b/src/web/CMakeLists.txt
new file mode 100644
index 0000000..ac66c81
--- /dev/null
+++ b/src/web/CMakeLists.txt
@@ -0,0 +1,136 @@
+set(SUB "sai-web")
+
+set(CPACK_DEBIAN_SERVER_PACKAGE_NAME ${SUB})
+
+set(SRCS
+ w-sai.c
+ w-conf.c
+ w-comms.c
+ w-central.c
+ w-artifact.c
+ w-ws-browser.c
+ w-websrv.c
+)
+
+set(requirements 1)
+require_lws_config(LWS_WITH_STRUCT_SQLITE3 1 requirements)
+require_lws_config(LWS_WITH_SERVER 1 requirements)
+require_lws_config(LWS_WITH_GENCRYPTO 1 requirements)
+require_lws_config(LWS_WITH_UNIX_SOCK 1 requirements)
+
+if (requirements)
+ add_executable(${SUB} ${SRCS})
+ if (APPLE)
+ set_property(TARGET sai-web PROPERTY MACOSX_RPATH YES)
+ endif()
+
+ #
+ # sqlite3 paths (web)
+ #
+ find_path( SQLITE3_INC_PATH NAMES "sqlite3.h")
+ find_library(SQLITE3_LIB_PATH NAMES "sqlite3")
+
+ if (SQLITE3_INC_PATH AND SQLITE3_LIB_PATH)
+ include_directories(BEFORE "${SQLITE3_INC_PATH}")
+ else()
+ message(FATAL_ERROR " Unable to find sqlite3")
+ endif()
+
+ target_link_libraries(${SUB} ${SQLITE3_LIB_PATH})
+
+ include_directories(BEFORE "${SAI_LWS_INC_PATH}")
+
+ CHECK_C_SOURCE_COMPILES("#include <libwebsockets.h>\nint
+ main(void) {\n#if defined(LWS_HAVE_LIBCAP)\n return
+ 0;\n#else\n fail;\n#endif\n return 0;\n}\n" HAS_LIBCAP)
+ if (HAS_LIBCAP)
+ find_library(CAP_LIB_PATH NAMES "cap")
+ endif()
+
+ target_link_libraries(${SUB} websockets ${SAI_LWS_LIB_PATH})
+
+ if (LWS_OPENSSL_LIBRARIES)
+ target_link_libraries(${SUB} ${LWS_OPENSSL_LIBRARIES})
+ endif()
+
+ if (SAI_EXT_PTHREAD_LIBRARIES)
+ target_link_libraries(${SUB} ${SAI_EXT_PTHREAD_LIBRARIES})
+ endif()
+
+ if (HAS_LIBCAP)
+ target_link_libraries(${SUB} ${CAP_LIB_PATH})
+ endif()
+
+ if (MSVC)
+ target_link_libraries(${SUB} ws2_32.lib userenv.lib psapi.lib iphlpapi.lib)
+ endif()
+
+ install(TARGETS ${SUB}
+ RUNTIME DESTINATION "${BIN_DIR}" COMPONENT web)
+ install(
+ FILES ../../assets/index.html
+ ../../assets/OpenSans-Light.ttf
+ ../../assets/rebuild.png
+ ../../assets/delete.png
+ ../../assets/passed.svg
+ ../../assets/failed.svg
+ ../../assets/sai.css
+ ../../assets/sai.js
+ ../../assets/sai.svg
+ ../../assets/sai-icon.svg
+ ../../assets/builder.png
+ ../../assets/favicon.ico
+ ../../assets/strict-csp.svg
+ ../../assets/lws-common.js
+ ../../assets/gs/lwsgs.js
+ ../../assets/gs/lwsgs.css
+ ../../assets/arch-aarch64-a72-bcm2711-rpi4.svg
+ ../../assets/arch-aarch64.svg
+ ../../assets/arch-arm32-m4-mt7697-usi.svg
+ ../../assets/arch-arm32.svg
+ ../../assets/arch-freertos.svg
+ ../../assets/arch-riscv64-virt.svg
+ ../../assets/arch-riscv.svg
+ ../../assets/arch-x86_64-amd.svg
+ ../../assets/arch-x86_64-intel-i3.svg
+ ../../assets/arch-x86_64-intel.svg
+ ../../assets/arch-xl6-esp32.svg
+ ../../assets/artifact.svg
+ ../../assets/builder.svg
+ ../../assets/decal-2.svg
+ ../../assets/decal-3.svg
+ ../../assets/decal-4.svg
+ ../../assets/decal-5.svg
+ ../../assets/decal-6.svg
+ ../../assets/freertos-espidf.svg
+ ../../assets/freertos-linkit.svg
+ ../../assets/git.svg
+ ../../assets/jsplease.svg
+ ../../assets/linux-android.svg
+ ../../assets/linux-centos-8.svg
+ ../../assets/linux-fedora-32-riscv.svg
+ ../../assets/linux-fedora-32.svg
+ ../../assets/linux-gentoo.svg
+ ../../assets/linux-ubuntu-1804.svg
+ ../../assets/linux-ubuntu-2004.svg
+ ../../assets/linux-ubuntu-xenial-amd64.svg
+ ../../assets/netbsd-iOS.svg
+ ../../assets/netbsd-OSX-catalina.svg
+ ../../assets/sai-event.svg
+ ../../assets/sai-icon.svg
+ ../../assets/sai.svg
+ ../../assets/stop.svg
+ ../../assets/strict-csp.svg
+ ../../assets/tc-gcc.svg
+ ../../assets/tc-llvm.svg
+ ../../assets/tc-mingw32.svg
+ ../../assets/tc-mingw64.svg
+ ../../assets/tc-msvc.svg
+ ../../assets/ubuntu-focal-aarch64.svg
+ ../../assets/virt-qemu.svg
+ ../../assets/windows-10.svg
+
+ DESTINATION "${DATA_DIR}/sai/assets"
+ COMPONENT web)
+
+endif(requirements)
diff --git a/src/web/w-artifact.c b/src/web/w-artifact.c
new file mode 100644
index 0000000..2b87270
--- /dev/null
+++ b/src/web/w-artifact.c
@@ -0,0 +1,151 @@
+/*
+ * Sai server
+ *
+ * Copyright (C) 2019 - 2020 Andy Green <andy@warmcat.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation:
+ * version 2.1 of the License.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301 USA
+ */
+
+#include <libwebsockets.h>
+#include <string.h>
+#include <signal.h>
+#include <time.h>
+
+#include "w-private.h"
+
+void
+sai_task_uuid_to_event_uuid(char *event_uuid33, const char *task_uuid65)
+{
+ memcpy(event_uuid33, task_uuid65, 32);
+ event_uuid33[32] = '\0';
+}
+
+int
+saiw_get_blob(struct vhd *vhd, const char *url, sqlite3 **pdb,
+ sqlite3_blob **blob, uint64_t *length)
+{
+ char task_uuid[66], event_uuid[34], nonce[34], qu[200],
+ esc[66], esc1[34];
+ struct lwsac *ac = NULL;
+ lws_dll2_owner_t o;
+ sai_artifact_t *a;
+ uint64_t rid = 0;
+ const char *p;
+ int n;
+
+ /*
+ * We get passed the RHS of a URL string like
+ *
+ * <task_uuid>/<down_nonce>/filename
+ *
+ * filename is not used for matching, but make sure the client saves it
+ * using the name generated along with the link.
+ *
+ * Extract the pieces from the URL
+ */
+
+ n = 0;
+ p = url;
+ while (*p && *p != '/' && n < (int)sizeof(task_uuid) - 1)
+ task_uuid[n++] = *p++;
+
+ if (n != sizeof(task_uuid) - 2 || *p != '/') {
+ lwsl_notice("%s: url layout 1: %d %s\n", __func__, n, url);
+ return -1;
+ }
+
+ task_uuid[n] = '\0';
+ p++; /* skip the / */
+
+ n = 0;
+ while (*p && *p != '/' && n < (int)sizeof(nonce) - 1)
+ nonce[n++] = *p++;
+
+ if (n != sizeof(nonce) - 2 || *p != '/') {
+ lwsl_notice("%s: url layout 2\n", __func__);
+ return -1;
+ }
+
+ sai_task_uuid_to_event_uuid(event_uuid, task_uuid);
+
+ /* open the event-specific database object */
+
+ if (sais_event_db_ensure_open(vhd, event_uuid, 0, pdb)) {
+ lwsl_info("%s: unable to open event-specific database\n",
+ __func__);
+
+ return -1;
+ }
+
+ /*
+ * Query the artifact he's asking for
+ */
+
+ lws_sql_purify(esc, task_uuid, sizeof(esc));
+ lws_sql_purify(esc1, nonce, sizeof(esc1));
+ lws_snprintf(qu, sizeof(qu),
+ " and task_uuid='%s' and artifact_down_nonce='%s'",
+ esc, esc1);
+ n = lws_struct_sq3_deserialize(*pdb, qu, NULL,
+ lsm_schema_sq3_map_artifact,
+ &o, &ac, 0, 1);
+ if (n < 0 || !o.head) {
+ lwsl_err("%s: no result from %s\n", __func__, qu);
+ goto fail;
+ }
+
+ a = (sai_artifact_t *)o.head;
+
+ *length = a->len;
+
+ /*
+ * recover the rowid the blob api requires
+ */
+
+ lws_snprintf(qu, sizeof(qu), "select rowid from artifacts "
+ "where artifact_down_nonce='%s'", esc1);
+
+ if (sqlite3_exec(*pdb, qu, sai_sql3_get_uint64_cb, &rid, NULL) != SQLITE_OK) {
+ lwsl_err("%s: %s: %s: fail\n", __func__, qu, sqlite3_errmsg(*pdb));
+ goto fail;
+ }
+
+ /*
+ * Get a read-only handle on the blob
+ */
+
+ if (sqlite3_blob_open(*pdb, "main", "artifacts", "blob", rid, 0, blob) != SQLITE_OK) {
+ lwsl_err("%s: unable to open blob, rid %d\n", __func__, (int)rid);
+ goto fail;
+ }
+
+ lwsac_free(&ac); /* drop the lwsac holding that result */
+
+ /*
+ * It was successful, *length was set, *pdb and *blob are live handles
+ * to the db and the blob itself.
+ */
+
+ return 0;
+
+fail:
+ lwsac_free(&ac);
+ sais_event_db_close(vhd, pdb);
+
+ lwsl_notice("%s: couldn't find blob %s\n", __func__, url);
+
+ return -1;
+}
diff --git a/src/web/w-central.c b/src/web/w-central.c
new file mode 100644
index 0000000..c94634d
--- /dev/null
+++ b/src/web/w-central.c
@@ -0,0 +1,45 @@
+/*
+ * Sai server
+ *
+ * Copyright (C) 2019 - 2020 Andy Green <andy@warmcat.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation:
+ * version 2.1 of the License.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301 USA
+ *
+ *
+ * Central dispatcher for jobs from events that made it into the database. This
+ * is done in an event-driven way in m-task.c, but management of it also has to
+ * be done in the background for when there are no events coming,
+ */
+
+#include <libwebsockets.h>
+#include <string.h>
+#include <signal.h>
+#include <time.h>
+
+#include "w-private.h"
+
+extern struct lws_context *context;
+
+void
+saiw_central_cb(lws_sorted_usec_list_t *sul)
+{
+ struct vhd *vhd = lws_container_of(sul, struct vhd, sul_central);
+
+ /* check again in 1s */
+
+ lws_sul_schedule(context, 0, &vhd->sul_central, saiw_central_cb,
+ 1 * LWS_US_PER_SEC);
+}
diff --git a/src/web/w-comms.c b/src/web/w-comms.c
new file mode 100644
index 0000000..49f6d38
--- /dev/null
+++ b/src/web/w-comms.c
@@ -0,0 +1,1123 @@
+/*
+ * Sai server
+ *
+ * Copyright (C) 2019 - 2020 Andy Green <andy@warmcat.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation:
+ * version 2.1 of the License.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301 USA
+ *
+ * The same ws interface is connected-to by builders (on path /builder), and
+ * provides the query transport for browsers (on path /browse).
+ *
+ * There's a single server slite3 database containing events, and a separate
+ * sqlite3 database file for each event, it only contains tasks and logs for
+ * the event and can be deleted when the event record associated with it is
+ * deleted. This is to keep is scalable when there may be thousands of events
+ * and related tasks and logs stored.
+ */
+
+#include <libwebsockets.h>
+#include <string.h>
+#include <signal.h>
+#include <time.h>
+#include <stdio.h>
+#include <fcntl.h>
+
+#include "w-private.h"
+
+#include "../common/struct-metadata.c"
+
+typedef enum {
+ SJS_CLONING,
+ SJS_ASSIGNING,
+ SJS_WAITING,
+ SJS_DONE
+} sai_job_state_t;
+
+typedef struct sai_job {
+ struct lws_dll2 jobs_list;
+ char reponame[64];
+ char ref[64];
+ char head[64];
+
+ time_t requested;
+
+ sai_job_state_t state;
+
+} sai_job_t;
+
+const lws_struct_map_t lsm_schema_map_ta[] = {
+ LSM_SCHEMA (sai_task_t, NULL, lsm_task, "com-warmcat-sai-ta"),
+};
+
+typedef struct sai_auth {
+ lws_dll2_t list;
+ char name[33];
+ char passphrase[65];
+ unsigned long since;
+ unsigned long last_updated;
+} sai_auth_t;
+
+const lws_struct_map_t lsm_auth[] = {
+ LSM_CARRAY (sai_auth_t, name, "name"),
+ LSM_CARRAY (sai_auth_t, passphrase, "passphrase"),
+ LSM_UNSIGNED (sai_auth_t, since, "since"),
+ LSM_UNSIGNED (sai_auth_t, last_updated, "last_updated"),
+};
+
+const lws_struct_map_t lsm_schema_sq3_map_auth[] = {
+ LSM_SCHEMA_DLL2 (sai_auth_t, list, NULL, lsm_auth, "auth"),
+};
+
+extern const lws_struct_map_t lsm_schema_sq3_map_event[];
+
+
+/* len is typically 16 (event uuid is 32 chars + NUL)
+ * But eg, task uuid is concatenated 32-char eventid and 32-char taskid
+ */
+
+int
+sai_uuid16_create(struct lws_context *context, char *dest33)
+{
+ return lws_hex_random(context, dest33, 33);
+}
+
+int
+sai_sqlite3_statement(sqlite3 *pdb, const char *cmd, const char *desc)
+{
+ sqlite3_stmt *sm;
+ int n;
+
+ if (sqlite3_prepare_v2(pdb, cmd, -1, &sm, NULL) != SQLITE_OK) {
+ lwsl_err("%s: Unable to %s: %s\n",
+ __func__, desc, sqlite3_errmsg(pdb));
+
+ return 1;
+ }
+
+ n = sqlite3_step(sm);
+ sqlite3_reset(sm);
+ sqlite3_finalize(sm);
+ if (n != SQLITE_DONE) {
+ n = sqlite3_extended_errcode(pdb);
+ if (!n) {
+ lwsl_info("%s: failed '%s'\n", __func__, cmd);
+ return 0;
+ }
+
+ lwsl_err("%s: %d: Unable to perform \"%s\": %s\n", __func__,
+ n, desc, sqlite3_errmsg(pdb));
+ puts(cmd);
+
+ return 1;
+ }
+
+ return 0;
+}
+
+int
+sais_event_db_ensure_open(struct vhd *vhd, const char *event_uuid,
+ char create_if_needed, sqlite3 **ppdb)
+{
+ char filepath[256], saf[33];
+ sais_sqlite_cache_t *sc;
+
+ if (*ppdb)
+ return 0;
+
+ /* do we have this guy cached? */
+
+ lws_start_foreach_dll(struct lws_dll2 *, p, vhd->sqlite3_cache.head) {
+ sc = lws_container_of(p, sais_sqlite_cache_t, list);
+
+ if (!strcmp(event_uuid, sc->uuid)) {
+ sc->refcount++;
+ *ppdb = sc->pdb;
+ return 0;
+ }
+
+ } lws_end_foreach_dll(p);
+
+ /* ... nope, well, let's open and cache him then... */
+
+ lws_strncpy(saf, event_uuid, sizeof(saf));
+ lws_filename_purify_inplace(saf);
+
+ lws_snprintf(filepath, sizeof(filepath), "%s-event-%s.sqlite3",
+ vhd->sqlite3_path_lhs, saf);
+
+ if (lws_struct_sq3_open(vhd->context, filepath, create_if_needed, ppdb)) {
+ lwsl_err("%s: Unable to open db %s: %s\n", __func__,
+ filepath, sqlite3_errmsg(*ppdb));
+
+ return 1;
+ }
+
+ /* create / add to the schema for the tables we will have in here */
+
+ if (lws_struct_sq3_create_table(*ppdb, lsm_schema_sq3_map_task))
+ return 1;
+
+ sai_sqlite3_statement(*ppdb, "PRAGMA journal_mode=WAL;", "set WAL");
+
+ if (lws_struct_sq3_create_table(*ppdb, lsm_schema_sq3_map_log))
+ return 1;
+
+ if (lws_struct_sq3_create_table(*ppdb, lsm_schema_sq3_map_artifact))
+ return 1;
+
+ sc = malloc(sizeof(*sc));
+ memset(sc, 0, sizeof(*sc));
+ if (!sc) {
+ lws_struct_sq3_close(ppdb);
+ *ppdb = NULL;
+ return 1;
+ }
+
+ lws_strncpy(sc->uuid, event_uuid, sizeof(sc->uuid));
+ sc->refcount = 1;
+ sc->pdb = *ppdb;
+ lws_dll2_add_tail(&sc->list, &vhd->sqlite3_cache);
+
+ return 0;
+}
+
+void
+sais_event_db_close(struct vhd *vhd, sqlite3 **ppdb)
+{
+ sais_sqlite_cache_t *sc;
+
+ if (!*ppdb)
+ return;
+
+ /* look for him in the cache */
+
+ lws_start_foreach_dll(struct lws_dll2 *, p, vhd->sqlite3_cache.head) {
+ sc = lws_container_of(p, sais_sqlite_cache_t, list);
+
+ if (sc->pdb == *ppdb) {
+ *ppdb = NULL;
+ if (--sc->refcount) {
+ lwsl_notice("%s: zero refcount to idle\n", __func__);
+ /*
+ * He's not currently in use then... don't
+ * close him immediately, m-central.c has a
+ * timer that closes and removes sqlite3
+ * cache entries idle for longer than 60s
+ */
+ sc->idle_since = lws_now_usecs();
+ }
+
+ return;
+ }
+
+ } lws_end_foreach_dll(p);
+
+ lws_struct_sq3_close(ppdb);
+ *ppdb = NULL;
+}
+
+int
+sais_event_db_delete_database(struct vhd *vhd, const char *event_uuid)
+{
+ char filepath[256], saf[33];
+
+ lws_strncpy(saf, event_uuid, sizeof(saf));
+ lws_filename_purify_inplace(saf);
+
+ lws_snprintf(filepath, sizeof(filepath), "%s-event-%s.sqlite3",
+ vhd->sqlite3_path_lhs, saf);
+
+ return unlink(filepath);
+}
+
+
+#if 0
+static void
+sais_all_browser_on_writable(struct vhd *vhd)
+{
+ lws_start_foreach_dll(struct lws_dll2 *, mp, vhd->browsers.head) {
+ struct pss *pss = lws_container_of(mp, struct pss, same);
+
+ lws_callback_on_writable(pss->wsi);
+ } lws_end_foreach_dll(mp);
+}
+#endif
+
+typedef enum {
+ SHMUT_NONE = -1,
+ SHMUT_HOOK,
+ SHMUT_BROWSE,
+ SHMUT_STATUS,
+ SHMUT_ARTIFACTS,
+ SHMUT_LOGIN
+} sai_http_murl_t;
+
+static const char * const well_known[] = {
+ "/update-hook",
+ "/sai/browse",
+ "/status",
+ "/artifacts/", /* HTTP api for accessing build artifacts */
+ "/login"
+};
+
+int
+saiw_task_cancel(struct vhd *vhd, const char *task_uuid)
+{
+ sai_cancel_t *can = malloc(sizeof(*can));
+
+ memset(can, 0, sizeof(*can));
+
+ lws_strncpy(can->task_uuid, task_uuid, sizeof(can->task_uuid));
+
+ lws_dll2_add_tail(&can->list, &vhd->web_to_srv_owner);
+
+
+ return 0;
+}
+
+int
+saiw_sched_destroy(struct lws_dll2 *d, void *user)
+{
+ saiw_scheduled_t *sch = lws_container_of(d, saiw_scheduled_t, list);
+
+ saiw_dealloc_sched(sch);
+
+ return 0;
+}
+
+int
+sai_get_head_status(struct vhd *vhd, const char *projname)
+{
+ struct lwsac *ac = NULL;
+ lws_dll2_owner_t o;
+ sai_event_t *e;
+ int state;
+
+ if (lws_struct_sq3_deserialize(vhd->pdb, NULL, "created ",
+ lsm_schema_sq3_map_event,
+ &o, &ac, 0, -1))
+ return -1;
+
+ if (!o.head)
+ return -1;
+
+ e = lws_container_of(o.head, sai_event_t, list);
+ state = e->state;
+
+ lwsac_free(&ac);
+
+ return state;
+}
+
+
+static int
+sai_login_cb(void *data, const char *name, const char *filename,
+ char *buf, int len, enum lws_spa_fileupload_states state)
+{
+ lwsl_notice("%s: name '%s'\n", __func__, name);
+ return 0;
+}
+
+static const char * const auth_param_names[] = {
+ "lname",
+ "lpass",
+ "success_redir",
+};
+
+enum enum_param_names {
+ EPN_LNAME,
+ EPN_LPASS,
+ EPN_SUCCESS_REDIR,
+};
+
+static int
+saiw_event_db_close_all_now(struct vhd *vhd)
+{
+ sais_sqlite_cache_t *sc;
+
+ lws_start_foreach_dll_safe(struct lws_dll2 *, p, p1,
+ vhd->sqlite3_cache.head) {
+ sc = lws_container_of(p, sais_sqlite_cache_t, list);
+
+ lws_struct_sq3_close(&sc->pdb);
+ lws_dll2_remove(&sc->list);
+ free(sc);
+
+ } lws_end_foreach_dll_safe(p, p1);
+
+ return 0;
+}
+
+static int
+callback_ws(struct lws *wsi, enum lws_callback_reasons reason, void *user,
+ void *in, size_t len)
+{
+ struct vhd *vhd = (struct vhd *)lws_protocol_vh_priv_get(
+ lws_get_vhost(wsi), lws_get_protocol(wsi));
+ uint8_t buf[LWS_PRE + 8192], *start = &buf[LWS_PRE], *p = start,
+ *end = &buf[sizeof(buf) - LWS_PRE - 1];
+ struct pss *pss = (struct pss *)user;
+ struct lws_jwt_sign_set_cookie ck;
+ sai_http_murl_t mu = SHMUT_NONE;
+ char projname[64];
+ int n, resp, r;
+ const char *cp;
+ size_t cml;
+
+ (void)end;
+ (void)p;
+
+ switch (reason) {
+ case LWS_CALLBACK_PROTOCOL_INIT:
+ vhd = lws_protocol_vh_priv_zalloc(lws_get_vhost(wsi),
+ lws_get_protocol(wsi),
+ sizeof(struct vhd));
+ if (!vhd)
+ return -1;
+
+ vhd->context = lws_get_context(wsi);
+ vhd->vhost = lws_get_vhost(wsi);
+
+ if (lws_pvo_get_str(in, "database", &vhd->sqlite3_path_lhs)) {
+ lwsl_err("%s: database pvo required\n", __func__);
+ return -1;
+ }
+
+ lws_snprintf((char *)buf, sizeof(buf), "%s-events.sqlite3",
+ vhd->sqlite3_path_lhs);
+
+ if (lws_struct_sq3_open(vhd->context, (char *)buf, 1, &vhd->pdb)) {
+ lwsl_err("%s: Unable to open session db %s: %s\n",
+ __func__, vhd->sqlite3_path_lhs, sqlite3_errmsg(
+ vhd->pdb));
+
+ return -1;
+ }
+
+ sai_sqlite3_statement(vhd->pdb,
+ "PRAGMA journal_mode=WAL;", "set WAL");
+
+ if (lws_struct_sq3_create_table(vhd->pdb,
+ lsm_schema_sq3_map_event)) {
+ lwsl_err("%s: unable to create event table\n", __func__);
+ return -1;
+ }
+
+ /* auth database */
+
+ lws_snprintf((char *)buf, sizeof(buf), "%s-auth.sqlite3",
+ vhd->sqlite3_path_lhs);
+
+ if (lws_struct_sq3_open(vhd->context, (char *)buf, 1,
+ &vhd->pdb_auth)) {
+ lwsl_err("%s: Unable to open auth db %s: %s\n",
+ __func__, vhd->sqlite3_path_lhs, sqlite3_errmsg(
+ vhd->pdb_auth));
+
+ return -1;
+ }
+
+ if (lws_struct_sq3_create_table(vhd->pdb_auth,
+ lsm_schema_sq3_map_auth)) {
+ lwsl_err("%s: unable to create auth table\n", __func__);
+ return -1;
+ }
+
+ /*
+ * jwt-iss
+ */
+
+ if (lws_pvo_get_str(in, "jwt-iss", &vhd->jwt_issuer)) {
+ lwsl_err("%s: jwt-iss required\n", __func__);
+ return -1;
+ }
+
+ /*
+ * jwt-aud
+ */
+
+ if (lws_pvo_get_str(in, "jwt-aud", &vhd->jwt_audience)) {
+ lwsl_err("%s: jwt-aud required\n", __func__);
+ return -1;
+ }
+
+ /*
+ * auth-alg
+ */
+
+ if (lws_pvo_get_str(in, "jwt-auth-alg", &cp)) {
+ lwsl_err("%s: jwt-auth-alg required\n", __func__);
+ return -1;
+ }
+
+ lws_strncpy(vhd->jwt_auth_alg, cp, sizeof(vhd->jwt_auth_alg));
+
+ /*
+ * auth-jwk-path
+ */
+
+ if (lws_pvo_get_str(in, "jwt-auth-jwk-path", &cp)) {
+ lwsl_err("%s: jwt-auth-jwk-path required\n", __func__);
+ return -1;
+ }
+
+ n = open(cp, LWS_O_RDONLY);
+ if (!n) {
+ lwsl_err("%s: can't open auth JWK %s\n", __func__, cp);
+ return -1;
+ }
+ r = read(n, buf, sizeof(buf));
+ close(n);
+ if (r < 0) {
+ lwsl_err("%s: can't read auth JWK %s\n", __func__, cp);
+ return -1;
+ }
+
+ if (lws_jwk_import(&vhd->jwt_jwk_auth, NULL, NULL,
+ (const char *)buf, r)) {
+ lwsl_notice("%s: Failed to parse JWK key\n", __func__);
+ return -1;
+ }
+
+ lwsl_notice("%s: Auth JWK type %d\n", __func__,
+ vhd->jwt_jwk_auth.kty);
+
+ lws_sul_schedule(vhd->context, 0, &vhd->sul_central,
+ saiw_central_cb, 500 * LWS_US_PER_MS);
+
+ /*
+ * Reach out to the sai-server part over the SS ws websrv link
+ */
+
+ if (lws_ss_create(lws_get_context(wsi), 0, &ssi_saiw_websrv, vhd,
+ &vhd->h_ss_websrv, NULL, NULL)) {
+ lwsl_err("%s: failed to create SS for websrv\n",
+ __func__);
+
+ return 1;
+ }
+
+ lws_ss_set_metadata(vhd->h_ss_websrv, "sockpath",
+ "@com.warmcat.sai-websrv", 23);
+ lws_ss_client_connect(vhd->h_ss_websrv);
+
+ break;
+
+ case LWS_CALLBACK_PROTOCOL_DESTROY:
+ saiw_event_db_close_all_now(vhd);
+ lws_struct_sq3_close(&vhd->pdb);
+ lws_struct_sq3_close(&vhd->pdb_auth);
+ lws_jwk_destroy(&vhd->jwt_jwk_auth);
+ goto passthru;
+
+ /*
+ * receive http hook notifications
+ */
+
+ case LWS_CALLBACK_HTTP:
+
+ // lwsl_notice("%s: HTTP\n", __func__);
+
+ resp = HTTP_STATUS_FORBIDDEN;
+ pss->vhd = vhd;
+
+ /*
+ * What's the situation with a JWT cookie? Normal users won't
+ * have any, but privileged users will have one, and we should
+ * try to confirm it and set the pss auth level accordingly
+ */
+
+ memset(&ck, 0, sizeof(ck));
+ ck.jwk = &vhd->jwt_jwk_auth;
+ ck.alg = vhd->jwt_auth_alg;
+ ck.iss = vhd->jwt_issuer;
+ ck.aud = vhd->jwt_audience;
+ ck.cookie_name = "__Host-sai_jwt";
+
+ cml = sizeof(buf);
+ if (!lws_jwt_get_http_cookie_validate_jwt(wsi, &ck,
+ (char *)buf, &cml) &&
+ ck.extra_json &&
+ !lws_json_simple_strcmp(ck.extra_json, ck.extra_json_len,
+ "\"authorized\":", "1")) {
+ /* the token allows him to manage us */
+ pss->authorized = 1;
+ pss->expiry_unix_time = ck.expiry_unix_time;
+ lws_strncpy(pss->auth_user, ck.sub,
+ sizeof(pss->auth_user));
+ } else
+ lwsl_info("%s: cookie rejected\n", __func__);
+
+ for (n = 0; n < (int)LWS_ARRAY_SIZE(well_known); n++)
+ if (!strncmp((const char *)in, well_known[n],
+ strlen(well_known[n]))) {
+ mu = n;
+ break;
+ }
+
+ pss->our_form = 0;
+
+ // lwsl_notice("%s: HTTP: '%s' mu = %d\n", __func__, (const char *)in, n);
+
+ switch (mu) {
+
+ case SHMUT_NONE:
+ goto passthru;
+
+ case SHMUT_HOOK:
+ pss->our_form = 1;
+ lwsl_notice("LWS_CALLBACK_HTTP: sees hook\n");
+ return 0;
+
+ case SHMUT_STATUS:
+ /*
+ * in is a string like /libwebsockets/status.svg
+ */
+ cp = ((const char *)in) + 7;
+ while (*cp == '/')
+ cp++;
+ n = 0;
+ while (*cp != '/' && *cp && (size_t)n < sizeof(projname) - 1)
+ projname[n++] = *cp++;
+ projname[n] = '\0';
+
+ // lwsl_notice("%s: status %s\n", __func__, projname);
+
+ r = sai_get_head_status(vhd, projname);
+ if (r < 2)
+ r = 2;
+ n = lws_snprintf(projname, sizeof(projname),
+ "../decal-%d.svg", r);
+
+ if (lws_http_redirect(wsi, 307,
+ (unsigned char *)projname, n,
+ &p, end) < 0)
+ return -1;
+
+ goto passthru;
+
+ case SHMUT_LOGIN:
+ pss->login_form = 1;
+ lwsl_notice("LWS_CALLBACK_HTTP: sees login\n");
+ return 0;
+
+ case SHMUT_ARTIFACTS:
+ /*
+ * HTTP Bulk GET interface for artifact download
+ *
+ * /artifacts/<taskhash>/<down_nonce>/filename
+ */
+ lwsl_notice("%s: SHMUT_ARTIFACTS\n", __func__);
+ pss->artifact_offset = 0;
+ if (saiw_get_blob(vhd, (const char *)in + 11,
+ &pss->pdb_artifact,
+ &pss->blob_artifact,
+ &pss->artifact_length)) {
+ lwsl_notice("%s: get_blob failed\n", __func__);
+ resp = 404;
+ goto http_resp;
+ }
+
+ /*
+ * Well, it seems what he wanted exists..
+ */
+
+ if (lws_add_http_header_status(wsi, 200, &p, end))
+ goto bail;
+ if (lws_add_http_header_content_length(wsi,
+ (unsigned long)pss->artifact_length,
+ &p, end))
+ goto bail;
+
+ if (lws_add_http_header_by_token(wsi,
+ WSI_TOKEN_HTTP_CONTENT_TYPE,
+ (uint8_t *)"application/octet-stream",
+ 24, &p, end))
+ goto bail;
+ if (lws_finalize_write_http_header(wsi, start, &p, end))
+ goto bail;
+
+ lwsl_notice("%s: started artifact transaction %d\n", __func__,
+ (int)pss->artifact_length);
+
+ lws_callback_on_writable(wsi);
+ return 0;
+
+ default:
+ lwsl_notice("%s: DEFAULT!!!\n", __func__);
+ return 0;
+ }
+
+ resp = HTTP_STATUS_OK;
+
+ /* faillthru */
+
+http_resp:
+ if (lws_add_http_header_status(wsi, resp, &p, end))
+ goto bail;
+ if (lws_add_http_header_content_length(wsi, 0, &p, end))
+ goto bail;
+ if (lws_finalize_write_http_header(wsi, start, &p, end))
+ goto bail;
+ goto try_to_reuse;
+
+
+ case LWS_CALLBACK_HTTP_WRITEABLE:
+
+ // lwsl_notice("%s: HTTP_WRITEABLE\n", __func__);
+
+ if (!pss || !pss->blob_artifact)
+ break;
+
+ n = lws_ptr_diff(end, start);
+ if ((int)(pss->artifact_length - pss->artifact_offset) < n)
+ n = (int)(pss->artifact_length - pss->artifact_offset);
+
+ if (sqlite3_blob_read(pss->blob_artifact, start, n,
+ pss->artifact_offset)) {
+ lwsl_err("%s: blob read failed\n", __func__);
+ return -1;
+ }
+
+ pss->artifact_offset += n;
+
+ if (lws_write(wsi, start, n,
+ pss->artifact_offset != pss->artifact_length ?
+ LWS_WRITE_HTTP : LWS_WRITE_HTTP_FINAL) != n)
+ return -1;
+
+ if (pss->artifact_offset != pss->artifact_length)
+ lws_callback_on_writable(wsi);
+
+ break;
+
+ /*
+ * Notifcation POSTs
+ */
+
+ case LWS_CALLBACK_HTTP_BODY:
+
+ // lwsl_notice("%s: HTTP_BODY\n", __func__);
+
+ if (pss->login_form) {
+
+ if (!pss->spa) {
+ pss->spa = lws_spa_create(wsi, auth_param_names,
+ LWS_ARRAY_SIZE(auth_param_names),
+ 1024, sai_login_cb, pss);
+ if (!pss->spa) {
+ lwsl_err("failed to create spa\n");
+ return -1;
+ }
+ }
+
+ /* let it parse the POST data */
+
+ lwsl_hexdump_notice(in, len);
+
+ if (!pss->spa_failed &&
+ lws_spa_process(pss->spa, in, (int)len)) {
+
+ lwsl_notice("%s: spa failed\n", __func__);
+
+ /*
+ * mark it as failed, and continue taking body until
+ * completion, and return error there
+ */
+ pss->spa_failed = 1;
+ }
+ }
+
+ break;
+
+ case LWS_CALLBACK_HTTP_BODY_COMPLETION:
+ lwsl_user("%s: LWS_CALLBACK_HTTP_BODY_COMPLETION: %d\n",
+ __func__, (int)len);
+
+ if (!pss->our_form && !pss->login_form) {
+ lwsl_user("%s: no sai form\n", __func__);
+ goto passthru;
+ }
+
+ /* inform the spa no more payload data coming */
+ if (pss->spa)
+ lws_spa_finalize(pss->spa);
+
+ if (pss->login_form) {
+ const char *un, *pw, *sr;
+ lws_dll2_owner_t o;
+ struct lwsac *ac = NULL;
+
+ if (lws_add_http_header_status(wsi,
+ HTTP_STATUS_SEE_OTHER, &p, end))
+ goto clean_spa;
+ if (lws_add_http_header_content_length(wsi, 0, &p, end))
+ goto clean_spa;
+
+ if (pss->spa_failed)
+ goto final;
+
+ if (pss->authorized) {
+
+ char temp[128];
+ /*
+ * It means, logout then
+ */
+
+ n = lws_snprintf(temp, sizeof(temp), "__Host-sai_jwt=deleted;"
+ "HttpOnly;"
+ "Secure;"
+ "SameSite=strict;"
+ "Path=/;"
+ "expires=Sun, 06 Nov 1994 08:49:37 GMT;");
+
+ sr = "x/..";
+
+ if (lws_add_http_header_by_token(wsi, WSI_TOKEN_HTTP_SET_COOKIE,
+ (uint8_t *)temp, n, &p, end)) {
+ lwsl_err("%s: failed to add JWT cookie header\n", __func__);
+ return 1;
+ }
+
+ goto back;
+ }
+
+ un = lws_spa_get_string(pss->spa, EPN_LNAME);
+ pw = lws_spa_get_string(pss->spa, EPN_LPASS);
+ sr = lws_spa_get_string(pss->spa, EPN_SUCCESS_REDIR);
+
+ if (!un || !pw || !sr) {
+ lwsl_notice("%s: missing form args %p %p %p\n",__func__, un, pw, sr);
+ pss->spa_failed = 1;
+ goto final;
+ }
+
+ // lwsl_notice("%s: login attempt %s %s %s\n", __func__,
+ // un, pw, sr);
+
+ /*
+ * Try to look up his credentials
+ */
+
+ lws_sql_purify((char *)buf + 512, un, 34);
+ lws_sql_purify((char *)buf + 768, pw, 66);
+ lws_snprintf((char *)buf + 256, 256,
+ " and name='%s' and passphrase='%s'",
+ (const char *)buf + 512,
+ (const char *)buf + 768);
+ lws_dll2_owner_clear(&o);
+ n = lws_struct_sq3_deserialize(pss->vhd->pdb_auth,
+ (const char *)buf + 256,
+ NULL,
+ lsm_schema_sq3_map_auth,
+ &o, &ac, 0, 1);
+ if (n < 0 || !o.head) {
+ /* no results, failed */
+ // lwsl_notice("%s: login attempt %s failed %d\n",
+ // __func__, (const char *)buf, n);
+ lwsac_free(&ac);
+ pss->spa_failed = 1;
+ goto final;
+ }
+
+ /* any result in o means a successful match */
+
+ lwsac_free(&ac);
+
+ /*
+ * Produce a signed JWT allowing managing this Sai
+ * instance for a short time, and redirect ourselves
+ * back to the page we were on
+ */
+
+
+
+ lwsl_notice("%s: setting cookie\n", __func__);
+ /* un is invalidated by destroying the spa */
+ memset(&ck, 0, sizeof(ck));
+ lws_strncpy(ck.sub, un, sizeof(ck.sub));
+ ck.jwk = &vhd->jwt_jwk_auth;
+ ck.alg = vhd->jwt_auth_alg;
+ ck.iss = vhd->jwt_issuer;
+ ck.aud = vhd->jwt_audience;
+ ck.cookie_name = "sai_jwt";
+ ck.extra_json = "\"authorized\": 1";
+ ck.expiry_unix_time = 2 * 60 * 60; /* 2 days */
+
+ if (lws_jwt_sign_token_set_http_cookie(wsi, &ck, &p, end))
+ goto clean_spa;
+
+back:
+ /*
+ * Auth succeeded, go to the page the form was on
+ */
+
+ if (lws_add_http_header_by_token(wsi,
+ WSI_TOKEN_HTTP_LOCATION,
+ (unsigned char *)sr,
+ strlen((const char *)sr),
+ &p, end)) {
+ goto clean_spa;
+ }
+
+ if (pss->spa) {
+ lws_spa_destroy(pss->spa);
+ pss->spa = NULL;
+ }
+
+ if (lws_finalize_write_http_header(wsi, start, &p, end))
+ goto bail;
+
+ lwsl_notice("%s: set / delete cookie OK\n", __func__);
+ // lwsl_hexdump_notice(start, lws_ptr_diff(p, start));
+ return 0;
+
+final:
+ lwsl_notice("%s: auth failed, login_form %d\n", __func__, pss->login_form);
+ /*
+ * Auth failed, go back to /
+ */
+ if (lws_add_http_header_by_token(wsi,
+ WSI_TOKEN_HTTP_LOCATION,
+ (unsigned char *)"/", 1,
+ &p, end)) {
+ goto clean_spa;
+ }
+ if (lws_finalize_write_http_header(wsi, start, &p, end))
+ goto bail;
+ return 0;
+ }
+
+ if (pss->spa) {
+ lws_spa_destroy(pss->spa);
+ pss->spa = NULL;
+ }
+
+ if (pss->spa_failed)
+ lwsl_notice("%s: POST failed\n", __func__);
+
+ if (lws_return_http_status(wsi,
+ pss->spa_failed ? HTTP_STATUS_FORBIDDEN :
+ HTTP_STATUS_OK,
+ NULL) < 0)
+ return -1;
+ break;
+
+clean_spa:
+ if (pss->spa) {
+ lws_spa_destroy(pss->spa);
+ pss->spa = NULL;
+ }
+ pss->spa_failed = 1;
+ goto final;
+
+ /*
+ * ws connections from builders and browsers
+ */
+
+ case LWS_CALLBACK_FILTER_PROTOCOL_CONNECTION:
+ n = lws_hdr_copy(wsi, (char *)buf, sizeof(buf) - 1,
+ WSI_TOKEN_GET_URI);
+ if (!n)
+ buf[0] = '\0';
+ //lwsl_notice("%s: checking with lwsgs for ws conn: %s\n",
+ // __func__, (const char *)buf);
+
+ /*
+ * Builders don't authenticate using sessions...
+ */
+
+ if (n >= 8 && !strncmp((const char *)buf + n - 8,
+ "/builder", 8))
+ return 0;
+
+ return 0;
+#if 0
+ /* but everything else does */
+
+ car_args.max_len = LWSGS_AUTH_LOGGED_IN | LWSGS_AUTH_VERIFIED;
+ car_args.final = 0;
+ car_args.chunked = 1; /* ie, we are ws */
+
+ in = &car_args;
+ goto passthru;
+#endif
+
+ case LWS_CALLBACK_ESTABLISHED:
+
+ if (!vhd)
+ return -1;
+
+ /*
+ * What's the situation with a JWT cookie? Normal users won't
+ * have any, but privileged users will have one, and we should
+ * try to confirm it and set the pss auth level accordingly
+ */
+
+ memset(&ck, 0, sizeof(ck));
+ ck.jwk = &vhd->jwt_jwk_auth;
+ ck.alg = vhd->jwt_auth_alg;
+ ck.iss = vhd->jwt_issuer;
+ ck.aud = vhd->jwt_audience;
+ ck.cookie_name = "__Host-sai_jwt";
+
+ lws_dll2_add_head(&pss->same, &vhd->browsers);
+
+ cml = sizeof(buf);
+ if (!lws_jwt_get_http_cookie_validate_jwt(wsi, &ck,
+ (char *)buf, &cml) &&
+ ck.extra_json &&
+ !lws_json_simple_strcmp(ck.extra_json, ck.extra_json_len,
+ "\"authorized\":", "1")) {
+ /* the token allows him to manage us */
+ pss->authorized = 1;
+ pss->expiry_unix_time = ck.expiry_unix_time;
+ lws_strncpy(pss->auth_user, ck.sub,
+ sizeof(pss->auth_user));
+ } else
+ lwsl_info("%s: cookie rejected\n", __func__);
+ pss->wsi = wsi;
+ pss->vhd = vhd;
+ pss->alang[0] = '\0';
+ lws_hdr_copy(wsi, pss->alang, sizeof(pss->alang),
+ WSI_TOKEN_HTTP_ACCEPT_LANGUAGE);
+
+ if (lws_hdr_total_length(wsi, WSI_TOKEN_GET_URI)) {
+ if (lws_hdr_copy(wsi, (char *)start, 64,
+ WSI_TOKEN_GET_URI) < 0)
+ return -1;
+ }
+#if defined(LWS_ROLE_H2)
+ else
+ if (lws_hdr_copy(wsi, (char *)start, 64,
+ WSI_TOKEN_HTTP_COLON_PATH) < 0)
+ return -1;
+#endif
+
+ if (!memcmp((char *)start, "/sai", 4))
+ start += 4;
+
+ if (!strncmp((char *)start, "/browse/specific", 16)) {
+ const char *spe;
+
+ lwsl_info("%s: ESTABLISHED: browser (specific)\n", __func__);
+ pss->wsi = wsi;
+ pss->specific_project[0] = '\0';
+ spe = (const char *)start + 16;
+ while (*spe == '/')
+ spe++;
+ n = 0;
+ while(*spe && *spe != '/' &&
+ (size_t)n < sizeof(pss->specific_project) - 2)
+ pss->specific_project[n++] = *spe++;
+
+ pss->specific_project[n] = '\0';
+
+ pss->specific_task[0] = '\0';
+ pss->specific_ref[0] = '\0';
+ if (lws_get_urlarg_by_name(wsi, "h", pss->specific_ref + 9,
+ sizeof(pss->specific_ref) - 9)) {
+ memcpy(pss->specific_ref, "refs/heads/", 11);
+ pss->specificity = SAIM_SPECIFIC_H;
+ } else
+ if (lws_get_urlarg_by_name(wsi, "id", (char *)buf,
+ sizeof(buf))) {
+ lws_strncpy(pss->specific_ref, (const char *)buf + 3,
+ sizeof(pss->specific_ref));
+ pss->specificity = SAIM_SPECIFIC_ID;
+ } else {
+ if (lws_get_urlarg_by_name(wsi, "task",
+ (char *)pss->specific_task,
+ sizeof(pss->specific_task))) {
+ pss->specificity = SAIM_SPECIFIC_TASK;
+ } else {
+ pss->specificity = SAIM_SPECIFIC_H;
+ lws_strncpy(pss->specific_ref,
+ "refs/heads/master",
+ sizeof(pss->specific_ref));
+ }
+ }
+
+ //lwsl_notice("%s: spec %d, '%s'\n", __func__,
+ // pss->specificity, pss->specific_ref);
+ break;
+ }
+
+ if (!strcmp((char *)start, "/browse")) {
+ lwsl_info("%s: ESTABLISHED: browser\n", __func__);
+ pss->wsi = wsi;
+ break;
+ }
+
+ lwsl_err("%s: unknown URL '%s'\n", __func__, start);
+
+ return -1;
+
+ case LWS_CALLBACK_CLOSED:
+
+ lwsl_info("%s: CLOSED browse conn\n", __func__);
+ lws_dll2_remove(&pss->same);
+ lws_dll2_remove(&pss->subs_list);
+
+ lws_dll2_foreach_safe(&pss->sched, NULL, saiw_sched_destroy);
+ lwsac_free(&pss->logs_ac);
+
+ break;
+
+ case LWS_CALLBACK_RECEIVE:
+
+ if (!pss->vhd)
+ pss->vhd = vhd;
+
+ // lwsl_user("SWT_BROWSE RX: %d\n", (int)len);
+ /*
+ * Browser UI sent us something on websockets
+ */
+ if (saiw_ws_json_rx_browser(vhd, pss, in, len))
+ return -1;
+
+ break;
+
+ case LWS_CALLBACK_SERVER_WRITEABLE:
+ if (!vhd) {
+ lwsl_notice("%s: no vhd\n", __func__);
+ break;
+ }
+
+ return saiw_ws_json_tx_browser(vhd, pss, buf, sizeof(buf));
+
+ default:
+passthru:
+ // if (!pss || !vhd)
+ break;
+
+ // return vhd->gsp->callback(wsi, reason, pss->pss_gs, in, len);
+ }
+
+ return lws_callback_http_dummy(wsi, reason, user, in, len);
+
+bail:
+ return 1;
+
+try_to_reuse:
+ if (lws_http_transaction_completed(wsi))
+ return -1;
+
+ return 0;
+}
+
+const struct lws_protocols protocol_ws =
+ { "com-warmcat-sai", callback_ws, sizeof(struct pss), 0 };
diff --git a/src/web/w-conf.c b/src/web/w-conf.c
new file mode 100644
index 0000000..b229ace
--- /dev/null
+++ b/src/web/w-conf.c
@@ -0,0 +1,84 @@
+/*
+ * Sai server src/server/conf.c
+ *
+ * Copyright (C) 2019 - 2020 Andy Green <andy@warmcat.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation:
+ * version 2.1 of the License.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301 USA
+ */
+#include <libwebsockets.h>
+#include <string.h>
+#include <signal.h>
+#include <time.h>
+
+#define SAI_CONFIG_STRING_SIZE (16 * 1024)
+
+struct lws_context *
+sai_lws_context_from_json(const char *config_dir,
+ struct lws_context_creation_info *info,
+ const struct lws_protocols **pprotocols,
+ const char *pol)
+{
+ int cs_len = SAI_CONFIG_STRING_SIZE - 1;
+ struct lws_context *context;
+ char *cs, *config_strings;
+
+ cs = config_strings = malloc(SAI_CONFIG_STRING_SIZE);
+ if (!config_strings) {
+ lwsl_err("Unable to allocate config strings heap\n");
+
+ return NULL;
+ }
+
+ memset(info, 0, sizeof(*info));
+
+ info->external_baggage_free_on_destroy = config_strings;
+ info->pt_serv_buf_size = 8192;
+ info->options = LWS_SERVER_OPTION_DO_SSL_GLOBAL_INIT |
+ LWS_SERVER_OPTION_EXPLICIT_VHOSTS |
+ LWS_SERVER_OPTION_HTTP_HEADERS_SECURITY_BEST_PRACTICES_ENFORCE |
+ LWS_SERVER_OPTION_VALIDATE_UTF8;
+ info->pss_policies_json = pol;
+
+ lwsl_notice("Using config dir: \"%s\"\n", config_dir);
+
+ /*
+ * first go through the config for creating the outer context
+ */
+ if (lwsws_get_config_globals(info, config_dir, &cs, &cs_len))
+ goto init_failed;
+
+ context = lws_create_context(info);
+ if (context == NULL) {
+ /* config_strings freed as 'external baggage' */
+ return NULL;
+ }
+
+ info->pprotocols = pprotocols;
+
+ if (lwsws_get_config_vhosts(context, info, config_dir, &cs, &cs_len)) {
+ lwsl_err("%s: sai_lws_context_from_json failed\n", __func__);
+ lws_context_destroy(context);
+
+ return NULL;
+ }
+
+ return context;
+
+init_failed:
+ free(config_strings);
+
+ return NULL;
+}
diff --git a/src/web/w-private.h b/src/web/w-private.h
new file mode 100644
index 0000000..8e3b27d
--- /dev/null
+++ b/src/web/w-private.h
@@ -0,0 +1,316 @@
+/*
+ * Sai server definitions src/server/private.h
+ *
+ * Copyright (C) 2019 Andy Green <andy@warmcat.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation:
+ * version 2.1 of the License.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301 USA
+ */
+
+#include "../common/include/private.h"
+#include <sqlite3.h>
+#include <sys/stat.h>
+
+struct sai_plat;
+
+typedef struct sai_platm {
+ struct lws_dll2_owner builder_owner;
+ struct lws_dll2_owner subs_owner;
+
+ sqlite3 *pdb;
+ sqlite3 *pdb_auth;
+} sais_t;
+
+typedef struct sai_platform {
+ struct lws_dll2 list;
+
+ const char *name;
+ const char *build;
+
+ uint8_t nondefault;
+
+ /* build and name over-allocated here */
+} sai_platform_t;
+
+typedef enum {
+ SAIN_ACTION_INVALID,
+ SAIN_ACTION_REPO_UPDATED
+} sai_notification_action_t;
+
+typedef struct {
+
+ sai_event_t e;
+ sai_task_t t;
+
+ char platbuild[4096];
+ char platname[96];
+ char explicit_platforms[2048];
+
+ int event_task_index;
+
+ struct lws_b64state b64;
+ char *saifile;
+ uint64_t when;
+ size_t saifile_in_len;
+ size_t saifile_out_len;
+ size_t saifile_out_pos;
+ size_t saifile_in_seen;
+ sai_notification_action_t action;
+
+ uint8_t nondefault;
+} sai_notification_t;
+
+typedef enum {
+ WSS_IDLE,
+ WSS_PREPARE_OVERVIEW,
+ WSS_SEND_OVERVIEW,
+ WSS_PREPARE_BUILDER_SUMMARY,
+ WSS_SEND_BUILDER_SUMMARY,
+
+ WSS_PREPARE_TASKINFO,
+ WSS_SEND_ARTIFACT_INFO,
+
+ WSS_PREPARE_EVENTINFO,
+ WSS_SEND_EVENTINFO,
+} ws_state;
+
+typedef struct sai_builder {
+ sais_t c;
+} saib_t;
+
+struct vhd;
+
+enum {
+ SAIM_NOT_SPECIFIC,
+ SAIM_SPECIFIC_H,
+ SAIM_SPECIFIC_ID,
+ SAIM_SPECIFIC_TASK,
+};
+
+typedef struct saiw_scheduled {
+ lws_dll2_t list;
+
+ char task_uuid[65];
+
+ const sai_task_t *one_task; /* only for browser */
+ const sai_event_t *one_event;
+
+ lws_dll2_t *walk;
+
+ lws_dll2_owner_t owner;
+
+ struct lwsac *ac;
+ struct lwsac *query_ac; /* taskinfo event only */
+
+ ws_state action;
+ int task_index;
+
+ uint8_t ovstate; /* SOS_ substate when doing overview */
+
+ uint8_t subsequent:1; /* for individual JSON */
+ uint8_t ov_db_done:1; /* for individual JSON */
+
+} saiw_scheduled_t;
+
+struct pss {
+ struct vhd *vhd;
+ struct lws *wsi;
+
+ struct lws_spa *spa;
+ struct lejp_ctx ctx;
+ sai_notification_t sn;
+ struct lws_dll2 same; /* owner: vhd.browsers */
+
+ struct lws_dll2 subs_list;
+
+ uint64_t sub_timestamp;
+ char sub_task_uuid[65];
+ char specific_ref[65];
+ char specific_task[65];
+ char specific_project[96];
+ char auth_user[33];
+
+ sqlite3 *pdb_artifact;
+ sqlite3_blob *blob_artifact;
+
+ lws_dll2_owner_t platform_owner; /* sai_platform_t builder offers */
+ lws_dll2_owner_t task_cancel_owner; /* sai_platform_t builder offers */
+ lws_dll2_owner_t logs_owner;
+ lws_struct_args_t a;
+
+ union {
+ sai_plat_t *b;
+ sai_plat_owner_t *o;
+ } u;
+ const char *server_name;
+
+ lws_dll2_owner_t sched; /* scheduled messages */
+
+ struct lwsac *logs_ac;
+ lws_dll2_owner_t issue_task_owner; /* list of sai_task_t */
+
+ int log_cache_index;
+ int log_cache_size;
+ int authorized;
+ int specificity;
+ unsigned long expiry_unix_time;
+
+ /* notification hmac information */
+ char notification_sig[128];
+ char alang[128];
+ struct lws_genhmac_ctx hmac;
+ enum lws_genhmac_types hmac_type;
+ char our_form;
+ char login_form;
+
+ uint64_t first_log_timestamp;
+ uint64_t artifact_offset;
+ uint64_t artifact_length;
+
+ ws_state send_state;
+
+ unsigned int spa_failed:1;
+ unsigned int dry:1;
+ unsigned int frag:1;
+ unsigned int mark_started:1;
+ unsigned int wants_event_updates:1;
+ unsigned int announced:1;
+ unsigned int bulk_binary_data:1;
+ unsigned int toggle_favour_sch:1;
+};
+
+typedef struct sais_sqlite_cache {
+ lws_dll2_t list;
+ char uuid[65];
+ sqlite3 *pdb;
+ lws_usec_t idle_since;
+ int refcount;
+} sais_sqlite_cache_t;
+
+struct vhd {
+ struct lws_context *context;
+ struct lws_vhost *vhost;
+
+ /* pss lists */
+ struct lws_dll2_owner browsers;
+
+ struct lws_dll2_owner *builders_owner;
+ struct lwsac *builders;
+
+ /* our keys */
+ struct lws_jwk jwt_jwk_auth;
+ char jwt_auth_alg[16];
+ const char *jwt_issuer;
+ const char *jwt_audience;
+
+ lws_dll2_owner_t web_to_srv_owner;
+ lws_dll2_owner_t subs_owner;
+ sqlite3 *pdb;
+ sqlite3 *pdb_auth;
+
+ struct lws_ss_handle *h_ss_websrv; /* client */
+
+ const char *sqlite3_path_lhs;
+
+ lws_dll2_owner_t sqlite3_cache; /* sais_sqlite_cache_t */
+ lws_dll2_owner_t tasklog_cache;
+ lws_sorted_usec_list_t sul_logcache;
+ lws_sorted_usec_list_t sul_central; /* background task allocation sul */
+};
+
+extern struct lws_context *
+sai_lws_context_from_json(const char *config_dir,
+ struct lws_context_creation_info *info,
+ const struct lws_protocols **pprotocols,
+ const char *pol);
+extern const struct lws_protocols protocol_ws;
+extern const lws_ss_info_t ssi_saiw_websrv;
+
+int
+sai_notification_file_upload_cb(void *data, const char *name,
+ const char *filename, char *buf, int len,
+ enum lws_spa_fileupload_states state);
+
+int
+sai_sqlite3_statement(sqlite3 *pdb, const char *cmd, const char *desc);
+
+int
+sai_uuid16_create(struct lws_context *context, char *dest33);
+
+int
+sais_event_db_ensure_open(struct vhd *vhd, const char *event_uuid, char can_create, sqlite3 **ppdb);
+
+void
+sais_event_db_close(struct vhd *vhd, sqlite3 **ppdb);
+
+int
+sais_event_db_delete_database(struct vhd *vhd, const char *event_uuid);
+
+int
+sai_sq3_event_lookup(sqlite3 *pdb, uint64_t start, lws_struct_args_cb cb, void *ca);
+
+int
+sai_sql3_get_uint64_cb(void *user, int cols, char **values, char **name);
+
+int
+saiw_ws_json_tx_browser(struct vhd *vhd, struct pss *pss, uint8_t *buf, size_t bl);
+
+int
+lws_struct_map_set(const lws_struct_map_t *map, char *u);
+
+int
+saiw_ws_json_rx_browser(struct vhd *vhd, struct pss *pss,
+ uint8_t *buf, size_t bl);
+
+void
+sai_task_uuid_to_event_uuid(char *event_uuid33, const char *task_uuid65);
+
+int
+sais_ws_json_tx_builder(struct vhd *vhd, struct pss *pss, uint8_t *buf, size_t bl);
+
+int
+saiw_subs_request_writeable(struct vhd *vhd, const char *task_uuid);
+
+int
+saiw_event_state_change(struct vhd *vhd, const char *event_uuid);
+
+int
+saiw_subs_task_state_change(struct vhd *vhd, const char *task_uuid);
+
+void
+saiw_central_cb(lws_sorted_usec_list_t *sul);
+
+int
+saiw_task_cancel(struct vhd *vhd, const char *task_uuid);
+
+int
+saiw_websrv_queue_tx(struct lws_ss_handle *h, void *buf, size_t len);
+
+int
+saiw_get_blob(struct vhd *vhd, const char *url, sqlite3 **pdb,
+ sqlite3_blob **blob, uint64_t *length);
+
+int
+saiw_browsers_task_state_change(struct vhd *vhd, const char *task_uuid);
+
+saiw_scheduled_t *
+saiw_alloc_sched(struct pss *pss, ws_state action);
+
+void
+saiw_dealloc_sched(saiw_scheduled_t *sch);
+
+int
+saiw_sched_destroy(struct lws_dll2 *d, void *user);
+
diff --git a/src/web/w-sai.c b/src/web/w-sai.c
new file mode 100644
index 0000000..cdcfddb
--- /dev/null
+++ b/src/web/w-sai.c
@@ -0,0 +1,97 @@
+/*
+ * Sai web
+ *
+ * Copyright (C) 2019 - 2020 Andy Green <andy@warmcat.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation:
+ * version 2.1 of the License.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301 USA
+ */
+
+#include <libwebsockets.h>
+#include <string.h>
+#include <signal.h>
+#include <time.h>
+
+#include "w-private.h"
+
+static int interrupted;
+struct lws_context *context;
+
+static const char * const default_ss_policy =
+ "{"
+ "\"retry\": [{"
+ "\"default\": {"
+ "\"backoff\": [1000, 2000, 3000],"
+ "\"conceal\": 5,"
+ "\"jitterpc\": 20,"
+ "\"svalidping\": 30,"
+ "\"svalidhup\": 35"
+ "}"
+ "}],"
+ "\"s\": ["
+ /*
+ * Unix Domain Socket connections to sai-server webevents
+ */
+ "{\"websrv\": {"
+ "\"endpoint\":" "\"+${sockpath}\","
+ "\"protocol\":" "\"ws\","
+ "\"metadata\": ["
+ "{\"sockpath\": \"\"}"
+ "],"
+ "\"retry\": \"default\","
+ "\"nailed_up\":" "true"
+ "}}"
+ "]}"
+;
+
+static const struct lws_protocols
+ *pprotocols[] = { &protocol_ws, NULL };
+
+static void sigint_handler(int sig)
+{
+ interrupted = 1;
+}
+
+int main(int argc, const char **argv)
+{
+ int logs = LLL_USER | LLL_ERR | LLL_WARN | LLL_NOTICE;
+ const char *p, *conf = "/etc/sai/web";
+ struct lws_context_creation_info info;
+
+ signal(SIGINT, sigint_handler);
+
+ if ((p = lws_cmdline_option(argc, argv, "-d")))
+ logs = atoi(p);
+
+ lws_set_log_level(logs, NULL);
+ lwsl_user("Sai Web - Copyright (C) 2019-2020 Andy Green <andy@warmcat.com>\n");
+
+ if ((p = lws_cmdline_option(argc, argv, "-c")))
+ conf = p;
+
+ context = sai_lws_context_from_json(conf, &info, pprotocols,
+ default_ss_policy);
+ if (!context) {
+ lwsl_err("lws init failed\n");
+ return 1;
+ }
+
+ while (!lws_service(context, 0) && !interrupted)
+ ;
+
+ lws_context_destroy(context);
+
+ return 0;
+}
diff --git a/src/web/w-websrv.c b/src/web/w-websrv.c
new file mode 100644
index 0000000..ab7bf88
--- /dev/null
+++ b/src/web/w-websrv.c
@@ -0,0 +1,253 @@
+/*
+ * Sai web websrv - saiw SS client private UDS link to sais SS server
+ *
+ * Copyright (C) 2019 - 2020 Andy Green <andy@warmcat.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation:
+ * version 2.1 of the License.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301 USA
+ *
+ * We copy JSON to heap and forward it in order to sais side.
+ */
+
+#include <libwebsockets.h>
+#include <string.h>
+#include <signal.h>
+#include <time.h>
+
+#include "w-private.h"
+
+typedef struct saiw_websrv {
+ struct lws_ss_handle *ss;
+ void *opaque_data;
+
+ lws_struct_args_t a;
+ struct lejp_ctx ctx;
+ //lws_dll2_t
+ struct lws_buflist *bltx;
+
+} saiw_websrv_t;
+
+static lws_struct_map_t lsm_websrv_evinfo[] = {
+ LSM_CARRAY (sai_browse_rx_evinfo_t, event_hash, "event_hash"),
+};
+
+static const lws_struct_map_t lsm_schema_json_map[] = {
+ LSM_SCHEMA (sai_browse_rx_evinfo_t, NULL, lsm_websrv_evinfo,
+ /* shares struct */ "sai-taskchange"),
+ LSM_SCHEMA (sai_browse_rx_evinfo_t, NULL, lsm_websrv_evinfo,
+ /* shares struct */ "sai-eventchange"),
+ LSM_SCHEMA (sai_plat_owner_t, NULL, lsm_plat_list, "sai-builders"),
+ LSM_SCHEMA (sai_browse_rx_evinfo_t, NULL, lsm_websrv_evinfo,
+ /* shares struct */ "sai-overview"),
+ LSM_SCHEMA (sai_browse_rx_evinfo_t, NULL, lsm_websrv_evinfo,
+ /* shares struct */ "sai-tasklogs"),
+};
+
+enum {
+ SAIS_WS_WEBSRV_RX_TASKCHANGE,
+ SAIS_WS_WEBSRV_RX_EVENTCHANGE,
+ SAIS_WS_WEBSRV_RX_SAI_BUILDERS,
+ SAIS_WS_WEBSRV_RX_OVERVIEW, /* deleted or added event */
+ SAIS_WS_WEBSRV_RX_TASKLOGS /* new logs for task (ratelimited) */
+};
+
+
+static int
+saiw_lp_rx(void *userobj, const uint8_t *buf, size_t len, int flags)
+{
+ saiw_websrv_t *m = (saiw_websrv_t *)userobj;
+ struct vhd *vhd = (struct vhd *)m->opaque_data;
+ sai_browse_rx_evinfo_t *ei;
+ int n;
+
+ // lwsl_user("%s: len %d, flags: %d\n", __func__, (int)len, flags);
+ // lwsl_hexdump_info(buf, len);
+
+ if (flags & LWSSS_FLAG_SOM) {
+ memset(&m->a, 0, sizeof(m->a));
+ m->a.map_st[0] = lsm_schema_json_map;
+ m->a.map_entries_st[0] = LWS_ARRAY_SIZE(lsm_schema_json_map);
+ m->a.map_entries_st[1] = LWS_ARRAY_SIZE(lsm_schema_json_map);
+ m->a.ac_block_size = 128;
+
+ lws_struct_json_init_parse(&m->ctx, NULL, &m->a);
+ }
+
+ n = lejp_parse(&m->ctx, (uint8_t *)buf, len);
+ if (n < LEJP_CONTINUE || (n >= 0 && !m->a.dest)) {
+ lwsac_free(&m->a.ac);
+ lwsl_hexdump_notice(buf, len);
+ lwsl_notice("%s: srv->web JSON decode failed '%s'\n",
+ __func__, lejp_error_to_string(n));
+ return LWSSSSRET_DISCONNECT_ME;
+ }
+
+ if (!(flags & LWSSS_FLAG_EOM))
+ return 0;
+
+// lwsl_notice("%s: schema idx %d\n", __func__, m->a.top_schema_index);
+
+ switch (m->a.top_schema_index) {
+
+ case SAIS_WS_WEBSRV_RX_TASKCHANGE:
+ ei = (sai_browse_rx_evinfo_t *)m->a.dest;
+ /* server has told us of a task change */
+ lwsl_notice("%s: TASKCHANGE %s\n", __func__, ei->event_hash);
+ saiw_browsers_task_state_change(vhd, ei->event_hash);
+ break;
+
+ case SAIS_WS_WEBSRV_RX_EVENTCHANGE:
+ ei = (sai_browse_rx_evinfo_t *)m->a.dest;
+ lwsl_notice("%s: EVENTCHANGE %s\n", __func__, ei->event_hash);
+ saiw_event_state_change(vhd, ei->event_hash);
+ break;
+
+ case SAIS_WS_WEBSRV_RX_SAI_BUILDERS:
+ lwsl_notice("%s: updated sai builder list\n", __func__);
+ if (vhd->builders)
+ lwsac_detach(&vhd->builders);
+ vhd->builders = m->a.ac;
+ m->a.ac = NULL;
+ vhd->builders_owner =
+ &((sai_plat_owner_t *)m->a.dest)->plat_owner;
+ lws_start_foreach_dll(struct lws_dll2 *, p, vhd->browsers.head) {
+ struct pss *pss = lws_container_of(p, struct pss, same);
+
+ saiw_alloc_sched(pss, WSS_PREPARE_BUILDER_SUMMARY);
+
+ } lws_end_foreach_dll(p);
+ break;
+
+ case SAIS_WS_WEBSRV_RX_OVERVIEW:
+ lwsl_notice("%s: force overview\n", __func__);
+ lws_start_foreach_dll(struct lws_dll2 *, p, vhd->browsers.head) {
+ struct pss *pss = lws_container_of(p, struct pss, same);
+
+ saiw_alloc_sched(pss, WSS_PREPARE_OVERVIEW);
+ } lws_end_foreach_dll(p);
+ break;
+
+ case SAIS_WS_WEBSRV_RX_TASKLOGS:
+ ei = (sai_browse_rx_evinfo_t *)m->a.dest;
+ /*
+ * ratelimited indication that logs for a particular task
+ * changed... for each connected browser subscribed to logs for
+ * that task, let them know
+ */
+ lws_start_foreach_dll(struct lws_dll2 *, p,
+ vhd->subs_owner.head) {
+ struct pss *pss = lws_container_of(p, struct pss,
+ subs_list);
+
+ if (!strcmp(pss->sub_task_uuid, ei->event_hash))
+ lws_callback_on_writable(pss->wsi);
+ } lws_end_foreach_dll(p);
+ break;
+ }
+
+ if (flags & LWSSS_FLAG_EOM)
+ lwsac_free(&m->a.ac);
+
+ return 0;
+}
+
+static int
+saiw_lp_tx(void *userobj, lws_ss_tx_ordinal_t ord, uint8_t *buf, size_t *len,
+ int *flags)
+{
+ saiw_websrv_t *m = (saiw_websrv_t *)userobj;
+ char som, eom;
+ int used;
+
+ if (!m->bltx)
+ return LWSSSSRET_TX_DONT_SEND;
+
+ used = lws_buflist_fragment_use(&m->bltx, buf, *len, &som, &eom);
+ if (!used)
+ return LWSSSSRET_TX_DONT_SEND;
+
+ *flags = (som ? LWSSS_FLAG_SOM : 0) | (eom ? LWSSS_FLAG_EOM : 0);
+ *len = (size_t)used;
+
+ if (m->bltx)
+ lws_ss_request_tx(m->ss);
+
+ return 0;
+}
+
+static int
+saiw_lp_state(void *userobj, void *sh, lws_ss_constate_t state,
+ lws_ss_tx_ordinal_t ack)
+{
+ saiw_websrv_t *m = (saiw_websrv_t *)userobj;
+ struct vhd *vhd = (struct vhd *)m->opaque_data;
+
+ lwsl_user("%s: %s, ord 0x%x\n", __func__, lws_ss_state_name(state),
+ (unsigned int)ack);
+
+ switch (state) {
+ case LWSSSCS_DESTROYING:
+ break;
+
+ case LWSSSCS_CONNECTED:
+ lwsl_notice("%s: connected to websrv uds\n", __func__);
+ lws_ss_request_tx(m->ss);
+ break;
+
+ case LWSSSCS_DISCONNECTED:
+ lws_buflist_destroy_all_segments(&m->bltx);
+ lwsac_detach(&vhd->builders);
+ break;
+
+ case LWSSSCS_ALL_RETRIES_FAILED:
+ lws_ss_client_connect(m->ss);
+ break;
+
+ case LWSSSCS_QOS_ACK_REMOTE:
+ break;
+
+ default:
+ break;
+ }
+
+ return 0;
+}
+
+const lws_ss_info_t ssi_saiw_websrv = {
+ .handle_offset = offsetof(saiw_websrv_t, ss),
+ .opaque_user_data_offset = offsetof(saiw_websrv_t, opaque_data),
+ .rx = saiw_lp_rx,
+ .tx = saiw_lp_tx,
+ .state = saiw_lp_state,
+ .user_alloc = sizeof(saiw_websrv_t),
+ .streamtype = "websrv"
+};
+
+/*
+ * send to server
+ */
+
+int
+saiw_websrv_queue_tx(struct lws_ss_handle *h, void *buf, size_t len)
+{
+ saiw_websrv_t *m = (saiw_websrv_t *)lws_ss_to_user_object(h);
+
+ if (lws_buflist_append_segment(&m->bltx, buf, len) < 0)
+ return 1;
+
+ lws_ss_request_tx(h);
+
+ return 0;
+}
diff --git a/src/web/w-ws-browser.c b/src/web/w-ws-browser.c
new file mode 100644
index 0000000..1db984a
--- /dev/null
+++ b/src/web/w-ws-browser.c
@@ -0,0 +1,1164 @@
+/*
+ * Sai server - ./src/server/m-ws-browser.c
+ *
+ * Copyright (C) 2019 - 2020 Andy Green <andy@warmcat.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation:
+ * version 2.1 of the License.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301 USA
+ *
+ * These are ws rx and tx handlers related to browser ws connections, on
+ * /broswe URLs.
+ */
+
+#include <libwebsockets.h>
+#include <string.h>
+#include <signal.h>
+#include <assert.h>
+#include <time.h>
+
+#include "w-private.h"
+
+/*
+ * For decoding specific event data request from browser
+ */
+
+static lws_struct_map_t lsm_browser_evinfo[] = {
+ LSM_CARRAY (sai_browse_rx_evinfo_t, event_hash, "event_hash"),
+};
+
+static lws_struct_map_t lsm_browser_taskreset[] = {
+ LSM_CARRAY (sai_browse_rx_evinfo_t, event_hash, "uuid"),
+};
+
+static lws_struct_map_t lsm_browser_taskinfo[] = {
+ LSM_CARRAY (sai_browse_rx_taskinfo_t, task_hash, "task_hash"),
+ LSM_UNSIGNED (sai_browse_rx_taskinfo_t, logs, "logs"),
+};
+
+
+/*
+ * Schema list so lws_struct can pick the right object to create based on the
+ * incoming schema name
+ */
+
+static const lws_struct_map_t lsm_schema_json_map_bwsrx[] = {
+ LSM_SCHEMA (sai_browse_rx_taskinfo_t, NULL, lsm_browser_taskinfo,
+ "com.warmcat.sai.taskinfo"),
+ LSM_SCHEMA (sai_browse_rx_evinfo_t, NULL, lsm_browser_evinfo,
+ "com.warmcat.sai.eventinfo"),
+ LSM_SCHEMA (sai_browse_rx_evinfo_t, NULL, lsm_browser_taskreset,
+ /* shares struct */ "com.warmcat.sai.taskreset"),
+ LSM_SCHEMA (sai_browse_rx_evinfo_t, NULL, lsm_browser_taskreset,
+ /* shares struct */ "com.warmcat.sai.eventreset"),
+ LSM_SCHEMA (sai_browse_rx_evinfo_t, NULL, lsm_browser_taskreset,
+ /* shares struct */ "com.warmcat.sai.eventdelete"),
+ LSM_SCHEMA (sai_cancel_t, NULL, lsm_task_cancel,
+ "com.warmcat.sai.taskcan"),
+};
+
+enum {
+ SAIM_WS_BROWSER_RX_TASKINFO,
+ SAIM_WS_BROWSER_RX_EVENTINFO,
+ SAIM_WS_BROWSER_RX_TASKRESET,
+ SAIM_WS_BROWSER_RX_EVENTRESET,
+ SAIM_WS_BROWSER_RX_EVENTDELETE,
+ SAIM_WS_BROWSER_RX_TASKCANCEL
+};
+
+
+/*
+ * For issuing combined task and event data back to browser
+ */
+
+typedef struct sai_browse_taskreply {
+ const sai_event_t *event;
+ const sai_task_t *task;
+ char auth_user[33];
+ int authorized;
+ int auth_secs;
+} sai_browse_taskreply_t;
+
+static lws_struct_map_t lsm_taskreply[] = {
+ LSM_CHILD_PTR (sai_browse_taskreply_t, event, sai_event_t, NULL,
+ lsm_event, "e"),
+ LSM_CHILD_PTR (sai_browse_taskreply_t, task, sai_task_t, NULL,
+ lsm_task, "t"),
+ LSM_CARRAY (sai_browse_taskreply_t, auth_user, "auth_user"),
+ LSM_UNSIGNED (sai_browse_taskreply_t, authorized, "authorized"),
+ LSM_UNSIGNED (sai_browse_taskreply_t, auth_secs, "auth_secs"),
+};
+
+const lws_struct_map_t lsm_schema_json_map_taskreply[] = {
+ LSM_SCHEMA (sai_browse_taskreply_t, NULL, lsm_taskreply,
+ "com.warmcat.sai.taskinfo"),
+};
+
+enum sai_overview_state {
+ SOS_EVENT,
+ SOS_TASKS,
+};
+
+int
+sai_sql3_get_uint64_cb(void *user, int cols, char **values, char **name)
+{
+ uint64_t *pui = (uint64_t *)user;
+
+ *pui = (uint64_t)atoll(values[0]);
+
+ return 0;
+}
+
+/* 1 == authorized */
+
+static int
+sais_conn_auth(struct pss *pss)
+{
+ if (!pss->authorized)
+ return 0;
+ if (pss->expiry_unix_time < (unsigned long)lws_now_secs())
+ return 0;
+
+ return 1;
+}
+
+/*
+ * Ask for writeable cb on all browser connections subscribed to a particular
+ * task (so we can send them some more logs)
+ */
+
+int
+saiw_subs_request_writeable(struct vhd *vhd, const char *task_uuid)
+{
+ lws_start_foreach_dll(struct lws_dll2 *, p,
+ vhd->subs_owner.head) {
+ struct pss *pss = lws_container_of(p, struct pss, subs_list);
+
+ if (!strcmp(pss->sub_task_uuid, task_uuid))
+ lws_callback_on_writable(pss->wsi);
+
+ } lws_end_foreach_dll(p);
+
+ return 0;
+}
+
+saiw_scheduled_t *
+saiw_alloc_sched(struct pss *pss, ws_state action)
+{
+ saiw_scheduled_t *sch = malloc(sizeof(*sch));
+
+ if (sch) {
+ memset(sch, 0, sizeof(*sch));
+ sch->action = action;
+ lws_dll2_add_tail(&sch->list, &pss->sched);
+ lws_callback_on_writable(pss->wsi);
+ }
+
+ return sch;
+}
+
+void
+saiw_dealloc_sched(saiw_scheduled_t *sch)
+{
+ if (!sch)
+ return;
+
+ lws_dll2_remove(&sch->list);
+
+ lwsac_free(&sch->ac);
+ lwsac_free(&sch->query_ac);
+
+ free(sch);
+}
+
+static int
+saiw_pss_schedule_eventinfo(struct pss *pss, const char *event_uuid)
+{
+ saiw_scheduled_t *sch = saiw_alloc_sched(pss, WSS_PREPARE_OVERVIEW);
+ char qu[80], esc[66], esc2[96];
+ int n;
+
+ if (!sch)
+ return -1;
+
+ /*
+ * This pss may be locked to a specific event
+ */
+
+ if (pss->specific_task[0] && memcmp(pss->specific_task, event_uuid, 32))
+ goto bail;
+
+ /*
+ * This pss may be locked to a specific project, qualify the db lookup
+ * vs any project name specificity.
+ *
+ * Just collect the event struct into pss->query_owner to dump
+ */
+
+ lws_sql_purify(esc, event_uuid, sizeof(esc));
+
+ if (pss->specific_project[0]) {
+ lws_sql_purify(esc2, pss->specific_project, sizeof(esc2));
+ lws_snprintf(qu, sizeof(qu), " and uuid='%s' and repo_name='%s'", esc, esc2);
+ } else
+ lws_snprintf(qu, sizeof(qu), " and uuid='%s'", esc);
+ n = lws_struct_sq3_deserialize(pss->vhd->pdb, qu, NULL,
+ lsm_schema_sq3_map_event,
+ &sch->owner, &sch->ac, 0, 1);
+ if (n < 0 || !sch->owner.head)
+ goto bail;
+
+ sch->ov_db_done = 1;
+ saiw_alloc_sched(pss, WSS_PREPARE_BUILDER_SUMMARY);
+
+ return 0;
+
+bail:
+ saiw_dealloc_sched(sch);
+ return 1;
+}
+
+static int
+saiw_pss_schedule_taskinfo(struct pss *pss, const char *task_uuid, int logsub)
+{
+ saiw_scheduled_t *sch = saiw_alloc_sched(pss, WSS_PREPARE_TASKINFO);
+ char qu[192], esc[66], event_uuid[33], esc2[96];
+ sqlite3 *pdb = NULL;
+ lws_dll2_owner_t o;
+ int n, m;
+
+ if (!sch)
+ return -1;
+
+ sai_task_uuid_to_event_uuid(event_uuid, task_uuid);
+
+ /*
+ * This pss may be locked to a specific event and not want to hear
+ * anything unrelated to that event... lock to task is same deal but
+ * we will also send it non-log info about other tasks, so it can
+ * keep its event summary alive
+ */
+
+ if (pss->specific_task[0] &&
+ memcmp(pss->specific_task, event_uuid, 32)) {
+ lwsl_info("%s: specific_task '%s' vs event_uuid '%s\n",
+ __func__, pss->specific_task, event_uuid);
+ goto bail;
+ }
+
+ /* open the event-specific database object */
+
+ if (sais_event_db_ensure_open(pss->vhd, event_uuid, 0, &pdb)) {
+ /* no longer exists, nothing to do */
+ saiw_dealloc_sched(sch);
+ return 0;
+ }
+
+ /*
+ * get the related task object into its own ac... there might
+ * be a lot of related data, so we hold the ac in the sch for
+ * as long as needed to send it out
+ */
+
+ lws_sql_purify(esc, task_uuid, sizeof(esc));
+ lws_snprintf(qu, sizeof(qu), " and uuid='%s'", esc);
+ n = lws_struct_sq3_deserialize(pdb, qu, NULL, lsm_schema_sq3_map_task,
+ &o, &sch->query_ac, 0, 1);
+ sais_event_db_close(pss->vhd, &pdb);
+ // lwsl_notice("%s: n %d, o.head %p\n", __func__, n, o.head);
+ if (n < 0 || !o.head)
+ goto bail;
+
+ sch->one_task = lws_container_of(o.head, sai_task_t, list);
+
+ lwsl_info("%s: browser ws asked for task hash: %s, plat %s\n",
+ __func__, task_uuid, sch->one_task->platform);
+
+ /* let the pss take over the task info ac and schedule sending */
+
+ lws_dll2_remove((struct lws_dll2 *)&sch->one_task->list);
+
+ /*
+ * let's also get the event object the task relates to into
+ * its own event struct, additionally qualify this task against any
+ * pss reponame-specific constraint and bail if doesn't match
+ */
+
+ lws_sql_purify(esc, event_uuid, sizeof(esc));
+ m = lws_snprintf(qu, sizeof(qu), " and uuid='%s'", esc);
+ if (pss->specific_project[0]) {
+ lws_sql_purify(esc2, pss->specific_project, sizeof(esc2));
+ m += lws_snprintf(qu + m, sizeof(qu) - m, " and repo_name='%s'", esc2);
+ }
+
+ if (pss->specific_ref[0]) {
+ lws_sql_purify(esc2, pss->specific_ref, sizeof(esc2));
+ if (pss->specific_ref[0] == 'r') {
+ /* check event ref against, eg, ref/heads/xxx */
+ if (!strcmp(pss->specific_ref, "refs/heads/master"))
+ m += lws_snprintf(qu + m, sizeof(qu) - m,
+ " and (ref='refs/heads/master' or ref='refs/heads/main')");
+ else
+ m += lws_snprintf(qu + m, sizeof(qu) - m, " and ref='%s'", esc2);
+ } else
+ /* check event hash against, eg, 12341234abcd... */
+ m += lws_snprintf(qu + m, sizeof(qu) - m, " and hash='%s'", esc2);
+ }
+
+ n = lws_struct_sq3_deserialize(pss->vhd->pdb, qu, NULL,
+ lsm_schema_sq3_map_event, &o,
+ &sch->query_ac, 0, 1);
+ if (n < 0 || !o.head) {
+ lwsl_notice("%s: no result\n", __func__);
+ goto bail;
+ }
+
+ /* does he want to subscribe to logs? */
+ if (logsub) {
+ strcpy(pss->sub_task_uuid, sch->one_task->uuid);
+ lws_dll2_add_head(&pss->subs_list, &pss->vhd->subs_owner);
+ pss->sub_timestamp = 0; /* where we got up to */
+ lws_callback_on_writable(pss->wsi);
+
+ lwsl_notice("%s: subscribed to logs for %s\n", __func__,
+ pss->sub_task_uuid);
+ }
+
+ sch->one_event = lws_container_of(o.head, sai_event_t, list);
+
+ saiw_alloc_sched(pss, WSS_PREPARE_BUILDER_SUMMARY);
+
+ return 0;
+
+bail:
+ saiw_dealloc_sched(sch);
+ return 1;
+}
+
+/*
+ * We need to schedule re-sending out task and event state to anyone subscribed
+ * to the task that changed or its associated event
+ */
+
+int
+saiw_subs_task_state_change(struct vhd *vhd, const char *task_uuid)
+{
+ lws_start_foreach_dll(struct lws_dll2 *, p,
+ vhd->subs_owner.head) {
+ struct pss *pss = lws_container_of(p, struct pss, subs_list);
+
+ if (!strcmp(pss->sub_task_uuid, task_uuid))
+ saiw_pss_schedule_taskinfo(pss, task_uuid, 0);
+
+ } lws_end_foreach_dll(p);
+
+ return 0;
+}
+
+
+int
+saiw_browsers_task_state_change(struct vhd *vhd, const char *task_uuid)
+{
+ lws_start_foreach_dll(struct lws_dll2 *, p, vhd->browsers.head) {
+ struct pss *pss = lws_container_of(p, struct pss, same);
+
+ saiw_pss_schedule_taskinfo(pss, task_uuid, 0);
+ } lws_end_foreach_dll(p);
+
+ return 0;
+}
+
+
+int
+saiw_event_state_change(struct vhd *vhd, const char *event_uuid)
+{
+ lws_start_foreach_dll(struct lws_dll2 *, p, vhd->browsers.head) {
+ struct pss *pss = lws_container_of(p, struct pss, same);
+
+ saiw_pss_schedule_eventinfo(pss, event_uuid);
+ } lws_end_foreach_dll(p);
+
+ return 0;
+}
+
+/*
+ * browser has sent us a request for either overview, or data on a specific
+ * task
+ */
+
+int
+saiw_ws_json_rx_browser(struct vhd *vhd, struct pss *pss, uint8_t *buf,
+ size_t bl)
+{
+ sai_browse_rx_taskinfo_t *ti;
+ sai_browse_rx_evinfo_t *ei;
+ lws_struct_args_t a;
+ sai_cancel_t *can;
+ int m, ret = -1;
+
+ memset(&a, 0, sizeof(a));
+ a.map_st[0] = lsm_schema_json_map_bwsrx;
+ a.map_entries_st[0] = LWS_ARRAY_SIZE(lsm_schema_json_map_bwsrx);
+ a.map_entries_st[1] = LWS_ARRAY_SIZE(lsm_schema_json_map_bwsrx);
+ a.ac_block_size = 128;
+
+ lws_struct_json_init_parse(&pss->ctx, NULL, &a);
+ m = lejp_parse(&pss->ctx, (uint8_t *)buf, bl);
+ if (m < 0 || !a.dest) {
+ lwsl_hexdump_notice(buf, bl);
+ lwsl_notice("%s: browser->web JSON decode failed '%s'\n",
+ __func__, lejp_error_to_string(m));
+ return m;
+ }
+
+ /*
+ * Which object we ended up with depends on the schema that came in...
+ * a.top_schema_index is the index in lsm_schema_json_map_bwsrx it
+ * matched on
+ */
+
+ switch (a.top_schema_index) {
+
+ case SAIM_WS_BROWSER_RX_TASKINFO:
+ ti = (sai_browse_rx_taskinfo_t *)a.dest;
+
+ lwsl_info("%s: schema index %d, task hash %s\n", __func__,
+ a.top_schema_index, ti->task_hash);
+
+ if (!ti->task_hash[0]) {
+ /*
+ * he's asking for the overview schema
+ */
+
+ saiw_alloc_sched(pss, WSS_PREPARE_OVERVIEW);
+ saiw_alloc_sched(pss, WSS_PREPARE_BUILDER_SUMMARY);
+ ret = 0;
+ goto bail;
+ }
+
+ /*
+ * get the related task object into its own ac... there might
+ * be a lot of related data, so we hold the ac in the pss for
+ * as long as needed to send it out
+ */
+
+ if (saiw_pss_schedule_taskinfo(pss, ti->task_hash, !!ti->logs))
+ goto soft_error;
+
+ return 0;
+
+ case SAIM_WS_BROWSER_RX_EVENTINFO:
+
+ ei = (sai_browse_rx_evinfo_t *)a.dest;
+
+ if (saiw_pss_schedule_eventinfo(pss, ei->event_hash))
+ goto soft_error;
+
+ lwsac_free(&a.ac);
+
+ return 0;
+
+ case SAIM_WS_BROWSER_RX_TASKRESET:
+
+ if (!sais_conn_auth(pss))
+ goto soft_error;
+
+ /*
+ * User is asking us to reset / rebuild this task
+ */
+
+ ei = (sai_browse_rx_evinfo_t *)a.dest;
+
+ saiw_websrv_queue_tx(vhd->h_ss_websrv, buf, bl);
+ lwsac_free(&a.ac);
+
+ return 0;
+
+ case SAIM_WS_BROWSER_RX_EVENTRESET:
+
+ if (!sais_conn_auth(pss))
+ goto soft_error;
+
+ /*
+ * User is asking us to reset / rebuild every task in the event
+ */
+
+ ei = (sai_browse_rx_evinfo_t *)a.dest;
+
+ lwsl_notice("%s: received request to reset event %s\n",
+ __func__, ei->event_hash);
+
+ saiw_websrv_queue_tx(vhd->h_ss_websrv, buf, bl);
+ lwsac_free(&a.ac);
+
+ return 0;
+
+ case SAIM_WS_BROWSER_RX_EVENTDELETE:
+ /*
+ * User is asking us to delete the whole event
+ */
+
+ if (!sais_conn_auth(pss))
+ goto soft_error;
+
+ ei = (sai_browse_rx_evinfo_t *)a.dest;
+
+ lwsl_notice("%s: received request to delete event %s\n",
+ __func__, ei->event_hash);
+
+ saiw_websrv_queue_tx(vhd->h_ss_websrv, buf, bl);
+ lwsac_free(&a.ac);
+
+ goto bail;
+
+ case SAIM_WS_BROWSER_RX_TASKCANCEL:
+
+ if (!sais_conn_auth(pss))
+ goto soft_error;
+
+ /*
+ * Browser is informing us of task's STOP button clicked, we
+ * need to inform any builder that might be building it
+ */
+ can = (sai_cancel_t *)a.dest;
+
+ lwsl_notice("%s: received request to cancel task %s\n",
+ __func__, can->task_uuid);
+
+ saiw_task_cancel(vhd, can->task_uuid);
+ break;
+
+ default:
+ assert(0);
+ break;
+ }
+
+ ret = 0;
+
+bail:
+ lwsac_free(&a.ac);
+
+ return ret;
+
+soft_error:
+ lwsac_free(&a.ac);
+
+ return 0;
+}
+
+
+/*
+ * We're sending something on a browser ws connection. Returning nonzero from
+ * here drops the connection, necessary if we fail partway through a message
+ * but undesirable if a browser tab will keep reconnecting and asking for the
+ * same, no-longer-existant thing.
+ */
+
+int
+saiw_ws_json_tx_browser(struct vhd *vhd, struct pss *pss, uint8_t *buf, size_t bl)
+{
+ uint8_t *start = buf + LWS_PRE, *p = start, *end = p + bl - LWS_PRE - 1;
+ int n, flags = LWS_WRITE_TEXT, first = 0, iu, endo;
+ char esc[256], esc1[33], filt[128];
+ sai_browse_taskreply_t task_reply;
+ struct lwsac *task_ac = NULL;
+ lws_dll2_owner_t task_owner;
+ lws_struct_serialize_t *js;
+ saiw_scheduled_t *sch;
+ char event_uuid[33];
+ sqlite3 *pdb = NULL;
+ sai_event_t *e;
+ sai_task_t *t;
+ char any, lg;
+ size_t w;
+
+again:
+
+ start = buf + LWS_PRE;
+ p = start;
+ end = p + bl - LWS_PRE - 1;
+ flags = LWS_WRITE_TEXT;
+ first = 0;
+ lg = 0;
+ endo = 0;
+
+ // lwsl_notice("%s: send_state %d, pss %p, wsi %p\n", __func__,
+ // pss->send_state, pss, pss->wsi);
+
+ if (pss->sched.count)
+ sch = lws_container_of(pss->sched.head, saiw_scheduled_t, list);
+ else
+ sch = NULL;
+
+ switch (pss->send_state) {
+ case WSS_IDLE:
+
+ /*
+ * Anything from a task log he's subscribed to?
+ *
+ * If so, let's prioritize that first...
+ */
+
+ if ((!pss->sched.count || !pss->toggle_favour_sch) &&
+ pss->subs_list.owner) {
+
+ sch = NULL;
+
+ lws_snprintf(esc, sizeof(esc),
+ "and task_uuid='%s' and timestamp > %llu",
+ pss->sub_task_uuid,
+ (unsigned long long)pss->sub_timestamp);
+
+ /*
+ * For efficiency, let's try to grab the next 100 at
+ * once from sqlite and work our way through sending
+ * them
+ */
+
+ if (pss->log_cache_index == pss->log_cache_size) {
+ int sr;
+
+ sai_task_uuid_to_event_uuid(event_uuid,
+ pss->sub_task_uuid);
+
+ lws_dll2_owner_clear(&task_owner);
+ lwsac_free(&pss->logs_ac);
+
+ lwsl_info("%s: collecting logs %s\n",
+ __func__, esc);
+
+ if (sais_event_db_ensure_open(vhd, event_uuid, 0,
+ &pdb)) {
+ lwsl_notice("%s: unable to open event-specific database\n",
+ __func__);
+
+ return 0;
+ }
+
+ sr = lws_struct_sq3_deserialize(pdb, esc,
+ "uid,timestamp ",
+ lsm_schema_sq3_map_log,
+ &pss->logs_owner,
+ &pss->logs_ac, 0, 100);
+
+ sais_event_db_close(vhd, &pdb);
+
+ if (sr) {
+
+ lwsl_err("%s: subs failed\n", __func__);
+
+ return 0;
+ }
+
+ pss->log_cache_index = 0;
+ pss->log_cache_size = pss->logs_owner.count;
+ }
+
+ if (pss->log_cache_index < pss->log_cache_size) {
+ sai_log_t *log = lws_container_of(
+ pss->logs_owner.head,
+ sai_log_t, list);
+
+ lws_dll2_remove(&log->list);
+ pss->log_cache_index++;
+
+ /*
+ * Turn it back into JSON so we can give it to
+ * the browser
+ */
+
+ js = lws_struct_json_serialize_create(
+ lsm_schema_json_map_log, 1, 0, log);
+ if (!js) {
+ lwsl_notice("%s: json ser fail\n", __func__);
+ return 0;
+ }
+
+ n = lws_struct_json_serialize(js, p, end - p, &w);
+ lws_struct_json_serialize_destroy(&js);
+ if (n == LSJS_RESULT_ERROR) {
+ lwsl_notice("%s: json ser error\n", __func__);
+ return 0;
+ }
+
+ p += w;
+ first = 1;
+ lg = 1;
+ pss->toggle_favour_sch = 1;
+
+ /*
+ * Record that this was the most recent log we
+ * saw so far
+ */
+ pss->sub_timestamp = log->timestamp;
+ goto send_it;
+ }
+ }
+
+ /*
+ * ...then do we have anything on the scheduled ll for this pss?
+ */
+
+ if (!sch)
+ /* ... nope... */
+ return 0;
+
+ pss->toggle_favour_sch = 0;
+ pss->send_state = sch->action;
+ goto again;
+
+ case WSS_PREPARE_OVERVIEW:
+
+ filt[0] = '\0';
+ if (pss->specific_project[0]) {
+ lws_sql_purify(esc, pss->specific_project, sizeof(esc) - 1);
+ lws_snprintf(filt, sizeof(filt), " and repo_name=\"%s\"", esc);
+ }
+ pss->wants_event_updates = 1;
+ if (!sch->ov_db_done && lws_struct_sq3_deserialize(vhd->pdb,
+ filt[0] ? filt : NULL, "created ",
+ lsm_schema_sq3_map_event, &sch->owner,
+ &sch->ac, 0, -8)) {
+ lwsl_notice("%s: OVERVIEW 2 failed\n", __func__);
+
+ pss->send_state = WSS_IDLE;
+ saiw_dealloc_sched(sch);
+
+ return 0;
+ }
+
+ /*
+ * we get zero or more sai_event_t laid out in pss->query_ac,
+ * and listed in pss->query_owner
+ */
+
+ lwsl_debug("%s: WSS_PREPARE_OVERVIEW: %d results %p\n",
+ __func__, sch->owner.count, sch->ac);
+
+ p += lws_snprintf((char *)p, end - p,
+ "{\"schema\":\"sai.warmcat.com.overview\","
+ " \"alang\":\"%s\","
+ " \"authorized\": %d,"
+ " \"auth_secs\": %ld,"
+ " \"auth_user\": \"%s\","
+ "\"overview\":[",
+ lws_json_purify(esc, pss->alang, sizeof(esc) - 1, &iu),
+ pss->authorized, pss->authorized ? pss->expiry_unix_time - lws_now_secs() : 0,
+ lws_json_purify(esc1, pss->auth_user, sizeof(esc1) - 1, &iu)
+ );
+
+ /*
+ * "authorized" here is used to decide whether to render the
+ * additional controls clientside. The events the controls
+ * cause if used are separately checked for coming from an
+ * authorized pss when they are received.
+ */
+
+ if (pss->specificity)
+ sch->walk = lws_dll2_get_head(&sch->owner);
+ else
+ sch->walk = lws_dll2_get_tail(&sch->owner);
+ sch->subsequent = 0;
+ first = 1;
+
+ pss->send_state = WSS_SEND_OVERVIEW;
+
+ if (!sch->owner.count)
+ goto so_finish;
+
+ /* fallthru */
+
+ case WSS_SEND_OVERVIEW:
+
+ if (sch->ovstate == SOS_TASKS)
+ goto enum_tasks;
+
+ any = 0;
+ while (end - p > 2048 && sch->walk &&
+ pss->send_state == WSS_SEND_OVERVIEW) {
+
+ e = lws_container_of(sch->walk, sai_event_t, list);
+
+ if (pss->specificity) {
+ lwsl_debug("%s: Specificity: e->hash: %s, "
+ "e->ref: '%s', pss->specific_ref: '%s'\n",
+ __func__, e->hash, e->ref,
+ pss->specific_ref);
+
+ if (!strcmp(pss->specific_ref, "refs/heads/master") &&
+ !strcmp(e->ref, "refs/heads/main")) {
+ // lwsl_notice("master->main\n");
+ any = 1;
+ } else {
+
+ if (strcmp(e->hash, pss->specific_ref) &&
+ strcmp(e->ref, pss->specific_ref)) {
+ sch->walk = sch->walk->next;
+ continue;
+ }
+ //lwsl_notice("%s: match\n", __func__);
+ any = 1;
+ }
+ }
+
+ js = lws_struct_json_serialize_create(
+ lsm_schema_json_map_event,
+ LWS_ARRAY_SIZE(lsm_schema_json_map_event), 0, e);
+ if (!js) {
+ lwsl_err("%s: json ser fail\n", __func__);
+ return 1;
+ }
+ if (sch->subsequent)
+ *p++ = ',';
+ sch->subsequent = 1;
+
+ p += lws_snprintf((char *)p, end - p, "{\"e\":");
+
+ n = lws_struct_json_serialize(js, p, end - p, &w);
+ lws_struct_json_serialize_destroy(&js);
+ switch (n) {
+ case LSJS_RESULT_ERROR:
+ pss->send_state = WSS_IDLE;
+ saiw_dealloc_sched(sch);
+ lwsl_err("%s: json ser error\n", __func__);
+ return 1;
+
+ case LSJS_RESULT_FINISH:
+ case LSJS_RESULT_CONTINUE:
+ p += w;
+ sch->ovstate = SOS_TASKS;
+ sch->task_index = 0;
+ p += lws_snprintf((char *)p, end - p, ", \"t\":[");
+ goto enum_tasks;
+ }
+ }
+ if (!any) {
+ pss->send_state = WSS_IDLE;
+ saiw_dealloc_sched(sch);
+
+ return 0;
+ }
+ break;
+
+enum_tasks:
+ /*
+ * Enumerate the tasks associated with this event... we will
+ * come back here as often as needed to dump all the tasks
+ */
+
+ e = lws_container_of(sch->walk, sai_event_t, list);
+
+ do {
+ task_ac = NULL;
+
+ if (sais_event_db_ensure_open(vhd, e->uuid, 0, &pdb)) {
+ lwsl_err("%s: unable to open event-specific database\n",
+ __func__);
+
+ break;
+ }
+
+ lws_dll2_owner_clear(&task_owner);
+ if (lws_struct_sq3_deserialize(pdb, NULL, NULL,
+ lsm_schema_sq3_map_task, &task_owner,
+ &task_ac, sch->task_index, 1)) {
+ lwsl_err("%s: OVERVIEW 1 failed\n", __func__);
+ sais_event_db_close(vhd, &pdb);
+
+ break;
+ }
+ sais_event_db_close(vhd, &pdb);
+
+ if (!task_owner.count)
+ break;
+
+ if (sch->task_index)
+ *p++ = ',';
+
+ /*
+ * We don't want to send everyone the artifact nonces...
+ * the up nonce is a key for uploading artifacts on to
+ * this task, it should only be stored in the server db
+ * and sent to the builder to use.
+ *
+ * The down nonce is used in generated links, but still
+ * you should have to acquire such a link via whatever
+ * auth rather than be able to cook them up yourself
+ * from knowing the task uuid.
+ */
+
+ t = (sai_task_t *)task_owner.head;
+ t->art_up_nonce[0] = '\0';
+ t->art_down_nonce[0] = '\0';
+
+ /* only one in it at a time */
+ t = lws_container_of(task_owner.head, sai_task_t, list);
+
+ js = lws_struct_json_serialize_create(
+ lsm_schema_json_map_task,
+ LWS_ARRAY_SIZE(lsm_schema_json_map_task), 0, t);
+
+ n = lws_struct_json_serialize(js, p, end - p, &w);
+ lws_struct_json_serialize_destroy(&js);
+ lwsac_free(&task_ac);
+ p += w;
+
+ sch->task_index++;
+ } while ((end - p > 2048) && task_owner.count);
+
+ if (task_owner.count)
+ /* may be more left to do */
+ break;
+
+ /* none left to do, go back up a level */
+
+ p += lws_snprintf((char *)p, end - p, "]}");
+
+ sch->ovstate = SOS_EVENT;
+ if (pss->specificity)
+ sch->walk = sch->walk->next;
+ else
+ sch->walk = sch->walk->prev;
+ if (!sch->walk || pss->specificity) {
+ while (sch->walk)
+ sch->walk = sch->walk->next;
+ goto so_finish;
+ }
+ break;
+
+so_finish:
+ p += lws_snprintf((char *)p, end - p, "]}");
+ pss->send_state = WSS_IDLE;
+ endo = 1;
+ break;
+
+ case WSS_PREPARE_BUILDER_SUMMARY:
+ p += lws_snprintf((char *)p, end - p,
+ "{\"schema\":\"com.warmcat.sai.builders\","
+ " \"alang\":\"%s\","
+ " \"authorized\":%d,"
+ " \"auth_secs\":%ld,"
+ " \"auth_user\": \"%s\","
+ " \"builders\":[",
+ lws_sql_purify(esc, pss->alang, sizeof(esc) - 1),
+ pss->authorized, pss->authorized ? pss->expiry_unix_time - lws_now_secs() : 0,
+ lws_json_purify(esc1, pss->auth_user, sizeof(esc1) - 1, &iu));
+
+ if (vhd && vhd->builders) {
+ lwsac_reference(vhd->builders);
+ sch->walk = lws_dll2_get_head(vhd->builders_owner);
+ }
+ sch->subsequent = 0;
+ pss->send_state = WSS_SEND_BUILDER_SUMMARY;
+ first = 1;
+
+ /* fallthru */
+
+ case WSS_SEND_BUILDER_SUMMARY:
+ if (!sch->walk)
+ goto b_finish;
+
+ /*
+ * We're going to send the browser some JSON about all the
+ * builders / platforms we feel are connected to us
+ */
+
+ while (end - p > 512 && sch->walk &&
+ pss->send_state == WSS_SEND_BUILDER_SUMMARY) {
+
+ sai_plat_t *b = lws_container_of(sch->walk, sai_plat_t,
+ sai_plat_list);
+
+ js = lws_struct_json_serialize_create(
+ lsm_schema_map_plat_simple,
+ LWS_ARRAY_SIZE(lsm_schema_map_plat_simple),
+ 0, b);
+ if (!js) {
+ lwsl_err("a\n");
+ lwsac_unreference(&vhd->builders);
+ return 1;
+ }
+ if (sch->subsequent)
+ *p++ = ',';
+ sch->subsequent = 1;
+
+ switch (lws_struct_json_serialize(js, p, end - p, &w)) {
+ case LSJS_RESULT_ERROR:
+ lws_struct_json_serialize_destroy(&js);
+ pss->send_state = WSS_IDLE;
+ saiw_dealloc_sched(sch);
+ return 1;
+ case LSJS_RESULT_FINISH:
+ p += w;
+ lws_struct_json_serialize_destroy(&js);
+ sch->walk = sch->walk->next;
+ if (!sch->walk)
+ goto b_finish;
+ break;
+
+ case LSJS_RESULT_CONTINUE:
+ p += w;
+ lws_struct_json_serialize_destroy(&js);
+ sch->walk = sch->walk->next;
+ if (!sch->walk)
+ goto b_finish;
+ break;
+ }
+ }
+ break;
+b_finish:
+ p += lws_snprintf((char *)p, end - p, "]}");
+ lwsac_unreference(&vhd->builders);
+ endo = 1;
+ break;
+
+
+ case WSS_PREPARE_TASKINFO:
+ /*
+ * We're sending a browser the specific task info that he
+ * asked for.
+ *
+ * We already got the task struct out of the db in .one_task
+ * (all in .query_ac)
+ */
+
+ task_reply.event = sch->one_event;
+ task_reply.task = sch->one_task;
+ task_reply.auth_secs = pss->authorized ? pss->expiry_unix_time - lws_now_secs() : 0;
+ task_reply.authorized = pss->authorized;
+ lws_strncpy(task_reply.auth_user, pss->auth_user,
+ sizeof(task_reply.auth_user));
+
+ js = lws_struct_json_serialize_create(lsm_schema_json_map_taskreply,
+ LWS_ARRAY_SIZE(lsm_schema_json_map_taskreply),
+ 0, &task_reply);
+ if (!js) {
+ lwsl_warn("%s: couldn't create\n", __func__);
+ return 1;
+ }
+
+ n = lws_struct_json_serialize(js, p, end - p, &w);
+ lws_struct_json_serialize_destroy(&js);
+
+ /*
+ * Let's also try to fetch any artifacts into pss->aft_owner...
+ * no db or no artifacts can also be a normal situation...
+ */
+
+ if (sch->one_task) {
+
+ sai_task_uuid_to_event_uuid(event_uuid,
+ sch->one_task->uuid);
+
+ lwsl_debug("%s: ---------------- event uuid '%s'\n",
+ __func__, event_uuid);
+
+ lws_dll2_owner_clear(&sch->owner);
+ if (!sais_event_db_ensure_open(vhd, event_uuid, 0, &pdb)) {
+
+ lws_snprintf(filt, sizeof(filt), " and (task_uuid == '%s')",
+ sch->one_task->uuid);
+
+ lwsl_debug("%s: ---------------- %s\n", __func__, filt);
+
+ if (lws_struct_sq3_deserialize(pdb, filt, NULL,
+ lsm_schema_sq3_map_artifact,
+ &sch->owner,
+ &sch->ac, 0, 10)) {
+ lwsl_err("%s: get afcts failed\n", __func__);
+ }
+ sais_event_db_close(vhd, &pdb);
+ }
+ }
+
+ first = 1;
+ sch->walk = NULL;
+ pss->send_state = WSS_SEND_ARTIFACT_INFO;
+ if (!sch->owner.head) {
+ lwsl_debug("%s: ---------------- no artifacts\n", __func__);
+ /* there's no artifact stuff to do */
+ endo = 1;
+ }
+ sch->one_task = NULL;
+ if (n == LSJS_RESULT_ERROR) {
+ lwsl_notice("%s: taskinfo: error generating json\n", __func__);
+ return 1;
+ }
+ p += w;
+ if (!lws_ptr_diff(p, start)) {
+ lwsl_notice("%s: taskinfo: empty json\n", __func__);
+ return 0;
+ }
+ break;
+
+ case WSS_SEND_ARTIFACT_INFO:
+ if (sch->owner.head) {
+ sai_artifact_t *aft = (sai_artifact_t *)sch->owner.head;
+
+ lws_dll2_remove(&aft->list);
+
+ /* we don't want to disclose this to browsers */
+ aft->artifact_up_nonce[0] = '\0';
+
+ js = lws_struct_json_serialize_create(lsm_schema_json_map_artifact,
+ LWS_ARRAY_SIZE(lsm_schema_json_map_artifact),
+ 0, aft);
+ if (!js) {
+ lwsl_err("%s ----------------- failed to render artifact json\n", __func__);
+ return 1;
+ }
+
+ n = lws_struct_json_serialize(js, p, end - p, &w);
+ lws_struct_json_serialize_destroy(&js);
+ if (n == LSJS_RESULT_ERROR) {
+ lwsl_notice("%s: taskinfo: ---------- error generating json\n", __func__);
+ return 1;
+ }
+ first = 1;
+ p += w;
+ lwsl_warn("%s: --------------------- %.*s\n", __func__, (int)w, start);
+ }
+
+ if (!sch->owner.head)
+ endo = 1;
+ break;
+
+ default:
+ lwsl_err("%s: pss state %d\n", __func__, pss->send_state);
+ return 0;
+ }
+
+send_it:
+ flags = lws_write_ws_flags(LWS_WRITE_TEXT, first, endo || lg || !sch->walk);
+
+ if (lg || !sch->walk || endo) {
+ pss->send_state = WSS_IDLE;
+ saiw_dealloc_sched(sch);
+ }
+
+ if (lws_write(pss->wsi, start, p - start, flags) < 0)
+ return -1;
+
+ /*
+ * We get a bad ratio of reads to write when the builder spams us
+ * with rx... we have to try to clear as much as we can in one go.
+ */
+
+// if (!lws_send_pipe_choked(pss->wsi))
+// goto again;
+
+ lws_callback_on_writable(pss->wsi);
+
+ return 0;
+}
![[]](/git/avatar/c50933ca2aa61e0fe2c43d46bb6b59cb_avatar){kind=avatar}
Andy Green <andy@warmcat.com> 2020-09-02 12:39 UTC