Author: Andy Green Date: Tue Apr 21 22:46:28 2020 +0100 sai development Also add .sai.json diff --git a/.sai.json b/.sai.json new file mode 100644 index 0000000..e284804 --- /dev/null +++ b/.sai.json @@ -0,0 +1,37 @@ +{ + "schema": "sai-1", + + "platforms": { + "linux-ubuntu-1804/x86_64-amd/gcc": { + "build": "mkdir build destdir;cd build;export CCACHE_DISABLE=1;cmake .. ${cmake};make -j4;cpack -G DEB" + }, + "linux-ubuntu-2004/x86_64-amd/gcc": { + "build": "mkdir build destdir;cd build;export CCACHE_DISABLE=1;cmake .. ${cmake};make -j;cpack -G DEB" + }, + "linux-fedora-32/x86_64-amd/gcc": { + "build": "mkdir build destdir;cd build;export CCACHE_DISABLE=1;cmake .. ${cmake};make -j;cpack -G RPM" + }, + "netbsd-OSX-catalina/x86_64-intel-i3/llvm": { + "build": "mkdir build destdir; cd build; export SAI_CPACK=\"-G ZIP\";export MACOSX_DEPLOYMENT_TARGET=10.14 ; cmake .. -DCMAKE_MAKE_PROGRAM=/usr/bin/make -DLWS_OPENSSL_INCLUDE_DIRS=/usr/local/opt/openssl@1.1/include -DLWS_OPENSSL_LIBRARIES=\"/usr/local/opt/openssl/lib/libssl.dylib;/usr/local/opt/openssl/lib/libcrypto.dylib\" ${cmake} && make -j && make -j DESTDIR=../destdir install && ctest -j4 --output-on-failure ${cpack}", + "default": false + }, + "windows-10/x86_64-amd/msvc": { + "build": "mkdir build && cd build && set SAI_CPACK=\"-G ZIP\" && cmake .. -DSAI_LWS_INC_PATH=\"C:\\Program Files (x86)\\libwebsockets\\include\" -DSAI_LWS_LIB_PATH=\"C:\\Program Files (x86)\\libwebsockets\\lib\\websockets_static.lib\" -DSAI_EXT_PTHREAD_INCLUDE_DIR=\"C:\\Program Files (x86)\\pthreads\\include\" -DSAI_EXT_PTHREAD_LIBRARIES=\"C:\\Program Files (x86)\\pthreads\\lib\\x64\\libpthreadGC2.a\" -DLWS_OPENSSL_INCLUDE_DIRS=\"C:\\Program Files\\OpenSSL\\include\" -DSAI_GIT2_INC_PATH=\"C:\\Program Files (x86)\\libgit2\\include\" -DSAI_GIT2_LIB_PATH=\"C:\\Program Files (x86)\\libgit2\\lib\\git2.lib\" -DLWS_OPENSSL_LIBRARIES=\"C:\\Program Files\\OpenSSL\\lib\\libcrypto.lib;C:\\Program Files\\OpenSSL\\lib\\libssl.lib\" ${cmake} && cmake --build . --config DEBUG && set CTEST_OUTPUT_ON_FAILURE=1 && ctest . -C DEBUG -j4 --output-on-failure ${cpack}", + "default": false + } + }, + + "configurations": { + "master+builder": { + "cmake": "", + "cpack": "&& cpack -C DEBUG $SAI_CPACK", + "artifacts": "build/sai-*.deb, build/sai-*.rpm" + }, + "builder-only": { + "cmake": "-DSAI_MASTER=0 ", + "platforms": "windows-10/x86_64-amd/msvc, netbsd-OSX-catalina/x86_64-intel-i3/llvm", + "cpack": "&& cpack -C DEBUG $SAI_CPACK", + "artifacts": "build/sai-*.deb, build/sai-*.rpm, build/sai-*.7z" + } + } +} diff --git a/CMakeLists.txt b/CMakeLists.txt index b46a3bc..3b08ae7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,26 +1,117 @@ +project(sai C) cmake_minimum_required(VERSION 2.8) include(CheckCSourceCompiles) -set(SRCS_M - src/master/sai-master.c - src/master/conf.c - src/master/ws.c -) -set(SRCS_B - src/builder/sai-builder.c - src/builder/conf.c - src/builder/protocol.c -) +set(CMAKE_SKIP_RPATH 1) + +find_package(Git) +if(GIT_EXECUTABLE) + execute_process( + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" + COMMAND "${GIT_EXECUTABLE}" describe --tags --always + OUTPUT_VARIABLE GIT_HASH + OUTPUT_STRIP_TRAILING_WHITESPACE + ) +endif() + + +set(PACKAGE "sai") +set(CPACK_PACKAGE_NAME "${PACKAGE}") +set(CPACK_PACKAGE_VERSION_MAJOR "0") +set(CPACK_PACKAGE_VERSION_MINOR "1") +set(CPACK_PACKAGE_VERSION_PATCH "99") +set(CPACK_PACKAGE_RELEASE 1) +if (WIN32 OR MSVC) + set(CPACK_GENERATOR "7Z") +else() + set(CPACK_GENERATOR "RPM") +endif() +set(CPACK_PACKAGE_VERSION "${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_MINOR}.${CPACK_PACKAGE_VERSION_PATCH}") +set(LWS_BUILD_HASH ${CPACK_PACKAGE_VERSION}-${GIT_HASH}) +message("Git commit hash: ${LWS_BUILD_HASH}") + +set(CPACK_PACKAGE_VENDOR "andy@warmcat.com") +set(CPACK_PACKAGE_CONTACT "andy@warmcat.com") +set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "${PACKAGE} ${PACKAGE_VERSION}") +set(CPACK_RPM_COMPONENT_INSTALL ON) +set(CPACK_RPM_PACKAGE_RELEASE_DIST ON) +set(CPACK_RPM_FILE_NAME "RPM-DEFAULT") +set(CPACK_PACKAGE_DESCRIPTION_FILE "${PROJECT_SOURCE_DIR}/cmake/Description") + +#set(CPACK_DEBIAN_PACKAGE_RELEASE_DIST 1) +set(CPACK_DEB_COMPONENT_INSTALL 1) +set(CPACK_DEBIAN_FILE_NAME "DEB-DEFAULT") +set(CPACK_DEBIAN_PACKAGE_COMPONENT 1) +set(CPACK_DEBIAN_PACKAGE_SHLIBDEPS ON) +set(CPACK_DEBIAN_MASTER_PACKAGE_NAME "sai-master") +set(CPACK_DEBIAN_BUILDER_PACKAGE_NAME "sai-builder") + +option(SAI_MASTER "Build the master / server part" ON) +option(SAI_BUILDER "Build the builder / client part" ON) + +set(LWS_OPENSSL_LIBRARIES CACHE PATH "Path to the OpenSSL library") +set(LWS_OPENSSL_INCLUDE_DIRS CACHE PATH "Path to the OpenSSL include directory") + +if (NOT SAI_MASTER AND NOT SAI_BUILDER) + message(FATAL_ERROR "Have to build both or one or the other") +endif() set(LIB_DIR lib CACHE PATH "Install dir for libraries") set(BIN_DIR bin CACHE PATH "Install dir for executables") set(INCLUDE_DIR include CACHE PATH "Install dir for header files") set(DATA_DIR share CACHE PATH "Install dir for data files") +set(SAI_LWS_INC_PATH CACHE PATH "Path to lws headers dir") +set(SAI_LWS_LIB_PATH CACHE PATH "Path to lws library dir") +set(SAI_EXT_PTHREAD_INCLUDE_DIR CACHE PATH "Path to an external pthreads include directory") +set(SAI_EXT_PTHREAD_LIBRARIES CACHE PATH "Path to an external pthreads library") + +if (NOT SAI_LWS_INC_PATH) + find_path(XSAI_LWS_INC_PATH "libwebsockets.h") + set(SAI_LWS_INC_PATH ${XSAI_LWS_INC_PATH}) + if (NOT XSAI_LWS_INC_PATH) + message(FATAL_ERROR "Can't find lws headers") + endif() +endif() +if (NOT SAI_LWS_LIB_PATH) + find_library(XSAI_LWS_LIB_PATH NAMES "websockets") + set(SAI_LWS_LIB_PATH ${XSAI_LWS_LIB_PATH}) + if (NOT SAI_LWS_LIB_PATH) + message(FATAL_ERROR "Can't find lws lib") + endif() +endif() + +if (SAI_EXT_PTHREAD_INCLUDE_DIR) + set(CMAKE_REQUIRED_INCLUDES ${CMAKE_REQUIRED_INCLUDES};${SAI_EXT_PTHREAD_INCLUDE_DIR}) + include_directories(${SAI_EXT_PTHREAD_INCLUDE_DIR}) + + set(CMAKE_REQUIRED_FLAGS ${CMAKE_REQUIRED_FLAGS} " -DHAVE_STRUCT_TIMESPEC=1") +endif() + +message("LWS: header path ${SAI_LWS_INC_PATH}, lib path ${SAI_LWS_LIB_PATH}") + # # let the code know where the assets were installed # -set(CMAKE_C_FLAGS "-D LWS_DATA_DIR=\\\"${CMAKE_INSTALL_PREFIX}/${DATA_DIR}\\\" ${CMAKE_C_FLAGS}") +if (NOT MSVC) + set(CMAKE_C_FLAGS "-pthread ${CMAKE_C_FLAGS}") +endif() + +if (APPLE) + set(CMAKE_C_FLAGS "-I/usr/local/include ${CMAKE_C_FLAGS}") + link_directories("/usr/local/lib") +endif() + +if (LWS_OPENSSL_INCLUDE_DIRS) + set(CMAKE_C_FLAGS "-I\"${LWS_OPENSSL_INCLUDE_DIRS}\" ${CMAKE_C_FLAGS}") +endif() + + +set(CMAKE_C_FLAGS "-DBUILD_INFO=\\\"${LWS_BUILD_HASH}\\\" ${CMAKE_C_FLAGS}") + +CHECK_C_SOURCE_COMPILES("#include \nint + main(void) {\n#if defined(LWS_HAVE_LIBCAP)\n return + 0;\n#else\n fail;\n#endif\n return 0;\n}\n" HAS_LIBCAP) # If we are being built as part of lws, confirm current build config supports @@ -81,61 +172,32 @@ MACRO(require_lws_config reqconfig _val result) endif() ENDMACRO() +set(CMAKE_REQUIRED_INCLUDES ${SAI_LWS_INC_PATH}) + set(requirements 1) -require_lws_config(LWS_ROLE_H1 1 requirements) -require_lws_config(LWS_ROLE_WS 1 requirements) -require_lws_config(LWS_WITHOUT_SERVER 0 requirements) -require_lws_config(LWS_WITHOUT_CLIENT 0 requirements) -require_lws_config(LWS_WITH_UNIX_SOCK 1 requirements) +require_lws_config(LWS_ROLE_H1 1 requirements) +require_lws_config(LWS_ROLE_WS 1 requirements) if (requirements) - add_executable("sai-master" ${SRCS_M}) - add_executable("sai-builder" ${SRCS_B}) - - if (CMAKE_COMPILER_IS_GNUCC OR CMAKE_COMPILER_IS_GNUCXX OR (CMAKE_C_COMPILER_ID MATCHES "Clang") OR (CMAKE_CXX_COMPILER_ID MATCHES "Clang")) + + if (CMAKE_COMPILER_IS_GNUCC OR + CMAKE_COMPILER_IS_GNUCXX OR + (CMAKE_C_COMPILER_ID MATCHES "Clang") OR + (CMAKE_CXX_COMPILER_ID MATCHES "Clang")) set(CMAKE_C_FLAGS "-Wall -Wsign-compare -Wignored-qualifiers -Wtype-limits -Wuninitialized -Werror -Wundef ${CMAKE_C_FLAGS}" ) endif() - if (websockets_shared) - target_link_libraries("sai-master" websockets_shared) - add_dependencies("sai-master" websockets_shared) - - target_link_libraries("sai-builder" websockets_shared) - add_dependencies("sai-builder" websockets_shared) - else() - target_link_libraries("sai-master" websockets) - target_link_libraries("sai-builder" websockets) + if (SAI_MASTER) + add_subdirectory(src/master) endif() + if (SAI_BUILDER) + add_subdirectory(src/builder) + if (NOT MSVC AND NOT WIN32) + add_subdirectory(src/device) + add_subdirectory(src/expect) + endif() + endif() + +endif(requirements) +include(CPack) -# -# libgit2 paths -# -find_path( GIT2_INC_PATH NAMES "git2.h") -find_library(GIT2_LIB_PATH NAMES "git2") - -if (GIT2_INC_PATH AND GIT2_LIB_PATH) - include_directories(BEFORE "${GIT2_INC_PATH}") -else() - message(FATAL_ERROR " Unable to find libgit2") -endif() - -target_link_libraries("sai-master" ${GIT2_LIB_PATH}) -target_link_libraries("sai-builder" ${GIT2_LIB_PATH}) - - install(TARGETS sai-master sai-builder - - LIBRARY DESTINATION "${LIB_DIR}${LIB_SUFFIX}" COMPONENT libraries - ARCHIVE DESTINATION "${LIB_DIR}${LIB_SUFFIX}" COMPONENT libraries - RUNTIME DESTINATION "${BIN_DIR}" COMPONENT libraries - PUBLIC_HEADER DESTINATION "${INCLUDE_DIR}" COMPONENT dev) - - install(FILES assets/index.html - assets/sai.css - assets/sai.js - assets/lws-common.js - assets/sai.svg - assets/favicon.ico - assets/strict-csp.svg - DESTINATION "${DATA_DIR}/sai/assets") - -endif() diff --git a/README.md b/README.md index 8a9b677..c1be368 100644 --- a/README.md +++ b/README.md @@ -1,61 +1,293 @@ ![Sai](./assets/sai.svg) `Sai` (pronouced like 'sigh', "Trial" in Japanese) is a very lightweight -network-aware remote CI runner. It knows how to use `systemd-nspawn` and -`overlayfs` to build remote git repos on a variety of distro versions very -cheaply. +network-aware distributed CI builder and coordinating server. It knows how to +use `systemd-nspawn` and `overlayfs` to build remote git repos on a variety of +distro versions very cheaply. On small targets, it can also build on the host rootfs directly, and can also run on an RPi3-class 'buddy' to flash and collect results (eg, over serial, or gpio) from even smaller targets. -A self-assembling constellation of Sai Builders make their own connections to a -master, who then received hook notifications, distributes work concurrently over -idle builders that have the required environment, and logs results. +A self-assembling constellation of Sai Builder clients make their own +connections to one or more Sai Masters, who then receive hook notifications, +read JSON from the project describing what set of build variations and tests it +should run on which platforms, and distributes work concurrently over idle +builders that have the required environment, and logs results at the master +accessible by a web interface. + +![sai overview](./READMEs/sai-overview.png) ## General approach - - Sai builders for various platforms and scenarios are configured and run - independently... they maintain "nailed-up" connections to the master over - wss, and announce their capabilities. The builders do not need any listening - ports or other central management this way. + - Distributed "builders" run the `sai-builder` daemon from inside the build + platform they want to provide builds for (ie, inside a systemd-nspawn or VM). + + - The `sai-builder` daemons maintain nailed-up outgoing client wss connections + to a central `sai-master`, which manages the ad-hoc collection of builders + it finds, and provides a web UI (http/2 and wss2). + +![sai git flow](./READMEs/sai-ov2.png) + + - When a git repo that wants sai tests is updated, a push hook performs a POST + notiftying the Sai master of a new push event which creates an entry in an + event sqlite3 database. (The master does not need to access the repo that + sends the git notification). The hook sends the master information about the + push and the revision of `.sai.json` from the new commit in the POST body... + the master parses that JSON to fill an sqlite3 database with tasks and + commandline options for the build on platforms mentioned in `.sai.json`. - - When a git repo being monitored is updated, a hook performs a GET notiftying - the Sai master. + - Builders typically build many variations of the same push, so they use a + local git mirror only on the builder to reduce the load on the repo that + was updated. For large trees, being able to already have previous trees + as a starting point for git dramatically reduces the time compared to a full + git fetch. - - The Sai master fetches `.sai.json` from the repo and queues jobs on connected - builders that have the platform and OS versions requested for build in - the `.sai.json`. + - The master hands out waiting tasks on connected idle builders that offer the + requested platforms, which build them concurrently. The builders may be + inside a protected network along with the repos they connect to; both the + builders and the repo only make outgoing https or wss connections to the + master. - - On the builders, result channels (like stdout, stderr) with logs and - results are first listed and then streamed back to the master and the build - proceeds. Discrete files may also be streamed to the master like this. + - On the builders, result channels (like stdout, stderr, eventually others like + `/dev/acm0`) with logs and results are streamed back to the master over a wss + link as the build proceeds, and are stored in an event-specific sqlite3 + database for scalability. + + - Builders can run CTest or other tests after the build and collect the + results (CTest has the advantages it's lightweight and crossplatform). - The master makes human readable current and historical results available - in realtime over https + in realtime over https web interface + + - One `sai-builder` daemon can be configured to offer multiple instances of + independent platform build, and multiple platform builds (eg, cross + toolchains). + +## Build flow and support for embedded + +![build flow](READMEs/sai-build-test-flow.png) + +Testing is based around CTest, it can either run on the build host inside the +container, or run on a separate embedded device. In the separate case, the flow +can include steps to flash the image that was built and to observe and drive +testing via usually USB tty devices. IO on these additional ttys is logged +separately than IO from build host subprocess stdout and stderr. + +### Sharing embedded devices on the test host + +Embedded devices are actually build host-wide assets that may be called upon +and shared by different containers and different build platforms. For +example, a cross-built flash image on Centos8 and another cross-built on +Ubuntu Bionic for the same platform may want to flash and test on the same +pool of embedded devices. Even images from different build platforms for the +same kind of device may wish to flash the same embedded device, where the +device can be flashed to completely different OSes. + +Devices may be needed by post-build actions, but they are not something +a sai-builder for a platform can "own" or manage by itself. Instead they are +requested from inside the build action by another tool built with `sai-builder`, +`sai-device`, which reads shared JSON config describing the available devices +and platforms they are appropriate for. + +Rather than reserve the device when the build is spawned, the reservation +needs to happen only when the build inside the build context has completed. +That in turn means that a different sai utility has to run at that time from +inside the build process, in order that it can set things in the already- +existing subprocess environment. + +``` +"devices": [ + { + "name": "esp32-heltec1", + "type": "esp32", + "compatible": "freertos-esp32", + "description": "ESP32 8MByte SPI flash plus display", + "ttys": [ + "/dev/serial/by-path/pci-0000:03:00.3-usb-0:2:1.0-port0" + ] + } +] +``` + +Devices are logically defined inside a separate conf file +`/etc/sai/devices/conf` on the host or vm, and containers should bind a ro +mount of the file at the same place in their /. This avoids having to maintain +a bunch of different files every time a new device is added. For platform +builders based in a VM, these have a boolean relationship with IO ports, they +either must wholly own them or are unaware of them: this means they can't +participate in sharing device pools but must be allocated their own with its +own config file inside the VM listing those. + +When the build process wants to acquire an embedded device of a particular type +for testing, it runs in the building context, eg, `sai-device esp32 ctest`. + +This waits until it can flock() all the ttys of one of the given type of +configured devices ("esp32" in the example), sets up environment vars for each +`SAI_DEVICE_`, eg, +`SAI_DEVICE_TTY0=/dev/serial/by-id/usb-Silicon_Labs_CP2102_USB_to_UART_Bridge_Controller_0001-if00-port0`, +then executes the given program (`ctest` in the example) as a child process +inside the build context. Although it has an fd on the tty so it can flock() +it, `sai-device` does not read or write on the fd itself. + +Baud rate is not considered an attribute of the tty definition but something set +for each sai-expect. + +When the child process or build process ends, the locking is undone and the +device may be acquired by another waiting `sai-device` instance in the same or +different platform build context. + +The underlying locking is done hostwide using flock() on bind mounts of +the tty devices, the other containers will observe the locking no matter who +did it. + +Availability of the device node via an environment variable means that CTest +or other scripts are able to directly write to the device. + +### Logging of device tty activity + +The `sai-builder` instance opens three local listening Unix Domain Sockets for +every platform, these accept raw data which is turned into logs on the +respective logging channel and passed up to the `sai-master` for storage and +display like the other logs. + +The paths of these "log proxy" Unix Domain Sockets are exported as environment +variables to the child build and test process as follows + +Environment Var|Ch#|Meaning|Example +---|---|---|--- +SAI_LOGPROXY|3|Build progress logging|@com.warmcat.com.saib.logproxy.warmcat_com-freertos-esp32.0 +SAI_LOGPROXY_TTY0|4|Device tty0|@com.warmcat.com.saib.logproxy.warmcat_com-freertos-esp32.0.tty0 +SAI_LOGPROXY_TTY1|5|Optional Device tty1|@com.warmcat.com.saib.logproxy.warmcat_com-freertos-esp32.0.tty1 + +Because some kinds of device share the same tty for flashing the device, at +which time nothing else must be reading from the tty, tty activity is only +captured and proxied during actual user testing by `sai-expect`, which is run at +will be the CTest script. In this way, device ttys are only monitored while +test are ongoing; however the kernel will buffer traffic that nobody has read +until the next reading consumes it. - - The Sai master node also has a builder integrated. So to get started you - can do it all on one machine. +### Sai device tty logging -## Creating a selfsigned TLS cert +Devices may have multiple ttys defined, for example a device with separate ttys +and log channels for a main cpu and a coprocessor is supported. -You can create a P-521 EC key and self-signed certificate for use with Sai -master like this: +The ttys listed on devices have their own log channel index and are timestamped +according to when they were read from the tty. In the event many channels are +"talking at once", in the web UI the different log channel content appears in +different css colours and in chunks of 100 bytes or so, which tends to keep +isolated lines of logging intact. + +## Non-Linux: use /home/sai in the main rootfs + +For OSX and other cases that doesn't support overlayfs, the same flow occurs +just in the main rootfs /home/sai instead of the overlayfs /home/sai. + +It means things can only be built in the context of the main OS, but since OSX +doesn't have different distros, which is the main use of the Linux overlayfs +feature, it's still okay. + +## Builder instances + +The config JSON for sai-builder can specify the number of build instances for +each platform. These instances do not have any relationship about what they +are building, just they run in the same platform context (and are managed by +the one `sai-builder` process). They each check out their own build tree +independently, so they can be engaged building different versions or different +trees concurrently inside the platform. + +Tests have to take care to disambiguate which instance they are running on, +since the network namespace is shared between instances that are running in the +same sai-builder process on the same platform. An environment var +`SAI_INSTANCE_IDX` is available inside the each build context set to 0, 1, etc +according to the builder instance. + +For network related tests, `SAI_INSTANCE_IDX` should be referred to when +choosing, eg, a test server port so it will not conflict with what other +builders may be doing in parallel. + +## Builder git caching + +For each ``, the builder maintains a local git cache. This is +updated once when the new ref appears and then the related tests check out a +fresh image of their ref from that each time. This is very fast after the first +update, because it doesn't even involve the network but fetching from the local +filesystem. + +## systemd-nspawn support + +On Linux, it's recommended to use systemd-nspawn to provide multiple distro +environments conveniently on one machine. There are instructions for setting +up individual virtual ethernet devices managed by nmcli on the host. + +`sai-builder` also supports running inside a KVM / QEMU VM transparently as well, +eg for windows or emulated architecture VMs. + +## `sai` builder user + +Builds happen using a user `sai` and on the builder, files are only created +down `/home/sai` or `\Users\sai`. ``` -$ openssl ecparam -name secp521r1 -genkey -param_enc explicit -out sai-selfsigned-key.pem -$ echo -e "\n\n\n\n\n\n" | openssl req -new -x509 -key sai-selfsigned-key.pem -out sai-selfsigned-cert.pem +# useradd -u883 -gnogroup sai -d/home/sai -m -r ``` -There's a helper script `sudo ./scripts/gen-selfsigned-certs.sh` to do this and -install the results in `/usr/local/share/sai`, you can override the install -directory by giving it as a commandline argument to the script. +## build filesystem layout -Or alternatively you can use your own PEM certificate and key from a trusted -certifcate vendor, which matches your domain name. + - /home/sai/ + - git-mirror/ + - `remote git url`_`project name` -- individual git mirrors + - jobs/ + - `master hostname`-`platform name`-`instance index`/ + - `project_name`/ - checkouts and builds occur in here -Self-signed is secure, but your browser will make you explicitly trust it before -it will let you use it. Afterwards, your browser will notice if the certificate -is different and untrusted for that site and warn you. Because you must -generate it after install, your selfsigned cert will be unique. +## Build steps +Building sai produces two different apps, `sai-master` and `sai-builder`. +`sai-master` is a ws and http server that coordinates activities, and +`sai-builder` is run on each machine that wants to offer build services to +one or more masters. By default, both are built, but you can use a cmake +option `-DSAI_MASTER=0` and `-DSAI_BUILDER=0` to disable one or the other. + +First you must build lws with appropriate options. + +For redhat type distros, you probably need to add /usr/local/lib to the +/etc/ld.so.conf before ldconfig can rgister the new libwebsockets.so + +``` +$ git clone https://libwebsockets.org/repo/libwebsockets +$ cd libwebsockets && mkdir build && cd build && \ + cmake .. -DLWS_UNIX_SOCK=1 -DLWS_WITH_STRUCT_JSON=1 \ + -DLWS_WITH_STRUCT_SQLITE3=1 -DLWS_WITH_GENCRYPTO=1 -DLWS_WITH_SPAWN=1 \ + -DLWS_WITH_SECURE_STREAMS=1 -DLWS_WITH_THREADPOOL=1 +$ make -j && sudo make -j install && sudo ldconfig +``` + +The actual cmake options needed depends on if you are building sai-master and / or +sai-builder. + +Feature|lws options +---|--- +either|`-DLWS_WITH_STRUCT_JSON=1` `-DLWS_WITH_SECURE_STREAMS=1` +master|`-DLWS_UNIX_SOCK=1` `-DLWS_WITH_GENCRYPTO=1` `-DLWS_WITH_STRUCT_SQLITE3=1` +builder|`-DLWS_WITH_SPAWN=1` `-DLWS_WITH_THREADPOOL=1` + +Similarly the two daemons bring in different dependencies + +Feature|dependency +---|--- +either|libwebsockets +master|libsqlite3 +builder|libgit2 pthreads + +#### Linux + +``` +$ git clone https://warmcat.com/repo/sai +$ cd sai && mkdir build && cd build && cmake .. && make && make install +$ sudo cp ../scripts/sai-builder.service /etc/systemd/system +$ sudo mkdir -p /etc/sai/builder +$ sudo cp ../scripts/builder-conf /etc/sai/builder/conf +$ sudo vim /etc/sai/builder/conf +$ sudo systemctl enable sai-builder +``` diff --git a/READMEs/README-build-windows.md b/READMEs/README-build-windows.md new file mode 100644 index 0000000..e709a07 --- /dev/null +++ b/READMEs/README-build-windows.md @@ -0,0 +1,60 @@ +# Building Sai Builder for windows + +The overall flow is + + - build lws + - build libgit2 + - build sai-builder piggybacking on pthreads built for lws + +## Building lws + +Refer to [the instructions in lws](https://libwebsockets.org/git/libwebsockets/tree/READMEs/README.build-windows.md) +for how to install `cmake`, `git` and the toolchain on windows +also used in the next steps here. + +For windows + sai-builder, the following lws cmake config is suitable + +``` +> cmake .. -DLWS_HAVE_PTHREAD_H=1 -DLWS_EXT_PTHREAD_INCLUDE_DIR="C:\Program Files (x86)\pthreads\include" -DLWS_EXT_PTHREAD_LIBRARIES="C:\Program Files (x86)\pthreads\lib\x64\libpthreadGC2.a" -DLWS_WITH_MINIMAL_EXAMPLES=1 -DLWS_WITH_THREADPOOL=1 -DLWS_UNIX_SOCK=1 -DLWS_WITH_STRUCT_JSON=1 -DLWS_WITH_SPAWN=1 -DLWS_WITH_SECURE_STREAMS=1 -DLWS_WITH_DIR=1 +``` + +## Building libgit2 + +``` +> git clone https://github.com/libgit2/libgit2.git +> cd libgit2 ; mkdir build ; cd build +> cmake .. -DBUILD_CLAR=OFF +> cmake --build . --config DEBUG +``` + +I couldn't get runas to do anything useful, I opened an Admin-privileged +cmd window and did + +``` +> cd \Users\agreen\libgit2\build +> cmake --install . --config DEBUG +> exit +``` + +## Building Sai-builder + +``` +> git clone https://warmcat.com/repo/sai +> cd sai ; mkdir build ; cd build +> cmake .. -DSAI_MASTER=0 -DSAI_LWS_INC_PATH="\Users\agreen\libwebsockets\build\include" -DSAI_LWS_LIB_PATH="\Users\agreen\libwebsockets\build\lib\Debug\websockets_static.lib" -DLWS_OPENSSL_INCLUDE_DIRS="\Program Files\OpenSSL\include" -DSAI_GIT2_INC_PATH="\Program Files (x86)\libgit2\include" -DSAI_GIT2_LIB_PATH="\Program Files (x86)\libgit2\lib\git2.lib" -DSAI_EXT_PTHREAD_INCLUDE_DIR="C:\Program Files (x86)\pthreads\include" -DLWS_OPENSSL_LIBRARIES="\Program Files\OpenSSL\lib\libcrypto.lib;\Program Files\OpenSSL\lib\libssl.lib" -DSAI_EXT_PTHREAD_LIBRARIES="C:\Program Files (x86)\pthreads\lib\x64\libpthreadGC2.a" +``` + +## Configuring Sai-builder + +``` +>net user sai /add +``` + +As administrator + +``` +> mkdir \ProgramData\sai\builder +``` + +Then create the config JSON in `\ProgramData\sai\builder\conf`, using the +platform name `windows-10`. diff --git a/READMEs/README-gitolite-hook.md b/READMEs/README-gitolite-hook.md new file mode 100644 index 0000000..bf566ea --- /dev/null +++ b/READMEs/README-gitolite-hook.md @@ -0,0 +1,50 @@ +# gitolite hook + +``` +#!/bin/sh + +if [ $(git rev-parse --is-bare-repository) = true ] +then + RN=$(basename "$PWD") +else + RN=$(basename $(readlink -nf "$PWD"/..)) +fi + +RN=${RN%.git} + +echo sai update hook $RN... +rm -f .sai.json +git show $3 -- .sai.json | patch -p1 --merge +cat .sai.json + +TF=`mktemp` + +echo "{\"schema\":\"com-warmcat-sai-notification\"," > $TF +echo " \"action\":\"repo-update\"," >> $TF +echo " \"repository\":{" >> $TF +echo " \"name\":\"$RN\"" >> $TF +echo " }," >> $TF +echo " \"nonce\":\"`dd if=/dev/urandom bs=32 count=1 | sha256sum | cut -d' ' -f1`\"," >> $TF +echo " \"ref\":\"$1\"," >> $TF +echo " \"hash\":\"$3\"" >> $TF +echo -n " \"saifile\":\"" >> $TF +# disallow any nested JSON monkey business by base64-encoding it +cat .sai.json | base64 >> $TF +echo "\"" >> $TF +echo "}" >> $TF + +HM=`cat $TF | sha256hmac -k /etc/sai/private/lws-sai-notification-token | cut -d' ' -f1` + +cat $TF +echo $HM + +curl --header "authorization: sai sha256=$HM" \ + -F"file=@$TF;type=application/json" \ + -A "sai-notifier" \ + https://warmcat.com/sai/update-hook +#curl --header "X-Sai-Signature: sha256=$HM" -F"file=@$TF;name=notification;type=application/json" -A "sai-notifier" http://127.0.0.1:4444 + +rm -f $TF + +exit 0 +``` diff --git a/READMEs/README-platform-naming.md b/READMEs/README-platform-naming.md new file mode 100644 index 0000000..7d10302 --- /dev/null +++ b/READMEs/README-platform-naming.md @@ -0,0 +1,130 @@ +# Sai platform tuples + +Platform names in sai capture the OS, arch and toolchain details in a string in +a way that allows more or less specificity for matching. + +The general format is "os/arch[/toolchain]", eg, "linux-fedora/x86_64/gcc". The +elements may contain increasingly specific details by -, eg, +"linux-fedora-32/x86_64-amd", or "freertos-espidf/xl6-esp32" + +Builders should describe their platform in full detail, applications that want +to use the platforms to perform tests should only provide the detail level they +care about. + +Instead of direct string matching between what the builders offer and the +project wants to be built on, the builders are matched from the left for both +the platform and architecture individually. So, a project wanting +"linux-fedora/x86_64" will successfully match on a builder offering +"linux-fedora-32/x86_64-amd". But a project requesting "linux-fedora-31/x86_64" +will not match it. + +**NOTE:** dots may not be used in platform tuple strings + +## OS part + +The OS class name should be first and is mandatory, eg, "linux" or "freertos". +The OS name is next, eg, "linux-fedora" or "freertos-espidf". + +Builders should go on to give more details, eg, versioning if that is relevant +for the platform, eg, "fedora-32". + +Projects using `.sai.json` to specify tests may choose how much specificity to +provide, eg, just "linux-fedora" to run on whatever fedora version the builders +provide. + +### Well-known OS strings + +OS|meaning +---|--- +linux-fedora| +linux-ubuntu| +linux-debian| +linux-gentoo| +linux-centos| +linux-android| +freertos-espidf|FreeRTOS-based espressif sdk +freertos-linkit|FreeRTOS-based Mediatek linkit SDK +netbsd-OSX| +netbsd-iOS| +windows| + + +## Architecture part + +Similarly builders should specify at least both the arch and the vendor name of +the architecture, eg, "x86_64-amd", or "x86_64-intel". Projects that don't care +can just specify "x86_64", ones that do care can control what it can build on +more specifically, eg, "x86_64-amd". + +### Well-known architecture strings + +arch|meaning +---|--- +`arm32`|32-bit arm, -m0, -m0+, -m3, -m4, -a9 etc may follow +`arm32-m4-mt7697`|Mediatek linkit chip +`aarch64`|any 64-bit arm +`aarch64-a72-bcm2711`|Raspberry Pi 4 CPU +`aarch64-a72-bcm2711-rpi4`|Raspberry Pi 4 CPU in an actual RPi4 +`x86_64`|Intel 64-bit (aka amd64) +`x86_64-amd`|Intel 64-bit implemented in an AMD chip +`x86_64-intel`|Intel 64-bit implemented in an Intel chip +`xl6`|Xtensa xl6 +`xl6-esp32`|Xtensa xl6 implemented in an ESP32 +`riscv32`|32-bit RiscV +`riscv64`|64-bit RiscV + +## Toolchain part + +On many platforms there is a choice of toolchain between, eg, gcc and llvm now, +and sometimes others like Intel compiler, mingw, etc. For that reason it has to +be treated as another "dimension" in the platform tuple. + +### Well-known toolchain strings + +toolchain|meaning +---|--- +`gcc`|OS gcc version +`llvm`|OS llvm version +`msvc`|Visual Studio toolchain +`mingw32`|Mingw compiler 32-bit +`mingw64`|Mingw compiler 64-bit + +`-cross` may be appended indicating it's a cross-build rather than targeted at +the same host. + +## Examples + +Platform|Description +---|--- +`linux-fedora/x86_64`|matches any version of fedora on x86_64 the builders provide +`linux-fedora/x86_64-amd`|as above but only on an amd builder +`linux-fedora-32`|f32 on any arch +`freertos/arm32`|any freertos-based RTOS on any arm32 +`freertos-espidf/xl6-esp32/gcc-cross`|the usual esp-idf based esp32 project +`freertos-arduino/xl6-esp32/gcc-cross`|arduino OS on esp32 +`linux-ubuntu-2004/aarch64-a72-bcm2711-rpi4/gcc`|Raspberry Pi 4 + +## Sai builder conf host names + +The `"host":` member of the builder configuration also has structure. +It indicates build host platform details and the location and virtualization +situation of the `sai-builder` instance using that conf. + +Its format is `"[,]"`; the platform specification follows the +platform description above, except the toolchain part is optionally replaced by +the hostname. + +Eg, `"linux-fedora-32/riscv-qemu,linux-fedora/x86_64-amd/noi"`. + +### Indicating virtualization + +The host part of the string can indicate its virtualization by adding +a well-known suffix to the arch part, ie, indicating that the "implementor" of +the arch "device" is qemu. + +Well-known virt name|Meaning +---|--- +-nspawn|systemd-nspawn hosted by the string to the right +-qemu|qemu instance hosted by the string to the right + +Eg, `"windows-10/x86_64-amd-qemu, linux-fedora/x86_64-amd/noi"` diff --git a/READMEs/README-sai-json.md b/READMEs/README-sai-json.md new file mode 100644 index 0000000..12a6686 --- /dev/null +++ b/READMEs/README-sai-json.md @@ -0,0 +1,96 @@ +# .sai.json format + +Projects can inform Sai of what build and test variations they want to run +on push using a `.sai.json` file in the top level dir of the project. + +The git push hook isolated this file from the pushed tree and passes it to +the sai-master along with the rest of the push notification information in +a signed JSON structure. + +## Simple .sai.json file + +``` +{ + "schema": "sai-1", + + # comments are OK + # + "platforms": { + "linux-ubuntu-xenial-amd64": { + "build": "mkdir build destdir;cd build;export CCACHE_DISABLE=1;cmake .. ${cmake};make -j;make -j DESTDIR=../destdir install && ctest -j4 --output-on-failure" + } + # you can continue to list platforms here + }, + + "configurations": { + "plain": { + "cmake": "", + } + } +} +``` + +The effect of the above is a single build using a `linux-ubuntu-xenial-amd64` +platform builder with the arguments + +``` + mkdir build destdir; + cd build; + export CCACHE_DISABLE=1; + cmake ..; + make -j; + make -j DESTDIR=../destdir install && + ctest -j4 --output-on-failure +``` + +### Platforms section and default + +In the top level `"platforms"` section you describe the different build +platforms you want to build and test on... the specified platforms must have +builders that have connected themselves to the sai-master. + +If you don't give a `"default": false` entry in the platform definition, the +platform is assumed to apply to all the configurations listed afterwards. If +you do set `"default": false` on a platform, it's taken to mean you will list +configurations you want it to apply to manually and that platform is not +automatically used otherwise. + +### Configuration section + +#### cmake + +The configuration section lists build variations, the `"cmake"` entry allows +you to give additional cmake options like `-DSOMEFEATURE=1` that together define +the build variation. These are applied additionally to the platform-specific +cmake options where the `${cmake}` token appears. + +#### platforms + +You can also give a `"platforms"` entry in each configuration specifiying which +platforms should build this configuration in a comma-separated list. If you +don't give a `"platforms"` entry on a configuration, all default platforms will +be applied. You can also list additional, non-default platform names you +want to apply, like + +``` + "platforms": "windows-10, linkit-cross, linux-debian-buster-arm32" +``` + +those will be applied on top of the defaults. Other options are to disable +given default platforms: + +``` + "platforms": "not linux-ubuntu-xenial-amd64" +``` + +or to disable all default platforms and just use the specified ones: + +``` + "platforms": "none, windows-10" +``` + +#### artifacts + +You can also give a comma-separated list of build artifacts, these are +arbitrary binary files which will be uploaded to sai-master and made available +for download over https. \ No newline at end of file diff --git a/READMEs/README-systemd-nspawn.md b/READMEs/README-systemd-nspawn.md new file mode 100644 index 0000000..1f4b11a --- /dev/null +++ b/READMEs/README-systemd-nspawn.md @@ -0,0 +1,535 @@ +# Setting up and administering systemd-nspawn containers + +Sai's approach is to run `sai-builder` from inside the environment it will do +builds from. It reaches out to the sai-master instance and git repos, and +otherwise there is no ingress into the container or VM from outside. + +So with `systemd-nspawn`, it means the containers should start on the host at +boot and run `sai-builder` inside eg from a systemd service file. You have to: + + - one-time create the containers you want to build in + - set them up with a `sai` user and minimal packageset + - build libwebsockets / sai in there + - configure `sai-builder` to run from systemd when the container starts + +Systemd only runs on Linux, so this document is about how to do all those steps +for Linux hosts and Linux guests. For Windows and OSX, the approach is similar +but using an actual machine or qemu / kvm VM. + +## Create the container on the host + +This will create, furnish and enter a container for Ubuntu Xenial (2016-era +long-term stable). Other distros are basically the same with minor changes + +![surprised](../assets/surprised.png)Restrictions! Systemd with create a veth device based off your machine name +chosen here, it's limited to 11 chars. So two machines here must not have the +same first 11 chars in their name! + +![surprised](../assets/surprised.png)Restrictions! Systemd won't let your machine name have an underscore, so no +`...x86_64`. It's OK with `-`. + +Pick your machine name, in this example `xenial-amd64`. + +### Step 1: Create the initial host config for the machine + +On the host, `mkdir /etc/systemd/nspawn` and then create in +`/etc/systemd/nspawn/xenial-amd64.nspawn` + +``` +[Exec] +Boot=no +[Network] +VirtualEthernet=no +``` + +We will come back and set both of those to `yes` after we have set the container +up manually from the inside. + +### Step 2: Create the machine + +``` +Host # cd /var/lib/machines +Host # debootstrap --arch=amd64 --variant=buildd xenial xenial-amd64 http://archive.ubuntu.com/ubuntu/ +Host # systemd-nspawn -M xenial-amd64 +Spawning container xenial-amd64 on /var/lib/machines/xenial-amd64. +Press ^] three times within 1s to kill container. +root@myhost:~# +``` + +ah that's pretty nice so far, right! We have a shell up inside the container +already. + +#### Variations: Fedora Container + +Replace the debootstrap part above with + +``` +Host # dnf -y --releasever=32 --installroot=`pwd`/fedora32-x8664 install systemd passwd dnf fedora-release +``` + +#### Variations: Debian Container + +Replace the debootstrap part above with + +``` +Host # debootstrap --arch=amd64 stable buster-amd64 +``` + +#### Variations: Centos 8 + +Create a file "centos8.repo" + +``` +[centos8-base] +name=CentOS-8-Base +baseurl=http://mirror.centos.org/centos-8/8.1.1911/BaseOS/x86_64/os +gpgcheck=1 +``` + +... then... + +``` +$ dnf -c centos8.repo --releasever=8 --disablerepo=* --enablerepo=centos8-base --installroot=/var/lib/machines/centos8 groups install 'Minimal Install' +``` + +## Step 3: Install things into the container + +While we're borrowing the host network, install things we want in the +container to configure and debug it. + +(You might think you can install these at the bootstrap step, but I found this +can make trouble both in the right deps not being brought in and it violating +the ownership of `/usr/lib/.build-id` with real `root` instead of the fake +root of the inside of the container.) + +Create the sai user and home dir + +``` +root@myhost:~# useradd -u883 -gnogroup sai -d/home/sai -m -r +``` + +For redhat type systems, use `-gnobody` instead + +### Specific to Ubuntu + +We have to enable the "universe" packageset to get normal things. + +``` +Container # apt update && apt install vim && apt edit-sources && apt update +``` + +add " universe" to the end of the line in the vim session it pops up and save it. + +### Specific to Xenial + +``` +Container # apt install sudo make git gcc cmake net-tools libssl-dev vim libc6-i386 dbus systemd-container libmbedtls-dev glib2.0-dev libevent-dev libev4 libev-dev libuv1 libuv1-dev libmount-dev libsqlite3-dev libgit2-dev inetutils-ping psmisc libsqlite-dev sqlite3 libsqlite3-0 libgit2-24 libgit2-dev libmbedcrypto0 libmbedtls10 libmbedx509-0 libmbedtls-dev libev4 libev-dev libevent-2.0 libevent-dev glib-2.0 pkg-config +``` + +### Specific to Bionic + +``` +apt install sudo make git gcc cmake net-tools libssl-dev vim libc6-i386 dbus systemd-container libmbedtls-dev libev4 libev-dev libuv1 libuv1-dev libmount-dev libsqlite0-dev libgit2-dev inetutils-ping psmisc libgit2-26 libmbedcrypto1 libmbedtls10 libmbedx509-0 libmbedtls-dev libevent-2.1-6 libevent-dev libglib2.0-0 libsqlite3-0 libsqlite3-dev pkg-config libglib2.0-dev libglib2.0-0 libdbus-1-dev libdbus-1-3 +``` + +### Specific to Focal + +``` +apt install sudo make git gcc cmake net-tools libssl-dev vim libc6-i386 dbus systemd-container libmbedtls-dev libevent-2.1-7 libev4 libev-dev libuv1 libuv1-dev libmount-dev libsqlite0-dev libgit2-dev inetutils-ping psmisc libgit2-28 libmbedcrypto3 libmbedtls12 libmbedx509-0 libmbedtls-dev libev4 libev-dev libevent-2.1-7 libevent-dev libglib2.0-0 libsqlite3-0 libsqlite3-dev pkg-config libglib2.0-dev libglib2.0-0 +``` + +### Specific to Fedora + +``` +Container # dnf install sudo make git gcc gcc-c++ cmake net-tools dhclient openssl-devel vim glibc.i686 dbus mbedtls-devel glib2-devel libevent-devel libev-devel libuv-devel libmount-devel sqlite-devel libgit2 libgit2-devel dbus systemd-container vim libpkgconf procps-ng +``` + +### Specific to Centos 8 + +Centos / RHEL 8 simply doesn't provide some -devel packages like mbedtls, libgit2... + +``` +Container # dnf install sudo make git gcc gcc-c++ cmake net-tools dhclient openssl-devel vim glibc.i686 dbus glib2-devel libevent-devel libev-devel sqlite-devel libgit2 dbus systemd-container vim libpkgconf rpm-build +``` + +### Specific to Gentoo + +``` +Container # emerge dev-db/sqlite libgit2 gcc net-libs/mbedtls dev-libs/openssl cmake libuv libevent dev-libs/glib dev-libs/libev dbus pkgconf libcap +``` + +## Step 3: Configure the container private networking + +Set up a hostname for the container by editing its `/etc/hostname`, ideally to +a name like `machinename-hostname`, eg, `xenial-myhost` if your host is `myhost`. + +Configure the container's systemd-networkd to set up the private network for +the container we will switch to next. + +In the container, create `/etc/systemd/network/50-host0.network` with the +contents + +``` +[Match] +Name=host0 +[Network] +DHCP=no +IPForward=1 +Gateway=192.168.80.1 +Address=192.168.80.9/24 +``` + +If you have multiple containers, give each one a different 192.168.xx network. +Enable systemd pieces to control setting the interface, route and dns. + +You can avoid systemd-resolved if you like, `rm /etc/resolv.conf` to remove +its symlink, and then something like `echo "nameserver 1.2.3.4" > /etc/resolv.conf` +Either way at least enable `systemd-networkd` + +``` +Container # ln -sf /run/systemd/resolve/stub-resolv.conf /etc/resolv.conf +Container # systemctl enable systemd-networkd systemd-resolved +``` + +### Specific to Centos 8 + +Centos 8 has no systemd-networkd. The base install instead has `nmcli`, you +can set up an ipv4.method=static solution that way using the same network +layout as above. + +## Step 4: Switch over to systemd management of the container + +Exit the container session. + +On the host, edit `/etc/systemd/nspawn/xenial-amd64.nspawn` we created earlier +and set both `Boot` and `VirtualEthernet` to `yes` now. + +Tell the host we want this container up when the host is up, and that we want +to start it now. + +``` +Host # systemctl enable systemd-nspawn@xenial-amd64 +Host # systemctl start systemd-nspawn@xenial-amd64 +``` + +Then, we can reopen a shell prompt in the now-running container + +``` +Host # machinectl -M xenial-amd64 shell +Connected to the local host. Press ^] three times within 1s to exit session. +[root@xenial-amd64 ~]# +``` + +You can look around with ps if you have it or systemctl status and see that it +was started inside by systemd. + +## Step 5: Configure the host-side networking + +systemd will have created a matching network interface on your host named +`ve-machinename`, where machinename is your (truncated if neccessary) +machine name. + +We need to explain to Network Manager how to deal with this so the container +will get networking services cleanly. + +We will call the related NetworkManager connection `nm-...` + +``` +Host # nmcli c add type ethernet autoconnect no con-name nm-ve-xenial ifname ve-xenial-amd64 +Host # nmcli c m nm-ve-xenial ipv4.method shared +Host # nmcli c m nm-ve-xenial ipv4.addresses 192.168.80.1/24 +Host # nmcli c m nm-ve-xenial connection.zone vm +Host # nmcli c m nm-ve-xenial connection.autoconnect true +Host # nmcli c down nm-ve-xenial ; nmcli c up nm-ve-xenial +``` + +## Step 6: Restart the container + +``` +Host # systemctl stop systemd-nspawn@xenial-amd64 +Host # systemctl start systemd-nspawn@xenial-amd64 +Host # machinectl -M xenial-amd64 shell +``` + +If you hit selinux AVCs on the start step, as I did on Fedora 32, you can get +around them with three policy additions + +``` +Host # echo 'AVC avc: denied { search } for pid=1502 comm="systemd-machine" name="21786" dev="proc" ino=22140 scontext=system_u:system_r:systemd_machined_t:s0 tcontext=system_u:system_r:unconfined_service_t:s0 tclass=dir permissive=0' | audit2allow -M machine-search ; semodule -i machine-search.pp +Host # echo 'AVC avc: denied { read } for pid=1502 comm="systemd-machine" name="cgroup" dev="proc" ino=60301 scontext=system_u:system_r:systemd_machined_t:s0 tcontext=system_u:system_r:unconfined_service_t:s0 tclass=file permissive=0' | audit2allow -M machine-read ; semodule -i machine-read.pp +Host # echo 'AVC avc: denied { open } for pid=1502 comm="systemd-machine" path="/proc/21937/cgroup" dev="proc" ino=46779 scontext=system_u:system_r:systemd_machined_t:s0 tcontext=system_u:system_r:unconfined_service_t:s0 tclass=file permissive=0' | audit2allow -M machine-open ; semodule -i machine-open.pp +``` + +In the container, you should see `host0` has ben set up + +``` +Container # ifconfig host0 +host0: flags=4163 mtu 1500 + inet 192.168.80.9 netmask 255.255.255.0 broadcast 192.168.80.255 + inet6 fe80::fc27:1eff:fe53:4d93 prefixlen 64 scopeid 0x20 + ether fe:27:1e:53:4d:93 txqueuelen 1000 (Ethernet) + RX packets 1117 bytes 102933 (100.5 KiB) + RX errors 0 dropped 0 overruns 0 frame 0 + TX packets 754 bytes 71331 (69.6 KiB) + TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0 +``` + +Whether you can ping things depends on your host routing arrangements for its +side of the virtual ethernet. If you pointed the container to your local +resolver you need to enable that in the "vm" zone, along with masquerading. +You'll only need to do this for the first vm, afterwards it enough to tell nmcli +that the connection is in `vm` zone to inherit the functionality. + +``` +Host # firewall-cmd --permanent --new-zone vm +Host # firewall-cmd --permanent --zone vm --add-masquerade +Host # firewall-cmd --permanent --zone vm --add-port 53/udp +Host # firewall-cmd --permanent --zone vm --add-port 53/tcp +Host # systemctl restart firewalld +``` + +This might seem complicated but nmcli and firewalld (and systemd) have made +all this a lot easier to set up than a few years ago. + +## Get, build and install lws + +We're going to do it as root in the container... I produce and host this code, +you may feel its riskier in which case create a new user with sudo rights to +handle it. + +For redhat type distros, you probably need to add /usr/local/lib to the +/etc/ld.so.conf before ldconfig can rgister the new libwebsockets.so + +``` +Container # git clone https://libwebsockets.org/repo/libwebsockets +Container # cd libwebsockets && mkdir build && cd build && \ + cmake .. -DLWS_UNIX_SOCK=1 -DLWS_WITH_STRUCT_JSON=1 \ + -DLWS_WITH_STRUCT_SQLITE3=1 -DLWS_WITH_GENCRYPTO=1 -DLWS_WITH_SPAWN=1 \ + -DLWS_WITH_SECURE_STREAMS=1 -DLWS_WITH_THREADPOOL=1 +Container # make -j && make -j install && ldconfig +``` + +For OSX, you'll need to add something like `-DLWS_OPENSSL_INCLUDE_DIRS=/usr/local/opt/openssl@1.1/include -DLWS_OPENSSL_LIBRARIES="/usr/local/opt/openssl/lib/libssl.dylib;/usr/local/opt/openssl/lib/libcrypto.dylib"` to the cmake line + +### Windows: build libgit2 + +For Windows, after following the way to build lws in +`READMEs/README.build-windows.md` in the lws tree, you need to build libgit2 + +``` +$ git clone https://github.com/libgit2/libgit2 +$ cd libgit2 && mkdir build && cd build && cmake .. -DBUILD_CLAR=OFF +$ cmake --build . --config DEBUG +``` + +Then, in and Administrator-privs cmd window + +``` +$ cmake --install . --config DEBUG +``` + +### Get, build and install sai + +#### Linux + +``` +Container # cd ~ +Container # git clone https://warmcat.com/repo/sai +Container # cd sai && mkdir build && cd build && cmake .. && make && make install +Container # cp ../scripts/sai-builder.service /etc/systemd/system +Container # mkdir -p /etc/sai/builder +Container # cp ../scripts/builder-conf /etc/sai/builder/conf +Container # vim /etc/sai/builder/conf +Container # systemctl enable sai-builder +``` + +#### Windows + +``` +$ git clone https://warmcat.com/repo/sai +$ cd sai && mkdir build && cd build +$ cmake .. -DSAI_MASTER=0 -DSAI_LWS_INC_PATH="C:/Program Files (x86)/libwebsockets/include" -DSAI_LWS_LIB_PATH="C:/Program Files (x86)/libwebsockets/lib/websockets.lib" -DSAI_EXT_PTHREAD_INCLUDE_DIR="C:\Program Files (x86)\pthreads\include" -DSAI_EXT_PTHREAD_LIBRARIES="C:\Program Files (x86)\pthreads\lib\x64\libpthreadGC2.a" -DLWS_OPENSSL_INCLUDE_DIRS="C:\Program Files\OpenSSL\include" -DLWS_OPENSSL_LIBRARIES="C:\Program Files\OpenSSL\lib\libssl.lib;C:\Program Files\OpenSSL\lib\libcrypto.lib" -DSAI_GIT2_LIB_PATH="C:\Program Files (x86)\libgit2\lib\git2.lib" -DSAI_GIT2_INC_PATH="C:\Program Files (x86)\libgit2\include" +$ cmake --build . --config DEBUG +``` + +Hack the libgit2 and lws dlls into visibility (or do it a better non hacky +way if you know how...) using an Administrator cmd window + +``` +# cp "C:\Program Files (x86)\libgit2\bin\git2.dll" C:\Windows\system32 +# cp "C:\Program Files (x86)\libwebsockets\bin\websockets.dll" C:\Windows\system32 +``` + +Also in the Administrator window, copy and edit the sai-builder config + +``` +# mkdir -p C:\ProgramData\sai\builder\ +# cp ../scripts/builder-conf C:\ProgramData\sai\builder\conf +# vim C:\ProgramData\sai\builder\conf +``` + +#### OSX + +For OSX, you'll need to adapt the cmake to something like + +cmake .. -DLWS_OPENSSL_INCLUDE_DIRS=/usr/local/opt/openssl@1.1/include -DLWS_OPENSSL_LIBRARIES="/usr/local/opt/openssl/lib/libssl.dylib;/usr/local/opt/openssl/lib/libcrypto.dylib" + +Make sai-builder run as a daemon, create the file `/Library/LaunchDaemons/com.warmcat.sai-builder.plist` + +``` + + + + + Label + com.warmcat.sai-builder + ServiceDescription + Sai Builder Daemon + ProgramArguments + + /usr/local/bin/sai-builder + + RunAtLoad + + + +``` + +``` +% sudo launchctl load /Library/LaunchDaemons/com.warmcat.sai-builder.plist +``` + +That's it, he should be running in the background. To stop him + +``` +% sudo launchctl stop com.warmcat.sai-builder +``` + +### Make the container start at boot + +Let's leave the container and go back to the host + +``` +Container # exit +Host # +``` + +Then, we can configure it to start on boot + +``` +Host # systemctl enable systemd-nspawn@xenial-amd64 +``` + +## Administration of containers + +The container should be up and running sai-builder, and should come back up +subsequent boots too. You can use machinectl to open a shell on the running +container like this: + +``` +Host # machinectl -M xenial-amd64 shell +Connected to the local host. Press ^] three times within 1s to exit session. +Container # ps fax + PID TTY STAT TIME COMMAND + 1 ? Ss 0:00 /lib/systemd/systemd + 14 ? Ss 0:00 /lib/systemd/systemd-journald + 22 ? Ss 0:00 /lib/systemd/systemd-logind + 23 ? Ss 0:00 /usr/bin/dbus-daemon --system --address=systemd: --nofork --nopidfile --systemd-activation + 25 ? Ssl 0:00 /usr/local/bin/sai-builder + 28 pts/0 Ss+ 0:00 /sbin/agetty --noclear --keep-baud console 115200 38400 9600 vt220 + 42 ? Ss 0:00 /lib/systemd/systemd-machined + 43 pts/1 Ss 0:00 -bin/sh + 52 pts/1 S 0:00 \_ bash + 77 pts/1 R+ 0:00 \_ ps fax +Container # +``` + +You can use journalctl etc in there to see what's happening with sai-builder +there, etc. + +## Distro age considerations + +Really old distros are not maintained any more for updates, and their tls +certs have expired so they can't do https... it's pretty hopeless. + +Xenial (16.04) has problems with cmake (not all CTest functionality) and +libgit2 (remote fetch broken on its 0.24 version). You can rapidly build +both of these by cloning, they have CMake-based flow that's easy to use. + +Package|upstream +---|--- +cmake|https://github.com/Kitware/CMake.git +libgit2|https://github.com/libgit2/libgit2.git + +Install into the default /usr/local, and remove the conflicting old distro +packages. + +## Other Linux distros + +Other modern Linux distros with systemd can be done in their own containers and +also made to start at boot... these are very lightweight compared to full VMs +and yet you can administer them using all the usual systemd goodies including +randomly opening shells into their universe if needed. + +# Flashing and testing on embedded devices + +## Exposing host assets in the containers + +For the embedded case you want to flash and test on an external board, you need +to expose typically USB tty devices inside the nspawn container safely. + +That's not so trivial or as useful as it could be and has some limitations. + +### Step 1: Bind the related devices + +Device binding inside the container is like a one-time thing at container +start, wildcards are not supported and if the device doesn't exist, the +container won't start. All this makes supporting ephemeral USB serial ports +that come and go impossible. + +If your USB tty device is always available, then you can elect to expose it to +the container by overriding the systemd service definition for your container, +eg + +``` +Host # mkdir /etc/systemd/system/systemd-nspawn@YOURMACHINE.service.d +Host # vim /etc/systemd/system/systemd-nspawn@YOURMACHINE.service.d/override.conf +``` + +and add in there something like + +``` +[Service] +DeviceAllow=char-ttyUSB rwm +DeviceAllow=char-usb_device rwm +``` + +### Step 2: loosen the permissions on the device at the host + +I wasn't able to find a way to let the container unprivileged user seem to join +the host `dialout` group. The only workaround was to allow general access to +the device node. + +Create a file `vim /etc/udev/rules.d/50-allow-nspawn-ttyUSB.rules` and add in +there + +``` +KERNEL=="ttyUSB[0-9]*",MODE="0666" +KERNEL=="ttyACM[0-9]*",MODE="0666" +``` + +### Step 3: Bind the device nodes into the container + +Edit the container's nspawn file, like `/etc/systemd/nspawn/MYMACHINE.nspawn` +and in the `[Files]` section add + +``` +Bind=/dev/serial +Bind=/dev/ttyUSB0 +``` + +This is very awkward because the device nodes like `/dev/ttyUSB0` must exist at +container start time or it will fail to start, and you can't expose them via the +`/dev/serial/by-id` symlinks either, or use wildcards since it's done as a +one-time init at container start. diff --git a/READMEs/README-using-overlayfs.md b/READMEs/README-using-overlayfs.md new file mode 100644 index 0000000..0c89448 --- /dev/null +++ b/READMEs/README-using-overlayfs.md @@ -0,0 +1,225 @@ +# Overview + +This shows how to create the layered nspawn overlay fs dirs +for use with sai-builder, from scratch with the base distro +files. + +# prerequisites + +Install `systemd-container` package containing `systemd-nspawn` tool. + +# conventions + + - `base` is the completely unchanged base dir + - `env` is an overlayfs with the default packages (like gcc, git etc) + - `session` is an overlayfs with changes from a session + - `work` contains temp files used by eth overlayfs process + - `overlay` is a mountpoint for the combined overlayfs image + + - "Build host": the actual build machine's OS + filesystem + - "overlayfs": the guest filesystem + +# 1) Build host: creating the sai user and sai-nspawn directory + +Normal \*nix + +``` +$ sudo useradd sai +$ sudo mkdir -p /home/sai/sai-nspawn +``` +OSX + +``` +$ sudo dscl . -create /Users/_sai +$ sudo dscl . -create /Users/_sai UniqueID 773 +$ sudo dscl . -create /Users/_sai PrimaryGroupID -2 +$ sudo dscl . -create /Users/_sai UserShell /bin/bash +$ sudo chown _sai:nobody /Users/sai +``` + +# 2) Build host: Creating base layers for various OSes + +** Notice we're doing this on the build host, not in an overlayfs ** + +At least on Fedora, debootstrap is a package available from the default repos, +even though it's alien to Fedora package system itself. + +## Ubuntu + +``` +# cd /home/sai/sai-nspawn +# mkdir linux-ubuntu-xenial-amd64 +# cd linux-ubuntu-xenial-amd64 +# mkdir env work +# debootstrap --arch=amd64 --variant=buildd xenial base http://archive.ubuntu.com/ubuntu/ +``` + +Ubuntu only has amd64 and i386 + +## Debian + +``` +# cd /home/sai/sai-nspawn +# mkdir linux-debian-jessie-amd64 +# cd linux-debian-jessie-amd64 +# mkdir env work +# debootstrap --arch=amd64 stable base +``` + +Debian also has arm64 arch available. + +## fedora + +``` +# cd /home/sai/sai-nspawn +# mkdir linux-fedora-32-x86_64 +# cd linux-fedora-32-x86_64 +# mkdir base env work +# dnf -y --releasever=32 --installroot=`pwd`/base install systemd passwd dnf fedora-release +``` + +If asked to import a key, accept it. + +Fedora also has aarch64 + +## OSX + +Install brew, take care to install `pkg-config` via brew along with gcc etc + +# Overlayfs: Creating the base + env overlay to prepare env for building + +Now we're going to do stuff based on mounting the overlay temporarily +rather than the initial setup, let's move to /tmp (or anywhere else +you prefer like your home dir). + +When env is the active layer, we must use a work dir in the same partition. +(Later, we'll add a throwaway layer /tmp/mysession on top of base+env, then +the work dir can be in /tmp/mywork. But when changing env the work dir +must be in the same partition.) + +``` +$ cd /tmp +$ sudo mkdir -p mywork myoverlay mysession +$ export MYBASE=/home/sai/sai-nspawn/linux-ubuntu-xenial-amd64 +$ sudo mount -t overlay -o lowerdir=$MYBASE/base,upperdir=$MYBASE/env,workdir=$MYBASE/work none /tmp/myoverlay +``` + +At this point, ./myoverlay is the merged filesystem of base + the currently +empty env overlay. Files created in ./myoverlay will not affect base, but be +isolated into ./env. We're going to use env as a layer containing all the +build tools we will need. + +Run a shell with its / in ./myoverlay... + +``` +$ sudo systemd-nspawn -D myoverlay +Spawning container myoverlay on /tmp/myoverlay. +Press ^] three times within 1s to kill container. +root@myoverlay:~# +``` + +## Overlayfs: Create sai user + +Install into env the things needed for the environment as root + +``` +# useradd -u883 -gnogroup sai -d/home/sai -m -r + (or -gnobody for fedora) +``` + +If you need to do things as root, use this to let the sai user use sudo + +``` +# echo "sai ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers +``` + +## Overlayfs: Fedora: install build env + +glibc.i686 is included so you can use prebuilt 32-bit embedded toolchains, eg, linkit + +``` +# dnf install -y sudo make git gcc gcc-c++ cmake net-tools dhclient openssl-devel vim glibc.i686 dbus +``` + +Additional things for eg, building lws can be added like + +``` +# dnf install mbedtls-devel glib2-devel libevent-devel libev-devel libuv-devel libmount-devel sqlite-devel libgit2-devel +``` + + +## Overlayfs: Debian / Ubuntu: install build env + +Ensure vim is available + +``` +# apt update && apt install vim +``` + +Then enable "universe" repo + +``` +# apt edit-sources +``` + +and add `universe` to the end of the line after `main` + +``` +# apt update +``` + +Then install general build prerequisties, libc-i386 is included so you can use +prebuilt 32-bit embedded toolchains, eg, linkit + +``` +# apt install sudo make git gcc cmake net-tools libssl-dev vim libc6-i386 dbus +``` +You probably want other things if you're building something with optional deps, +eg, these for various build options on lws + +``` +# apt install libmbedtls-dev glib2.0-dev libevent-dev libev4 libev-dev libuv1 libuv1-dev libmount-dev libsqlite3-dev libgit2-dev +``` + +### Overlayfs: Finishing with layer modifications + +Type `exit` to exit the overlay shell + +## Build host: Unmount the overlayfs + +Having left the chrooted shell, you can unmount the overlay + +``` +# umount myoverlay +``` + +The changes we make like the package installs were all captured into env + +## Build host: optionally Remount so the overlay is base + env + /tmp/mysession + +``` +# mount -t overlay -o "lowerdir=$MYBASE/env:$MYBASE/base,upperdir=mysession,workdir=mywork" none myoverlay +``` + +Enter a shell again with changes going into the throwaway overlay "session" and this time as `sai` user. +From the changes we made earlier in env layer, sai has nopassword sudo access. + +``` +# systemd-nspawn -u sai -D myoverlay +``` + +From there, you can git clone your CI projects and configure, make and make install them + + +# Uploading nspawn layers to git + +``` +$ cd linux-ubuntu-xenial-amd64+base +$ sudo git init +$ sudo git add * +$ sudo git commit -m "initial commit" +$ sudo git remote add origin ssh://git@example.com/linux-ubuntu-xenial-amd64+base +$ sudo git push origin +master:master +``` + + diff --git a/READMEs/sai-build-test-flow.png b/READMEs/sai-build-test-flow.png new file mode 100644 index 0000000..a3c439b Binary files /dev/null and b/READMEs/sai-build-test-flow.png differ diff --git a/READMEs/sai-embedded-test.png b/READMEs/sai-embedded-test.png new file mode 100644 index 0000000..e617c07 Binary files /dev/null and b/READMEs/sai-embedded-test.png differ diff --git a/READMEs/sai-ov2.png b/READMEs/sai-ov2.png new file mode 100644 index 0000000..a885094 Binary files /dev/null and b/READMEs/sai-ov2.png differ diff --git a/READMEs/sai-overview.png b/READMEs/sai-overview.png new file mode 100644 index 0000000..1577021 Binary files /dev/null and b/READMEs/sai-overview.png differ diff --git a/assets/OSX-catalina.png b/assets/OSX-catalina.png new file mode 100644 index 0000000..ed3d45a Binary files /dev/null and b/assets/OSX-catalina.png differ diff --git a/assets/OpenSans-Light.ttf b/assets/OpenSans-Light.ttf new file mode 100644 index 0000000..6580d3a Binary files /dev/null and b/assets/OpenSans-Light.ttf differ diff --git a/assets/apple.svg b/assets/apple.svg new file mode 100644 index 0000000..aebe91e --- /dev/null +++ b/assets/apple.svg @@ -0,0 +1,16 @@ + + + + + + +image/svg+xml + + + + + + + + + diff --git a/assets/arch-aarch64-a72-bcm2711-rpi4.svg b/assets/arch-aarch64-a72-bcm2711-rpi4.svg new file mode 100644 index 0000000..3193351 --- /dev/null +++ b/assets/arch-aarch64-a72-bcm2711-rpi4.svg @@ -0,0 +1 @@ + diff --git a/assets/arch-aarch64.svg b/assets/arch-aarch64.svg new file mode 100644 index 0000000..3193351 --- /dev/null +++ b/assets/arch-aarch64.svg @@ -0,0 +1 @@ + diff --git a/assets/arch-arm32-m4-mt7697-usi.svg b/assets/arch-arm32-m4-mt7697-usi.svg new file mode 100644 index 0000000..8575247 --- /dev/null +++ b/assets/arch-arm32-m4-mt7697-usi.svg @@ -0,0 +1 @@ +32 \ No newline at end of file diff --git a/assets/arch-arm32.svg b/assets/arch-arm32.svg new file mode 100644 index 0000000..8575247 --- /dev/null +++ b/assets/arch-arm32.svg @@ -0,0 +1 @@ +32 \ No newline at end of file diff --git a/assets/arch-freertos.svg b/assets/arch-freertos.svg new file mode 100644 index 0000000..1f73977 --- /dev/null +++ b/assets/arch-freertos.svg @@ -0,0 +1,66 @@ + + + + + + +image/svg+xml + + + + + + + + + + + + + + + + + + + diff --git a/assets/arch-riscv.svg b/assets/arch-riscv.svg new file mode 100644 index 0000000..659b4e0 --- /dev/null +++ b/assets/arch-riscv.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/assets/arch-riscv64-virt.svg b/assets/arch-riscv64-virt.svg new file mode 100644 index 0000000..659b4e0 --- /dev/null +++ b/assets/arch-riscv64-virt.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/assets/arch-x86_64-amd.svg b/assets/arch-x86_64-amd.svg new file mode 100644 index 0000000..39eaa23 --- /dev/null +++ b/assets/arch-x86_64-amd.svg @@ -0,0 +1 @@ + diff --git a/assets/arch-x86_64-intel-i3.svg b/assets/arch-x86_64-intel-i3.svg new file mode 100644 index 0000000..450e32c --- /dev/null +++ b/assets/arch-x86_64-intel-i3.svg @@ -0,0 +1 @@ + diff --git a/assets/arch-x86_64-intel.svg b/assets/arch-x86_64-intel.svg new file mode 100644 index 0000000..450e32c --- /dev/null +++ b/assets/arch-x86_64-intel.svg @@ -0,0 +1 @@ + diff --git a/assets/arch-xl6-esp32.svg b/assets/arch-xl6-esp32.svg new file mode 100644 index 0000000..83cc4d8 --- /dev/null +++ b/assets/arch-xl6-esp32.svg @@ -0,0 +1 @@ + diff --git a/assets/artifact.svg b/assets/artifact.svg new file mode 100644 index 0000000..2efd25d --- /dev/null +++ b/assets/artifact.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/assets/builder-instance.png b/assets/builder-instance.png new file mode 100644 index 0000000..8beb672 Binary files /dev/null and b/assets/builder-instance.png differ diff --git a/assets/builder.png b/assets/builder.png new file mode 100644 index 0000000..6585d31 Binary files /dev/null and b/assets/builder.png differ diff --git a/assets/builder.svg b/assets/builder.svg new file mode 100644 index 0000000..18b6949 --- /dev/null +++ b/assets/builder.svg @@ -0,0 +1,3918 @@ + + + + + + + + + + + +image/svg+xml + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/assets/decal-2.svg b/assets/decal-2.svg new file mode 100644 index 0000000..dff950e --- /dev/null +++ b/assets/decal-2.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/assets/decal-3.svg b/assets/decal-3.svg new file mode 100644 index 0000000..494e2ad --- /dev/null +++ b/assets/decal-3.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/assets/decal-4.svg b/assets/decal-4.svg new file mode 100644 index 0000000..cda47df --- /dev/null +++ b/assets/decal-4.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/assets/decal-5.svg b/assets/decal-5.svg new file mode 100644 index 0000000..fbb1892 --- /dev/null +++ b/assets/decal-5.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/assets/decal-6.svg b/assets/decal-6.svg new file mode 100644 index 0000000..76b8282 --- /dev/null +++ b/assets/decal-6.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/assets/delete.png b/assets/delete.png new file mode 100644 index 0000000..262eab9 Binary files /dev/null and b/assets/delete.png differ diff --git a/assets/failed-login.html b/assets/failed-login.html new file mode 100644 index 0000000..9ab065b --- /dev/null +++ b/assets/failed-login.html @@ -0,0 +1,3 @@ + +This is an example destination that will appear after a failed login + diff --git a/assets/failed.svg b/assets/failed.svg new file mode 100644 index 0000000..9c04372 --- /dev/null +++ b/assets/failed.svg @@ -0,0 +1,18 @@ + + + + + +image/svg+xml + + + + + + + + + + + + diff --git a/assets/freertos-espidf.svg b/assets/freertos-espidf.svg new file mode 100644 index 0000000..78ef16a --- /dev/null +++ b/assets/freertos-espidf.svg @@ -0,0 +1 @@ + diff --git a/assets/freertos-linkit.svg b/assets/freertos-linkit.svg new file mode 100644 index 0000000..a564d05 --- /dev/null +++ b/assets/freertos-linkit.svg @@ -0,0 +1 @@ + diff --git a/assets/git.svg b/assets/git.svg new file mode 100644 index 0000000..aad1e3b --- /dev/null +++ b/assets/git.svg @@ -0,0 +1,21 @@ + + + + + + +image/svg+xml + + + + + + + + + + + + + + diff --git a/assets/gs/admin-login.html b/assets/gs/admin-login.html new file mode 100644 index 0000000..113df9c --- /dev/null +++ b/assets/gs/admin-login.html @@ -0,0 +1,5 @@ + +This is an example destination that will appear after successful Admin login. + +This URL cannot be served if you're not logged in as admin. + diff --git a/assets/gs/index.html b/assets/gs/index.html new file mode 100644 index 0000000..d40904a --- /dev/null +++ b/assets/gs/index.html @@ -0,0 +1,56 @@ + + + + + + + + + + + + + + + + +
+
+
+ +
+ + This is a demo application for lws generic-sessions.

+ It's a simple messageboard.

+ What's interesting about it is there is no serverside scripting,
+ instead client js makes a wss:// connection back to the server
+ and then reacts to JSON from the ws protocol. Sessions stuff is
+ handled by lws generic sessions, making the actual
+ test application very small.

+ And because it's natively websocket, it's naturally connected
+ for dynamic events and easy to maintain. +

+ Register / Login at the top right to see and create new messages. + + +
+
+ New message
+
+ +
+
+ +
+ +
+ +
+
+ + + + diff --git a/assets/gs/lwsgs-logo.png b/assets/gs/lwsgs-logo.png new file mode 100644 index 0000000..723a124 Binary files /dev/null and b/assets/gs/lwsgs-logo.png differ diff --git a/assets/gs/lwsgs.css b/assets/gs/lwsgs.css new file mode 100644 index 0000000..9dfde75 --- /dev/null +++ b/assets/gs/lwsgs.css @@ -0,0 +1,134 @@ +.body { font-size: 12px } +.gstitle { font-size: 18px } + +.group1 { + vertical-align:middle; + text-align:center; + background:#f0f0e0; + padding:12px; + border-radius:10px; +} +.group2 { + display:none; + vertical-align:middle; + font-size: 22px; + text-align:center; + margin:auto; + align:center; + background-color: rgba(255, 255, 255, 0.8); + padding:12px; + border-radius:10px; +} + +body.seats { + background-image:url(seats.jpg) +} + +div.lwsgs { + z-index: 3; + text-align:right; + background-color: rgba(255, 255, 255, 0.8); +} + +table.lwsgs { + width:100%; + height:100%; + transition: max-height 2s; +} +table.c100 { + text-align:center; + width:100%; +} + +table.r { + vertical-align:top; + text-align:right; +} + +table.l { + vertical-align:top; + text-align:left; +} + +table.fixed { + table-layout: fixed; +} + +td.logo { + vertical-align:top; + text-align:left; + width:200px +} + +td.lwsgs { + vertical-align:top; + float:right; +} + +td.h99 { + height:99%; + vertical-align:middle; +} + +td.c { + margin:auto; + align:center +} + +td.tac { + text-align:center +} + +td.ava { + display:inline-block; + vertical-align:top; + word-wrap:break-word; +} + +iframe.hidden { + display:none; +} + +div.hidden { + display:none; +} + +div.hiddenr { + display:none; + text-align:right; +} + +input { + margin: 2px; + padding: 2px; +} + +input.em { + margin: 4px; + font-weight:bold; +} + +input.wide { + margin: 6px; + padding: 6px; +} + +input.hidden { + display: none; +} + +form.r { + text-align:right; +} + +span.bad { + color: red; +} + +span.small { + font-size:8pt; +} + +.green { + color: green; +} diff --git a/assets/gs/lwsgs.js b/assets/gs/lwsgs.js new file mode 100644 index 0000000..b3ae489 --- /dev/null +++ b/assets/gs/lwsgs.js @@ -0,0 +1,617 @@ + + +var lwsgs_user = "$lwsgs_user"; +var lwsgs_auth = parseInt("$lwsgs_auth", 10); +var lwsgs_email = "$lwsgs_email"; + +/* + * auth: + * b0: LWSGS_AUTH_LOGGED_IN + * b1: LWSGS_AUTH_ADMIN + * b2: LWSGS_AUTH_VERIFIED + * b3: LWSGS_AUTH_FORGOT_FLOW + */ + +var lwsgs_html = '\ + \ +\ +
\ +
\ + \ + \ + \ + \ +
\ + \ +
\ + \ +
\ +
\ +\ +
\ + \ + \ + \ +
\ + \ +
\ + \ +
Awaiting account verification
\ +
\ +\ + \ + \ + \ + \ + \ +'; + +/*-- this came from + -- https://raw.githubusercontent.com/blueimp/JavaScript-MD5/master/js/md5.min.js + -- under MIT license */ +!function(n){"use strict";function t(n,t){var r=(65535&n)+(65535&t),e=(n>>16)+(t>>16)+(r>>16);return e<<16|65535&r}function r(n,t){return n<>>32-t}function e(n,e,o,u,c,f){return t(r(t(t(e,n),t(u,f)),c),o)}function o(n,t,r,o,u,c,f){return e(t&r|~t&o,n,t,u,c,f)}function u(n,t,r,o,u,c,f){return e(t&o|r&~o,n,t,u,c,f)}function c(n,t,r,o,u,c,f){return e(t^r^o,n,t,u,c,f)}function f(n,t,r,o,u,c,f){return e(r^(t|~o),n,t,u,c,f)}function i(n,r){n[r>>5]|=128<>>9<<4)+14]=r;var e,i,a,h,d,l=1732584193,g=-271733879,v=-1732584194,m=271733878;for(e=0;e>5]>>>t%32&255);return r}function h(n){var t,r=[];for(r[(n.length>>2)-1]=void 0,t=0;t>5]|=(255&n.charCodeAt(t/8))<16&&(o=i(o,8*n.length)),r=0;16>r;r+=1)u[r]=909522486^o[r],c[r]=1549556828^o[r];return e=i(u.concat(h(t)),512+8*t.length),a(i(c.concat(e),640))}function g(n){var t,r,e="0123456789abcdef",o="";for(r=0;r>>4&15)+e.charAt(15&t);return o}function v(n){return unescape(encodeURIComponent(n))}function m(n){return d(v(n))}function p(n){return g(m(n))}function s(n,t){return l(v(n),v(t))}function C(n,t){return g(s(n,t))}function A(n,t,r){return t?r?s(t,n):C(t,n):r?m(n):p(n)}"function"==typeof define&&define.amd?define(function(){return A}):"object"==typeof module&&module.exports?module.exports=A:n.md5=A}(this); + +if (lwsgs_user.substring(0, 1) == "$") { + alert("lwsgs.js: lws generic sessions misconfigured and not providing vars"); +} +function lwsgs_san(s) +{ + if (s.search("<") != -1) + return "invalid string"; + + return s; +} + +function lwsgs_update() +{ + var en_login = 1, en_forgot = 1; + + if (document.getElementById('password').value.length && + document.getElementById('password').value.length < 8) + en_login = 0; + + if (!document.getElementById('username').value || + !document.getElementById('password').value) + en_login = 0; + + if (!document.getElementById('username').value || + document.getElementById('password').value) + en_forgot = 0; + + document.getElementById('login').disabled = !en_login; + document.getElementById('forgot').disabled = !en_forgot; + + if (lwsgs_user) + document.getElementById("curuser1").innerHTML = + document.getElementById("curuser").innerHTML = + lwsgs_san(lwsgs_user); + + if (lwsgs_user === "") + document.getElementById("dlogin").style.display = "inline"; + else { + if (lwsgs_auth & 4) + document.getElementById("dlogout").style.display = "inline"; + else + document.getElementById("dwaitverify").style.display = "inline"; + } + } + +function lwsgs_open_registration() +{ + document.getElementById("dadmin").style.display = "none"; + document.getElementById("dlogin").style.display = "none"; + document.getElementById("dlogout").style.display = "none"; + document.getElementById("dchange").style.display = "none"; + document.getElementById("dregister").style.display = "inline"; +} + +function lwsgs_cancel_registration() +{ + document.getElementById("dadmin").style.display = "none"; + document.getElementById("dregister").style.display = "none"; + document.getElementById("dchange").style.display = "none"; + + if (lwsgs_user === "" || !(lwsgs_auth & 1)) + document.getElementById("dlogin").style.display = "inline"; + else + document.getElementById("dlogout").style.display = "inline"; +} + +function lwsgs_select_change() +{ + document.getElementById("dlogin").style.display = "none"; + document.getElementById("dlogout").style.display = "none"; + document.getElementById("dregister").style.display = "none"; + if (lwsgs_auth & 2) { + document.getElementById("dadmin").style.display = "inline"; + document.getElementById("dchange").style.display = "none"; + } else { + document.getElementById("dadmin").style.display = "none"; + document.getElementById("dchange").style.display = "inline"; + } + + event.preventDefault() +} + +var lwsgs_user_check = '0'; +var lwsgs_email_check = '0'; + +function lwsgs_rupdate() +{ + var en_register = 1, en_forgot = 0; + + if (document.getElementById('rpassword').value == + document.getElementById('password2').value) { + if (document.getElementById('rpassword').value.length) + document.getElementById('match').innerHTML = + "\u2713"; + else + document.getElementById('match').innerHTML = ""; + document.getElementById('pw2').style = ""; + } else { + if (document.getElementById('password2').value || + document.getElementById('email').value) { // ie, he is filling in "register" path and cares + document.getElementById('match').innerHTML = + "\u2718 Passwords do not match"; + } else + document.getElementById('match').innerHTML = + "\u2718 Passwords do not match"; + + en_register = 0; + } + + if (document.getElementById('rpassword').value.length && + document.getElementById('rpassword').value.length < 8) { + en_register = 0; + document.getElementById('rpw1').innerHTML = "Need 8 chars"; + } else + if (document.getElementById('rpassword').value.length) + document.getElementById('rpw1').innerHTML = "\u2713"; + else + document.getElementById('rpw1').innerHTML = ""; + + if (!document.getElementById('rpassword').value || + !document.getElementById('password2').value || + !document.getElementById('rusername').value || + !document.getElementById('email').value || + lwsgs_email_check === '1'|| + lwsgs_user_check === '1') + en_register = 0; + + document.getElementById('register').disabled = !en_register; + document.getElementById('rpassword').disabled = lwsgs_user_check === '1'; + document.getElementById('password2').disabled = lwsgs_user_check === '1'; + document.getElementById('email').disabled = lwsgs_user_check === '1'; + + if (lwsgs_user_check === '0') { + var uc = document.getElementById('uchk'); + + if (uc) { + if (document.getElementById('rusername').value) + uc.innerHTML = "\u2713"; + else + uc.innerHTML = ""; + } + } else { + if (document.getElementById('uchk')) + ocument.getElementById('uchk').innerHTML = "\u2718 Already registered"; + en_forgot = 1; + } + + if (lwsgs_email_check === '0') { + var ec = document.getElementById('echk'); + + if (ec) { + if (document.getElementById('email').value) + ec.innerHTML = "\u2713"; + else + ec.innerHTML = ""; + } + } else { + if (document.getElementById('echk')) + document.getElementById('echk').innerHTML = "\u2718 Already registered"; + en_forgot = 1; + } + + if (en_forgot) + document.getElementById('rforgot').style.display = "inline"; + else + document.getElementById('rforgot').style.display = "none"; + + if (lwsgs_user_check === '1') + op = '0.5'; + else + op = '1.0'; + document.getElementById('rpassword').style.opacity = op; + document.getElementById('password2').style.opacity = op; + document.getElementById('email').style.opacity = op; + } + +function lwsgs_cupdate() +{ + var en_change = 1, en_forgot = 1, pwok = 1; + + if (lwsgs_auth & 8) { + document.getElementById('ccurpw').style.display = "none"; + document.getElementById('ccurpw_name').style.display = "none"; + } else { + if (!document.getElementById('ccurpw').value || + document.getElementById('ccurpw').value.length < 8) { + en_change = 0; + pwok = 0; + document.getElementById('cuchk').innerHTML = "\u2718"; + } else { + en_forgot = 0; + document.getElementById('cuchk').innerHTML = ""; + } + document.getElementById('ccurpw').style.display = "inline"; + document.getElementById('ccurpw_name').style.display = "inline"; + } + + if (document.getElementById('cpassword').value == + document.getElementById('cpassword2').value) { + if (document.getElementById('cpassword').value.length) + document.getElementById('cmatch').innerHTML = "\u2713"; + else + document.getElementById('cmatch').innerHTML = ""; + document.getElementById('pw2').style = ""; + } else { + if (document.getElementById('cpassword2').value //|| + //document.getElementById('cemail').value + ) { // ie, he is filling in "register" path and cares + document.getElementById('cmatch').innerHTML = + "\u2718 Passwords do not match"; + } else + document.getElementById('cmatch').innerHTML = "\u2718 Passwords do not match"; + + en_change = 0; + } + + if (document.getElementById('cpassword').value.length && + document.getElementById('cpassword').value.length < 8) { + en_change = 0; + document.getElementById('cpw1').innerHTML = "Need 8 chars"; + } else { + var cpw = document.getElementById('cpw1'); + + if (cpw) { + if (document.getElementById('cpassword').value.length) + cpw.innerHTML = "\u2713"; + else + cpw.innerHTML = ""; + } + } + + if (!document.getElementById('cpassword').value || + !document.getElementById('cpassword2').value || + pwok === 0) + en_change = 0; + + if (document.getElementById('showdel').checked) + document.getElementById('delete').style.display = "inline"; + else + document.getElementById('delete').style.display = "none"; + + document.getElementById('change').disabled = !en_change; + document.getElementById('cpassword').disabled = pwok === 0; + document.getElementById('cpassword2').disabled = pwok === 0; + document.getElementById('showdel').disabled = pwok === 0; + document.getElementById('delete').disabled = pwok === 0; + //document.getElementById('cemail').disabled = pwok === 0; + + /* + if (lwsgs_auth.parseInt() & 8) { + document.getElementById('cemail').style.display = "none"; + document.getElementById('cemail_name').style.display = "none"; + } else { + document.getElementById('cemail').style.display = "inline"; + document.getElementById('cemail_name').style.display = "inline"; + if (lwsgs_email_check === '0' && + document.getElementById('cemail').value != lwsgs_email) { + if (document.getElementById('cemail').value) + document.getElementById('cechk').innerHTML = "\u2713"; + else + document.getElementById('cechk').innerHTML = ""; + } else { + document.getElementById('cechk').innerHTML = "\u2718 Already registered"; + en_forgot = 1; + } + } */ + + if (lwsgs_auth & 8) + en_forgot = 0; + + if (en_forgot) + document.getElementById('cforgot').style.display = "inline"; + else + document.getElementById('cforgot').style.display = "none"; + + if (pwok === 0) + op = '0.5'; + else + op = '1.0'; + document.getElementById('cpassword').style.opacity = op; + document.getElementById('cpassword2').style.opacity = op; + // document.getElementById('cemail').style.opacity = op; + } + +function lwsgs_check_user() +{ + var xmlHttp = new XMLHttpRequest(); + xmlHttp.onreadystatechange = function() { + if (xmlHttp.readyState === 4 && xmlHttp.status === 200) { + lwsgs_user_check = xmlHttp.responseText; + lwsgs_rupdate(); + } + } + xmlHttp.open("GET", "lwsgs-check/username="+document.getElementById('rusername').value, true); + xmlHttp.send(null); +} + +function lwsgs_check_email(id) +{ + var xmlHttp = new XMLHttpRequest(); + xmlHttp.onreadystatechange = function() { + if (xmlHttp.readyState === 4 && xmlHttp.status === 200) { + lwsgs_email_check = xmlHttp.responseText; + lwsgs_rupdate(); + } + } + xmlHttp.open("GET", "lwsgs-check/email="+document.getElementById(id).value, true); + xmlHttp.send(null); +} + +function rupdate_user() +{ + lwsgs_rupdate(); + lwsgs_check_user(); +} + +function rupdate_email() +{ + lwsgs_rupdate(); + lwsgs_check_email('email'); +} + +function cupdate_email() +{ + lwsgs_cupdate(); + lwsgs_check_email('cemail'); +} + + +function lwsgs_initial() +{ + document.getElementById('lwsgs').innerHTML = lwsgs_html; + + if (lwsgs_user) { + document.getElementById("curuser").innerHTML = + "currently logged in as " + lwsgs_san(lwsgs_user) + "
"; + + document.getElementById("ccuruser").innerHTML = + "Login settings for " + + lwsgs_san(lwsgs_user) + "
"; + } + + document.getElementById('username').oninput = lwsgs_update; + document.getElementById('username').onchange = lwsgs_update; + document.getElementById('password').oninput = lwsgs_update; + document.getElementById('password').onchange = lwsgs_update; + document.getElementById('doreg').onclick = lwsgs_open_registration; + document.getElementById('clink').onclick = lwsgs_select_change; + document.getElementById('cancel').onclick =lwsgs_cancel_registration; + document.getElementById('cancel2').onclick =lwsgs_cancel_registration; + document.getElementById('rpassword').oninput = lwsgs_rupdate; + document.getElementById('password2').oninput = lwsgs_rupdate; + document.getElementById('rusername').oninput = rupdate_user; + document.getElementById('email').oninput = rupdate_email; + document.getElementById('ccurpw').oninput = lwsgs_cupdate; + document.getElementById('cpassword').oninput = lwsgs_cupdate; + document.getElementById('cpassword2').oninput = lwsgs_cupdate; + + document.getElementById('showdel').onchange = lwsgs_cupdate; + + if (lwsgs_email) + document.getElementById('grav1').innerHTML = + document.getElementById('grav').innerHTML = + ""; + //if (lwsgs_email) + //document.getElementById('cemail').placeholder = lwsgs_email; + document.getElementById('cusername').value = lwsgs_user; + lwsgs_update(); + lwsgs_cupdate(); +} + +window.addEventListener("load", function() { + lwsgs_initial(); + document.getElementById("nolog").style.display = !!lwsgs_user ? "none" : "inline-block"; + document.getElementById("logged").style.display = !lwsgs_user ? "none" : "inline-block"; + + //document.getElementById("msg").onkeyup = mupd; + //document.getElementById("msg").onchange = mupd; + + var ws; + + function mb_format(s) + { + var r = "", n, wos = 0; + + for (n = 0; n < s.length; n++) { + if (s[n] == ' ') + wos = 0; + else { + wos++; + if (wos === 40) { + wos = 0; + r = r + ' '; + } + } + if (s[n] == '<') { + r = r + "<"; + continue; + } + if (s[n] == '\n') { + r = r + "
"; + continue; + } + + r = r + s[n]; + } + + return r; + } + + function add_div(n, m) + { + var q = document.getElementById(n); + var d = new Date(m.time * 1000); + + q.innerHTML = "
" + + "
" + + "" + lwsgs_san(m.username) + "
" + + "" + d.toDateString() + + "
" + d.toTimeString() + "
" + + "IP: " + lwsgs_san(m.ip) + + "		" + + mb_format(m.content) + + "

" + q.innerHTML; + } + + function mupd() + { + //document.getElementById("send").disabled = !document.getElementById("msg").value; + } +}, false); diff --git a/assets/gs/md5.min.js b/assets/gs/md5.min.js new file mode 100644 index 0000000..4bd9de1 --- /dev/null +++ b/assets/gs/md5.min.js @@ -0,0 +1,2 @@ +!function(n){"use strict";function t(n,t){var r=(65535&n)+(65535&t),e=(n>>16)+(t>>16)+(r>>16);return e<<16|65535&r}function r(n,t){return n<>>32-t}function e(n,e,o,u,c,f){return t(r(t(t(e,n),t(u,f)),c),o)}function o(n,t,r,o,u,c,f){return e(t&r|~t&o,n,t,u,c,f)}function u(n,t,r,o,u,c,f){return e(t&o|r&~o,n,t,u,c,f)}function c(n,t,r,o,u,c,f){return e(t^r^o,n,t,u,c,f)}function f(n,t,r,o,u,c,f){return e(r^(t|~o),n,t,u,c,f)}function i(n,r){n[r>>5]|=128<>>9<<4)+14]=r;var e,i,a,h,d,l=1732584193,g=-271733879,v=-1732584194,m=271733878;for(e=0;e>5]>>>t%32&255);return r}function h(n){var t,r=[];for(r[(n.length>>2)-1]=void 0,t=0;t>5]|=(255&n.charCodeAt(t/8))<16&&(o=i(o,8*n.length)),r=0;16>r;r+=1)u[r]=909522486^o[r],c[r]=1549556828^o[r];return e=i(u.concat(h(t)),512+8*t.length),a(i(c.concat(e),640))}function g(n){var t,r,e="0123456789abcdef",o="";for(r=0;r>>4&15)+e.charAt(15&t);return o}function v(n){return unescape(encodeURIComponent(n))}function m(n){return d(v(n))}function p(n){return g(m(n))}function s(n,t){return l(v(n),v(t))}function C(n,t){return g(s(n,t))}function A(n,t,r){return t?r?s(t,n):C(t,n):r?m(n):p(n)}"function"==typeof define&&define.amd?define(function(){return A}):"object"==typeof module&&module.exports?module.exports=A:n.md5=A}(this); +//# sourceMappingURL=md5.min.js.map \ No newline at end of file diff --git a/assets/gs/post-forgot-fail.html b/assets/gs/post-forgot-fail.html new file mode 100644 index 0000000..ead3d13 --- /dev/null +++ b/assets/gs/post-forgot-fail.html @@ -0,0 +1,5 @@ + +Sorry, something went wrong. + +Click here to continue. + diff --git a/assets/gs/post-forgot-ok.html b/assets/gs/post-forgot-ok.html new file mode 100644 index 0000000..3e8e9cf --- /dev/null +++ b/assets/gs/post-forgot-ok.html @@ -0,0 +1,6 @@ + +This is a one-time password recovery login. + +Please click here and click your username at the top to reset your password. + + diff --git a/assets/gs/post-verify-fail.html b/assets/gs/post-verify-fail.html new file mode 100644 index 0000000..d1d89ca --- /dev/null +++ b/assets/gs/post-verify-fail.html @@ -0,0 +1,20 @@ + + + + + + + + + + + + +
+ +
+ Sorry, the link was invalid. +
+ + + diff --git a/assets/gs/post-verify-ok.html b/assets/gs/post-verify-ok.html new file mode 100644 index 0000000..ae647fc --- /dev/null +++ b/assets/gs/post-verify-ok.html @@ -0,0 +1,25 @@ + + + + + + + + + + + + +
+ +
+ Thanks for signing up, your registration as is verified.
+
+ Click here to continue. +
+ + + + diff --git a/assets/gs/sent-forgot-fail.html b/assets/gs/sent-forgot-fail.html new file mode 100644 index 0000000..ead3d13 --- /dev/null +++ b/assets/gs/sent-forgot-fail.html @@ -0,0 +1,5 @@ + +Sorry, something went wrong. + +Click here to continue. + diff --git a/assets/gs/sent-forgot-ok.html b/assets/gs/sent-forgot-ok.html new file mode 100644 index 0000000..83df751 --- /dev/null +++ b/assets/gs/sent-forgot-ok.html @@ -0,0 +1,4 @@ +An email has been sent to your registered address. + +Please follow the instructions to reset your password. + diff --git a/assets/gs/successful-login.html b/assets/gs/successful-login.html new file mode 100644 index 0000000..dfc25cf --- /dev/null +++ b/assets/gs/successful-login.html @@ -0,0 +1,4 @@ + +This is an example destination that will appear after successful non-Admin login + + diff --git a/assets/index.html b/assets/index.html index 11a3f7b..1125093 100644 --- a/assets/index.html +++ b/assets/index.html @@ -2,16 +2,34 @@ - - - + + + + Sai -
+
+ + + + + +
+
+
+
+
-
+ + + +
+
+ + + diff --git a/assets/iosevka-term-ss01-regular.ttf b/assets/iosevka-term-ss01-regular.ttf new file mode 100644 index 0000000..30c2355 Binary files /dev/null and b/assets/iosevka-term-ss01-regular.ttf differ diff --git a/assets/jsplease.svg b/assets/jsplease.svg new file mode 100644 index 0000000..910f1c8 --- /dev/null +++ b/assets/jsplease.svg @@ -0,0 +1,801 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + image/svg+xml + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/assets/linux-android.svg b/assets/linux-android.svg new file mode 100644 index 0000000..1cddd64 --- /dev/null +++ b/assets/linux-android.svg @@ -0,0 +1 @@ + diff --git a/assets/linux-centos-8.svg b/assets/linux-centos-8.svg new file mode 100644 index 0000000..a54a759 --- /dev/null +++ b/assets/linux-centos-8.svg @@ -0,0 +1 @@ +8 diff --git a/assets/linux-fedora-32-riscv.svg b/assets/linux-fedora-32-riscv.svg new file mode 100644 index 0000000..8b826dd --- /dev/null +++ b/assets/linux-fedora-32-riscv.svg @@ -0,0 +1 @@ + diff --git a/assets/linux-fedora-32.svg b/assets/linux-fedora-32.svg new file mode 100644 index 0000000..bfd8b52 --- /dev/null +++ b/assets/linux-fedora-32.svg @@ -0,0 +1 @@ +32 diff --git a/assets/linux-gentoo.svg b/assets/linux-gentoo.svg new file mode 100644 index 0000000..293551a --- /dev/null +++ b/assets/linux-gentoo.svg @@ -0,0 +1 @@ + diff --git a/assets/linux-ubuntu-1804.svg b/assets/linux-ubuntu-1804.svg new file mode 100644 index 0000000..e795c3a --- /dev/null +++ b/assets/linux-ubuntu-1804.svg @@ -0,0 +1 @@ +b \ No newline at end of file diff --git a/assets/linux-ubuntu-2004.svg b/assets/linux-ubuntu-2004.svg new file mode 100644 index 0000000..8c389d5 --- /dev/null +++ b/assets/linux-ubuntu-2004.svg @@ -0,0 +1 @@ +f \ No newline at end of file diff --git a/assets/linux-ubuntu-xenial-amd64.svg b/assets/linux-ubuntu-xenial-amd64.svg new file mode 100644 index 0000000..f4387a7 --- /dev/null +++ b/assets/linux-ubuntu-xenial-amd64.svg @@ -0,0 +1 @@ +x \ No newline at end of file diff --git a/assets/linux.svg b/assets/linux.svg new file mode 100644 index 0000000..ce7d516 --- /dev/null +++ b/assets/linux.svg @@ -0,0 +1,19 @@ + + + + + + +image/svg+xml + + + + + + + + + + + + diff --git a/assets/netbsd-OSX-catalina.svg b/assets/netbsd-OSX-catalina.svg new file mode 100644 index 0000000..3f5276c --- /dev/null +++ b/assets/netbsd-OSX-catalina.svg @@ -0,0 +1 @@ + diff --git a/assets/netbsd-iOS.svg b/assets/netbsd-iOS.svg new file mode 100644 index 0000000..2e283dd --- /dev/null +++ b/assets/netbsd-iOS.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/assets/passed.svg b/assets/passed.svg new file mode 100644 index 0000000..75ad325 --- /dev/null +++ b/assets/passed.svg @@ -0,0 +1,18 @@ + + + + + +image/svg+xml + + + + + + + + + + + + diff --git a/assets/post-register-fail.html b/assets/post-register-fail.html new file mode 100644 index 0000000..063c3c5 --- /dev/null +++ b/assets/post-register-fail.html @@ -0,0 +1 @@ +Registration failed, sorry diff --git a/assets/post-register-ok.html b/assets/post-register-ok.html new file mode 100644 index 0000000..f5d70b8 --- /dev/null +++ b/assets/post-register-ok.html @@ -0,0 +1,22 @@ + + + + + + + + + +
+ +
+ Your registration is accepted:
+ you will receive an email shortly with instructions
+ to verify and enable the account for normal use.

+ + The link is only valid for an hour, after that if it has
+ not been verified your account will be deleted. +
+ + + diff --git a/assets/rebuild.png b/assets/rebuild.png new file mode 100644 index 0000000..7c5df4a Binary files /dev/null and b/assets/rebuild.png differ diff --git a/assets/sai-event.svg b/assets/sai-event.svg new file mode 100644 index 0000000..1dccc6c --- /dev/null +++ b/assets/sai-event.svg @@ -0,0 +1,48 @@ + + + + + +image/svg+xml + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/assets/sai-icon.svg b/assets/sai-icon.svg new file mode 100644 index 0000000..de7980c --- /dev/null +++ b/assets/sai-icon.svg @@ -0,0 +1,45 @@ + + + + + +image/svg+xml + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/assets/sai.css b/assets/sai.css index c0be30b..f505808 100644 --- a/assets/sai.css +++ b/assets/sai.css @@ -1,122 +1,50 @@ -span.title { - font-size:18pt; - font-family: Arial; - font-weight:normal; - text-align:center; - color:#000000; -} -span.mount { - font-size:10pt; - font-family: Arial; - font-weight:normal; - text-align:center; - color:#000000; -} -span.mountname { - font-size:14pt; - font-family: Arial; - font-weight:bold; - text-align:center; - color:#404010; +@font-face { + font-family: 'OpenSans'; + src: url('OpenSans-Light.ttf') format('truetype'); } -span.n { - font-size:12pt; - font-family: Arial; - font-weight:normal; - text-align:center; - color:#808020; -} -span.sn { - font-size:12pt; - font-family: Arial; - font-weight:bold; - text-align:center; - color:#808020; -} -span.v { - font-size:12pt; - font-family: Arial; - font-weight:bold; - text-align:center; - color:#202020; + +@font-face { + font-family: 'Iosevka'; + font-style: normal; + font-weight: 400; + font-display: block; + src: url('/iosevka-term-ss01-regular.ttf') format('truetype'); } -span.m1 { - font-size:12pt; - font-family: Arial; - font-weight:bold; - text-align:center; - color:#202020; + + +body { + font-family: 'OpenSans', Fallback, sans-serif; } -span.m2 { - font-size:12pt; - font-family: Arial; - font-weight:normal; - text-align:center; - color:#202020; + + +.content { vertical-align:top; text-align:center; background:#fffff0; padding:12px; -webkit-border-radius:10px; border-radius:10px; } +.canvas { vertical-align:top; text-align:center; background:#efefd0; padding:12px; -webkit-border-radius:10px; border-radius:10px; } + + +div.zi { + z-index: 1000 } -.browser { font-size:12pt; font-family: Arial; font-weight:normal; text-align:center; color:#ffff00; vertical-align:middle; text-align:center; background:#d0b070; padding:12px; -webkit-border-radius:10px; border-radius:10px;} -.group2 { vertical-align:middle; - text-align:center; - background:#f0f0e0; - padding:12px; - -webkit-border-radius:10px; - border-radius:10px; } -.explain { vertical-align:middle; - text-align:center; - background:#f0f0c0; padding:12px; - -webkit-border-radius:10px; - border-radius:10px; - color:#404000; - padding:3px; +td.al { + text-align: left } -td.wsstatus { vertical-align:middle; width:200px; height:50px; - text-align:center; - background:#f0f0c0; padding:6px; - -webkit-border-radius:8px; - border-radius:8px; - color:#404000; } -.tdform { vertical-align:middle; width:200px; height:50px; - text-align:center; - background:#f0f0d0; padding:6px; - -webkit-border-radius:8px; - margin:10px; - border-radius:8px; - border: 1px solid black; - border-collapse: collapse;font-size:18pt; font-family: Arial; font-weight:normal; text-align:center; color:#000000; - color:#404000; } - -td.l { vertical-align:middle; - text-align:center; - background:#d0d0b0; - padding:3px; - -webkit-border-radius:3px; - border-radius:3px; } - -td.bigger { font-size:120%; } -div.bgw { background:white } -div.conninfo { - border: solid 2px #e0d040; - padding: 4px; - width: 500px; - height:350px; - overflow: auto; +img.tread1 { + margin-left: -6px; + z-index: 10 } -span.f12 { font-size:12pt } - -.content { vertical-align:top; text-align:center; background:#fffff0; padding:12px; -webkit-border-radius:10px; border-radius:10px; } -.canvas { vertical-align:top; text-align:center; background:#efefd0; padding:12px; -webkit-border-radius:10px; border-radius:10px; } -.tabs { - position: relative; - min-height: 750px; /* This part sucks */ - clear: both; - margin: 25px 0; + +img.tread2 { + margin-left: -6px; + z-index: 9 } -.tab { - float: left; + +img.zup { + position: relative; + z-index: 1001 } + .tab label { background: #eee; padding: 10px; @@ -147,70 +75,530 @@ span.f12 { font-size:12pt } z-index: 1; } -td.wsstatus { vertical-align:middle; width:200px; height:50px; - text-align:center; - background:#f0f0c0; padding:6px; - -webkit-border-radius:8px; - border-radius:8px; - color:#404000; } -td.l { vertical-align:middle; - text-align:center; - background:#d0d0b0; - padding:3px; - -webkit-border-radius:3px; - border-radius:3px; } -td.dl { vertical-align:middle; - text-align:center; - background:#c0c0c0; - padding:3px; - -webkit-border-radius:3px; - border-radius:3px; } -td.c { vertical-align:middle; - text-align:center; - background:#c0c0a0; - padding:3px; - -webkit-border-radius:3px; - border-radius:3px; } -td.c0 { vertical-align:middle; - text-align:center; - background:#b0b090; - padding:3px; - -webkit-border-radius:3px; - border-radius:3px; } -td.dc0 { vertical-align:middle; - text-align:center; - background:#a0a0a0; - padding:3px; - -webkit-border-radius:3px; - border-radius:3px; } -td.c1 { vertical-align:middle; +td.waiting { + vertical-align:top; + text-align:left; + color:#000000; + padding:1px; + font-size: 9pt; + font-weight: bold; + border-radius:5px; +} + +table.comp { + border-radius:7px; +} + +table.comp_pass { + background:#01ff70; +} + +table.comp_fail { + background:#ff4136; + color:#fff; +} + +div.ov_good { + background:#2ecc40; +} + +div.ov_bad { + background: #ff4136; + color:#fff; +} + +span.ov_bad { + color:#fff +} + +div.ov_dunno { + background:#d0d0d0; +} + +img.deemph { + opacity: 0.6; +} + +td.tasks { + vertical-align:top; text-align:center; - background:#c0c0c0; - padding:3px; - -webkit-border-radius:3px; - border-radius:3px; } -td.t { vertical-align:middle; + background:#f0f0f0; + color:#000000; + padding:0px; + font-weight: normal; + border-radius:6px; +} + +td.ti { + vertical-align: top; + margin-left: 4px; +} + +td.atop { + vertical-align: top; +} + +td.keepline { + display: inline-flex; +} + +td.jumble { + position: relative; + vertical-align: top; + white-space: nowrap; +} + +span.group2 { + vertical-align:middle; text-align:center; - background:#e0e0c0; - padding:3px; + background:#f0f0f0; + color:#000000; + padding:3px; + font-weight: normal; -webkit-border-radius:3px; border-radius:3px; } +span.stderr { + color: #44f; +} +span.saibuild { + color: #68a; +} +span.tty0 { + color: #884; +} +span.tty1 { + color: #848; +} + + +span.noscript { + text-size:200%; +} + +span.nowrap { + white-space: nowrap; +} + +div.taskinfo { + vertical-align: top; +} + +div.taskstate { + font-weight: normal; + font-size: 7pt; + padding:1px; + margin-right:1px; + border-radius:3px; + white-space: nowrap; +} + +div.nailed { + position:fixed; + background: #fff; + width:100%; + z-index:1001; + margin-top:-12px; +} + +div.undernailed { + position:relative; + top: 180px; +} + + +div.evr { + position: absolute; + top: 16px; + left: 16px; +} + + +.taskstate0 { + color:#000000; + opacity:0.5; +} + +.taskstate1 { + + color:#000000; + opacity:0.6; +} + +img.ip { + width: 16px; + height: 16px; + vertical-align:middle; +} + +img.ip0 { + width: 16px; + height: 16px; + vertical-align:middle; +} + +img.ip1 { + width: 18px; + height: 18px; + vertical-align:top; +} + +img.ip2 { + width: 19px; + height: 19px; + vertical-align:text-bottom; +} + +img.rebuild { + width: 16px; + height: 16px; + vertical-align:middle; + cursor: pointer; +} + +img.bico { + width: 12px; + height: 12px; + vertical-align:middle; +} + +@keyframes stretch { + 0%, 100% { + opacity: 0.8; + } + 0% { + opacity: 0.3; + } +} + +.taskstate1 { + background:#a0a0a0; + color:#000000; + opacity:0.75; + animation-name: stretch; + animation-duration: 2s; + animation-timing-function: ease-out; + animation-delay: 0s; + animation-direction: alternate; + animation-iteration-count: infinite; + animation-fill-mode: none; + animation-play-state: running; +} + +.taskstate2 { + background:#a0a0a0; + color:#000000; + opacity:0.75; + animation-name: stretch; + animation-duration: 1s; + animation-timing-function: ease-out; + animation-delay: 0s; + animation-direction: alternate; + animation-iteration-count: infinite; + animation-fill-mode: none; + animation-play-state: running; +} + +.taskstate3 { + background:#01ff70; + color:#003000; + opacity:1; + /* + animation-name: fadeInOpacity; + animation-iteration-count: 1; + animation-timing-function: ease-in; + animation-duration: 2s; */ +} + +@keyframes fadeInOpacity { + 0% { + opacity: 0; + } + 100% { + opacity: 1; + } +} +} + +.taskstate4 { + background:#ff851b; + color:#300000; + opacity:1; +} + +.taskstate5 { + background:#808080; + color:#202020; + opacity:1; +} + +.taskstate6 { + background:#df3030; + color:#300000; + opacity:1; +} + + +div.dlogs { + vertical-align: top; + font-size: 11px; + font-family: "Iosevka"; + +-webkit-text-size-adjust: none; + -moz-text-size-adjust: none; + -ms-text-size-adjust: none; + text-size-adjust: none; +} +div.dlogsn { + vertical-align: top; + font-size: 11px; + font-family: "Iosevka"; + text-align:right; + width: 5em; + margin-top: 0px; + margin-right: 3px; +-webkit-text-size-adjust: none; + -moz-text-size-adjust: none; + -ms-text-size-adjust: none; + text-size-adjust: none; + +} +div.dlogst { + vertical-align: top; + font-size: 11px; + color:#808080; + font-family: "Iosevka"; + text-align:right; + width: 5em; + margin-top: 0px; + margin-right: 3px; +-webkit-text-size-adjust: none; + -moz-text-size-adjust: none; + -ms-text-size-adjust: none; + text-size-adjust: none; + +} + +div.sai_arts { + display: inline-flex; + vertical-align: middle; + font-size: 7pt; + background-color:#ddd; + color:#3d9970; + text-align:left; + margin: 3px 2px; + padding: 2px 3px; + border-radius:5px; +} + +span.ti1 { + font-weight: normal; + font-size: 9pt; + background:#d0c0b0; + color:#000000; + padding:3px; + border-radius:3px; + white-space: nowrap; +} + +span.ti2 { + font-weight: normal; + font-size: 7pt; +} + +span.ti3 { + font-weight: normal; + font-size: 7pt; +} + +span.ti4 { + font-weight: normal; + font-size: 7pt; +} +span.ti5 { + font-weight: normal; + font-size: 7pt; +} + +span.e1 { + font-weight: bold; + font-size: 11pt; +} + +span.e2 { + font-weight: normal; + font-size: 10pt; +} + +span.e3 { + font-weight: normal; + font-size: 8pt; +} + +td.e4 { + font-weight: normal; + font-size: 8pt; + text-align: right; +} + table.summary { width: 100%; + padding: 1px; + margin: 1px; +} + +table.builders { + height: 100%; + margin-left: 32px; } + +img.bsvg { + height: 50px; + width: auto; + padding:3px; +} + +td.builder { + width: 99%; +} + + td.logo { - margin: 16px; - margin-right: 32px; - padding: 16px; + margin: 3px; + margin-right: 4px; + padding: 2px; + vertical-align: top; + z-index: 1000; } td.summary { background: #f8f8f8; width: 100%; - margin: 16px; - padding: 16px; + margin: 4px; + padding: 4px; +} + +td.bn { + font-weight: normal; + font-size: 7pt; + padding: 1px; +} + +td.bins { + font-weight: normal; + font-size: 9pt; + text-align: center; +} + +td.tn { + font-weight: normal; + font-size: 7pt; + padding-left:0px; +} + +table.nomar { + padding: 0px; + margin: -3px; +} + +td.nomar { + padding: 0px; + margin: 0px; +} + +div.ib { + display: inline-block; + vertical-align:middle; + text-align:left; + line-height: 50%; + float:left; + margin: 0.1em; + font-weight: normal; + font-size: 7pt; + padding: 1.1px 1.5px 1.1px 1.5px; + border-radius:3px; +} + +div.ibuil { + display: inline-block; + vertical-align:middle; + text-align:left; + line-height: 50%; + float:left; + margin: 0.1em; + font-weight: normal; + font-size: 7pt; + background:#c0c0e0; + color:#000000; + padding:3px; + border-radius:10px; +} + +div.ibuilctr { + font-weight: normal; + font-size: 9pt; + background:#e0e0ff; +} + +div.ibuilctrname { + display: inline-block; + vertical-align:middle; + text-align:left; + line-height: 50%; + float:left; + margin: 0.1em; + font-weight: normal; + font-size: 9pt; + color:#000000; + padding:3px; + +} + +div.plat { + display: inline-block; + vertical-align:middle; + text-align:left; + line-height: 50%; + float:left; + margin: 0.1em; + font-weight: normal; + font-size: 7pt; + color:#000000; + padding:3px; + border-radius:3px; +} + +div.pass { + display: inline-block; + background:#40e040; +} + +div.fail { + display: inline-block; + background:#40e040; +} + +div.awaiting { + display: inline-block; +} + +div.bdr { + display: inline-block; + opacity: 1; +} + + + +img.branch { + display: inline-block; + float:left; + background: url("data:image/svg+xml;base64,PD94bWwgdmVyc2lvbj0iMS4wIiBlbmNvZGluZz0iVVRGLTgiPz4KPHN2ZyB2ZXJzaW9uPSIxLjEiIHZpZXdCb3g9IjAgMCAyNi43NDkgMjcuMjIyIiB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHhtbG5zOmNjPSJodHRwOi8vY3JlYXRpdmVjb21tb25zLm9yZy9ucyMiIHhtbG5zOmRjPSJodHRwOi8vcHVybC5vcmcvZGMvZWxlbWVudHMvMS4xLyIgeG1sbnM6cmRmPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5LzAyLzIyLXJkZi1zeW50YXgtbnMjIj4KPGRlZnM+CjxmaWx0ZXIgaWQ9ImIiIHg9Ii0uMDcwOCIgeT0iLS4wNzA4IiB3aWR0aD0iMS4xNDE2IiBoZWlnaHQ9IjEuMTQxNiIgY29sb3ItaW50ZXJwb2xhdGlvbi1maWx0ZXJzPSJzUkdCIj4KPGZlR2F1c3NpYW5CbHVyIHN0ZERldmlhdGlvbj0iMC4yNTQyNzk1MyIvPgo8L2ZpbHRlcj4KPGZpbHRlciBpZD0iYSIgeD0iLS4wOTA1NjgiIHk9Ii0uMDQ2MjE0IiB3aWR0aD0iMS4xODExIiBoZWlnaHQ9IjEuMDkyNCIgY29sb3ItaW50ZXJwb2xhdGlvbi1maWx0ZXJzPSJzUkdCIj4KPGZlR2F1c3NpYW5CbHVyIHN0ZERldmlhdGlvbj0iMC40Nzk4NDA4OSIvPgo8L2ZpbHRlcj4KPC9kZWZzPgo8bWV0YWRhdGE+CjxyZGY6UkRGPgo8Y2M6V29yayByZGY6YWJvdXQ9IiI+CjxkYzpmb3JtYXQ+aW1hZ2Uvc3ZnK3htbDwvZGM6Zm9ybWF0Pgo8ZGM6dHlwZSByZGY6cmVzb3VyY2U9Imh0dHA6Ly9wdXJsLm9yZy9kYy9kY21pdHlwZS9TdGlsbEltYWdlIi8+CjxkYzp0aXRsZS8+CjwvY2M6V29yaz4KPC9yZGY6UkRGPgo8L21ldGFkYXRhPgo8ZyB0cmFuc2Zvcm09InRyYW5zbGF0ZSgtNzAuNzQ3IC02OS40NTQpIj4KPGNpcmNsZSBjeD0iODQuMTIxIiBjeT0iODMuMDIyIiByPSIxMy4zNzQiIGZpbGw9Im5vbmUiLz4KPHBhdGggZD0ibTgzLjc3MyA3MC42MDZjLTEuMTg2MiAwLjAxNTQtMi4zNDE4IDAuODQ2ODgtMi40Nzc0IDEuOTk4OGwtMC4wMTI0IDAuMDQ5MWMtMC4xNDk5IDUuNjExMi0wLjA0NzQgMTEuMjUyLTAuMTExNjIgMTYuODc0IDAuMDU4MSAxLjg3NzctMC40NDMzNyAzLjg0NDEgMC44NjgxNyA1LjIwMTIgMS43NDYyIDEuNzQ0NSA0LjIzMTcgMC40OTM5MyA0LjE0MjQtMi41MTU2IDAuMDQ1NS0wLjkyODQyIDAuMDExNy0xLjQwMiAwLjExMDA3LTMuMjM0OSAxLjUwNTIgMS4xMzQ4IDIuOTYwNCAyLjk5NjYgNC44MTk5IDMuNTIzOCAyLjAzNzEgMC4zMTUyOCAzLjQ3NDgtMi4yMDU1IDIuMzEyLTMuODU1MS0yLjI5OTItMi4yMDI1LTQuODIwMi00LjA1Ny03LjE2Ni02LjIyODYgMC4xMzg5LTMuMyAwLjAxNzYtNi4xNDIgMC4wNDY1LTkuNjA2Ny0wLjQxNDMtMS41OTgtMS40ODQ5LTIuMjE5Ni0yLjUzMTYtMi4yMDYxem0wLjE4NDQ5IDExLjg4M2EyLjA3ODkgMi4wNzg5IDAgMCAxIDIuMDc4OSAyLjA3ODkgMi4wNzg5IDIuMDc4OSAwIDAgMS0yLjA3ODkgMi4wNzkgMi4wNzg5IDIuMDc4OSAwIDAgMS0yLjA3ODQtMi4wNzkgMi4wNzg5IDIuMDc4OSAwIDAgMSAyLjA3ODQtMi4wNzg5eiIgZmlsdGVyPSJ1cmwoI2EpIiBzdHJva2U9IiMwMDAiIHN0cm9rZS13aWR0aD0iLjI2NDU4cHgiLz4KPHBhdGggZD0ibTgzLjUxOCA3MC4yODNjLTEuMTg2MiAwLjAxNTQtMi4zNDE4IDAuODQ2ODgtMi40Nzc0IDEuOTk4OGwtMC4wMTI0IDAuMDQ5MWMtMC4xNDk5IDUuNjExMi0wLjA0NzQgMTEuMjUyLTAuMTExNjIgMTYuODc0IDAuMDU4MSAxLjg3NzctMC40NDMzNyAzLjg0NDEgMC44NjgxNyA1LjIwMTIgMS43NDYyIDEuNzQ0NSA0LjIzMTcgMC40OTM5MyA0LjE0MjQtMi41MTU2IDAuMDQ1NS0wLjkyODQyIDAuMDExNy0xLjQwMiAwLjExMDA3LTMuMjM0OSAxLjUwNTIgMS4xMzQ4IDIuOTYwNCAyLjk5NjYgNC44MTk5IDMuNTIzOCAyLjAzNzEgMC4zMTUyOCAzLjQ3NDgtMi4yMDU1IDIuMzEyLTMuODU1MS0yLjI5OTItMi4yMDI1LTQuODIwMi00LjA1Ny03LjE2Ni02LjIyODYgMC4xMzg5LTMuMyAwLjAxNzYtNi4xNDIgMC4wNDY1LTkuNjA2Ny0wLjQxNDMtMS41OTgtMS40ODQ5LTIuMjE5Ni0yLjUzMTYtMi4yMDYxem0wLjE4NDQ5IDExLjg4M2EyLjA3ODkgMi4wNzg5IDAgMCAxIDIuMDc4OSAyLjA3ODkgMi4wNzg5IDIuMDc4OSAwIDAgMS0yLjA3ODkgMi4wNzkgMi4wNzg5IDIuMDc4OSAwIDAgMS0yLjA3ODQtMi4wNzkgMi4wNzg5IDIuMDc4OSAwIDAgMSAyLjA3ODQtMi4wNzg5eiIgZmlsbD0iI2ZmZiIgc3Ryb2tlPSIjMDAwIiBzdHJva2Utd2lkdGg9Ii4yNjQ1OHB4Ii8+CjxwYXRoIGQ9Im04My43MTYgODAuMThhNC4zMDk3IDQuMzA5NyAwIDAgMC00LjMwOTggNC4zMDk4IDQuMzA5NyA0LjMwOTcgMCAwIDAgNC4zMDk4IDQuMzA5OCA0LjMwOTcgNC4zMDk3IDAgMCAwIDQuMzA5OC00LjMwOTggNC4zMDk3IDQuMzA5NyAwIDAgMC00LjMwOTgtNC4zMDk4em0wLjAzMzYgMi4xMDQ4YTIuMTcxNiAyLjE3MTYgMCAwIDEgMi4xNzE1IDIuMTcxNSAyLjE3MTYgMi4xNzE2IDAgMCAxLTIuMTcxNSAyLjE3MTQgMi4xNzE2IDIuMTcxNiAwIDAgMS0yLjE3MTQtMi4xNzE0IDIuMTcxNiAyLjE3MTYgMCAwIDEgMi4xNzE0LTIuMTcxNXoiIGZpbGw9IiNmZmYiIGZpbHRlcj0idXJsKCNiKSIgc3Ryb2tlPSIjMDAwIiBzdHJva2Utd2lkdGg9Ii41Ii8+CjxwYXRoIGQ9Im04My42NjkgODAuMDYyYTQuMzA5NyA0LjMwOTcgMCAwIDAtNC4zMDk4IDQuMzA5OCA0LjMwOTcgNC4zMDk3IDAgMCAwIDQuMzA5OCA0LjMwOTggNC4zMDk3IDQuMzA5NyAwIDAgMCA0LjMwOTgtNC4zMDk4IDQuMzA5NyA0LjMwOTcgMCAwIDAtNC4zMDk4LTQuMzA5OHptMC4wMzM2IDIuMTA0OGEyLjE3MTYgMi4xNzE2IDAgMCAxIDIuMTcxNSAyLjE3MTQgMi4xNzE2IDIuMTcxNiAwIDAgMS0yLjE3MTUgMi4xNzE0IDIuMTcxNiAyLjE3MTYgMCAwIDEtMi4xNzE0LTIuMTcxNCAyLjE3MTYgMi4xNzE2IDAgMCAxIDIuMTcxNC0yLjE3MTR6IiBmaWxsPSIjZmZmIiBzdHJva2U9IiMwMDAiIHN0cm9rZS13aWR0aD0iLjYiLz4KPC9nPgo8L3N2Zz4K"); + width:0px; + height:0px; + cursor:pointer; + padding: 0.6em; + background-repeat: no-repeat; + vertical-align:middle; + color: rgba(0, 0, 0, 0); } - - \ No newline at end of file diff --git a/assets/sai.js b/assets/sai.js index 59477d8..2f523cd 100644 --- a/assets/sai.js +++ b/assets/sai.js @@ -5,6 +5,395 @@ * that has HTML stuff in it */ +/* http://i18njs.com/ this from http://i18njs.com/js/i18n.js */ +(function() { + var Translator, i18n, translator, + __bind = function(fn, me){ return function(){ return fn.apply(me, arguments); }; }; + + Translator = (function() { + function Translator() { + this.translate = __bind(this.translate, this); this.data = { + values: {}, + contexts: [] + }; + this.globalContext = {}; + } + + Translator.prototype.translate = function(text, defaultNumOrFormatting, + numOrFormattingOrContext, formattingOrContext, context) { + var defaultText, formatting, isObject, num; + + if (context == null) { + context = this.globalContext; + } + isObject = function(obj) { + var type; + + type = typeof obj; + return type === "function" || type === "object" && !!obj; + }; + if (isObject(defaultNumOrFormatting)) { + defaultText = null; + num = null; + formatting = defaultNumOrFormatting; + context = numOrFormattingOrContext || this.globalContext; + } else { + if (typeof defaultNumOrFormatting === "number") { + defaultText = null; + num = defaultNumOrFormatting; + formatting = numOrFormattingOrContext; + context = formattingOrContext || this.globalContext; + } else { + defaultText = defaultNumOrFormatting; + if (typeof numOrFormattingOrContext === "number") { + num = numOrFormattingOrContext; + formatting = formattingOrContext; + context = context; + } else { + num = null; + formatting = numOrFormattingOrContext; + context = formattingOrContext || this.globalContext; + } + } + } + if (isObject(text)) { + if (isObject(text['i18n'])) { + text = text['i18n']; + } + return this.translateHash(text, context); + } else { + return this.translateText(text, num, formatting, context, defaultText); + } + }; + + Translator.prototype.add = function(d) { + var c, v, _i, _len, _ref, _ref1, _results; + + if ((d.values != null)) { + _ref = d.values; + var k; + for (k in _ref) { + if ({}.hasOwnProperty.call(_ref, k)) { + v = _ref[k]; + this.data.values[k] = v; + } + } + } + if ((d.contexts != null)) { + _ref1 = d.contexts; + _results = []; + for (_i = 0, _len = _ref1.length; _i < _len; _i++) { + c = _ref1[_i]; + _results.push(this.data.contexts.push(c)); + } + return _results; + } + }; + + Translator.prototype.setContext = function(key, value) { + return this.globalContext[key] = value; + }; + + Translator.prototype.clearContext = function(key) { + return this.lobalContext[key] = null; + }; + + Translator.prototype.reset = function() { + this.data = { + values: {}, + contexts: [] + }; + return this.globalContext = {}; + }; + + Translator.prototype.resetData = function() { + return this.data = { + values: {}, + contexts: [] + }; + }; + + Translator.prototype.resetContext = function() { + return this.globalContext = {}; + }; + + Translator.prototype.translateHash = function(hash, context) { + var k, v; + + for (k in hash) { + if ({}.hasOwnProperty.call(hash, k)) { + v = hash[k]; + if (typeof v === "string") { + hash[k] = this.translateText(v, null, null, context); + } + } + } + return hash; + }; + + Translator.prototype.translateText = function(text, num, formatting, + context, defaultText) { + var contextData, result; + + if (context == null) { + context = this.globalContext; + } + if (this.data == null) { + return this.useOriginalText(defaultText || text, num, formatting); + } + contextData = this.getContextData(this.data, context); + if (contextData != null) { + result = this.findTranslation(text, num, formatting, contextData.values, + defaultText); + } + if (result == null) { + result = this.findTranslation(text, num, formatting, this.data.values, + defaultText); + } + if (result == null) { + return this.useOriginalText(defaultText || text, num, formatting); + } + return result; + }; + + Translator.prototype.findTranslation = function(text, num, formatting, data) { + var result, triple, value, _i, _len; + + value = data[text]; + if (value == null) { + return null; + } + if (num == null) { + if (typeof value === "string") { + return this.applyFormatting(value, num, formatting); + } + } else { + if (value instanceof Array || value.length) { + for (_i = 0, _len = value.length; _i < _len; _i++) { + triple = value[_i]; + if ((num >= triple[0] || triple[0] === null) && + (num <= triple[1] || triple[1] === null)) { + result = this.applyFormatting(triple[2].replace("-%n", + String(-num)), num, formatting); + return this.applyFormatting(result.replace("%n", + String(num)), num, formatting); + } + } + } + } + return null; + }; + + Translator.prototype.getContextData = function(data, context) { + var c, equal, key, value, _i, _len, _ref, _ref1; + + if (data.contexts == null) { + return null; + } + _ref = data.contexts; + for (_i = 0, _len = _ref.length; _i < _len; _i++) { + c = _ref[_i]; + equal = true; + _ref1 = c.matches; + for (key in _ref1) { + if ({}.hasOwnProperty.call(_ref1, key)) { + value = _ref1[key]; + equal = equal && value === context[key]; + } + } + if (equal) { + return c; + } + } + return null; + }; + + Translator.prototype.useOriginalText = function(text, num, formatting) { + if (num == null) { + return this.applyFormatting(text, num, formatting); + } + return this.applyFormatting(text.replace("%n", String(num)), + num, formatting); + }; + + Translator.prototype.applyFormatting = function(text, num, formatting) { + var ind, regex; + + for (ind in formatting) { + if ({}.hasOwnProperty.call(formatting, ind)) { + regex = new RegExp("%{" + ind + "}", "g"); + text = text.replace(regex, formatting[ind]); + } + } + return text; + }; + + return Translator; + + })(); + + translator = new Translator(); + + i18n = translator.translate; + + i18n.translator = translator; + + i18n.create = function(data) { + var trans; + + trans = new Translator(); + if (data != null) { + trans.add(data); + } + trans.translate.create = i18n.create; + return trans.translate; + }; + + (typeof module !== "undefined" && module !== null ? module.exports = i18n : void 0) || + (this.i18n = i18n); + +}.call(this)); + +var lang_ja = "{" + + "\"values\":{" + + "\"Summary\": \"概要\"," + + "\"Log\": \"ログ\"," + + "\"Tree\": \"木構造\"," + + "\"Blame\": \"責任\"," + + "\"Copy Lines\": \"コピーライン\"," + + "\"Copy Link\": \"リンクをコピーする\"," + + "\"View Blame\": \"責任がある\"," + + "\"Remove Blame\": \"責任を取り除く\"," + + "\"Mode\": \"モード\"," + + "\"Size\": \"サイズ\"," + + "\"Name\": \"名\"," + + "\"s\": \"秒\"," + + "\"m\": \"分\"," + + "\"h\": \"時間\"," + + "\" days\": \"日々\"," + + "\" weeks\": \"週\"," + + "\" months\": \"数ヶ月\"," + + "\" years\": \"年\"," + + "\"Branch Snapshot\": \"ブランチスナップショット\"," + + "\"Tag Snapshot\": \"タグスナップショット\"," + + "\"Commit Snapshot\": \"スナップショットをコミットする\"," + + "\"Description\": \"説明\"," + + "\"Owner\": \"オーナー\"," + + "\"Branch\": \"ブランチ\"," + + "\"Tag\": \"タグ\"," + + "\"Author\": \"著者\"," + + "\"Age\": \"年齢\"," + + "\"Page fetched\": \"ページを取得した\"," + + "\"creation time\": \"作成時間\"," + + "\"created\": \"作成した\"," + + "\"ago\": \"前\"," + + "\"Message\": \"メッセージ\"," + + "\"Download\": \"ダウンロード\"," + + "\"root\": \"ルート\"," + + "\"Committer\": \"コミッター\"," + + "\"Raw Patch\": \"生パッチ\"," + + "\"Page fetched %{pf} ago, creation time: %{ct}ms " + + "(vhost etag hits: %{ve}%, cache hits: %{ch}%)\": " + + "\"%{pf}間前に取得されたページ, 作成にかかった時間: %{ct}ms " + + "(vhost etag キャッシュヒット: %{ve}%, キャッシュヒット: %{ch}%)\"," + + "\"Created %{pf} ago, creation time: %{ct}ms \":" + + "\"%{pf}間前に作成されました, 作成にかかった時間: %{ct}ms\"" + + "}}"; + +var lang_zht = "{" + +"\"values\":{" + + "\"Summary\": \"概要\"," + + "\"Log\": \"日誌\"," + + "\"Tree\": \"樹\"," + + "\"Blame\": \"責怪\"," + + "\"Copy Lines\": \"複製線\"," + + "\"Copy Link\": \"複製鏈接\"," + + "\"View Blame\": \"看責怪\"," + + "\"Remove Blame\": \"刪除責怪\"," + + "\"Mode\": \"模式\"," + + "\"Size\": \"尺寸\"," + + "\"Name\": \"名稱\"," + + "\"s\": \"秒\"," + + "\"m\": \"分鐘\"," + + "\"h\": \"小時\"," + + "\" days\": \"天\"," + + "\" weeks\": \"週\"," + + "\" months\": \"個月\"," + + "\" years\": \"年份\"," + + "\"Branch Snapshot\": \"科快照\"," + + "\"Tag Snapshot\": \"标签快照\"," + + "\"Commit Snapshot\": \"提交快照\"," + + "\"Description\": \"描述\"," + + "\"Owner\": \"所有者\"," + + "\"Branch\": \"科\"," + + "\"Tag\": \"標籤\"," + + "\"Author\": \"作者\"," + + "\"Age\": \"年齡\"," + + "\"Page fetched\": \"頁面已獲取\"," + + "\"creation time\": \"創作時間\"," + + "\"created\": \"創建\"," + + "\"ago\": \"前\"," + + "\"Message\": \"信息\"," + + "\"Download\": \"下載\"," + + "\"root\": \"根源\"," + + "\"Committer\": \"提交者\"," + + "\"Raw Patch\": \"原始補丁\"," + + "\"Page fetched %{pf} ago, creation time: %{ct}ms " + + "(vhost etag hits: %{ve}%, cache hits: %{ch}%)\": " + + "\"頁面%{pf}前獲取, 創作時間: %{ct}ms " + + "(vhost etag 緩存命中: %{ve}%, 緩存命中: %{ch}%)\"," + + "\"Created %{pf} ago, creation time: %{ct}ms \":" + + "\"%{pf}前創建, 創作時間: %{ct}ms \"" + +"}}"; + +var lang_zhs = "{" + +"\"values\":{" + + "\"Summary\": \"概要\"," + + "\"Log\": \"日志\"," + + "\"Tree\": \"木\"," + + "\"Blame\": \"归咎\"," + + "\"Copy Lines\": \"复制线\"," + + "\"Copy Link\": \"复制链接\"," + + "\"View Blame\": \"看责备\"," + + "\"Remove Blame\": \"删除责备\"," + + "\"Mode\": \"模式\"," + + "\"Size\": \"尺寸\"," + + "\"Name\": \"名称\"," + + "\"s\": \"秒\"," + + "\"m\": \"分钟\"," + + "\"h\": \"小时\"," + + "\" days\": \"天\"," + + "\" weeks\": \"周\"," + + "\" months\": \"个月\"," + + "\" years\": \"年份\"," + + "\"Branch Snapshot\": \"科快照\"," + + "\"Tag Snapshot\": \"标签快照\"," + + "\"Commit Snapshot\": \"提交快照\"," + + "\"Description\": \"描述\"," + + "\"Owner\": \"所有者\"," + + "\"Branch\": \"科\"," + + "\"Tag\": \"标签\"," + + "\"Author\": \"作者\"," + + "\"Age\": \"年龄\"," + + "\"Page fetched\": \"页面已获取\"," + + "\"creation time\": \"创作时间\"," + + "\"created\": \"创建\"," + + "\"ago\": \"前\"," + + "\"Message\": \"信息\"," + + "\"Download\": \"下载\"," + + "\"root\": \"根源\"," + + "\"Committer\": \"提交者\"," + + "\"Raw Patch\": \"原始补丁\"," + + "\"Page fetched %{pf} ago, creation time: %{ct}ms " + + "(vhost etag hits: %{ve}%, cache hits: %{ch}%)\": " + + "\"页面%{pf}前获取, 创作时间: %{ct}ms " + + "(vhost etag 缓存命中: %{ve}%, 缓存命中: %{ch}%)\"," + + "\"Created %{pf} ago, creation time: %{ct}ms \":" + + "\"%{pf}前创建, 创作时间: %{ct}ms \"" + +"}}"; + +var logs = "", redpend = 0, gitohashi_integ = 0; + function san(s) { if (s.search("<") !== -1) @@ -29,7 +418,23 @@ function humanize(s) return s; } - var pos = 0; +function hsanitize(s) +{ + var table = { + '<': 'lt', + '>': 'gt', + '"': 'quot', + '\'': 'apos', + '&': 'amp' + }; + + return s.toString().replace(/[<>"'&]/g, function(chr) { + return '&' + table[chr] + ';'; + }).replace(/\n/g, '
'); +} + +var pos = 0, lli = 1, lines = "", times = "", locked = 1, tfirst = 0, + cont = [ 0, 0, 0, 0, 0]; function get_appropriate_ws_url() { @@ -55,182 +460,701 @@ function get_appropriate_ws_url() return pcol + u[0]; } +var age_names = [ "s", "m", "h", " days", " weeks", " months", " years" ]; +var age_div = [ 1, 60, 3600, 86400, 604800, 2419200, 31536000 ]; +var age_limit = [ 120, 7200, 172800, 1209600, 4838400, 63072000, 0 ]; +var age_upd = [ 5, 10, 300, 1800, 3600, 12 * 3600, 12 * 3600 ]; + +function agify(now, secs) +{ + var d = now - secs, n; + + if (!secs) + return ""; + + for (n = 0; n < age_names.length; n++) + if (d < age_limit[n] || age_limit[n] === 0) + return "" + Math.ceil(d / age_div[n]) + + i18n(age_names[n]) + ""; +} + +function aging() +{ + var n, next = 24 * 3600, + now_ut = Math.round((new Date().getTime() / 1000)); + + for (n = 0; n < age_names.length; n++) { + var m, elems = document.getElementsByClassName( + "age-" + n), ne = elems.length, + a = new Array(), ee = new Array(); + + if (elems.length && age_upd[n] < next) + next = age_upd[n]; + + for (m = 0; m < ne; m++) { + var e = elems[m], secs = elems[m].getAttribute("ut"); + + a.push(agify(now_ut, secs)); + ee.push(e); + } + + for (m = 0; m < ee.length; m++) { + ee[m].innerHTML = a[m]; + } + } + + /* + * We only need to come back when the age might have changed. + * Eg, if everything is counted in hours already, once per + * 5 minutes is accurate enough. + */ + window.setTimeout(aging, next * 1000); +} +var sai, jso, s, sai_arts = ""; + +function sai_plat_icon(plat, size) +{ + var s, s1 = ""; + + s = plat.split('/'); + if (s[0]) { + console.log("plat " + plat + " plat[0] " + s[0]); + s1 = ""; + + if (s[1]) + s1 += ""; + } + + if (s[2]) { + s1 += ""; + } + return s1; +} + +function sai_stateful_taskname(state, nm, sf) +{ + var tp = ""; + + if (sf) + return " " + san(nm) + "  "; + + if (state == 4 || state == 6) + tp = " ov_bad"; + + return "" + san(nm) + ""; +} + +function sai_taskinfo_render(t, now_ut) +{ + var now_ut = Math.round((new Date().getTime() / 1000)); + var s = ""; + + s = "
"; + + s += "
" + + sai_event_render(t, now_ut, 0) + "
" + + "" + sai_plat_icon(t.t.platform, 2) + + san(t.t.platform) + " "; + if (t.t.state != 0 && t.t.state != 3 && t.t.state != 4 && t.t.state != 5) + s += "\"stop "; + s += "\"rebuild\" " + + sai_stateful_taskname(t.t.state, t.t.taskname, 1); + + if (t.t.builder_name) { + var now_ut = Math.round((new Date().getTime() / 1000)); + + s += "   " + + san(t.t.builder_name) + ""; + if (t.t.started) + /* started is a unix time, in seconds */ + s += ", " + + agify(now_ut, t.t.started) + " ago, Dur: " + + (t.t.duration ? t.t.duration / 1000000 : + now_ut - t.t.started).toFixed(1) + + "s
"; + sai_arts = ""; + } + + s += "
"; + + return s; +} + +function sai_event_summary_render(o, now_ut, reset_all_icon) +{ + var s, q, ctn = "", wai, s1 = "", n, e = o.e; + + s = "" + + "
"; + if (e.state == 4) + s += "
"; + + s += ""; + if (reset_all_icon && !gitohashi_integ) { + s += "
\"rebuild "; + s += "\"delete"; + } + s += "		" + + ""; + if (o.t.length > 1) + s += ""; + s += "
" + + san(e.repo_name) + + "
"; + + if (e.ref.substr(0, 11) === "refs/heads/") { + s += "" + + san(e.ref.substr(11)); + } else + if (e.ref.substr(0, 10) === "refs/tags/") { + s += "" + + san(e.ref.substr(10)); + } else + s += san(e.ref); + + s += "
" + + san(e.hash.substr(0, 8)) + + "" + + agify(now_ut, e.created) + "
" + o.t.length + " builds
" + + "
"; + + return s; +} + +function sai_event_render(o, now_ut, reset_all_icon) +{ + var s, q, ctn = "", wai, s1 = "", n, e = o.e; + + s = sai_event_summary_render(o, now_ut, reset_all_icon); -var sai, jso, s; + if (o.t.length) { + var all_good, has_bad, all_done; + + /* + * SAIES_WAITING, + * SAIES_PASSED_TO_BUILDER, + * SAIES_BEING_BUILT, + * SAIES_SUCCESS, done --> + * SAIES_FAIL, + * SAIES_CANCELLED, + * SAIES_BEING_BUILT_HAS_FAILURES + */ + + s += "
"; -function ws_open_server_status() + for (q = 0; q < o.t.length; q++) { + var t = o.t[q]; + + if (t.taskname !== ctn) { + if (ctn !== "") { + var st = 0; + /* + * We defer issuing the task overview + * until we know the class element to + * use for the container + */ + + s += "
" + + "
" + s1; + s += "" + + sai_stateful_taskname(st, ctn, 0) + + "
"; + s1 = ""; + } + all_good = 1; + has_bad = 0; + all_done = 1; + ctn = t.taskname; + wai = 0; + } + + if (t.state < 3) + all_done = 0; + if (t.state !== 3) + all_good = 0; + if (t.state === 4 || t.state == 6) + has_bad = 1; + + s1 += "
"; + if (t.state === 0) + wai = 1; + + s1 += "" + + sai_plat_icon(t.platform, 0) + ""; + + s1 += "
"; + } + + if (ctn !== "") { + var st = 0; + + s += "
" + + "
" + s1; + s += "" + + sai_stateful_taskname(st, ctn, 0) + + "
"; + } + + s += "
"; + } + + s += ""; + + return s; +} + +function render_builders(jso) +{ + var s, n, conts = [], nc = 0; + + s = ""; + s = s + "
"; + +// s = "
"; + + for (n = 0; n < jso.builders.length; n++) { + var e = jso.builders[n], host; + + host = e.name.split('.')[0]; + if (host.split('-')[1]) { + var m; + + for (m = 0; m < conts.length; m++) + if (host.split('-')[1] === conts[m]) + m = 999; + if (m < 999 || !conts.length) + conts.push(host.split('-')[1]); + } + } + + for (nc = 0; nc <= conts.length; nc++) { + var samplat = ""; + + /* + * nc == conts.length means those not + * in a container + */ + + if (nc < conts.length) + s += "
" + conts[nc] + "
"; + + for (n = 0; n < jso.builders.length; n++) { + var e = jso.builders[n], nn, host, plat, cia, sen; + + sen = e.name; + host = sen.split('.')[0]; + cia = host.split('-')[1]; + + console.log("host " + host + ", cia " + cia); + + if ((nc == conts.length && !cia) || + (cia && cia === conts[nc])) { + var did = 0, arc; + + if (!n || + host !== jso.builders[n - 1].name.split('.')[0] || + e.platform != samplat) { + s += "
" + + sai_plat_icon(e.platform, 1); + + samplat = e.platform; + did = 1; + } + + if (n + 1 == jso.builders.length || did) + s += "
"; + } + } + if (nc < conts.length) + s += "
"; + } + + s += "
"; + + return s; +} + +function ws_open_sai() { - sai = new WebSocket(get_appropriate_ws_url() + "/browse", "com-warmcat-sai"); + var s = "", q, qa, qi, q5, q5s; - console.log(get_appropriate_ws_url() + "/browse"); + q = window.location.href; + console.log(q); + qi = q.indexOf("/git/"); + if (qi !== -1) { + /* it has the /git/... does it have the project? */ + s += "/specific"; + q5 = q.substring(qi + 5); + console.log("q5 = " + q5); + q5s = q5.indexOf("/"); + if (q5s !== -1) + s += "/" + q5.substring(0, q5s); + else + s += "/" + q5; + + /* + * gitohashi has ?h=branch and ?id=hash possible + */ + qa = q.split("?"); + if (qa[1]) + s += "?" + qa[1]; + console.log(s); + gitohashi_integ = 1; + } + + sai = new WebSocket(get_appropriate_ws_url() + "/sai/browse" + s, + "com-warmcat-sai"); try { sai.onopen = function() { - // document.getElementById("title").innerHTML = "Server Status (Active)"; - lws_gray_out(false); + var par = new URLSearchParams(window.location.search), + tid, eid; + tid = par.get('task'); + eid = par.get('event'); + + + if (tid) { + /* + * We're being the page monitoring / reporting + * on what happened with a specific task... ask + * about the specific task on the ws link + */ + + console.log("tid " + tid); + + sai.send("{\"schema\":" + + "\"com.warmcat.sai.taskinfo\"," + + "\"logs\": 1," + + "\"task_hash\":" + + JSON.stringify(tid) + "}"); + + return; + } + + if (eid) { + /* + * We're being the page monitoring / reporting + * on what happened with a specific event... ask + * about the specific event on the ws link + */ + + console.log("eid " + eid); + + sai.send("{\"schema\":" + + "\"com.warmcat.sai.eventinfo\"," + + "\"event_hash\":" + + JSON.stringify(eid) + "}"); + + return; + } + + /* + * request the overview schema + */ + + sai.send("{\"schema\":" + + "\"com.warmcat.sai.taskinfo\"}"); }; sai.onmessage =function got_packet(msg) { var u, ci, n; + var now_ut = Math.round((new Date().getTime() / 1000)); + console.log(msg.data); - if (msg.data.length < 100) - return; + // if (msg.data.length < 10) + // return; jso = JSON.parse(msg.data); - u = parseInt(san(jso.i.uptime), 10); - - if (parseInt(jso.i.contexts[0].deprecated, 10) === 0) - s = "
"; - else - s = ""; - - for (ci = 0; ci < jso.i.contexts.length; ci++) { - - if (parseInt(jso.i.contexts[ci].deprecated, 10) === 0) - s += ""; + if (document.getElementById("dlogsn")) + document.getElementById("dlogsn").innerHTML = lines; - } // context - s = s + "
"; - s += - "Server" + - "Server Version: " + - san(jso.i.version) + "
" + - "Host Uptime: " + - ((u / (24 * 3600)) | 0) + "d " + - (((u % (24 * 3600)) / 3600) | 0) + "h " + - (((u % 3600) / 60) | 0) + "m"; - if (jso.i.l1) - s = s + ", Host Load: " + san(jso.i.l1) + " "; - if (jso.i.l2) - s = s + san(jso.i.l2) + " "; - if (jso.i.l3) - s = s + san(jso.i.l3); - if (jso.i.l1) - s =s + ""; - - if (jso.i.statm) { - var sm = jso.i.statm.split(" "); - s += ", Virt stack + heap Usage: " + - humanize(parseInt(sm[0], 10) * 4096) + "B"; + console.log(jso.schema); + + if (jso.alang) { + var a = jso.alang.split(","), n; + + for (n = 0; n < a.length; n++) { + var b = a[n].split(";"); + switch (b[0]) { + case "ja": + i18n.translator.add(JSON.parse(lang_ja)); + n = a.length; + break; + case "zh_TW": + case "zh_HK": + case "zh_SG": + case "zh_HANT": + case "zh-TW": + case "zh-HK": + case "zh-SG": + case "zh-HANT": + i18n.translator.add(JSON.parse(lang_zht)); + n = a.length; + break; + case "zh": + case "zh_CN": + case "zh_HANS": + case "zh-CN": + case "zh-HANS": + i18n.translator.add(JSON.parse(lang_zhs)); + n = a.length; + break; + case "en": + case "en_US": + case "en-US": + n = a.length; + break; + } + } } + + if (jso.schema == "com.warmcat.sai.builders") { + + s = render_builders(jso); - for (n = 0; n < jso.files.length; n++) { - s += "
" + san(jso.files[n].path) + ":
" + san(jso.files[n].val); + if (document.getElementById("sai_builders")) + document.getElementById("sai_builders").innerHTML = s; } - s += "
" + - "Active Context"; - else - s += "
" + - "Deprecated Context " + ci + ""; - - u = parseInt(san(jso.i.contexts[ci].context_uptime), 10); - s += "Server Uptime: " + - ((u / (24 * 3600)) | 0) + "d " + - (((u % (24 * 3600)) / 3600) | 0) + "h " + - (((u % 3600) / 60) | 0) + "m"; - - s = s + - "
" + - "Listening wsi: " + san(jso.i.contexts[ci].listen_wsi) + ", " + - "Current wsi alive: " + (parseInt(san(jso.i.contexts[ci].wsi_alive), 10) - - parseInt(san(jso.i.contexts[ci].listen_wsi), 10)) + "
" + - "Total Rx: " + humanize(san(jso.i.contexts[ci].rx)) +"B, " + - "Total Tx: " + humanize(san(jso.i.contexts[ci].tx)) +"B
" + - - "CONNECTIONS: HTTP/1.x: " + san(jso.i.contexts[ci].h1_conn) +", " + - "Websocket: " + san(jso.i.contexts[ci].ws_upg) +", " + - "H2 upgrade: " + san(jso.i.contexts[ci].h2_upg) +", " + - "H2 ALPN: " + san(jso.i.contexts[ci].h2_alpn) +", " + - "Rejected: " + san(jso.i.contexts[ci].rejected) +"
" + - - "TRANSACTIONS: HTTP/1.x: " + san(jso.i.contexts[ci].h1_trans) + ", " + - "H2: " + san(jso.i.contexts[ci].h2_trans) +", " + - "Total H2 substreams: " + san(jso.i.contexts[ci].h2_subs) +"
" + - - "CGI: alive: " + san(jso.i.contexts[ci].cgi_alive) + ", " + - "spawned: " + san(jso.i.contexts[ci].cgi_spawned) + - ""; - - for (n = 0; n < jso.i.contexts[ci].pt.length; n++) { - - if (parseInt(jso.i.contexts[ci].deprecated, 10) === 0) - s += ""; - + + if (jso.schema == "sai.warmcat.com.overview") { + s = "
  service thread " + (n + 1); - else - s += "
  service thread " + (n + 1); - s += "" + - "fds: " + san(jso.i.contexts[ci].pt[n].fds_count) + " / " + - san(jso.i.contexts[ci].pt_fd_max) + ", "; - s = s + "ah pool: " + san(jso.i.contexts[ci].pt[n].ah_pool_inuse) + " / " + - san(jso.i.contexts[ci].ah_pool_max) + ", " + - "ah waiting list: " + san(jso.i.contexts[ci].pt[n].ah_wait_list); - - s = s + "
"; + + if (jso.overview.length) + for (n = jso.overview.length - 1; n >= 0; n--) + s += sai_event_render(jso.overview[n], now_ut, 1); + + s = s + "
"; + + if (document.getElementById("sai_sticky")) + document.getElementById("sai_sticky").innerHTML = s; + aging(); + + if (jso.overview.length) + for (n = jso.overview.length - 1; n >= 0; n--) { + if (document.getElementById("rebuild-ev-" + san(jso.overview[n].e.uuid))) + document.getElementById("rebuild-ev-" + san(jso.overview[n].e.uuid)). + addEventListener("click", function(e) { + console.log(e); + var rs= "{\"schema\":" + + "\"com.warmcat.sai.eventreset\"," + + "\"uuid\": " + + JSON.stringify(san(e.srcElement.id.substring(11))) + "}"; + + console.log(rs); + sai.send(rs); + }); + if (document.getElementById("delete-ev-" + san(jso.overview[n].e.uuid))) + document.getElementById("delete-ev-" + san(jso.overview[n].e.uuid)). + addEventListener("click", function(e) { + console.log(e); + var rs= "{\"schema\":" + + "\"com.warmcat.sai.eventdelete\"," + + "\"uuid\": " + + JSON.stringify(san(e.srcElement.id.substring(10))) + "}"; + + console.log(rs); + sai.send(rs); + }); + } + } + + if (jso.schema == "com.warmcat.sai.taskinfo") { + s = sai_taskinfo_render(jso); + if (document.getElementById("sai_sticky")) + document.getElementById("sai_sticky").innerHTML = s; + + + s = ""; + while (li--) { + en += "" + + lli + "
"; + tn += "
" + lli++; } + + lines += en; + times += tn; + + if (!redpend) { + redpend = 1; + setTimeout(function() { + redpend = 0; + locked = document.body.scrollHeight - + document.body.clientHeight <= + document.body.scrollTop + 1; + + if (document.getElementById("logs")) { + document.getElementById("logs").innerHTML = logs; - s += "
" +
+		"" +
+		"" +
+	     "
" +
+		"
" + lines + "
" +
+		"
" + times + "
" +
+	     "" + logs +
+	     	""+
+		"
"; + + if (document.getElementById("sai_overview")) + document.getElementById("sai_overview").innerHTML = s; + + if (document.getElementById("rebuild-" + san(jso.t.uuid))) { + document.getElementById("rebuild-" + san(jso.t.uuid)). + addEventListener("click", function(e) { + var rs= "{\"schema\":" + + "\"com.warmcat.sai.taskreset\"," + + "\"uuid\": " + + JSON.stringify(san(e.srcElement.id.substring(8))) + "}"; + + console.log(rs); + sai.send(rs); + document.getElementById("dlogsn").innerHTML = ""; + document.getElementById("dlogst").innerHTML = ""; + document.getElementById("logs").innerHTML = ""; + lines = times = logs = ""; + tfirst = 0; + lli = 1; + }); } - for (n = 0; n < jso.i.contexts[ci].vhosts.length; n++) { - if (parseInt(jso.i.contexts[ci].deprecated, 10) === 0) - s += "
  vhost " + (n + 1); - else - s += "
  vhost " + (n + 1); - s += ""; - if (jso.i.contexts[ci].vhosts[n].use_ssl === "1") - s = s + "https://"; - else - s = s + "http://"; - s = s + san(jso.i.contexts[ci].vhosts[n].name) + ":" + - san(jso.i.contexts[ci].vhosts[n].port) + ""; - if (jso.i.contexts[ci].vhosts[n].sts === "1") - s = s + " (STS)"; - s = s +"
" + - - "Total Rx: " + humanize(san(jso.i.contexts[ci].vhosts[n].rx)) +"B, " + - "Total Tx: " + humanize(san(jso.i.contexts[ci].vhosts[n].tx)) +"B
" + - - "CONNECTIONS: HTTP/1.x: " + san(jso.i.contexts[ci].vhosts[n].h1_conn) +", " + - "Websocket: " + san(jso.i.contexts[ci].vhosts[n].ws_upg) +", " + - "H2 upgrade: " + san(jso.i.contexts[ci].vhosts[n].h2_upg) +", " + - "H2 ALPN: " + san(jso.i.contexts[ci].vhosts[n].h2_alpn) +", " + - "Rejected: " + san(jso.i.contexts[ci].vhosts[n].rejected) +"
" + + + if (document.getElementById("stop-" + san(jso.t.uuid))) { + document.getElementById("stop-" + san(jso.t.uuid)). + addEventListener("click", function(e) { + var rs= "{\"schema\":" + + "\"com.warmcat.sai.taskcan\"," + + "\"task_uuid\": " + + JSON.stringify(san(e.srcElement.id.substring(5))) + "}"; + console.log(rs); + sai.send(rs); + }); + } + + aging(); + } + + + if (jso.schema == "com-warmcat-sai-artifact") { + console.log(jso); + + sai_arts += "
 " + + san(jso.blob_filename) + " " + + humanize(jso.len) + "B
"; + + if (document.getElementById("sai_arts")) + document.getElementById("sai_arts").innerHTML = sai_arts; + + } + + if (jso.schema == "com-warmcat-sai-logs") { + var s1 = atob(jso.log), s = hsanitize(s1), li, + en = "", yo, dh, ce, tn = ""; - "TRANSACTIONS: HTTP/1.x: " + san(jso.i.contexts[ci].vhosts[n].h1_trans) + ", " + - "H2: " + san(jso.i.contexts[ci].vhosts[n].h2_trans) +", " + - "Total H2 substreams: " + san(jso.i.contexts[ci].vhosts[n].h2_subs) +"
"; + if (!tfirst) + tfirst = jso.timestamp; + + li = (s1.match(/\n/g)||[]).length; + + switch (jso.channel) { + case 1: + logs += s; + break; + case 2: + logs += "" + s + + ""; + break; + case 3: + logs += "\u{25a0} " + s + + ""; + break; + case 4: + logs += "" + s + + ""; + break; + default: + logs += "" + s + + ""; + + + } + + if (!cont[jso.channel] && jso.len) + tn = ((jso.timestamp - tfirst) / 1000000).toFixed(4); + + cont[jso.channel] = (li == 0); - if (jso.i.contexts[ci].vhosts[n].mounts) { - s = s + ""; - - var m; - for (m = 0; m < jso.i.contexts[ci].vhosts[n].mounts.length; m++) { - s = s + ""; - } - s = s + "
MountpointOriginCache Policy
"; - s = s + "" + san(jso.i.contexts[ci].vhosts[n].mounts[m].mountpoint) + - "" + - san(jso.i.contexts[ci].vhosts[n].mounts[m].origin) + - ""; - if (parseInt(san(jso.i.contexts[ci].vhosts[n].mounts[m].cache_max_age), 10)) - s = s + "max-age: " + - san(jso.i.contexts[ci].vhosts[n].mounts[m].cache_max_age) + - ", reuse: " + - san(jso.i.contexts[ci].vhosts[n].mounts[m].cache_reuse) + - ", reval: " + - san(jso.i.contexts[ci].vhosts[n].mounts[m].cache_revalidate) + - ", inter: " + - san(jso.i.contexts[ci].vhosts[n].mounts[m].cache_intermediaries); - s = s + "
"; - } - s = s + "
"; - - document.getElementById("conninfo").innerHTML = s; + if (document.getElementById("dlogst")) + document.getElementById("dlogst").innerHTML = times; + } + + if (locked) + document.body.scrollTop = + document.body.scrollHeight - + document.body.clientHeight; + }, 500); + } + } }; sai.onclose = function(){ -// document.getElementById("title").innerHTML = "Server Status (Disconnected)"; - lws_gray_out(true,{"zindex":"499"}); +// document.getElementById("title").innerHTML = +// "Server Status (Disconnected)"; +// lws_gray_out(true,{"zindex":"499"}); - myVar = setTimeout(ws_open_server_status, 3000); + myVar = setTimeout(ws_open_sai, 10000); }; } catch(exception) { alert("

Error" + exception); @@ -241,9 +1165,22 @@ function ws_open_server_status() window.addEventListener("load", function() { - lws_gray_out(true,{"zindex":"499"}); + if (document.getElementById("noscript")) + document.getElementById("noscript").display = "none"; + ws_open_sai(); + aging(); + - ws_open_server_status(); + setInterval(function() { + + var locked = document.body.scrollHeight - + document.body.clientHeight <= document.body.scrollTop + 1; + + if (locked) + document.body.scrollTop = document.body.scrollHeight - + document.body.clientHeight; + + }, 500) }, false); diff --git a/assets/sai.svg b/assets/sai.svg index bf913b2..cdc6d1b 100644 --- a/assets/sai.svg +++ b/assets/sai.svg @@ -1,84 +1,52 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + +image/svg+xml + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/assets/stop.svg b/assets/stop.svg new file mode 100644 index 0000000..462efaf --- /dev/null +++ b/assets/stop.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/assets/surprised.png b/assets/surprised.png new file mode 100644 index 0000000..973273a Binary files /dev/null and b/assets/surprised.png differ diff --git a/assets/tc-gcc.svg b/assets/tc-gcc.svg new file mode 100644 index 0000000..564b0c1 --- /dev/null +++ b/assets/tc-gcc.svg @@ -0,0 +1 @@ + diff --git a/assets/tc-llvm.svg b/assets/tc-llvm.svg new file mode 100644 index 0000000..8817783 --- /dev/null +++ b/assets/tc-llvm.svg @@ -0,0 +1 @@ +LLVM diff --git a/assets/tc-mingw32.svg b/assets/tc-mingw32.svg new file mode 100644 index 0000000..5524cfa --- /dev/null +++ b/assets/tc-mingw32.svg @@ -0,0 +1 @@ + diff --git a/assets/tc-mingw64.svg b/assets/tc-mingw64.svg new file mode 100644 index 0000000..a004222 --- /dev/null +++ b/assets/tc-mingw64.svg @@ -0,0 +1 @@ + diff --git a/assets/tc-msvc.svg b/assets/tc-msvc.svg new file mode 100644 index 0000000..dd47f12 --- /dev/null +++ b/assets/tc-msvc.svg @@ -0,0 +1 @@ + diff --git a/assets/ubuntu-focal-aarch64.svg b/assets/ubuntu-focal-aarch64.svg new file mode 100644 index 0000000..0a4b1e3 --- /dev/null +++ b/assets/ubuntu-focal-aarch64.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/assets/virt-qemu.svg b/assets/virt-qemu.svg new file mode 100644 index 0000000..9d9cd02 --- /dev/null +++ b/assets/virt-qemu.svg @@ -0,0 +1,167 @@ + + + + + + + + + + + + + + + + + + + + + + +image/svg+xml + + + + + + + + + + + + + + + + + + + + + + diff --git a/assets/windows-10.png b/assets/windows-10.png new file mode 100644 index 0000000..79b9c67 Binary files /dev/null and b/assets/windows-10.png differ diff --git a/assets/windows-10.svg b/assets/windows-10.svg new file mode 100644 index 0000000..fbdd1a2 --- /dev/null +++ b/assets/windows-10.svg @@ -0,0 +1 @@ + diff --git a/cmake/Description b/cmake/Description new file mode 100644 index 0000000..593fcb6 --- /dev/null +++ b/cmake/Description @@ -0,0 +1,2 @@ +Sai is a CI build tool to coordinate and perform distributed +building of git commits on a variety of platforms. diff --git a/etc-sai-EXAMPLE/builder/conf b/etc-sai-EXAMPLE/builder/conf index 83360cd..b371b51 100644 --- a/etc-sai-EXAMPLE/builder/conf +++ b/etc-sai-EXAMPLE/builder/conf @@ -4,12 +4,13 @@ # "systemd-nspawn": { "path": "/home/sai/sai-nspawn", + "overlaypath": "/home/sai/sai-nspawn", "instances": 3, "timeout": 1800 }, # - # physically connected targets, like microconstroller boards + # physically connected targets, like microcontroller boards # "target": [ { @@ -31,7 +32,7 @@ # "masters": [ { -# "url": "wss://warmcat.com/builder", +# "url": "wss://warmcat.com/sai/builder", "url": "wss://localhost:443/builder", "selfsigned": 1, "priority": 1 diff --git a/etc-sai-EXAMPLE/master/conf.d/unixskt b/etc-sai-EXAMPLE/master/conf.d/unixskt index 6c5fddc..527045f 100644 --- a/etc-sai-EXAMPLE/master/conf.d/unixskt +++ b/etc-sai-EXAMPLE/master/conf.d/unixskt @@ -1,21 +1,24 @@ -# you should create one file like this per vhost in /etc/sai/conf.d +# you should create one file like this per vhost in /etc/sai/master/conf.d { "vhosts": [{ # this has no special meaning but needs to be unique "name": "unixskt", "unix-socket": 1, + # this should be owned by sai and allow group access from + # whatever group your webserver runs under + "unix-socket-perms": "sai:apache", # multiple vhosts can exist with different unix skt names - "interface": "/var/run/sai-unixskt", + "interface": "/var/run/sai", # required for avatar cache "enable-client-ssl": "on", "mounts": [ { - # static assets like js, css, fonts "mountpoint": "/sai", "default": "index.html", "origin": "file:///usr/local/share/sai/assets", + "origin": "callback://com-warmcat-sai", "cache-max-age": "7200", "cache-reuse": "1", "cache-revalidate": "0", @@ -45,16 +48,33 @@ # "headers": [{ - "content-security-policy": "default-src 'none'; img-src 'self' data: https://travis-ci.org https://api.travis-ci.org https://ci.appveyor.com https://scan.coverity.com; script-src 'self'; font-src 'self'; style-src 'self'; connect-src 'self'; frame-ancestors 'none'; base-uri 'none';", + "content-security-policy": "default-src 'none'; img-src 'self' data:; script-src 'self'; font-src 'self'; style-src 'self'; connect-src 'self'; frame-ancestors 'none'; base-uri 'none';", "x-content-type-options": "nosniff", "x-xss-protection": "1; mode=block", - "x-frame-options": "deny", "referrer-policy": "no-referrer" }], "ws-protocols": [ - {"com-warmcat-sai": { + # this is the lhs of the path to use, not a full filepath + # the events go in ...-events.sqlite3 and the related + # info for an event goes in its own database file in + # ...-event-xxxx.sqlite3 where xxxx is the event uuid. + # + "database": "/var/lib/sai/sai-master", + # + # the push hook notification script creates a JSON + # object and signs it with a secret... "notification- + # key" should match the secret the notification hook + # script has that we're willing to accept. + # + # It should be a 64-char hex-ascii representation of + # a random 32-byte sequence, you can produce a strong + # random key in the correct format like this + # + # dd if=/dev/random bs=32 count=1 | sha256sum | cut -d' ' -f1 + # + "notification-key": "xxxxxxxxxxxx", # template HTML to use for this vhost. You'd normally # copy this to gitohashi-vhostname.html and modify it # to show the content, logos, links, fonts, css etc for @@ -105,8 +125,10 @@ # this dir is served via avatar-proxy # "cache-dir": "/var/cache/libjsongit2" - }} + } + } ] } ] } + diff --git a/hooks/sai.sh b/hooks/sai.sh new file mode 100755 index 0000000..b01e315 --- /dev/null +++ b/hooks/sai.sh @@ -0,0 +1,65 @@ +#!/bin/sh + +REPO_FETCH_URL_BASE="https://libwebsockets.org/repo" + +if [ $(git rev-parse --is-bare-repository) = true ] +then + RN=$(basename "$PWD") +else + RN=$(basename $(readlink -nf "$PWD"/..)) +fi + +RN=${RN%.git} + +echo sai update hook $RN... +rm -f .sai.json .sai.json.b64 +git show $3 -- .sai.json | patch -p1 --merge +cat .sai.json +cat .sai.json | base64 -w0 > .sai.json.b64 +SJL=`stat .sai.json.b64 -c %s` + +if [ -z $SJL -o $SJL = "0" ] ; then + cat /home/sai/.sai.json | base64 -w0 > .sai.json.b64 + SJL=`stat .sai.json.b64 -c %s` +fi + + + +TF=`mktemp` + +echo "{\"schema\":\"com-warmcat-sai-notification\"," > $TF +echo " \"action\":\"repo-update\"," >> $TF +echo " \"repository\":{" >> $TF +echo " \"name\":\"$RN\"" >> $TF +echo " \"fetchurl\":\"$REPO_FETCH_URL_BASE\"" >> $TF +echo " }," >> $TF +echo " \"nonce\":\"`dd if=/dev/urandom bs=32 count=1 | sha256sum | cut -d' ' -f1`\"," >> $TF +echo " \"ref\":\"$1\"," >> $TF +echo " \"hash\":\"$3\"," >> $TF +echo " \"saifile_len\":$SJL," >> $TF +echo -n " \"saifile\":\"" >> $TF +# disallow any nested JSON monkey business by base64-encoding it +cat .sai.json.b64 >> $TF +echo "\"" >> $TF +echo "}" >> $TF + +HM=`cat $TF | sha256hmac -k /etc/sai/private/lws-sai-notification-token | cut -d' ' -f1` + +if [ $SJL = "0" ] ; then + echo "No saifile" + exit 0 +fi + +cat $TF +echo $HM + +curl --header "authorization: sai sha256=$HM" \ + -F"file=@$TF;type=application/json" \ + -A "sai-notifier" \ + https://warmcat.com/sai/update-hook +#curl --header "X-Sai-Signature: sha256=$HM" -F"file=@$TF;name=notification;type=application/json" -A "sai-notifier" http://127.0.0.1:4444 + +rm -f $TF + +exit 0 + diff --git a/scripts/builder-conf b/scripts/builder-conf new file mode 100644 index 0000000..c5ea5c1 --- /dev/null +++ b/scripts/builder-conf @@ -0,0 +1,14 @@ +{ + "perms": "sai:nobody", + "home": "/home/sai", + "host": "myhostname", + + "platforms": [ + { + "name": "linux-ubuntu-xenial-amd64", + "instances": 1, + "masters": [ "wss://warmcat.com/sai/builder" ] + } + ] +} + diff --git a/scripts/gen-selfsigned-certs.sh b/scripts/gen-selfsigned-certs.sh index 261826b..2ea08bd 100755 --- a/scripts/gen-selfsigned-certs.sh +++ b/scripts/gen-selfsigned-certs.sh @@ -5,13 +5,20 @@ if [ ! -z "$1" ] ; then DEST=$1 fi -sudo openssl ecparam -name secp256r1 -genkey -out $DEST/sai-selfsigned-key.pem +sudo openssl ecparam -name secp256r1 -genkey -out $DEST/sai-selfsigned-key.pem.1 echo -e "\n\n\n\n\n\n" | \ sudo openssl req -new -x509 \ -sha256 \ -subj "/CN=localhost" \ -days 10000 \ - -key $DEST/sai-selfsigned-key.pem \ + -key $DEST/sai-selfsigned-key.pem.1 \ -out $DEST/sai-selfsigned-cert.pem + +# +# mbedtls can't handle the extra EC parameters stanza +# +sudo sed '/^-----BEGIN\ EC\ PARAMETERS-----/,/^-----END\ EC\ PARAMETERS-----/{/^#/!{/^\$/!d}}' $DEST/sai-selfsigned-key.pem.1 > sai-selfsigned-key.pem +sudo cp sai-selfsigned-key.pem $DEST +sudo rm sai-selfsigned-key.pem echo diff --git a/scripts/sai-builder.service b/scripts/sai-builder.service new file mode 100644 index 0000000..96f3c30 --- /dev/null +++ b/scripts/sai-builder.service @@ -0,0 +1,10 @@ +[Unit] +Description=Sai Builder + +[Service] +Environment="LWS_NTP_SERVER=192.168.4.1" +ExecStart=/usr/local/bin/sai-builder + +[Install] +WantedBy=multi-user.target + diff --git a/scripts/sai-master.service b/scripts/sai-master.service new file mode 100644 index 0000000..041c738 --- /dev/null +++ b/scripts/sai-master.service @@ -0,0 +1,13 @@ +[Unit] +Description=Sai Master +After=syslog.target + +[Service] +ExecStart=/usr/local/bin/sai-master +ExecReload=/usr/bin/kill -HUP $MAINPID +ExecStop=/usr/bin/killall sai-master + +[Install] +WantedBy=multi-user.target + + diff --git a/src/builder/CMakeLists.txt b/src/builder/CMakeLists.txt new file mode 100644 index 0000000..62d9b30 --- /dev/null +++ b/src/builder/CMakeLists.txt @@ -0,0 +1,89 @@ +set(CPACK_DEBIAN_BUILDER_PACKAGE_NAME "sai-builder") + +set(SRCS + b-sai.c + b-conf.c + b-comms.c + b-mirror.c + b-nspawn.c + b-task.c + b-artifacts.c + b-logproxy.c +) + +set(requirements 1) +require_lws_config(LWS_WITH_CLIENT 1 requirements) +require_lws_config(LWS_WITH_SPAWN 1 requirements) +require_lws_config(LWS_WITH_STRUCT_JSON 1 requirements) +require_lws_config(LWS_WITH_SECURE_STREAMS 1 requirements) +require_lws_config(LWS_WITH_DIR 1 requirements) + +if (requirements) + add_executable("sai-builder" ${SRCS}) + if (APPLE) + set_property(TARGET sai-builder PROPERTY MACOSX_RPATH YES) + endif() + + # + # libgit2 paths (builder) + # + if (SAI_BUILDER) + # + # libgit2 paths + # + find_path( SAI_GIT2_INC_PATH NAMES "git2.h") + find_library(SAI_GIT2_LIB_PATH NAMES "git2") + + if (SAI_GIT2_INC_PATH AND SAI_GIT2_LIB_PATH) + set(SAI_DEPLIBS "${SAI_GIT2_LIB_PATH}" ${SAI_DEPLIBS}) + include_directories(BEFORE "${SAI_GIT2_INC_PATH}") + else() + message(FATAL_ERROR " Unable to find libgit2") + endif() + + target_link_libraries("sai-builder" ${SAI_GIT2_LIB_PATH}) + endif() + + set(CMAKE_REQUIRED_LIBRARIES ${SAI_GIT2_LIB_PATH}) + + CHECK_C_SOURCE_COMPILES("#include + int main(void) { + const git_error *e = git_error_last(); + return !!e->message; + }" SAI_HAVE_LIBGIT2_GIT_ERROR) + + if (SAI_HAVE_LIBGIT2_GIT_ERROR) + set(CMAKE_C_FLAGS "-DSAI_HAVE_LIBGIT2_GIT_ERROR ${CMAKE_C_FLAGS}") + endif() + + + if (SAI_LWS_INC_PATH) + target_include_directories("sai-builder" PRIVATE ${SAI_LWS_INC_PATH}) + endif() + + if (HAS_LIBCAP) + find_library(CAP_LIB_PATH NAMES "cap") + endif() + + target_link_libraries("sai-builder" ${SAI_LWS_LIB_PATH}) + + if (LWS_OPENSSL_LIBRARIES) + target_link_libraries("sai-builder" ${LWS_OPENSSL_LIBRARIES}) + endif() + + if (SAI_EXT_PTHREAD_LIBRARIES) + target_link_libraries("sai-builder" ${SAI_EXT_PTHREAD_LIBRARIES}) + endif() + if (HAS_LIBCAP) + target_link_libraries("sai-builder" ${CAP_LIB_PATH}) + endif() + + if (MSVC OR WIN32) + target_link_libraries("sai-builder" ws2_32.lib userenv.lib psapi.lib iphlpapi.lib) + endif() + + install(TARGETS "sai-builder" + RUNTIME DESTINATION "${BIN_DIR}" COMPONENT builder) + +endif(requirements) +include(CPack) diff --git a/src/builder/b-artifacts.c b/src/builder/b-artifacts.c new file mode 100644 index 0000000..b16767f --- /dev/null +++ b/src/builder/b-artifacts.c @@ -0,0 +1,183 @@ +/* + * sai-builder com-warmcat-sai client protocol implementation + * + * Copyright (C) 2019 - 2020 Andy Green + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation: + * version 2.1 of the License. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + * MA 02110-1301 USA + * + * This deals with POSTing artifacts to the related sai-master, using a secret + * that was passed to the builder as part of the task that was allocated + */ + +#include + +#include "b-private.h" + +#include +#include +#include + +#ifdef _MSC_VER +/* don't complain about our use of open */ +#pragma warning(disable:4996) +#endif + +static int +saib_artifact_rx(void *userobj, const uint8_t *buf, size_t len, int flags) +{ + // sai_artifact_t *ap = (sai_artifact_t *)userobj; + + return 0; +} + +static int +saib_artifact_tx(void *userobj, lws_ss_tx_ordinal_t ord, uint8_t *buf, + size_t *len, int *flags) +{ + sai_artifact_t *ap = (sai_artifact_t *)userobj; + lws_struct_serialize_t *js; + size_t w; + int n; + + *flags = 0; + + if (!ap->sent_json) { + *flags |= LWSSS_FLAG_SOM; + + js = lws_struct_json_serialize_create( + lsm_schema_json_map_artifact, + LWS_ARRAY_SIZE(lsm_schema_json_map_artifact), + 0, ap); + if (!js) + return -1; + + lws_struct_json_serialize(js, buf, *len, &w); + *len = w; + + ap->sent_json = 1; + lws_ss_request_tx(ap->ss); + lwsl_notice("%s: sent JSON %s\n", __func__, (const char *)buf); + + return 0; + } + + if (ap->fd == -1) + return 1; /* nothing to send */ + + n = read(ap->fd, buf, +#if defined(WIN32) + (unsigned int) +#endif + *len); + if (n < 0) { + lwsl_err("%s: artifact read failed: fd: %d, errno: %d\n", + __func__, ap->fd, errno); + *len = 0; + close(ap->fd); + ap->fd = -1; + return -1; + } + if (!n) { + lwsl_notice("%s: file EOF\n", __func__); + return 1; /* nothing to send */ + } + + ap->ofs += n; + lwsl_debug("%s: %p: writing %d at +%llu / %llu\n", __func__, ap->ss, n, + (unsigned long long)ap->ofs, (unsigned long long)ap->len); + *len = (size_t)n; + if (ap->ofs == ap->len) { + lwsl_notice("%s: reached logical end of artifact\n", __func__); + *flags |= LWSSS_FLAG_EOM; + close(ap->fd); + ap->fd = -1; + } + + /* even if we finished, we want to come back to close */ + lws_ss_request_tx(ap->ss); + + return 0; +} + +static int +saib_artifact_state(void *userobj, void *sh, lws_ss_constate_t state, + lws_ss_tx_ordinal_t ack) +{ + sai_artifact_t *ap = (sai_artifact_t *)userobj; + struct stat s; + + lwsl_info("%s: %s, ord 0x%x\n", __func__, lws_ss_state_name(state), + (unsigned int)ack); + + switch (state) { + case LWSSSCS_CREATING: + // lwsl_notice("%s: CREATING '%s'\n", __func__, ap->path); + ap->fd = open((const char *)ap->opaque_data, O_RDONLY +#if defined(WIN32) + /* ugh */ + | _O_BINARY +#endif + ); + if (ap->fd == -1) + return -1; + if (fstat(ap->fd, &s)) { + close(ap->fd); + ap->fd = -1; + return -1; + } + ap->len = s.st_size; + break; + + case LWSSSCS_DESTROYING: + + // lwsl_notice("%s: LWSSSCS_DESTROYING\n", __func__); + /* + * We clean up everything about the upload here + */ + lws_dll2_remove(&ap->list); + if (ap->fd != -1) { + close(ap->fd); + ap->fd = -1; + } + unlink(ap->path); + break; + + case LWSSSCS_CONNECTED: + // lwsl_user("%s: CONNECTED: %p\n", __func__, ap->ss); + lws_ss_request_tx_len(ap->ss, (unsigned long)ap->len); + break; + + case LWSSSCS_DISCONNECTED: + // lwsl_notice("%s: LWSSSCS_DISCONNECTED\n", __func__); + /* don't retry */ + return -1; + + default: + break; + } + + return 0; +} + +const lws_ss_info_t ssi_sai_artifact = { + .handle_offset = offsetof(sai_artifact_t, ss), + .opaque_user_data_offset = offsetof(sai_artifact_t, opaque_data), + .rx = saib_artifact_rx, + .tx = saib_artifact_tx, + .state = saib_artifact_state, + .user_alloc = sizeof(sai_artifact_t), + .streamtype = "sai_artifact" +}; diff --git a/src/builder/b-comms.c b/src/builder/b-comms.c new file mode 100644 index 0000000..3acb7f6 --- /dev/null +++ b/src/builder/b-comms.c @@ -0,0 +1,571 @@ +/* + * sai-builder com-warmcat-sai client protocol implementation + * + * Copyright (C) 2019 - 2020 Andy Green + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation: + * version 2.1 of the License. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + * MA 02110-1301 USA + */ + +#include +#include +#include + +#include "b-private.h" + +#include "../common/struct-metadata.c" + +static int +saib_m_rx(void *userobj, const uint8_t *buf, size_t len, int flags) +{ + struct sai_plat_master *spm = (struct sai_plat_master *)userobj; + //struct sai_plat *sp = (struct sai_plat *)spm->sai_plat; + + lwsl_info("%s: len %d, flags: %d\n", __func__, (int)len, flags); + lwsl_hexdump_info(buf, len); + + if (saib_ws_json_rx_builder(spm, buf, len)) + return 1; + + return 0; +} + +/* + * We come here for every platform's threadpool sync + */ + +static int +tp_sync_check(struct lws_dll2 *d, void *user) +{ + sai_plat_t *sp = lws_container_of(d, sai_plat_t, sai_plat_list); + struct sai_plat_master *spm = (struct sai_plat_master *)user; + struct sai_nspawn *ns; + int n, soe; + void *vp; + + /* + * Let's look into every nspawn for each platform then... + */ + + lws_start_foreach_dll_safe(struct lws_dll2 *, d, d1, + sp->nspawn_owner.head) { + + ns = lws_container_of(d, struct sai_nspawn, list); + soe = ns->state; + + /* + * We can't deal with nspawns bound to a different master or + * nspawns not with an active threadpool task + */ + + if (!ns->tp_task || spm != ns->spm) + goto next; + + /* + * We may not be the only threadpool task that wants + * to sync... so bear in mind we want to loop after + * handling this particular one + */ + + // lwsl_notice("%s: tp svc, state %d '%s'\n", __func__, + // ns->state, ns->pending_mirror_log); + + /* + * We got here by threadpool sync... logify the + * >saib> message from the thread... + */ + + if (ns->pending_mirror_log[0]) { + lwsl_notice("%s: logging %s\n", __func__, + ns->pending_mirror_log); + saib_log_chunk_create(ns, ns->pending_mirror_log, + strlen(ns->pending_mirror_log), 3); + ns->pending_mirror_log[0] = 0; + } + + //(soe == NSSTATE_WAIT_REMOTE_MIRROR || + // soe == NSSTATE_FAILED || + // soe == NSSTATE_CHECKOUT || + // soe == NSSTATE_CHECKEDOUT) && + + n = lws_threadpool_task_status(ns->tp_task, &vp); + lwsl_info("%s: WRITEABLE: ss=%p: " + "task %p, priv %p, status %d\n", __func__, spm->ss, + ns->tp_task, vp, n); + switch (n) { + case LWS_TP_STATUS_FINISHED: + case LWS_TP_STATUS_STOPPED: + case LWS_TP_STATUS_QUEUED: + case LWS_TP_STATUS_RUNNING: + case LWS_TP_STATUS_STOPPING: + goto next; + + case LWS_TP_STATUS_SYNCING: + /* + * This is what the threadpool thread wants to hear from + * us in order to continue on. The choice in the second + * arg is whether to ask the thread to stop or not. + * + * MIRROR: let thread continue on to CHECKOUT + * WAIT_REMOTE_MIRROR: continue to wait + * CHECKOUT: go back into CHECKOUT + * FAILED or CHECKEDOUT: we're done, stop the thread + * + * This wakes the stalled task, we can't read its + * state after this + */ + + lws_threadpool_task_sync(ns->tp_task, + soe == NSSTATE_CHECKEDOUT || + soe == NSSTATE_FAILED); + + if (soe != NSSTATE_CHECKEDOUT && + soe != NSSTATE_FAILED) { + lwsl_notice("%s: task still going, status %d, " + "sp = %p, sp->ss = %p\n", + __func__, n, sp, ns->spm->ss); + goto next; + } + /* + * We asked for the task to stop... let's move on + * while that's happening + */ + break; + + default: + return 1; + } + + /* + * The thread is over... either FAILED... + */ + + if (soe == NSSTATE_FAILED) { + lwsl_notice("%s: thread over with FAILED\n", + __func__); + saib_set_ns_state(ns, NSSTATE_FAILED); + saib_task_grace(ns); + goto next; + } + + /* + * ...or we did the mirror and let's spawn the actual task now + */ + + lwsl_notice("%s: Destroying checkout thread, spawning task\n", + __func__); + ns->tp_task = NULL; + + lws_sul_cancel(&ns->sul_task_cancel); + + saib_set_ns_state(ns, NSSTATE_BUILD); + + n = saib_spawn(ns); + if (!n) + goto next; + + lwsl_err("%s: spawn failed: %d\n", __func__, n); + + lwsl_notice("%s: failing spawn cleanly\n", __func__); + saib_set_ns_state(ns, NSSTATE_FAILED); + saib_task_grace(ns); + +next: + + if (ns->artifact_owner.head) { + /* + * This nspawn has an outstanding artifact to upload + */ + } + + lwsl_debug("%s: next tp sync\n", __func__); + + } lws_end_foreach_dll_safe(d, d1); + + return 0; +} + +/* + * We cover requested tx for any instance of a platform that can takes tasks + * from the same master... it means just by coming here, no particular + * platform / sai_plat is implied... + */ + +static int +saib_m_tx(void *userobj, lws_ss_tx_ordinal_t ord, uint8_t *buf, size_t *len, + int *flags) +{ + struct sai_plat_master *spm = (struct sai_plat_master *)userobj; + uint8_t *start = buf, *end = buf + (*len) - 1, *p = start; + struct ws_capture_chunk *chunk; + struct sai_plat *sp = NULL; + lws_struct_serialize_t *js; + lws_dll2_t *star, *walk; + struct sai_nspawn *ns; + size_t w = 0; + int n = 0; + + /* + * We are the ss / wsi that any threadpool instances on any platform + * with tasks for this master are trying to sync to. We need to handle + * and resume them all. + */ + + lws_dll2_foreach_safe(&builder.sai_plat_owner, spm, tp_sync_check); + + /* + * Any builder state updates / rejections to process? + */ + + if (spm->rejection_list.count) { + struct lws_dll2 *d = lws_dll2_get_head(&spm->rejection_list); + struct sai_rejection *rej = + lws_container_of(d, struct sai_rejection, list); + + lwsl_notice("%s: issuing %s\n", __func__, + rej->task_uuid[0] ? "task rejection" : "load update"); + + js = lws_struct_json_serialize_create(lsm_schema_json_task_rej, + LWS_ARRAY_SIZE(lsm_schema_json_task_rej), 0, rej); + if (!js) + return -1; + + n = lws_struct_json_serialize(js, start, + lws_ptr_diff(end, start), &w); + lws_struct_json_serialize_destroy(&js); + + lwsl_hexdump_notice(start, w); + + n = (int)w; + + lws_dll2_remove(&rej->list); + free(rej); + + lws_ss_request_tx(spm->ss); + goto sendify; + } + + switch (spm->phase) { + case PHASE_BUILDING: + case PHASE_IDLE: + break; + + default: + + /* + * Update master with platform status + */ + + js = lws_struct_json_serialize_create(lsm_schema_map_plat, + LWS_ARRAY_SIZE(lsm_schema_map_plat), 0, + &builder.sai_plat_owner); + if (!js) + return -1; + + n = lws_struct_json_serialize(js, start, + lws_ptr_diff(end, start), &w); + lws_struct_json_serialize_destroy(&js); + + sp = (sai_plat_t *)builder.sai_plat_owner.head; + lwsl_hexdump_warn(sp, sizeof(*sp)); + lwsl_hexdump_notice(start, w); + + *len = (int)w; + spm->phase = PHASE_IDLE; + *flags = LWSSS_FLAG_SOM | LWSSS_FLAG_EOM; + + if (spm->logs_in_flight) + lws_ss_request_tx(spm->ss); + + return 0; + } + + /* + * Are there some logs to dump? + */ + + if (!spm->logs_in_flight) + return 1; /* nothing to send */ + + /* + * Yes somebody has some logs... since we handle all logs on any + * platform doing tasks for the same master, we have to take care not + * to favour draining logs for any busy tasks over letting others + * getting a chance at the mic. If we just scan for guys with logs + * from the start of the list each time, we will never deal with guys + * far from the list head while anybody closer has logs. + * + * For that reason we remember the last dll2 who wrote logs, and start + * looking for the next nspawn with pending logs after him next time. + * + * That requires statefully rotating through... + * + * platform in platform list : nspawn in platform's nspawn list + * = = + * spm->last_logging_platform : spm->last_logging_nspawn + * + * ... filtered for nspawns associated with our spm / master SS link. + * + * The platforms and nspawns are allocated at conf-time statically. + */ + + star = NULL; + do { + if (!spm->last_logging_nspawn) { + /* start at the start */ + sp = spm->last_logging_platform = lws_container_of( + builder.sai_plat_owner.head, + sai_plat_t, sai_plat_list); + walk = spm->last_logging_nspawn = sp->nspawn_owner.head; + } else { + /* if we can move on, move on */ + sp = spm->last_logging_platform; + walk = spm->last_logging_nspawn->next; + } + + /* if no more nspawns, try moving to next platform */ + if (!walk) { + if (!sp->sai_plat_list.next) + /* if no more platforms, wrap around to first */ + sp = spm->last_logging_platform = + lws_container_of( + builder.sai_plat_owner.head, + sai_plat_t, sai_plat_list); + else + sp = spm->last_logging_platform = + lws_container_of( + sp->sai_plat_list.next, + sai_plat_t, sai_plat_list); + + /* use the first nspawn in our new platform */ + walk = sp->nspawn_owner.head; + } + + spm->last_logging_nspawn = walk; + + if (walk == star) { + lwsl_notice("%s: did not find logs: %d expected\n", + __func__, spm->logs_in_flight); + return 1; /* nothing to do */ + } + + if (!star) /* take first usable one as the starting point */ + star = walk; + + ns = lws_container_of(walk, struct sai_nspawn, list); + if (spm != ns->spm || !ns->chunk_cache.count || !ns->chunk_cache.tail) + continue; + + /* + * We're going to process a chunk + */ + + chunk = lws_container_of(ns->chunk_cache.tail, + struct ws_capture_chunk, list); + + lws_dll2_remove(&chunk->list); + + if (ns->task) { + + n = lws_snprintf((char *)p, lws_ptr_diff(end, p), + "{\"schema\":\"com-warmcat-sai-logs\"," + "\"task_uuid\":\"%s\", \"timestamp\": %llu," + "\"channel\": %d, \"len\": %d, ", + ns->task->uuid, (unsigned long long)lws_now_usecs(), + chunk->stdfd, (int)chunk->len); + + if (ns->finished_when_logs_drained && !ns->chunk_cache.count) + /* + * Let the last guy report the finished state + */ + n += lws_snprintf((char *)p + n, lws_ptr_diff(end, p) - n, + "\"finished\":%d,", ns->retcode); + + n += lws_snprintf((char *)p + n, lws_ptr_diff(end, p) - n, + "\"log\":\""); + + // puts((const char *)&chunk[1]); + // puts((const char *)start); + + n += lws_b64_encode_string((const char *)&chunk[1], + (int)chunk->len, (char *)&start[n], + (int)lws_ptr_diff(end, p) - n - 5); + + p[n++] = '\"'; + p[n++] = '}'; + p[n] = '\0'; + // puts((const char *)start); + } + + ns->chunk_cache_size -= sizeof(*chunk) + chunk->len; + free(chunk); + + spm->logs_in_flight--; + lwsl_debug("%s: spm logs_in_flight %d\n", __func__, + spm->logs_in_flight); + + if (ns->finished_when_logs_drained && !ns->chunk_cache.count) { + + /* + * He's in DONE state, and the draining he was waiting + * for has now happened + */ + lwsl_notice("%s: drained and empty\n", __func__); + saib_task_destroy(ns); + } + + break; + + } while (walk != star); + + +sendify: + + *flags = LWSSS_FLAG_SOM | LWSSS_FLAG_EOM; + *len = n; + + if (spm->phase != PHASE_IDLE || spm->logs_in_flight) + lws_ss_request_tx(spm->ss); + + if (!n) + return 1; + + return 0; +} + +static int +cleanup_on_ss_destroy(struct lws_dll2 *d, void *user) +{ + struct sai_plat_master *spm = (struct sai_plat_master *)user; + sai_plat_t *sp = lws_container_of(d, sai_plat_t, sai_plat_list); + + lws_start_foreach_dll_safe(struct lws_dll2 *, d, d1, + sp->nspawn_owner.head) { + struct sai_nspawn *ns = + lws_container_of(d, struct sai_nspawn, list); + + if (ns->spm == spm && ns->tp) { + lwsl_notice("%s: calling threadpool_destroy\n", __func__); + lws_threadpool_finish(ns->tp); + lws_threadpool_destroy(ns->tp); + ns->tp = NULL; + ns->tp_task = NULL; + } + } lws_end_foreach_dll_safe(d, d1); + + return 0; +} + +static int +cleanup_on_ss_disconnect(struct lws_dll2 *d, void *user) +{ + struct sai_plat_master *spm = (struct sai_plat_master *)user; + sai_plat_t *sp = lws_container_of(d, sai_plat_t, sai_plat_list); + struct ws_capture_chunk *cc; + + lws_start_foreach_dll_safe(struct lws_dll2 *, d, d1, + sp->nspawn_owner.head) { + struct sai_nspawn *ns = lws_container_of(d, + struct sai_nspawn, list); + + if (ns->spm == spm) { + + /* + * This pss is about to go away, make sure the ns + * can't reference it any more no matter what happens + */ + + ns->spm = NULL; + + //if (ns->lsp) + // lws_spawn_piped_kill_child_process(ns->lsp); + + /* clean up any capture chunks */ + + lws_start_foreach_dll_safe(struct lws_dll2 *, e, e1, + ns->chunk_cache.head) { + cc = lws_container_of(e, + struct ws_capture_chunk, list); + lws_dll2_remove(&cc->list); + free(cc); + } lws_end_foreach_dll_safe(e, e1); + + } + } lws_end_foreach_dll_safe(d, d1); + + return 0; +} + +static int +saib_m_state(void *userobj, void *sh, lws_ss_constate_t state, + lws_ss_tx_ordinal_t ack) +{ + struct sai_plat_master *spm = (struct sai_plat_master *)userobj; + + lwsl_user("%s: %s, ord 0x%x\n", __func__, lws_ss_state_name(state), + (unsigned int)ack); + + switch (state) { + case LWSSSCS_DESTROYING: + + /* + * If the logical SS itself is going down, every platform that + * used us to connect to their master and has nspawns are also + * going down + */ + lws_dll2_foreach_safe(&builder.sai_plat_owner, spm, + cleanup_on_ss_destroy); + + break; + + case LWSSSCS_CONNECTED: + lwsl_user("%s: CONNECTED: %p\n", __func__, spm->ss); + spm->phase = PHASE_START_ATTACH; + lws_ss_request_tx(spm->ss); + break; + + case LWSSSCS_DISCONNECTED: + /* + * clean up any ongoing spawns related to this connection + */ + + lwsl_user("%s: DISCONNECTED\n", __func__); + lws_dll2_foreach_safe(&builder.sai_plat_owner, spm, + cleanup_on_ss_disconnect); + break; + + case LWSSSCS_ALL_RETRIES_FAILED: + break; + + case LWSSSCS_QOS_ACK_REMOTE: + lwsl_notice("%s: LWSSSCS_QOS_ACK_REMOTE\n", __func__); + break; + + default: + break; + } + + return 0; +} + +const lws_ss_info_t ssi_sai_builder = { + .handle_offset = offsetof(struct sai_plat_master, ss), + .opaque_user_data_offset = offsetof(struct sai_plat_master, opaque_data), + .rx = saib_m_rx, + .tx = saib_m_tx, + .state = saib_m_state, + .user_alloc = sizeof(struct sai_plat_master), + .streamtype = "sai_builder" +}; diff --git a/src/builder/b-conf.c b/src/builder/b-conf.c new file mode 100644 index 0000000..f037a5f --- /dev/null +++ b/src/builder/b-conf.c @@ -0,0 +1,427 @@ +/* + * sai-builder conf.c + * + * Copyright (C) 2019 - 2020 Andy Green + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation: + * version 2.1 of the License. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + * MA 02110-1301 USA + */ + +#include +#include +#include +#include +#include + +#include "b-private.h" + +/* global part */ + +static const char * const paths_global[] = { + "home", + "perms", + "host", +}; + +enum enum_paths_global { + LEJPM_HOME, + LEJPM_PERMS, + LEJPM_HOST, +}; + +/* platform-related part */ + +static const char * const paths[] = { + + "platforms[].name", + "platforms[].instances", + "platforms[].env[].*", + "platforms[].env[]", + "platforms[].masters", + "platforms[]", +}; + +enum enum_paths { + + LEJPM_PLATFORMS_NAME, + LEJPM_PLATFORMS_INSTANCES, + LEJPM_PLATFORMS_ENV_ITEM, + LEJPM_PLATFORMS_ENV, + LEJPM_PLATFORMS_MASTERS, + LEJPM_PLATFORMS, +}; + +struct jpargs { + struct sai_builder *builder; + + struct sai_nspawn *nspawn; + struct sai_plat *sai_plat; + + struct sai_platform *pl; + + int next_master_index; + int next_plat_index; +}; + +static signed char +saib_conf_cb(struct lejp_ctx *ctx, char reason) +{ + struct jpargs *a = (struct jpargs *)ctx->user; + sai_plat_master_ref_t *mref; + struct sai_plat_master *cm; + struct lws_ss_handle *h; + const char **pp, *pq; + char temp[65]; + int n; + +#if 0 + lwsl_notice(" %d: %s (%d)\n", reason, ctx->path, ctx->path_match); + for (n = 0; n < ctx->wildcount; n++) + lwsl_notice(" %d\n", ctx->wild[n]); +#endif + + if (reason == LEJPCB_OBJECT_START) { + switch (ctx->path_match - 1) { + + case LEJPM_PLATFORMS: + /* + * Create the sai_plat object + */ + a->sai_plat = lwsac_use_zero(&a->builder->conf_head, + sizeof(*a->sai_plat), 4096); + if (!a->sai_plat) + return -1; + + a->sai_plat->instances = 1; /* default */ + + lws_dll2_add_tail(&a->sai_plat->sai_plat_list, + &a->builder->sai_plat_owner); + break; + + default: + return 0; + } + } + + /* we only match on the prepared path strings */ + if (!(reason & LEJP_FLAG_CB_IS_VALUE) || !ctx->path_match) + return 0; + + if (reason == LEJPCB_VAL_NUM_INT) { + n = atoi(ctx->buf); + switch (ctx->path_match - 1) { + + case LEJPM_PLATFORMS_INSTANCES: + lwsl_notice("%s: instances %d\n", __func__, n); + a->sai_plat->instances = n; + /* + * Instantiate n nspawns bound to this platform + */ + + for (n = 0; n < a->sai_plat->instances; n++) { + struct sai_nspawn *ns = malloc(sizeof(*ns)); + if (!ns) + return -1; + + memset(ns, 0, sizeof(*ns)); + ns->instance_idx = n; + ns->builder = &builder; + ns->sp = a->sai_plat; + ns->task = NULL; + lws_dll2_add_head(&ns->list, + &a->sai_plat->nspawn_owner); + } + break; + + default: + break; + } + + return 0; + } + + if (reason != LEJPCB_VAL_STR_END) + return 0; + + /* only the end part of the string, where we know the length */ + + switch (ctx->path_match - 1) { + + case LEJPM_PLATFORMS_ENV: + lwsl_notice("env %s %s\n", ctx->path, ctx->buf); + return 0; + // break; + + case LEJPM_PLATFORMS_NAME: + n = lws_snprintf(temp, sizeof(temp), "%s.%.*s", + builder.host, ctx->npos, ctx->buf); + a->sai_plat->name = lwsac_use(&a->builder->conf_head, n + 1, 512); + memcpy((char *)a->sai_plat->name, temp, n + 1); + lwsl_notice("%s: platform: %.*s, name %s\n", __func__, + ctx->npos, ctx->buf, a->sai_plat->name); + pp = &a->sai_plat->platform; + a->sai_plat->index = a->next_plat_index++; + break; + + case LEJPM_PLATFORMS_MASTERS: + + mref = lwsac_use_zero(&a->builder->conf_head, sizeof(*mref), + 512); + + /* + * The builder as a whole maintains only one SS connection to + * each master. If there are multiple platforms supported by + * the builder that want to accept tasks from the same master, + * only the first platform creates the SS to the master, and + * the others just use that. + * + * The sai_plat_master is instantiated as the ss userdata. + * + * So let's see if we already have the connection created + * for this master... + */ + + lws_start_foreach_dll(struct lws_dll2 *, p, + a->builder->sai_plat_master_owner.head) { + cm = lws_container_of(p, sai_plat_master_t, list); + + if (!strncmp(ctx->buf, cm->url, ctx->npos)) { + /* we already have a logical connection... */ + mref->spm = cm; + cm->refcount++; + lws_dll2_add_tail(&mref->list, + &a->sai_plat->masters); + + return 0; + } + } lws_end_foreach_dll(p); + + /* + * This is the first plat that wants to talk to this master, + * we need to create the logical SS connection + */ + + if (lws_ss_create(builder.context, 0, &ssi_sai_builder, 0, &h, + NULL, NULL)) { + lwsl_err("%s: failed to create secure stream\n", + __func__); + return -1; + } + + cm = lws_ss_to_user_object(h); /* the sai_plat_master object */ + cm->index = a->next_master_index++; + + /* hook the ss up to the master url */ + + cm->url = lwsac_use(&a->builder->conf_head, + 2 *(ctx->npos + 1), 512); + memcpy((char *)cm->url, ctx->buf, ctx->npos); + ((char *)cm->url)[ctx->npos] = '\0'; + + lwsl_notice("%s: binding ss to %s\n", __func__, cm->url); + lws_ss_set_metadata(h, "url", cm->url, strlen(cm->url)); + + pq = cm->url; + while (*pq && (pq[0] != '/' || pq[1] != '/')) + pq++; + + if (*pq) { + n = 0; + pq += 2; + while (pq[n] && pq[n] != '/') + n++; + } else { + pq = cm->url; + n = ctx->npos; + } + + cm->name = cm->url + ctx->npos + 1; + memcpy((char *)cm->name, pq, n); + ((char *)cm->name)[n] = '\0'; + + while (strchr(cm->name, '.')) + *strchr(cm->name, '.') = '_'; + while (strchr(cm->name, '/')) + *strchr(cm->name, '/') = '_'; + + /* add us to the builder list of unique masters */ + lws_dll2_add_head(&cm->list, &a->builder->sai_plat_master_owner); + + /* add us to this platforms's list of masters it accepts */ + mref->spm = cm; + cm->refcount++; + lws_dll2_add_tail(&mref->list, &a->sai_plat->masters); + + lws_ss_client_connect(h); + + return 0; + + default: + return 0; + } + + *pp = lwsac_use(&a->builder->conf_head, ctx->npos + 1, 512); + if (!*pp) + return 1; + memcpy((char *)(*pp), ctx->buf, ctx->npos); + ((char *)(*pp))[ctx->npos] = '\0'; + + return 0; +} + +static signed char +saib_conf_global_cb(struct lejp_ctx *ctx, char reason) +{ + struct jpargs *a = (struct jpargs *)ctx->user; + const char **pp; +#if 0 + int n; + + lwsl_notice("%s: reason: %d, path: %s, match %d\n", __func__, + reason, ctx->path, ctx->path_match); + for (n = 0; n < ctx->wildcount; n++) + lwsl_notice(" %d\n", ctx->wild[n]); +#endif + + + /* we only match on the prepared path strings */ + if (!(reason & LEJP_FLAG_CB_IS_VALUE) || !ctx->path_match) + return 0; + + if (reason != LEJPCB_VAL_STR_END) + return 0; + + /* only the end part of the string, where we know the length */ + + switch (ctx->path_match - 1) { + case LEJPM_HOME: + pp = &a->builder->home; + break; + + case LEJPM_PERMS: + pp = &a->builder->perms; + break; + + case LEJPM_HOST: + pp = &a->builder->host; + break; + + default: + return 0; + } + + *pp = lwsac_use(&a->builder->conf_head, ctx->npos + 1, 512); + if (!*pp) + return 1; + memcpy((char *)(*pp), ctx->buf, ctx->npos); + ((char *)(*pp))[ctx->npos] = '\0'; + + return 0; +} + +int +saib_config_global(struct sai_builder *builder, const char *d) +{ + unsigned char buf[128]; + struct lejp_ctx ctx; + int n, m, fd; + struct jpargs a; + + memset(&a, 0, sizeof(a)); + a.builder = builder; + +#if defined(WIN32) + lws_snprintf((char *)buf, sizeof(buf) - 1, "%s\\conf", d); +#else + lws_snprintf((char *)buf, sizeof(buf) - 1, "%s/conf", d); +#endif + + fd = lws_open((char *)buf, O_RDONLY); + if (fd < 0) { + lwsl_err("Cannot open %s\n", (char *)buf); + return 2; + } + lwsl_info("%s: %s\n", __func__, (char *)buf); + lejp_construct(&ctx, saib_conf_global_cb, &a, + paths_global, LWS_ARRAY_SIZE(paths_global)); + + do { + n = read(fd, buf, sizeof(buf)); + if (!n) + break; + + m = lejp_parse(&ctx, buf, n); + } while (m == LEJP_CONTINUE); + + close(fd); + n = ctx.line; + lejp_destruct(&ctx); + + return 0; +} + +int +saib_config(struct sai_builder *builder, const char *d) +{ + unsigned char buf[128]; + struct lejp_ctx ctx; + int n, m, fd; + struct jpargs a; + + memset(&a, 0, sizeof(a)); + a.builder = builder; + +#if defined(WIN32) + lws_snprintf((char *)buf, sizeof(buf) - 1, "%s\\conf", d); +#else + lws_snprintf((char *)buf, sizeof(buf) - 1, "%s/conf", d); +#endif + + fd = lws_open((char *)buf, O_RDONLY); + if (fd < 0) { + lwsl_err("Cannot open %s\n", (char *)buf); + return 2; + } + lwsl_notice("%s: %s\n", __func__, (char *)buf); + lejp_construct(&ctx, saib_conf_cb, &a, paths, LWS_ARRAY_SIZE(paths)); + + do { + n = read(fd, buf, sizeof(buf)); + if (!n) + break; + + m = lejp_parse(&ctx, buf, n); + } while (m == LEJP_CONTINUE); + + close(fd); + n = ctx.line; + lejp_destruct(&ctx); + + if (m < 0) { + lwsl_err("%s/conf(%u): parsing error %d: %s\n", d, n, m, + lejp_error_to_string(m)); + return 2; + } + + lwsl_notice("%s: parsing completed\n", __func__); + + return 0; +} + +void +saib_config_destroy(struct sai_builder *builder) +{ + lwsac_free(&builder->conf_head); +} diff --git a/src/builder/b-logproxy.c b/src/builder/b-logproxy.c new file mode 100644 index 0000000..0fb9732 --- /dev/null +++ b/src/builder/b-logproxy.c @@ -0,0 +1,61 @@ +/* + * sai-builder - logproxy + * + * Copyright (C) 2019 - 2020 Andy Green + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation: + * version 2.1 of the License. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + * MA 02110-1301 USA + * + * We create a listening UDS socket for each platform / instance / log channel + * so subprocesses can just dump raw log content and have it understood and + * handled correctly, without having to worry about datagram size or coalescing. + */ +#include +#include +#include +#include "b-private.h" + +static int +callback_logproxy(struct lws *wsi, enum lws_callback_reasons reason, + void *user, void *in, size_t len) +{ + struct saib_logproxy *slp = lws_vhost_user(lws_get_vhost(wsi)); + + switch (reason) { + + case LWS_CALLBACK_RAW_RX: + if (!slp->ns->spm) + break; + + /* + * We just turn whatever comes into a log of the appropriate + * channel according to the vhost it appeared on + */ + saib_log_chunk_create(slp->ns, in, len, slp->log_channel_idx); + break; + + default: + break; + } + + return 0; +} + +const struct lws_protocols protocol_logproxy = { + "protocol-logproxy", + callback_logproxy, + 0, + 2048, 2048, NULL, 0 +}; diff --git a/src/builder/b-mirror.c b/src/builder/b-mirror.c new file mode 100644 index 0000000..f83deb2 --- /dev/null +++ b/src/builder/b-mirror.c @@ -0,0 +1,671 @@ +/* + * sai-builder com-warmcat-sai client protocol implementation + * + * Copyright (C) 2019 - 2020 Andy Green + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation: + * version 2.1 of the License. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + * MA 02110-1301 USA + */ + +#include +#include +#include +#include + +#include "b-private.h" + +#include + +enum { + SRFS_REQUESTING, + SRFS_PROCESSING, + SRFS_FAILED, + SRFS_SUCCEEDED, +}; + +/* + * Request to do the remote -> local mirror action + */ + +typedef struct sai_mirror_req { + lws_dll2_t list; + + struct sai_nspawn *ns; + + char path[100]; /* local mirror path */ + char url[100]; + char hash[130]; + char ref[96]; + + int state; +} sai_mirror_req_t; + +static void +sai_mirror_req_state_set(sai_mirror_req_t *req, int n) +{ + lwsl_notice("%s: req %p: %d -> %d\n", __func__, req, req->state, n); + req->state = n; +} + +enum { + SAIB_CHECKOUT_OK, + SAIB_CHECKOUT_NOT_IN_LOCAL_MIRROR, + SAIB_CHECKOUT_CHECKOUT_FAILED, +}; + +static int +sai_mirror_local_checkout(struct sai_nspawn *ns) +{ + git_checkout_options co_opts = GIT_CHECKOUT_OPTIONS_INIT; + git_fetch_options opts = GIT_FETCH_OPTIONS_INIT; + git_repository *git_repo_build_specific = NULL; + char dp[384], spec[96], *paths[] = { spec }; + git_strarray rfs = { paths, 1 }; + git_object *treeish; + git_remote *remote; + int n, tries = 2; + + /* + * Remove anything that was already in the build-specific dir + */ + + lwsl_notice("%s: rm -rf %s\n", __func__, ns->inp); + lws_dir(ns->inp, NULL, lws_dir_rm_rf_cb); + + /* + * Make sure the build-specific dir itself is left standing in there. + * + * We can only create files and dirs using the global sai:nobody + * credentials since we have dropped root long ago + */ + + mkdir(ns->inp, 0755); + + /* + * Create the build-specific git dir and init it + */ + + n = git_repository_init(&git_repo_build_specific, ns->inp, 0); + if (n) { +#if defined(SAI_HAVE_LIBGIT2_GIT_ERROR) + const git_error *e = git_error_last(); + + lwsl_err("%s: unable to init temp repo %s: %d %s\n", + __func__, ns->inp, n, e ? e->message : "?"); +#else + lwsl_err("%s: unable to init temp repo %s: %d\n", + __func__, ns->inp, n); +#endif + + return 1; + } + + /* + * Attempt to fetch the ref we are interested in from our local mirror + * + * Create a temp remote against the destination repo + */ + + if (git_remote_create_anonymous(&remote, git_repo_build_specific, + ns->path)) { + lwsl_err("%s: cant find remote %s\n", __func__, + ns->git_repo_url); + + git_repository_free(git_repo_build_specific); + return 1; + } + + lws_snprintf(spec, sizeof(spec), "ref-%s:ref-%s", ns->hash, ns->hash); + + lwsl_notice("%s: Attempting to fetch %s from mirror to %s\n", + __func__, spec, ns->inp); + + /* + * Fetch the tree from the local mirror to our local build repo + * This may take an open-ended amount of time + */ + + n = git_remote_fetch(remote, &rfs, &opts, "fetch"); + git_remote_free(remote); + if (n) { + lwsl_notice("%s: git_remote_fetch() says %d\n", __func__, n); +#if defined(SAI_HAVE_LIBGIT2_GIT_ERROR) + const git_error *e = git_error_last(); + + lwsl_err("%s: git_remote_fetch libgit err: %d %s\n", + __func__, n, e ? e->message : "?"); +#endif + git_repository_free(git_repo_build_specific); + return SAIB_CHECKOUT_NOT_IN_LOCAL_MIRROR; + } + + /* + * Check out the commit we fetched into the ephemeral local build dir... + * this should be a formality since we already retreived the ref into + * ephemeral local build dir's repo. + */ + + lws_snprintf(spec, sizeof(spec), "ref-%s", ns->hash); + n = git_revparse_single(&treeish, git_repo_build_specific, spec); + if (n) { + lwsl_notice("%s: revparse %s failed: %d\n", __func__, spec, n); + git_repository_free(git_repo_build_specific); + return SAIB_CHECKOUT_NOT_IN_LOCAL_MIRROR; + } + +again: + co_opts.checkout_strategy = GIT_CHECKOUT_FORCE; + n = git_checkout_tree(git_repo_build_specific, treeish, &co_opts); + git_object_free(treeish); + if (n) { + if (n == GIT_EUNBORNBRANCH && tries--) { + lwsl_warn("%s: git checkout says HEAD is empty branch\n", + __func__); + + /* + * It's telling us we need to delete the local mirror + * HEAD and retry + */ + + lws_snprintf(dp, sizeof(dp), "%s/HEAD", ns->inp); + unlink(dp); + + goto again; + } + goto co_failed; + } + + lws_snprintf(spec, sizeof(spec), "refs/heads/ref-%s", ns->hash); + git_repository_set_head(git_repo_build_specific, spec); + + lwsl_notice("%s: checkout OK\n", __func__); + + git_repository_free(git_repo_build_specific); + + return SAIB_CHECKOUT_OK; + +co_failed: + lwsl_err("%s: git checkout failed: %d\n", __func__, n); + git_repository_free(git_repo_build_specific); + + return SAIB_CHECKOUT_CHECKOUT_FAILED; +} + + +/* + * We are a threadpool thread handling queueing on remote mirror "thread_repo", + * and afterwards checking out a ref from the local copy. + * + * The main git2 process just goes away and does the fetch atomically. Wrapping + * it in an extra threadpool thread lets us continue to be resposive while we're + * waiting for the remote mirror fetch. + */ + +enum lws_threadpool_task_return +saib_mirror_task(void *user, enum lws_threadpool_task_status s) +{ + struct sai_nspawn *ns = (struct sai_nspawn *)user; + sai_mirror_instance_t *mi; + sai_mirror_req_t *req; + int n, m; + + lwsl_warn("%s: entry: state %d\n", __func__, ns->state); + + mi = &builder.mi; + + if (s == LWS_TP_STATUS_STOPPING) + return LWS_TP_RETURN_STOPPED; + + switch (ns->state) { + case NSSTATE_INIT: + case NSSTATE_MOUNTING: + + case NSSTATE_STARTING_MIRROR: + lwsl_notice("%s: NSSTATE_STARTING_MIRROR\n", __func__); + ns->state = NSSTATE_CHECKOUT_SPEC; + /* fallthru */ + + case NSSTATE_CHECKOUT_SPEC: + /* + * First attempt to checkout from local mirror, in case we + * already fetched it to the local mirror for an earlier task + */ + + lwsl_notice("%s: NSSTATE_CHECKOUT_SPEC\n", __func__); + switch (sai_mirror_local_checkout(ns)) { + case SAIB_CHECKOUT_OK: + /* + * oh we have it then... do a final sync and have the + * foreground handler STOP us when we resume + */ + + lwsl_notice("%s: syncing checkout %s done\n", __func__, + ns->inp); + ns->state = NSSTATE_CHECKEDOUT; + + lws_snprintf(ns->pending_mirror_log, + sizeof(ns->pending_mirror_log), + ">saib> Local mirror checkout %s\n", ns->hash); + + return LWS_TP_RETURN_SYNC; + + case SAIB_CHECKOUT_NOT_IN_LOCAL_MIRROR: + /* + * Basically it means we have to get it from the remote + * repo into the local mirror first... clear down the + * local build dir + */ + + lws_dir(ns->inp, NULL, lws_dir_rm_rf_cb); + + /* + * Make sure the dir itself is left standing in there + */ + + mkdir(ns->inp, 0755); + + /* + * Form a request, kick the remoting thread and then + * wait on the condition in the request + */ + + req = malloc(sizeof(*req)); + if (!req) + goto fail; + + /* + * The request needs to exist with a standalone + * lifetime regardless of what's happening to the ns or + * threadpool thread while it waits, or proceeds + * asynchronously + */ + + lwsl_notice("%s: NOT_IN_LOCAL_MIRROR %s\n", __func__, ns->ref); + memset(req, 0, sizeof(*req)); + req->state = SRFS_REQUESTING; + req->ns = ns; + lws_strncpy(req->url, ns->git_repo_url, sizeof(req->url)); + lws_strncpy(req->hash, ns->hash, sizeof(req->hash)); + lws_strncpy(req->ref, ns->ref, sizeof(req->ref)); + lws_strncpy(req->path, ns->path, sizeof(req->path)); + + pthread_mutex_lock(&mi->mut); + lws_dll2_add_tail(&req->list, &mi->pending_req); + + pthread_cond_broadcast(&mi->cond); + pthread_mutex_unlock(&mi->mut); + + ns->state = NSSTATE_WAIT_REMOTE_MIRROR; + ns->mirror_wait_budget = 600; + + lws_snprintf(ns->pending_mirror_log, + sizeof(ns->pending_mirror_log), + ">saib> Starting REMOTE MIRROR FETCH\n"); + + return LWS_TP_RETURN_SYNC; + + case SAIB_CHECKOUT_CHECKOUT_FAILED: + + lws_snprintf(ns->pending_mirror_log, + sizeof(ns->pending_mirror_log), + ">saib> Checkout failed...\n"); + + goto fail; + } + + break; + + case NSSTATE_CHECKOUT: + /* + * We come here after the remote -> local mirror operation + * seemed to go well, to complete the flow by checking out + * the newly-fetched commit. We can only go OK or fail. + */ + + lwsl_notice("%s: NSSTATE_CHECKOUT\n", __func__); + m = sai_mirror_local_checkout(ns); + lwsl_notice("%s: sai_mirror_local_checkout() says %d\n", __func__, m); + switch (m) { + case SAIB_CHECKOUT_OK: + /* + * oh we have it then... do a final sync and have the + * foreground handler STOP us when we resume + */ + + lwsl_notice("%s: syncing checkout %s done\n", __func__, + ns->inp); + ns->state = NSSTATE_CHECKEDOUT; + + lws_snprintf(ns->pending_mirror_log, + sizeof(ns->pending_mirror_log), + ">saib> Remote->Local->Checked out %s\n", ns->hash); + + return LWS_TP_RETURN_SYNC; + + case SAIB_CHECKOUT_NOT_IN_LOCAL_MIRROR: + /* fallthru */ + lwsl_notice("%s: checkout not it local mirror\n", __func__); + case SAIB_CHECKOUT_CHECKOUT_FAILED: + + lws_snprintf(ns->pending_mirror_log, + sizeof(ns->pending_mirror_log), + ">saib> Checkout failed after mirror...\n"); + lwsl_notice("%s: checkout failed\n", __func__); + + goto fail; + } + break; + + case NSSTATE_CHECKEDOUT: + case NSSTATE_BUILD: + case NSSTATE_DONE: + case NSSTATE_FAILED: + break; + + case NSSTATE_WAIT_REMOTE_MIRROR: + + /* + * We are waiting for our request to be handled... we need to + * watch the req completed list at intervals. Anything on the + * completed_req owner has either completed or failed. + */ + + lwsl_notice("%s: NSSTATE_WAIT_REMOTE_MIRROR\n", __func__); + pthread_mutex_lock(&mi->mut); + + lws_start_foreach_dll_safe(struct lws_dll2 *, d, d1, + mi->completed_req.head) { + sai_mirror_req_t *r = lws_container_of(d, + sai_mirror_req_t, list); + + if (r->ns == ns) { + + /* + * It's our request that has completed + */ + + n = r->state; + + lws_dll2_remove(&r->list); + pthread_mutex_unlock(&mi->mut); + free(r); + + if (n == SRFS_FAILED) { + lwsl_notice("%s: mirror req failed\n", + __func__); + goto fail; + } + + /* + * Leave in NSSTATE_CHECKOUT and come back to + * continue with checking out after we have + * sync'd with comms to master + */ + + ns->state = NSSTATE_CHECKOUT; + lws_snprintf(ns->pending_mirror_log, + sizeof(ns->pending_mirror_log), + ">saib> Remote -> Local OK\n"); + + return LWS_TP_RETURN_SYNC; + } + + } lws_end_foreach_dll_safe(d, d1); + + pthread_mutex_unlock(&mi->mut); /* --- mi->mut unlock */ + +#if defined(WIN32) + Sleep(1000); +#else + sleep(1); +#endif + + if (!--ns->mirror_wait_budget) { + lws_snprintf(ns->pending_mirror_log, sizeof(ns->pending_mirror_log), + ">saib> timed out waiting for mirror\n"); + lwsl_notice("%s: timed out waiting for mirror\n", __func__); + + goto fail; + } + + lws_snprintf(ns->pending_mirror_log, sizeof(ns->pending_mirror_log), + ">saib> Waiting on MIRROR...\n"); + + return LWS_TP_RETURN_SYNC; + } + + return LWS_TP_RETURN_SYNC; + +fail: + + /* + * If we have an incomplete request on the mirror thread, remove it + */ + + pthread_mutex_lock(&mi->mut); + lws_start_foreach_dll_safe(struct lws_dll2 *, d, d1, + mi->pending_req.head) { + sai_mirror_req_t *r = lws_container_of(d, + sai_mirror_req_t, list); + + if (r->ns == ns) { + lws_dll2_remove(&r->list); + free(r); + } + + } lws_end_foreach_dll_safe(d, d1); + pthread_mutex_unlock(&mi->mut); + + lwsl_err("%s: failed\n", __func__); + ns->retcode = SAISPRF_EXIT | 253; + saib_set_ns_state(ns, NSSTATE_FAILED); + saib_task_grace(ns); + + if (saib_queue_task_status_update(ns->sp, ns->spm, NULL)) + return -1; + + return LWS_TP_RETURN_SYNC; +} + +/* + * This thread serially handles requests for remote mirrors, since there's a + * write lock in the filesystem copy when someone is updating the mirror. + * + * It picks takes queued requests from the head + */ + +void * +thread_repo(void *d) +{ + sai_mirror_instance_t *mi = (sai_mirror_instance_t *)d; + sai_mirror_req_t *req, rcopy; + int new_state; + int n; + + lwsl_notice("%s: repo thread start\n", __func__); + + git_libgit2_init(); + + while (!mi->finish) { + git_fetch_options opts = GIT_FETCH_OPTIONS_INIT; + char spec[96], *paths[] = { spec }; + git_strarray rfs = { paths, 1 }; + git_repository *repo_mirror; + git_remote *remote; + + pthread_mutex_lock(&mi->mut); + + /* we sleep if there are no requests pending or ongoing */ + + while (!mi->pending_req.count && !mi->finish) + pthread_cond_wait(&mi->cond, &mi->mut); + + if (mi->finish) { + pthread_mutex_unlock(&mi->mut); + break; + } + + /* + * Starting from the head, look for the first request that's in + * a state we should start the remote for it + */ + + req = NULL; + lws_start_foreach_dll_safe(struct lws_dll2 *, d, d1, + mi->pending_req.head) { + sai_mirror_req_t *r = lws_container_of(d, + sai_mirror_req_t, list); + + if (r->state == SRFS_REQUESTING) { + req = r; + lwsl_notice("%s: setting processing\n", + __func__); + sai_mirror_req_state_set(req, SRFS_PROCESSING); + break; + } + + } lws_end_foreach_dll_safe(d, d1); + + //req = lws_dll2_get_head(&bi->requests); + + //lws_dll2_remove(&->overall_list); + + /* + * The request object on the list might be removed and destroyed + * while we do this long-term mirroring action. So take a temp + * copy of it to set the action up, before releasing the mutex. + */ + if (req) + rcopy = *req; + + pthread_mutex_unlock(&mi->mut); + if (!req) + continue; + + /* + * ... we cannot dereference req after releasing the mutex. + * We made a temp copy of it in rcopy, to set the transaction + * up we can use that so we can be sure it's around for that. + */ + + /* + * git init --bare + */ + + new_state = SRFS_FAILED; + repo_mirror = NULL; + if (git_repository_init(&repo_mirror, rcopy.path, 1)) { + fprintf(stderr, "%s: unable to init sticky repo %s, errno %d\n", + __func__, rcopy.path, errno); + + goto fail_out; + } + + /* + * Fetch over the branch or tag we're interested in from the + * remote repo and into our local sticky mirror + * + * git fetch +: + */ + + if (git_remote_create_anonymous(&remote, repo_mirror, rcopy.url)) { + fprintf(stderr, "%s: cant find remote %s\n", __func__, + rcopy.url); + + git_repository_free(repo_mirror); + goto fail_out; + } + + lws_snprintf(spec, sizeof(spec), "%s:ref-%s", rcopy.ref, rcopy.hash); + fprintf(stderr, "%s: fetching %s %s\n", __func__, rcopy.url, spec); + + /* + * This may take an open-ended amount of time + */ + + n = git_remote_fetch(remote, &rfs, &opts, "fetch"); + + git_remote_free(remote); + git_repository_free(repo_mirror); + + if (n) { +#if defined(SAI_HAVE_LIBGIT2_GIT_ERROR) + const git_error *e = git_error_last(); + + if (e) + fprintf(stderr, "%s: git error %s\n", __func__, + e->message); +#endif + + fprintf(stderr, "%s: failed to fetch %s, n: %d\n", + __func__, spec, n); + goto fail_out; + } + + fprintf(stderr, "%s: syncing mirror fetch %s %s successful\n", + __func__, rcopy.url, rcopy.hash); + + new_state = SRFS_SUCCEEDED; + +fail_out: + /* + * We notify the threadpool monitoring thread + */ + + pthread_mutex_lock(&mi->mut); + + /* + * The requestor and its request may have gone. And, it's + * possible we got several queued requests that were actually + * waiting for the same mirror transaction which has now + * succeeded or failed the same for all of them. + * + * Let's take the approach to lock the queue and tell everybody + * at once who's asking for the same thing what the outcome was. + */ + lws_start_foreach_dll_safe(struct lws_dll2 *, d, d1, + mi->pending_req.head) { + sai_mirror_req_t *r = lws_container_of(d, + sai_mirror_req_t, list); + + if ((r->state == SRFS_REQUESTING || + r->state == SRFS_PROCESSING) && + !strcmp(r->url, rcopy.url) && + !strcmp(r->hash, rcopy.hash) && + !strcmp(r->path, rcopy.path) && + !strcmp(r->ref, rcopy.ref)) { + /* + * Change the state of matching guys and put + * them on to the completed_req list owner + */ + fprintf(stderr, "%s: setting req state %d\n", + __func__, new_state); + sai_mirror_req_state_set(r, new_state); + lws_dll2_remove(&r->list); + lws_dll2_add_tail(&r->list, &mi->completed_req); + } + + } lws_end_foreach_dll_safe(d, d1); + + pthread_mutex_unlock(&mi->mut); + } + + lwsl_notice("%s: repo thread exiting\n", __func__); + + git_libgit2_shutdown(); + pthread_exit(NULL); + + return NULL; +} diff --git a/src/builder/b-nspawn.c b/src/builder/b-nspawn.c new file mode 100644 index 0000000..308c7a2 --- /dev/null +++ b/src/builder/b-nspawn.c @@ -0,0 +1,424 @@ +/* + * sai-builder + * + * Copyright (C) 2019 - 2020 Andy Green + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation: + * version 2.1 of the License. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + * MA 02110-1301 USA + * + * Generic unix spawn and stdxxx pipe to wsi mapping + */ + +#include + +#include +#include + +#include +#include +#if !defined(WIN32) +#include +#endif +#include +#include + +#include "b-private.h" + +#if !defined(WIN32) +static char csep = '/'; +#else +static char csep = '\\'; +#endif + +extern struct lws_vhost *builder_vhost; + +struct ws_capture_chunk * +saib_log_chunk_create(struct sai_nspawn *ns, void *buf, size_t len, int channel) +{ + struct ws_capture_chunk *chunk = malloc(sizeof(*chunk) + len); + + if (!chunk || !ns->spm) + return NULL; + + memset(chunk, 0, sizeof(*chunk)); + chunk->us = lws_now_usecs(); + chunk->len = len; + chunk->stdfd = channel; + if (len) + memcpy(&chunk[1], buf, len); + + ns->chunk_cache_size += sizeof(*chunk) + len; + + lws_dll2_add_head(&chunk->list, &ns->chunk_cache); + ns->spm->logs_in_flight++; + + return chunk; +} + +static int +callback_sai_stdwsi(struct lws *wsi, enum lws_callback_reasons reason, + void *user, void *in, size_t len) +{ + struct sai_nspawn *ns = (struct sai_nspawn *)lws_get_opaque_user_data(wsi); + uint8_t buf[600]; + int ilen; + + switch (reason) { + + case LWS_CALLBACK_RAW_CLOSE_FILE: + lwsl_user("%s: RAW_CLOSE_FILE wsi %p: fd: %d, stdfd: %d\n", + __func__, wsi, lws_get_socket_fd(wsi), + lws_spawn_get_stdfd(wsi)); + + ilen = lws_snprintf((char *)buf, sizeof(buf), "Stdwsi %d close\n", lws_spawn_get_stdfd(wsi)); + saib_log_chunk_create(ns, buf, (size_t)ilen, 3); + + if (ns->lsp) + lws_spawn_stdwsi_closed(ns->lsp, wsi); + break; + + case LWS_CALLBACK_RAW_RX_FILE: + if (!ns->spm) + return -1; +#if defined(WIN32) + { + DWORD rb; + if (!ReadFile((HANDLE)lws_get_socket_fd(wsi), buf, sizeof(buf), &rb, NULL)) { + lwsl_debug("%s: read on stdwsi failed\n", __func__); + return -1; + } + ilen = (int)rb; + } +#else + ilen = read((int)(intptr_t)lws_get_socket_fd(wsi), buf, sizeof(buf)); + if (ilen < 1) { + lwsl_debug("%s: read on stdwsi failed\n", __func__); + return -1; + } +#endif + + len = ilen; + + // printf("(%d) %.*s\n", (int)len, (int)len, buf); + + if (!saib_log_chunk_create(ns, buf, len, lws_spawn_get_stdfd(wsi))) + return -1; + + lws_ss_request_tx(ns->spm->ss); + break; + + default: + break; + } + + return 0; +} + +struct lws_protocols protocol_stdxxx = + { "sai-stdxxx", callback_sai_stdwsi, 0, 0 }; + + +static void +sai_lsp_reap_cb(void *opaque, lws_usec_t *accounting, siginfo_t *si, + int we_killed_him) +{ + struct sai_nspawn *ns = (struct sai_nspawn *)opaque; + + saib_log_chunk_create(ns, ">saib> Reaping build process\n", 29, 3); + +#if !defined(WIN32) + lwsl_notice("%s: reaped: timing %dms %dms %dms %dms\n", __func__, + (int)(accounting[0] / 1000), (int)(accounting[1] / 1000), + (int)(accounting[2] / 1000), (int)(accounting[3] / 1000)); + + if (we_killed_him & 1) { + lwsl_notice("%s: Process TIMED OUT by Sai\n", __func__); + ns->retcode = SAISPRF_TIMEDOUT; + goto ok; + } + + if (we_killed_him & 2) { + lwsl_notice("%s: Process killed by Sai due to spew\n", __func__); + ns->retcode = SAISPRF_TERMINATED; + goto ok; + } + + switch (si->si_code) { + case CLD_EXITED: + lwsl_notice("%s: Process Exited with exit code %d\n", + __func__, si->si_status); + ns->retcode = SAISPRF_EXIT | si->si_status; + if (ns->user_cancel) + ns->retcode = SAISPRF_TERMINATED; + break; + case CLD_KILLED: + case CLD_DUMPED: + lwsl_notice("%s: Process Terminated by signal %d / %d\n", + __func__, si->si_status, si->si_signo); + ns->retcode = SAISPRF_SIGNALLED | si->si_signo; + break; + default: + lwsl_notice("%s: SI code %d\n", __func__, si->si_code); + break; + } + +ok: +#else + ns->retcode = SAISPRF_EXIT | (si->retcode & 0xff); +#endif + + saib_task_grace(ns); + saib_set_ns_state(ns, NSSTATE_DONE); + + /* + * add a final zero-length log with the retcode to the list of pending + * logs + */ + + saib_log_chunk_create(ns, NULL, 0, 2); + + lwsl_notice("%s: finished, waiting to drain logs (this ns %d, spm in flight %d)\n", + __func__, ns->chunk_cache.count, + ns->spm ? ns->spm->logs_in_flight : -99); +} + +#if defined(WIN32) + +static const char * const runscript = + "set SAI_INSTANCE_IDX=%d\n" + "set SAI_LOGPROXY=%s\n" + "set SAI_LOGPROXY_TTY0=%s\n" + "set SAI_LOGPROXY_TTY1=%s\n" + "cd %s\\jobs\\%s\\%s &&" + "%s" +; + +#else + +static const char * const runscript = + "#!/bin/bash -x\n" +#if defined(__APPLE__) + "export PATH=/usr/local/bin:/usr/bin:/bin:/sbin:/usr/sbin\n" +#endif + "export SAI_OVN=%s\n" + "export SAI_PROJECT=%s\n" + "export SAI_REMOTE_REF=%s\n" + "export SAI_INSTANCE_IDX=%d\n" + "export SAI_LOGPROXY=%s\n" + "export SAI_LOGPROXY_TTY0=%s\n" + "export SAI_LOGPROXY_TTY1=%s\n" + "set -e\n" + "cd %s/jobs/$SAI_OVN/$SAI_PROJECT\n" + + "%s\n" + "exit $?\n" +; + +#endif + +int +saib_spawn(struct sai_nspawn *ns) +{ + struct lws_spawn_piped_info info; + char args[290], st[2048], *p; + int fd, n; + const char * cmd[] = { + "/bin/ps", + NULL + }; + char *env[] = { + "PATH=/usr/bin:/bin", + "LANG=en_US.UTF-8", + NULL + }; + + lws_strncpy(st, ns->sp->name, sizeof(st)); + lws_filename_purify_inplace(st); + p = st; + while ((p = strchr(p, '/'))) + *p = '_'; + +#if defined(WIN32) + lws_snprintf(args, sizeof(args), "%s\\sai-build-script-%s-%d.bat", + builder.home, st, ns->instance_idx); +#else + lws_snprintf(args, sizeof(args), "%s/sai-build-script-%s-%d.sh", + builder.home, st, ns->instance_idx); +#endif + + lwsl_hexdump_notice(ns->task->build, strlen(ns->task->build)); + +#if defined(WIN32) + if (_sopen_s(&fd, args, _O_CREAT | _O_TRUNC | _O_WRONLY, + _SH_DENYNO, _S_IWRITE)) + fd = -1; +#else + fd = open(args, O_CREAT | O_TRUNC | O_WRONLY, 0755); +#endif + if (fd < 0) { + lwsl_err("%s: unable to open %s for write\n", __func__, args); + return 1; + } + +#if defined(WIN32) + n = lws_snprintf(st, sizeof(st), runscript, ns->instance_idx, + ns->slp_control.sockpath, + ns->slp[0].sockpath, ns->slp[1].sockpath, + builder.home, ns->fsm.ovname, ns->project_name, + ns->task->build); +#else + n = lws_snprintf(st, sizeof(st), runscript, ns->fsm.ovname, + ns->project_name, ns->ref, ns->instance_idx, + ns->slp_control.sockpath, + ns->slp[0].sockpath, ns->slp[1].sockpath, + builder.home, ns->task->build); +#endif + + /* but from the script's pov, it's chrooted at /home/sai */ + + if (write(fd, st, n) != n) { + close(fd); + lwsl_err("%s: failed to write runscript to %s\n", __func__, args); + return 1; + } + + close(fd); + + cmd[0] = args; + + memset(&info, 0, sizeof(info)); + info.vh = builder.vhost; + info.env_array = env; + info.exec_array = cmd; + info.protocol_name = "sai-stdxxx"; + info.max_log_lines = 10000; + info.timeout_us = 30 * 60 * LWS_US_PER_SEC; + info.reap_cb = sai_lsp_reap_cb; + info.opaque = ns; + info.plsp = &ns->lsp; + + ns->lsp = lws_spawn_piped(&info); + if (!ns->lsp) { + lwsl_err("%s: failed\n", __func__); + + return 1; + } + + lwsl_notice("%s: lws_spawn_piped started\n", __func__); + + return 0; +} + +/* + * This prepares the same filesystem layout whether we can use it for overlayfs, + * or have to use it directly / with chroot + */ + +int +saib_prepare_mount(struct sai_builder *b, struct sai_nspawn *ns) +{ + char homedir[384]; + int n, m; + + /* create the top level overlays container if not already there */ + + n = lws_snprintf(ns->fsm.mp, sizeof(ns->fsm.mp), "%s%coverlays", + b->home, csep); + + mkdir(ns->fsm.mp, 0770); + + /* create a subdir for our overlay pieces */ + + n += lws_snprintf(ns->fsm.mp + n, sizeof(ns->fsm.mp) - n, "%c%s", csep, + ns->fsm.ovname); + m = mkdir(ns->fsm.mp, 0777); + if (m && errno != EEXIST) + goto bail_dir; + lws_strncpy(homedir, ns->fsm.mp, sizeof(homedir)); + + /* create work dir, session layer and mountpoint, retain mountpoint */ + +#if defined(__linux__) && 0 + + lws_snprintf(ns->fsm.mp + n, sizeof(ns->fsm.mp) - n, "%cwork", csep); + m = mkdir(ns->fsm.mp, 0770); + if (m && errno != EEXIST) + goto bail_dir; + + lws_snprintf(ns->fsm.mp + n, sizeof(ns->fsm.mp) - n, "%csession", csep); + m = mkdir(ns->fsm.mp, 0777); + if (m && errno != EEXIST) + goto bail_dir; + + n += lws_snprintf(ns->fsm.mp + n, sizeof(ns->fsm.mp) - n, "%cmountpoint", + csep); + m = mkdir(ns->fsm.mp, 0777); + if (m && errno != EEXIST) + goto bail_dir; + + lws_snprintf(ns->fsm.mp + n, sizeof(ns->fsm.mp) - n, "%chome%csai", + csep, csep); + lws_strncpy(homedir, ns->fsm.mp, sizeof(homedir)); + ns->fsm.mp[n] = '\0'; + + n = lws_fsmount_mount(&ns->fsm); + + if (!n) +#endif + { + + /* these are ephemeral on top of the mountpoint path, we snip + * them off later */ + + n = (int)strlen(ns->fsm.mp); + lws_snprintf(ns->fsm.mp + n, sizeof(ns->fsm.mp) - n, "%chome", + csep); + m = mkdir(ns->fsm.mp, 0700); + if (m && errno != EEXIST) + goto bail_dir; + + lws_snprintf(ns->fsm.mp + n, sizeof(ns->fsm.mp) - n, + "%chome%csai", csep, csep); + lws_strncpy(homedir, ns->fsm.mp, sizeof(homedir)); + + m = mkdir(ns->fsm.mp, 0700); + if (m && errno != EEXIST) + goto bail_dir; + + lws_snprintf(ns->fsm.mp + n, sizeof(ns->fsm.mp) - n, + "%chome%csai%cgit-mirror", csep, csep, csep); + m = mkdir(ns->fsm.mp, 0755); + ns->fsm.mp[n] = '\0'; + if (m && errno != EEXIST) + goto bail_dir; + + return 0; + } +#if defined(__linux__) && 0 + else { + lwsl_err("%s: mount err %d, errno %d\n", __func__, n, errno); + } + + return n; +#else + return 0; +#endif + +bail_dir: + lwsl_err("%s: failed to create dir %s: errno %d\n", __func__, + ns->fsm.mp, errno); + + return 1; +} diff --git a/src/builder/b-private.h b/src/builder/b-private.h new file mode 100644 index 0000000..b01cceb --- /dev/null +++ b/src/builder/b-private.h @@ -0,0 +1,228 @@ +/* + * Sai builder definitions src/builder/b-private.h + * + * Copyright (C) 2019 - 2020 Andy Green + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation: + * version 2.1 of the License. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + * MA 02110-1301 USA + */ + +#include "../common/include/private.h" + +#include +#if defined(WIN32) +#include +#define read _read +#define open _open +#define close _close +#define write _write +#define mkdir(x,y) _mkdir(x) +#define rmdir _rmdir +#define unlink _unlink +#define HAVE_STRUCT_TIMESPEC +#if defined(pid_t) +#undef pid_t +#endif +#endif +#include +#include + +typedef enum { + PHASE_IDLE, + + PFL_FIRST = 128, + + PHASE_START_ATTACH = PFL_FIRST | 1, + PHASE_SUMM_PLATFORMS = 2, + + PHASE_BUILDING + +} cursor_phase_t; + + + +struct saib_ws_pss; + +enum { + NSSTATE_INIT, + NSSTATE_MOUNTING, + NSSTATE_STARTING_MIRROR, + /* speculatively see if we can already check out locally */ + NSSTATE_CHECKOUT_SPEC, + /* we are stuck waiting for remote mirror thread */ + NSSTATE_WAIT_REMOTE_MIRROR, + /* remote mirror completed, we can check out locally */ + NSSTATE_CHECKOUT, + /* we have finished with threadpool / wait for remote */ + NSSTATE_CHECKEDOUT, + NSSTATE_BUILD, + NSSTATE_DONE, + NSSTATE_FAILED, +}; + +struct saib_logproxy { + char sockpath[128]; + struct sai_nspawn *ns; + int log_channel_idx; +}; + +struct sai_nspawn; + +struct sai_nspawn { + lws_dll2_owner_t chunk_cache; + + char inp[384]; + char path[256]; + char pending_mirror_log[128]; + + /* convenient place to store it */ + struct saib_logproxy slp_control; + struct saib_logproxy slp[2]; + + lws_dll2_t list; /* sai_plat owner lists sai_nspawns */ + struct sai_builder *builder; + struct lws_fsmount fsm; + struct lws_spawn_piped *lsp; + sai_task_t *task; + + lws_dll2_owner_t artifact_owner; /* struct artifact_path */ + + struct lws_threadpool *tp; + struct lws_threadpool_task *tp_task; + lws_sorted_usec_list_t sul_cleaner; + lws_sorted_usec_list_t sul_task_cancel; + + sai_plat_t *sp; /* the sai_plat */ + struct sai_plat_master *spm; /* the sai plat / master with the ss / wsi */ + + size_t chunk_cache_size; + + const char *master_name; /* sai-master name who triggered this, eg, 'warmcat' */ + const char *project_name; /* name of the git project, eg, 'libwebsockets' */ + const char *ref; /* remote refname, eg 'master' */ + const char *hash; /* remote hash */ + const char *git_repo_url; + + int retcode; + int instance_idx; + int mirror_wait_budget; + + uint8_t spins; + uint8_t state; /* NSSTATE_ */ + uint8_t stdcount; + uint8_t term_budget; + + uint8_t finished_when_logs_drained:1; + uint8_t state_changed:1; + uint8_t user_cancel:1; +}; + +typedef struct sai_mirror_instance { + pthread_mutex_t mut; + pthread_cond_t cond; + + pthread_t repo_thread; + + lws_dll2_owner_t pending_req; + lws_dll2_owner_t completed_req; + + uint8_t finish; +} sai_mirror_instance_t; + + +/* + * This represents this builder process as a whole + */ + +struct sai_builder { + lws_dll2_owner_t sai_plat_owner; /* list of platforms we offer */ + lws_dll2_owner_t sai_plat_master_owner; /* masters we connect to */ + lws_dll2_owner_t devices_owner; /* sai_serial_t */ + + struct lwsac *conf_head; + struct lws_context *context; + struct lws_vhost *vhost; + + const char *home; /* home dir, usually /sai/home */ + const char *perms; /* user:group */ + + const char *host; /* prepended before hostname */ + + sai_mirror_instance_t mi; +}; + +struct ws_capture_chunk { + struct lws_dll2 list; + + lws_usec_t us; /* builder time that we saw this */ + size_t len; + uint8_t stdfd; /* 1 = stdout, 2 = stderr */ + + /* len bytes of data is overallocated after this */ +}; + + +extern struct sai_builder builder; +extern const lws_ss_info_t ssi_sai_builder, ssi_sai_mirror, ssi_sai_artifact; +extern const struct lws_protocols protocol_com_warmcat_sai; +int +saib_config_global(struct sai_builder *builder, const char *d); +extern int saib_config(struct sai_builder *builder, const char *d); +extern void saib_config_destroy(struct sai_builder *builder); + +int +saib_overlay_mount(struct sai_builder *b, struct sai_nspawn *ns); + +int +saib_overlay_unmount(struct sai_nspawn *ns); + +int +saib_spawn(struct sai_nspawn *ns); + +int +saib_prepare_mount(struct sai_builder *b, struct sai_nspawn *ns); + +int +saib_ws_json_rx_builder(struct sai_plat_master *spm, const void *in, size_t len); + +int +saib_generate(struct sai_plat *sp, char *buf, int len); + +enum lws_threadpool_task_return +saib_mirror_task(void *user, enum lws_threadpool_task_status s); + +int +saib_set_ns_state(struct sai_nspawn *ns, int state); + +void +saib_task_destroy(struct sai_nspawn *ns); + +void +saib_task_grace(struct sai_nspawn *ns); + +struct ws_capture_chunk * +saib_log_chunk_create(struct sai_nspawn *ns, void *buf, size_t len, int channel); + +int +saib_queue_task_status_update(sai_plat_t *sp, struct sai_plat_master *spm, + const char *rej_task_uuid); + +int +rm_rf_cb(const char *dirpath, void *user, struct lws_dir_entry *lde); + +extern const struct lws_protocols protocol_logproxy; + +void * +thread_repo(void *d); diff --git a/src/builder/b-sai.c b/src/builder/b-sai.c new file mode 100644 index 0000000..b3d0b3d --- /dev/null +++ b/src/builder/b-sai.c @@ -0,0 +1,509 @@ +/* + * sai-builder + * + * Copyright (C) 2019 - 2020 Andy Green + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation: + * version 2.1 of the License. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + * MA 02110-1301 USA + * + * Sai-builder uses a secure streams template to make the client connections to + * the masters listed in /etc/sai/builder/conf JSON config. The URL in the + * config is substituted for the endpoint URL at runtime. + * + * See b-comms.c for the secure stream template and callbacks for this. + */ + +#include +#include +#include + +#if defined(__APPLE__) +#include /* for mkdir() */ +#include /* for chown() */ +#endif + +#if defined(WIN32) +#include +#include +#include +#endif + +#include "b-private.h" + +static const char *config_dir = "/etc/sai/builder"; +static int interrupted; +static lws_state_notify_link_t nl; + +struct sai_builder builder; + +extern struct lws_protocols protocol_stdxxx; + +static const char * const default_ss_policy = + "{" + "\"retry\": [" /* named backoff / retry strategies */ + "{\"default\": {" + "\"backoff\": [" "1000," + "2000," + "3000," + "5000," + "10000" + "]," + "\"conceal\":" "99999," + "\"jitterpc\":" "20," + "\"svalidping\":" "300," + "\"svalidhup\":" "350" + "}}" + "]," + + /* + * No certs / trust stores because we will validate using system trust + * store... metadata.url should be set at runtime to something like + * https://warmcat.com/sai + */ + + "\"s\": [" + /* + * The main connection to a master carrying events and logs + */ + "{\"sai_builder\": {" + "\"endpoint\":" "\"${url}\"," + "\"port\":" "443," + "\"protocol\":" "\"ws\"," + "\"ws_subprotocol\":" "\"com-warmcat-sai\"," + "\"http_url\":" "\"\"," /* filled in by url */ + "\"tls\":" "true," + "\"retry\":" "\"default\"," + "\"metadata\": [" + "{\"url\": \"\"}" + "]" + "}}," + /* + * Ephemeral connections to the same master carrying artifact + * JSON + bulk data + */ + "{\"sai_artifact\": {" + "\"endpoint\":" "\"${url}\"," + "\"port\":" "443," + "\"protocol\":" "\"ws\"," + "\"ws_subprotocol\":" "\"com-warmcat-sai\"," + "\"http_url\":" "\"\"," /* filled in by url */ + "\"tls\":" "true," + "\"opportunistic\":" "true," + "\"retry\":" "\"default\"," + "\"metadata\": [" + "{\"url\": \"\"}" + "]" + "}}" + "]}" +; + +static const struct lws_protocols *pprotocols[] = { + &protocol_stdxxx, + &protocol_logproxy, + NULL +}; + +static int lpidx; +static char vhnames[256], *pv = vhnames; + +static struct lws_protocol_vhost_options pvo = { + NULL, /* "next" pvo linked-list */ + NULL, /* "child" pvo linked-list */ + "protocol-logproxy", /* protocol name we belong to on this vhost */ + "ok" /* ignored */ +}; + +static int +saib_create_listen_uds(struct lws_context *context, struct saib_logproxy *lp) +{ + struct lws_context_creation_info info; + + memset(&info, 0, sizeof(info)); + + info.vhost_name = pv; + pv += lws_snprintf(pv, sizeof(vhnames) - (pv - vhnames), "logproxy.%d", lpidx++) + 1; + info.options = LWS_SERVER_OPTION_ADOPT_APPLY_LISTEN_ACCEPT_CONFIG | + LWS_SERVER_OPTION_UNIX_SOCK; + info.iface = lp->sockpath; + info.listen_accept_role = "raw-skt"; + info.listen_accept_protocol = "protocol-logproxy"; + info.user = lp; + info.pvo = &pvo; + info.pprotocols = pprotocols; + +#if !defined(__linux__) + unlink(lp->sockpath); +#endif + + lwsl_notice("%s: %s\n", __func__, lp->sockpath); + + if (!lws_create_vhost(context, &info)) { + lwsl_notice("%s: failed to create vh %s\n", __func__, + info.vhost_name); + return -1; + } + + return 0; +} + +static int +app_system_state_nf(lws_state_manager_t *mgr, lws_state_notify_link_t *link, + int current, int target) +{ + struct lws_context *context = lws_system_context_from_system_mgr(mgr); + char pur[128]; + + /* + * For the things we care about, let's notice if we are trying to get + * past them when we haven't solved them yet, and make the system + * state wait while we trigger the dependent action. + */ + switch (target) { + + case LWS_SYSTATE_OPERATIONAL: + if (current != LWS_SYSTATE_OPERATIONAL) + break; + + /* + * The builder JSON conf listed masters we want to connect to, + * let's collect the config, make a ss for each and add the + * saim into an lws_dll2 list owned by + * builder->builder->sai_plat_owner + */ + + lwsl_notice("%s: starting platform config\n", __func__); + if (saib_config(&builder, config_dir)) { + lwsl_err("%s: config failed\n", __func__); + + return 1; + } + + /* + * For each platform... + */ + + lws_start_foreach_dll_safe(struct lws_dll2 *, mp, mp1, + builder.sai_plat_owner.head) { + struct sai_plat *sp = lws_container_of(mp, struct sai_plat, + sai_plat_list); + + /* + * ... for each nspawn on the platform... + */ + + lws_start_foreach_dll_safe(struct lws_dll2 *, np, np1, + sp->nspawn_owner.head) { + struct sai_nspawn *ns = + lws_container_of(np, struct sai_nspawn, list); + char *p; + int n; + + lws_strncpy(pur, sp->name, sizeof(pur)); + lws_filename_purify_inplace(pur); + p = pur; + while ((p = strchr(p, '/'))) + *p = '_'; + + /* + * Proxy the control logging channel (this + * is the one that has sai progress info) + */ + + lws_snprintf(ns->slp_control.sockpath, + sizeof(ns->slp_control.sockpath), +#if defined(__linux__) + UDS_PATHNAME_LOGPROXY".%s.%d.saib", +#else + UDS_PATHNAME_LOGPROXY"/%s.%d.saib", +#endif + pur, ns->instance_idx); + + ns->slp_control.ns = ns; + ns->slp_control.log_channel_idx = 3; + + if (saib_create_listen_uds(context, &ns->slp_control)) { + lwsl_err("%s: Failed to create ctl log proxy listen UDS %s\n", + __func__, ns->slp_control.sockpath); + return -1; + } + + /* + * For each additional logging channel... + */ + + for (n = 0; n < (int)LWS_ARRAY_SIZE(ns->slp); n++) { + + /* + * ... create a UDS listening proxy + */ + + lws_snprintf(ns->slp[n].sockpath, + sizeof(ns->slp[n].sockpath), +#if defined(__linux__) + UDS_PATHNAME_LOGPROXY".%s.%d.tty%d", +#else + UDS_PATHNAME_LOGPROXY"/%s.%d.tty%d", +#endif + pur, ns->instance_idx, n); + + ns->slp[n].ns = ns; + ns->slp[n].log_channel_idx = n + 4; + + if (saib_create_listen_uds(context, &ns->slp[n])) { + lwsl_err("%s: Failed to create log proxy listen UDS %s\n", + __func__, ns->slp[n].sockpath); + return -1; + } + } + } lws_end_foreach_dll_safe(np, np1); + } lws_end_foreach_dll_safe(mp, mp1); + + break; + } + + return 0; +} + +static lws_state_notify_link_t * const app_notifier_list[] = { + &nl, NULL +}; + +#if !defined(LWS_WITHOUT_EXTENSIONS) +static const struct lws_extension extensions[] = { + { + "permessage-deflate", + lws_extension_callback_pm_deflate, + "permessage-deflate" + "; client_no_context_takeover" + "; client_max_window_bits" + }, + { NULL, NULL, NULL /* terminator */ } +}; +#endif + +void sigint_handler(int sig) +{ + interrupted = 1; +} + +void +sai_ns_destroy(struct sai_nspawn *ns) +{ + + lws_dll2_remove(&ns->list); + free(ns); +} + + +int main(int argc, const char **argv) +{ + int logs = LLL_USER | LLL_ERR | LLL_WARN | LLL_NOTICE; + struct lws_context_creation_info info; +#if defined(WIN32) + char temp[256], stg_config_dir[256]; +#endif + struct stat sb; + const char *p; + void *retval; + + if ((p = lws_cmdline_option(argc, argv, "-d"))) + logs = atoi(p); + + lws_set_log_level(logs, NULL); + +#if defined(WIN32) + { + PWSTR wdi = NULL; + + if (SHGetKnownFolderPath(&FOLDERID_ProgramData, + 0, NULL, &wdi) != S_OK) { + lwsl_err("%s: unable to get config dir\n", __func__); + return 1; + } + + if (WideCharToMultiByte(CP_ACP, 0, wdi, -1, temp, + sizeof(temp), 0, NULL) <= 0) { + lwsl_err("%s: problem with string encoding\n", __func__); + return 1; + } + + lws_snprintf(stg_config_dir, sizeof(stg_config_dir), + "%s\\sai\\builder\\", temp); + + config_dir = stg_config_dir; + CoTaskMemFree(wdi); + } +#endif + + /* + * Let's parse the global bits out of the config + */ + + lwsl_notice("%s: config dir %s\n", __func__, config_dir); + if (saib_config_global(&builder, config_dir)) { + lwsl_err("%s: global config failed\n", __func__); + + return 1; + } + + /* + * We need to sample the true uid / gid we should use inside + * the mountpoint for sai:nobody or sai:sai, by looking at + * what the uid and gid are on /home/sai before anything changes + * it + */ + if (stat(builder.home, &sb)) { + lwsl_err("%s: Can't find %s\n", __func__, builder.home); + return 1; + } + +#if defined(__APPLE__) + /* we are still root */ + mkdir(UDS_PATHNAME_LOGPROXY, 0700); + chown(UDS_PATHNAME_LOGPROXY, sb.st_uid, sb.st_gid); +#endif + + /* if we don't do this, libgit2 looks in /root/.gitconfig */ +#if defined(WIN32) + _putenv_s("HOME", builder.home); +#else + setenv("HOME", builder.home, 1); +#endif + + lwsl_user("Sai Builder - " + "Copyright (C) 2019-2020 Andy Green \n"); + lwsl_user(" sai-builder [-c ]\n"); + + memset(&info, 0, sizeof info); + info.port = CONTEXT_PORT_NO_LISTEN; + info.pprotocols = pprotocols; + + info.uid = sb.st_uid; + info.gid = sb.st_gid; + + +#if !defined(LWS_WITHOUT_EXTENSIONS) + if (!lws_cmdline_option(argc, argv, "-n")) + info.extensions = extensions; +#endif + info.pt_serv_buf_size = 32 * 1024; + info.options = LWS_SERVER_OPTION_DO_SSL_GLOBAL_INIT | + LWS_SERVER_OPTION_VALIDATE_UTF8 | + LWS_SERVER_OPTION_EXPLICIT_VHOSTS; + + signal(SIGINT, sigint_handler); + + info.pss_policies_json = default_ss_policy; + info.fd_limit_per_thread = 1 + 128 + 1; + + /* hook up our lws_system state notifier */ + + nl.name = "sai-builder"; + nl.notify_cb = app_system_state_nf; + info.register_notifier_list = app_notifier_list; + + /* create the lws context */ + + builder.context = lws_create_context(&info); + if (!builder.context) { + lwsl_err("lws init failed\n"); + return 1; + } + + /* ... and our vhost... */ + + builder.vhost = lws_create_vhost(builder.context, &info); + if (!builder.vhost) { + lwsl_err("Failed to create tls vhost\n"); + goto bail; + } + + pthread_mutex_init(&builder.mi.mut, NULL); + pthread_cond_init(&builder.mi.cond, NULL); + + /* + * Our approach is to split off a thread to do the git remote handling + * in a serialized way blocking the related threadpool threads until it + * completes, without blocking the main (lws) event loop thread. + * + * It has to be segregated because there is a repo write lock with one + * owner at a time for remote -> local mirror write operation. + * + * local mirror -> build-specific checkout doesn't need the write lock + * and can happen concurrently at the threadpool threads. + */ + + if (pthread_create(&builder.mi.repo_thread, NULL, thread_repo, + &builder.mi)) { + lwsl_err("%s: repo thread creation failed\n", __func__); + return 1; + } + + while (!lws_service(builder.context, 0) && !interrupted) + ; + +bail: + + /* destroy the unique masters */ + + lws_start_foreach_dll_safe(struct lws_dll2 *, p, p1, + builder.sai_plat_master_owner.head) { + struct sai_plat_master *cm = lws_container_of(p, + struct sai_plat_master, list); + + lws_dll2_remove(&cm->list); + lws_ss_destroy(&cm->ss); + + } lws_end_foreach_dll_safe(p, p1); + + saib_config_destroy(&builder); + + /* + * Clean up after the threads + */ + + builder.mi.finish = 1; + + pthread_mutex_lock(&builder.mi.mut); + pthread_cond_broadcast(&builder.mi.cond); + pthread_mutex_unlock(&builder.mi.mut); + + pthread_join(builder.mi.repo_thread, &retval); + + pthread_mutex_destroy(&builder.mi.mut); + pthread_cond_destroy(&builder.mi.cond); + + lws_start_foreach_dll_safe(struct lws_dll2 *, mp, mp1, + builder.sai_plat_owner.head) { + struct sai_plat *sp = lws_container_of(mp, struct sai_plat, + sai_plat_list); + + lws_dll2_remove(&sp->sai_plat_list); + + lws_start_foreach_dll_safe(struct lws_dll2 *, p, p1, + sp->nspawn_owner.head) { + struct sai_nspawn *ns = lws_container_of(p, + struct sai_nspawn, list); + + sai_ns_destroy(ns); + + } lws_end_foreach_dll_safe(p, p1); + + } lws_end_foreach_dll_safe(mp, mp1); + + lws_context_destroy(builder.context); + + return 0; +} diff --git a/src/builder/b-task.c b/src/builder/b-task.c new file mode 100644 index 0000000..d20dcdd --- /dev/null +++ b/src/builder/b-task.c @@ -0,0 +1,757 @@ +/* + * sai-builder task acquisition + * + * Copyright (C) 2019 - 2020 Andy Green + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation: + * version 2.1 of the License. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + * MA 02110-1301 USA + */ + +#include +#include +#include +#include +#include + +#include "b-private.h" + +#if !defined(WIN32) +static char csep = '/'; +#else +static char csep = '\\'; +#endif + +const lws_struct_map_t lsm_schema_map_m_to_b[] = { + LSM_SCHEMA (sai_task_t, NULL, lsm_task, "com-warmcat-sai-ta"), + LSM_SCHEMA (sai_cancel_t, NULL, lsm_task_cancel, "com.warmcat.sai.taskcan"), +}; + +enum { + SAIB_RX_TASK_ALLOCATION, + SAIB_RX_TASK_CANCEL, +}; + +static const char * const nsstates[] = { + "NSSTATE_INIT", + "NSSTATE_MOUNTING", + "NSSTATE_STARTING_MIRROR", + "NSSTATE_CHECKOUT_SPEC", /* initial speculative checkout */ + "NSSTATE_WAIT_REMOTE_MIRROR", + "NSSTATE_CHECKOUT", + "NSSTATE_CHECKEDOUT", + "NSSTATE_BUILD", + "NSSTATE_DONE", + "NSSTATE_FAILED", +}; + +int +saib_set_ns_state(struct sai_nspawn *ns, int state) +{ + char log[100]; + int n; + + ns->state = state; + ns->state_changed = 1; + + n = lws_snprintf(log, sizeof(log), ">saib> %s\n", nsstates[state]); + + saib_log_chunk_create(ns, log, n, 3); + + if (state == NSSTATE_FAILED) { + ns->retcode = SAISPRF_EXIT | 254; + saib_task_grace(ns); + } + + if (ns->spm && ns->spm->ss) + lws_ss_request_tx(ns->spm->ss); + + return 0; +} + +/* + * update all masters we're connected to about builder status / optional reject + */ + +int +saib_queue_task_status_update(sai_plat_t *sp, struct sai_plat_master *spm, + const char *rej_task_uuid) +{ + struct sai_rejection *rej; + + if (!spm) + return -1; + + rej = malloc(sizeof(*rej)); + + if (!rej) + return -1; + + if (!spm->ss) + return 0; + + memset(rej, 0, sizeof(*rej)); + + /* + * Queue a builder status update / + * optional task rejection + */ + + if (rej_task_uuid) { + lwsl_notice("%s: builder .%d occupied reject\n", + __func__, (int)(intptr_t)spm->opaque_data); + + lws_strncpy(rej->task_uuid, rej_task_uuid, + sizeof(rej->task_uuid)); + } + + lws_snprintf(rej->host_platform, sizeof(rej->host_platform), "%s", + sp->name); + + rej->limit = sp->instances; + rej->ongoing = sp->ongoing; + + lws_dll2_add_tail(&rej->list, &spm->rejection_list); + lws_ss_request_tx(spm->ss); + + return 0; +} + +void +saib_task_destroy(struct sai_nspawn *ns) +{ + ns->finished_when_logs_drained = 0; + lws_sul_cancel(&ns->sul_cleaner); + lws_sul_cancel(&ns->sul_task_cancel); + + /* + * If able, builder should reintroduce himself to get + * another task + */ + if (ns->spm) { + ns->spm->phase = PHASE_START_ATTACH; + lws_ss_request_tx(ns->spm->ss); + } + + if (ns->tp) { + ns->tp_task = NULL; + lwsl_notice("%s: calling threadpool_finish\n", __func__); + lws_threadpool_finish(ns->tp); + lwsl_notice("%s: calling threadpool_destroy\n", __func__); + lws_threadpool_destroy(ns->tp); + ns->tp = NULL; + } + + if (ns->task && ns->task->told_ongoing) { + + /* + * Account that we're not doing this task any more + */ + + lwsl_notice("%s: ongoing %d -> %d\n", __func__, + ns->sp->ongoing, ns->sp->ongoing - 1); + ns->sp->ongoing--; + + /* + * Schedule informing all the masters we're connected to + */ + + saib_queue_task_status_update(ns->sp, ns->spm, NULL); + } + + if (ns->task && ns->task->ac_task_container) { + /* contains the task object */ + lwsac_free(&ns->task->ac_task_container); + ns->task = NULL; + } +} + +static void +saib_sub_cleaner_cb(lws_sorted_usec_list_t *sul) +{ + struct sai_nspawn *ns = lws_container_of(sul, struct sai_nspawn, + sul_cleaner); + lwsl_warn("%s: .%d: Destroying task after grace period\n", + __func__, ns->instance_idx); + + saib_task_destroy(ns); +} + +static int +artifact_glob_cb(void *data, const char *path) +{ + struct sai_nspawn *ns = (struct sai_nspawn *)data; + const char *p, *ph = NULL; + struct lws_ss_handle *h; + char upp[256], s[384]; + sai_artifact_t *ap; + int n; + + /* + * "path" passed the filter... + * + * In order that we can maximize usage of the CI builder, first mv + * the artifact from path to ns->inp + "uploads/" + * + filename part + */ + + p = path; + while (*p) { + if (*p == '/' || *p == '\\') + ph = p + 1; + p++; + } + + if (!ph) + return 1; + + /* + * This builder might complete another task that happens to create the + * same- named artifact before we finish uploading this one, trashing + * the temp copy on disk. So we add the timestamp in the temp upload + * copy filename that this can't happen. + */ + + lws_snprintf(upp, sizeof(upp), "%s../.sai-uploads/%llu-%s", ns->inp, + (unsigned long long)lws_now_usecs(), ph); + + n = lws_snprintf(s, sizeof(s), ">saib> Artifact: %s\n", upp); + saib_log_chunk_create(ns, s, (size_t)n, 3); + + lwsl_notice("%s: moving %s -> %s\n", __func__, path, upp); + if (rename(path, upp)) { + lwsl_err("%s: mv artifact %s %s failed\n", __func__, path, upp); + return 1; + } + + /* pass upp in as the opaque data... we use it during CREATING */ + + if (lws_ss_create(builder.context, 0, &ssi_sai_artifact, upp, &h, + NULL, NULL)) { + lwsl_err("%s: failed to create secure stream\n", + __func__); + return -1; + } + + ap = lws_ss_to_user_object(h); + /* take a copy so we can unlink the path later */ + lws_strncpy(ap->path, upp, sizeof(ap->path)); + + lws_strncpy(ap->task_uuid, ns->task->uuid, sizeof(ap->task_uuid)); + lws_strncpy(ap->artifact_up_nonce, ns->task->art_up_nonce, + sizeof(ap->artifact_up_nonce)); + lws_strncpy(ap->blob_filename, ph, sizeof(ap->blob_filename)); + ap->timestamp = lws_now_usecs(); + + lwsl_notice("%s: artifact ss created '%s'\n", __func__, ap->path); + + + /* + * We need to set the metadata items for the post urlargs. spm->url is + * something like "wss://warmcat.com/sai/builder"... we will send JSON + * on this connection first and that will be understood by the master + * as meaning the bulk data follows. + */ + + lws_ss_set_metadata(h, "url", ns->spm->url, strlen(ns->spm->url)); + lws_ss_client_connect(h); + + return 0; +} + +/* + * We're finished with the nspawn / task one way or the other, but there's + * still stuff we need to send out. Give it some time then force destruction + * of the task and reset the nspawn. + */ + +void +saib_task_grace(struct sai_nspawn *ns) +{ + char filt[32], scandir[256]; + struct lws_tokenize ts; + uint8_t *p, *p1, *ps; + lws_dir_glob_t g; + int m; + + ns->finished_when_logs_drained = 1; + lws_sul_schedule(builder.context, 0, &ns->sul_cleaner, + saib_sub_cleaner_cb, 20 * LWS_USEC_PER_SEC); + + if (!ns->spm) + return; + + /* + * Let's look for any artifacts the saifile lists... + * they're done as globs so the package filenames or + * whatever may contain substrings like git commit hash + * without having to know it in the saifile. + * + * The file search context is ns->inp, which is where + * the instance and platform-specific build takes place. + * + * We get a comma-separated list of globs possibly each + * with a path offset like "build/\*.rpm". Let's parse + * out each in turn, and do an lws_dir at any path + * offset, matching on the remaining glob. + */ + + lws_tokenize_init(&ts, ns->task->artifacts, + LWS_TOKENIZE_F_NO_INTEGERS | + LWS_TOKENIZE_F_NO_FLOATS | + LWS_TOKENIZE_F_SLASH_NONTERM | + LWS_TOKENIZE_F_DOT_NONTERM | + LWS_TOKENIZE_F_MINUS_NONTERM); + + m = 0; + filt[0] = '\0'; + while ((ts.e = lws_tokenize(&ts)) >= 0) { + switch (ts.e) { + case LWS_TOKZE_ENDED: + if (filt[0]) + goto scan; + break; + case LWS_TOKZE_DELIMITER: + if (*ts.token == ',') + goto scan; + else + filt[m++] = *ts.token; + break; + case LWS_TOKZE_TOKEN: + lws_strnncpy(&filt[m], ts.token, ts.token_len, + sizeof(filt) - 1 - m); + m = (int)strlen(filt); + break; + } + if (ts.e == LWS_TOKZE_ENDED) + break; + continue; +scan: + lws_strncpy(scandir, ns->inp, sizeof(scandir)); + m = (int)strlen(scandir); + + /* + * if the filter has a fully-defined subdir, + * append it to the start path + */ + + ps = p1 = p = (uint8_t *)filt; + while (*p) { + if (*p == '/') { + if (lws_ptr_diff(p, p1) + 2 < + (int)sizeof(scandir) - m) { + memcpy(scandir + m, p1, + lws_ptr_diff(p, p1)); + m += lws_ptr_diff(p, p1); + scandir[m] = '\0'; + } else + break; + p1 = p; + ps = p + 1; + } + if (*p == '*') + break; + p++; + } + + lwsl_notice("%s: scan path %s, filter %s\n", __func__, + scandir, (const char *)ps); + + g.filter = (char *)ps; + g.user = (void *)ns; + g.cb = artifact_glob_cb; + + lws_dir(scandir, &g, lws_dir_glob_cb); + + filt[0] = '\0'; + m = 0; + + if (ts.e == LWS_TOKZE_ENDED) + break; + } +} + +static void +saib_sul_task_cancel(struct lws_sorted_usec_list *sul) +{ + struct sai_nspawn *ns = lws_container_of(sul, + struct sai_nspawn, sul_task_cancel); + char s[64]; + int n; + + if (!ns->lsp) + return; + + n = lws_snprintf(s, sizeof(s), ">saib> Cancelling...\n"); + saib_log_chunk_create(ns, s, (size_t)n, 3); + + lws_spawn_piped_kill_child_process(ns->lsp); + if (!--ns->term_budget) + return; + + lws_sul_schedule(ns->builder->context, 0, &ns->sul_task_cancel, + saib_sul_task_cancel, 500 * LWS_US_PER_MS); +} + +int +saib_ws_json_rx_builder(struct sai_plat_master *spm, const void *in, size_t len) +{ + struct lws_threadpool_create_args tca; + struct lws_threadpool_task_args tpa; + sai_plat_t *sp = NULL; + struct sai_nspawn *ns; + struct lejp_ctx ctx; + lws_struct_args_t a; + sai_cancel_t *can; + char url[128], *p; + sai_task_t *task; + int n, m; + + /* + * use the schema name on the incoming JSON to decide what kind of + * structure to instantiate + */ + + memset(&a, 0, sizeof(a)); + a.map_st[0] = lsm_schema_map_m_to_b; + a.map_entries_st[0] = LWS_ARRAY_SIZE(lsm_schema_map_m_to_b); + a.ac_block_size = 512; + +// lwsl_hexdump_warn(in, len); + + lws_struct_json_init_parse(&ctx, NULL, &a); + m = lejp_parse(&ctx, (uint8_t *)in, (int)len); + if (m < 0) { + printf("%.*s\n", (int)len, (const char *)in); + lwsl_notice("%s: builder rx JSON decode failed '%s'\n", + __func__, lejp_error_to_string(m)); + return m; + } + + if (!a.dest) { + lwsac_free(&a.ac); + return 1; + } + + switch (a.top_schema_index) { + case SAIB_RX_TASK_ALLOCATION: + task = (sai_task_t *)a.dest; + task->ac_task_container = a.ac; /* bequeath lwsac responsibility */ + + /* + * Master is requesting that a platform adopt a task... + * + * Multiple platforms may be using this connection to a given + * master so we have to disambiguate which platform he's + * tasking first. + */ + + lws_start_foreach_dll_safe(struct lws_dll2 *, d, d1, + builder.sai_plat_owner.head) { + sp = lws_container_of(d, sai_plat_t, sai_plat_list); + + if (!strcmp(sp->platform, task->platform)) + break; + sp = NULL; + } lws_end_foreach_dll_safe(d, d1); + + if (!sp) { + lwsl_err("%s: can't identify req task plat '%s'\n", + __func__, task->platform); + + return 1; + } + + /* + * store a copy of the toplevel ac used for the deserialization + * into the outer part of the c builder wrapper + */ + + sp->deserialization_ac = a.ac; + + /* + * Look for a spare nspawn... + * + * We may connect to multiple masters and it's asynchronous + * which master may have tasked us first, so it's not that + * unusual to reject a task the master thought we could have + * taken + */ + + n = 0; + ns = NULL; + lws_start_foreach_dll_safe(struct lws_dll2 *, d, d1, + sp->nspawn_owner.head) { + struct sai_nspawn *xns = lws_container_of(d, + struct sai_nspawn, list); + + n++; + if (!xns->task) { + ns = xns; + break; + } + + } lws_end_foreach_dll_safe(d, d1); + + if (!ns) { + + /* + * Full up... reject the task and update every + * master's model of our task load status + */ + + lwsl_notice("%s: plat '%s': no idle nspawn (of %d), " + "plat load %d / %d\n", __func__, sp->name, + n, sp->ongoing, sp->instances); + if (saib_queue_task_status_update(sp, spm, task->uuid)) + return -1; + + return 0; + } + +// lwsl_hexdump_warn(task->build, strlen(task->build)); + + /* create a taskqueue just for preparing this specific spawn */ + + memset(&tpa, 0, sizeof(tpa)); + tca.threads = 1; + tca.max_queue_depth = 1; + ns->tp = lws_threadpool_create(builder.context, &tca, "nsp-%s", + task->uuid); + + lws_strncpy(ns->fsm.distro, task->platform, + sizeof(ns->fsm.distro)); + lws_filename_purify_inplace(ns->fsm.distro); + p = ns->fsm.distro; + while ((p = strchr(p, '/'))) + *p = '_'; + + /* + * unique for remote master name ("warmcat"), + * project name ("libwebsockets") + */ + ns->master_name = spm->name; + ns->project_name = task->repo_name; + if (!strncmp(task->git_ref, "refs/heads/", 11)) + ns->ref = task->git_ref + 11; + else + if (!strncmp(task->git_ref, "refs/tags/", 10)) + ns->ref = task->git_ref + 10; + else + ns->ref = task->git_ref; + ns->hash = task->git_hash; + ns->git_repo_url = task->git_repo_url; + ns->task = task; /* we are owning this nspawn for the duration */ + ns->spm = spm; /* bind this task to the spm the req came in on */ + + { + int ml; + char mb[96]; + + ml = lws_snprintf(mb, sizeof(mb), + ">saib> Sai Builder Version: %s, lws: %s\n", + BUILD_INFO, LWS_BUILD_HASH); + saib_log_chunk_create(ns, mb, ml, 3); + } + saib_set_ns_state(ns, NSSTATE_INIT); + +#if defined(__linux__) + ns->fsm.layers[0] = "base"; + ns->fsm.layers[1] = "env"; +#endif + + lws_snprintf(ns->fsm.ovname, sizeof(ns->fsm.ovname), "%d-%d.%d", + spm->index, ns->sp->index, ns->instance_idx); + + lwsl_notice("%s: master %s\n", __func__, ns->master_name); + lwsl_notice("%s: project %s\n", __func__, ns->project_name); + lwsl_notice("%s: ref %s\n", __func__, ns->ref); + lwsl_notice("%s: hash %s\n", __func__, ns->hash); + lwsl_notice("%s: distro %s\n", __func__, ns->fsm.distro); + lwsl_notice("%s: ovname %s\n", __func__, ns->fsm.ovname); + lwsl_notice("%s: git_repo_url %s\n", __func__, ns->git_repo_url); + lwsl_notice("%s: mountpoint %s\n", __func__, ns->fsm.mp); + + spm->phase = PHASE_BUILDING; + n = lws_snprintf(ns->inp, sizeof(ns->inp), "%s%c", + builder.home, csep); + + n += lws_snprintf(ns->inp + n, sizeof(ns->inp) - n, "jobs%c", + csep); + lws_filename_purify_inplace(ns->inp); + if (mkdir(ns->inp, 0755) && errno != EEXIST) + goto ebail; + + n += lws_snprintf(ns->inp + n, sizeof(ns->inp) - n, "%s%c", + ns->fsm.ovname, csep); + lws_filename_purify_inplace(ns->inp); + if (mkdir(ns->inp, 0755) && errno != EEXIST) + goto ebail; + + n += lws_snprintf(ns->inp + n, sizeof(ns->inp) - n, "%s%c", + ns->project_name, csep); + lws_filename_purify_inplace(ns->inp); + if (mkdir(ns->inp, 0755) && errno != EEXIST) + goto ebail; + + /* + * Create a pending upload dir to mv artifacts into while + * we get on with the next job. + */ + lws_snprintf(ns->inp + n, sizeof(ns->inp) - n, "../.sai-uploads"); + if (mkdir(ns->inp, 0755) && errno != EEXIST) + goto ebail; + /* + * Snip that last bit off so ns->inp is the fully qualified + * builder instance base dir + */ + ns->inp[n] = '\0'; + + /* the git mirror is in the build host's /home/sai, NOT the + * mounted overlayfs */ + + m = lws_snprintf(ns->path, sizeof(ns->path), + "%s%cgit-mirror", builder.home, csep); + + if (mkdir(ns->path, 0755) && errno != EEXIST) + goto ebail; + + lws_strncpy(url, ns->git_repo_url, sizeof(url)); + + /* + * We want to convert the repo url to a unique subdir name + * that's safe, like http___warmcat.com_repo_sai + */ + + lws_filename_purify_inplace(url); + { + char *q = url; + while (*q) { + if (*q == '/') + *q = '_'; + if (*q == '.') + *q = '_'; + q++; + } + } + m += lws_snprintf(ns->path + m, sizeof(ns->path) - m, "%c%s", + csep, url); + if (mkdir(ns->path, 0755) && errno != EEXIST) { + lwsl_err("%s: unable to create %s\n", __func__, + ns->path); + goto bail; + } + lwsl_notice("%s: created %s\n", __func__, ns->path); + + /* manage the mirror dir using build host credentials */ + + saib_set_ns_state(ns, NSSTATE_STARTING_MIRROR); + + memset(&tpa, 0, sizeof(tpa)); + tpa.ss = spm->ss; + tpa.user = ns; + tpa.name = "nsp"; + tpa.task = saib_mirror_task; + + ns->user_cancel = 0; + ns->spins = 0; + + lwsl_warn("%s: enqueuing mirror thread\n", __func__); + + ns->tp_task = lws_threadpool_enqueue(ns->tp, &tpa, "tptask-%s", + task->uuid); + if (!ns->tp_task) { + lwsl_err("%s: threadpool enqueue failed\n", __func__); + goto bail; + } + + /* + * If we successfully set the threadpool task up, then we + * bump our idea of what is ongoing... completing or failing + * after that needs to adjust sp->ongoing accordingly + */ + + lwsl_warn("%s: enqueued mirror thread\n", __func__); + + lwsl_notice("%s: ongoing %d -> %d\n", __func__, sp->ongoing, + sp->ongoing + 1); + sp->ongoing++; + ns->task->told_ongoing = 1; + + /* + * Let the mirror thread get on with things... + * + * When we took on a task, we should inform any masters we're + * connected to about our change in task load status + */ + + if (saib_queue_task_status_update(sp, spm, NULL)) { + goto bail; + } + + break; + + case SAIB_RX_TASK_CANCEL: + + can = (sai_cancel_t *)a.dest; + + lwsl_notice("%s: SAIB_RX_TASK_CANCEL: %s\n", __func__, can->task_uuid); + + lws_start_foreach_dll_safe(struct lws_dll2 *, mp, mp1, + builder.sai_plat_owner.head) { + struct sai_plat *sp = lws_container_of(mp, struct sai_plat, + sai_plat_list); + + lws_start_foreach_dll_safe(struct lws_dll2 *, p, p1, + sp->nspawn_owner.head) { + struct sai_nspawn *ns = lws_container_of(p, + struct sai_nspawn, list); + + if (ns->task && + !strcmp(can->task_uuid, ns->task->uuid)) { + lwsl_notice("%s: trying to cancel %s\n", + __func__, can->task_uuid); + + /* + * We're going to send a few signals + * at 500ms intervals + */ + ns->user_cancel = 1; + ns->term_budget = 5; + lws_sul_schedule(ns->builder->context, 0, + &ns->sul_task_cancel, + saib_sul_task_cancel, 1); + } + + } lws_end_foreach_dll_safe(p, p1); + + } lws_end_foreach_dll_safe(mp, mp1); + break; + + default: + break; + } + + return 0; + +ebail: + lwsl_err("%s: unable to create %s\n", __func__, ns->inp); + +bail: + lwsl_notice("%s: failing spawn cleanly\n", __func__); + saib_set_ns_state(ns, NSSTATE_FAILED); + saib_task_grace(ns); + + return 1; +} diff --git a/src/builder/conf.c b/src/builder/conf.c deleted file mode 100644 index deb8588..0000000 --- a/src/builder/conf.c +++ /dev/null @@ -1,340 +0,0 @@ -/* - * sai-builder conf.c - * - * Copyright (C) 2019 Andy Green - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation: - * version 2.1 of the License. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, - * MA 02110-1301 USA - */ - -#include -#include -#include -#include - -#include "private.h" - -static const char * const paths[] = { - "systemd-nspawn.path", - "systemd-nspawn.instances", - "systemd-nspawn.timeout", - - "target[]", - "target[].name", - "target[].timeout", - "target[].consoles[]", - "target[].consoles[].name", - "target[].consoles[].path", - "target[].consoles[].baudrate", - "target[].consoles[].mode", - - "masters[]", - "masters[].url", - "masters[].interface", - "masters[].selfsigned", - "masters[].priority", /* array of builder names */ -}; - -enum enum_paths { - LEJPM_NSPAWN_PATH, - LEJPM_NSPAWN_INSTANCES, - LEJPM_NSPAWN_TIMEOUT, - - LEJPM_TARGET, - LEJPM_TARGET_NAME, - LEJPM_TARGET_TIMEOUT, - LEJPM_TARGET_CONSOLES, - LEJPM_TARGET_CONSOLES_NAME, - LEJPM_TARGET_CONSOLES_PATH, - LEJPM_TARGET_CONSOLES_BAUDRATE, - LEJPM_TARGET_CONSOLES_MODE, - - LEJPM_MASTERS, - LEJPM_MASTERS_URL, - LEJPM_MASTERS_INTERFACE, - LEJPM_MASTERS_SELFSIGNED, - LEJPM_MASTERS_PRIORITY, -}; - -struct jpargs { - struct sai_builder *builder; - - struct sai_nspawn *nspawn; - struct sai_master *master; - struct sai_target *target; - struct sai_console *console; -}; - -#define SAI_BUILDER_CONF_CHUNK 512 -#define SAI_BUILDER_PATH_MAX 128 - -static int -sai_builder_scan_overlays_cb(const char *dirpath, void *user, - struct lws_dir_entry *lde) -{ - struct sai_nspawn *nspawn = (struct sai_nspawn *)user; - size_t s = strlen(lde->name); - struct sai_overlay *overlay; - - if (lde->type != LDOT_DIR) - return 0; - - if (strncmp(lde->name, "env-", 4)) - return 0; - - overlay = lwsac_use(&nspawn->builder->conf_head, sizeof(*overlay), - SAI_BUILDER_CONF_CHUNK); - if (!overlay) - return 1; - - memset(overlay, 0, sizeof(*overlay)); - overlay->name = lwsac_use(&nspawn->builder->conf_head, s + 1, - SAI_BUILDER_CONF_CHUNK); - if (!overlay->name) - return 1; - memcpy(overlay->name, lde->name, s + 1); - - lws_dll_add_front(&overlay->overlay_list, &nspawn->overlay_head); - - return 0; -} - -static int -sai_builder_scan_nspawn_cb(const char *dirpath, void *user, - struct lws_dir_entry *lde) -{ - struct sai_builder *builder = (struct sai_builder *)user; - size_t s = strlen(lde->name); - struct sai_nspawn *nspawn; - char path[256]; - - if (lde->type != LDOT_DIR) - return 0; - - nspawn = lwsac_use(&builder->conf_head, sizeof(*nspawn), - SAI_BUILDER_CONF_CHUNK); - if (!nspawn) - return 1; - - memset(nspawn, 0, sizeof(*nspawn)); - nspawn->builder = builder; - nspawn->name = lwsac_use(&builder->conf_head, s + 1, - SAI_BUILDER_CONF_CHUNK); - if (!nspawn->name) { - free(nspawn); - return 1; - } - memcpy((void *)nspawn->name, lde->name, s + 1); - - lws_dll_add_front(&nspawn->nspawn_list, &builder->nspawn_head); - - lws_snprintf(path, sizeof(path) - 1, "%s/%s", dirpath, lde->name); - - if (lws_dir(path, nspawn, sai_builder_scan_overlays_cb)) { - lwsl_err("%s: unable to scan nspawn dir %s\n", __func__, - builder->nspawn_path); - return 1; - } - - return 0; -} - -static signed char -sai_builder_conf_cb(struct lejp_ctx *ctx, char reason) -{ - struct jpargs *a = (struct jpargs *)ctx->user; - const char **pp; - int n; - -#if 0 - lwsl_notice(" %d: %s (%d)\n", reason, ctx->path, ctx->path_match); - for (n = 0; n < ctx->wildcount; n++) - lwsl_notice(" %d\n", ctx->wild[n]); -#endif - - if (reason == LEJPCB_OBJECT_START) { - switch (ctx->path_match - 1) { - case LEJPM_TARGET: - a->target = lwsac_use_zeroed(&a->builder->conf_head, - sizeof(*a->target), - SAI_BUILDER_CONF_CHUNK); - if (!a->target) - return 1; - lws_dll_add_front(&a->target->target_list, - &a->builder->target_head); - return 0; - - case LEJPM_TARGET_CONSOLES: - a->console = lwsac_use_zeroed(&a->builder->conf_head, - sizeof(*a->console), - SAI_BUILDER_CONF_CHUNK); - if (!a->console) - return 1; - lws_dll_add_front(&a->console->console_list, - &a->target->console_head); - return 0; - - case LEJPM_MASTERS: - a->master = lwsac_use_zeroed(&a->builder->conf_head, - sizeof(*a->master), - SAI_BUILDER_CONF_CHUNK); - if (!a->master) - return 1; - lws_dll_add_front(&a->master->master_list, - &a->builder->master_head); - return 0; - - default: - return 0; - } - } - - /* we only match on the prepared path strings */ - if (!(reason & LEJP_FLAG_CB_IS_VALUE) || !ctx->path_match) - return 0; - - if (reason == LEJPCB_VAL_NUM_INT) { - n = atoi(ctx->buf); - switch (ctx->path_match - 1) { - case LEJPM_NSPAWN_INSTANCES: - a->builder->nspawn_instances = n; - break; - case LEJPM_NSPAWN_TIMEOUT: - a->builder->nspawn_timeout = n; - break; - case LEJPM_TARGET_TIMEOUT: - a->target->timeout = n; - break; - case LEJPM_TARGET_CONSOLES_BAUDRATE: - a->console->baudrate = n; - break; - case LEJPM_MASTERS_SELFSIGNED: - a->master->accept_selfsigned = n; - break; - case LEJPM_MASTERS_PRIORITY: - a->master->priority = n; - break; - default: - break; - } - - return 0; - } - - if (reason != LEJPCB_VAL_STR_END) - return 0; - - /* only the end part of the string, where we know the length */ - - switch (ctx->path_match - 1) { - case LEJPM_NSPAWN_PATH: - pp = &a->builder->nspawn_path; - break; - - case LEJPM_TARGET_NAME: - pp = &a->target->name; - break; - - case LEJPM_TARGET_CONSOLES_NAME: - pp = &a->console->name; - break; - - case LEJPM_TARGET_CONSOLES_PATH: - pp = &a->console->path; - break; - - case LEJPM_TARGET_CONSOLES_MODE: - pp = &a->console->mode; - break; - - case LEJPM_MASTERS_URL: - pp = &a->master->url; - break; - - case LEJPM_MASTERS_INTERFACE: - pp = &a->master->interface; - break; - - default: - return 0; - } - - *pp = lwsac_use(&a->builder->conf_head, ctx->npos + 1, - SAI_BUILDER_CONF_CHUNK); - if (!*pp) - return 1; - memcpy((char *)(*pp), ctx->buf, ctx->npos); - ((char *)(*pp))[ctx->npos] = '\0'; - - return 0; -} - -int -sai_builder_config(struct sai_builder *builder, const char *d) -{ - unsigned char buf[128]; - struct lejp_ctx ctx; - struct jpargs a; - int n, m, fd; - - memset(&a, 0, sizeof(a)); - a.builder = builder; - - lws_snprintf((char *)buf, sizeof(buf) - 1, "%s/conf", d); - - fd = lws_open((char *)buf, O_RDONLY); - if (fd < 0) { - lwsl_err("Cannot open %s\n", (char *)buf); - return 2; - } - lwsl_info("%s: %s\n", __func__, (char *)buf); - lejp_construct(&ctx, sai_builder_conf_cb, &a, paths, LWS_ARRAY_SIZE(paths)); - - do { - n = read(fd, buf, sizeof(buf)); - if (!n) - break; - - m = (int)(signed char)lejp_parse(&ctx, buf, n); - } while (m == LEJP_CONTINUE); - - close(fd); - n = ctx.line; - lejp_destruct(&ctx); - - if (m < 0) { - lwsl_err("%s/conf(%u): parsing error %d: %s\n", d, n, m, - lejp_error_to_string(m)); - return 2; - } - - /* scan the configured nspawn directory for what we support */ - - if (builder->nspawn_path && - lws_dir(builder->nspawn_path, builder, sai_builder_scan_nspawn_cb)) { - lwsl_err("%s: unable to scan nspawn dir %s\n", __func__, - builder->nspawn_path); - - return 1; - } - - return 0; -} - -void -sai_builder_config_destroy(struct sai_builder *builder) -{ - lwsac_free(&builder->conf_head); -} diff --git a/src/builder/private.h b/src/builder/private.h deleted file mode 100644 index 0bc1347..0000000 --- a/src/builder/private.h +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Sai builder definitions src/builder/private.h - * - * Copyright (C) 2019 Andy Green - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation: - * version 2.1 of the License. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, - * MA 02110-1301 USA - */ - -struct sai_master; -struct sai_builder; - -enum sai_lease_states { - SOS_IDLE, - SOS_RESERVED, - SOS_CONFIRMED -}; - -struct sai_lease { - struct sai_master *lessor; - enum sai_lease_states state; -}; - -struct sai_console { - struct lws_dll console_list; - - const char *name; - const char *path; - const char *mode; - int baudrate; -}; - -struct sai_target { - struct lws_dll target_list; - struct lws_dll console_head; - - const char *name; - - struct sai_lease lease; - - int timeout; -}; - -struct sai_master { - struct lws_dll master_list; - - struct lws *cwsi; - - const char *url; - const char *interface; - int priority; - - unsigned int accept_selfsigned:1; -}; - -struct sai_overlay { - struct lws_dll overlay_list; - - char *name; -}; - -struct sai_nspawn { - struct lws_dll nspawn_list; - struct lws_dll overlay_head; - - struct sai_builder *builder; - - const char *name; - - struct sai_lease lease; -}; - -struct sai_builder { - struct lws_dll master_head; - struct lws_dll target_head; - struct lws_dll nspawn_head; - - struct lwsac *conf_head; - - const char *nspawn_path; - int nspawn_instances; - int nspawn_timeout; -}; - -extern const struct lws_protocols protocol_com_warmcat_sai; -extern int sai_builder_config(struct sai_builder *builder, const char *d); -extern void sai_builder_config_destroy(struct sai_builder *builder); diff --git a/src/builder/protocol.c b/src/builder/protocol.c deleted file mode 100644 index 736b35a..0000000 --- a/src/builder/protocol.c +++ /dev/null @@ -1,344 +0,0 @@ -/* - * sai-builder com-warmcat-sai client protocol implementation - * - * Copyright (C) 2019 Andy Green - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation: - * version 2.1 of the License. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, - * MA 02110-1301 USA - */ - -#include -#include -#include - -#include "private.h" - -typedef enum { - PHASE_IDLE, - - PFL_FIRST = 128, - - PHASE_START_ATTACH = PFL_FIRST | 1, - PHASE_SUMM_TARGETS = 2, - PHASE_SUMM_NSPAWNS = 3, - PHASE_SUMM_NSPAWNS_OVERLAYS = 4, - -} cursor_phase_t; - -struct pss { - struct sai_target *target; - struct sai_console *console; - - struct sai_nspawn *nspawn; - struct sai_overlay *overlay; - - cursor_phase_t phase; -}; - -struct vhd { - struct lws_context *context; - struct lws_vhost *vhost; - - struct sai_builder builder; - - int *interrupted; - const char **config_dir; -}; - -static int -generate(struct vhd *vhd, struct pss *pss, char *buf, int len) -{ - struct sai_builder *b = &vhd->builder; - char *p = buf, *end = buf + len - 10; - - while (pss->phase != PHASE_IDLE && lws_ptr_diff(end, p) > 100) { - - switch (pss->phase) { - case PHASE_IDLE: - break; - - case PHASE_START_ATTACH: - p += lws_snprintf(p, lws_ptr_diff(end, p), - "{\"schema\":\"com-warmcat-sai-ba\",\n" - " \"hostname\":\"%s\",\n" - " \"nspawn-timeout\":%d,\n" - " \"targets\": [", - lws_canonical_hostname(vhd->context), - b->nspawn_timeout); - pss->target = lws_container_of(b->target_head.next, - struct sai_target, - target_list); - pss->phase = PHASE_SUMM_TARGETS; - break; - - case PHASE_SUMM_TARGETS: - if (pss->target) { - p += lws_snprintf(p, lws_ptr_diff(end, p), - "{\"name\":\"%s\"}", - pss->target->name); - if (pss->target->target_list.next) - *p++ = ','; - *p++ = '\n'; - *p = '\0'; - - pss->target = lws_container_of( - pss->target->target_list.next, - struct sai_target, target_list); - } - if (!pss->target) { - p += lws_snprintf(p, lws_ptr_diff(end, p), - "],\n\"nspawns\": ["); - pss->nspawn = lws_container_of( - b->nspawn_head.next, - struct sai_nspawn, nspawn_list); - pss->phase = PHASE_SUMM_NSPAWNS; - } - break; - - case PHASE_SUMM_NSPAWNS: - if (pss->nspawn) { - *p++ = '{'; - if (pss->nspawn->name) - p += lws_snprintf(p, lws_ptr_diff(end, p), - "\"name\":\"%s\",\n", - pss->nspawn->name); - p += lws_snprintf(p, lws_ptr_diff(end, p), - " \"overlays\": ["); - - pss->overlay = lws_container_of( - pss->nspawn->overlay_head.next, - struct sai_overlay, overlay_list); - if (pss->overlay) { - pss->phase = PHASE_SUMM_NSPAWNS_OVERLAYS; - break; - } -next_nspawn: - *p++ = ']'; - *p++ = '}'; - if (pss->nspawn->nspawn_list.next) - *p++ = ','; - *p++ = '\n'; - *p = '\0'; - pss->nspawn = lws_container_of( - pss->nspawn->nspawn_list.next, - struct sai_nspawn, nspawn_list); - } - if (!pss->nspawn) { - p += lws_snprintf(p, lws_ptr_diff(end, p), "]\n"); - p += lws_snprintf(p, lws_ptr_diff(end, p), "}\n"); - pss->phase = PHASE_IDLE; - } - break; - - case PHASE_SUMM_NSPAWNS_OVERLAYS: - if (pss->overlay) { - p += lws_snprintf(p, lws_ptr_diff(end, p), - "{\"name\":\"%s\"}\n", - pss->overlay->name); - pss->overlay = lws_container_of( - pss->overlay->overlay_list.next, - struct sai_overlay, overlay_list); - if (pss->overlay) { - - } - } - if (!pss->overlay) { - pss->phase = PHASE_SUMM_NSPAWNS; - goto next_nspawn; - } - break; - - default: - break; - } - } - - return lws_ptr_diff(p, buf); -} - -static int -connect_client(struct vhd *vhd, struct sai_master *master) -{ - struct lws_client_connect_info i; - const char *prot; - char url[128], host[64]; - - memset(&i, 0, sizeof(i)); - - strncpy(url, master->url, sizeof(url) - 1); - url[sizeof(url) - 1] = '\0'; - - if (lws_parse_uri(url, &prot, &i.address, &i.port, &i.path)) { - lwsl_err("%s: unable to parse url '%s'\n", __func__, url); - - return 1; - } - - lws_snprintf(host, sizeof(host), "%s:%u", i.address, i.port); - - i.context = vhd->context; - i.host = host; - i.origin = host; - - if (!strcmp(prot, "https") || !strcmp(prot, "wss")) - i.ssl_connection = LCCSCF_USE_SSL; - if (master->accept_selfsigned) - i.ssl_connection |= LCCSCF_ALLOW_SELFSIGNED; - i.vhost = vhd->vhost; - i.iface = master->interface; - i.pwsi = &master->cwsi; - i.opaque_user_data = (void *)master; - - lwsl_user("connecting to %s://%s:%d/%s\n", prot, i.address, i.port, i.path); - - return !lws_client_connect_via_info(&i); -} - -static void -schedule_callback(struct lws *wsi, int reason, int secs) -{ - lws_timed_callback_vh_protocol(lws_get_vhost(wsi), - lws_get_protocol(wsi), reason, secs); -} - -static int -callback_com_warmcat_sai(struct lws *wsi, enum lws_callback_reasons reason, - void *user, void *in, size_t len) -{ - struct pss *pss = (struct pss *)user; - struct vhd *vhd = (struct vhd *) - lws_protocol_vh_priv_get(lws_get_vhost(wsi), - lws_get_protocol(wsi)); - struct sai_master *master = - (struct sai_master *)lws_get_opaque_user_data(wsi); - uint8_t buf[1024 + LWS_PRE], *start = buf + LWS_PRE, - *end = buf + sizeof(buf) -1; - int n, m, f; - - switch (reason) { - - case LWS_CALLBACK_PROTOCOL_INIT: - vhd = lws_protocol_vh_priv_zalloc(lws_get_vhost(wsi), - lws_get_protocol(wsi), - sizeof(struct vhd)); - if (!vhd) - return -1; - - vhd->context = lws_get_context(wsi); - vhd->vhost = lws_get_vhost(wsi); - - /* get the pointer to "interrupted" we were passed in pvo */ - vhd->interrupted = (int *)lws_pvo_search( - (const struct lws_protocol_vhost_options *)in, - "interrupted")->value; - vhd->config_dir = (const char **)lws_pvo_search( - (const struct lws_protocol_vhost_options *)in, - "configdir")->value; - - if (sai_builder_config(&vhd->builder, *vhd->config_dir)) { - lwsl_err("%s: config failed\n", __func__); - - return 1; - } - - schedule_callback(wsi, LWS_CALLBACK_USER, 1); - break; - - case LWS_CALLBACK_PROTOCOL_DESTROY: - sai_builder_config_destroy(&vhd->builder); - break; - - case LWS_CALLBACK_CLIENT_ESTABLISHED: - pss->phase = PHASE_START_ATTACH; - lws_callback_on_writable(wsi); - break; - - case LWS_CALLBACK_CLIENT_WRITEABLE: - lwsl_user("LWS_CALLBACK_CLIENT_WRITEABLE\n"); - - if (pss->phase == PHASE_IDLE) - break; - - f = pss->phase & PFL_FIRST; - n = generate(vhd, pss, (char *)start, lws_ptr_diff(end, start)); - - m = lws_write(wsi, start, n, - lws_write_ws_flags(LWS_WRITE_TEXT, f, - pss->phase == PHASE_IDLE)); - if (m < n) { - lwsl_err("ERROR %d writing to ws socket\n", m); - return -1; - } - - if (pss->phase != PHASE_IDLE) - lws_callback_on_writable(wsi); - break; - - case LWS_CALLBACK_CLIENT_RECEIVE: - - break; - - case LWS_CALLBACK_CLIENT_CONNECTION_ERROR: - lwsl_err("CLIENT_CONNECTION_ERROR: %s\n", - in ? (char *)in : "(null)"); - master->cwsi = NULL; - schedule_callback(wsi, LWS_CALLBACK_USER, 1); - break; - - case LWS_CALLBACK_CLIENT_CLOSED: - lwsl_user("LWS_CALLBACK_CLIENT_CLOSED\n"); - master->cwsi = NULL; - schedule_callback(wsi, LWS_CALLBACK_USER, 1); - break; - - /* rate-limited client connect retries */ - - case LWS_CALLBACK_USER: - /* - * let's retry any master connections that aren't either - * ongoing or happy - */ - - m = 0; - lws_start_foreach_dll(struct lws_dll *, mp, - vhd->builder.master_head.next) { - struct sai_master *ma = lws_container_of(mp, - struct sai_master, master_list); - - if (!ma->cwsi && connect_client(vhd, ma)) - m = 1; - } lws_end_foreach_dll(mp); - - /* - * if somebody's already unhappy, schedule a retry. Otherwise - * wait until a connection close or error to do it then. - */ - if (m) - schedule_callback(wsi, LWS_CALLBACK_USER, 1); - break; - - default: - break; - } - - return 0; -} - -const struct lws_protocols protocol_com_warmcat_sai = { - "com-warmcat-sai", - callback_com_warmcat_sai, - sizeof(struct pss), - 1024, 0, NULL, 0 -}; diff --git a/src/builder/sai-builder.c b/src/builder/sai-builder.c deleted file mode 100644 index 1760330..0000000 --- a/src/builder/sai-builder.c +++ /dev/null @@ -1,114 +0,0 @@ -/* - * sai-builder - * - * Copyright (C) 2019 Andy Green - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation: - * version 2.1 of the License. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, - * MA 02110-1301 USA - */ - -#include -#include -#include - -#include "private.h" - -static struct lws_context *context; -static const char *config_dir = "/etc/sai/builder"; -static int interrupted; - -static const struct lws_protocols *pprotocols[] = { - &protocol_com_warmcat_sai, - NULL -}; - -static const struct lws_protocol_vhost_options pvo_configdir = { - NULL, - NULL, - "configdir", /* pvo name */ - (void *)&config_dir /* pvo value */ -}; - -static const struct lws_protocol_vhost_options pvo_interrupted = { - &pvo_configdir, - NULL, - "interrupted", /* pvo name */ - (void *)&interrupted /* pvo value */ -}; - -static const struct lws_protocol_vhost_options pvo = { - NULL, /* "next" pvo linked-list */ - &pvo_interrupted, /* "child" pvo linked-list */ - "com-warmcat-sai", /* protocol name we belong to on this vhost */ - "" /* ignored */ -}; -static const struct lws_extension extensions[] = { - { - "permessage-deflate", - lws_extension_callback_pm_deflate, - "permessage-deflate" - "; client_no_context_takeover" - "; client_max_window_bits" - }, - { NULL, NULL, NULL /* terminator */ } -}; - -void sigint_handler(int sig) -{ - interrupted = 1; -} - -int main(int argc, const char **argv) -{ - struct lws_context_creation_info info; - const char *p; - int logs = LLL_USER | LLL_ERR | LLL_WARN | LLL_NOTICE; - - if ((p = lws_cmdline_option(argc, argv, "-d"))) - logs = atoi(p); - - lws_set_log_level(logs, NULL); - lwsl_user("Sai Builder - " - "Copyright (C) 2019 Andy Green \n"); - lwsl_user(" sai-builder [-c ]\n"); - - memset(&info, 0, sizeof info); /* otherwise uninitialized garbage */ - info.port = CONTEXT_PORT_NO_LISTEN; - info.pprotocols = pprotocols; - info.pvo = &pvo; - if (!lws_cmdline_option(argc, argv, "-n")) - info.extensions = extensions; - info.pt_serv_buf_size = 32 * 1024; - info.options = LWS_SERVER_OPTION_DO_SSL_GLOBAL_INIT | - LWS_SERVER_OPTION_VALIDATE_UTF8; - - if (lws_cmdline_option(argc, argv, "--libuv")) - info.options |= LWS_SERVER_OPTION_LIBUV; - else - signal(SIGINT, sigint_handler); - - context = lws_create_context(&info); - if (!context) { - lwsl_err("lws init failed\n"); - return 1; - } - - while (!lws_service(context, 1000) && !interrupted) - ; - - lws_context_destroy(context); - - return 0; -} diff --git a/src/common/include/private.h b/src/common/include/private.h new file mode 100644 index 0000000..f4ec979 --- /dev/null +++ b/src/common/include/private.h @@ -0,0 +1,284 @@ +/* + * Sai - ./src/common/include/private.h + * + * Copyright (C) 2019-2020 Andy Green + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation: + * version 2.1 of the License. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + * MA 02110-1301 USA + * + * structs common to builder and master + */ + +#if defined(__linux__) +#define UDS_PATHNAME_LOGPROXY "@com.warmcat.com.saib.logproxy" +#else +#define UDS_PATHNAME_LOGPROXY "/var/run/com.warmcat.com.saib.logproxy" +#endif + +struct sai_plat; +struct sai_builder; + +typedef enum { + SAIES_WAITING = 0, + SAIES_PASSED_TO_BUILDER = 1, + SAIES_BEING_BUILT = 2, + SAIES_SUCCESS = 3, + SAIES_FAIL = 4, + SAIES_CANCELLED = 5, + SAIES_BEING_BUILT_HAS_FAILURES = 6, +} sai_event_state_t; + +enum { + SAISPRF_TIMEDOUT = 0x1000, + SAISPRF_TERMINATED = 0x2000, + SAISPRF_EXIT = 0x8000, + SAISPRF_SIGNALLED = 0x4000, +}; + +/* + * Builder is indicating he can't take the task and master should free it up + * and try another builder. + */ + +typedef struct sai_rejection { + struct lws_dll2 list; + char host_platform[65]; + char task_uuid[65]; + int ongoing; + int limit; +} sai_rejection_t; + +/* + * Master is broadcasting that builders should stop work on the given task, + * because, eg, the task was reset + */ + +typedef struct sai_cancel { + struct lws_dll2 list; + char task_uuid[65]; +} sai_cancel_t; + + +struct sai_event; + +typedef struct { + lws_dll2_t list; + lws_dll2_t pending_assign_list; + const struct sai_event *one_event; /* event we are associated with */ + char platform[96]; + char build[4096]; /* strsubst and serialized */ + char taskname[96]; + char packages[2048]; + char artifacts[256]; + char prep[4096]; /* only build is serialized */ + char cmake[4096]; /* only build is serialized */ + char builder[65 + 5]; + char event_uuid[65]; + char art_up_nonce[33]; + char art_down_nonce[33]; + char uuid[65]; + char builder_name[96]; + char cpack[128]; + + struct lwsac *ac_task_container; + + const char *master_name; /* used in offer */ + const char *repo_name; /* used in offer */ + const char *git_ref; /* used in offer */ + const char *git_hash; /* used in offer */ + const char *git_repo_url; /* used in offer */ + uint64_t last_updated; + uint64_t started; + uint64_t duration; + int state; + int uid; + + char told_ongoing; +} sai_task_t; + +typedef struct sai_event { + struct lws_dll2 list; + lws_dll2_owner_t task_owner; + char repo_name[65]; + char repo_fetchurl[96]; + char ref[65]; + char hash[65]; + char uuid[65]; + char source_ip[32]; + void *pdb; /* master only, sqlite3 */ + uint64_t created; + uint64_t last_updated; + sai_event_state_t state; + int uid; +} sai_event_t; + +typedef struct { + struct lws_dll2 list; + char task_uuid[65]; + char *log; + uint64_t timestamp; + size_t len; + int finished; + int channel; + int uid; +} sai_log_t; + +typedef struct { + struct lws_dll2 list; + + struct lws_ss_handle *ss; + void *opaque_data; + + char task_uuid[65]; + char artifact_up_nonce[33]; + char artifact_down_nonce[33]; + char blob_filename[65]; + char path[256]; /* for unlink on completion */ + void *blob; + uint64_t ofs; + uint64_t timestamp; + size_t len; + int uid; + int fd; + char sent_json; +} sai_artifact_t; + +struct sai_plat; + +/* + * One SS per unique master the builder connects to; one of these as the SS + * userdata object + * + * May be in use by multiple plats offered by same builder to same master + */ + +typedef struct sai_plat_master { + lws_dll2_t list; + + lws_dll2_owner_t rejection_list; + + const char *url; + const char *name; + + struct lws_ss_handle *ss; + void *opaque_data; + + lws_dll2_t *last_logging_nspawn; + struct sai_plat *last_logging_platform; + + int logs_in_flight; + int phase; + int refcount; + + int index; /* used to create unique build dir path */ + + uint16_t retries; +} sai_plat_master_t; + +struct sai_env { + lws_dll2_t list; + + const char *name; + const char *value; +}; + +typedef struct sai_plat_master_ref { + lws_dll2_t list; + sai_plat_master_t *spm; +} sai_plat_master_ref_t; + +/* + * One of these instantiated per platform instance + * + * It lists a sai_plat_master per master / ss that can use the platform + * + * It contains an nspawn for each platform builder instance + * + * It's also used as the object on master side that represents a builder / + * platform instance and status. + */ + +typedef struct sai_plat { + lws_dll2_t sai_plat_list; + + lws_dll2_owner_t masters; /* list of sai_plat_master_ref_t */ + lws_dll2_owner_t chunk_cache; + + const char *name; + const char *platform; + + struct lejp_ctx ctx; + lws_dll2_owner_t nspawn_owner; + struct lwsac *deserialization_ac; + + struct lws *wsi; /* master side only */ + + lws_dll2_owner_t env_head; + + int instances; + int ongoing; + + int index; /* used to create unique build dir path */ +} sai_plat_t; + +typedef struct sai_plat_owner { + lws_dll2_owner_t plat_owner; +} sai_plat_owner_t; + +typedef struct sai_repo { + char *name; + char *fetch_url; + char *notification_key; +} sai_repo_t; + +typedef enum { + SAILOGA_STARTED, +} sai_log_action_t; + +extern const lws_struct_map_t + lsm_schema_json_map_task[], + lsm_schema_sq3_map_task[], + lsm_schema_sq3_map_event[], + lsm_schema_json_map_log[], + lsm_schema_sq3_map_log[], + lsm_schema_json_map_artifact[1], + lsm_schema_sq3_map_artifact[1], + lsm_schema_map_ta[1], + lsm_schema_map_plat_simple[1], + lsm_event[9], + lsm_task[21], + lsm_log[7], + lsm_artifact[8], + lsm_plat_list[1], + lsm_task_rej[4], + lsm_task_cancel[1], + lsm_schema_json_map_can[1], + lsm_schema_json_map_task[1], + lsm_schema_json_map_event[1] +; + +extern const lws_ss_info_t ssi_said_logproxy; +extern struct lws_ss_handle *ssh[3]; + +typedef void (*saicom_drain_cb)(void *opaque); + +int +saicom_lp_add(struct lws_ss_handle *h, const char *buf, size_t len); + +struct lws_ss_handle * +saicom_lp_ss_from_env(struct lws_context *context, const char *env_name); + +int +saicom_lp_callback_on_drain(saicom_drain_cb cb, void *opaque); diff --git a/src/common/ss-client-logproxy.c b/src/common/ss-client-logproxy.c new file mode 100644 index 0000000..df42962 --- /dev/null +++ b/src/common/ss-client-logproxy.c @@ -0,0 +1,206 @@ +/* + * sai-device ss logproxy link implementation + * + * Copyright (C) 2019 - 2020 Andy Green + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation: + * version 2.1 of the License. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + * MA 02110-1301 USA + */ + +#if defined(WIN32) /* complaints about getenv */ +#define _CRT_SECURE_NO_WARNINGS +#define read _read +#define close _close +#endif + +#include +#include +#include + +#include "../common/include/private.h" + +typedef struct said_log { + lws_dll2_t list; + size_t pos; + size_t len; +} said_log_t; + +typedef struct said_logproxy { + lws_dll2_owner_t logs; + struct lws_ss_handle *ss; + void *opaque_data; +} said_logproxy_t; + +static saicom_drain_cb cb; +static void *opaque; + +struct lws_ss_handle * +saicom_lp_ss_from_env(struct lws_context *context, const char *env_name) +{ + char *e = getenv(env_name); + struct lws_ss_handle *h; + + if (!e) { + lwsl_notice("%s: no env var %s\n", __func__, env_name); + return NULL; + } + + lwsl_info("%s: connecting to %s\n", __func__, e); + + if (lws_ss_create(context, 0, &ssi_said_logproxy, 0, &h, NULL, NULL)) { + lwsl_err("%s: failed to create SS for %s\n", __func__, env_name); + return NULL; + } + + lws_ss_set_metadata(h, "sockpath", e, strlen(e)); + lws_ss_client_connect(h); + + return h; +} + +int +saicom_lp_add(struct lws_ss_handle *h, const char *buf, size_t len) +{ + said_logproxy_t *lp = lws_ss_to_user_object(h); + said_log_t *log = malloc(sizeof(*log) + len); + + if (!log) + return 1; + memset(log, 0, sizeof(*log)); + memcpy(&log[1], buf, len); + + log->len = len; + lws_dll2_add_tail(&log->list, &lp->logs); + lws_ss_request_tx(h); + + return 0; +} + +void +check_drained(void) +{ + int n, m = 0; + + if (!cb) + return; + + for (n = 0; n < (int)LWS_ARRAY_SIZE(ssh); n++) { + + said_logproxy_t *lp = lws_ss_to_user_object(ssh[n]); + + if (lp->logs.head) + m = 1; + } + + if (!m) { + cb(opaque); + cb = NULL; + } +} + +int +saicom_lp_callback_on_drain(saicom_drain_cb _cb, void *_opaque) +{ + cb = _cb; + opaque = _opaque; + + check_drained(); + + return 0; +} + +static int +saicom_lp_rx(void *userobj, const uint8_t *buf, size_t len, int flags) +{ + return 0; +} + +static int +saicom_lp_tx(void *userobj, lws_ss_tx_ordinal_t ord, uint8_t *buf, size_t *len, + int *flags) +{ + said_logproxy_t *lp = (said_logproxy_t *)userobj; + said_log_t *log = lws_container_of(lp->logs.head, said_log_t, list); + + if (!lp->logs.head) + return 1; /* nothing to send */ + + if (!log->pos) + *flags = LWSSS_FLAG_SOM; + + if (*len > log->len - log->pos) + *len = log->len - log-> pos; + + memcpy(buf, ((char *)&log[1]) + log->pos, *len); + log->pos += *len; + if (log->pos == log->len) { + lws_dll2_remove(&log->list); + free(log); + *flags |= LWSSS_FLAG_EOM; + } + + if (lp->logs.head) { + lws_ss_request_tx(lp->ss); + + return 0; + } + + check_drained(); + + return 0; +} + +static int +saicom_lp_state(void *userobj, void *sh, lws_ss_constate_t state, + lws_ss_tx_ordinal_t ack) +{ + said_logproxy_t *lp = (said_logproxy_t *)userobj; + + lwsl_info("%s: %s, ord 0x%x\n", __func__, lws_ss_state_name(state), + (unsigned int)ack); + + switch (state) { + case LWSSSCS_DESTROYING: + break; + + case LWSSSCS_CONNECTED: + lws_ss_request_tx(lp->ss); + break; + + case LWSSSCS_DISCONNECTED: + break; + + case LWSSSCS_ALL_RETRIES_FAILED: + break; + + case LWSSSCS_QOS_ACK_REMOTE: + break; + + default: + break; + } + + return 0; +} + +const lws_ss_info_t ssi_said_logproxy = { + .handle_offset = offsetof(said_logproxy_t, ss), + .opaque_user_data_offset = offsetof(said_logproxy_t, opaque_data), + .rx = saicom_lp_rx, + .tx = saicom_lp_tx, + .state = saicom_lp_state, + .user_alloc = sizeof(said_logproxy_t), + .streamtype = "logproxy" +}; diff --git a/src/common/struct-metadata.c b/src/common/struct-metadata.c new file mode 100644 index 0000000..8bfbdde --- /dev/null +++ b/src/common/struct-metadata.c @@ -0,0 +1,192 @@ +/* + * Sai master - ./src/master/struct-metadata.c + * + * Copyright (C) 2019 - 2020 Andy Green + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation: + * version 2.1 of the License. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + * MA 02110-1301 USA + * + * lws_struct metadata for structs common to builder and master + */ + +static const lws_struct_map_t lsm_plat[] = { + LSM_STRING_PTR (sai_plat_t, name, "name"), + LSM_UNSIGNED (sai_plat_t, ongoing, "ongoing"), + LSM_UNSIGNED (sai_plat_t, instances, "instances"), + LSM_STRING_PTR (sai_plat_t, platform, "platform"), +}; + +const lws_struct_map_t lsm_schema_map_plat_simple[] = { + LSM_SCHEMA (sai_plat_t, NULL, lsm_plat, "com-warmcat-sai-ba"), +}; + +const lws_struct_map_t lsm_plat_list[] = { + LSM_LIST (sai_plat_owner_t, plat_owner, sai_plat_t, + sai_plat_list, NULL, lsm_plat, "platforms"), +}; + +const lws_struct_map_t lsm_schema_map_plat[] = { + LSM_SCHEMA_DLL2 (sai_plat_owner_t, plat_owner, NULL, lsm_plat_list, + "com-warmcat-sai-ba"), +}; + +const lws_struct_map_t lsm_event[] = { + LSM_UNSIGNED (sai_event_t, uid, "uid"), + LSM_CARRAY (sai_event_t, repo_name, "repo_name"), + LSM_CARRAY (sai_event_t, ref, "ref"), + LSM_CARRAY (sai_event_t, hash, "hash"), + LSM_CARRAY (sai_event_t, uuid, "uuid"), + LSM_CARRAY (sai_event_t, source_ip, "source_ip"), + LSM_UNSIGNED (sai_event_t, created, "created"), + LSM_UNSIGNED (sai_event_t, state, "state"), + LSM_UNSIGNED (sai_event_t, last_updated, "last_updated"), +}; + +const lws_struct_map_t lsm_schema_json_map_event[] = { + LSM_SCHEMA_DLL2 (sai_event_t, list, NULL, lsm_event, + "com.warmcat.sai.events"), +}; + +const lws_struct_map_t lsm_schema_sq3_map_event[] = { + LSM_SCHEMA_DLL2 (sai_event_t, list, NULL, lsm_event, "events"), +}; + +/* + * tasks are bound to an event, many tasks may be generated from one event. + * + * platform string subst is already done by the time we elaborate the saifile + * into tasks + */ + +const lws_struct_map_t lsm_task[] = { + LSM_UNSIGNED (sai_task_t, uid, "uid"), + LSM_UNSIGNED (sai_task_t, state, "state"), + LSM_UNSIGNED (sai_task_t, last_updated, "last_updated"), + LSM_UNSIGNED (sai_task_t, started, "started"), + LSM_UNSIGNED (sai_task_t, duration, "duration"), + LSM_CARRAY (sai_task_t, platform, "platform"), + LSM_CARRAY (sai_task_t, build, "build"), + LSM_CARRAY (sai_task_t, taskname, "taskname"), + LSM_CARRAY (sai_task_t, packages, "packages"), + LSM_CARRAY (sai_task_t, builder, "builder"), + LSM_CARRAY (sai_task_t, artifacts, "artifacts"), + LSM_CARRAY (sai_task_t, art_up_nonce, "art_up_nonce"), + LSM_CARRAY (sai_task_t, art_down_nonce, "art_down_nonce"), + LSM_CARRAY (sai_task_t, event_uuid, "event_uuid"), + LSM_CARRAY (sai_task_t, uuid, "uuid"), + LSM_CARRAY (sai_task_t, builder_name, "builder_name"), + LSM_STRING_PTR (sai_task_t, master_name, "master_name"), + LSM_STRING_PTR (sai_task_t, repo_name, "repo_name"), + LSM_STRING_PTR (sai_task_t, git_ref, "git_ref"), + LSM_STRING_PTR (sai_task_t, git_hash, "git_hash"), + LSM_STRING_PTR (sai_task_t, git_repo_url, "git_repo_url"), +}; + +const lws_struct_map_t lsm_schema_json_map_task[] = { + LSM_SCHEMA_DLL2 (sai_task_t, list, NULL, lsm_task, + "com.warmcat.sai.tasks"), +}; + +const lws_struct_map_t lsm_schema_sq3_map_task[] = { + LSM_SCHEMA_DLL2 (sai_task_t, list, NULL, lsm_task, "tasks"), +}; + +/* builder -> master */ + +const lws_struct_map_t lsm_task_rej[] = { + LSM_CARRAY (sai_rejection_t, host_platform, "host_platform"), + LSM_CARRAY (sai_rejection_t, task_uuid, "task_uuid"), + LSM_UNSIGNED (sai_rejection_t, ongoing, "ongoing"), + LSM_UNSIGNED (sai_rejection_t, limit, "limit"), +}; + +const lws_struct_map_t lsm_schema_json_task_rej[] = { + LSM_SCHEMA (sai_event_t, NULL, lsm_task_rej, + "com.warmcat.sai.taskrej") +}; + +/* master -> builder */ + +const lws_struct_map_t lsm_task_cancel[] = { + LSM_CARRAY (sai_cancel_t, task_uuid, "task_uuid"), +}; + +const lws_struct_map_t lsm_schema_json_map_can[] = { + LSM_SCHEMA (sai_cancel_t, NULL, lsm_task_cancel, + "com.warmcat.sai.taskcan") +}; + +/* + * logs are bound to a task... many log entries will typically belong to one + * task, one is created each time the builder reads something on stdout, stderr + * or an auxiliary logging channel. + * + * This requires more storage than squashing it into a simple string, but allows + * individual timestamping and log channel source for each part to be shown in + * the UI. + */ + +const lws_struct_map_t lsm_log[] = { + LSM_UNSIGNED (sai_log_t, uid, "uid"), + LSM_UNSIGNED (sai_log_t, len, "len"), + LSM_UNSIGNED (sai_log_t, timestamp, "timestamp"), + LSM_UNSIGNED (sai_log_t, channel, "channel"), + LSM_UNSIGNED (sai_log_t, finished, "finished"), + LSM_CARRAY (sai_log_t, task_uuid, "task_uuid"), + LSM_STRING_PTR (sai_log_t, log, "log"), +}; + +const lws_struct_map_t lsm_schema_json_map_log[] = { + LSM_SCHEMA_DLL2 (sai_log_t, list, NULL, lsm_log, "com-warmcat-sai-logs"), +}; + +const lws_struct_map_t lsm_schema_sq3_map_log[] = { + LSM_SCHEMA_DLL2 (sai_log_t, list, NULL, lsm_log, "logs"), +}; + +/* + * Artifacts live in their own table in the event-specific db, and refer back to + * a task uuid. The build decides how many artifacts exist for a task (the + * saifile may give globs like "sai-*.rpm"), the exact filenames and lengths, + * and uploads them after the task completes. + * + * Notice the LSM_BLOB_PTR is just there to ensure the column exists in the + * table schema. The bulk data for that is not serialized and deserialized + * as part of lws_struct like the other types, since its size is unbounded and + * it may not make sense to base64 encode it inside JSON or even take it all + * out of the database in one hit at all. + */ + +const lws_struct_map_t lsm_artifact[] = { + LSM_UNSIGNED (sai_artifact_t, uid, "uid"), + LSM_CARRAY (sai_artifact_t, task_uuid, "task_uuid"), + LSM_CARRAY (sai_artifact_t, blob_filename, "blob_filename"), + /* created master-side, sent back with valid upload, checked at master */ + LSM_CARRAY (sai_artifact_t, artifact_up_nonce, "artifact_up_nonce"), + /* created master-side (not sent to builder), used in artifact links we generate */ + LSM_CARRAY (sai_artifact_t, artifact_down_nonce, "artifact_down_nonce"), + LSM_BLOB_PTR (sai_artifact_t, blob, "blob"), + LSM_UNSIGNED (sai_artifact_t, timestamp, "timestamp"), + LSM_UNSIGNED (sai_artifact_t, len, "len"), +}; + +const lws_struct_map_t lsm_schema_json_map_artifact[] = { + LSM_SCHEMA_DLL2 (sai_artifact_t, list, NULL, lsm_artifact, + "com-warmcat-sai-artifact"), +}; + +const lws_struct_map_t lsm_schema_sq3_map_artifact[] = { + LSM_SCHEMA_DLL2 (sai_artifact_t, list, NULL, lsm_artifact, "artifacts"), +}; diff --git a/src/device/CMakeLists.txt b/src/device/CMakeLists.txt new file mode 100644 index 0000000..fd2391e --- /dev/null +++ b/src/device/CMakeLists.txt @@ -0,0 +1,44 @@ +set(CPACK_DEBIAN_BUILDER_PACKAGE_NAME "sai-device") + +set(SRCS + d-sai.c + d-conf.c + ../common/ss-client-logproxy.c +) + +set(requirements 1) +require_lws_config(LWS_WITH_CLIENT 1 requirements) +require_lws_config(LWS_WITH_SPAWN 1 requirements) +require_lws_config(LWS_WITH_STRUCT_JSON 1 requirements) +require_lws_config(LWS_WITH_SECURE_STREAMS 1 requirements) +require_lws_config(LWS_WITH_DIR 1 requirements) + +if (requirements) + + add_executable("sai-device" ${SRCS}) + if (APPLE) + set_property(TARGET sai-device PROPERTY MACOSX_RPATH YES) + endif() + + target_link_libraries("sai-device" ${SAI_LWS_LIB_PATH}) + + if (LWS_OPENSSL_LIBRARIES) + target_link_libraries("sai-device" ${LWS_OPENSSL_LIBRARIES}) + endif() + + if (SAI_EXT_PTHREAD_LIBRARIES) + target_link_libraries("sai-device" ${SAI_EXT_PTHREAD_LIBRARIES}) + endif() + if (HAS_LIBCAP) + target_link_libraries("sai-device" ${CAP_LIB_PATH}) + endif() + + if (MSVC OR WIN32) + target_link_libraries("sai-device" ws2_32.lib userenv.lib psapi.lib iphlpapi.lib) + endif() + + install(TARGETS "sai-device" + RUNTIME DESTINATION "${BIN_DIR}" COMPONENT device) + +endif(requirements) +include(CPack) diff --git a/src/device/d-conf.c b/src/device/d-conf.c new file mode 100644 index 0000000..078a25c --- /dev/null +++ b/src/device/d-conf.c @@ -0,0 +1,246 @@ +/* + * sai-device conf.c + * + * Copyright (C) 2019 - 2020 Andy Green + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation: + * version 2.1 of the License. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + * MA 02110-1301 USA + */ + +#if defined(WIN32) /* complaints about getenv */ +#define _CRT_SECURE_NO_WARNINGS +#define read _read +#define close _close +#endif + +#include +#include +#include +#include +#include + +#include "d-private.h" + +/* global part */ + +static const char * const paths_global[] = { + "home", + + "devices[].name", + "devices[].type", + "devices[].compatible", + "devices[].description", + "devices[].ttys[].name", + "devices[].ttys[].tty", + "devices[].ttys[].rate", + "devices[].ttys[].initial_monitor", + "devices[].ttys[]", + "devices[]", +}; + +enum enum_paths_global { + LEJPM_HOME, + + LEJPM_DEVICES__NAME, + LEJPM_DEVICES__TYPE, + LEJPM_DEVICES__COMPATIBLE, + LEJPM_DEVICES__DESCRIPTION, + LEJPM_DEVICES__TTYS__NAME, + LEJPM_DEVICES__TTYS__TTY, + LEJPM_DEVICES__TTYS__RATE, + LEJPM_DEVICES__TTYS__INITIAL_MONITOR, + LEJPM_DEVICES__TTYS, + LEJPM_DEVICES, + +}; + + +struct jpargs { + sai_devices_t *devices; + + sai_device_t *dev; + sai_tty_t *tty; +}; + + +static signed char +said_conf_global_cb(struct lejp_ctx *ctx, char reason) +{ + struct jpargs *a = (struct jpargs *)ctx->user; + const char **pp; +#if 0 + int n; + + lwsl_notice("%s: reason: %d, path: %s, match %d\n", __func__, + reason, ctx->path, ctx->path_match); + for (n = 0; n < ctx->wildcount; n++) + lwsl_notice(" %d\n", ctx->wild[n]); +#endif + + if (reason == LEJPCB_OBJECT_START && + ctx->path_match - 1 == LEJPM_DEVICES) { + + /* + * We're starting on defining a new device + */ + + a->dev = lwsac_use_zero(&a->devices->conf_head, sizeof(*a->dev), 512); + if (!a->dev) + return -1; + lws_dll2_add_tail(&a->dev->list, &a->devices->devices_owner); + a->tty = NULL; + return 0; + } + + if (reason == LEJPCB_OBJECT_START && + ctx->path_match - 1 == LEJPM_DEVICES__TTYS) { + + /* + * We're starting on defining a new tty for our device + */ + + a->tty = lwsac_use_zero(&a->devices->conf_head, sizeof(*a->tty), 512); + if (!a->tty) + return -1; + + a->tty->initial_monitor = 1; + a->tty->index = a->dev->ttys_owner.count; + lws_dll2_add_tail(&a->tty->list, &a->dev->ttys_owner); + return 0; + } + + /* we only match on the prepared path strings */ + if (!(reason & LEJP_FLAG_CB_IS_VALUE) || !ctx->path_match) + return 0; + + if (reason == LEJPCB_VAL_NUM_INT && + ctx->path_match - 1 == LEJPM_DEVICES__TTYS__RATE) { + a->tty->rate = atoi(ctx->buf); + return 0; + } + + if (reason == LEJPCB_VAL_FALSE && + ctx->path_match - 1 == LEJPM_DEVICES__TTYS__INITIAL_MONITOR) { + a->tty->initial_monitor = 0; + return 0; + } + + if (reason != LEJPCB_VAL_STR_END) + return 0; + + /* only the end part of the string, where we know the length */ + + switch (ctx->path_match - 1) { + case LEJPM_HOME: + pp = &a->devices->home; + break; + + case LEJPM_DEVICES__NAME: + pp = &a->dev->name; + break; + case LEJPM_DEVICES__TYPE: + pp = &a->dev->type; + break; + case LEJPM_DEVICES__COMPATIBLE: + pp = &a->dev->compatible; + break; + case LEJPM_DEVICES__DESCRIPTION: + pp = &a->dev->description; + break; + case LEJPM_DEVICES__TTYS__NAME: + pp = &a->tty->name; + break; + case LEJPM_DEVICES__TTYS__TTY: + pp = &a->tty->tty_path; + break; + + default: + return 0; + } + + *pp = lwsac_use(&a->devices->conf_head, ctx->npos + 1, 512); + if (!*pp) + return 1; + memcpy((char *)(*pp), ctx->buf, ctx->npos); + ((char *)(*pp))[ctx->npos] = '\0'; + + return 0; +} + +int +said_config_global(struct sai_devices *devices, const char *d) +{ + unsigned char buf[128]; + struct lejp_ctx ctx; + int n, m, fd; + struct jpargs a; + + memset(&a, 0, sizeof(a)); + a.devices = devices; + +#if defined(WIN32) + lws_snprintf((char *)buf, sizeof(buf) - 1, "%s\\conf", d); +#else + lws_snprintf((char *)buf, sizeof(buf) - 1, "%s/conf", d); +#endif + + fd = lws_open((char *)buf, O_RDONLY); + if (fd < 0) { + lwsl_err("Cannot open %s\n", (char *)buf); + return 2; + } + lwsl_info("%s: %s\n", __func__, (char *)buf); + lejp_construct(&ctx, said_conf_global_cb, &a, + paths_global, LWS_ARRAY_SIZE(paths_global)); + + do { + n = read(fd, buf, sizeof(buf)); + if (!n) + break; + + m = lejp_parse(&ctx, buf, n); + } while (m == LEJP_CONTINUE); + + close(fd); + n = ctx.line; + lejp_destruct(&ctx); + + lws_start_foreach_dll_safe(struct lws_dll2 *, p, p1, + devices->devices_owner.head) { + sai_device_t *d = lws_container_of(p, sai_device_t, list); + + lwsl_info("%s: device '%s', %s, %s, %s\n", __func__, + d->name, d->type, d->compatible, d->description); + + lws_start_foreach_dll_safe(struct lws_dll2 *, q, q1, + d->ttys_owner.head) { + sai_tty_t *t = lws_container_of(q, sai_tty_t, list); + + lwsl_info("%s: tty %s\n", __func__, + t->tty_path); + + } lws_end_foreach_dll_safe(q, q1); + + } lws_end_foreach_dll_safe(p, p1); + + return 0; +} + + +void +said_config_destroy(struct sai_devices *devices) +{ + lwsac_free(&devices->conf_head); +} diff --git a/src/device/d-private.h b/src/device/d-private.h new file mode 100644 index 0000000..232b4d7 --- /dev/null +++ b/src/device/d-private.h @@ -0,0 +1,68 @@ +/* + * Sai device definitions src/device/d-private.h + * + * Copyright (C) 2019 - 2020 Andy Green + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation: + * version 2.1 of the License. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + * MA 02110-1301 USA + */ + +#include "../common/include/private.h" + +typedef struct sai_tty { + lws_dll2_t list; + + const char *name; + const char *tty_path; + struct lws *wsi; + lws_sorted_usec_list_t sul_serial_acq; + int rate; + int initial_monitor; + int index; + +} sai_tty_t; /* owned by sai_device_t's ttys_owner */ + +typedef struct sai_device { + lws_dll2_t list; + + lws_dll2_owner_t ttys_owner; /* sai_tty_t */ + + const char * name; + const char * type; + const char * compatible; + const char * description; +} sai_device_t; /* owned by sai_devices devices_owner */ + +typedef struct sai_devices { + + lws_dll2_owner_t devices_owner; + + struct lws_context *context; + struct lws_vhost *vhost; + + struct lwsac *conf_head; + + const char *home; + +} sai_devices_t; + + +extern struct lws_protocols protocol_serial; + +int +said_config_global(struct sai_devices *devices, const char *d); + +void +said_config_destroy(struct sai_devices *devices); diff --git a/src/device/d-sai.c b/src/device/d-sai.c new file mode 100644 index 0000000..80d1b38 --- /dev/null +++ b/src/device/d-sai.c @@ -0,0 +1,434 @@ +/* + * sai-device + * + * Copyright (C) 2019 - 2020 Andy Green + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation: + * version 2.1 of the License. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + * MA 02110-1301 USA + * + * + * sai-device simply tries to acquire exclusive file locks on ttys belonging + * to a given type of device, sets up some env vars representing what it locked + * eg, + * + * SAI_DEVICE_TTY0=/dev/serial/by-path/pci-0000:03:00.3-usb-0:2:1.0-port0 + * + * and then spawns a given argument. + * + * It doesn't do any network IO or tty IO, although it holds an fd on the ttys + * as part of locking them until sai-device terminates. + */ + +#if defined(WIN32) /* complaints about getenv */ +#define _CRT_SECURE_NO_WARNINGS +#endif + +#include +#include +#include +#include + +#if defined(WIN32) +#include +#include +#include +#else +#include +#include +#endif + +#if defined(__linux__) +#include +#endif + +#include "d-private.h" + +static const char *config_dir = "/etc/sai/devices"; +static const char * _argv[9], *alias; +struct lws_ss_handle *ssh[3]; +static struct lws_context *context; +static lws_sorted_usec_list_t sul; +static lws_state_notify_link_t nl; +static int interrupted, xarg = 2; +struct sai_devices devices; +static const char *devtype; +static pid_t pid; + +static const char * const default_ss_policy = + "{" + "\"s\": [" + /* + * Unix Domain Socket connections to sai-builder logging + */ + "{\"logproxy\": {" + "\"endpoint\":" "\"+${sockpath}\"," + "\"protocol\":" "\"raw\"," + "\"metadata\": [" + "{\"sockpath\": \"\"}" + "]" + "}}" + "]}" +; + + +void sigint_handler(int sig) +{ + interrupted = 1; +} + +static void +said_check_device(lws_sorted_usec_list_t *sul) +{ + int n; + + if (pid) { + int w; + + /* + * We already locked the device and ran the child process + */ + + if (waitpid(pid, &w, WNOHANG) == pid) { + + if (WIFEXITED(w)) + exit(WEXITSTATUS(w)); + if (WIFSIGNALED(w)) + exit(1); + } + + goto again; + } + + lws_start_foreach_dll_safe(struct lws_dll2 *, p, p1, + devices.devices_owner.head) { + sai_device_t *d = lws_container_of(p, sai_device_t, list); + struct lws_tokenize ts; + + lwsl_debug("%s: device '%s', %s, %s, %s\n", __func__, + d->name, d->type, d->compatible, d->description); + + /* + * is this device compatible with what we were asked for? + */ + lws_tokenize_init(&ts, d->compatible, + LWS_TOKENIZE_F_COMMA_SEP_LIST | + LWS_TOKENIZE_F_MINUS_NONTERM); + ts.len = (int)strlen(d->compatible); + + do { + ts.e = lws_tokenize(&ts); + switch (ts.e) { + case LWS_TOKZE_TOKEN: + if (!strncmp(ts.token, devtype, ts.token_len)) + ts.e = LWS_TOKZE_ENDED; + break; + + default: /* includes ENDED found by the tokenizer itself */ + ts.e = -99; + break; + } + } while (ts.e > 0); + + if (ts.e == LWS_TOKZE_ENDED) { + char ename[1024], *p = ename; + int try = 0, locked[8], fd; + + /* + * this matches what he was asking for + */ + lws_start_foreach_dll_safe(struct lws_dll2 *, q, q1, + d->ttys_owner.head) { + sai_tty_t *t = lws_container_of(q, + sai_tty_t, list); + + if (try >= (int)LWS_ARRAY_SIZE(locked)) + break; + try++; + + fd = lws_open(t->tty_path, O_RDWR); + if (fd == -1) { + try--; + goto nope; + } + + locked[try - 1] = fd; + + /* + * We need to insist on exclusive use of it, not + * just for the obvious reason we want to see + * all the RX and not have someone else's TX + * disrupting us, but also because we can expect + * other processes to want to use this to flash + * the same device. + * + * We need to lock it for the whole time between + * wanting to flash and sai-expect using it + * opportunistically for tty IO, so we just open + * it for locking and don't do anything with it + * ourselves. + */ + + if (flock(fd, LOCK_EX | LOCK_NB) == -1) { + close(fd); + try--; + + goto nope; + } + + } lws_end_foreach_dll_safe(q, q1); /* each tty */ + + /* + * if we made it here, we locked them all and we can go + * on and set the env and spawn the payload + */ + + try = 0; + lws_start_foreach_dll_safe(struct lws_dll2 *, q, q1, + d->ttys_owner.head) { + sai_tty_t *t = lws_container_of(q, + sai_tty_t, list); + char *p1 = p; + + p += lws_snprintf(p, sizeof(ename) - + lws_ptr_diff(p, ename), + "SAI_DEVICE_TTY%d", + try); + setenv(p1, t->tty_path, 1); + + if (alias) { + p1 = strchr(alias, '='); + if (p1 && atoi(&p1[1]) == try) { + *p1 = '\0'; + setenv(alias, t->tty_path, 1); + } + } + + p++; + try++; + + } lws_end_foreach_dll_safe(q, q1); /* each tty */ + + pid = fork(); + if (pid) { + char log[128]; + n = lws_snprintf(log, sizeof(log), + "sai-device: acquired device '%s' (%s)\n", + d->name, d->type); + saicom_lp_add(ssh[0], log, n); + /* + * We'll check in the background if the child + * exited, in the meanwhile we can send any + * logs + */ + goto again; + } + + /* + * We're the child process + */ + + if (execvp(_argv[0], (char * const *)_argv)) { + lwsl_err("%s: spawn %s failed: errno %d\n", + __func__, _argv[0], errno); + return; + } + + /* unreachable */ + +nope: + while (try--) { + flock(locked[try], LOCK_UN); + close(locked[try]); + } + + } /* device matches type */ + + } lws_end_foreach_dll_safe(p, p1); /* each device */ + +again: + lws_sul_schedule(context, 0, sul, said_check_device, + 100 * LWS_US_PER_MS); +} + +static int +app_system_state_nf(lws_state_manager_t *mgr, lws_state_notify_link_t *link, + int current, int target) +{ + struct lws_context *context = lws_system_context_from_system_mgr(mgr); + char log[128]; + int n; + + switch (target) { + + case LWS_SYSTATE_OPERATIONAL: + if (current != LWS_SYSTATE_OPERATIONAL) + break; + + n = lws_snprintf(log, sizeof(log), + "sai-device: queueing for device compatible with '%s'\n", + devtype); + saicom_lp_add(ssh[0], log, n); + + lws_sul_schedule(context, 0, &sul, said_check_device, 1); + break; + } + + return 0; +} + + +static lws_state_notify_link_t * const app_notifier_list[] = { + &nl, NULL +}; + +int main(int argc, const char **argv) +{ + int logs = LLL_USER | LLL_ERR | LLL_WARN | LLL_NOTICE; + struct lws_context_creation_info info; +#if defined(WIN32) + char temp[256], stg_config_dir[256]; +#endif + const char *p; + int n; + + if (argc < 2) { + lwsl_notice("usage: %s [ENVNAME=] [args...]\n", + argv[0]); + + return 1; + } + +#if defined(__linux__) + n = prctl(PR_SET_PDEATHSIG, SIGTERM); + if (n == -1) { + perror(0); + return 1; + } + + if (getppid() == 1) + /* parent process already dead */ + return 1; +#endif + + devtype = argv[1]; + + if (strchr(argv[2], '=')) { + alias = argv[2]; + xarg++; + } + + _argv[0] = argv[xarg]; + for (n = 1; n < argc && + n + xarg < (int)LWS_ARRAY_SIZE(_argv); n++) + _argv[n] = argv[xarg + n]; + _argv[n] = NULL; + + if ((p = lws_cmdline_option(argc, argv, "-d"))) + logs = atoi(p); + + lws_set_log_level(logs, NULL); + +#if defined(WIN32) + { + PWSTR wdi = NULL; + + if (SHGetKnownFolderPath(&FOLDERID_ProgramData, + 0, NULL, &wdi) != S_OK) { + lwsl_err("%s: unable to get config dir\n", __func__); + return 1; + } + + if (WideCharToMultiByte(CP_ACP, 0, wdi, -1, temp, + sizeof(temp), 0, NULL) <= 0) { + lwsl_err("%s: problem with string encoding\n", __func__); + return 1; + } + + lws_snprintf(stg_config_dir, sizeof(stg_config_dir), + "%s\\sai\\devices\\", temp); + + config_dir = stg_config_dir; + CoTaskMemFree(wdi); + } +#endif + + /* + * Let's parse the global bits out of the config + */ + + lwsl_info("%s: config dir %s\n", __func__, config_dir); + if (said_config_global(&devices, config_dir)) { + lwsl_err("%s: global config failed\n", __func__); + + return 1; + } + + lwsl_info("Sai Devices - " + "Copyright (C) 2019-2020 Andy Green \n"); + lwsl_info(" sai-devices devtype payload ...\n"); + + + memset(&info, 0, sizeof info); + info.port = CONTEXT_PORT_NO_LISTEN; + info.pt_serv_buf_size = 32 * 1024; + info.options = LWS_SERVER_OPTION_DO_SSL_GLOBAL_INIT | + LWS_SERVER_OPTION_VALIDATE_UTF8 | + LWS_SERVER_OPTION_EXPLICIT_VHOSTS; + + signal(SIGINT, sigint_handler); + + info.pss_policies_json = default_ss_policy; + info.fd_limit_per_thread = 1 + 128 + 1; + + /* hook up our lws_system state notifier */ + + nl.name = "sai-devices"; + nl.notify_cb = app_system_state_nf; + info.register_notifier_list = app_notifier_list; + + /* create the lws context */ + + context = lws_create_context(&info); + if (!context) { + lwsl_err("lws init failed\n"); + return 1; + } + + /* ... and our vhost... */ + + if (!lws_create_vhost(context, &info)) { + lwsl_err("Failed to create tls vhost\n"); + goto bail; + } + + ssh[0] = saicom_lp_ss_from_env(context, "SAI_LOGPROXY"); + if (!ssh[0]) + goto bail; + ssh[1] = saicom_lp_ss_from_env(context, "SAI_LOGPROXY_TTY0"); + if (!ssh[1]) + goto bail; + ssh[2] = saicom_lp_ss_from_env(context, "SAI_LOGPROXY_TTY1"); + if (!ssh[2]) + goto bail; + + while (!lws_service(context, 0) && !interrupted) + ; + +bail: + lws_context_destroy(context); + said_config_destroy(&devices); + + return 0; +} diff --git a/src/expect/CMakeLists.txt b/src/expect/CMakeLists.txt new file mode 100644 index 0000000..4c146f8 --- /dev/null +++ b/src/expect/CMakeLists.txt @@ -0,0 +1,51 @@ +set(CPACK_DEBIAN_BUILDER_PACKAGE_NAME "sai-expect") + +set(SRCS + e-sai.c + e-serial.c + ../common/ss-client-logproxy.c +) + +set(requirements 1) +require_lws_config(LWS_WITH_CLIENT 1 requirements) +require_lws_config(LWS_WITH_SPAWN 1 requirements) +require_lws_config(LWS_WITH_STRUCT_JSON 1 requirements) +require_lws_config(LWS_WITH_SECURE_STREAMS 1 requirements) +require_lws_config(LWS_WITH_DIR 1 requirements) + +if (requirements AND NOT MSVC) + + CHECK_C_SOURCE_COMPILES("#include \nint + main(void) {\n#if defined(LWS_HAVE_LIBCAP)\n return + 0;\n#else\n fail;\n#endif\n return 0;\n}\n" HAS_LIBCAP) + if (HAS_LIBCAP) + find_library(CAP_LIB_PATH NAMES "cap") + endif() + + add_executable("sai-expect" ${SRCS}) + if (APPLE) + set_property(TARGET sai-expect PROPERTY MACOSX_RPATH YES) + endif() + + target_link_libraries("sai-expect" ${SAI_LWS_LIB_PATH}) + + if (LWS_OPENSSL_LIBRARIES) + target_link_libraries("sai-expect" ${LWS_OPENSSL_LIBRARIES}) + endif() + + if (SAI_EXT_PTHREAD_LIBRARIES AND NOT MSVC) + target_link_libraries("sai-expect" ${SAI_EXT_PTHREAD_LIBRARIES}) + endif() + if (HAS_LIBCAP) + target_link_libraries("sai-expect" ${CAP_LIB_PATH}) + endif() + + if (MSVC OR WIN32) + target_link_libraries("sai-expect" ws2_32.lib userenv.lib psapi.lib iphlpapi.lib) + endif() + + install(TARGETS "sai-expect" + RUNTIME DESTINATION "${BIN_DIR}" COMPONENT expect) + +endif(requirements AND NOT MSVC) +include(CPack) diff --git a/src/expect/e-private.h b/src/expect/e-private.h new file mode 100644 index 0000000..1d1c92d --- /dev/null +++ b/src/expect/e-private.h @@ -0,0 +1,28 @@ +/* + * Sai expect definitions src/expect/e-private.h + * + * Copyright (C) 2019 - 2020 Andy Green + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation: + * version 2.1 of the License. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + * MA 02110-1301 USA + */ + +#include "../common/include/private.h" + +extern struct lws_protocols protocol_serial; +extern int interrupted, bad; + +struct lws * +sai_serial_try_open(struct lws_vhost *vh, const char *devpath, int rate, int i); diff --git a/src/expect/e-sai.c b/src/expect/e-sai.c new file mode 100644 index 0000000..56585a9 --- /dev/null +++ b/src/expect/e-sai.c @@ -0,0 +1,179 @@ +/* + * sai-expect + * + * Copyright (C) 2019 - 2020 Andy Green + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation: + * version 2.1 of the License. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + * MA 02110-1301 USA + * + * sai-builder : exports logproxy unix domain paths in SAI_LOGPROXY_TTY0/1 + * sai-device : exports raw tty node path in SAI_DEVICE_TTY0/1 + * sai-expect : proxies SAI_DEVICE_TTYx to SAI_LOGPROXY_TTYx, performs TX, + * waits for and assesses RX + * + * Sai-expect opportunistically opens ttys exported by sai-device in the + * environment, proxies logs to sai-builder (which forwards them to the sai- + * master that is storing them with the task), and parses the received tty + * data and emits strings as requested by its arguments. + */ + +#include +#include +#include + +#include "e-private.h" + +struct lws_ss_handle *ssh[3]; +static lws_state_notify_link_t nl; +static const char *serpath[2]; +static struct lws_vhost *vh; +int interrupted, bad = 1; + +static const char * const default_ss_policy = + "{" + "\"s\": [" + /* + * Unix Domain Socket connections to sai-builder logging + */ + "{\"logproxy\": {" + "\"endpoint\":" "\"+${sockpath}\"," + "\"protocol\":" "\"raw\"," + "\"metadata\": [" + "{\"sockpath\": \"\"}" + "]" + "}}" + "]}" +; + +static const struct lws_protocols *pprotocols[] = { + &protocol_serial, + NULL +}; + +static lws_state_notify_link_t * const app_notifier_list[] = { + &nl, NULL +}; + +static int +app_system_state_nf(lws_state_manager_t *mgr, lws_state_notify_link_t *link, + int current, int target) +{ + struct lws_context *context = lws_system_context_from_system_mgr(mgr); + char log[128]; + int n; + + switch (target) { + + case LWS_SYSTATE_OPERATIONAL: + if (current != LWS_SYSTATE_OPERATIONAL) + break; + + ssh[0] = saicom_lp_ss_from_env(context, "SAI_LOGPROXY"); + if (!ssh[0]) + goto bail; + ssh[1] = saicom_lp_ss_from_env(context, "SAI_LOGPROXY_TTY0"); + if (!ssh[1]) + goto bail; + ssh[2] = saicom_lp_ss_from_env(context, "SAI_LOGPROXY_TTY1"); + if (!ssh[2]) + goto bail; + + if (!sai_serial_try_open(vh, serpath[0], 115200, 0)) + goto bail; + if (serpath[1]) + if (!sai_serial_try_open(vh, serpath[1], 115200, 1)) + goto bail; + + n = lws_snprintf(log, sizeof(log), "sai-expect: starting\n"); + saicom_lp_add(ssh[0], log, n); + break; + } + + return 0; + +bail: + return 1; +} + +void sigint_handler(int sig) +{ + interrupted = 1; +} + +int main(int argc, const char **argv) +{ + int logs = LLL_USER | LLL_ERR | LLL_WARN | LLL_NOTICE; + struct lws_context_creation_info info; + struct lws_context *context; + const char *p; + + serpath[0] = getenv("SAI_DEVICE_TTY0"); + serpath[1] = getenv("SAI_DEVICE_TTY1"); + + if (!serpath[0]) { + lwsl_err("%s: need at least SAI_DEVICE_TTY0\n", __func__); + return 1; + } + + if ((p = lws_cmdline_option(argc, argv, "-d"))) + logs = atoi(p); + + lws_set_log_level(logs, NULL); + + lwsl_user("Sai Expect - " + "Copyright (C) 2019-2020 Andy Green \n"); + + memset(&info, 0, sizeof info); + info.port = CONTEXT_PORT_NO_LISTEN; + info.pprotocols = pprotocols; + + info.pss_policies_json = default_ss_policy; + info.pt_serv_buf_size = 8 * 1024; + info.options = LWS_SERVER_OPTION_VALIDATE_UTF8 | + LWS_SERVER_OPTION_EXPLICIT_VHOSTS; + + signal(SIGINT, sigint_handler); + info.fd_limit_per_thread = 1 + 8 + 1; + + /* hook up our lws_system state notifier */ + + nl.name = "sai-devices"; + nl.notify_cb = app_system_state_nf; + info.register_notifier_list = app_notifier_list; + + /* create the lws context */ + + context = lws_create_context(&info); + if (!context) { + lwsl_err("lws init failed\n"); + return 1; + } + + /* ... and our vhost... */ + + vh = lws_create_vhost(context, &info); + if (!vh) { + lwsl_err("%s: Failed to create vhost\n", __func__); + goto bail1; + } + + while (!lws_service(context, 0) && !interrupted) + ; + +bail1: + lws_context_destroy(context); + + return bad; +} diff --git a/src/expect/e-serial.c b/src/expect/e-serial.c new file mode 100644 index 0000000..2de2443 --- /dev/null +++ b/src/expect/e-serial.c @@ -0,0 +1,246 @@ +/* + * sai-device tty monitoring + * + * Copyright (C) 2019 - 2020 Andy Green + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation: + * version 2.1 of the License. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + * MA 02110-1301 USA + * + * Logging / parsing of device ttys + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "e-private.h" + +static const char *pass = "Completed: PASS"; + +struct pss { + struct lws_context *context; + lws_sorted_usec_list_t sul; + char collation[128]; + lws_usec_t earliest; + int pos; + int i; + + int pass_match; + char draining; + char match; +}; + +struct lws * +sai_serial_try_open(struct lws_vhost *vh, const char *devpath, int _rate, int i) +{ + lws_sock_file_fd_type u; + struct termios tio; + struct lws *wsi; + int fd, rate; + + switch (_rate) { + case 115200: + rate = B115200; + break; + default: + lwsl_err("%s: unknown rate %d\n", __func__, _rate); + + return NULL; + } + + /* enforce suitable tty state */ + + memset(&tio, 0, sizeof tio); + tio.c_cflag = CS8 | CREAD | CLOCAL; +// tio.c_cc[VMIN] = 1; +// tio.c_cc[VTIME] = 5; + + fd = lws_open(devpath, O_RDWR); + if (fd == -1) { + /* + * Even when it's locked, we ought to be able to open it + */ + lwsl_err("%s: Unable to open %s\n", __func__, devpath); + + return NULL; + } + + cfsetospeed(&tio, rate); + cfsetispeed(&tio, rate); + + tcsetattr(fd, TCSANOW, &tio); + tcflush(fd, TCIOFLUSH); + + u.filefd = (lws_filefd_type)(long long)fd; + wsi = lws_adopt_descriptor_vhost(vh, LWS_ADOPT_RAW_FILE_DESC, u, + "sai-serial", NULL); + if (!wsi) { + lwsl_err("%s: Failed to adopt fifo descriptor\n", __func__); + close(fd); + + return NULL; + } + + lws_set_opaque_user_data(wsi, (void *)(intptr_t)i); + + return wsi; +} + +static void +finished_cb(void *opaque) +{ + interrupted = 1; + bad = 0; +} + +static void +spill(struct pss *pss) +{ + saicom_lp_add(ssh[pss->i + 1], pss->collation, pss->pos); + pss->pos = 0; + pss->earliest = 0; + lws_sul_cancel(&pss->sul); + + /* + * We have seen the pass string + */ + if (pss->match) { + saicom_lp_add(ssh[0], "sai-expect: completed\n", 22); + pss->match = 0; + pss->draining = 1; + saicom_lp_callback_on_drain(finished_cb, NULL); + } +} + +static void +sul_cb(lws_sorted_usec_list_t *sul) +{ + struct pss *pss = lws_container_of(sul, struct pss, sul); + + spill(pss); +} + +static int +callback_serial(struct lws *wsi, enum lws_callback_reasons reason, + void *user, void *in, size_t len) +{ + struct pss *pss = (struct pss *)user; + lws_usec_t now = lws_now_usecs(); + uint8_t buf[128]; + int n, m; + + switch (reason) { + + case LWS_CALLBACK_RAW_ADOPT_FILE: + pss->i = (int)(intptr_t)lws_get_opaque_user_data(wsi); + pss->context = lws_get_context(wsi); + break; + + case LWS_CALLBACK_RAW_RX_FILE: + /* + * Typically we can read this much faster than it gets new data, + * 115200 is only 12KB/sec. If we don't do something about + * collation locally, we will end up generating and sending a + * huge number of discrete log fragments just a few chars long + * each. + * + * Let's collate them up to 128 bytes, with a 150ms limit to + * holding pieces + */ + n = read((int)(intptr_t)lws_get_socket_fd(wsi), buf, sizeof(buf)); + if (n < 1) { + lwsl_debug("%s: read on stdwsi failed\n", __func__); + return -1; + } + + if (pss->draining) + break; + + for (m = 0; m < n; m++) { + pss->collation[pss->pos++] = buf[m]; + + if (buf[m] == pass[pss->pass_match]) { + pss->pass_match++; + if (pss->pass_match == (int)strlen(pass)) { + /* + * We have seen the EOT match string, + * start that process when we spill + * this line so we can log it + */ + pss->match = 1; + pss->pass_match = 0; + } + } else + pss->pass_match = 0; + + /* + * Let's spill on line boundaries too, if the line + * was long enough to take 2ms (24 x 115Kbps octets) to + * collect + */ + if ((buf[m] == '\n' && pss->pos > 1 && pss->earliest && + (now - pss->earliest) > 2 * LWS_US_PER_MS) || + pss->pos == sizeof(pss->collation)) + /* + * spill due to fill collation buffer + */ + spill(pss); + } + + if (pss->pos && !pss->earliest) + pss->earliest = now; + + if (now - pss->earliest > 150 * LWS_US_PER_MS) + /* + * Spill because the earliest is stale + */ + spill(pss); + + else + /* + * if we are holding on to something, and nothing else + * happens, spill 150ms after earliest + */ + if (pss->pos) + lws_sul_schedule(pss->context, 0, &pss->sul, + sul_cb, (150 * LWS_US_PER_MS) - + (now - pss->earliest)); + break; + + case LWS_CALLBACK_RAW_CLOSE_FILE: + break; + + case LWS_CALLBACK_RAW_WRITEABLE_FILE: +// lwsl_notice("LWS_CALLBACK_RAW_WRITEABLE_FILE\n"); +// if (lws_write(wsi, (uint8_t *)"hello-this-is-written-every-couple-of-seconds\r\n", 47, LWS_WRITE_RAW) != 47) +// return -1; + break; + + default: + break; + } + + return 0; +} + +struct lws_protocols protocol_serial = { + "sai-serial", callback_serial, sizeof(struct pss), 0, +}; diff --git a/src/master/CMakeLists.txt b/src/master/CMakeLists.txt new file mode 100644 index 0000000..5c41dc7 --- /dev/null +++ b/src/master/CMakeLists.txt @@ -0,0 +1,141 @@ +set(SUB "sai-master") + +set(CPACK_DEBIAN_MASTER_PACKAGE_NAME ${SUB}) + +set(SRCS + m-sai.c + m-conf.c + m-notification.c + m-comms.c + m-ws-builder.c + m-task.c + m-central.c + m-artifact.c + m-overview.c + m-ws-browser.c +) + +set(requirements 1) +require_lws_config(LWS_WITH_STRUCT_SQLITE3 1 requirements) +require_lws_config(LWS_WITH_SERVER 1 requirements) +require_lws_config(LWS_WITH_GENCRYPTO 1 requirements) +require_lws_config(LWS_WITH_UNIX_SOCK 1 requirements) + +if (requirements) + add_executable(${SUB} ${SRCS}) + if (APPLE) + set_property(TARGET sai-master PROPERTY MACOSX_RPATH YES) + endif() + + # + # sqlite3 paths (master) + # + if (SAI_MASTER) + find_path( SQLITE3_INC_PATH NAMES "sqlite3.h") + find_library(SQLITE3_LIB_PATH NAMES "sqlite3") + + if (SQLITE3_INC_PATH AND SQLITE3_LIB_PATH) + include_directories(BEFORE "${SQLITE3_INC_PATH}") + else() + message(FATAL_ERROR " Unable to find sqlite3") + endif() + + target_link_libraries(${SUB} ${SQLITE3_LIB_PATH}) + endif() + + include_directories(BEFORE "${SAI_LWS_INC_PATH}") + + CHECK_C_SOURCE_COMPILES("#include \nint + main(void) {\n#if defined(LWS_HAVE_LIBCAP)\n return + 0;\n#else\n fail;\n#endif\n return 0;\n}\n" HAS_LIBCAP) + if (HAS_LIBCAP) + find_library(CAP_LIB_PATH NAMES "cap") + endif() + + target_link_libraries(${SUB} ${SAI_LWS_LIB_PATH}) + + if (LWS_OPENSSL_LIBRARIES) + target_link_libraries(${SUB} ${LWS_OPENSSL_LIBRARIES}) + endif() + + if (SAI_EXT_PTHREAD_LIBRARIES) + target_link_libraries(${SUB} ${SAI_EXT_PTHREAD_LIBRARIES}) + endif() + + if (HAS_LIBCAP) + target_link_libraries(${SUB} ${CAP_LIB_PATH}) + endif() + + if (MSVC) + target_link_libraries(${SUB} ws2_32.lib userenv.lib psapi.lib iphlpapi.lib) + endif() + + install(TARGETS ${SUB} + RUNTIME DESTINATION "${BIN_DIR}" COMPONENT master) + install( + FILES ../../assets/index.html + ../../assets/OpenSans-Light.ttf + ../../assets/rebuild.png + ../../assets/delete.png + ../../assets/passed.svg + ../../assets/failed.svg + ../../assets/sai.css + ../../assets/sai.js + ../../assets/sai.svg + ../../assets/sai-icon.svg + ../../assets/builder.png + ../../assets/favicon.ico + ../../assets/strict-csp.svg + ../../assets/lws-common.js + ../../assets/gs/lwsgs.js + ../../assets/gs/lwsgs.css + ../../assets/arch-aarch64-a72-bcm2711-rpi4.svg + ../../assets/arch-aarch64.svg + ../../assets/arch-arm32-m4-mt7697-usi.svg + ../../assets/arch-arm32.svg + ../../assets/arch-freertos.svg + ../../assets/arch-riscv64-virt.svg + ../../assets/arch-riscv.svg + ../../assets/arch-x86_64-amd.svg + ../../assets/arch-x86_64-intel-i3.svg + ../../assets/arch-x86_64-intel.svg + ../../assets/arch-xl6-esp32.svg + ../../assets/artifact.svg + ../../assets/builder.svg + ../../assets/decal-2.svg + ../../assets/decal-3.svg + ../../assets/decal-4.svg + ../../assets/decal-5.svg + ../../assets/decal-6.svg + ../../assets/freertos-espidf.svg + ../../assets/freertos-linkit.svg + ../../assets/git.svg + ../../assets/jsplease.svg + ../../assets/linux-android.svg + ../../assets/linux-centos-8.svg + ../../assets/linux-fedora-32-riscv.svg + ../../assets/linux-fedora-32.svg + ../../assets/linux-gentoo.svg + ../../assets/linux-ubuntu-1804.svg + ../../assets/linux-ubuntu-2004.svg + ../../assets/linux-ubuntu-xenial-amd64.svg + ../../assets/netbsd-iOS.svg + ../../assets/netbsd-OSX-catalina.svg + ../../assets/sai-event.svg + ../../assets/sai-icon.svg + ../../assets/sai.svg + ../../assets/stop.svg + ../../assets/strict-csp.svg + ../../assets/tc-gcc.svg + ../../assets/tc-llvm.svg + ../../assets/tc-mingw32.svg + ../../assets/tc-mingw64.svg + ../../assets/tc-msvc.svg + ../../assets/ubuntu-focal-aarch64.svg + ../../assets/virt-qemu.svg + ../../assets/windows-10.svg + + DESTINATION "${DATA_DIR}/sai/assets" + COMPONENT master) + +endif(requirements) diff --git a/src/master/conf.c b/src/master/conf.c deleted file mode 100644 index 6aacd4d..0000000 --- a/src/master/conf.c +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Sai master src/master/conf.c - * - * Copyright (C) 2019 Andy Green - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation: - * version 2.1 of the License. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, - * MA 02110-1301 USA - */ -#include -#include -#include -#include - -#define SAI_CONFIG_STRING_SIZE (16 * 1024) - -struct lws_context * -sai_lws_context_from_json(const char *config_dir, - const struct lws_protocols **pprotocols) -{ - int cs_len = SAI_CONFIG_STRING_SIZE - 1; - struct lws_context_creation_info info; - struct lws_context *context; - char *cs, *config_strings; - - cs = config_strings = malloc(SAI_CONFIG_STRING_SIZE); - if (!config_strings) { - lwsl_err("Unable to allocate config strings heap\n"); - - return NULL; - } - - memset(&info, 0, sizeof(info)); - - info.external_baggage_free_on_destroy = config_strings; - info.pt_serv_buf_size = 8192; - info.options = LWS_SERVER_OPTION_DO_SSL_GLOBAL_INIT | - LWS_SERVER_OPTION_EXPLICIT_VHOSTS | - LWS_SERVER_OPTION_HTTP_HEADERS_SECURITY_BEST_PRACTICES_ENFORCE | - LWS_SERVER_OPTION_VALIDATE_UTF8; - - lwsl_notice("Using config dir: \"%s\"\n", config_dir); - - /* - * first go through the config for creating the outer context - */ - if (lwsws_get_config_globals(&info, config_dir, &cs, &cs_len)) - goto init_failed; - - context = lws_create_context(&info); - if (context == NULL) { - lwsl_err("lws init failed\n"); - /* config_strings freed as 'external baggage' */ - return NULL; - } - - info.pprotocols = pprotocols; - - if (lwsws_get_config_vhosts(context, &info, config_dir, &cs, &cs_len)) { - lws_context_destroy(context); - - return NULL; - } - - return context; - -init_failed: - free(config_strings); - - return NULL; -} diff --git a/src/master/m-artifact.c b/src/master/m-artifact.c new file mode 100644 index 0000000..527abc2 --- /dev/null +++ b/src/master/m-artifact.c @@ -0,0 +1,144 @@ +/* + * Sai master + * + * Copyright (C) 2019 - 2020 Andy Green + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation: + * version 2.1 of the License. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + * MA 02110-1301 USA + */ + +#include +#include +#include +#include + +#include "m-private.h" + +int +saim_get_blob(struct vhd *vhd, const char *url, sqlite3 **pdb, + sqlite3_blob **blob, uint64_t *length) +{ + char task_uuid[66], event_uuid[34], nonce[34], qu[200], + esc[66], esc1[34]; + struct lwsac *ac = NULL; + lws_dll2_owner_t o; + sai_artifact_t *a; + const char *p; + uint64_t rid = 0; + int n; + + /* + * We get passed the RHS of a URL string like + * + * //filename + * + * filename is not used for matching, but make sure the client saves it + * using the name generated along with the link. + * + * Extract the pieces from the URL + */ + + n = 0; + p = url; + while (*p && *p != '/' && n < (int)sizeof(task_uuid) - 1) + task_uuid[n++] = *p++; + + if (n != sizeof(task_uuid) - 2 || *p != '/') { + lwsl_notice("%s: url layout 1: %d %s\n", __func__, n, url); + return -1; + } + + task_uuid[n] = '\0'; + p++; /* skip the / */ + + n = 0; + while (*p && *p != '/' && n < (int)sizeof(nonce) - 1) + nonce[n++] = *p++; + + if (n != sizeof(nonce) - 2 || *p != '/') { + lwsl_notice("%s: url layout 2\n", __func__); + return -1; + } + + sai_task_uuid_to_event_uuid(event_uuid, task_uuid); + + /* open the event-specific database object */ + + if (saim_event_db_ensure_open(vhd, event_uuid, 0, pdb)) { + lwsl_info("%s: unable to open event-specific database\n", + __func__); + + return -1; + } + + /* + * Query the artifact he's asking for + */ + + lws_sql_purify(esc, task_uuid, sizeof(esc)); + lws_sql_purify(esc1, nonce, sizeof(esc1)); + lws_snprintf(qu, sizeof(qu), + " and task_uuid='%s' and artifact_down_nonce='%s'", + esc, esc1); + n = lws_struct_sq3_deserialize(*pdb, qu, NULL, + lsm_schema_sq3_map_artifact, + &o, &ac, 0, 1); + if (n < 0 || !o.head) { + lwsl_err("%s: no result from %s\n", __func__, qu); + goto fail; + } + + a = (sai_artifact_t *)o.head; + + *length = a->len; + + /* + * recover the rowid the blob api requires + */ + + lws_snprintf(qu, sizeof(qu), "select rowid from artifacts " + "where artifact_down_nonce='%s'", esc1); + + if (sqlite3_exec(*pdb, qu, sai_sql3_get_uint64_cb, &rid, NULL) != SQLITE_OK) { + lwsl_err("%s: %s: %s: fail\n", __func__, qu, sqlite3_errmsg(*pdb)); + goto fail; + } + + /* + * Get a read-only handle on the blob + */ + + if (sqlite3_blob_open(*pdb, "main", "artifacts", "blob", rid, 0, blob) != SQLITE_OK) { + lwsl_err("%s: unable to open blob, rid %d\n", __func__, (int)rid); + goto fail; + } + + lwsac_free(&ac); /* drop the lwsac holding that result */ + + /* + * It was successful, *length was set, *pdb and *blob are live handles + * to the db and the blob itself. + */ + + return 0; + +fail: + lwsac_free(&ac); + saim_event_db_close(vhd, pdb); + + lwsl_notice("%s: couldn't find blob %s\n", __func__, url); + + return -1; +} diff --git a/src/master/m-central.c b/src/master/m-central.c new file mode 100644 index 0000000..c967625 --- /dev/null +++ b/src/master/m-central.c @@ -0,0 +1,159 @@ +/* + * Sai master + * + * Copyright (C) 2019 - 2020 Andy Green + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation: + * version 2.1 of the License. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + * MA 02110-1301 USA + * + * + * Central dispatcher for jobs from events that made it into the database. This + * is done in an event-driven way in m-task.c, but management of it also has to + * be done in the background for when there are no events coming, + */ + +#include +#include +#include +#include + +#include "m-private.h" + +extern struct lws_context *context; + +static void +saim_central_clean_abandoned(struct vhd *vhd) +{ + saim_sqlite_cache_t *sc; + struct lwsac *ac = NULL; + lws_usec_t now = lws_now_usecs(); + lws_dll2_owner_t o; + char s[160]; + int n, nzr; + + /* + * Sqlite cache cleaning + */ + + nzr = 0; + lws_start_foreach_dll_safe(struct lws_dll2 *, p, p1, + vhd->sqlite3_cache.head) { + sc = lws_container_of(p, saim_sqlite_cache_t, list); + + if (!sc->refcount && + (now - sc->idle_since) > (60 * LWS_USEC_PER_SEC)) { + lwsl_notice("%s: delayed db pool clean %s\n", __func__, + sc->uuid); + lws_struct_sq3_close(&sc->pdb); + lws_dll2_remove(&sc->list); + free(sc); + } else + if (sc->refcount) + nzr++; + + } lws_end_foreach_dll_safe(p, p1); + + if (vhd->sqlite3_cache.count) + lwsl_notice("%s: db pool items: in-use: %d, total: %d\n", + __func__, nzr, vhd->sqlite3_cache.count); + + /* + * Collect the most recent <=10 events that still feel they're + * incomplete and may be running something + */ + + n = lws_struct_sq3_deserialize(vhd->master.pdb, + " and (state != 3 and state != 4 and state != 5)", + NULL, lsm_schema_sq3_map_event, &o, + &ac, 0, 10); + if (n < 0 || !o.head) + return; + + /* + * For each of those events, look for tasks that have been running + * "too long", eg, builder restarted or lost connection etc + */ + + lws_start_foreach_dll(struct lws_dll2 *, p, o.head) { + sai_event_t *e = lws_container_of(p, sai_event_t, list); + sqlite3 *pdb = NULL; + + if (!saim_event_db_ensure_open(vhd, e->uuid, 0, &pdb)) { + /* + * Such tasks should go into CANCELLED + */ + + lws_snprintf(s, sizeof(s), + "update tasks set state=%d where " + "(state=1 OR state=2) and started < %llu", + SAIES_CANCELLED, (unsigned long long) + (lws_now_secs() - (30 * 60))); + + if (sqlite3_exec(pdb, s, NULL, NULL, NULL) != + SQLITE_OK) + lwsl_err("%s: %s: %s: fail\n", __func__, s, + sqlite3_errmsg(pdb)); + + saim_event_db_close(vhd, &pdb); + } + + } lws_end_foreach_dll(p); + + lwsac_free(&ac); +} + +void +saim_central_cb(lws_sorted_usec_list_t *sul) +{ + struct vhd *vhd = lws_container_of(sul, struct vhd, sul_central); + + /* + * For each builder connected to us, see if it can handle a new task, + * and if so, try to select one matching its supported platforms + */ + + lws_start_foreach_dll(struct lws_dll2 *, p, + vhd->master.builder_owner.head) { + sai_plat_t *cb = lws_container_of(p, sai_plat_t, sai_plat_list); + + lwsl_debug("%s: checking tasks %s %d %d %p\n", __func__, + cb->name, cb->ongoing, cb->instances, cb->wsi); + + if (cb->wsi && lws_wsi_user(cb->wsi) && + cb->ongoing < cb->instances) + saim_allocate_task(vhd, + (struct pss *)lws_wsi_user(cb->wsi), + cb, cb->platform); + + } lws_end_foreach_dll(p); + + /* + * Need to globally check for abandoned tasks periodically + */ + + if (!vhd->last_check_abandoned_tasks || + lws_now_usecs() > (vhd->last_check_abandoned_tasks + + (20 * LWS_USEC_PER_SEC))) { + + saim_central_clean_abandoned(vhd); + + vhd->last_check_abandoned_tasks = lws_now_usecs(); + } + + /* check again in 1s */ + + lws_sul_schedule(context, 0, &vhd->sul_central, saim_central_cb, + 1 * LWS_US_PER_SEC); +} diff --git a/src/master/m-comms.c b/src/master/m-comms.c new file mode 100644 index 0000000..3a8eef8 --- /dev/null +++ b/src/master/m-comms.c @@ -0,0 +1,917 @@ +/* + * Sai master + * + * Copyright (C) 2019 - 2020 Andy Green + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation: + * version 2.1 of the License. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + * MA 02110-1301 USA + * + * The same ws interface is connected-to by builders (on path /builder), and + * provides the query transport for browsers (on path /browse). + * + * There's a single master slite3 database containing events, and a separate + * sqlite3 database file for each event, it only contains tasks and logs for + * the event and can be deleted when the event record associated with it is + * deleted. This is to keep is scalable when there may be thousands of events + * and related tasks and logs stored. + */ + +#include +#include +#include +#include + +#include "m-private.h" + +#include "../common/struct-metadata.c" + +typedef enum { + SJS_CLONING, + SJS_ASSIGNING, + SJS_WAITING, + SJS_DONE +} sai_job_state_t; + +typedef struct sai_job { + struct lws_dll2 jobs_list; + char reponame[64]; + char ref[64]; + char head[64]; + + time_t requested; + + sai_job_state_t state; + +} sai_job_t; + +const lws_struct_map_t lsm_schema_map_ta[] = { + LSM_SCHEMA (sai_task_t, NULL, lsm_task, "com-warmcat-sai-ta"), +}; + +extern const lws_struct_map_t lsm_schema_sq3_map_event[]; + + +static int +sai_destroy_builder(struct lws_dll2 *d, void *user) +{ +// saib_t *b = lws_container_of(d, saib_t, c.builder_list); + + lws_dll2_remove(d); + + return 0; +} + +static void +saim_master_destroy(saim_t *master) +{ + lws_dll2_foreach_safe(&master->builder_owner, NULL, sai_destroy_builder); + + lws_struct_sq3_close(&master->pdb); +} + +static char *hexch = "0123456789abcdef"; + +/* len is typically 16 (event uuid is 32 chars + NUL) + * But eg, task uuid is concatenated 32-char eventid and 32-char taskid + */ + +int +sai_uuid16_create(struct lws_context *context, char *dest33) +{ + uint8_t *r = ((uint8_t *)dest33) + 33 - 16; + size_t n = 16; + + if (lws_get_random(context, r, n) != n) + return 1; + + while (n--) { + *dest33++ = hexch[(*r) >> 4]; + *dest33++ = hexch[(*r++) & 0xf]; + } + + *dest33 = '\0'; + + return 0; +} + +int +sai_sqlite3_statement(sqlite3 *pdb, const char *cmd, const char *desc) +{ + sqlite3_stmt *sm; + int n; + + if (sqlite3_prepare_v2(pdb, cmd, -1, &sm, NULL) != SQLITE_OK) { + lwsl_err("%s: Unable to %s: %s\n", + __func__, desc, sqlite3_errmsg(pdb)); + + return 1; + } + + n = sqlite3_step(sm); + sqlite3_reset(sm); + sqlite3_finalize(sm); + if (n != SQLITE_DONE) { + lwsl_err("%s: %d: Unable to perform \"%s\": %s\n", __func__, + sqlite3_extended_errcode(pdb), desc, + sqlite3_errmsg(pdb)); + puts(cmd); + + return 1; + } + + return 0; +} + +int +saim_event_db_ensure_open(struct vhd *vhd, const char *event_uuid, + char create_if_needed, sqlite3 **ppdb) +{ + char filepath[256], saf[33]; + saim_sqlite_cache_t *sc; + + if (*ppdb) + return 0; + + /* do we have this guy cached? */ + + lws_start_foreach_dll(struct lws_dll2 *, p, vhd->sqlite3_cache.head) { + sc = lws_container_of(p, saim_sqlite_cache_t, list); + + if (!strcmp(event_uuid, sc->uuid)) { + sc->refcount++; + *ppdb = sc->pdb; + return 0; + } + + } lws_end_foreach_dll(p); + + /* ... nope, well, let's open and cache him then... */ + + lws_strncpy(saf, event_uuid, sizeof(saf)); + lws_filename_purify_inplace(saf); + + lws_snprintf(filepath, sizeof(filepath), "%s-event-%s.sqlite3", + vhd->sqlite3_path_lhs, saf); + + if (lws_struct_sq3_open(vhd->context, filepath, create_if_needed, ppdb)) { + lwsl_err("%s: Unable to open db %s: %s\n", __func__, + filepath, sqlite3_errmsg(*ppdb)); + + return 1; + } + + /* create / add to the schema for the tables we will have in here */ + + if (lws_struct_sq3_create_table(*ppdb, lsm_schema_sq3_map_task)) + return 1; + + sai_sqlite3_statement(*ppdb, "PRAGMA journal_mode=WAL;", "set WAL"); + + if (lws_struct_sq3_create_table(*ppdb, lsm_schema_sq3_map_log)) + return 1; + + if (lws_struct_sq3_create_table(*ppdb, lsm_schema_sq3_map_artifact)) + return 1; + + sc = malloc(sizeof(*sc)); + memset(sc, 0, sizeof(*sc)); + if (!sc) { + lws_struct_sq3_close(ppdb); + *ppdb = NULL; + return 1; + } + + lws_strncpy(sc->uuid, event_uuid, sizeof(sc->uuid)); + sc->refcount = 1; + sc->pdb = *ppdb; + lws_dll2_add_tail(&sc->list, &vhd->sqlite3_cache); + + return 0; +} + +void +saim_event_db_close(struct vhd *vhd, sqlite3 **ppdb) +{ + saim_sqlite_cache_t *sc; + + if (!*ppdb) + return; + + /* look for him in the cache */ + + lws_start_foreach_dll(struct lws_dll2 *, p, vhd->sqlite3_cache.head) { + sc = lws_container_of(p, saim_sqlite_cache_t, list); + + if (sc->pdb == *ppdb) { + *ppdb = NULL; + if (--sc->refcount) { + lwsl_notice("%s: zero refcount to idle\n", __func__); + /* + * He's not currently in use then... don't + * close him immediately, m-central.c has a + * timer that closes and removes sqlite3 + * cache entries idle for longer than 60s + */ + sc->idle_since = lws_now_usecs(); + } + + return; + } + + } lws_end_foreach_dll(p); + + lws_struct_sq3_close(ppdb); + *ppdb = NULL; +} + +int +saim_event_db_delete_database(struct vhd *vhd, const char *event_uuid) +{ + char filepath[256], saf[33]; + + lws_strncpy(saf, event_uuid, sizeof(saf)); + lws_filename_purify_inplace(saf); + + lws_snprintf(filepath, sizeof(filepath), "%s-event-%s.sqlite3", + vhd->sqlite3_path_lhs, saf); + + return unlink(filepath); +} + + +#if 0 +static void +saim_all_browser_on_writable(struct vhd *vhd) +{ + lws_start_foreach_dll(struct lws_dll2 *, mp, vhd->browsers.head) { + struct pss *pss = lws_container_of(mp, struct pss, same); + + lws_callback_on_writable(pss->wsi); + } lws_end_foreach_dll(mp); +} +#endif + +typedef enum { + SHMUT_NONE = -1, + SHMUT_HOOK, + SHMUT_BROWSE, + SHMUT_STATUS, + SHMUT_ARTIFACTS, +} sai_http_murl_t; + +static const char * const well_known[] = { + "/update-hook", + "/sai/browse", + "/status", + "/artifacts/" /* HTTP api for accessing build artifacts */ +}; + +static const char *hmac_names[] = { + "sai sha256=", + "sai sha384=", + "sai sha512=" +}; + +void +mark_pending(struct pss *pss, ws_state state) +{ + pss->pending |= 1 << state; + lws_callback_on_writable(pss->wsi); +} + +static int +browser_upd(struct lws_dll2 *d, void *user) +{ + struct pss *pss = lws_container_of(d, struct pss, same); + ws_state state = (ws_state)(intptr_t)user; + + mark_pending(pss, state); + + return 0; +} + +int +sai_get_head_status(struct vhd *vhd, const char *projname) +{ + struct lwsac *ac = NULL; + lws_dll2_owner_t o; + sai_event_t *e; + int state; + + if (lws_struct_sq3_deserialize(vhd->master.pdb, NULL, "created ", + lsm_schema_sq3_map_event, &o, &ac, 0, -1)) + return -1; + + if (!o.head) + return -1; + + e = lws_container_of(o.head, sai_event_t, list); + state = e->state; + + lwsac_free(&ac); + + return state; +} + + +static int +callback_ws(struct lws *wsi, enum lws_callback_reasons reason, void *user, + void *in, size_t len) +{ + struct vhd *vhd = (struct vhd *)lws_protocol_vh_priv_get( + lws_get_vhost(wsi), lws_get_protocol(wsi)); + uint8_t buf[LWS_PRE + 6144], *start = &buf[LWS_PRE], *p = start, + *end = &buf[sizeof(buf) - LWS_PRE - 1]; + struct pss *pss = (struct pss *)user; + sai_http_murl_t mu = SHMUT_NONE; + char projname[64]; + int n, resp, r; + const char *cp; + + (void)end; + (void)p; + + switch (reason) { + case LWS_CALLBACK_PROTOCOL_INIT: + vhd = lws_protocol_vh_priv_zalloc(lws_get_vhost(wsi), + lws_get_protocol(wsi), + sizeof(struct vhd)); + if (!vhd) + return -1; + + vhd->context = lws_get_context(wsi); + vhd->vhost = lws_get_vhost(wsi); + + if (lws_pvo_get_str(in, "notification-key", + &vhd->notification_key)) { + lwsl_err("%s: notification_key pvo required\n", __func__); + return -1; + } + + if (lws_pvo_get_str(in, "database", &vhd->sqlite3_path_lhs)) { + lwsl_err("%s: database pvo required\n", __func__); + return -1; + } + + lws_snprintf((char *)buf, sizeof(buf), "%s-events.sqlite3", + vhd->sqlite3_path_lhs); + + if (lws_struct_sq3_open(vhd->context, (char *)buf, 1, + &vhd->master.pdb)) { + lwsl_err("%s: Unable to open session db %s: %s\n", + __func__, vhd->sqlite3_path_lhs, sqlite3_errmsg( + vhd->master.pdb)); + + return -1; + } + + sai_sqlite3_statement(vhd->master.pdb, + "PRAGMA journal_mode=WAL;", "set WAL"); + + if (lws_struct_sq3_create_table(vhd->master.pdb, + lsm_schema_sq3_map_event)) { + lwsl_err("%s: unable to create event table\n", __func__); + return -1; + } + + lws_sul_schedule(vhd->context, 0, &vhd->sul_central, + saim_central_cb, 500 * LWS_US_PER_MS); + + break; + + case LWS_CALLBACK_PROTOCOL_DESTROY: + saim_master_destroy(&vhd->master); + goto passthru; + + /* + * receive http hook notifications + */ + + case LWS_CALLBACK_HTTP: + + resp = HTTP_STATUS_FORBIDDEN; + pss->vhd = vhd; + + for (n = 0; n < (int)LWS_ARRAY_SIZE(well_known); n++) + if (!strncmp((const char *)in, well_known[n], + strlen(well_known[n]))) { + mu = n; + break; + } + + pss->our_form = 0; + + lwsl_notice("%s: HTTP: xmu = %d\n", __func__, n); + + switch (mu) { + + case SHMUT_NONE: + goto passthru; + + case SHMUT_HOOK: + pss->our_form = 1; + lwsl_notice("LWS_CALLBACK_HTTP: sees hook\n"); + return 0; + + case SHMUT_STATUS: + /* + * in is a string like /libwebsockets/status.svg + */ + cp = ((const char *)in) + 7; + while (*cp == '/') + cp++; + n = 0; + while (*cp != '/' && *cp && (size_t)n < sizeof(projname) - 1) + projname[n++] = *cp++; + projname[n] = '\0'; + + lwsl_notice("%s: status %s\n", __func__, projname); + + r = sai_get_head_status(vhd, projname); + if (r < 2) + r = 2; + n = lws_snprintf(projname, sizeof(projname), + "../decal-%d.svg", r); + + if (lws_http_redirect(wsi, 307, + (unsigned char *)projname, n, + &p, end) < 0) + return -1; + + goto passthru; + + case SHMUT_ARTIFACTS: + /* + * HTTP Bulk GET interface for artifact download + * + * /artifacts///filename + */ + lwsl_notice("%s: SHMUT_ARTIFACTS\n", __func__); + pss->artifact_offset = 0; + if (saim_get_blob(vhd, (const char *)in + 11, + &pss->pdb_artifact, + &pss->blob_artifact, + &pss->artifact_length)) { + lwsl_notice("%s: get_blob failed\n", __func__); + resp = 404; + goto http_resp; + } + + /* + * Well, it seems what he wanted exists.. + */ + + if (lws_add_http_header_status(wsi, 200, &p, end)) + goto bail; + if (lws_add_http_header_content_length(wsi, + (unsigned long)pss->artifact_length, + &p, end)) + goto bail; + + if (lws_add_http_header_by_token(wsi, + WSI_TOKEN_HTTP_CONTENT_TYPE, + (uint8_t *)"application/octet-stream", + 24, &p, end)) + goto bail; + if (lws_finalize_write_http_header(wsi, start, &p, end)) + goto bail; + + lwsl_notice("%s: started artifact transaction %d\n", __func__, + (int)pss->artifact_length); + + lws_callback_on_writable(wsi); + return 0; + + default: + lwsl_notice("%s: DEFAULT!!!\n", __func__); + return 0; + } + + resp = HTTP_STATUS_OK; + + /* faillthru */ + +http_resp: + if (lws_add_http_header_status(wsi, resp, &p, end)) + goto bail; + if (lws_add_http_header_content_length(wsi, 0, &p, end)) + goto bail; + if (lws_finalize_write_http_header(wsi, start, &p, end)) + goto bail; + goto try_to_reuse; + + + case LWS_CALLBACK_HTTP_WRITEABLE: + + lwsl_notice("%s: HTTP_WRITEABLE\n", __func__); + + if (!pss || !pss->blob_artifact) + break; + + n = lws_ptr_diff(end, start); + if ((int)(pss->artifact_length - pss->artifact_offset) < n) + n = (int)(pss->artifact_length - pss->artifact_offset); + + if (sqlite3_blob_read(pss->blob_artifact, start, n, + pss->artifact_offset)) { + lwsl_err("%s: blob read failed\n", __func__); + return -1; + } + + pss->artifact_offset += n; + + if (lws_write(wsi, start, n, + pss->artifact_offset != pss->artifact_length ? + LWS_WRITE_HTTP : LWS_WRITE_HTTP_FINAL) != n) + return -1; + + if (pss->artifact_offset != pss->artifact_length) + lws_callback_on_writable(wsi); + + break; + + /* + * Notifcation POSTs + */ + + case LWS_CALLBACK_HTTP_BODY: + + if (!pss->our_form) { + lwsl_notice("%s: not our form\n", __func__); + goto passthru; + } + + lwsl_user("LWS_CALLBACK_HTTP_BODY: %d\n", (int)len); + /* create the POST argument parser if not already existing */ + + if (!pss->spa) { + pss->wsi = wsi; + if (lws_hdr_copy(wsi, pss->notification_sig, + sizeof(pss->notification_sig), + WSI_TOKEN_HTTP_AUTHORIZATION) < 0) { + lwsl_err("%s: failed to get signature hdr\n", + __func__); + return -1; + } + + if (lws_hdr_copy(wsi, pss->sn.e.source_ip, + sizeof(pss->sn.e.source_ip), + WSI_TOKEN_X_FORWARDED_FOR) < 0) + lws_get_peer_simple(wsi, pss->sn.e.source_ip, + sizeof(pss->sn.e.source_ip)); + + pss->spa = lws_spa_create(wsi, NULL, 0, 1024, + sai_notification_file_upload_cb, pss); + if (!pss->spa) { + lwsl_err("failed to create spa\n"); + return -1; + } + + /* find out the hmac used to sign it */ + + pss->hmac_type = LWS_GENHMAC_TYPE_UNKNOWN; + for (n = 0; n < (int)LWS_ARRAY_SIZE(hmac_names); n++) + if (!strncmp(pss->notification_sig, + hmac_names[n], + strlen(hmac_names[n]))) { + pss->hmac_type = n + 1; + break; + } + + if (pss->hmac_type == LWS_GENHMAC_TYPE_UNKNOWN) { + lwsl_notice("%s: unknown sig hash type\n", + __func__); + return -1; + } + + /* convert it to binary */ + + n = lws_hex_to_byte_array( + pss->notification_sig + strlen(hmac_names[n]), + (uint8_t *)pss->notification_sig, 64); + + if (n != (int)lws_genhmac_size(pss->hmac_type)) { + lwsl_notice("%s: notifcation hash bad length\n", + __func__); + + return -1; + } + } + + /* let it parse the POST data */ + + if (!pss->spa_failed && + lws_spa_process(pss->spa, in, (int)len)) + /* + * mark it as failed, and continue taking body until + * completion, and return error there + */ + pss->spa_failed = 1; + + break; + + case LWS_CALLBACK_HTTP_BODY_COMPLETION: + lwsl_user("%s: LWS_CALLBACK_HTTP_BODY_COMPLETION: %d\n", + __func__, (int)len); + + if (!pss->our_form) { + lwsl_user("%s: no sai form\n", __func__); + goto passthru; + } + + + /* inform the spa no more payload data coming */ + lws_spa_finalize(pss->spa); + if (pss->spa) { + lws_spa_destroy(pss->spa); + pss->spa = NULL; + } + + if (pss->spa_failed) + lwsl_notice("%s: notification failed\n", __func__); + else { + lwsl_notice("%s: notification: %d %s %s %s\n", __func__, + pss->sn.action, pss->sn.e.hash, + pss->sn.e.ref, pss->sn.e.repo_name); + + /* update connected browsers */ + + lws_dll2_foreach_safe(&vhd->browsers, + (void *)WSS_PREPARE_OVERVIEW, browser_upd); + } + + if (lws_return_http_status(wsi, + pss->spa_failed ? HTTP_STATUS_FORBIDDEN : + HTTP_STATUS_OK, + NULL) < 0) + return -1; + break; + + /* + * ws connections from builders and browsers + */ + + case LWS_CALLBACK_FILTER_PROTOCOL_CONNECTION: + n = lws_hdr_copy(wsi, (char *)buf, sizeof(buf) - 1, + WSI_TOKEN_GET_URI); + if (!n) + buf[0] = '\0'; + //lwsl_notice("%s: checking with lwsgs for ws conn: %s\n", + // __func__, (const char *)buf); + + /* + * Builders don't authenticate using sessions... + */ + + if (n >= 8 && !strncmp((const char *)buf + n - 8, + "/builder", 8)) + return 0; + + return 0; +#if 0 + /* but everything else does */ + + car_args.max_len = LWSGS_AUTH_LOGGED_IN | LWSGS_AUTH_VERIFIED; + car_args.final = 0; + car_args.chunked = 1; /* ie, we are ws */ + + in = &car_args; + goto passthru; +#endif + + case LWS_CALLBACK_ESTABLISHED: + + // lwsl_notice("%s: wsi %p: ESTABLISHED\n", __func__, wsi); +#if 0 + vhd->gsp->callback(wsi, LWS_CALLBACK_SESSION_INFO, + pss->pss_gs, &pss->sinfo, 0); + if (!pss->sinfo.username[0]) { + lwsl_notice("sai ws attempt with no session\n"); + + return -1; + } +#endif + + pss->wsi = wsi; + pss->vhd = vhd; + pss->alang[0] = '\0'; + lws_hdr_copy(wsi, pss->alang, sizeof(pss->alang), + WSI_TOKEN_HTTP_ACCEPT_LANGUAGE); + + if (lws_hdr_total_length(wsi, WSI_TOKEN_GET_URI)) { + if (lws_hdr_copy(wsi, (char *)start, 64, + WSI_TOKEN_GET_URI) < 0) + return -1; + } +#if defined(LWS_ROLE_H2) + else + if (lws_hdr_copy(wsi, (char *)start, 64, + WSI_TOKEN_HTTP_COLON_PATH) < 0) + return -1; +#endif + + if (!memcmp((char *)start, "/sai", 4)) + start += 4; + + if (!strncmp((char *)start, "/browse/specific", 16)) { + const char *spe; + + lwsl_info("%s: ESTABLISHED: browser (specific)\n", __func__); + pss->type = SWT_BROWSE; + pss->wsi = wsi; + pss->specific_project[0] = '\0'; + spe = (const char *)start + 16; + while(*spe == '/') + spe++; + n = 0; + while(*spe && *spe != '/' && + (size_t)n < sizeof(pss->specific_project) - 2) + pss->specific_project[n++] = *spe++; + + pss->specific_project[n] = '\0'; + + strncpy(pss->specific, "refs/heads/", + sizeof(pss->specific)); + if (lws_get_urlarg_by_name(wsi, "h", pss->specific + 9, + sizeof(pss->specific) - 9)) { + memcpy(pss->specific, "refs/heads/", 11); + pss->specificity = SAIM_SPECIFIC_H; + } else + if (lws_get_urlarg_by_name(wsi, "id", pss->specific + 8, + sizeof(pss->specific))) { + memcpy(pss->specific, "refs/heads/", 11); + pss->specificity = SAIM_SPECIFIC_ID; + } else { + pss->specificity = SAIM_SPECIFIC_H; + strncpy(pss->specific, "refs/heads/master", + sizeof(pss->specific)); + } + break; + } + + if (!strcmp((char *)start, "/browse")) { + lwsl_info("%s: ESTABLISHED: browser\n", __func__); + pss->type = SWT_BROWSE; + pss->wsi = wsi; + break; + } + if (!strcmp((char *)start, "/builder")) { + lwsl_info("%s: ESTABLISHED: builder\n", __func__); + pss->type = SWT_BUILDER; + pss->wsi = wsi; + /* + * this adds our pss part, but not the logical builder + * yet, until we get the ws rx + */ + lws_dll2_add_head(&pss->same, &vhd->builders); + break; + } + + lwsl_err("%s: unknown URL '%s'\n", __func__, start); + + return -1; + + case LWS_CALLBACK_CLOSED: + lwsac_free(&pss->query_ac); + switch (pss->type) { + case SWT_BROWSE: + lwsl_info("%s: CLOSED browse conn\n", __func__); + lws_dll2_remove(&pss->same); + lws_dll2_remove(&pss->subs_list); + lwsac_free(&pss->task_ac); + lwsac_free(&pss->logs_ac); + break; + + case SWT_BUILDER: + lwsl_user("%s: CLOSED builder conn\n", __func__); + /* remove pss from vhd->builders */ + lws_dll2_remove(&pss->same); + + /* + * Destroy any the builder-tracking objects that + * were using this departing connection + */ + + lws_start_foreach_dll_safe(struct lws_dll2 *, p, p1, + vhd->master.builder_owner.head) { + sai_plat_t *cb = lws_container_of(p, sai_plat_t, + sai_plat_list); + + if (cb->wsi == wsi) { + /* remove builder object itself from master list */ + cb->wsi = NULL; + lws_dll2_remove(&cb->sai_plat_list); + /* + * free the deserialized builder object, + * everything he pointed to was overallocated + * when his deep copy was made + */ + free(cb); + } + + } lws_end_foreach_dll_safe(p, p1); + + if (pss->blob_artifact) { + sqlite3_blob_close(pss->blob_artifact); + pss->blob_artifact = NULL; + } + + if (pss->pdb_artifact) { + saim_event_db_close(pss->vhd, &pss->pdb_artifact); + pss->pdb_artifact = NULL; + } + + /* update the browsers about the builder removal */ + lws_dll2_foreach_safe(&vhd->browsers, + (void *)WSS_PREPARE_BUILDER_SUMMARY, browser_upd); + break; + } + break; + + case LWS_CALLBACK_RECEIVE: + + if (!pss->vhd) + pss->vhd = vhd; + + switch (pss->type) { + case SWT_BROWSE: + // lwsl_user("SWT_BROWSE RX: %d\n", (int)len); + /* + * Browser UI sent us something on websockets + */ + if (saim_ws_json_rx_browser(vhd, pss, in, len)) + return -1; + break; + + case SWT_BUILDER: + lwsl_info("SWT_BUILDER RX: %d\n", (int)len); + /* + * Builder sent us something on websockets + */ + pss->wsi = wsi; + if (saim_ws_json_rx_builder(vhd, pss, in, len)) + return -1; + if (!pss->announced) { + lws_dll2_foreach_safe(&vhd->browsers, + (void *)WSS_PREPARE_BUILDER_SUMMARY, + browser_upd); + pss->announced = 1; + } + break; + default: + break; + } + break; + + case LWS_CALLBACK_SERVER_WRITEABLE: + if (!vhd) { + lwsl_notice("%s: no vhd\n", __func__); + break; + } + switch (pss->type) { + case SWT_BROWSE: + return saim_ws_json_tx_browser(vhd, pss, buf, sizeof(buf)); + break; + + case SWT_BUILDER: + return saim_ws_json_tx_builder(vhd, pss, buf, sizeof(buf)); + break; + default: + break; + } + break; + + default: +passthru: + // if (!pss || !vhd) + break; + + // return vhd->gsp->callback(wsi, reason, pss->pss_gs, in, len); + } + + return lws_callback_http_dummy(wsi, reason, user, in, len); + +bail: + return 1; + +try_to_reuse: + if (lws_http_transaction_completed(wsi)) + return -1; + + return 0; +} + +const struct lws_protocols protocol_ws = + { "com-warmcat-sai", callback_ws, sizeof(struct pss), 0 }; diff --git a/src/master/m-conf.c b/src/master/m-conf.c new file mode 100644 index 0000000..498378f --- /dev/null +++ b/src/master/m-conf.c @@ -0,0 +1,82 @@ +/* + * Sai master src/master/conf.c + * + * Copyright (C) 2019 - 2020 Andy Green + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation: + * version 2.1 of the License. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + * MA 02110-1301 USA + */ +#include +#include +#include +#include + +#define SAI_CONFIG_STRING_SIZE (16 * 1024) + +struct lws_context * +sai_lws_context_from_json(const char *config_dir, + struct lws_context_creation_info *info, + const struct lws_protocols **pprotocols) +{ + int cs_len = SAI_CONFIG_STRING_SIZE - 1; + struct lws_context *context; + char *cs, *config_strings; + + cs = config_strings = malloc(SAI_CONFIG_STRING_SIZE); + if (!config_strings) { + lwsl_err("Unable to allocate config strings heap\n"); + + return NULL; + } + + memset(info, 0, sizeof(*info)); + + info->external_baggage_free_on_destroy = config_strings; + info->pt_serv_buf_size = 8192; + info->options = LWS_SERVER_OPTION_DO_SSL_GLOBAL_INIT | + LWS_SERVER_OPTION_EXPLICIT_VHOSTS | + LWS_SERVER_OPTION_HTTP_HEADERS_SECURITY_BEST_PRACTICES_ENFORCE | + LWS_SERVER_OPTION_VALIDATE_UTF8; + + lwsl_notice("Using config dir: \"%s\"\n", config_dir); + + /* + * first go through the config for creating the outer context + */ + if (lwsws_get_config_globals(info, config_dir, &cs, &cs_len)) + goto init_failed; + + context = lws_create_context(info); + if (context == NULL) { + /* config_strings freed as 'external baggage' */ + return NULL; + } + + info->pprotocols = pprotocols; + + if (lwsws_get_config_vhosts(context, info, config_dir, &cs, &cs_len)) { + lwsl_err("%s: sai_lws_context_from_json failed\n", __func__); + lws_context_destroy(context); + + return NULL; + } + + return context; + +init_failed: + free(config_strings); + + return NULL; +} diff --git a/src/master/m-notification.c b/src/master/m-notification.c new file mode 100644 index 0000000..2412054 --- /dev/null +++ b/src/master/m-notification.c @@ -0,0 +1,938 @@ +/* + * Sai master - src/master/notification.c + * + * Copyright (C) 2019 - 2020 Andy Green + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation: + * version 2.1 of the License. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + * MA 02110-1301 USA + * + * The same ws interface is connected-to by builders (on path /builder), and + * provides the query transport for browsers (on path /browse). + */ + +#include +#include "m-private.h" + +/* starts from +1 of sai_notification_action_t */ + +static const char *notifaction_action_names[] = { + "repo-update" +}; + +static const char * const paths[] = { + "schema", + "action", + "repository.name", + "repository.fetchurl", + "ref", + "hash", + "nonce", + "saifile_len", + "saifile", +}; + +enum enum_paths { + LEJPN_SCHEMA, + LEJPN_ACTION, + LEJPN_REPOSITORY_NAME, + LEJPN_REPOSITORY_FETCHURL, + LEJPN_REF, + LEJPN_HASH, + LEJPN_NONCE, + LEJPN_SAIFILE_LEN, + LEJPN_SAIFILE, +}; + +/* + * Saifile parser + */ + +static const char * const saifile_paths[] = { + "schema", + "platforms.*.build", + "platforms.*.default", + "platforms.*", + "configurations.*.prep", + "configurations.*.cmake", + "configurations.*.deps", + "configurations.*.platforms", + "configurations.*.artifacts", + "configurations.*.cpack", + "configurations.*", +}; + +enum enum_saifile_paths { + LEJPNSAIF_SCHEMA, + LEJPNSAIF_PLAT_BUILD, + LEJPNSAIF_PLAT_DEFAULT, + LEJPNSAIF_PLAT_NAME, + LEJPNSAIF_CONFIGURATIONS_PREP, + LEJPNSAIF_CONFIGURATIONS_CMAKE, + LEJPNSAIF_CONFIGURATIONS_DEPS, + LEJPNSAIF_CONFIGURATIONS_PLATFORMS, + LEJPNSAIF_CONFIGURATIONS_ARTIFACTS, + LEJPNSAIF_CONFIGURATIONS_CPACK, + LEJPNSAIF_CONFIGURATIONS_NAME, +}; + +/* do the string subst for ${cmake} etc */ + +static int +exp_cmake(void *priv, const char *name, char *out, size_t *pos, size_t olen, + size_t *exp_ofs) +{ + sai_notification_t *sn = (sai_notification_t *)priv; + const char *replace = NULL; + size_t replace_len, rem_out; + + if (!strcmp(name, "prep")) { + replace = sn->t.prep; + replace_len = strlen(sn->t.prep); + goto expand; + } + + if (!strcmp(name, "cmake")) { + replace = sn->t.cmake; + replace_len = strlen(sn->t.cmake); + goto expand; + } + + if (!strcmp(name, "cpack")) { + replace = sn->t.cpack; + replace_len = strlen(sn->t.cpack); + goto expand; + } + + return LSTRX_FATAL_NAME_UNKNOWN; + +expand: + rem_out = olen - *pos; /* remaining rhs */ + replace_len -= *exp_ofs; + if (replace_len < rem_out) + rem_out = replace_len; + + memcpy(out + *pos, replace + (*exp_ofs), rem_out); + *exp_ofs += rem_out; + *pos += rem_out; + + if (rem_out == replace_len) + return LSTRX_DONE; + + return LSTRX_FILLED_OUT; +} + +static int +arg_to_bool(const char *s) +{ + static const char * const on[] = { "on", "yes", "true" }; + int n = atoi(s); + + if (n) + return 1; + + for (n = 0; n < (int)LWS_ARRAY_SIZE(on); n++) + if (!strcasecmp(s, on[n])) + return 1; + + return 0; +} + +static int +sai_tuple_element_compare(const char *e1, const char *e2) +{ + char *p; + + // lwsl_notice("%s: comp '%s' '%s'\n", __func__, e1, e2); + + if (strlen(e1) > strlen(e2)) + /* can't match if req is longer than plat */ + return -1; + + p = strchr(e2, '/'); + if (p && lws_ptr_diff(p, e2) < (int)strlen(e1)) + /* e2 contains a / section terminal inside e1 scope */ + return 1; + + return strncmp(e1, e2, strlen(e1)); +} + +static int +sai_tuple_compare(const char *req, int req_len, const char *plat) +{ + const char *pos = req; + char e1[96]; + int n; + + do { + n = 0; + while (n < (int)sizeof(e1) - 1 && req_len && *pos != '/') { + e1[n++] = *pos++; + req_len--; + } + e1[n] = '\0'; + if (*pos == '/' && req_len) { + req_len--; + pos++; + } + + if (n == sizeof(e1) - 1) { + // lwsl_notice("%s: NOMATCH 3 '%s' '%s'\n", __func__, req, plat); + /* element too long */ + return 1; + } + + if (n && /* let it pass if empty match section, eg, linux//gcc */ + sai_tuple_element_compare(e1, plat)) { + // lwsl_notice("%s: NOMATCH 2 '%s' '%s'\n", __func__, req, plat); + /* section does not match */ + return 1; + } + + while (*plat && *plat != '/') + plat++; + if (*plat == '/') + plat++; + + if (!*plat && req_len) { + // lwsl_notice("%s: NOMATCH 1 '%s' '%s'\n", __func__, req, plat); + /* FAIL: req had more than we had */ + return 1; + } + + if (!req_len) { + // lwsl_notice("%s: MATCH '%s' '%s'\n", __func__, req, plat); + /* MATCH: we matched at least as far as we had */ + return 0; + } + + } while (1); + + return 1; +} + +/* + * We parse the saifile JSON + */ + +static signed char +sai_saifile_lejp_cb(struct lejp_ctx *ctx, char reason) +{ + struct pss *pss = (struct pss *)ctx->user; + sai_notification_t *sn = (sai_notification_t *)&pss->sn; + sqlite3 *pdb = NULL; + size_t n; + + // lwsl_notice("%s: reason %d, %s\n", __func__, reason, ctx->path); + + if (reason == LEJPCB_COMPLETE) { + + if (!pss->dry) + lwsl_notice("%s: saifile decode completed\n", __func__); + + return 0; + } + + if (reason == LEJPCB_OBJECT_START && + ctx->path_match - 1 == LEJPNSAIF_CONFIGURATIONS_NAME) { + /* + * We're at the testname part of "testname" : { } + */ + lws_strncpy(sn->t.taskname, &ctx->path[15], + sizeof(sn->t.taskname)); + sn->t.prep[0] = '\0'; + sn->t.packages[0] = '\0'; + sn->t.cmake[0] = '\0'; + sn->t.artifacts[0] = '\0'; + sn->explicit_platforms[0] = '\0'; + return 0; + } + + if (reason == LEJPCB_OBJECT_END && + ctx->path_match - 1 == LEJPNSAIF_CONFIGURATIONS_NAME && + sn->t.taskname[0]) { + lws_dll2_owner_t owner; + + /* + * We're at the testname part of "testname" : { } + */ + if (pss->dry) { + sn->t.taskname[0] = '\0'; + + return 0; + } + + /* + * Iterate through each platform creating task entries for this + * configuration customized for each platform... first time just + * check the string subst for all of them is OK... + */ + + lwsl_notice("%s: Creating task entries for notification\n", __func__); + + lws_start_foreach_dll(struct lws_dll2 *, p, + pss->platform_owner.head) { + sai_platform_t *pl = + lws_container_of(p, sai_platform_t, list); + size_t used_in, used_out; + struct lws_tokenize ts; + int n, match = 0; + lws_strexp_t sx; + + if (!pl->nondefault) + /* + * If it's a default platform, then match + * by default + */ + match = 1; + + if (sn->explicit_platforms[0]) { + char not = 0; + + /* + * If the configuration gives explicit + * platforms, we have to filter the platform + * against the comma-separated list. + * + * "none" - don't match any plats just because + * they are default + * + * "not plat" - disallow specific plat "plat", + * implies you're not using "none" + * + * "plat" - allow specific plat "plat" + */ + memset(&ts, 0, sizeof(ts)); + ts.start = (char *)sn->explicit_platforms; + ts.len = strlen(ts.start); + ts.flags = LWS_TOKENIZE_F_DOT_NONTERM | + LWS_TOKENIZE_F_SLASH_NONTERM | + LWS_TOKENIZE_F_MINUS_NONTERM; + do { + ts.e = lws_tokenize(&ts); + if (ts.e != LWS_TOKZE_TOKEN) + continue; + + lwsl_notice("%s: check %.*s\n", __func__, + (int)ts.token_len, ts.token); + + if (!strncmp(ts.token, "none", + ts.token_len)) { + not = match = 0; + continue; + } + if (!strncmp(ts.token, "not", + ts.token_len)) { + not = 1; + continue; + } + /* + * We need to check with per-slash + * minimal matching, ie, "linux" + * matches "linux-ubuntu/x86_64/gcc" + */ + if (!sai_tuple_compare(ts.token, + ts.token_len, pl->name)) { + match = !not; + if (match) + break; + } + not = 0; + } while (ts.e > 0); + } + + if (match) { + + /* + * The platform build string (pl->build) is the + * base, with optional entries like ${cmake} + * that are filled in with the corresponding + * info from the specific configuration's + * "cmake" entry (sn->t.cmake)... + * + * sn->t.build becomes the string-substituted + * copy that is serialized + */ + + lws_strexp_init(&sx, sn, exp_cmake, sn->t.build, + sizeof(sn->t.build)); + + n = lws_strexp_expand(&sx, pl->build, + strlen(pl->build), + &used_in, &used_out); + if (n != LSTRX_DONE) { + lwsl_notice("%s: strsubst fail n=%d %s %s\n", + __func__, n, pl->build, + sn->t.cmake); + return -1; + } + } + } lws_end_foreach_dll(p); + + /* + * ...now we know the string handling will go well, create the + * event database file and create task entries in there for this + * configuration's tasks for each platform + */ + + if (saim_event_db_ensure_open(pss->vhd, pss->sn.e.uuid, 1, &pdb)) { + lwsl_err("%s: unable to open event-specific db\n", __func__); + return -1; + } + + lws_start_foreach_dll(struct lws_dll2 *, p, + pss->platform_owner.head) { + sai_platform_t *pl = + lws_container_of(p, sai_platform_t, list); + size_t used_in, used_out; + struct lws_tokenize ts; + int n, match = 0; + lws_strexp_t sx; + + if (!pl->nondefault) + /* + * If it's a nondefault platform, then match + * by default + */ + match = 1; + + if (sn->explicit_platforms[0]) { + char not = 0; + + /* + * If the configuration gives explicit + * platforms, we have to filter the platform + * against the comma-separated list. + * + * "none" - don't match any plats just because + * they are default + * + * "not plat" - disallow specific plat "plat", + * implies you're not using "none" + * + * "plat" - allow specific plat "plat" + */ + + memset(&ts, 0, sizeof(ts)); + ts.start = (char *)sn->explicit_platforms; + ts.len = strlen(ts.start); + ts.flags = LWS_TOKENIZE_F_DOT_NONTERM | + LWS_TOKENIZE_F_SLASH_NONTERM | + LWS_TOKENIZE_F_MINUS_NONTERM; + + do { + ts.e = lws_tokenize(&ts); + if (ts.e != LWS_TOKZE_TOKEN) + continue; + + lwsl_notice("%s: check %.*s\n", __func__, + (int)ts.token_len, ts.token); + + if (!strncmp(ts.token, "none", + ts.token_len)) { + not = match = 0; + continue; + } + if (!strncmp(ts.token, "not", + ts.token_len)) { + not = 1; + continue; + } + if (!sai_tuple_compare(ts.token, + ts.token_len, pl->name)) { + match = !not; + if (match) + break; + } + not = 0; + } while (ts.e > 0); + } + + if (match) { + /* + * For this platform, we want to create a task + * associated with this event. Tasks and logs + * associated with an event go in an event- + * specific database file for scalability. + */ + + lws_strexp_init(&sx, sn, exp_cmake, sn->t.build, + sizeof(sn->t.build)); + + n = lws_strexp_expand(&sx, pl->build, + strlen(pl->build), + &used_in, &used_out); + if (n != LSTRX_DONE) { + lwsl_notice("%s: strsubst failed %s %s\n", + __func__, pl->build, + sn->t.cmake); + saim_event_db_close(pss->vhd, &pdb); + return -1; + } + + /* + * Prepare a struct of the task object... + * task uuid is the event uuid and another + * random 32 chars, so you can always recover + * the related event uuid from the task uuid + */ + + memcpy(pss->sn.t.uuid, pss->sn.e.uuid, 32); + sai_uuid16_create(lws_get_context(pss->wsi), + pss->sn.t.uuid + 32); + strcpy(pss->sn.t.event_uuid, pss->sn.e.uuid); + pss->sn.t.uid = pss->sn.event_task_index++; + + /* + * This is basically a secret that anything + * trying to upload an artifact for the task + * must provide to authenticate. + */ + sai_uuid16_create(lws_get_context(pss->wsi), + pss->sn.t.art_up_nonce); + /* + * An unrelated secret that anything + * trying to download an artifact for the task + * must provide to identify it. + */ + sai_uuid16_create(lws_get_context(pss->wsi), + pss->sn.t.art_down_nonce); + + pss->sn.t.git_repo_url = + pss->sn.e.repo_fetchurl; + pss->sn.e.last_updated = + (unsigned long long)lws_now_secs(); + pss->sn.e.state = 0; + lws_strncpy(pss->sn.t.platform, pl->name, + sizeof(pss->sn.t.platform)); + + memset(&pss->sn.t.list, 0, sizeof(pss->sn.t.list)); + lws_dll2_owner_clear(&owner); + lws_dll2_add_head(&pss->sn.t.list, &owner); + + /* + * Create the task in event-specific database + */ + + lws_struct_sq3_serialize(pdb, + lsm_schema_sq3_map_task, + &owner, pss->sn.t.uid); + } + + } lws_end_foreach_dll(p); + + saim_event_db_close(pss->vhd, &pdb); + + lwsl_notice("%s: New test '%s', '%s', '%s'\n", __func__, + sn->t.taskname, sn->t.cmake, sn->t.packages); + + sn->t.taskname[0] = '\0'; + return 0; + } + + if (reason == LEJPCB_OBJECT_START && + ctx->path_match - 1 == LEJPNSAIF_PLAT_NAME) { + /* + * We're at the platformname part of "platformname" : { } + */ + lws_strncpy(sn->platname, &ctx->path[10], sizeof(sn->platname)); + sn->t.prep[0] = '\0'; + sn->t.cmake[0] = '\0'; + sn->t.cpack[0] = '\0'; + sn->platbuild[0] = '\0'; + sn->nondefault = 0; + return 0; + } + + if (reason == LEJPCB_OBJECT_END && + ctx->path_match - 1 == LEJPNSAIF_PLAT_NAME && + sn->platname[0] && sn->platbuild[0]) { + sai_platform_t *pl; + uint8_t *plb; + size_t pnl; + + if (pss->dry) { + sn->t.platform[0] = '\0'; + return 0; + } + + /* + * We create a platform object with space for its strings after + */ + + pnl = strlen(sn->platname); + pl = (sai_platform_t *)malloc(sizeof(*pl) + pnl + + strlen(sn->platbuild) + 2); + if (!pl) + return -1; + + memset(pl, 0, sizeof(*pl)); + plb = (uint8_t *)&pl[1]; + + /* + * Platforms are malloc'd up and added to pss + * .platform_owner + */ + + pl->name = (const char *)plb; + memcpy(plb, sn->platname, pnl + 1); + plb += pnl + 1; + + pl->build = (const char *)plb; + memcpy(plb, sn->platbuild, strlen(sn->platbuild) + 1); + + pl->nondefault = sn->nondefault; + + lws_dll2_add_head(&pl->list, &pss->platform_owner); + + lwsl_notice("%s: New platform '%s', build '%s', notdefault %d\n", + __func__, pl->name, pl->build, pl->nondefault); + + sn->platbuild[0] = '\0'; + return 0; + } + + /* we only match on the prepared path strings */ + if (!(reason & LEJP_FLAG_CB_IS_VALUE) || !ctx->path_match) + return 0; + +// if (reason != LEJPCB_VAL_STR_END) +// return 0; + + /* only the end part of the string, where we know the length */ + + switch (ctx->path_match - 1) { + case LEJPNSAIF_SCHEMA: + sn->event_task_index = 0; + return 0; + + case LEJPNSAIF_CONFIGURATIONS_PREP: + /* the additional cmake options for this test configuration */ + n = strlen(sn->t.prep); + if (n < sizeof(sn->t.prep) - 2) + lws_strnncpy(sn->t.prep + n, ctx->buf, ctx->npos, + sizeof(sn->t.prep) - n); + break; + + case LEJPNSAIF_CONFIGURATIONS_CMAKE: + /* the additional cmake options for this test configuration */ + n = strlen(sn->t.cmake); + if (n < sizeof(sn->t.cmake) - 2) + lws_strnncpy(sn->t.cmake + n, ctx->buf, ctx->npos, + sizeof(sn->t.cmake) - n); + break; + + case LEJPNSAIF_CONFIGURATIONS_DEPS: + /* the necessary dependent package strings */ + n = strlen(sn->t.packages); + if (n < sizeof(sn->t.packages) - 2) { + if (n) + sn->t.packages[n++] = ','; + lws_strncpy(sn->t.packages + n, ctx->buf, + sizeof(sn->t.packages) - n); + } + break; + + case LEJPNSAIF_CONFIGURATIONS_PLATFORMS: + /* the necessary dependent package strings */ + lws_strncpy(sn->explicit_platforms, ctx->buf, + sizeof(sn->explicit_platforms)); + break; + + case LEJPNSAIF_CONFIGURATIONS_ARTIFACTS: + lws_strncpy(sn->t.artifacts, ctx->buf, sizeof(sn->t.artifacts)); + break; + + case LEJPNSAIF_CONFIGURATIONS_CPACK: + lws_strncpy(sn->t.cpack, ctx->buf, sizeof(sn->t.cpack)); + break; + + case LEJPNSAIF_PLAT_BUILD: + /* + * The overall build script for this platform + * is appended into the temp sn.platbuild + */ + n = strlen(sn->platbuild); + if (n < sizeof(sn->platbuild) - 2) + lws_strnncpy(sn->platbuild + n, ctx->buf, ctx->npos, + sizeof(sn->platbuild) - n); + break; + + case LEJPNSAIF_PLAT_DEFAULT: + sn->nondefault = !arg_to_bool(ctx->buf); + break; + + default: + return 0; + } + + return 0; +} + +static signed char +sai_notification_lejp_cb(struct lejp_ctx *ctx, char reason) +{ + struct pss *pss = (struct pss *)ctx->user; + sai_notification_t *sn = (sai_notification_t *)&pss->sn; + size_t ile, ole; + int n; + + /* we only match on the prepared path strings */ + if (!(reason & LEJP_FLAG_CB_IS_VALUE) || !ctx->path_match) + return 0; + +// if (reason != LEJPCB_VAL_STR_END) +// return 0; + + /* only the end part of the string, where we know the length */ + + switch (ctx->path_match - 1) { + case LEJPN_SCHEMA: + if (strcmp(ctx->buf, "com-warmcat-sai-notification")) { + lwsl_err("%s: unknown schema '%s'\n", __func__, ctx->buf); + return -1; + } + return 0; + + case LEJPN_ACTION: + for (n = 0; n < (int)LWS_ARRAY_SIZE(notifaction_action_names);n++) + if (!strcmp(ctx->buf, notifaction_action_names[n])) { + sn->action = n + 1; + + return 0; + } + lwsl_notice("%s: unknown action '%s' ignored\n", + __func__, ctx->buf); + return -1; + + case LEJPN_REPOSITORY_NAME: + lws_strncpy(sn->e.repo_name, ctx->buf, sizeof(sn->e.repo_name)); + break; + + case LEJPN_REPOSITORY_FETCHURL: + lws_strncpy(sn->e.repo_fetchurl, ctx->buf, sizeof(sn->e.repo_fetchurl)); + break; + + case LEJPN_REF: + lws_strncpy(sn->e.ref, ctx->buf, sizeof(sn->e.ref)); + break; + + case LEJPN_HASH: + lws_strncpy(sn->e.hash, ctx->buf, sizeof(sn->e.hash)); + break; + + case LEJPN_NONCE: + break; + + case LEJPN_SAIFILE_LEN: + sn->saifile_in_len = atoi(ctx->buf); + /* only accept sane base64 size */ + if (sn->saifile_in_len < 8 || sn->saifile_in_len > 65536) { + lwsl_err("%s: bad saifile_len %u\n", __func__, + (unsigned int)sn->saifile_in_len); + + return -1; + } + sn->saifile_out_len = (sn->saifile_in_len * 3) / 4 + 4; + sn->saifile = malloc(sn->saifile_out_len); + if (!sn->saifile) { + lwsl_err("%s: OOM\n", __func__); + return -1; + } + sn->saifile_in_seen = sn->saifile_out_pos = 0; + lws_b64_decode_state_init(&sn->b64); + break; + + case LEJPN_SAIFILE: + /* + * Base64 encoding of the commit's .sai.json file contents... we + * have to treat this with caution since it can be anything, + * including unparsable JSON. + * + * Let's base64-decode it and collect it into an buffer first, + * before sn test parse and then sn second parse to extract the + * tasks + */ + + sn->saifile_in_seen += ctx->npos; + if (sn->saifile_in_seen > sn->saifile_in_len) { + lwsl_err("%s: SAIFILE: too large in %u vs %u\n", __func__, + (unsigned int)sn->saifile_in_seen, + (unsigned int)sn->saifile_in_len); + return -1; + } + ile = ctx->npos; + ole = sn->saifile_out_len - sn->saifile_out_pos; + lws_b64_decode_stateful(&sn->b64, ctx->buf, &ile, + (uint8_t *)sn->saifile + sn->saifile_out_pos, &ole, + sn->saifile_in_seen == sn->saifile_in_len); + + sn->saifile_out_pos += ole; + break; + + default: + return 0; + } + + return 0; +} + +int +sai_notification_file_upload_cb(void *data, const char *name, + const char *filename, char *buf, int len, + enum lws_spa_fileupload_states state) +{ + struct pss *pss = (struct pss *)data; + struct lejp_ctx saictx; + lws_dll2_owner_t owner; + uint8_t result[64]; + int m; + + switch (state) { + case LWS_UFS_OPEN: + lwsl_notice("%s: LWS_UFS_OPEN\n", __func__); + lejp_construct(&pss->ctx, sai_notification_lejp_cb, + pss, paths, LWS_ARRAY_SIZE(paths)); + + if (lws_genhmac_init(&pss->hmac, pss->hmac_type, + (uint8_t *)pss->vhd->notification_key, + strlen(pss->vhd->notification_key))) { + lwsl_err("%s: failed to init hmac\n", __func__); + + return -1; + } + break; + case LWS_UFS_FINAL_CONTENT: + case LWS_UFS_CONTENT: + lwsl_notice("%s: LWS_UFS_[]CONTENT: %p %p, \n", __func__, pss, buf); + if (len && lws_genhmac_update(&pss->hmac, buf, len)) + return -1; + + printf("%.*s", (int)len, buf); + + m = lejp_parse(&pss->ctx, (uint8_t *)buf, len); + if (m < 0 && m != LEJP_CONTINUE) { + lwsl_notice("%s: notif JSON decode failed '%s' (%d)\n", + __func__, lejp_error_to_string(m), m); + return m; + } + + if (m != 1) + break; + + lws_genhmac_destroy(&pss->hmac, result); + + if (memcmp(result, pss->notification_sig, + lws_genhmac_size(pss->hmac_type))) { + lwsl_err("%s: hmac mismatch\n", __func__); + + return -1; + } + lwsl_notice("%s: hmac OK\n", __func__); + + if (!pss->sn.saifile) + return -1; + + /* + * We processed the notification JSON, but we only decoded the + * base64'd copy of the .sai.json so far... now's the time we + * want to process that and break it down into tasks. + * + * We don't trust it since it's controlled by the guy who pushed + * the commit, there can be anything at all in there. We made + * sure he can't attack us until now by base64-ing it at the + * server hook, so he's just dumb payload. + * + * Let's try to parse it in two passes, first without acting on + * the content to confirm it's going to succeed... + */ + + sai_uuid16_create(lws_get_context(pss->wsi), pss->sn.e.uuid); + if (saim_event_db_ensure_open(pss->vhd, pss->sn.e.uuid, 1, + (sqlite3 **)&pss->sn.e.pdb)) { + lwsl_err("%s: unable to open event-specific database\n", + __func__); + + return -1; + } + + pss->dry = 1; + lejp_construct(&saictx, sai_saifile_lejp_cb, pss, saifile_paths, + LWS_ARRAY_SIZE(saifile_paths)); + m = lejp_parse(&saictx, (uint8_t *)pss->sn.saifile, + pss->sn.saifile_out_pos); + saim_event_db_close(pss->vhd, (sqlite3 **)&pss->sn.e.pdb); + if (m < 0) { + lwsl_notice("%s: saifile JSON decode failed '%s' (%d)\n", + __func__, lejp_error_to_string(m), m); + return m; + } + + /* ... then add the 32-char event object in the database ... */ + + pss->sn.e.created = (unsigned long long)lws_now_secs(); + pss->sn.e.state = SAIES_WAITING; + + memset(&pss->sn.e.list, 0, sizeof(pss->sn.e.list)); + lws_dll2_owner_clear(&owner); + lws_dll2_add_head(&pss->sn.e.list, &owner); + + /* + * This is our new event going into the event database... + */ + + lws_struct_sq3_serialize(pss->vhd->master.pdb, + lsm_schema_sq3_map_event, &owner, 0); + + /* + * ... process the saifile JSON again creating tasks for each + * entry in the saifile, for each platform, against that event + * object... + */ + + pss->dry = 0; + lejp_construct(&saictx, sai_saifile_lejp_cb, pss, saifile_paths, + LWS_ARRAY_SIZE(saifile_paths)); + m = lejp_parse(&saictx, (uint8_t *)pss->sn.saifile, + pss->sn.saifile_out_pos); + if (m < 0) { + lwsl_notice("%s: saifile JSON decode failed '%s' (%d)\n", + __func__, lejp_error_to_string(m), m); + return m; + } + + free(pss->sn.saifile); + pss->sn.saifile = NULL; + + lwsl_notice("%s: notification inserted into db\n", __func__); + + /* + * Reassess now if there's a builder we can match to a pending task + */ + + lws_sul_schedule(pss->vhd->context, 0, &pss->vhd->sul_central, + saim_central_cb, 1); + + return 0; + + + case LWS_UFS_CLOSE: + // lwsl_info("%s: LWS_UFS_CLOSE\n", __func__); + lejp_destruct(&pss->ctx); + break; + } + + return 0; +} + diff --git a/src/master/m-overview.c b/src/master/m-overview.c new file mode 100644 index 0000000..e650c19 --- /dev/null +++ b/src/master/m-overview.c @@ -0,0 +1,28 @@ +/* + * Sai master - src/master/overview.c + * + * Copyright (C) 2019 Andy Green + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation: + * version 2.1 of the License. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + * MA 02110-1301 USA + * + * The same ws interface is connected-to by builders (on path /builder), and + * provides the query transport for browsers (on path /browse). + */ + +#include +#include "m-private.h" + + diff --git a/src/master/m-private.h b/src/master/m-private.h new file mode 100644 index 0000000..de1883d --- /dev/null +++ b/src/master/m-private.h @@ -0,0 +1,286 @@ +/* + * Sai master definitions src/master/private.h + * + * Copyright (C) 2019 Andy Green + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation: + * version 2.1 of the License. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + * MA 02110-1301 USA + */ + +#include "../common/include/private.h" +#include +#include + +struct sai_plat; + +typedef struct sai_platm { + struct lws_dll2_owner builder_owner; + struct lws_dll2_owner subs_owner; + + sqlite3 *pdb; +} saim_t; + +typedef struct sai_platform { + struct lws_dll2 list; + + const char *name; + const char *build; + + uint8_t nondefault; + + /* build and name over-allocated here */ +} sai_platform_t; + +typedef enum { + SAIN_ACTION_INVALID, + SAIN_ACTION_REPO_UPDATED +} sai_notification_action_t; + +typedef struct { + + sai_event_t e; + sai_task_t t; + + char platbuild[4096]; + char platname[96]; + char explicit_platforms[2048]; + + int event_task_index; + + struct lws_b64state b64; + char *saifile; + uint64_t when; + size_t saifile_in_len; + size_t saifile_out_len; + size_t saifile_out_pos; + size_t saifile_in_seen; + sai_notification_action_t action; + + uint8_t nondefault; +} sai_notification_t; + +typedef enum { + SWT_BUILDER, + SWT_BROWSE +} ws_type; + +typedef enum { + WSS_IDLE, + WSS_PREPARE_OVERVIEW, + WSS_SEND_OVERVIEW, + WSS_PREPARE_BUILDER_SUMMARY, + WSS_SEND_BUILDER_SUMMARY, + + WSS_PREPARE_TASKINFO, + WSS_SEND_ARTIFACT_INFO, + + WSS_PREPARE_EVENTINFO, + WSS_SEND_EVENTINFO, +} ws_state; + +typedef struct sai_builder { + saim_t c; +} saib_t; + +struct vhd; + +enum { + SAIM_NOT_SPECIFIC, + SAIM_SPECIFIC_H, + SAIM_SPECIFIC_ID +}; + +struct pss { + struct vhd *vhd; + struct lws *wsi; + + struct lws_spa *spa; + struct lejp_ctx ctx; + sai_notification_t sn; + struct lws_dll2 same; /* owner: vhd.builders */ + + struct lws_dll2 subs_list; + + uint64_t sub_timestamp; + char sub_task_uuid[65]; + char specific[65]; + char specific_project[96]; + + sqlite3 *pdb_artifact; + sqlite3_blob *blob_artifact; + + lws_dll2_owner_t platform_owner; /* sai_platform_t builder offers */ + lws_dll2_owner_t task_cancel_owner; /* sai_platform_t builder offers */ + lws_dll2_owner_t aft_owner; /* for statefully spooling artifact info */ + lws_struct_args_t a; + + union { + sai_plat_t *b; + sai_plat_owner_t *o; + } u; + const char *master_name; + + struct lwsac *query_ac; + struct lwsac *task_ac; /* tasks for an event */ + struct lwsac *logs_ac; + lws_dll2_owner_t issue_task_owner; /* list of sai_task_t */ + const sai_task_t *one_task; /* only for browser */ + const sai_event_t *one_event; + lws_dll2_owner_t query_owner; + lws_dll2_t *walk; + + int task_index; + int log_cache_index; + int log_cache_size; + int authorized; + int specificity; + + /* notification hmac information */ + char notification_sig[128]; + char alang[128]; + struct lws_genhmac_ctx hmac; + enum lws_genhmac_types hmac_type; + char our_form; + + uint64_t first_log_timestamp; + uint64_t artifact_offset; + uint64_t artifact_length; + + ws_type type; + ws_state send_state; + + uint32_t pending; /* bitmap of things that need sending */ + + unsigned int spa_failed:1; + unsigned int subsequent:1; /* for individual JSON */ + unsigned int dry:1; + unsigned int query_already_done:1; + unsigned int frag:1; + unsigned int mark_started:1; + unsigned int wants_event_updates:1; + unsigned int announced:1; + unsigned int bulk_binary_data:1; + + uint8_t ovstate; /* SOS_ substate when doing overview */ +}; + +typedef struct saim_sqlite_cache { + lws_dll2_t list; + char uuid[65]; + sqlite3 *pdb; + lws_usec_t idle_since; + int refcount; +} saim_sqlite_cache_t; + +struct vhd { + struct lws_context *context; + struct lws_vhost *vhost; + + /* pss lists */ + struct lws_dll2_owner browsers; + struct lws_dll2_owner builders; + + const char *sqlite3_path_lhs; + + lws_dll2_owner_t sqlite3_cache; /* saim_sqlite_cache_t */ + lws_dll2_owner_t tasklog_cache; + lws_sorted_usec_list_t sul_logcache; + lws_sorted_usec_list_t sul_central; /* background task allocation sul */ + + lws_usec_t last_check_abandoned_tasks; + + const char *notification_key; + + saim_t master; +}; + +extern struct lws_context * +sai_lws_context_from_json(const char *config_dir, + struct lws_context_creation_info *info, + const struct lws_protocols **pprotocols); +extern const struct lws_protocols protocol_ws; + +int +sai_notification_file_upload_cb(void *data, const char *name, + const char *filename, char *buf, int len, + enum lws_spa_fileupload_states state); + +int +sai_sqlite3_statement(sqlite3 *pdb, const char *cmd, const char *desc); + +int +sai_uuid16_create(struct lws_context *context, char *dest33); + +int +saim_event_db_ensure_open(struct vhd *vhd, const char *event_uuid, char can_create, sqlite3 **ppdb); + +void +saim_event_db_close(struct vhd *vhd, sqlite3 **ppdb); + +int +saim_event_db_delete_database(struct vhd *vhd, const char *event_uuid); + +int +sai_sq3_event_lookup(sqlite3 *pdb, uint64_t start, lws_struct_args_cb cb, void *ca); + +int +sai_sql3_get_uint64_cb(void *user, int cols, char **values, char **name); + +int +saim_ws_json_tx_browser(struct vhd *vhd, struct pss *pss, uint8_t *buf, size_t bl); + +int +saim_ws_json_rx_builder(struct vhd *vhd, struct pss *pss, uint8_t *buf, size_t bl); + +int +lws_struct_map_set(const lws_struct_map_t *map, char *u); + +int +saim_ws_json_rx_browser(struct vhd *vhd, struct pss *pss, + uint8_t *buf, size_t bl); + +void +sai_task_uuid_to_event_uuid(char *event_uuid33, const char *task_uuid65); + +int +saim_ws_json_tx_builder(struct vhd *vhd, struct pss *pss, uint8_t *buf, size_t bl); + +void +mark_pending(struct pss *pss, ws_state state); + +int +saim_subs_request_writeable(struct vhd *vhd, const char *task_uuid); + +void +saim_central_cb(lws_sorted_usec_list_t *sul); + +int +saim_task_reset(struct vhd *vhd, const char *task_uuid); + +int +saim_task_cancel(struct vhd *vhd, const char *task_uuid); + +int +saim_allocate_task(struct vhd *vhd, struct pss *pss, sai_plat_t *cb, + const char *cns_name); + +int +saim_set_task_state(struct vhd *vhd, const char *builder_name, + const char *builder_uuid, const char *task_uuid, int state, + uint64_t started, uint64_t duration); + +int +saim_get_blob(struct vhd *vhd, const char *url, sqlite3 **pdb, + sqlite3_blob **blob, uint64_t *length); diff --git a/src/master/m-sai.c b/src/master/m-sai.c new file mode 100644 index 0000000..5e753d6 --- /dev/null +++ b/src/master/m-sai.c @@ -0,0 +1,69 @@ +/* + * Sai master + * + * Copyright (C) 2019 - 2020 Andy Green + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation: + * version 2.1 of the License. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + * MA 02110-1301 USA + */ + +#include +#include +#include +#include + +#include "m-private.h" + +static int interrupted; +struct lws_context *context; + +static const struct lws_protocols + *pprotocols[] = { &protocol_ws, NULL }; + +static void sigint_handler(int sig) +{ + interrupted = 1; +} + +int main(int argc, const char **argv) +{ + int logs = LLL_USER | LLL_ERR | LLL_WARN | LLL_NOTICE; + const char *p, *conf = "/etc/sai/master"; + struct lws_context_creation_info info; + + signal(SIGINT, sigint_handler); + + if ((p = lws_cmdline_option(argc, argv, "-d"))) + logs = atoi(p); + + lws_set_log_level(logs, NULL); + lwsl_user("Sai Master - Copyright (C) 2019-2020 Andy Green \n"); + + if ((p = lws_cmdline_option(argc, argv, "-c"))) + conf = p; + + context = sai_lws_context_from_json(conf, &info, pprotocols); + if (!context) { + lwsl_err("lws init failed\n"); + return 1; + } + + while (!lws_service(context, 0) && !interrupted) + ; + + lws_context_destroy(context); + + return 0; +} diff --git a/src/master/m-task.c b/src/master/m-task.c new file mode 100644 index 0000000..76ff2ae --- /dev/null +++ b/src/master/m-task.c @@ -0,0 +1,467 @@ +/* + * Sai master + * + * Copyright (C) 2019 - 2020 Andy Green + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation: + * version 2.1 of the License. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + * MA 02110-1301 USA + */ + +#include +#include +#include +#include +#include + +#include "m-private.h" + +static int +browser_upd(struct lws_dll2 *d, void *user) +{ + struct pss *pss = lws_container_of(d, struct pss, same); + ws_state state = (ws_state)(intptr_t)user; + + mark_pending(pss, state); + + return 0; +} + +static int +sql3_get_integer_cb(void *user, int cols, char **values, char **name) +{ + unsigned int *pui = (unsigned int *)user; + + lwsl_warn("%s: values[0] '%s'\n", __func__, values[0]); + *pui = atoi(values[0]); + + return 0; +} + +void +sai_task_uuid_to_event_uuid(char *event_uuid33, const char *task_uuid65) +{ + memcpy(event_uuid33, task_uuid65, 32); + event_uuid33[32] = '\0'; +} + +int +saim_set_task_state(struct vhd *vhd, const char *builder_name, + const char *builder_uuid, const char *task_uuid, int state, + uint64_t started, uint64_t duration) +{ + char update[384], esc[96], esc1[96], esc2[96], esc3[32], esc4[32], + event_uuid[33]; + unsigned int count = 0, count_good = 0, count_bad = 0; + sai_event_state_t oes, sta; + struct lwsac *ac = NULL; + sai_event_t *e = NULL; + lws_dll2_owner_t o; + int n; + + /* + * Extract the event uuid from the task uuid + */ + + sai_task_uuid_to_event_uuid(event_uuid, task_uuid); + + /* + * Look up the task's event in the event database... + */ + + lws_dll2_owner_clear(&o); + lws_sql_purify(esc1, event_uuid, sizeof(esc1)); + lws_snprintf(esc2, sizeof(esc2), " and uuid='%s'", esc1); + n = lws_struct_sq3_deserialize(vhd->master.pdb, esc2, NULL, + lsm_schema_sq3_map_event, &o, &ac, 0, 1); + if (n < 0 || !o.head) { + lwsl_err("%s: failed to get task_uuid %s\n", __func__, esc1); + goto bail; + } + + e = lws_container_of(o.head, sai_event_t, list); + oes = e->state; + + /* + * Open the event-specific database on the temporary event object + */ + + if (saim_event_db_ensure_open(vhd, event_uuid, 0, (sqlite3 **)&e->pdb)) { + lwsl_err("%s: unable to open event-specific database\n", + __func__); + + return -1; + } + + if (builder_name) + lws_sql_purify(esc, builder_name, sizeof(esc)); + else + esc[0] = '\0'; + + if (builder_uuid) + lws_sql_purify(esc1, builder_uuid, sizeof(esc1)); + else + esc1[0] = '\0'; + lws_sql_purify(esc2, task_uuid, sizeof(esc2)); + + esc3[0] = esc4[0] = '\0'; + + if (started) { + if (started == 1) + started = 0; + lws_snprintf(esc3, sizeof(esc3), ",started=%llu", + (unsigned long long)started); + } + if (duration) { + if (duration == 1) + duration = 0; + lws_snprintf(esc4, sizeof(esc4), ",duration=%llu", + (unsigned long long)duration); + } + + /* + * Update the task by uuid, in the event-specific database + */ + + lws_snprintf(update, sizeof(update), + "update tasks set state=%d%s%s%s%s%s%s%s%s where uuid='%s'", + state, builder_uuid ? ",builder='": "", + builder_uuid ? esc1 : "", + builder_uuid ? "'" : "", + builder_name ? ",builder_name='" : "", + builder_name ? esc : "", + builder_name ? "'" : "", + esc3, esc4, esc2); + + if (sqlite3_exec((sqlite3 *)e->pdb, update, NULL, NULL, NULL) != SQLITE_OK) { + lwsl_err("%s: %s: %s: fail\n", __func__, update, + sqlite3_errmsg(vhd->master.pdb)); + goto bail; + } + + /* + * So, how many tasks for this event? + */ + + if (sqlite3_exec((sqlite3 *)e->pdb, "select count(state) from tasks", + sql3_get_integer_cb, &count, NULL) != SQLITE_OK) { + lwsl_err("%s: %s: %s: fail\n", __func__, update, + sqlite3_errmsg(vhd->master.pdb)); + goto bail; + } + + /* + * ... how many completed well? + */ + + if (sqlite3_exec((sqlite3 *)e->pdb, "select count(state) from tasks where state == 3", + sql3_get_integer_cb, &count_good, NULL) != SQLITE_OK) { + lwsl_err("%s: %s: %s: fail\n", __func__, update, + sqlite3_errmsg(vhd->master.pdb)); + goto bail; + } + + /* + * ... how many failed? + */ + + if (sqlite3_exec((sqlite3 *)e->pdb, "select count(state) from tasks where state == 4", + sql3_get_integer_cb, &count_bad, NULL) != SQLITE_OK) { + lwsl_err("%s: %s: %s: fail\n", __func__, update, + sqlite3_errmsg(vhd->master.pdb)); + goto bail; + } + + /* + * Decide how to set the event state based on that + */ + + lwsl_warn("%s: ev %s, count %u, count_good %u, count_bad %u\n", + __func__, event_uuid, count, count_good, count_bad); + + sta = SAIES_BEING_BUILT; + + if (count) { + if (count == count_good) + sta = SAIES_SUCCESS; + else + if (count == count_bad) + sta = SAIES_FAIL; + else + if (count_bad) + sta = SAIES_BEING_BUILT_HAS_FAILURES; + } + + if (sta != oes) { + lwsl_notice("%s: event state changed\n", __func__); + } + + /* + * Update the event + */ + + lws_sql_purify(esc1, event_uuid, sizeof(esc1)); + lws_snprintf(update, sizeof(update), + "update events set state=%d where uuid='%s'", sta, esc1); + + if (sqlite3_exec(vhd->master.pdb, update, NULL, NULL, NULL) != SQLITE_OK) { + lwsl_err("%s: %s: %s: fail\n", __func__, update, + sqlite3_errmsg(vhd->master.pdb)); + goto bail; + } + + saim_event_db_close(vhd, (sqlite3 **)&e->pdb); + + /* + * We need to schedule sending an event info update to anyone + * subscribing to the task... + */ + + lws_dll2_foreach_safe(&vhd->browsers, + (void *)WSS_PREPARE_OVERVIEW, browser_upd); + + lwsac_free(&ac); + + return 0; + +bail: + if (e) + saim_event_db_close(vhd, (sqlite3 **)&e->pdb); + lwsac_free(&ac); + + return 1; +} + +/* + * Find the most recent task that still needs doing for platform, on any event + * + * If any, the task pointed-to lives inside *pac, along with its strings etc + */ + +static const sai_task_t * +saim_task_pending(struct vhd *vhd, struct lwsac **pac, const char *platform) +{ + struct lwsac *ac = NULL; + char esc[96], pf[96]; + lws_dll2_owner_t o; + int n; + + lws_sql_purify(esc, platform, sizeof(esc)); + + assert(platform); + assert(pac); + + /* + * Collect a list of events that still have any open tasks + * + * We don't put this list in the pac since we can dispose of it in this + * scope whether we find something or not + */ + + n = lws_struct_sq3_deserialize(vhd->master.pdb, + " and (state != 3 and state != 4 and state != 5)", + "created desc ", lsm_schema_sq3_map_event, &o, + &ac, 0, 10); + if (n < 0 || !o.head) { + // lwsl_notice("%s: all events complete\n", __func__); + /* error, or there are no events that aren't complete */ + goto bail; + } + + /* + * Iterate through the events looking at his event-specific database + * for tasks on the specified platform... + */ + + lws_start_foreach_dll(struct lws_dll2 *, p, o.head) { + sai_event_t *e = lws_container_of(p, sai_event_t, list); + lws_dll2_owner_t ot; + sqlite3 *pdb = NULL; + + if (!saim_event_db_ensure_open(vhd, e->uuid, 0, &pdb)) { + lws_snprintf(pf, sizeof(pf), + " and state=0 and platform='%s'", esc); + n = lws_struct_sq3_deserialize(pdb, pf, NULL, + lsm_schema_sq3_map_task, + &ot, pac, 0, 1); + saim_event_db_close(vhd, &pdb); + if (ot.count) { + + lwsl_notice("%s: found task for %s\n", __func__, esc); + + lwsac_free(&ac); + + return lws_container_of(ot.head, + sai_task_t, list); + } + } + + } lws_end_foreach_dll(p); + + lwsl_debug("%s: no free tasks matching '%s'\n", __func__, esc); + +bail: + lwsac_free(&ac); + + return NULL; +} + +int +saim_task_cancel(struct vhd *vhd, const char *task_uuid) +{ + sai_cancel_t *can; + + /* + * For every pss that we have from builders... + */ + lws_start_foreach_dll(struct lws_dll2 *, p, vhd->builders.head) { + struct pss *pss = lws_container_of(p, struct pss, same); + + + /* + * ... queue the task cancel message + */ + can = malloc(sizeof *can); + if (!can) + return -1; + memset(can, 0, sizeof(*can)); + + strncpy(can->task_uuid, task_uuid, sizeof(can->task_uuid)); + + lws_dll2_add_tail(&can->list, &pss->task_cancel_owner); + + lws_callback_on_writable(pss->wsi); + + } lws_end_foreach_dll(p); + + return 0; +} + +/* + * Keep the task record itself, but remove all logs and artifacts related to + * it and reset the task state back to WAITING. + */ + +int +saim_task_reset(struct vhd *vhd, const char *task_uuid) +{ + char esc[96], cmd[256], event_uuid[33]; + sqlite3 *pdb = NULL; + + if (!task_uuid[0]) + return 0; + + lwsl_notice("%s: received request to reset task %s\n", __func__, task_uuid); + + sai_task_uuid_to_event_uuid(event_uuid, task_uuid); + + if (saim_event_db_ensure_open(vhd, event_uuid, 0, &pdb)) { + lwsl_err("%s: unable to open event-specific database\n", + __func__); + + return -1; + } + + lws_sql_purify(esc, task_uuid, sizeof(esc)); + lws_snprintf(cmd, sizeof(cmd), "delete from logs where task_uuid='%s'", + esc); + + if (sqlite3_exec(pdb, cmd, NULL, NULL, NULL) != SQLITE_OK) { + saim_event_db_close(vhd, &pdb); + lwsl_err("%s: %s: %s: fail\n", __func__, cmd, + sqlite3_errmsg(vhd->master.pdb)); + return 1; + } + lws_snprintf(cmd, sizeof(cmd), "delete from artifacts where task_uuid='%s'", + esc); + + sqlite3_exec(pdb, cmd, NULL, NULL, NULL); + saim_event_db_close(vhd, &pdb); + + saim_set_task_state(vhd, NULL, NULL, task_uuid, SAIES_WAITING, 1, 1); + + saim_task_cancel(vhd, task_uuid); + + /* + * Reassess now if there's a builder we can match to a pending task + */ + + lws_sul_schedule(vhd->context, 0, &vhd->sul_central, saim_central_cb, 1); + + return 0; +} + +/* + * Look for any task on any event that needs building on platform_name + */ + +int +saim_allocate_task(struct vhd *vhd, struct pss *pss, sai_plat_t *cb, + const char *platform_name) +{ + char esc1[96], esc2[96]; + lws_dll2_owner_t o; + sai_task_t *task; + int n; + + if (cb->ongoing >= cb->instances) + return 1; + + /* + * Look for a task for this platform, on any event that needs building + */ + + task = (sai_task_t *)saim_task_pending(vhd, &pss->a.ac, platform_name); + + if (!task) + return 1; + + lwsl_notice("%s: %s: task found %s\n", __func__, platform_name, cb->name); + + /* yes, we will offer it to him */ + + if (saim_set_task_state(vhd, cb->name, cb->name, task->uuid, + SAIES_PASSED_TO_BUILDER, lws_now_secs(), 0)) + return -1; + + /* advance the task state first time we get logs */ + pss->mark_started = 1; + + /* let's get ahold of his event as well */ + + lws_sql_purify(esc1, task->event_uuid, sizeof(esc1)); + lws_snprintf(esc2, sizeof(esc2), " and uuid='%s'", esc1); + n = lws_struct_sq3_deserialize(vhd->master.pdb, esc2, NULL, + lsm_schema_sq3_map_event, &o, &pss->a.ac, + 0, 1); + if (n < 0 || !o.head) + return -1; + + task->one_event = lws_container_of(o.head, sai_event_t, list); + + task->master_name = pss->master_name; + task->repo_name = task->one_event->repo_name; + task->git_ref = task->one_event->ref; + task->git_hash = task->one_event->hash; + + /* + * add to master's estimate of builder's ongoing tasks... + */ + cb->ongoing++; + + lws_dll2_add_tail(&task->pending_assign_list, &pss->issue_task_owner); + lws_callback_on_writable(pss->wsi); + + return 0; +} diff --git a/src/master/m-ws-browser.c b/src/master/m-ws-browser.c new file mode 100644 index 0000000..cf16293 --- /dev/null +++ b/src/master/m-ws-browser.c @@ -0,0 +1,1077 @@ +/* + * Sai master - ./src/master/m-ws-browser.c + * + * Copyright (C) 2019 - 2020 Andy Green + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation: + * version 2.1 of the License. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + * MA 02110-1301 USA + * + * These are ws rx and tx handlers related to browser ws connections, on + * /broswe URLs. + */ + +#include +#include +#include +#include +#include + +#include "m-private.h" + +/* + * For decoding specific event data request from browser + */ + + +typedef struct sai_browse_rx_evinfo { + char event_hash[65]; +} sai_browse_rx_evinfo_t; + +static lws_struct_map_t lsm_browser_evinfo[] = { + LSM_CARRAY (sai_browse_rx_evinfo_t, event_hash, "event_hash"), +}; + + +/* + * For decoding specific task data request from browser + */ + +typedef struct sai_browse_rx_taskinfo { + char task_hash[65]; + unsigned int log_start; + uint8_t logs; +} sai_browse_rx_taskinfo_t; + +static lws_struct_map_t lsm_browser_taskinfo[] = { + LSM_CARRAY (sai_browse_rx_taskinfo_t, task_hash, "task_hash"), + LSM_UNSIGNED (sai_browse_rx_taskinfo_t, logs, "logs"), +}; + +static lws_struct_map_t lsm_browser_taskreset[] = { + LSM_CARRAY (sai_browse_rx_evinfo_t, event_hash, "uuid"), + +}; + +/* + * Schema list so lws_struct can pick the right object to create based on the + * incoming schema name + */ + +static const lws_struct_map_t lsm_schema_json_map_bwsrx[] = { + LSM_SCHEMA (sai_browse_rx_taskinfo_t, NULL, lsm_browser_taskinfo, + "com.warmcat.sai.taskinfo"), + LSM_SCHEMA (sai_browse_rx_evinfo_t, NULL, lsm_browser_evinfo, + "com.warmcat.sai.eventinfo"), + LSM_SCHEMA (sai_browse_rx_evinfo_t, NULL, lsm_browser_taskreset, + /* shares struct */ "com.warmcat.sai.taskreset"), + LSM_SCHEMA (sai_browse_rx_evinfo_t, NULL, lsm_browser_taskreset, + /* shares struct */ "com.warmcat.sai.eventreset"), + LSM_SCHEMA (sai_browse_rx_evinfo_t, NULL, lsm_browser_taskreset, + /* shares struct */ "com.warmcat.sai.eventdelete"), + LSM_SCHEMA (sai_cancel_t, NULL, lsm_task_cancel, + "com.warmcat.sai.taskcan"), +}; + +enum { + SAIM_WS_BROWSER_RX_TASKINFO, + SAIM_WS_BROWSER_RX_EVENTINFO, + SAIM_WS_BROWSER_RX_TASKRESET, + SAIM_WS_BROWSER_RX_EVENTRESET, + SAIM_WS_BROWSER_RX_EVENTDELETE, + SAIM_WS_BROWSER_RX_TASKCANCEL +}; + + +/* + * For issuing combined task and event data back to browser + */ + +typedef struct sai_browse_taskreply { + const sai_event_t *event; + const sai_task_t *task; +} sai_browse_taskreply_t; + +static lws_struct_map_t lsm_taskreply[] = { + LSM_CHILD_PTR (sai_browse_taskreply_t, event, sai_event_t, NULL, + lsm_event, "e"), + LSM_CHILD_PTR (sai_browse_taskreply_t, task, sai_task_t, NULL, + lsm_task, "t"), +}; + +const lws_struct_map_t lsm_schema_json_map_taskreply[] = { + LSM_SCHEMA (sai_browse_taskreply_t, NULL, lsm_taskreply, + "com.warmcat.sai.taskinfo"), +}; + +enum sai_overview_state { + SOS_EVENT, + SOS_TASKS, +}; + +/* + * Ask for writeable cb on all browser connections subscribed to a particular + * task (so we can send them some more logs) + */ + +int +saim_subs_request_writeable(struct vhd *vhd, const char *task_uuid) +{ + lws_start_foreach_dll(struct lws_dll2 *, p, + vhd->master.subs_owner.head) { + struct pss *pss = lws_container_of(p, struct pss, subs_list); + + if (!strcmp(pss->sub_task_uuid, task_uuid)) + lws_callback_on_writable(pss->wsi); + + } lws_end_foreach_dll(p); + + return 0; +} + +static int +saim_pss_schedule_eventinfo(struct pss *pss, const char *event_uuid) +{ + char qu[80], esc[66]; + int n; + + /* + * Just collect the event struct into pss->query_owner to dump + */ + + lws_sql_purify(esc, event_uuid, sizeof(esc)); + lws_snprintf(qu, sizeof(qu), " and uuid='%s'", esc); + n = lws_struct_sq3_deserialize(pss->vhd->master.pdb, qu, NULL, + lsm_schema_sq3_map_event, + &pss->query_owner, &pss->query_ac, 0, 1); + if (n < 0 || !pss->query_owner.head) + return 1; + + pss->query_already_done = 1; + + mark_pending(pss, WSS_PREPARE_OVERVIEW); + mark_pending(pss, WSS_PREPARE_BUILDER_SUMMARY); + + return 0; +} + +static int +saim_pss_schedule_taskinfo(struct pss *pss, const char *task_uuid, int logsub) +{ + char qu[80], esc[66], event_uuid[33]; + sqlite3 *pdb = NULL; + lws_dll2_owner_t o; + int n; + + sai_task_uuid_to_event_uuid(event_uuid, task_uuid); + + /* open the event-specific database object */ + + if (saim_event_db_ensure_open(pss->vhd, event_uuid, 0, &pdb)) + /* no longer exists, nothing to do */ + return 0; + + /* + * get the related task object into its own ac... there might + * be a lot of related data, so we hold the ac in the pss for + * as long as needed to send it out + */ + + lws_sql_purify(esc, task_uuid, sizeof(esc)); + lws_snprintf(qu, sizeof(qu), " and uuid='%s'", esc); + n = lws_struct_sq3_deserialize(pdb, qu, NULL, lsm_schema_sq3_map_task, + &o, &pss->query_ac, 0, 1); + saim_event_db_close(pss->vhd, &pdb); + lwsl_info("%s: n %d, o.head %p\n", __func__, n, o.head); + if (n < 0 || !o.head) + return 1; + + pss->one_task = lws_container_of(o.head, sai_task_t, list); + + lwsl_info("%s: browser ws asked for task hash: %s, plat %s\n", + __func__, task_uuid, pss->one_task->platform); + + /* let the pss take over the task info ac and schedule sending */ + + lws_dll2_remove((struct lws_dll2 *)&pss->one_task->list); + + /* + * let's also get the event object the task relates to into + * its own event struct + */ + + lws_sql_purify(esc, event_uuid, sizeof(esc)); + lws_snprintf(qu, sizeof(qu), " and uuid='%s'", esc); + n = lws_struct_sq3_deserialize(pss->vhd->master.pdb, qu, NULL, + lsm_schema_sq3_map_event, &o, + &pss->query_ac, 0, 1); + if (n < 0 || !o.head) + return 1; + + /* does he want to subscribe to logs? */ + if (logsub) { + strcpy(pss->sub_task_uuid, pss->one_task->uuid); + lws_dll2_add_head(&pss->subs_list, + &pss->vhd->master.subs_owner); + pss->sub_timestamp = 0; /* where we got up to */ + lws_callback_on_writable(pss->wsi); + lwsl_info("%s: subscribed to logs for %s\n", __func__, + pss->sub_task_uuid); + } + + pss->one_event = lws_container_of(o.head, sai_event_t, list); + + mark_pending(pss, WSS_PREPARE_TASKINFO); + mark_pending(pss, WSS_PREPARE_BUILDER_SUMMARY); + + return 0; +} + +/* + * We need to schedule re-sending out task and event state to anyone subscribed + * to the task that changed or its associated event + */ + +int +saim_subs_task_state_change(struct vhd *vhd, const char *task_uuid) +{ + lws_start_foreach_dll(struct lws_dll2 *, p, + vhd->master.subs_owner.head) { + struct pss *pss = lws_container_of(p, struct pss, subs_list); + + if (!strcmp(pss->sub_task_uuid, task_uuid)) + saim_pss_schedule_taskinfo(pss, task_uuid, 0); + + } lws_end_foreach_dll(p); + + return 0; +} + + +/* + * browser has sent us a request for either overview, or data on a specific + * task + */ + +int +saim_ws_json_rx_browser(struct vhd *vhd, struct pss *pss, uint8_t *buf, + size_t bl) +{ + sai_browse_rx_taskinfo_t *ti; + char qu[128], esc[96], *err; + sai_browse_rx_evinfo_t *ei; + sqlite3 *pdb = NULL; + lws_struct_args_t a; + lws_dll2_owner_t o; + sai_cancel_t *can; + int m, ret = -1; + + memset(&a, 0, sizeof(a)); + a.map_st[0] = lsm_schema_json_map_bwsrx; + a.map_entries_st[0] = LWS_ARRAY_SIZE(lsm_schema_json_map_bwsrx); + a.map_entries_st[1] = LWS_ARRAY_SIZE(lsm_schema_json_map_bwsrx); + a.ac_block_size = 128; + + lws_struct_json_init_parse(&pss->ctx, NULL, &a); + m = lejp_parse(&pss->ctx, (uint8_t *)buf, bl); + if (m < 0 || !a.dest) { + lwsl_hexdump_notice(buf, bl); + lwsl_notice("%s: notification JSON decode failed '%s'\n", + __func__, lejp_error_to_string(m)); + return m; + } + + /* + * Which object we ended up with depends on the schema that came in... + * a.top_schema_index is the index in lsm_schema_json_map_bwsrx it + * matched on + */ + + switch (a.top_schema_index) { + + case SAIM_WS_BROWSER_RX_TASKINFO: + ti = (sai_browse_rx_taskinfo_t *)a.dest; + + lwsl_info("%s: schema index %d, task hash %s\n", __func__, + a.top_schema_index, ti->task_hash); + + if (!ti->task_hash[0]) { + /* + * he's asking for the overview schema + */ + lws_dll2_add_head(&pss->same, &vhd->browsers); + mark_pending(pss, WSS_PREPARE_OVERVIEW); + mark_pending(pss, WSS_PREPARE_BUILDER_SUMMARY); + ret = 0; + goto bail; + } + + /* + * get the related task object into its own ac... there might + * be a lot of related data, so we hold the ac in the pss for + * as long as needed to send it out + */ + + if (saim_pss_schedule_taskinfo(pss, ti->task_hash, !!ti->logs)) + goto soft_error; + + return 0; + + case SAIM_WS_BROWSER_RX_EVENTINFO: + + ei = (sai_browse_rx_evinfo_t *)a.dest; + + if (saim_pss_schedule_eventinfo(pss, ei->event_hash)) + goto soft_error; + + lwsac_free(&a.ac); + + return 0; + + case SAIM_WS_BROWSER_RX_TASKRESET: + + /* + * User is asking us to reset / rebuild this task + */ + + ei = (sai_browse_rx_evinfo_t *)a.dest; + + if (saim_task_reset(vhd, ei->event_hash)) + goto soft_error; + + if (saim_pss_schedule_taskinfo(pss, ei->event_hash, 0)) + goto soft_error; + + lwsac_free(&a.ac); + + return 0; + + case SAIM_WS_BROWSER_RX_EVENTRESET: + + /* + * User is asking us to reset / rebuild every task in the event + */ + + ei = (sai_browse_rx_evinfo_t *)a.dest; + + lwsl_notice("%s: received request to reset event %s\n", + __func__, ei->event_hash); + + /* open the event-specific database object */ + + if (saim_event_db_ensure_open(pss->vhd, ei->event_hash, 0, &pdb)) { + lwsl_err("%s: unable to open event-specific database\n", + __func__); + /* + * hanging up isn't a good way to deal with browser + * tabs left open with a live connection to a + * now-deleted task... the page will reconnect endlessly + */ + goto soft_error; + } + + /* + * Retreive all the related structs into a dll2 list + */ + + lws_sql_purify(esc, ei->event_hash, sizeof(esc)); + + if (lws_struct_sq3_deserialize(pdb, NULL, NULL, + lsm_schema_sq3_map_task, + &o, &a.ac, 0, 999) >= 0) { + + sqlite3_exec(vhd->master.pdb, "BEGIN TRANSACTION", + NULL, NULL, &err); + + /* + * Walk the results list resetting all the tasks + */ + + lws_start_foreach_dll(struct lws_dll2 *, p, o.head) { + sai_task_t *t = lws_container_of(p, sai_task_t, + list); + + saim_task_reset(vhd, t->uuid); + + } lws_end_foreach_dll(p); + + sqlite3_exec(vhd->master.pdb, "END TRANSACTION", + NULL, NULL, &err); + } + + saim_event_db_close(vhd, &pdb); + lwsac_free(&a.ac); + + return 0; + + case SAIM_WS_BROWSER_RX_EVENTDELETE: + /* + * User is asking us to delete the whole event + */ + + ei = (sai_browse_rx_evinfo_t *)a.dest; + + lwsl_notice("%s: received request to delete event %s\n", + __func__, ei->event_hash); + + /* open the event-specific database object */ + + if (saim_event_db_ensure_open(pss->vhd, ei->event_hash, 0, &pdb)) { + lwsl_notice("%s: unable to open event-specific database\n", + __func__); + /* + * hanging up isn't a good way to deal with browser + * tabs left open with a live connection to a + * now-deleted task... the page will reconnect endlessly + */ + + goto soft_error; + } + + /* + * Retreive all the related structs into a dll2 list + */ + + lws_sql_purify(esc, ei->event_hash, sizeof(esc)); + + if (lws_struct_sq3_deserialize(pdb, NULL, NULL, + lsm_schema_sq3_map_task, + &o, &a.ac, 0, 999) >= 0) { + + sqlite3_exec(vhd->master.pdb, "BEGIN TRANSACTION", + NULL, NULL, &err); + + /* + * Walk the results list cancelling all the tasks + * that look like they might be ongoing + */ + + lws_start_foreach_dll(struct lws_dll2 *, p, o.head) { + sai_task_t *t = lws_container_of(p, sai_task_t, + list); + + if (t->state != SAIES_WAITING && + t->state != SAIES_SUCCESS && + t->state != SAIES_FAIL && + t->state != SAIES_CANCELLED) + saim_task_cancel(vhd, t->uuid); + + } lws_end_foreach_dll(p); + + sqlite3_exec(vhd->master.pdb, "END TRANSACTION", + NULL, NULL, &err); + } + + saim_event_db_close(vhd, &pdb); + + /* delete the event iself */ + + lws_sql_purify(esc, ei->event_hash, sizeof(esc)); + lws_snprintf(qu, sizeof(qu), "delete from events where uuid='%s'", + esc); + sqlite3_exec(vhd->master.pdb, qu, NULL, NULL, &err); + + /* remove the event-specific database */ + + saim_event_db_delete_database(vhd, ei->event_hash); + + ret = 0; + lwsac_free(&a.ac); + mark_pending(pss, WSS_PREPARE_OVERVIEW); + mark_pending(pss, WSS_PREPARE_BUILDER_SUMMARY); + + goto bail;; + + case SAIM_WS_BROWSER_RX_TASKCANCEL: + /* + * Browser is informing us of task's STOP button clicked, we + * need to inform any builder that might be building it + */ + can = (sai_cancel_t *)a.dest; + + lwsl_notice("%s: received request to cancel task %s\n", + __func__, can->task_uuid); + + saim_task_cancel(vhd, can->task_uuid); + break; + + default: + assert(0); + break; + } + + ret = 0; + +bail: + lwsac_free(&a.ac); + + return ret; + +soft_error: + lwsac_free(&a.ac); + + return 0; +} + + +/* + * We're sending something on a browser ws connection. Returning nonzero from + * here drops the connection, necessary if we fail partway through a message + * but undesirable if a browser tab will keep reconnecting and asking for the + * same, no-longer-existant thing. + */ + +int +saim_ws_json_tx_browser(struct vhd *vhd, struct pss *pss, uint8_t *buf, size_t bl) +{ + uint8_t *start = buf + LWS_PRE, *p = start, *end = p + bl - LWS_PRE - 1; + int n, flags = LWS_WRITE_TEXT, first = 0, iu; + sai_browse_taskreply_t task_reply; + lws_dll2_owner_t task_owner; + lws_struct_serialize_t *js; + char esc[256], filt[128]; + char event_uuid[33]; + sqlite3 *pdb = NULL; + sai_event_t *e; + sai_task_t *t; + char any; + size_t w; + +again: + + start = buf + LWS_PRE; + p = start; + end = p + bl - LWS_PRE - 1; + flags = LWS_WRITE_TEXT; + first = 0; + + // lwsl_notice("%s: send_state %d, pss %p, wsi %p\n", __func__, + // pss->send_state, pss, pss->wsi); + + switch (pss->send_state) { + case WSS_IDLE: + + /* + * Anything from a task log he's subscribed to? + */ + + if (pss->subs_list.owner) { + + lws_snprintf(esc, sizeof(esc), + "and task_uuid='%s' and timestamp > %llu", + pss->sub_task_uuid, + (unsigned long long)pss->sub_timestamp); + + /* + * For efficiency, let's try to grab the next 100 at + * once from sqlite and work our way through sending + * them + */ + + if (pss->log_cache_index == pss->log_cache_size) { + int sr; + + sai_task_uuid_to_event_uuid(event_uuid, + pss->sub_task_uuid); + + lws_dll2_owner_clear(&task_owner); + lwsac_free(&pss->logs_ac); + + lwsl_info("%s: collecting logs %s\n", + __func__, esc); + + if (saim_event_db_ensure_open(vhd, event_uuid, 0, + &pdb)) { + lwsl_notice("%s: unable to open event-specific database\n", + __func__); + + return 0; + } + + sr = lws_struct_sq3_deserialize(pdb, esc, + "uid,timestamp ", + lsm_schema_sq3_map_log, + &pss->query_owner, + &pss->logs_ac, 0, 100); + + saim_event_db_close(vhd, &pdb); + + if (sr) { + + lwsl_err("%s: subs failed\n", __func__); + + return 0; + } + + pss->log_cache_index = 0; + pss->log_cache_size = pss->query_owner.count; + } + + if (pss->log_cache_index < pss->log_cache_size) { + sai_log_t *log = lws_container_of( + pss->query_owner.head, + sai_log_t, list); + + lws_dll2_remove(&log->list); + pss->log_cache_index++; + + /* + * Turn it back into JSON so we can give it to + * the browser + */ + + js = lws_struct_json_serialize_create( + lsm_schema_json_map_log, 1, 0, log); + if (!js) { + lwsl_notice("%s: json ser fail\n", __func__); + return 0; + } + + n = lws_struct_json_serialize(js, p, end - p, &w); + lws_struct_json_serialize_destroy(&js); + if (n == LSJS_RESULT_ERROR) { + lwsl_notice("%s: json ser error\n", __func__); + return 0; + } + + p += w; + first = 1; + pss->walk = NULL; + + /* + * Record that this was the most recent log we + * saw so far + */ + pss->sub_timestamp = log->timestamp; + goto send_it; + } + } + + if (!pss->pending) + return 0; + + for (n = 0; n < 31; n++) { + if (pss->pending & (1 << n)) { + pss->pending &= ~(1 << n); + pss->send_state = n; + goto again; + } + } + return 0; + + case WSS_PREPARE_OVERVIEW: + + filt[0] = '\0'; + if (pss->specific_project[0]) { + lws_sql_purify(esc, pss->specific_project, sizeof(esc) - 1); + lws_snprintf(filt, sizeof(filt), " and repo_name=\"%s\"", esc); + } + pss->wants_event_updates = 1; + if (!pss->query_already_done && + lws_struct_sq3_deserialize(vhd->master.pdb, + filt[0] ? filt : NULL, "created ", + lsm_schema_sq3_map_event, &pss->query_owner, + &pss->query_ac, 0, -8)) { + lwsl_notice("%s: OVERVIEW 2 failed\n", __func__); + + pss->send_state = WSS_IDLE; + lwsac_free(&pss->task_ac); + lwsac_free(&pss->query_ac); + + return 0; + } + + pss->query_already_done = 0; + + /* + * we get zero or more sai_event_t laid out in pss->query_ac, + * and listed in pss->query_owner + */ + + lwsl_debug("%s: WSS_PREPARE_OVERVIEW: %d results %p\n", + __func__, pss->query_owner.count, pss->query_ac); + + p += lws_snprintf((char *)p, end - p, + "{\"schema\":\"sai.warmcat.com.overview\"," + " \"alang\":\"%s\"," + "\"overview\":[", + lws_json_purify(esc, pss->alang, sizeof(esc) - 1, &iu)); + + if (pss->specificity) + pss->walk = lws_dll2_get_head(&pss->query_owner); + else + pss->walk = lws_dll2_get_tail(&pss->query_owner); + pss->subsequent = 0; + first = 1; + + pss->send_state = WSS_SEND_OVERVIEW; + + if (!pss->query_owner.count) + goto so_finish; + + /* fallthru */ + + case WSS_SEND_OVERVIEW: + + if (pss->ovstate == SOS_TASKS) + goto enum_tasks; + + any = 0; + while (end - p > 2048 && pss->walk && + pss->send_state == WSS_SEND_OVERVIEW) { + + e = lws_container_of(pss->walk, sai_event_t, list); + + if (pss->specificity) { + //lwsl_notice("%s %s %s\n", e->hash, e->ref, pss->specific); + if (strcmp(e->hash, pss->specific) && + strcmp(e->ref, pss->specific)) { + pss->walk = pss->walk->next; + continue; + } + //lwsl_notice("%s: match\n", __func__); + any = 1; + } + + js = lws_struct_json_serialize_create( + lsm_schema_json_map_event, + LWS_ARRAY_SIZE(lsm_schema_json_map_event), 0, e); + if (!js) { + lwsl_err("%s: json ser fail\n", __func__); + return 1; + } + if (pss->subsequent) + *p++ = ','; + pss->subsequent = 1; + + p += lws_snprintf((char *)p, end - p, "{\"e\":"); + + n = lws_struct_json_serialize(js, p, end - p, &w); + lws_struct_json_serialize_destroy(&js); + switch (n) { + case LSJS_RESULT_ERROR: + pss->send_state = WSS_IDLE; + lwsl_err("%s: json ser error\n", __func__); + return 1; + + case LSJS_RESULT_FINISH: + case LSJS_RESULT_CONTINUE: + p += w; + pss->ovstate = SOS_TASKS; + pss->task_index = 0; + p += lws_snprintf((char *)p, end - p, ", \"t\":["); + goto enum_tasks; + } + } + if (!any) { + pss->send_state = WSS_IDLE; + lwsac_free(&pss->task_ac); + lwsac_free(&pss->query_ac); + + return 0; + } + break; + +enum_tasks: + /* + * Enumerate the tasks associated with this event... we will + * come back here as often as needed to dump all the tasks + */ + + e = lws_container_of(pss->walk, sai_event_t, list); + + do { + + if (saim_event_db_ensure_open(vhd, e->uuid, 0, &pdb)) { + lwsl_err("%s: unable to open event-specific database\n", + __func__); + + break; + } + + lws_dll2_owner_clear(&task_owner); + if (lws_struct_sq3_deserialize(pdb, NULL, NULL, + lsm_schema_sq3_map_task, &task_owner, + &pss->task_ac, pss->task_index, 1)) { + lwsl_err("%s: OVERVIEW 1 failed\n", __func__); + lwsac_free(&pss->task_ac); + saim_event_db_close(vhd, &pdb); + + break; + } + saim_event_db_close(vhd, &pdb); + + if (!task_owner.count) + break; + + if (pss->task_index) + *p++ = ','; + + /* + * We don't want to send everyone the artifact nonces... + * the up nonce is a key for uploading artifacts on to + * this task, it should only be stored in the master db + * and sent to the builder to use. + * + * The down nonce is used in generated links, but still + * you should have to acquire such a link via whatever + * auth rather than be able to cook them up yourself + * from knowing the task uuid. + */ + + t = (sai_task_t *)task_owner.head; + t->art_up_nonce[0] = '\0'; + t->art_down_nonce[0] = '\0'; + + /* only one in it at a time */ + t = lws_container_of(task_owner.head, sai_task_t, list); + + js = lws_struct_json_serialize_create( + lsm_schema_json_map_task, + LWS_ARRAY_SIZE(lsm_schema_json_map_task), 0, t); + + n = lws_struct_json_serialize(js, p, end - p, &w); + lws_struct_json_serialize_destroy(&js); + lwsac_free(&pss->task_ac); + p += w; + + pss->task_index++; + } while ((end - p > 2048) && task_owner.count); + + if (task_owner.count) + /* may be more left to do */ + break; + + /* none left to do, go back up a level */ + + p += lws_snprintf((char *)p, end - p, "]}"); + + pss->ovstate = SOS_EVENT; + if (pss->specificity) + pss->walk = pss->walk->next; + else + pss->walk = pss->walk->prev; + if (!pss->walk || pss->specificity) { + while (pss->walk) + pss->walk = pss->walk->next; + goto so_finish; + } + break; + +so_finish: + p += lws_snprintf((char *)p, end - p, "]}"); + pss->send_state = WSS_IDLE; + lwsac_free(&pss->task_ac); + lwsac_free(&pss->query_ac); + break; + + case WSS_PREPARE_BUILDER_SUMMARY: + p += lws_snprintf((char *)p, end - p, + "{\"schema\":\"com.warmcat.sai.builders\"," + " \"alang\":\"%s\"," + "\"builders\":[", + lws_sql_purify(esc, pss->alang, sizeof(esc) - 1)); + + pss->walk = lws_dll2_get_head(&vhd->master.builder_owner); + pss->subsequent = 0; + pss->send_state = WSS_SEND_BUILDER_SUMMARY; + first = 1; + + /* fallthru */ + + case WSS_SEND_BUILDER_SUMMARY: + if (!pss->walk) + goto b_finish; + + /* + * We're going to send the browser some JSON about all the + * builders / platforms we feel are connected to us + */ + + while (end - p > 512 && pss->walk && + pss->send_state == WSS_SEND_BUILDER_SUMMARY) { + + sai_plat_t *b = lws_container_of(pss->walk, sai_plat_t, + sai_plat_list); + + js = lws_struct_json_serialize_create( + lsm_schema_map_plat_simple, + LWS_ARRAY_SIZE(lsm_schema_map_plat_simple), + 0, b); + if (!js) + return 1; + if (pss->subsequent) + *p++ = ','; + pss->subsequent = 1; + + switch (lws_struct_json_serialize(js, p, end - p, &w)) { + case LSJS_RESULT_ERROR: + lws_struct_json_serialize_destroy(&js); + pss->send_state = WSS_IDLE; + return 1; + case LSJS_RESULT_FINISH: + p += w; + lws_struct_json_serialize_destroy(&js); + pss->walk = pss->walk->next; + if (!pss->walk) + goto b_finish; + break; + + case LSJS_RESULT_CONTINUE: + p += w; + lws_struct_json_serialize_destroy(&js); + pss->walk = pss->walk->next; + if (!pss->walk) + goto b_finish; + break; + } + } + break; +b_finish: + p += lws_snprintf((char *)p, end - p, "]}"); + pss->send_state = WSS_IDLE; +// lwsac_free(&pss->query_ac); + lwsac_free(&pss->task_ac); + break; + + + case WSS_PREPARE_TASKINFO: + /* + * We're sending a browser the specific task info that he + * asked for. + * + * We already got the task struct out of the db in .one_task + * (all in .query_ac) + */ + + task_reply.event = pss->one_event; + task_reply.task = pss->one_task; + + js = lws_struct_json_serialize_create(lsm_schema_json_map_taskreply, + LWS_ARRAY_SIZE(lsm_schema_json_map_taskreply), + 0, &task_reply); + if (!js) + return 1; + + n = lws_struct_json_serialize(js, p, end - p, &w); + lws_struct_json_serialize_destroy(&js); + + /* + * Let's also try to fetch any artifacts into pss->aft_owner... + * no db or no artifacts can also be a normal situation... + */ + + if (pss->one_task) { + + sai_task_uuid_to_event_uuid(event_uuid, pss->one_task->uuid); + + lwsl_notice("%s: ---------------- event uuid '%s'\n", __func__, + event_uuid); + + lws_dll2_owner_clear(&pss->aft_owner); + if (!saim_event_db_ensure_open(vhd, event_uuid, 0, &pdb)) { + + lws_snprintf(filt, sizeof(filt), " and (task_uuid == '%s')", + pss->one_task->uuid); + + lwsl_notice("%s: ---------------- %s\n", __func__, filt); + + if (lws_struct_sq3_deserialize(pdb, filt, NULL, + lsm_schema_sq3_map_artifact, + &pss->aft_owner, + &pss->task_ac, 0, 10)) { + lwsl_err("%s: get afcts failed\n", __func__); + } + saim_event_db_close(vhd, &pdb); + } + } + + first = 1; + pss->walk = NULL; + pss->send_state = WSS_SEND_ARTIFACT_INFO; + if (!pss->aft_owner.head) { + lwsl_notice("%s: ---------------- no artifacts\n", __func__); + /* there's no artifact stuff to do */ + pss->send_state = WSS_IDLE; + lwsac_free(&pss->query_ac); + } + pss->one_task = NULL; + if (n == LSJS_RESULT_ERROR) { + lwsl_notice("%s: taskinfo: error generating json\n", __func__); + return 1; + } + p += w; + if (!lws_ptr_diff(p, start)) { + lwsl_notice("%s: taskinfo: empty json\n", __func__); + return 0; + } + break; + + case WSS_SEND_ARTIFACT_INFO: + if (pss->aft_owner.head) { + sai_artifact_t *aft = (sai_artifact_t *)pss->aft_owner.head; + + lws_dll2_remove(&aft->list); + + /* we don't want to disclose this to browsers */ + aft->artifact_up_nonce[0] = '\0'; + + js = lws_struct_json_serialize_create(lsm_schema_json_map_artifact, + LWS_ARRAY_SIZE(lsm_schema_json_map_artifact), + 0, aft); + if (!js) { + lwsl_err("%s ----------------- failed to render artifact json\n", __func__); + return 1; + } + + n = lws_struct_json_serialize(js, p, end - p, &w); + lws_struct_json_serialize_destroy(&js); + if (n == LSJS_RESULT_ERROR) { + lwsl_notice("%s: taskinfo: ---------- error generating json\n", __func__); + return 1; + } + first = 1; + p += w; + lwsl_warn("%s: --------------------- %.*s\n", __func__, (int)w, start); + } + + if (!pss->aft_owner.head) { + pss->send_state = WSS_IDLE; + lwsac_free(&pss->query_ac); + } + break; + + default: + lwsl_err("%s: pss state %d\n", __func__, pss->send_state); + return 0; + } + +send_it: + flags = lws_write_ws_flags(LWS_WRITE_TEXT, first, !pss->walk); + + if (lws_write(pss->wsi, start, p - start, flags) < 0) + return -1; + + /* + * We get a bad ratio of reads to write when the builder spams us + * with rx... we have to try to clear as much as we can in one go. + */ + + if (!lws_send_pipe_choked(pss->wsi)) + goto again; + + lws_callback_on_writable(pss->wsi); + + return 0; +} diff --git a/src/master/m-ws-builder.c b/src/master/m-ws-builder.c new file mode 100644 index 0000000..0b717d0 --- /dev/null +++ b/src/master/m-ws-builder.c @@ -0,0 +1,728 @@ +/* + * Sai master - ./src/master/ws-json-rx.c + * + * Copyright (C) 2019 - 2020 Andy Green + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation: + * version 2.1 of the License. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + * MA 02110-1301 USA + * + * These are ws rx and tx handlers related to builder ws connections, on + * /builder + */ + +#include +#include +#include +#include + +#include "m-private.h" + +enum sai_overview_state { + SOS_EVENT, + SOS_TASKS, +}; + +typedef struct saim_logcache_pertask { + lws_dll2_t list; /* vhd->tasklog_cache is the owner */ + char uuid[65]; + lws_dll2_owner_t cache; /* sai_log_t */ +} saim_logcache_pertask_t; + +/* + * The Schema that may be sent to us by a builder + * + * Artifacts are sent on secondary SS connections so they don't block ongoing + * log delivery etc. The JSON is immediately followed by binary data to the + * length told in the JSON. + */ + +static const lws_struct_map_t lsm_schema_map_ba[] = { + LSM_SCHEMA_DLL2 (sai_plat_owner_t, plat_owner, NULL, lsm_plat_list, + "com-warmcat-sai-ba"), + LSM_SCHEMA (sai_log_t, NULL, lsm_log, + "com-warmcat-sai-logs"), + LSM_SCHEMA (sai_event_t, NULL, lsm_task_rej, + "com.warmcat.sai.taskrej"), + LSM_SCHEMA (sai_artifact_t, NULL, lsm_artifact, + "com-warmcat-sai-artifact"), +}; + +enum { + SAIM_WSSCH_BUILDER_PLATS, + SAIM_WSSCH_BUILDER_LOGS, + SAIM_WSSCH_BUILDER_TASKREJ, + SAIM_WSSCH_BUILDER_ARTIFACT +}; + +static void +saim_dump_logs_to_db(lws_sorted_usec_list_t *sul) +{ + struct vhd *vhd = lws_container_of(sul, struct vhd, sul_logcache); + saim_logcache_pertask_t *lcpt; + char event_uuid[33]; + sqlite3 *pdb = NULL; + sai_log_t *hlog; + char *err; + + /* + * for each task that acquired logs in the interval + */ + + lws_start_foreach_dll_safe(struct lws_dll2 *, p, p1, + vhd->tasklog_cache.head) { + lcpt = lws_container_of(p, saim_logcache_pertask_t, list); + + sai_task_uuid_to_event_uuid(event_uuid, lcpt->uuid); + + if (!saim_event_db_ensure_open(vhd, event_uuid, 0, &pdb)) { + + /* + * Empty the task-specific log cache into the event- + * specific db for the task in one go, this is much + * more efficient + */ + + sqlite3_exec(pdb, "BEGIN TRANSACTION", NULL, NULL, &err); + + lws_struct_sq3_serialize(pdb, lsm_schema_sq3_map_log, + &lcpt->cache, 0); + + sqlite3_exec(pdb, "END TRANSACTION", NULL, NULL, &err); + saim_event_db_close(vhd, &pdb); + + } else + lwsl_err("%s: unable to open event-specific database\n", + __func__); + + /* + * Destroy the logs in the task cache and the task cache + */ + + lws_start_foreach_dll_safe(struct lws_dll2 *, pq, pq1, + lcpt->cache.head) { + hlog = lws_container_of(pq, sai_log_t, list); + lws_dll2_remove(&hlog->list); + free(hlog); + } lws_end_foreach_dll_safe(pq, pq1); + + /* + * Inform anybody who's looking at this task's logs that + * something changed + */ + saim_subs_request_writeable(vhd, lcpt->uuid); + + /* + * Destroy the whole task-specific cache, it will regenerate + * if more logs come for it + */ + + lws_dll2_remove(&lcpt->list); + free(lcpt); + + } lws_end_foreach_dll_safe(p, p1); + +} + +/* + * We're going to stash these logs on a per-task list, and deal with them + * inside a single trasaction per task efficiently on a timer. + */ + +static void +saim_log_to_db(struct vhd *vhd, sai_log_t *log) +{ + saim_logcache_pertask_t *lcpt = NULL; + sai_log_t *hlog; + + /* + * find the pertask if one exists + */ + + lws_start_foreach_dll(struct lws_dll2 *, p, vhd->tasklog_cache.head) { + lcpt = lws_container_of(p, saim_logcache_pertask_t, list); + + if (!strcmp(lcpt->uuid, log->task_uuid)) + break; + lcpt = NULL; + + } lws_end_foreach_dll(p); + + if (!lcpt) { + /* + * Create a pertask and add it to the vhd list of them + */ + lcpt = malloc(sizeof(*lcpt)); + memset(lcpt, 0, sizeof(*lcpt)); + lws_strncpy(lcpt->uuid, log->task_uuid, sizeof(lcpt->uuid)); + lws_dll2_add_tail(&lcpt->list, &vhd->tasklog_cache); + } + + hlog = malloc(sizeof(*hlog) + log->len + strlen(log->log) + 1); + if (!hlog) + return; + + *hlog = *log; + memset(&hlog->list, 0, sizeof(hlog->list)); + memcpy(&hlog[1], log->log, strlen(log->log) + 1); + hlog->log = (char *)&hlog[1]; + + /* + * add our log copy to the task-specific cache + */ + + lws_dll2_add_tail(&hlog->list, &lcpt->cache); + + if (!vhd->sul_logcache.list.owner) + /* if not already scheduled, schedule it for 250ms */ + + lws_sul_schedule(vhd->context, 0, &vhd->sul_logcache, + saim_dump_logs_to_db, 250 * LWS_US_PER_MS); +} + +static sai_plat_t * +saim_builder_from_uuid(struct vhd *vhd, const char *hostname) +{ + lws_start_foreach_dll(struct lws_dll2 *, p, + vhd->master.builder_owner.head) { + sai_plat_t *cb = lws_container_of(p, sai_plat_t, + sai_plat_list); + + if (!strcmp(hostname, cb->name)) + return cb; + + } lws_end_foreach_dll(p); + + return NULL; +} + +int +sai_sql3_get_uint64_cb(void *user, int cols, char **values, char **name) +{ + uint64_t *pui = (uint64_t *)user; + + *pui = (uint64_t)atoll(values[0]); + + return 0; +} + +/* + * Master received a communication from a builder + */ + +int +saim_ws_json_rx_builder(struct vhd *vhd, struct pss *pss, uint8_t *buf, size_t bl) +{ + char event_uuid[33], s[128], esc[96]; + struct lwsac *ac = NULL; + sai_plat_t *build, *cb; + sai_rejection_t *rej; + lws_dll2_owner_t o; + sai_artifact_t *ap; + sai_task_t *task; + sai_log_t *log; + uint64_t rid; + int n, m; + + /* + * use the schema name on the incoming JSON to decide what kind of + * structure to instantiate + * + * We may have: + * + * - just received a fragment of the whole JSON + * + * - received the JSON and be handling appeneded blob data + */ + + if (pss->bulk_binary_data) + goto handle; + + if (!pss->frag) { + memset(&pss->a, 0, sizeof(pss->a)); + pss->a.map_st[0] = lsm_schema_map_ba; + pss->a.map_entries_st[0] = LWS_ARRAY_SIZE(lsm_schema_map_ba); + pss->a.ac_block_size = 4096; + + lws_struct_json_init_parse(&pss->ctx, NULL, &pss->a); + } else + pss->frag = 0; + + m = lejp_parse(&pss->ctx, (uint8_t *)buf, bl); + + /* + * returns negative, or unused amount... for us, we either had a + * (negative) error, had LEJP_CONTINUE, or if 0/positive, finished + */ + if (m < 0 && m != LEJP_CONTINUE) { + /* an explicit error */ + lwsl_hexdump_err(buf, bl); + lwsl_err("%s: rx JSON decode failed '%s', %d, %s, %s, %d\n", + __func__, lejp_error_to_string(m), m, + pss->ctx.path, pss->ctx.buf, pss->ctx.npos); + return 1; + } + + if (m == LEJP_CONTINUE) { + pss->frag = 1; + return 0; + } + + if (!pss->a.dest) { + lwsl_err("%s: json decode didn't make an object\n", __func__); + return 1; + } + +handle: + switch (pss->a.top_schema_index) { + case SAIM_WSSCH_BUILDER_PLATS: + + lwsl_hexdump_notice(buf, bl); + + /* + * builder is sending us an array of platforms it provides us + */ + + pss->u.o = (sai_plat_owner_t *)pss->a.dest; + + lwsl_notice("%s: seen platform list: count %d\n", __func__, + pss->u.o->plat_owner.count); + + lws_start_foreach_dll(struct lws_dll2 *, pb, + pss->u.o->plat_owner.head) { + build = lws_container_of(pb, sai_plat_t, sai_plat_list); + + lwsl_notice("%s: seeing plat %s\n", __func__, build->name); + + /* + * ... so is this one a new guy? + */ + + cb = saim_builder_from_uuid(vhd, build->name); + if (!cb) { + char *cp; + + /* + * We need to make a persistent, deep, copy of + * the (from JSON) builder object representing + * this client. + * + * "platform" is eg "linux-ubuntu-bionic-arm64" + * and "name" is "hostname.". + */ + + if (!build->name || !build->platform) { + lwsl_err("%s: missing build '%s'/hostname '%s'\n", + __func__, + build->name ? build->name : "null", + build->platform ? build->platform : "null"); + return -1; + } + + cb = malloc(sizeof(*cb) + + strlen(build->name) + 1 + + strlen(build->platform) + 1); + + memset(cb, 0, sizeof(*cb)); + cp = (char *)&cb[1]; + + memcpy(cp, build->name, strlen(build->name) + 1); + cb->name = cp; + cp += strlen(build->name) + 1; + + memcpy(cp, build->platform, strlen(build->platform) + 1); + cb->platform = cp; + cp += strlen(build->platform) + 1; + + cb->ongoing = build->ongoing; + cb->instances = build->instances; + + cb->wsi = pss->wsi; + + /* Then attach the copy to the master in the vhd + */ + lws_dll2_add_tail(&cb->sai_plat_list, + &vhd->master.builder_owner); + } + + /* + * Even if he's not new, we should use his updated info about + * builder load + */ + cb->ongoing = build->ongoing; + cb->instances = build->instances; + + lwsl_notice("%s: builder %s reports load %d/%d\n", + __func__, cb->name, cb->ongoing, + cb->instances); + + } lws_end_foreach_dll(pb); + + lwsac_free(&pss->a.ac); + + /* + * look if we should offer the builder a task, given the + * platforms he's offering + */ + + if (saim_allocate_task(vhd, pss, cb, cb->platform) < 0) + goto bail; + + break; + +bail: + lwsac_free(&pss->a.ac); + return -1; + + case SAIM_WSSCH_BUILDER_LOGS: + /* + * builder is sending us info about task logs + */ + + log = (sai_log_t *)pss->a.dest; + saim_log_to_db(vhd, log); + + if (pss->mark_started) { + pss->mark_started = 0; + pss->first_log_timestamp = log->timestamp; + if (saim_set_task_state(vhd, NULL, NULL, log->task_uuid, + SAIES_BEING_BUILT, 0, 0)) + goto bail; + } + + if (log->finished) { + /* + * We have reached the end of the logs for this task + */ + lwsl_info("%s: log->finished says 0x%x, dur %lluus\n", + __func__, log->finished, (unsigned long long)( + log->timestamp - pss->first_log_timestamp)); + if (log->finished & SAISPRF_EXIT) { + if ((log->finished & 0xff) == 0) + n = SAIES_SUCCESS; + else + n = SAIES_FAIL; + } else + if (log->finished & 8192) + n = SAIES_CANCELLED; + else + n = SAIES_FAIL; + + if (saim_set_task_state(vhd, NULL, NULL, log->task_uuid, + n, 0, log->timestamp - + pss->first_log_timestamp)) + goto bail; + } + + lwsac_free(&pss->a.ac); + + break; + + case SAIM_WSSCH_BUILDER_TASKREJ: + + /* + * builder is updating us about his status, and may be + * rejecting a task we tried to give him + */ + + rej = (sai_rejection_t *)pss->a.dest; + + cb = saim_builder_from_uuid(vhd, rej->host_platform); + if (!cb) { + lwsl_info("%s: unknown builder %s rejecting\n", + __func__, rej->host_platform); + break; + } + + /* update our info about builder state with reality */ + + cb->ongoing = rej->ongoing; + cb->instances = rej->limit; + + lwsl_notice("%s: builder %s reports load %d/%d (rej %s)\n", + __func__, cb->name, cb->ongoing, cb->instances, + rej->task_uuid[0] ? rej->task_uuid : "none"); + + if (rej->task_uuid[0]) + saim_task_reset(vhd, rej->task_uuid); + + break; + + case SAIM_WSSCH_BUILDER_ARTIFACT: + /* + * We get sent a JSON object immediately followed by binary + * data for the artifact. + * + * We place the binary data as a blob in the sql record in the + * artifact table. + */ + + lwsl_debug("%s: SAIM_WSSCH_BUILDER_ARTIFACT\n", __func__); + + if (!pss->bulk_binary_data) { + + ap = (sai_artifact_t *)pss->a.dest; + + sai_task_uuid_to_event_uuid(event_uuid, ap->task_uuid); + + /* + * Open the event-specific database object... the + * handle is closed when the stream closes for whatever + * reason. + */ + + if (saim_event_db_ensure_open(pss->vhd, event_uuid, 0, + &pss->pdb_artifact)) { + lwsl_err("%s: unable to open event-specific " + "database\n", __func__); + + return -1; + } + + /* + * Retreive the task object + */ + + lws_sql_purify(esc, ap->task_uuid, sizeof(esc)); + lws_snprintf(s, sizeof(s)," and uuid == \"%s\"", esc); + n = lws_struct_sq3_deserialize(pss->pdb_artifact, s, + NULL, lsm_schema_sq3_map_task, + &o, &ac, 0, 1); + if (n < 0 || !o.head) { + saim_event_db_close(vhd, &pss->pdb_artifact); + lwsl_notice("%s: no task of that id\n", __func__); + return -1; + } + + task = (sai_task_t *)o.head; + n = strcmp(task->art_up_nonce, ap->artifact_up_nonce); + + if (n) { + lwsl_err("%s: artifact nonce mismatch\n", + __func__); + goto afail; + } + + /* + * The task the sender is sending us an artifact for + * exists. The sender knows the random upload nonce + * for that task's artifacts. + * + * Create a random download nonce unrelated to the + * random upload nonce (so knowing the download one + * won't let you upload anything). + * + * Create the artifact's entry in the event-specific + * database + */ + + sai_uuid16_create(pss->vhd->context, + ap->artifact_down_nonce); + + lws_dll2_owner_clear(&o); + lws_dll2_add_head(&ap->list, &o); + + /* + * Create the task in event-specific database + */ + + if (lws_struct_sq3_serialize(pss->pdb_artifact, + lsm_schema_sq3_map_artifact, + &o, ap->uid)) { + lwsl_err("%s: failed artifact struct insert\n", + __func__); + + goto afail; + } + + /* + * recover the rowid + */ + + lws_snprintf(s, sizeof(s), + "select rowid from artifacts " + "where timestamp=%llu", + (unsigned long long)ap->timestamp); + + if (sqlite3_exec((sqlite3 *)pss->pdb_artifact, s, + sai_sql3_get_uint64_cb, &rid, NULL) != + SQLITE_OK) { + lwsl_err("%s: %s: %s: fail\n", __func__, s, + sqlite3_errmsg(pss->pdb_artifact)); + goto afail; + } + + /* + * Set the blob size on associated row + */ + + lws_snprintf(s, sizeof(s), + "update artifacts set blob=zeroblob(%llu) " + "where rowid=%llu", + (unsigned long long)ap->len, + (unsigned long long)rid); + + if (sqlite3_exec((sqlite3 *)pss->pdb_artifact, s, + NULL, NULL, NULL) != + SQLITE_OK) { + lwsl_err("%s: %s: %s: fail\n", __func__, s, + sqlite3_errmsg(pss->pdb_artifact)); + goto afail; + } + + /* + * Open a blob on the associated row... the blob handle + * is closed when this stream closes for whatever + * reason. + */ + + if (sqlite3_blob_open(pss->pdb_artifact, "main", + "artifacts", "blob", rid, 1, + &pss->blob_artifact) != SQLITE_OK) { + lwsl_err("%s: unable to open blob\n", __func__); + goto afail; + } + + /* + * First time around, m == number of bytes let in buf + * after JSON, (bl - m) offset + */ + pss->bulk_binary_data = 1; + pss->artifact_length = ap->len; + } else + m = bl; + + if (m) { + lwsl_notice("%s: blob write +%d, ofs %llu / %llu, len %d\n", + __func__, (int)(bl - m), + (unsigned long long)pss->artifact_offset, + (unsigned long long)pss->artifact_length, m); + if (sqlite3_blob_write(pss->blob_artifact, + (uint8_t *)buf + (bl - m), (int)m, + pss->artifact_offset)) { + lwsl_err("%s: writing blob failed\n", __func__); + goto afail; + } + + lws_set_timeout(pss->wsi, PENDING_TIMEOUT_HTTP_CONTENT, 5); + pss->artifact_offset += (int)m; + } + + if (pss->artifact_offset == pss->artifact_length) { + lwsl_notice("%s: blob upload finished\n", __func__); + + goto afail; + } + + } + + return 0; + +afail: + lwsac_free(&ac); + saim_event_db_close(vhd, &pss->pdb_artifact); + + return -1; +} + +/* + * We're sending something on a builder ws connection + */ + +int +saim_ws_json_tx_builder(struct vhd *vhd, struct pss *pss, uint8_t *buf, + size_t bl) +{ + uint8_t *start = buf + LWS_PRE, *p = start, *end = p + bl - LWS_PRE - 1; + int n, flags = LWS_WRITE_TEXT, first = 0; + lws_struct_serialize_t *js; + sai_task_t *task; + size_t w; + + if (pss->task_cancel_owner.head) { + /* + * Pending cancel message to send + */ + sai_cancel_t *c = lws_container_of(pss->task_cancel_owner.head, + sai_cancel_t, list); + + js = lws_struct_json_serialize_create(lsm_schema_json_map_can, + LWS_ARRAY_SIZE(lsm_schema_json_map_can), 0, c); + if (!js) + return 1; + + n = lws_struct_json_serialize(js, p, end - p, &w); + lws_struct_json_serialize_destroy(&js); + + lws_dll2_remove(&c->list); + free(c); + + first = 1; + pss->walk = NULL; + + goto send_json; + } + + if (!pss->issue_task_owner.count) + return 0; /* nothing to send */ + + /* + * We're sending a browser the specific task info that he + * asked for. + * + * We already got the task struct out of the db in .one_task + * (all in .query_ac) + */ + + task = lws_container_of(pss->issue_task_owner.head, sai_task_t, + pending_assign_list); + lws_dll2_remove(&task->pending_assign_list); + + js = lws_struct_json_serialize_create(lsm_schema_map_ta, + LWS_ARRAY_SIZE(lsm_schema_map_ta), + 0, task); + if (!js) + return 1; + + n = lws_struct_json_serialize(js, p, end - p, &w); + lws_struct_json_serialize_destroy(&js); + + first = 1; + pss->walk = NULL; + + //lwsac_free(&pss->query_ac); + +send_json: + p += w; + if (n == LSJS_RESULT_ERROR) { + lwsl_notice("%s: taskinfo: error generating json\n", + __func__); + return 1; + } + if (!lws_ptr_diff(p, start)) { + lwsl_notice("%s: taskinfo: empty json\n", __func__); + return 0; + } + + flags = lws_write_ws_flags(LWS_WRITE_TEXT, first, !pss->walk); + + lwsl_hexdump_notice(start, p - start); + + if (lws_write(pss->wsi, start, p - start, flags) < 0) + return -1; + + lws_callback_on_writable(pss->wsi); + + return 0; +} diff --git a/src/master/private.h b/src/master/private.h deleted file mode 100644 index 7ee6749..0000000 --- a/src/master/private.h +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Sai master definitions src/master/private.h - * - * Copyright (C) 2019 Andy Green - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation: - * version 2.1 of the License. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, - * MA 02110-1301 USA - */ - -struct sai_master; - -enum sai_lease_states { - SOS_IDLE, - SOS_RESERVED, - SOS_CONFIRMED, - SOS_OCCUPIED, - SOS_GOING_DOWN -}; - -typedef struct sai_console { - struct lws_dll console_list; - - const char *name; - const char *path; - const char *mode; - int baudrate; -} sai_console_t; - -typedef struct sai_target { - struct lws_dll target_list; - struct lws_dll console_head; - - const char *name; - - enum sai_lease_states state; -} sai_target_t; - -typedef struct sai_nspawn { - struct lws_dll nspawn_list; - - const char *name; - - enum sai_lease_states state; -} sai_nspawn_t; - -typedef struct sai_builder { - struct lws_dll builder_list; - struct lws_dll target_head; - struct lws_dll nspawn_head; - - struct lws *wsi; - - char *hostname; -} sai_builder_t; - -typedef enum { - SJS_CLONING, - SJS_ASSIGNING, - SJS_WAITING, - SJS_DONE -} sai_job_state_t; - -typedef struct sai_job { - struct lws_dll jobs_list; - char reponame[64]; - char ref[64]; - char head[64]; - - time_t requested; - - sai_job_state_t state; - -} sai_job_t; - -typedef struct sai_master { - struct lws_dll builder_head; - - struct lws_dll pending_jobs_head; - struct lws_dll ongoing_jobs_head; -} sai_master_t; - - -extern struct lws_context * -sai_lws_context_from_json(const char *config_dir, - const struct lws_protocols **pprotocols); -extern const struct lws_protocols protocol_ws; diff --git a/src/master/sai-master.c b/src/master/sai-master.c deleted file mode 100644 index 138703c..0000000 --- a/src/master/sai-master.c +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Sai master - * - * Copyright (C) 2019 Andy Green - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation: - * version 2.1 of the License. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, - * MA 02110-1301 USA - */ - -#include -#include -#include -#include - -#include "private.h" - -static int interrupted; - -static const struct lws_protocols - *pprotocols[] = { &protocol_ws, NULL }; - -static void sigint_handler(int sig) -{ - interrupted = 1; -} - -int main(int argc, const char **argv) -{ - int n = 0, logs = LLL_USER | LLL_ERR | LLL_WARN | LLL_NOTICE; - struct lws_context *context; - const char *p, *conf = "/etc/sai/master"; - - signal(SIGINT, sigint_handler); - - if ((p = lws_cmdline_option(argc, argv, "-d"))) - logs = atoi(p); - - lws_set_log_level(logs, NULL); - lwsl_user("Sai Master - Copyright (C) 2019 Andy Green \n"); - - if ((p = lws_cmdline_option(argc, argv, "-c"))) - conf = p; - - context = sai_lws_context_from_json(conf, pprotocols); - if (!context) { - lwsl_err("lws init failed\n"); - return 1; - } - - while (n >= 0 && !interrupted) - n = lws_service(context, 1000); - - lws_context_destroy(context); - - return 0; -} diff --git a/src/master/ws.c b/src/master/ws.c deleted file mode 100644 index 19aec34..0000000 --- a/src/master/ws.c +++ /dev/null @@ -1,317 +0,0 @@ -/* - * Sai master - * - * Copyright (C) 2019 Andy Green - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation: - * version 2.1 of the License. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, - * MA 02110-1301 USA - * - * The same ws interface is connected-to by builders (on path /builder), and - * provides the query transport for browsers (on path /browse). - */ - -#include -#include -#include -#include - -#include - -#include "private.h" - -typedef enum { - SWT_BUILDER, - SWT_BROWSE -} ws_type; - -struct pss { - ws_type type; -}; - -struct vhd { - struct lws_context *context; - struct lws_vhost *vhost; - - struct sai_master master; -}; - -/* - * {"schema":"com-warmcat-sai-builder-attach", - "hostname":"learn", "targets": [{"name":"optee-hikey-arm64"} -], -{"nspawns": []} - */ - - -static const char * const paths[] = { - "schema", - "hostname", - "nspawn-timeout", - - "targets[]", - "targets[].name", - - "nspawns[]", - "nspawns[].name", - "nspawns[].overlays[]", - "nspawns[].overlays[].name", -}; - -enum enum_paths { - SBAJP_SCHEMA, - SBAJP_HOSTNAME, - SBAJP_NSPAWN_TIMEOUT, - - SBAJP_TARGETS, - SBAJP_TARGETS_NAME, - - SBAJP_NSPAWNS, - SBAJP_NSPAWNS_NAME, - SBAJP_NSPAWNS_OVERLAYS, - SBAJP_NSPAWNS_OVERLAYS_NAME, -}; - -static int -sai_master_nspawn_destroy(struct lws_dll *d) -{ - sai_nspawn_t *n = lws_container_of(d, sai_nspawn_t, nspawn_list); - - lws_dll_remove(&n->nspawn_list); - free(n); - - return 0; -} - -static int -sai_master_console_destroy(struct lws_dll *d) -{ - sai_console_t *c = lws_container_of(d, sai_console_t, console_list); - - lws_dll_remove(&c->console_list); - free(c); - - return 0; -} - -static int -sai_master_target_destroy(struct lws_dll *d) -{ - sai_target_t *t = lws_container_of(d, sai_target_t, target_list); - - lws_dll_foreach_safe(&t->console_head, sai_master_console_destroy); - - free(t); - - return 0; -} - -static int -sai_master_builder_destroy(struct lws_dll *d) -{ - sai_builder_t *b = lws_container_of(d, sai_builder_t, builder_list); - - lws_dll_foreach_safe(&b->nspawn_head, sai_master_nspawn_destroy); - lws_dll_foreach_safe(&b->target_head, sai_master_target_destroy); - - if (b->hostname) - free(b->hostname); - - free(b); - - return 0; -} - -static int -sai_destroy_job(struct lws_dll *d) -{ - sai_job_t *job = lws_container_of(d, sai_job_t, jobs_list); - - lws_dll_remove(d); - free(job); - - return 0; -} - -static void -sai_master_master_destroy(struct sai_master *master) -{ - lws_dll_foreach_safe(&master->builder_head, sai_master_builder_destroy); - - lws_dll_foreach_safe(&master->pending_jobs_head, sai_destroy_job); - lws_dll_foreach_safe(&master->ongoing_jobs_head, sai_destroy_job); -} - -static int -callback_ws(struct lws *wsi, enum lws_callback_reasons reason, void *user, - void *in, size_t len) -{ - struct vhd *vhd = (struct vhd *)lws_protocol_vh_priv_get( - lws_get_vhost(wsi), lws_get_protocol(wsi)); - uint8_t buf[LWS_PRE + 2048], *start = &buf[LWS_PRE], *p = start, - *end = &buf[sizeof(buf) - LWS_PRE - 1]; - struct pss *pss = (struct pss *)user; - struct sai_job *job; - const char *ccp; - int n; - - (void)end; - (void)p; - (void)paths; - - switch (reason) { - case LWS_CALLBACK_PROTOCOL_INIT: - vhd = lws_protocol_vh_priv_zalloc(lws_get_vhost(wsi), - lws_get_protocol(wsi), - sizeof(struct vhd)); - if (!vhd) - return -1; - - vhd->context = lws_get_context(wsi); - vhd->vhost = lws_get_vhost(wsi); - break; - - case LWS_CALLBACK_PROTOCOL_DESTROY: - sai_master_master_destroy(&vhd->master); - break; - - case LWS_CALLBACK_HTTP: - - n = HTTP_STATUS_FORBIDDEN; - - if (strcmp((const char *)in, "/sai/update-hook")) { - lwsl_notice("%s: unknown path %s\n", __func__, - (const char *)in); - goto http_resp; - } - - job = malloc(sizeof(*job)); - if (!job) { - lwsl_err("%s: OOM\n", __func__); - - return 1; - } - - ccp = lws_get_urlarg_by_name(wsi, "rn=", (char *)start, - sizeof(job->reponame)); - if (!ccp || !*ccp) { - lwsl_notice("%s: missing rn=\n", __func__); - free(job); - goto http_resp; - } - lws_strncpy(job->reponame, ccp, sizeof(job->reponame)); - ccp = lws_get_urlarg_by_name(wsi, "ref=", (char *)start + 128, - sizeof(job->ref)); - if (!ccp || !*ccp) { - lwsl_notice("%s: missing ref=\n", __func__); - free(job); - goto http_resp; - } - lws_strncpy(job->ref, ccp, sizeof(job->ref)); - ccp = lws_get_urlarg_by_name(wsi, "head=", (char *)start + 256, - sizeof(job->head)); - if (!ccp || !*ccp) { - lwsl_notice("%s: missing ref=\n", __func__); - free(job); - goto http_resp; - } - lws_strncpy(job->head, ccp, sizeof(job->head)); - - job->requested = time(NULL); - - ccp = strrchr(job->ref + strlen(job->ref) - 1, '/'); - if (ccp) - ccp++; - - lwsl_user("%s: added job: %s %s %s\n", __func__, job->reponame, - ccp, job->head); - - { - git_clone_options opts; - git_repository *repo; - const char *url = "https://libwebsockets.org/repo/libwebsockets"; - - git_clone_init_options(&opts, GIT_CLONE_OPTIONS_VERSION); - opts.checkout_branch = ccp; - - if (git_clone(&repo, url, "/tmp", &opts)) { - lwsl_err("%s: unable to clone\n", __func__); - free(job); - goto http_resp; - } - - lwsl_user("%s: clone OK\n", __func__); - } - - lws_dll_add_front(&job->jobs_list, &vhd->master.pending_jobs_head); - n = HTTP_STATUS_OK; - - /* faillthru */ - -http_resp: - if (lws_add_http_header_status(wsi, n, &p, end)) - goto bail; - if (lws_add_http_header_content_length(wsi, 0, &p, end)) - goto bail; - if (lws_finalize_write_http_header(wsi, start, &p, end)) - goto bail; - goto try_to_reuse; - - case LWS_CALLBACK_ESTABLISHED: - - if (lws_hdr_total_length(wsi, WSI_TOKEN_GET_URI)) { - if (lws_hdr_copy(wsi, (char *)start, 64, WSI_TOKEN_GET_URI) < 0) - return -1; - } else - if (lws_hdr_copy(wsi, (char *)start, 64, WSI_TOKEN_HTTP_COLON_PATH) < 0) - return -1; - - if (!strcmp((char *)start, "/browse")) { - n = SWT_BUILDER; - } else { - if (!strcmp((char *)start, "builder")) { - n = SWT_BROWSE; - } else { - lwsl_err("%s: unknown URL\n", __func__); - - return -1; - } - } - - pss->type = n; - break; - - case LWS_CALLBACK_RECEIVE: - lwsl_user("RECEIVE: %d\n", (int)len); - lwsl_user("%.*s\n", (int)len, (const char *)in); - break; - - default: - break; - } - - return lws_callback_http_dummy(wsi, reason, user, in, len); - -bail: - return 1; - -try_to_reuse: - if (lws_http_transaction_completed(wsi)) - return -1; - - return 0; -} - -const struct lws_protocols protocol_ws = - { "com-warmcat-sai", callback_ws, sizeof(struct pss), 0 };