diff --git a/README.md b/README.md index b94d1ab..ba5be37 100644 --- a/README.md +++ b/README.md @@ -142,8 +142,15 @@ networks: | `RUNNERS_START` | `5` | Number of act\_runner instances to register | | `RUNNER_CACHE_PORT` | `44015` | Port for the act\_runner cache server | | `RUNNER_IP_ADDRESS` | container IP | IP address act\_runner registers with Forgejo | -| `RUNNER_DEFAULT_HOME` | `/config/act_runner/forgejo` | Directory where runner registration state is stored | -| `RUNNER_CONFIG_NAME` | `act_runner.yaml` | Runner config filename inside `RUNNER_DEFAULT_HOME` | +| `RUNNER_DEFAULT_HOME` | `/config/act_runner/forgejo` | Rendered runner configuration home; registration state remains under `/config/act_runner/reg` | +| `RUNNER_CONFIG_NAME` | `act_runner.yaml` | Base runner config filename inside `RUNNER_DEFAULT_HOME`; each runner gets an isolated `config.yaml` | +| `RUNNERS_CONFIG_BASE` | rendered config | Override the full base config used by every runner instance | +| `RUNNER_CLEANUP_ENABLED` | `yes` | Periodically prune stale Docker job resources and act caches | +| `RUNNER_CLEANUP_INTERVAL` | `60` | Cleanup interval in minutes | +| `RUNNER_CLEANUP_UNTIL` | `72h` | Age threshold for stopped containers, networks, and build cache | +| `RUNNER_CLEANUP_DISK_PERCENT` | `80` | Disk-use threshold for pruning all unused images | +| `RUNNER_CLEANUP_ACT_CACHE_DAYS` | `7` | Remove act cache entries older than this many days; `0` disables | +| `RUNNER_CLEANUP_LOG_FILE` | `/data/logs/act_runner/cleanup.log` | Cleanup log path | | `ACT_RUNNER_FALLBACK_VERSION` | `v13.1.0` | Pinned act\_runner version used if code.forgejo.org is unreachable during build | **Runner labels** are set automatically based on the host architecture. All jobs run inside Docker containers — no bare-metal execution. diff --git a/TODO.AI.md b/TODO.AI.md index 8fd0d05..54eaf42 100644 --- a/TODO.AI.md +++ b/TODO.AI.md @@ -428,6 +428,14 @@ Found while reading the full repo tree; not previously logged. (matching the build-stage `ENV HOSTNAME="casjaysdevdocker-forgejo"`, which was already correct). Fixed to `casjaysdevdocker-${IMAGE_NAME}`. +## Runner maintenance and configuration migration — applied + +- Switched first-run config seeding in the three hand-crafted init scripts to `cp -Rn`, so existing files under `/config` are not overwritten by baked defaults. +- `start-runners` now derives an isolated per-runner config from the rendered Forgejo base config, preserving full container/host/cache settings while giving each runner its own `.runner` registration state. The Forgejo-specific cache fallback continues to use `secret`. +- Added `cleanup-runners` and scheduled it through the existing `__cron` helper. Cleanup is bounded by age and disk thresholds, skips unreachable Docker, never prunes running containers, and removes only stale top-level act-cache entries. +- Gitea-specific retention/deprecation and passkey/cookie changes were not ported: the image resolves Forgejo dynamically and the reviewed Forgejo documentation does not establish equivalent version-safe behavior. `DISABLE_QUERY_AUTH_TOKEN` remains because Forgejo documents it as supported. +- Runtime Docker/runner verification remains pending until a fresh image build and privileged integration run are available. + ## Non-issue — confirmed intentional (`.gitea/workflows/docker.yaml`) - Uses a stale/unpinned action pattern (`@v2`-`@v4`, DockerHub-only, `catthehacker/ubuntu:act-latest`). diff --git a/rootfs/usr/local/bin/cleanup-runners b/rootfs/usr/local/bin/cleanup-runners new file mode 100755 index 0000000..c0fb046 --- /dev/null +++ b/rootfs/usr/local/bin/cleanup-runners @@ -0,0 +1,95 @@ +#!/usr/bin/env bash +# shellcheck shell=bash +# - - - - - - - - - - - - - - - - - - - - - - - - - +##@Version : 202610051200-git +# @@Author : Jason Hempstead +# @@Contact : jason@casjaysdev.pro +# @@License : LICENSE.md +# @@ReadME : cleanup-runners --help +# @@Copyright : Copyright: (c) 2026 Jason Hempstead, Casjays Developments +# @@Description : Prune docker and act cache leftovers from act runner jobs +# - - - - - - - - - - - - - - - - - - - - - - - - - +VERSION="202610051200-git" +set -eo pipefail + +RUNNER_CLEANUP_UNTIL="${RUNNER_CLEANUP_UNTIL:-72h}" +RUNNER_CLEANUP_DISK_PERCENT="${RUNNER_CLEANUP_DISK_PERCENT:-80}" +RUNNER_CLEANUP_DOCKER_DIR="${RUNNER_CLEANUP_DOCKER_DIR:-/data/docker}" +RUNNER_CLEANUP_ACT_CACHE_DAYS="${RUNNER_CLEANUP_ACT_CACHE_DAYS:-7}" +RUNNER_CLEANUP_ACT_CACHE_DIRS="${RUNNER_CLEANUP_ACT_CACHE_DIRS:-$HOME/.cache/act /root/.cache/act}" +RUNNER_CLEANUP_LOG_FILE="${RUNNER_CLEANUP_LOG_FILE:-/data/logs/act_runner/cleanup.log}" +if ! [[ "$RUNNER_CLEANUP_DISK_PERCENT" =~ ^(100|[0-9]{1,2})$ ]]; then + RUNNER_CLEANUP_DISK_PERCENT=80 +fi + +__log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" >>"$RUNNER_CLEANUP_LOG_FILE" 2>&1; } +__prune() { + local output="" + if output=$("$@" 2>&1); then + __log "$* :: $(printf '%s\n' "$output" | grep -i -- 'reclaimed' | tail -n1)" + else + __log "FAILED: $* :: $output" + fi +} +__disk_percent() { + df -P "$1" 2>/dev/null | awk 'NR==2 {gsub("%", "", $5); print $5}' +} +__help() { + cat <<'EOF' +Usage: cleanup-runners [--help|--version] + +Prune stale Docker runner resources, build cache, images, and act runner +cache entries. Only explicitly dangling volumes matching the act- prefix are removed. + +Environment: + RUNNER_CLEANUP_UNTIL Docker prune age (default: 72h) + RUNNER_CLEANUP_DISK_PERCENT Image cleanup threshold (default: 80) + RUNNER_CLEANUP_DOCKER_DIR Docker data path (default: /data/docker) + RUNNER_CLEANUP_ACT_CACHE_DAYS Cache age in days (default: 7) + RUNNER_CLEANUP_ACT_CACHE_DIRS Space-separated act cache directories + RUNNER_CLEANUP_LOG_FILE Cleanup log path +EOF +} + +case "${1:-}" in + -h|--help) + __help + exit 0 + ;; + --version) + echo "$VERSION" + exit 0 + ;; +esac + +mkdir -p "$(dirname -- "$RUNNER_CLEANUP_LOG_FILE")" +if ! docker info >/dev/null 2>&1; then + __log "docker is not reachable, skipping Docker cleanup" +else + __log "Starting cleanup" + __prune docker container prune -f --filter "until=$RUNNER_CLEANUP_UNTIL" + __prune docker network prune -f --filter "until=$RUNNER_CLEANUP_UNTIL" + act_volumes="$(docker volume ls -q --filter dangling=true --filter name=act- 2>/dev/null | grep -E -- '^act-' || true)" + if [ -n "$act_volumes" ]; then + # shellcheck disable=SC2086 + __prune docker volume rm $act_volumes + fi + __prune docker builder prune -af --filter "until=$RUNNER_CLEANUP_UNTIL" + __prune docker image prune -f --filter "until=$RUNNER_CLEANUP_UNTIL" + disk_used="$(__disk_percent "$RUNNER_CLEANUP_DOCKER_DIR" || true)" + if [[ "$disk_used" =~ ^[0-9]+$ ]] && [ "$disk_used" -ge "$RUNNER_CLEANUP_DISK_PERCENT" ]; then + __log "$RUNNER_CLEANUP_DOCKER_DIR is ${disk_used}% used (limit ${RUNNER_CLEANUP_DISK_PERCENT}%), removing unused images" + __prune docker image prune -af --filter "until=$RUNNER_CLEANUP_UNTIL" + fi +fi + +if [[ "$RUNNER_CLEANUP_ACT_CACHE_DAYS" =~ ^[1-9][0-9]*$ ]]; then + for act_cache_dir in $RUNNER_CLEANUP_ACT_CACHE_DIRS; do + [ -d "$act_cache_dir" ] || continue + removed="$(find "$act_cache_dir" -mindepth 1 -maxdepth 1 -mtime +"$RUNNER_CLEANUP_ACT_CACHE_DAYS" 2>/dev/null | wc -l || true)" + find "$act_cache_dir" -mindepth 1 -maxdepth 1 -mtime +"$RUNNER_CLEANUP_ACT_CACHE_DAYS" -exec rm -rf {} + 2>/dev/null || true + __log "Removed $removed stale entries from $act_cache_dir" + done +fi +__log "Cleanup finished" +exit 0 diff --git a/rootfs/usr/local/bin/start-runners b/rootfs/usr/local/bin/start-runners index 2e32aa3..6c438b7 100755 --- a/rootfs/usr/local/bin/start-runners +++ b/rootfs/usr/local/bin/start-runners @@ -28,6 +28,34 @@ echo "${emoji}Fatal error occurred: Exit code $retVal at line $LINENO in command kill -TERM 1' ERR trap 'retVal=$?;if [ "$SERVICE_IS_RUNNING" != "yes" ] && [ -f "$SERVICE_PID_FILE" ]; then rm -Rf "$SERVICE_PID_FILE"; fi;exit $retVal' SIGINT SIGTERM SIGPWR # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +__help() { + cat <<'EOF' +Usage: start-runners + +Register and start one or more Forgejo act_runner daemons in parallel. + +Environment: + SERVER_ADDRESS Forgejo server URL (defaults to http://$HOSTNAME) + SERVER_TOKEN Required runner registration token + RUNNERS_LOG_DIR Runner log directory + RUNNERS_START Number of runners to start (default: 1) + RUNNER_NAME_PREFIX Runner name prefix (default: runner) + RUNNER_LABELS Comma-separated runner labels + RUNNERS_CONFIG_BASE Base act_runner config file + RUNNER_CACHE_HOST External cache host (used with RUNNER_CACHE_PORT and + RUNNER_CACHE_SECRET when no base config is available) + RUNNER_CACHE_PORT External cache port + RUNNER_CACHE_SECRET External cache secret +EOF +} + +case "${1:-}" in + -h|--help) + __help + exit 0 + ;; +esac + # Function to __log messages with timestamp __log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" >>"$RUNNERS_LOG_DIR/runners" 2>&1; } # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -100,29 +128,55 @@ __log "Starting $RUNNERS_START act_runner instance(s)" __log "Server Address: $SERVER_ADDRESS" __log "Runner Name Prefix: ${RUNNER_NAME_PREFIX:-runner}" # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -# Shared cache config so runner-N instances use the same cache-server -# started by zz-act_runner.sh instead of an unshared per-process cache. +# Full base config rendered by zz-act_runner.sh. Fall back to a shared cache-only +# config when this script is invoked without a rendered Forgejo runner config. +RUNNERS_DEFAULT_HOME="${RUNNER_DEFAULT_HOME:-/config/act_runner/forgejo}" +# Keep the established registration path by default; callers may override it +# independently when relocating runner state. +RUNNERS_REGISTRATION_ROOT="${RUNNER_REGISTRATION_ROOT:-/config/act_runner/reg}" RUNNERS_CACHE_CONFIG="" -if [ -n "$RUNNER_CACHE_HOST" ] && [ -n "$RUNNER_CACHE_PORT" ] && [ -n "$RUNNER_CACHE_SECRET" ]; then +if [ -n "$RUNNERS_CONFIG_BASE" ] && [ -f "$RUNNERS_CONFIG_BASE" ]; then + __log "Base config: $RUNNERS_CONFIG_BASE" +elif [ -n "$RUNNER_CACHE_HOST" ] && [ -n "$RUNNER_CACHE_PORT" ] && [ -n "$RUNNER_CACHE_SECRET" ]; then + RUNNERS_CONFIG_BASE="" RUNNERS_CACHE_CONFIG="/config/act_runner/runners-cache.yaml" cat <"$RUNNERS_CACHE_CONFIG" +runner: + file: REPLACE_RUNNER_STATE/.runner cache: enabled: true external_server: 'http://${RUNNER_CACHE_HOST}:${RUNNER_CACHE_PORT}/' secret: '${RUNNER_CACHE_SECRET}' EOF +else + RUNNERS_CONFIG_BASE="" fi # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +# Create the config used by one runner, isolating its registration state. +__runner_config() { + local runner_dir=$1 + local runner_config="$runner_dir/config.yaml" + if [ -n "$RUNNERS_CONFIG_BASE" ]; then + sed "s|^\([[:space:]]*\)file:.*|\1file: $runner_dir/.runner|" "$RUNNERS_CONFIG_BASE" >"$runner_config" || return 1 + echo "$runner_config" + elif [ -n "$RUNNERS_CACHE_CONFIG" ]; then + sed "s|REPLACE_RUNNER_STATE|$runner_dir|" "$RUNNERS_CACHE_CONFIG" >"$runner_config" || return 1 + echo "$runner_config" + fi +} +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - # Register a single runner synchronously (no daemon start) __register_runner() { local runner_id=$1 local runner_name="${RUNNER_NAME_PREFIX:-runner}-${runner_id}" - local runner_dir="/config/act_runner/reg/${runner_name}" + local runner_dir="$RUNNERS_REGISTRATION_ROOT/${runner_name}" local config_args=() + local runner_config="" mkdir -p "$runner_dir" [ -d "$runner_dir" ] && cd "$runner_dir" || return 1 - [ -n "$RUNNERS_CACHE_CONFIG" ] && config_args=(--config "$RUNNERS_CACHE_CONFIG") + runner_config="$(__runner_config "$runner_dir")" || return 1 + [ -n "$runner_config" ] && config_args=(--config "$runner_config") if [ ! -f "$runner_dir/.runner" ]; then __log "Registering runner: $runner_name (ID: $runner_id)" @@ -139,11 +193,13 @@ __register_runner() { __start_runner_daemon() { local runner_id=$1 local runner_name="${RUNNER_NAME_PREFIX:-runner}-${runner_id}" - local runner_dir="/config/act_runner/reg/${runner_name}" + local runner_dir="$RUNNERS_REGISTRATION_ROOT/${runner_name}" local config_args=() + local runner_config="" [ -d "$runner_dir" ] && cd "$runner_dir" || return 1 - [ -n "$RUNNERS_CACHE_CONFIG" ] && config_args=(--config "$RUNNERS_CACHE_CONFIG") + runner_config="$(__runner_config "$runner_dir")" || return 1 + [ -n "$runner_config" ] && config_args=(--config "$runner_config") __log "Starting daemon for runner: $runner_name" exec act_runner daemon "${config_args[@]}" } @@ -155,20 +211,28 @@ done unset i # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - # Phase 2: start all daemons in parallel +runner_pids=() for i in $(seq 1 $RUNNERS_START); do (__start_runner_daemon "$i") & + runner_pids+=("$!") done unset i # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -RUNNERS_ID="$(jobs -p | tr '\n' ' ')" -if [ -n "$RUNNERS_ID" ]; then - __log "All $RUNNERS_START runners started successfully" - __log "Process IDs: $RUNNERS_ID" - printf 'All %s runners started successfully: %s\n' "$RUNNERS_START" "$RUNNERS_ID" -else - __log "The runners have failed to start" - printf '%s\n' "The runners have failed to start" +RUNNERS_ID="${runner_pids[*]}" +__log "All $RUNNERS_START runners launched" +__log "Process IDs: $RUNNERS_ID" +printf 'All %s runners launched: %s\n' "$RUNNERS_START" "$RUNNERS_ID" +# Wait for all background processes and preserve a failure status. A daemon that +# exits is reported as failed instead of being treated as successfully started. +runner_failures=0 +for runner_pid in "${runner_pids[@]}"; do + if ! wait "$runner_pid"; then + runner_failures=$((runner_failures + 1)) + __log "ERROR: Runner process $runner_pid exited unsuccessfully" + fi +done +if [ "$runner_failures" -gt 0 ]; then + __log "$runner_failures runner process(es) failed" + printf '%s\n' "$runner_failures runner process(es) failed" >&2 + exit 1 fi -# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -# Wait for all background processes -wait diff --git a/rootfs/usr/local/etc/docker/init.d/05-dockerd.sh b/rootfs/usr/local/etc/docker/init.d/05-dockerd.sh index b32c3ba..051b39c 100755 --- a/rootfs/usr/local/etc/docker/init.d/05-dockerd.sh +++ b/rootfs/usr/local/etc/docker/init.d/05-dockerd.sh @@ -279,7 +279,7 @@ __run_precopy() { if [ -d "$ETC_DIR" ] && ! [ -L "$ETC_DIR" ]; then if [ ! -f "$CONF_DIR/.initialized" ]; then mkdir -p "$CONF_DIR" - cp -Rf "$ETC_DIR/." "$CONF_DIR/" 2>/dev/null || true + cp -Rn "$ETC_DIR/." "$CONF_DIR/" 2>/dev/null || true fi rm -Rf "$ETC_DIR" ln -sf "$CONF_DIR" "$ETC_DIR" diff --git a/rootfs/usr/local/etc/docker/init.d/08-forgejo.sh b/rootfs/usr/local/etc/docker/init.d/08-forgejo.sh index 51d1f41..6c2e6c1 100755 --- a/rootfs/usr/local/etc/docker/init.d/08-forgejo.sh +++ b/rootfs/usr/local/etc/docker/init.d/08-forgejo.sh @@ -323,7 +323,7 @@ __run_precopy() { if [ -d "$ETC_DIR" ] && ! [ -L "$ETC_DIR" ]; then if [ ! -f "$CONF_DIR/.initialized" ]; then mkdir -p "$CONF_DIR" - cp -Rf "$ETC_DIR/." "$CONF_DIR/" 2>/dev/null || true + cp -Rn "$ETC_DIR/." "$CONF_DIR/" 2>/dev/null || true fi rm -Rf "$ETC_DIR" ln -sf "$CONF_DIR" "$ETC_DIR" diff --git a/rootfs/usr/local/etc/docker/init.d/zz-act_runner.sh b/rootfs/usr/local/etc/docker/init.d/zz-act_runner.sh index 4c2cdcd..f3b44e4 100755 --- a/rootfs/usr/local/etc/docker/init.d/zz-act_runner.sh +++ b/rootfs/usr/local/etc/docker/init.d/zz-act_runner.sh @@ -341,6 +341,17 @@ RUNNER_CACHE_HOST="${RUNNER_CACHE_HOST:-$IP4_ADDRESS}" CACHE_CONFIG_FILE="${CACHE_CONFIG_FILE:-$CONF_DIR/cache_server.yaml}" CACHE_LOG_FILE="${CACHE_LOG_FILE:-$LOG_DIR/cache.log}" RUNNER_CACHE_SECRET="${RUNNER_CACHE_SECRET:-}" +# Rendered full runner config consumed by each per-runner daemon. +RUNNERS_CONFIG_BASE="${RUNNERS_CONFIG_BASE:-$RUNNER_DEFAULT_HOME/$RUNNER_CONFIG_NAME}" +# Periodic cleanup of stopped job resources and stale act caches. +RUNNER_CLEANUP_ENABLED="${RUNNER_CLEANUP_ENABLED:-yes}" +RUNNER_CLEANUP_INTERVAL="${RUNNER_CLEANUP_INTERVAL:-60}" +RUNNER_CLEANUP_UNTIL="${RUNNER_CLEANUP_UNTIL:-72h}" +RUNNER_CLEANUP_DISK_PERCENT="${RUNNER_CLEANUP_DISK_PERCENT:-80}" +RUNNER_CLEANUP_DOCKER_DIR="${RUNNER_CLEANUP_DOCKER_DIR:-/data/docker}" +RUNNER_CLEANUP_ACT_CACHE_DAYS="${RUNNER_CLEANUP_ACT_CACHE_DAYS:-7}" +RUNNER_CLEANUP_ACT_CACHE_DIRS="${RUNNER_CLEANUP_ACT_CACHE_DIRS:-$HOME/.cache/act /root/.cache/act}" +RUNNER_CLEANUP_LOG_FILE="${RUNNER_CLEANUP_LOG_FILE:-$LOG_DIR/cleanup.log}" # - - - - - - - - - - - - - - - - - - - - - - - - - # Additional variables @@ -375,7 +386,7 @@ __run_precopy() { if [ -d "$ETC_DIR" ] && ! [ -L "$ETC_DIR" ]; then if [ ! -f "$CONF_DIR/.initialized" ]; then mkdir -p "$CONF_DIR" - cp -Rf "$ETC_DIR/." "$CONF_DIR/" 2>/dev/null || true + cp -Rn "$ETC_DIR/." "$CONF_DIR/" 2>/dev/null || true fi rm -Rf "$ETC_DIR" ln -sf "$CONF_DIR" "$ETC_DIR" @@ -523,6 +534,10 @@ __post_execute() { export SERVER_ADDRESS="$RUNNER_IP_ADDRESS:$FORGEJO_PORT" SERVER_TOKEN="${RUNNER_AUTH_TOKEN:-$SYS_AUTH_TOKEN}" export RUNNER_CACHE_HOST RUNNER_CACHE_PORT export RUNNER_CACHE_SECRET="${RUNNER_CACHE_SECRET:-$(__gen_cache_secret)}" + export RUNNERS_CONFIG_BASE + export RUNNER_CLEANUP_ENABLED RUNNER_CLEANUP_INTERVAL RUNNER_CLEANUP_UNTIL + export RUNNER_CLEANUP_DISK_PERCENT RUNNER_CLEANUP_DOCKER_DIR + export RUNNER_CLEANUP_ACT_CACHE_DAYS RUNNER_CLEANUP_ACT_CACHE_DIRS RUNNER_CLEANUP_LOG_FILE # wait sleep $waitTime @@ -568,6 +583,21 @@ __post_execute() { # subshell (itself the left side of the __post_execute pipe) reaches its end disown "$!" 2>/dev/null || true fi + if [ "${RUNNER_CLEANUP_ENABLED,,}" = "yes" ] || [ "${RUNNER_CLEANUP_ENABLED,,}" = "true" ] || [ "$RUNNER_CLEANUP_ENABLED" = "1" ]; then + if [ -x "/usr/local/bin/cleanup-runners" ]; then + cleanup_interval="${RUNNER_CLEANUP_INTERVAL:-60}" + if ! [[ "$cleanup_interval" =~ ^[1-9][0-9]*$ ]]; then + cleanup_interval=60 + fi + mkdir -p "$(dirname -- "$RUNNER_CLEANUP_LOG_FILE")" + ( + export RUNNER_CLEANUP_UNTIL RUNNER_CLEANUP_DISK_PERCENT RUNNER_CLEANUP_DOCKER_DIR + export RUNNER_CLEANUP_ACT_CACHE_DAYS RUNNER_CLEANUP_ACT_CACHE_DIRS RUNNER_CLEANUP_LOG_FILE + CRON_NAME="cleanup-runners" __cron "$cleanup_interval" /usr/local/bin/cleanup-runners + ) >>"$RUNNER_CLEANUP_LOG_FILE" 2>&1 & + disown "$!" 2>/dev/null || true + fi + fi # show exit message __banner "$postMessageEnd: Status $retVal" ) 2>"/dev/stderr" | tee -p -a "/data/logs/init.txt" &