casjaysdevdocker/forgejo
1
0 Fork 0
mirror of https://github.com/casjaysdevdocker/forgejo synced 2026-10-15 14:12:55 -04:00
Code Issues Actions Packages Projects Releases Wiki Activity

fix: harden Forgejo runner lifecycle

Browse Source 
- README.md: document full runner configuration, isolated state, and cleanup settings
- IDEA.md: N/A - no user-facing product behavior change
- TODO.AI.md: record runner cleanup and runtime verification status
- rootfs/usr/local/bin/cleanup-runners: add safe stale Docker and act-cache cleanup
- rootfs/usr/local/bin/start-runners: propagate per-runner config and report daemon failures
- rootfs/usr/local/etc/docker/init.d/05-dockerd.sh: seed Docker config without overwriting user files
- rootfs/usr/local/etc/docker/init.d/08-forgejo.sh: seed Forgejo config without overwriting user files
- rootfs/usr/local/etc/docker/init.d/zz-act_runner.sh: seed runner config safely and schedule validated cleanup
This commit is contained in:
casjay committed 2026-10-05 10:21:09 -04:00
1 parent 67db5abd86
commit 67d9624809
7 files changed
+227 -23
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

rootfs/usr/local/bin/cleanup-runners
Executable
+95
View File
@@ -0,0 +1,95 @@
#!/usr/bin/env bash
# shellcheck shell=bash
# - - - - - - - - - - - - - - - - - - - - - - - - -
##@Version : 202610051200-git
# @@Author : Jason Hempstead
# @@Contact : jason@casjaysdev.pro
# @@License : LICENSE.md
# @@ReadME : cleanup-runners --help
# @@Copyright : Copyright: (c) 2026 Jason Hempstead, Casjays Developments
# @@Description : Prune docker and act cache leftovers from act runner jobs
# - - - - - - - - - - - - - - - - - - - - - - - - -
VERSION="202610051200-git"
set -eo pipefail
RUNNER_CLEANUP_UNTIL="${RUNNER_CLEANUP_UNTIL:-72h}"
RUNNER_CLEANUP_DISK_PERCENT="${RUNNER_CLEANUP_DISK_PERCENT:-80}"
RUNNER_CLEANUP_DOCKER_DIR="${RUNNER_CLEANUP_DOCKER_DIR:-/data/docker}"
RUNNER_CLEANUP_ACT_CACHE_DAYS="${RUNNER_CLEANUP_ACT_CACHE_DAYS:-7}"
RUNNER_CLEANUP_ACT_CACHE_DIRS="${RUNNER_CLEANUP_ACT_CACHE_DIRS:-$HOME/.cache/act /root/.cache/act}"
RUNNER_CLEANUP_LOG_FILE="${RUNNER_CLEANUP_LOG_FILE:-/data/logs/act_runner/cleanup.log}"
if ! [[ "$RUNNER_CLEANUP_DISK_PERCENT" =~ ^(100|[0-9]{1,2})$ ]]; then
RUNNER_CLEANUP_DISK_PERCENT=80
fi
__log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" >>"$RUNNER_CLEANUP_LOG_FILE" 2>&1; }
__prune() {
local output=""
if output=$("$@" 2>&1); then
__log "$* :: $(printf '%s\n' "$output" | grep -i -- 'reclaimed' | tail -n1)"
else
__log "FAILED: $* :: $output"
fi
}
__disk_percent() {
df -P "$1" 2>/dev/null | awk 'NR==2 {gsub("%", "", $5); print $5}'
}
__help() {
cat <<'EOF'
Usage: cleanup-runners [--help|--version]
Prune stale Docker runner resources, build cache, images, and act runner
cache entries. Only explicitly dangling volumes matching the act- prefix are removed.
Environment:
RUNNER_CLEANUP_UNTIL Docker prune age (default: 72h)
RUNNER_CLEANUP_DISK_PERCENT Image cleanup threshold (default: 80)
RUNNER_CLEANUP_DOCKER_DIR Docker data path (default: /data/docker)
RUNNER_CLEANUP_ACT_CACHE_DAYS Cache age in days (default: 7)
RUNNER_CLEANUP_ACT_CACHE_DIRS Space-separated act cache directories
RUNNER_CLEANUP_LOG_FILE Cleanup log path
EOF
}
case "${1:-}" in
-h|--help)
__help
exit 0
;;
--version)
echo "$VERSION"
exit 0
;;
esac
mkdir -p "$(dirname -- "$RUNNER_CLEANUP_LOG_FILE")"
if ! docker info >/dev/null 2>&1; then
__log "docker is not reachable, skipping Docker cleanup"
else
__log "Starting cleanup"
__prune docker container prune -f --filter "until=$RUNNER_CLEANUP_UNTIL"
__prune docker network prune -f --filter "until=$RUNNER_CLEANUP_UNTIL"
act_volumes="$(docker volume ls -q --filter dangling=true --filter name=act- 2>/dev/null | grep -E -- '^act-' || true)"
if [ -n "$act_volumes" ]; then
# shellcheck disable=SC2086
__prune docker volume rm $act_volumes
fi
__prune docker builder prune -af --filter "until=$RUNNER_CLEANUP_UNTIL"
__prune docker image prune -f --filter "until=$RUNNER_CLEANUP_UNTIL"
disk_used="$(__disk_percent "$RUNNER_CLEANUP_DOCKER_DIR" || true)"
if [[ "$disk_used" =~ ^[0-9]+$ ]] && [ "$disk_used" -ge "$RUNNER_CLEANUP_DISK_PERCENT" ]; then
__log "$RUNNER_CLEANUP_DOCKER_DIR is ${disk_used}% used (limit ${RUNNER_CLEANUP_DISK_PERCENT}%), removing unused images"
__prune docker image prune -af --filter "until=$RUNNER_CLEANUP_UNTIL"
fi
fi
if [[ "$RUNNER_CLEANUP_ACT_CACHE_DAYS" =~ ^[1-9][0-9]*$ ]]; then
for act_cache_dir in $RUNNER_CLEANUP_ACT_CACHE_DIRS; do
[ -d "$act_cache_dir" ] || continue
removed="$(find "$act_cache_dir" -mindepth 1 -maxdepth 1 -mtime +"$RUNNER_CLEANUP_ACT_CACHE_DAYS" 2>/dev/null | wc -l || true)"
find "$act_cache_dir" -mindepth 1 -maxdepth 1 -mtime +"$RUNNER_CLEANUP_ACT_CACHE_DAYS" -exec rm -rf {} + 2>/dev/null || true
__log "Removed $removed stale entries from $act_cache_dir"
done
fi
__log "Cleanup finished"
exit 0
rootfs/usr/local/bin/start-runners
+82 -18
View File
@@ -28,6 +28,34 @@ echo "${emoji}Fatal error occurred: Exit code $retVal at line $LINENO in command
kill -TERM 1' ERR
trap 'retVal=$?;if [ "$SERVICE_IS_RUNNING" != "yes" ] && [ -f "$SERVICE_PID_FILE" ]; then rm -Rf "$SERVICE_PID_FILE"; fi;exit $retVal' SIGINT SIGTERM SIGPWR
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
__help() {
cat <<'EOF'
Usage: start-runners
Register and start one or more Forgejo act_runner daemons in parallel.
Environment:
SERVER_ADDRESS Forgejo server URL (defaults to http://$HOSTNAME)
SERVER_TOKEN Required runner registration token
RUNNERS_LOG_DIR Runner log directory
RUNNERS_START Number of runners to start (default: 1)
RUNNER_NAME_PREFIX Runner name prefix (default: runner)
RUNNER_LABELS Comma-separated runner labels
RUNNERS_CONFIG_BASE Base act_runner config file
RUNNER_CACHE_HOST External cache host (used with RUNNER_CACHE_PORT and
RUNNER_CACHE_SECRET when no base config is available)
RUNNER_CACHE_PORT External cache port
RUNNER_CACHE_SECRET External cache secret
EOF
}
case "${1:-}" in
-h|--help)
__help
exit 0
;;
esac
# Function to __log messages with timestamp
__log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" >>"$RUNNERS_LOG_DIR/runners" 2>&1; }
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
@@ -100,29 +128,55 @@ __log "Starting $RUNNERS_START act_runner instance(s)"
__log "Server Address: $SERVER_ADDRESS"
__log "Runner Name Prefix: ${RUNNER_NAME_PREFIX:-runner}"
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
# Shared cache config so runner-N instances use the same cache-server
# started by zz-act_runner.sh instead of an unshared per-process cache.
# Full base config rendered by zz-act_runner.sh. Fall back to a shared cache-only
# config when this script is invoked without a rendered Forgejo runner config.
RUNNERS_DEFAULT_HOME="${RUNNER_DEFAULT_HOME:-/config/act_runner/forgejo}"
# Keep the established registration path by default; callers may override it
# independently when relocating runner state.
RUNNERS_REGISTRATION_ROOT="${RUNNER_REGISTRATION_ROOT:-/config/act_runner/reg}"
RUNNERS_CACHE_CONFIG=""
if [ -n "$RUNNER_CACHE_HOST" ] && [ -n "$RUNNER_CACHE_PORT" ] && [ -n "$RUNNER_CACHE_SECRET" ]; then
if [ -n "$RUNNERS_CONFIG_BASE" ] && [ -f "$RUNNERS_CONFIG_BASE" ]; then
__log "Base config: $RUNNERS_CONFIG_BASE"
elif [ -n "$RUNNER_CACHE_HOST" ] && [ -n "$RUNNER_CACHE_PORT" ] && [ -n "$RUNNER_CACHE_SECRET" ]; then
RUNNERS_CONFIG_BASE=""
RUNNERS_CACHE_CONFIG="/config/act_runner/runners-cache.yaml"
cat <<EOF >"$RUNNERS_CACHE_CONFIG"
runner:
file: REPLACE_RUNNER_STATE/.runner
cache:
enabled: true
external_server: 'http://${RUNNER_CACHE_HOST}:${RUNNER_CACHE_PORT}/'
secret: '${RUNNER_CACHE_SECRET}'
EOF
else
RUNNERS_CONFIG_BASE=""
fi
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
# Create the config used by one runner, isolating its registration state.
__runner_config() {
local runner_dir=$1
local runner_config="$runner_dir/config.yaml"
if [ -n "$RUNNERS_CONFIG_BASE" ]; then
sed "s|^\([[:space:]]*\)file:.*|\1file: $runner_dir/.runner|" "$RUNNERS_CONFIG_BASE" >"$runner_config" || return 1
echo "$runner_config"
elif [ -n "$RUNNERS_CACHE_CONFIG" ]; then
sed "s|REPLACE_RUNNER_STATE|$runner_dir|" "$RUNNERS_CACHE_CONFIG" >"$runner_config" || return 1
echo "$runner_config"
fi
}
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
# Register a single runner synchronously (no daemon start)
__register_runner() {
local runner_id=$1
local runner_name="${RUNNER_NAME_PREFIX:-runner}-${runner_id}"
local runner_dir="/config/act_runner/reg/${runner_name}"
local runner_dir="$RUNNERS_REGISTRATION_ROOT/${runner_name}"
local config_args=()
local runner_config=""
mkdir -p "$runner_dir"
[ -d "$runner_dir" ] && cd "$runner_dir" || return 1
[ -n "$RUNNERS_CACHE_CONFIG" ] && config_args=(--config "$RUNNERS_CACHE_CONFIG")
runner_config="$(__runner_config "$runner_dir")" || return 1
[ -n "$runner_config" ] && config_args=(--config "$runner_config")
if [ ! -f "$runner_dir/.runner" ]; then
__log "Registering runner: $runner_name (ID: $runner_id)"
@@ -139,11 +193,13 @@ __register_runner() {
__start_runner_daemon() {
local runner_id=$1
local runner_name="${RUNNER_NAME_PREFIX:-runner}-${runner_id}"
local runner_dir="/config/act_runner/reg/${runner_name}"
local runner_dir="$RUNNERS_REGISTRATION_ROOT/${runner_name}"
local config_args=()
local runner_config=""
[ -d "$runner_dir" ] && cd "$runner_dir" || return 1
[ -n "$RUNNERS_CACHE_CONFIG" ] && config_args=(--config "$RUNNERS_CACHE_CONFIG")
runner_config="$(__runner_config "$runner_dir")" || return 1
[ -n "$runner_config" ] && config_args=(--config "$runner_config")
__log "Starting daemon for runner: $runner_name"
exec act_runner daemon "${config_args[@]}"
}
@@ -155,20 +211,28 @@ done
unset i
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
# Phase 2: start all daemons in parallel
runner_pids=()
for i in $(seq 1 $RUNNERS_START); do
(__start_runner_daemon "$i") &
runner_pids+=("$!")
done
unset i
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
RUNNERS_ID="$(jobs -p | tr '\n' ' ')"
if [ -n "$RUNNERS_ID" ]; then
__log "All $RUNNERS_START runners started successfully"
__log "Process IDs: $RUNNERS_ID"
printf 'All %s runners started successfully: %s\n' "$RUNNERS_START" "$RUNNERS_ID"
else
__log "The runners have failed to start"
printf '%s\n' "The runners have failed to start"
RUNNERS_ID="${runner_pids[*]}"
__log "All $RUNNERS_START runners launched"
__log "Process IDs: $RUNNERS_ID"
printf 'All %s runners launched: %s\n' "$RUNNERS_START" "$RUNNERS_ID"
# Wait for all background processes and preserve a failure status. A daemon that
# exits is reported as failed instead of being treated as successfully started.
runner_failures=0
for runner_pid in "${runner_pids[@]}"; do
if ! wait "$runner_pid"; then
runner_failures=$((runner_failures + 1))
__log "ERROR: Runner process $runner_pid exited unsuccessfully"
fi
done
if [ "$runner_failures" -gt 0 ]; then
__log "$runner_failures runner process(es) failed"
printf '%s\n' "$runner_failures runner process(es) failed" >&2
exit 1
fi
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
# Wait for all background processes
wait
rootfs/usr/local/etc/docker/init.d/05-dockerd.sh
+1 -1
View File
@@ -279,7 +279,7 @@ __run_precopy() {
if [ -d "$ETC_DIR" ] && ! [ -L "$ETC_DIR" ]; then
if [ ! -f "$CONF_DIR/.initialized" ]; then
mkdir -p "$CONF_DIR"
cp -Rf "$ETC_DIR/." "$CONF_DIR/" 2>/dev/null || true
cp -Rn "$ETC_DIR/." "$CONF_DIR/" 2>/dev/null || true
fi
rm -Rf "$ETC_DIR"
ln -sf "$CONF_DIR" "$ETC_DIR"
rootfs/usr/local/etc/docker/init.d/08-forgejo.sh
+1 -1
View File
@@ -323,7 +323,7 @@ __run_precopy() {
if [ -d "$ETC_DIR" ] && ! [ -L "$ETC_DIR" ]; then
if [ ! -f "$CONF_DIR/.initialized" ]; then
mkdir -p "$CONF_DIR"
cp -Rf "$ETC_DIR/." "$CONF_DIR/" 2>/dev/null || true
cp -Rn "$ETC_DIR/." "$CONF_DIR/" 2>/dev/null || true
fi
rm -Rf "$ETC_DIR"
ln -sf "$CONF_DIR" "$ETC_DIR"
rootfs/usr/local/etc/docker/init.d/zz-act_runner.sh
+31 -1
View File
@@ -341,6 +341,17 @@ RUNNER_CACHE_HOST="${RUNNER_CACHE_HOST:-$IP4_ADDRESS}"
CACHE_CONFIG_FILE="${CACHE_CONFIG_FILE:-$CONF_DIR/cache_server.yaml}"
CACHE_LOG_FILE="${CACHE_LOG_FILE:-$LOG_DIR/cache.log}"
RUNNER_CACHE_SECRET="${RUNNER_CACHE_SECRET:-}"
# Rendered full runner config consumed by each per-runner daemon.
RUNNERS_CONFIG_BASE="${RUNNERS_CONFIG_BASE:-$RUNNER_DEFAULT_HOME/$RUNNER_CONFIG_NAME}"
# Periodic cleanup of stopped job resources and stale act caches.
RUNNER_CLEANUP_ENABLED="${RUNNER_CLEANUP_ENABLED:-yes}"
RUNNER_CLEANUP_INTERVAL="${RUNNER_CLEANUP_INTERVAL:-60}"
RUNNER_CLEANUP_UNTIL="${RUNNER_CLEANUP_UNTIL:-72h}"
RUNNER_CLEANUP_DISK_PERCENT="${RUNNER_CLEANUP_DISK_PERCENT:-80}"
RUNNER_CLEANUP_DOCKER_DIR="${RUNNER_CLEANUP_DOCKER_DIR:-/data/docker}"
RUNNER_CLEANUP_ACT_CACHE_DAYS="${RUNNER_CLEANUP_ACT_CACHE_DAYS:-7}"
RUNNER_CLEANUP_ACT_CACHE_DIRS="${RUNNER_CLEANUP_ACT_CACHE_DIRS:-$HOME/.cache/act /root/.cache/act}"
RUNNER_CLEANUP_LOG_FILE="${RUNNER_CLEANUP_LOG_FILE:-$LOG_DIR/cleanup.log}"
# - - - - - - - - - - - - - - - - - - - - - - - - -
# Additional variables
@@ -375,7 +386,7 @@ __run_precopy() {
if [ -d "$ETC_DIR" ] && ! [ -L "$ETC_DIR" ]; then
if [ ! -f "$CONF_DIR/.initialized" ]; then
mkdir -p "$CONF_DIR"
cp -Rf "$ETC_DIR/." "$CONF_DIR/" 2>/dev/null || true
cp -Rn "$ETC_DIR/." "$CONF_DIR/" 2>/dev/null || true
fi
rm -Rf "$ETC_DIR"
ln -sf "$CONF_DIR" "$ETC_DIR"
@@ -523,6 +534,10 @@ __post_execute() {
export SERVER_ADDRESS="$RUNNER_IP_ADDRESS:$FORGEJO_PORT" SERVER_TOKEN="${RUNNER_AUTH_TOKEN:-$SYS_AUTH_TOKEN}"
export RUNNER_CACHE_HOST RUNNER_CACHE_PORT
export RUNNER_CACHE_SECRET="${RUNNER_CACHE_SECRET:-$(__gen_cache_secret)}"
export RUNNERS_CONFIG_BASE
export RUNNER_CLEANUP_ENABLED RUNNER_CLEANUP_INTERVAL RUNNER_CLEANUP_UNTIL
export RUNNER_CLEANUP_DISK_PERCENT RUNNER_CLEANUP_DOCKER_DIR
export RUNNER_CLEANUP_ACT_CACHE_DAYS RUNNER_CLEANUP_ACT_CACHE_DIRS RUNNER_CLEANUP_LOG_FILE
# wait
sleep $waitTime
@@ -568,6 +583,21 @@ __post_execute() {
# subshell (itself the left side of the __post_execute pipe) reaches its end
disown "$!" 2>/dev/null || true
fi
if [ "${RUNNER_CLEANUP_ENABLED,,}" = "yes" ] || [ "${RUNNER_CLEANUP_ENABLED,,}" = "true" ] || [ "$RUNNER_CLEANUP_ENABLED" = "1" ]; then
if [ -x "/usr/local/bin/cleanup-runners" ]; then
cleanup_interval="${RUNNER_CLEANUP_INTERVAL:-60}"
if ! [[ "$cleanup_interval" =~ ^[1-9][0-9]*$ ]]; then
cleanup_interval=60
fi
mkdir -p "$(dirname -- "$RUNNER_CLEANUP_LOG_FILE")"
(
export RUNNER_CLEANUP_UNTIL RUNNER_CLEANUP_DISK_PERCENT RUNNER_CLEANUP_DOCKER_DIR
export RUNNER_CLEANUP_ACT_CACHE_DAYS RUNNER_CLEANUP_ACT_CACHE_DIRS RUNNER_CLEANUP_LOG_FILE
CRON_NAME="cleanup-runners" __cron "$cleanup_interval" /usr/local/bin/cleanup-runners
) >>"$RUNNER_CLEANUP_LOG_FILE" 2>&1 &
disown "$!" 2>/dev/null || true
fi
fi
# show exit message
__banner "$postMessageEnd: Status $retVal"
) 2>"/dev/stderr" | tee -p -a "/data/logs/init.txt" &