mirror of
https://github.com/casjaysdevdocker/gitea
synced 2026-10-15 14:12:54 -04:00
✨ Add act_runner cleanup and apply full runner config ✨
Prune leftovers from CI jobs in the nested dockerd on a schedule, and make runner-N daemons use the full default_config.yaml instead of a cache-only config. - rootfs/usr/local/bin/cleanup-runners: new script that prunes stopped containers, networks, build cache, act-* volumes, dangling images, all unused images above a disk threshold, and stale ~/.cache/act clones - rootfs/usr/local/etc/docker/init.d/zz-act_runner.sh: add RUNNER_CLEANUP_* settings, run cleanup-runners via __cron, export RUNNERS_CONFIG_BASE - rootfs/usr/local/bin/start-runners: generate per-runner config.yaml from RUNNERS_CONFIG_BASE with its own runner.file; keep cache-only config as fallback - README.md: document RUNNER_CLEANUP_* variables, per-runner config, correct RUNNER_IP_ADDRESS default - TODO.AI.md: record both fixes and log newly found open issues Claude-Session: https://claude.ai/code/session_01Kae22tmhzhmfHyqc34PZbx
This commit is contained in:
5 files changed
+193
-8
No files matched your search
Executable
+106
@@ -0,0 +1,106 @@
|
||||
#!/usr/bin/env bash
|
||||
# shellcheck shell=bash
|
||||
# - - - - - - - - - - - - - - - - - - - - - - - - -
|
||||
##@Version : 202610031200-git
|
||||
# @@Author : Jason Hempstead
|
||||
# @@Contact : jason@casjaysdev.pro
|
||||
# @@License : LICENSE.md
|
||||
# @@ReadME : cleanup-runners --help
|
||||
# @@Copyright : Copyright: (c) 2026 Jason Hempstead, Casjays Developments
|
||||
# @@Created : Saturday, Oct 03, 2026 12:00 EDT
|
||||
# @@File : cleanup-runners
|
||||
# @@Description : Prune docker and act cache leftovers from act runner jobs
|
||||
# @@Changelog : New script
|
||||
# @@TODO : Better documentation
|
||||
# @@Other :
|
||||
# @@Resource :
|
||||
# @@Terminal App : no
|
||||
# @@sudo/root : no
|
||||
# @@Template : shell/bash
|
||||
# - - - - - - - - - - - - - - - - - - - - - - - - -
|
||||
# shellcheck disable=SC1001,SC1003,SC2001,SC2003,SC2016,SC2031,SC2090,SC2115,SC2120,SC2155,SC2199,SC2229,SC2317,SC2329
|
||||
# - - - - - - - - - - - - - - - - - - - - - - - - -
|
||||
VERSION="202610031200-git"
|
||||
set -eo pipefail
|
||||
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
|
||||
# Settings - all overridable via environment
|
||||
# Only prune containers/networks/build cache older than this (docker duration, e.g. 72h)
|
||||
RUNNER_CLEANUP_UNTIL="${RUNNER_CLEANUP_UNTIL:-72h}"
|
||||
# Remove all unused images (not just dangling) once the docker data dir reaches this usage percent
|
||||
RUNNER_CLEANUP_DISK_PERCENT="${RUNNER_CLEANUP_DISK_PERCENT:-80}"
|
||||
# Directory holding the nested dockerd's data, used for the disk usage check
|
||||
RUNNER_CLEANUP_DOCKER_DIR="${RUNNER_CLEANUP_DOCKER_DIR:-/data/docker}"
|
||||
# Remove cloned actions in act's cache that have not changed in this many days (0 disables)
|
||||
RUNNER_CLEANUP_ACT_CACHE_DAYS="${RUNNER_CLEANUP_ACT_CACHE_DAYS:-7}"
|
||||
# Space separated list of act action cache directories
|
||||
RUNNER_CLEANUP_ACT_CACHE_DIRS="${RUNNER_CLEANUP_ACT_CACHE_DIRS:-$HOME/.cache/act /root/.cache/act}"
|
||||
RUNNER_CLEANUP_LOG_FILE="${RUNNER_CLEANUP_LOG_FILE:-/data/logs/act_runner/cleanup.log}"
|
||||
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
|
||||
__log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" >>"$RUNNER_CLEANUP_LOG_FILE" 2>&1; }
|
||||
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
|
||||
# Run a prune command, log its reclaimed-space line; a failure is logged, never fatal
|
||||
__prune() {
|
||||
local output=""
|
||||
if output="$("$@" 2>&1)"; then
|
||||
__log "$* :: $(printf '%s\n' "$output" | grep -i -- 'reclaimed' | tail -n1)"
|
||||
else
|
||||
__log "FAILED: $* :: $output"
|
||||
fi
|
||||
}
|
||||
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
|
||||
# Print the usage percent of the filesystem holding $1
|
||||
__disk_percent() {
|
||||
df -P "$1" 2>/dev/null | awk 'NR==2 {gsub("%", "", $5); print $5}'
|
||||
}
|
||||
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
|
||||
if [ "$1" = "--help" ] || [ "$1" = "-h" ]; then
|
||||
printf '%s\n' "Usage: cleanup-runners [--help|--version]" \
|
||||
"Prunes docker containers, volumes, networks, build cache and images left by act runner jobs." \
|
||||
"Env: RUNNER_CLEANUP_UNTIL RUNNER_CLEANUP_DISK_PERCENT RUNNER_CLEANUP_DOCKER_DIR" \
|
||||
" RUNNER_CLEANUP_ACT_CACHE_DAYS RUNNER_CLEANUP_ACT_CACHE_DIRS RUNNER_CLEANUP_LOG_FILE"
|
||||
exit 0
|
||||
elif [ "$1" = "--version" ]; then
|
||||
echo "$VERSION"
|
||||
exit 0
|
||||
fi
|
||||
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
|
||||
mkdir -p "$(dirname -- "$RUNNER_CLEANUP_LOG_FILE")"
|
||||
if ! docker info >/dev/null 2>&1; then
|
||||
__log "docker is not reachable, skipping docker cleanup"
|
||||
else
|
||||
__log "Starting cleanup"
|
||||
# Exited job containers left behind when a runner died mid-job (running containers are never touched)
|
||||
__prune docker container prune -f --filter "until=$RUNNER_CLEANUP_UNTIL"
|
||||
# Job networks that no container uses anymore
|
||||
__prune docker network prune -f --filter "until=$RUNNER_CLEANUP_UNTIL"
|
||||
# Unused act-* job volumes; docker recreates a named volume if a starting job needs it
|
||||
act_volumes="$(docker volume ls -q --filter dangling=true --filter name=act- 2>/dev/null || true)"
|
||||
if [ -n "$act_volumes" ]; then
|
||||
# shellcheck disable=SC2086
|
||||
__prune docker volume rm $act_volumes
|
||||
fi
|
||||
# Anonymous volumes no container uses
|
||||
__prune docker volume prune -f
|
||||
# Build cache from jobs that run docker build
|
||||
__prune docker builder prune -af --filter "until=$RUNNER_CLEANUP_UNTIL"
|
||||
# Untagged (dangling) images
|
||||
__prune docker image prune -f
|
||||
# Every image not used by a container, only when disk is getting full; jobs re-pull what they need
|
||||
disk_used="$(__disk_percent "$RUNNER_CLEANUP_DOCKER_DIR" || true)"
|
||||
if [[ "$disk_used" =~ ^[0-9]+$ ]] && [ "$disk_used" -ge "$RUNNER_CLEANUP_DISK_PERCENT" ]; then
|
||||
__log "$RUNNER_CLEANUP_DOCKER_DIR is ${disk_used}% used (limit ${RUNNER_CLEANUP_DISK_PERCENT}%), removing unused images"
|
||||
__prune docker image prune -af
|
||||
fi
|
||||
fi
|
||||
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
|
||||
# Cloned actions (one directory per action@ref) that have not been updated recently
|
||||
if [[ "$RUNNER_CLEANUP_ACT_CACHE_DAYS" =~ ^[1-9][0-9]*$ ]]; then
|
||||
for act_cache_dir in $RUNNER_CLEANUP_ACT_CACHE_DIRS; do
|
||||
[ -d "$act_cache_dir" ] || continue
|
||||
removed="$(find "$act_cache_dir" -mindepth 1 -maxdepth 1 -mtime +"$RUNNER_CLEANUP_ACT_CACHE_DAYS" 2>/dev/null | wc -l || true)"
|
||||
find "$act_cache_dir" -mindepth 1 -maxdepth 1 -mtime +"$RUNNER_CLEANUP_ACT_CACHE_DAYS" -exec rm -rf {} + 2>/dev/null || true
|
||||
__log "Removed $removed stale entries from $act_cache_dir"
|
||||
done
|
||||
fi
|
||||
__log "Cleanup finished"
|
||||
exit 0
|
||||
@@ -100,10 +100,13 @@ __log "Starting $RUNNERS_START act_runner instance(s)"
|
||||
__log "Server Address: $SERVER_ADDRESS"
|
||||
__log "Runner Name Prefix: ${RUNNER_NAME_PREFIX:-runner}"
|
||||
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
|
||||
# Shared cache config so runner-N instances use the same cache-server
|
||||
# started by zz-act_runner.sh instead of an unshared per-process cache.
|
||||
# Full base config (container/runner/host/cache settings) rendered by zz-act_runner.sh.
|
||||
# Falls back to a cache-only config so runner-N instances still share the cache-server.
|
||||
RUNNERS_CACHE_CONFIG=""
|
||||
if [ -n "$RUNNER_CACHE_HOST" ] && [ -n "$RUNNER_CACHE_PORT" ] && [ -n "$RUNNER_CACHE_SECRET" ]; then
|
||||
if [ -n "$RUNNERS_CONFIG_BASE" ] && [ -f "$RUNNERS_CONFIG_BASE" ]; then
|
||||
__log "Base config: $RUNNERS_CONFIG_BASE"
|
||||
elif [ -n "$RUNNER_CACHE_HOST" ] && [ -n "$RUNNER_CACHE_PORT" ] && [ -n "$RUNNER_CACHE_SECRET" ]; then
|
||||
RUNNERS_CONFIG_BASE=""
|
||||
RUNNERS_CACHE_CONFIG="/config/act_runner/runners-cache.yaml"
|
||||
cat <<EOF >"$RUNNERS_CACHE_CONFIG"
|
||||
cache:
|
||||
@@ -111,18 +114,34 @@ cache:
|
||||
external_server: 'http://${RUNNER_CACHE_HOST}:${RUNNER_CACHE_PORT}/'
|
||||
external_secret: '${RUNNER_CACHE_SECRET}'
|
||||
EOF
|
||||
else
|
||||
RUNNERS_CONFIG_BASE=""
|
||||
fi
|
||||
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
|
||||
# Print the config a runner uses; per-runner copy of the base config with its own runner.file
|
||||
__runner_config() {
|
||||
local runner_dir=$1
|
||||
local runner_config="$runner_dir/config.yaml"
|
||||
if [ -n "$RUNNERS_CONFIG_BASE" ]; then
|
||||
sed "s|^\([[:space:]]*\)file:.*|\1file: $runner_dir/.runner|" "$RUNNERS_CONFIG_BASE" >"$runner_config" || return 1
|
||||
echo "$runner_config"
|
||||
elif [ -n "$RUNNERS_CACHE_CONFIG" ]; then
|
||||
echo "$RUNNERS_CACHE_CONFIG"
|
||||
fi
|
||||
}
|
||||
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
|
||||
# Register a single runner synchronously (no daemon start)
|
||||
__register_runner() {
|
||||
local runner_id=$1
|
||||
local runner_name="${RUNNER_NAME_PREFIX:-runner}-${runner_id}"
|
||||
local runner_dir="/config/act_runner/reg/${runner_name}"
|
||||
local config_args=()
|
||||
local runner_config=""
|
||||
|
||||
mkdir -p "$runner_dir"
|
||||
[ -d "$runner_dir" ] && cd "$runner_dir" || return 1
|
||||
[ -n "$RUNNERS_CACHE_CONFIG" ] && config_args=(--config "$RUNNERS_CACHE_CONFIG")
|
||||
runner_config="$(__runner_config "$runner_dir")" || return 1
|
||||
[ -n "$runner_config" ] && config_args=(--config "$runner_config")
|
||||
|
||||
if [ ! -f "$runner_dir/.runner" ]; then
|
||||
__log "Registering runner: $runner_name (ID: $runner_id)"
|
||||
@@ -141,9 +160,11 @@ __start_runner_daemon() {
|
||||
local runner_name="${RUNNER_NAME_PREFIX:-runner}-${runner_id}"
|
||||
local runner_dir="/config/act_runner/reg/${runner_name}"
|
||||
local config_args=()
|
||||
local runner_config=""
|
||||
|
||||
[ -d "$runner_dir" ] && cd "$runner_dir" || return 1
|
||||
[ -n "$RUNNERS_CACHE_CONFIG" ] && config_args=(--config "$RUNNERS_CACHE_CONFIG")
|
||||
runner_config="$(__runner_config "$runner_dir")" || return 1
|
||||
[ -n "$runner_config" ] && config_args=(--config "$runner_config")
|
||||
__log "Starting daemon for runner: $runner_name"
|
||||
exec act_runner daemon "${config_args[@]}"
|
||||
}
|
||||
|
||||
@@ -341,6 +341,10 @@ RUNNER_CACHE_HOST="${RUNNER_CACHE_HOST:-$IP4_ADDRESS}"
|
||||
CACHE_CONFIG_FILE="${CACHE_CONFIG_FILE:-$CONF_DIR/cache_server.yaml}"
|
||||
CACHE_LOG_FILE="${CACHE_LOG_FILE:-$LOG_DIR/cache.log}"
|
||||
RUNNER_CACHE_SECRET="${RUNNER_CACHE_SECRET:-}"
|
||||
# Periodic prune of the nested dockerd and act's action cache (see /usr/local/bin/cleanup-runners)
|
||||
RUNNER_CLEANUP_ENABLED="${RUNNER_CLEANUP_ENABLED:-yes}"
|
||||
RUNNER_CLEANUP_INTERVAL="${RUNNER_CLEANUP_INTERVAL:-60}"
|
||||
RUNNER_CLEANUP_LOG_FILE="${RUNNER_CLEANUP_LOG_FILE:-$LOG_DIR/cleanup.log}"
|
||||
# - - - - - - - - - - - - - - - - - - - - - - - - -
|
||||
# Additional variables
|
||||
|
||||
@@ -523,6 +527,8 @@ __post_execute() {
|
||||
export SERVER_ADDRESS="$RUNNER_IP_ADDRESS:$GITEA_PORT" SERVER_TOKEN="${RUNNER_AUTH_TOKEN:-$SYS_AUTH_TOKEN}"
|
||||
export RUNNER_CACHE_HOST RUNNER_CACHE_PORT
|
||||
export RUNNER_CACHE_SECRET="${RUNNER_CACHE_SECRET:-$(__gen_cache_secret)}"
|
||||
# full rendered config so runner-N daemons get the container/runner/host settings too
|
||||
export RUNNERS_CONFIG_BASE="$RUNNER_DEFAULT_HOME/$RUNNER_CONFIG_NAME"
|
||||
|
||||
# wait
|
||||
sleep $waitTime
|
||||
@@ -568,6 +574,16 @@ __post_execute() {
|
||||
# subshell (itself the left side of the __post_execute pipe) reaches its end
|
||||
disown "$!" 2>/dev/null || true
|
||||
fi
|
||||
# periodic cleanup loop; output goes to a real file for the same fd-leak reason as above
|
||||
if [ "$RUNNER_CLEANUP_ENABLED" = "yes" ] && [ -x "/usr/local/bin/cleanup-runners" ]; then
|
||||
[[ "$RUNNER_CLEANUP_INTERVAL" =~ ^[1-9][0-9]*$ ]] || RUNNER_CLEANUP_INTERVAL="60"
|
||||
(
|
||||
export RUNNER_CLEANUP_LOG_FILE
|
||||
CRON_NAME="cleanup-runners" __cron "$RUNNER_CLEANUP_INTERVAL" /usr/local/bin/cleanup-runners
|
||||
) >>"$RUNNER_CLEANUP_LOG_FILE" 2>&1 &
|
||||
disown "$!" 2>/dev/null || true
|
||||
echo "Runner cleanup scheduled every $RUNNER_CLEANUP_INTERVAL minutes"
|
||||
fi
|
||||
# show exit message
|
||||
__banner "$postMessageEnd: Status $retVal"
|
||||
) 2>"/dev/stderr" | tee -p -a "/data/logs/init.txt" &
|
||||
|
||||
Reference in new issue
Block a user