hmm

app/host_vars/hosts

@@ -1 +1 @@
-sydney
+sydney2

app/onboarding

@@ -71,4 +71,3 @@ rex doas mkdir -p /etc/doas.d
 
 rex doas rc-service sshd restart
 rex doas rc-service nebula restart
-rex doas reboot

app/sbin/api

@@ -386,40 +386,23 @@ def get_cpu_log():
         raise HTTPException(status_code=500, detail=f"Error reading CPU log: {e}")
 
 
-@app.get("/system/info", dependencies=[Depends(verify_api_key)])
-def get_system_info():
+@app.get("/server/audit", dependencies=[Depends(verify_api_key)])
+def server_audit():
+    output = run_command([f"{BIN_PATH}/audit"])
+    # The audit script outputs JSON; parse and return it
     try:
-        alpine_version = None
-        last_update = None
-        bump_dates = execute_db(
-            "SELECT MAX(bump) AS latest_bump FROM containers", fetch=True
-        )[0]["latest_bump"]
-        if os.path.exists("/4server/data/update"):
-            with open("/4server/data/update") as f:
-                last_update = f.read().strip()
-        if os.path.exists("/etc/alpine-release"):
-            with open("/etc/alpine-release") as f:
-                alpine_version = f.read().strip()
-        mem = psutil.virtual_memory()
-        disk = psutil.disk_usage("/")
-        cpu_count = psutil.cpu_count(logical=True)
-        return {
-            "alpine_version": alpine_version,
-            "last_update": last_update,
-            "latest_bump": bump_dates,
-            "version": VERSION,
-            "resources": {
-                "memory": {
-                    "total": mem.total,
-                    "available": mem.available,
-                    "used": mem.used,
-                },
-                "disk": {"total": disk.total, "used": disk.used, "free": disk.free},
-                "cpu_count": cpu_count,
-            },
-        }
-    except Exception as e:
-        raise HTTPException(status_code=500, detail=str(e))
+        return json.loads(output)
+    except json.JSONDecodeError:
+        return {"raw_output": output}
+
+
+@app.get("/server/stats", dependencies=[Depends(verify_api_key)])
+def server_stats():
+    output = run_command([f"{BIN_PATH}/stats"])
+    try:
+        return json.loads(output)
+    except json.JSONDecodeError:
+        return {"raw_output": output}
 
 
 @app.post("/system/pull", dependencies=[Depends(verify_api_key)])
@@ -576,6 +559,11 @@ def backup_check(request: BackupCheckRequest):
     return {"message": run_command([f"{BIN_PATH}/checkBackup", request.uuid])}
 
 
+@app.get("/backup/all", dependencies=[Depends(verify_api_key)])
+def backup_all():
+    return {"message": run_command([f"{BIN_PATH}/backupAll"])}
+
+
 @app.get("/backup/borgpush", dependencies=[Depends(verify_api_key)])
 def backup_borgpush():
     return {"message": run_command(["doas", "-u", "4server", f"{BIN_PATH}/borgpush"])}

app/sbin/audit

@@ -0,0 +1,139 @@
+#!/usr/bin/env python3
+"""Docker system audit: collect container diagnostics and return as JSON."""
+
+import json
+import os
+import subprocess
+
+
+def cmd(args, parse_json=False):
+    try:
+        result = subprocess.run(
+            args,
+            capture_output=True,
+            text=True,
+            timeout=30,
+        )
+        out = (result.stdout + result.stderr).strip()
+
+        if parse_json:
+            try:
+                return json.loads(out) if out else None
+            except json.JSONDecodeError:
+                return out
+
+        return out
+
+    except Exception as e:
+        return f"ERROR: {e}"
+
+
+def read_file(path):
+    try:
+        with open(path) as f:
+            return f.read().strip().splitlines()
+    except FileNotFoundError:
+        return None
+
+
+def main():
+    audit = {}
+
+    # Container identity
+    audit["container_id"] = os.environ.get("HOSTNAME")
+
+    # OS information inside container
+    audit["os_release"] = read_file("/etc/os-release")
+    audit["uname"] = cmd(["uname", "-a"])
+
+    # Installed packages (works for Debian/Ubuntu, Alpine, etc.)
+    audit["packages"] = {}
+
+    if os.path.exists("/etc/alpine-release"):
+        audit["packages"]["manager"] = "apk"
+        audit["packages"]["installed"] = cmd(["apk", "info", "-v"]).splitlines()
+        audit["packages"]["upgradeable"] = cmd(
+            ["apk", "version", "-l", "<"]
+        ).splitlines()
+
+    elif os.path.exists("/etc/debian_version"):
+        audit["packages"]["manager"] = "dpkg"
+        audit["packages"]["installed"] = cmd(
+            ["dpkg-query", "-W", "-f=${Package}-${Version}\n"]
+        ).splitlines()
+
+    elif os.path.exists("/etc/redhat-release"):
+        audit["packages"]["manager"] = "rpm"
+        audit["packages"]["installed"] = cmd(["rpm", "-qa"]).splitlines()
+
+    # Docker engine information
+    audit["docker_version"] = cmd(
+        ["docker", "version", "--format", "{{json .}}"],
+        parse_json=True,
+    )
+
+    audit["docker_info"] = cmd(
+        ["docker", "info", "--format", "{{json .}}"],
+        parse_json=True,
+    )
+
+    # Containers
+    containers = cmd(
+        ["docker", "ps", "-a", "--format", "{{json .}}"],
+        parse_json=True,
+    )
+
+    if isinstance(containers, list):
+        audit["docker_containers"] = containers
+    else:
+        audit["docker_containers"] = []
+
+    # Images
+    audit["docker_images"] = cmd(
+        ["docker", "images", "--format", "{{json .}}"],
+        parse_json=True,
+    )
+
+    # Runtime stats
+    audit["docker_stats"] = cmd(
+        [
+            "docker",
+            "stats",
+            "--no-stream",
+            "--format",
+            "table {{.Name}}\t{{.CPUPerc}}\t{{.MemUsage}}\t{{.NetIO}}",
+        ]
+    )
+
+    # Container logs
+    logs = {}
+
+    for c in audit["docker_containers"]:
+        name = c.get("Names") or c.get("ID")
+
+        if not name:
+            continue
+
+        logs[name] = cmd(
+            [
+                "docker",
+                "logs",
+                "--tail",
+                "100",
+                name,
+            ]
+        )
+
+    audit["docker_logs"] = logs
+
+    # SSH config if present
+    audit["sshd_config"] = cmd(["sshd", "-T"]).splitlines()
+
+    # Users
+    audit["passwd"] = read_file("/etc/passwd")
+
+    print(json.dumps(audit, indent=2))
+
+
+if __name__ == "__main__":
+    main()

app/sbin/backup/ODOO

@@ -13,7 +13,7 @@ source /4server/sbin/helpers
 get_contract_info
 
 log "UUID:         $UUID"
-log "Backup slots: $BACKUP_SLOTS"
+log "Retention: daily 14d, weekly 12w"
 log "DB host:      $POSTGRES_HOST"
 
 # Build paths
@@ -48,13 +48,51 @@ doas docker exec "$UUID" chown odoo:odoo -R /mnt/backup
 doas ln -sf "../$FILENAME" "$BB_DIR/current.zip"
 log "bb/current.zip -> ../$FILENAME"
 
-# Remove old backups beyond the configured slot count
-# Process substitution (< <(...)) keeps the loop in the main shell so
-# set -e / exit codes propagate correctly — unlike a pipe subshell.
-while IFS= read -r old_file; do
-    log "Deleting old backup: $old_file"
-    doas rm -f "$old_file"
-done < <(ls -t "$BACKUP_DIR"/[0-9]*.zip 2>/dev/null | tail -n +$((BACKUP_SLOTS + 3)))
+# --- Backup retention ---
+# Policy:
+#   - Keep all backups from the last 14 days (daily tier)
+#   - For backups older than 14 days but within 3 months (12 weeks),
+#     keep one per ISO week (the newest in each week)
+#   - Delete everything else
+BACKUP_DAYS_DAILY=14
+BACKUP_WEEKS_WEEKLY=12   # ~3 months
+
+# Map of kept files (absolute path -> 1)
+declare -A kept_files
+# Track which ISO weeks already have a kept weekly backup
+declare -A weekly_kept
+
+now_ts=$(date +%s)
+
+while IFS= read -r file; do
+    filename=$(basename "$file")
+    filedate="${filename:0:8}"          # YYYYMMDD from filename
+    # BusyBox date (Alpine) requires YYYY-MM-DD; compact YYYYMMDD is rejected
+    filedate_fmt="${filedate:0:4}-${filedate:4:2}-${filedate:6:2}"
+    file_ts=$(date -d "$filedate_fmt" +%s 2>/dev/null) || continue
+    age_days=$(( (now_ts - file_ts) / 86400 ))
+
+    if [ "$age_days" -le "$BACKUP_DAYS_DAILY" ]; then
+        # --- Daily tier: keep everything within the last 14 days ---
+        kept_files["$file"]=1
+    elif [ "$age_days" -le $((BACKUP_DAYS_DAILY + BACKUP_WEEKS_WEEKLY * 7)) ]; then
+        # --- Weekly tier: keep 1 per ISO week ---
+        week_key=$(date -d "$filedate_fmt" +%G%V)   # ISO year + week number
+        if [ -z "${weekly_kept[$week_key]-}" ]; then
+            weekly_kept[$week_key]=1
+            kept_files["$file"]=1
+        fi
+    fi
+done < <(ls -t "$BACKUP_DIR"/[0-9]*.zip 2>/dev/null)
+
+# Delete any backup not in the keep map
+for file in "$BACKUP_DIR"/[0-9]*.zip; do
+    [ -f "$file" ] || continue
+    if [ -z "${kept_files[$file]-}" ]; then
+        log "Deleting old backup: $file"
+        doas rm -f "$file"
+    fi
+done
 
 log "Backup finished successfully ."
 

app/sbin/checkBackup

@@ -17,6 +17,7 @@ if not directory.is_dir():
     sys.exit(1)
 
 latest_date = None
+latest_file = None
 
 for zip_file in directory.rglob("*.zip"):
     name = zip_file.name
@@ -29,10 +30,15 @@ for zip_file in directory.rglob("*.zip"):
 
     if latest_date is None or date_part > latest_date:
         latest_date = date_part
+        latest_file = zip_file
 
 if latest_date is None:
     print("NO BACKUPS")
     sys.exit(1)
 
+if latest_file.stat().st_size < 1000:
+    print("01-01-2000")
+    sys.exit(0)
+
 # Format as yyyy-mm-dd
 print(f"{latest_date[:4]}-{latest_date[4:6]}-{latest_date[6:8]}")

app/sbin/stats

@@ -0,0 +1,47 @@
+#!/usr/bin/env python3
+"""Collect server runtime stats (uptime, top, cgroup, mounts) and return as JSON."""
+
+import json
+import subprocess
+
+
+def cmd(args, splitlines=False):
+    try:
+        result = subprocess.run(
+            args,
+            capture_output=True,
+            text=True,
+            timeout=30,
+        )
+        out = (result.stdout + result.stderr).strip()
+
+        if splitlines:
+            return out.splitlines()
+
+        return out
+
+    except Exception as e:
+        return f"ERROR: {e}"
+
+
+def read_file(path):
+    try:
+        with open(path) as f:
+            return f.read().strip().splitlines()
+    except FileNotFoundError:
+        return None
+
+
+def main():
+    stats = {}
+
+    stats["uptime"] = cmd(["uptime"])
+    stats["top"] = cmd(["sh", "-c", "top -b -n 1 | head -n 3"], splitlines=True)
+    stats["cgroup"] = read_file("/proc/self/cgroup")
+    stats["mounts"] = cmd(["mount"], splitlines=True)
+
+    print(json.dumps(stats, indent=2))
+
+
+if __name__ == "__main__":
+    main()

app/sbin/upgrade_alpine_version

@@ -0,0 +1,42 @@
+#!/bin/sh
+# Set target Alpine version here
+TARGET_VERSION="3.24"
+
+echo "Current Alpine:"
+cat /etc/alpine-release
+
+echo "Upgrading to Alpine ${TARGET_VERSION}..."
+
+# Safety backup
+BACKUP="/etc/apk/repositories.backup.$(date +%Y%m%d-%H%M%S)"
+cp /etc/apk/repositories "$BACKUP"
+echo "Repository backup saved: $BACKUP"
+
+# Update repositories
+sed -i "s/v[0-9]\+\.[0-9]\+/v${TARGET_VERSION}/g" /etc/apk/repositories
+
+echo "New repositories:"
+cat /etc/apk/repositories
+
+# Refresh indexes
+apk update
+
+# Upgrade apk tooling first
+apk add --upgrade apk-tools
+
+# Upgrade all packages to target branch
+apk upgrade --available
+
+# Clean cache
+apk cache clean
+
+echo
+echo "Upgrade finished."
+echo "New Alpine version:"
+cat /etc/alpine-release
+
+echo
+echo "Kernel:"
+uname -r
+
+

app/update_alpine

@@ -0,0 +1,4 @@
+rex doas apk update
+rex doas apk fix
+rex doas apk upgrade
+rex doas reboot