From 5a92cac546f1b737df3f26acd428bc5bed82db94 Mon Sep 17 00:00:00 2001 From: Azat Date: Mon, 2 Feb 2026 23:44:24 +0100 Subject: [PATCH] Initial backup skill implementation --- SKILL.md | 169 +++++++++++++++++++++++++++++++++++++++++++++ scripts/autorun.sh | 91 ++++++++++++++++++++++++ scripts/backup.sh | 146 +++++++++++++++++++++++++++++++++++++++ scripts/restore.sh | 160 ++++++++++++++++++++++++++++++++++++++++++ scripts/run.sh | 67 ++++++++++++++++++ 5 files changed, 633 insertions(+) create mode 100644 SKILL.md create mode 100644 scripts/autorun.sh create mode 100644 scripts/backup.sh create mode 100644 scripts/restore.sh create mode 100644 scripts/run.sh diff --git a/SKILL.md b/SKILL.md new file mode 100644 index 0000000..e0a8ad2 --- /dev/null +++ b/SKILL.md @@ -0,0 +1,169 @@ +--- +name: backup +description: Automated backup and restore using restic +metadata: + version: "1.0.0" + vibestack: + main: false +--- + +# Backup Skill + +Automated backup and restore for all VibeStack data using [restic](https://restic.net/). + +## Features + +- Incremental, encrypted backups +- Multiple backup targets (local, S3, B2, SFTP) +- Scheduled automatic backups via cron +- Retention policy management +- Point-in-time restore +- PostgreSQL-aware backups (pg_dump) + +## Configuration + +### Environment Variables + +| Variable | Default | Description | +|----------|---------|-------------| +| `BACKUP_SCHEDULE` | `0 3 * * *` | Cron schedule (default: 3am daily) | +| `BACKUP_RETENTION` | `7d` | Retention period | +| `BACKUP_TARGET` | `/backups` | Local backup directory | +| `BACKUP_PASSWORD` | (required) | Encryption password | +| `BACKUP_S3_BUCKET` | (none) | S3 bucket URL (e.g., `s3:bucket-name/path`) | +| `BACKUP_S3_ACCESS_KEY` | (none) | S3 access key | +| `BACKUP_S3_SECRET_KEY` | (none) | S3 secret key | +| `BACKUP_B2_ACCOUNT_ID` | (none) | Backblaze B2 account ID | +| `BACKUP_B2_ACCOUNT_KEY` | (none) | Backblaze B2 account key | +| `BACKUP_B2_BUCKET` | (none) | B2 bucket name | +| `BACKUP_SFTP_HOST` | (none) | SFTP host for remote backup | +| `BACKUP_SFTP_USER` | (none) | SFTP username | +| `BACKUP_SFTP_PATH` | (none) | SFTP path | + +## What Gets Backed Up + +| Path | Description | +|------|-------------| +| `/data/postgres` | PostgreSQL data (via pg_dump) | +| `/data/redis` | Redis persistence files | +| `/data/duckdb` | DuckDB databases | +| `/data/loki` | Log data | +| `/data/caddy` | TLS certificates | +| `/personalities` | Agent personality configs | +| `/workspaces` | Agent workspaces | + +## Usage + +### Manual Backup + +```bash +# Trigger immediate backup +/skills/backup/scripts/backup.sh + +# Backup specific path +/skills/backup/scripts/backup.sh /data/postgres +``` + +### Manual Restore + +```bash +# List available snapshots +/skills/backup/scripts/restore.sh --list + +# Restore latest snapshot +/skills/backup/scripts/restore.sh --latest + +# Restore specific snapshot +/skills/backup/scripts/restore.sh --snapshot abc123 + +# Restore specific path +/skills/backup/scripts/restore.sh --latest --path /data/postgres +``` + +### Check Backup Status + +```bash +# Show backup stats +restic -r "$BACKUP_TARGET" stats + +# List snapshots +restic -r "$BACKUP_TARGET" snapshots +``` + +## Backup Targets + +### Local (default) + +```bash +BACKUP_TARGET=/backups +BACKUP_PASSWORD=your-secret-password +``` + +### Amazon S3 + +```bash +BACKUP_TARGET=s3:my-bucket/vibestack-backups +BACKUP_S3_ACCESS_KEY=AKIAIOSFODNN7EXAMPLE +BACKUP_S3_SECRET_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY +BACKUP_PASSWORD=your-secret-password +``` + +### Backblaze B2 + +```bash +BACKUP_TARGET=b2:my-bucket:/vibestack-backups +BACKUP_B2_ACCOUNT_ID=your-account-id +BACKUP_B2_ACCOUNT_KEY=your-account-key +BACKUP_PASSWORD=your-secret-password +``` + +### SFTP + +```bash +BACKUP_TARGET=sftp:user@host:/path/to/backups +BACKUP_SFTP_HOST=backup.example.com +BACKUP_SFTP_USER=backup +BACKUP_PASSWORD=your-secret-password +``` + +## Retention Policy + +The `BACKUP_RETENTION` variable controls how long backups are kept: + +| Format | Example | Description | +|--------|---------|-------------| +| `Xd` | `7d` | Keep backups for X days | +| `Xw` | `4w` | Keep backups for X weeks | +| `Xm` | `3m` | Keep backups for X months | + +Restic's `forget` command with `--keep-within` is used to enforce retention. + +## PostgreSQL Backups + +When PostgreSQL is detected, the backup skill: +1. Runs `pg_dump` to create a consistent SQL dump +2. Stores the dump at `/data/postgres/backup.sql` +3. Includes it in the restic backup + +This ensures database consistency during backup. + +## Monitoring + +Backup status is written to `/run/vibestack/backup-status.json`: + +```json +{ + "last_backup": "2024-01-15T03:00:00Z", + "last_status": "success", + "snapshot_id": "abc123def", + "duration_seconds": 45, + "bytes_added": 1048576 +} +``` + +## Security + +1. **Always set a strong `BACKUP_PASSWORD`** - backups are encrypted with this +2. Store credentials securely (use environment variables, not files) +3. Test restore procedure regularly +4. Keep backup target separate from primary data diff --git a/scripts/autorun.sh b/scripts/autorun.sh new file mode 100644 index 0000000..b26926f --- /dev/null +++ b/scripts/autorun.sh @@ -0,0 +1,91 @@ +#!/bin/bash +set -e + +BACKUP_TARGET="${BACKUP_TARGET:-/backups}" +BACKUP_PASSWORD="${BACKUP_PASSWORD:-}" + +# Idempotent restic installation +install_restic() { + if command -v restic &>/dev/null; then + echo "restic already installed: $(restic version)" + return 0 + fi + + echo "Installing restic..." + + # Install from official repository + apt-get update + apt-get install -y restic + + # Update to latest version + restic self-update 2>/dev/null || true + + echo "restic installed: $(restic version)" +} + +# Install cron for scheduled backups +install_cron() { + if command -v cron &>/dev/null; then + echo "cron already installed" + return 0 + fi + + echo "Installing cron..." + apt-get update + apt-get install -y cron + + echo "cron installed" +} + +# Setup directories +setup_dirs() { + # Create local backup directory if using local target + if [[ "$BACKUP_TARGET" == /* ]]; then + mkdir -p "$BACKUP_TARGET" + echo "Local backup directory: $BACKUP_TARGET" + fi + + # Create status directory + mkdir -p /run/vibestack +} + +# Initialize restic repository +init_repository() { + if [ -z "$BACKUP_PASSWORD" ]; then + echo "WARNING: BACKUP_PASSWORD not set - skipping repository init" + echo "Set BACKUP_PASSWORD to enable backups" + return 0 + fi + + export RESTIC_PASSWORD="$BACKUP_PASSWORD" + export RESTIC_REPOSITORY="$BACKUP_TARGET" + + # Setup cloud credentials if provided + if [ -n "$BACKUP_S3_ACCESS_KEY" ]; then + export AWS_ACCESS_KEY_ID="$BACKUP_S3_ACCESS_KEY" + export AWS_SECRET_ACCESS_KEY="$BACKUP_S3_SECRET_KEY" + fi + + if [ -n "$BACKUP_B2_ACCOUNT_ID" ]; then + export B2_ACCOUNT_ID="$BACKUP_B2_ACCOUNT_ID" + export B2_ACCOUNT_KEY="$BACKUP_B2_ACCOUNT_KEY" + fi + + # Check if repository exists + if restic cat config &>/dev/null; then + echo "Backup repository already initialized" + return 0 + fi + + echo "Initializing backup repository at $BACKUP_TARGET..." + restic init + + echo "Backup repository initialized" +} + +install_restic +install_cron +setup_dirs +init_repository + +echo "Backup setup complete" diff --git a/scripts/backup.sh b/scripts/backup.sh new file mode 100644 index 0000000..b60b93e --- /dev/null +++ b/scripts/backup.sh @@ -0,0 +1,146 @@ +#!/bin/bash +set -e + +BACKUP_TARGET="${BACKUP_TARGET:-/backups}" +BACKUP_PASSWORD="${BACKUP_PASSWORD:-}" +BACKUP_RETENTION="${BACKUP_RETENTION:-7d}" + +# Paths to backup +BACKUP_PATHS=( + "/data/postgres" + "/data/redis" + "/data/duckdb" + "/data/loki" + "/data/caddy" + "/personalities" + "/workspaces" +) + +# Validate password +if [ -z "$BACKUP_PASSWORD" ]; then + echo "ERROR: BACKUP_PASSWORD is required" + exit 1 +fi + +# Setup restic environment +export RESTIC_PASSWORD="$BACKUP_PASSWORD" +export RESTIC_REPOSITORY="$BACKUP_TARGET" + +# Setup cloud credentials if provided +[ -n "$BACKUP_S3_ACCESS_KEY" ] && export AWS_ACCESS_KEY_ID="$BACKUP_S3_ACCESS_KEY" +[ -n "$BACKUP_S3_SECRET_KEY" ] && export AWS_SECRET_ACCESS_KEY="$BACKUP_S3_SECRET_KEY" +[ -n "$BACKUP_B2_ACCOUNT_ID" ] && export B2_ACCOUNT_ID="$BACKUP_B2_ACCOUNT_ID" +[ -n "$BACKUP_B2_ACCOUNT_KEY" ] && export B2_ACCOUNT_KEY="$BACKUP_B2_ACCOUNT_KEY" + +START_TIME=$(date +%s) +echo "=== VibeStack Backup ===" +echo "Time: $(date -Iseconds)" +echo "Target: $BACKUP_TARGET" +echo "" + +# Pre-backup: dump PostgreSQL if running +pre_backup_postgres() { + if command -v pg_dump &>/dev/null && [ -d "/data/postgres" ]; then + echo "Creating PostgreSQL dump..." + + # Source postgres env if available + [ -f /run/vibestack/postgres.env ] && source /run/vibestack/postgres.env + + local pg_user="${POSTGRES_USER:-vibestack}" + local pg_db="${POSTGRES_DB:-vibestack}" + local dump_file="/data/postgres/backup.sql" + + if pg_dump -U "$pg_user" -d "$pg_db" -f "$dump_file" 2>/dev/null; then + echo " PostgreSQL dump created: $dump_file" + else + echo " PostgreSQL dump skipped (database not running or accessible)" + fi + fi +} + +# Pre-backup: trigger Redis BGSAVE if running +pre_backup_redis() { + if command -v redis-cli &>/dev/null; then + echo "Triggering Redis BGSAVE..." + + # Source redis env if available + [ -f /run/vibestack/redis.env ] && source /run/vibestack/redis.env + + local redis_pass="${REDIS_PASSWORD:-}" + local auth_args="" + [ -n "$redis_pass" ] && auth_args="-a $redis_pass" + + if redis-cli $auth_args BGSAVE 2>/dev/null; then + # Wait for save to complete + sleep 2 + echo " Redis BGSAVE triggered" + else + echo " Redis BGSAVE skipped (not running)" + fi + fi +} + +# Run pre-backup hooks +echo "Running pre-backup hooks..." +pre_backup_postgres +pre_backup_redis +echo "" + +# Build list of paths that exist +existing_paths=() +for path in "${BACKUP_PATHS[@]}"; do + if [ -e "$path" ]; then + existing_paths+=("$path") + echo "Including: $path" + fi +done + +# Allow overriding with specific path +if [ -n "$1" ] && [ -e "$1" ]; then + existing_paths=("$1") + echo "Backing up only: $1" +fi + +if [ ${#existing_paths[@]} -eq 0 ]; then + echo "WARNING: No paths to backup" + exit 0 +fi + +echo "" +echo "Starting backup..." + +# Run restic backup +BACKUP_OUTPUT=$(restic backup "${existing_paths[@]}" --json 2>&1 | tail -1) + +# Parse results +SNAPSHOT_ID=$(echo "$BACKUP_OUTPUT" | jq -r '.snapshot_id // empty' 2>/dev/null || echo "") +BYTES_ADDED=$(echo "$BACKUP_OUTPUT" | jq -r '.data_added // 0' 2>/dev/null || echo "0") + +END_TIME=$(date +%s) +DURATION=$((END_TIME - START_TIME)) + +echo "" +echo "Backup complete!" +echo " Snapshot: ${SNAPSHOT_ID:-unknown}" +echo " Duration: ${DURATION}s" +echo " Data added: $BYTES_ADDED bytes" + +# Apply retention policy +echo "" +echo "Applying retention policy: keep within $BACKUP_RETENTION..." +restic forget --keep-within "$BACKUP_RETENTION" --prune + +# Write status +cat > /run/vibestack/backup-status.json << EOF +{ + "status": "idle", + "last_backup": "$(date -Iseconds)", + "last_status": "success", + "snapshot_id": "$SNAPSHOT_ID", + "duration_seconds": $DURATION, + "bytes_added": $BYTES_ADDED +} +EOF + +echo "" +echo "=== Backup Complete ===" diff --git a/scripts/restore.sh b/scripts/restore.sh new file mode 100644 index 0000000..58bd2e4 --- /dev/null +++ b/scripts/restore.sh @@ -0,0 +1,160 @@ +#!/bin/bash +set -e + +BACKUP_TARGET="${BACKUP_TARGET:-/backups}" +BACKUP_PASSWORD="${BACKUP_PASSWORD:-}" + +# Validate password +if [ -z "$BACKUP_PASSWORD" ]; then + echo "ERROR: BACKUP_PASSWORD is required" + exit 1 +fi + +# Setup restic environment +export RESTIC_PASSWORD="$BACKUP_PASSWORD" +export RESTIC_REPOSITORY="$BACKUP_TARGET" + +# Setup cloud credentials if provided +[ -n "$BACKUP_S3_ACCESS_KEY" ] && export AWS_ACCESS_KEY_ID="$BACKUP_S3_ACCESS_KEY" +[ -n "$BACKUP_S3_SECRET_KEY" ] && export AWS_SECRET_ACCESS_KEY="$BACKUP_S3_SECRET_KEY" +[ -n "$BACKUP_B2_ACCOUNT_ID" ] && export B2_ACCOUNT_ID="$BACKUP_B2_ACCOUNT_ID" +[ -n "$BACKUP_B2_ACCOUNT_KEY" ] && export B2_ACCOUNT_KEY="$BACKUP_B2_ACCOUNT_KEY" + +usage() { + cat << EOF +VibeStack Restore Tool + +Usage: restore.sh [OPTIONS] + +Options: + --list List available snapshots + --latest Restore from latest snapshot + --snapshot ID Restore from specific snapshot + --path PATH Restore only specific path + --target DIR Restore to specific directory (default: /) + --dry-run Show what would be restored without doing it + --help Show this help message + +Examples: + # List all snapshots + restore.sh --list + + # Restore everything from latest snapshot + restore.sh --latest + + # Restore specific path from latest + restore.sh --latest --path /data/postgres + + # Restore from specific snapshot + restore.sh --snapshot abc123def + + # Restore to different directory + restore.sh --latest --target /tmp/restore + + # Preview restore + restore.sh --latest --dry-run +EOF +} + +# Parse arguments +SNAPSHOT="" +RESTORE_PATH="" +TARGET_DIR="/" +DRY_RUN="" +LIST_ONLY="" + +while [[ $# -gt 0 ]]; do + case $1 in + --list) + LIST_ONLY="true" + shift + ;; + --latest) + SNAPSHOT="latest" + shift + ;; + --snapshot) + SNAPSHOT="$2" + shift 2 + ;; + --path) + RESTORE_PATH="$2" + shift 2 + ;; + --target) + TARGET_DIR="$2" + shift 2 + ;; + --dry-run) + DRY_RUN="--dry-run" + shift + ;; + --help) + usage + exit 0 + ;; + *) + echo "Unknown option: $1" + usage + exit 1 + ;; + esac +done + +echo "=== VibeStack Restore ===" +echo "Repository: $BACKUP_TARGET" +echo "" + +# List snapshots +if [ "$LIST_ONLY" = "true" ]; then + echo "Available snapshots:" + echo "" + restic snapshots + exit 0 +fi + +# Validate snapshot +if [ -z "$SNAPSHOT" ]; then + echo "ERROR: Specify --latest or --snapshot ID" + echo "" + usage + exit 1 +fi + +# Show what we're restoring +echo "Snapshot: $SNAPSHOT" +[ -n "$RESTORE_PATH" ] && echo "Path: $RESTORE_PATH" +echo "Target: $TARGET_DIR" +[ -n "$DRY_RUN" ] && echo "Mode: DRY RUN" +echo "" + +# Build restore command +restore_args=("restore" "$SNAPSHOT" "--target" "$TARGET_DIR") +[ -n "$RESTORE_PATH" ] && restore_args+=("--include" "$RESTORE_PATH") +[ -n "$DRY_RUN" ] && restore_args+=("$DRY_RUN") + +# Confirm unless dry run +if [ -z "$DRY_RUN" ]; then + echo "WARNING: This will overwrite existing files!" + echo "" + read -p "Continue? [y/N] " -n 1 -r + echo "" + if [[ ! $REPLY =~ ^[Yy]$ ]]; then + echo "Aborted" + exit 0 + fi +fi + +echo "Restoring..." +restic "${restore_args[@]}" + +# Post-restore for PostgreSQL +if [ -z "$DRY_RUN" ] && [ -f "$TARGET_DIR/data/postgres/backup.sql" ]; then + echo "" + echo "PostgreSQL dump found at $TARGET_DIR/data/postgres/backup.sql" + echo "To restore the database, run:" + echo " psql -U vibestack -d vibestack -f $TARGET_DIR/data/postgres/backup.sql" +fi + +echo "" +echo "=== Restore Complete ===" diff --git a/scripts/run.sh b/scripts/run.sh new file mode 100644 index 0000000..6c0918f --- /dev/null +++ b/scripts/run.sh @@ -0,0 +1,67 @@ +#!/bin/bash +set -e + +BACKUP_SCHEDULE="${BACKUP_SCHEDULE:-0 3 * * *}" +SKILL_DIR="$(dirname "$(dirname "$0")")" + +# Validate password is set +if [ -z "$BACKUP_PASSWORD" ]; then + echo "ERROR: BACKUP_PASSWORD is required" + echo "Set BACKUP_PASSWORD environment variable to enable backups" + exit 1 +fi + +# Setup cron job for scheduled backups +setup_cron() { + local cron_file="/etc/cron.d/vibestack-backup" + local backup_script="$SKILL_DIR/scripts/backup.sh" + + # Build environment exports for cron + local env_exports="" + env_exports+="BACKUP_PASSWORD='$BACKUP_PASSWORD' " + env_exports+="BACKUP_TARGET='${BACKUP_TARGET:-/backups}' " + env_exports+="BACKUP_RETENTION='${BACKUP_RETENTION:-7d}' " + + [ -n "$BACKUP_S3_ACCESS_KEY" ] && env_exports+="BACKUP_S3_ACCESS_KEY='$BACKUP_S3_ACCESS_KEY' " + [ -n "$BACKUP_S3_SECRET_KEY" ] && env_exports+="BACKUP_S3_SECRET_KEY='$BACKUP_S3_SECRET_KEY' " + [ -n "$BACKUP_B2_ACCOUNT_ID" ] && env_exports+="BACKUP_B2_ACCOUNT_ID='$BACKUP_B2_ACCOUNT_ID' " + [ -n "$BACKUP_B2_ACCOUNT_KEY" ] && env_exports+="BACKUP_B2_ACCOUNT_KEY='$BACKUP_B2_ACCOUNT_KEY' " + + # Create cron job + cat > "$cron_file" << EOF +# VibeStack automated backup +SHELL=/bin/bash +PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + +$BACKUP_SCHEDULE root $env_exports $backup_script >> /var/log/vibestack-backup.log 2>&1 +EOF + + chmod 644 "$cron_file" + echo "Cron job configured: $BACKUP_SCHEDULE" +} + +# Write initial status +write_status() { + cat > /run/vibestack/backup-status.json << EOF +{ + "status": "running", + "schedule": "$BACKUP_SCHEDULE", + "target": "${BACKUP_TARGET:-/backups}", + "last_backup": null, + "last_status": null +} +EOF +} + +setup_cron +write_status + +echo "Starting cron daemon for scheduled backups..." +echo "Schedule: $BACKUP_SCHEDULE" +echo "Target: ${BACKUP_TARGET:-/backups}" +echo "" +echo "Manual backup: $SKILL_DIR/scripts/backup.sh" +echo "Manual restore: $SKILL_DIR/scripts/restore.sh --list" + +# Start cron in foreground +exec cron -f