spiderunderurbed@raspberrypi:~ $ cat restore-sql-backup.sh #!/usr/bin/env bash #set -euo pipefail # ------------------------------- # CONFIGURATION # ------------------------------- #DB_DIRS=("/home/spiderunderurbed/k8s/gitea-db" "/home/spiderunderurbed/k8s/immich-db" "/home/spiderunderurbed/k8s/wp_db") DB_DIRS=("/home/spiderunderurbed/k8s/immich-db") NAMESPACE="default" FUZZY=0 NEAREST=0 BACKUP_BASE_DIR="/home/spiderunderurbed/db-backups" declare -A PVC_OVERRIDE_DB # db_name -> PVC to lookup pod declare -A OVERRIDE_DEST_DB # db_name -> PVC to restore into # ------------------------------- # ARG PARSING # ------------------------------- while [[ "$#" -gt 0 ]]; do case "$1" in --fuzzy) FUZZY=1 ;; --nearest) NEAREST=1 ;; --pvc-override-db) IFS=":" read -r db pvc <<< "$2" PVC_OVERRIDE_DB["$db"]="$pvc" shift ;; --override-dest-db) IFS=":" read -r db pvc <<< "$2" OVERRIDE_DEST_DB["$db"]="$pvc" shift ;; *) echo "Unknown option: $1"; exit 1 ;; esac shift done # ------------------------------- # FUNCTIONS # ------------------------------- apply_fuzzy_matching() { local base="$1" local -n all_items="$2" local -n matched_items="$3" if [[ ${#matched_items[@]} -eq 0 && $FUZZY -eq 1 ]]; then >&2 echo "Applying fuzzy matching for: $base" local fuzzy_base fuzzy_item fuzzy_base=$(echo "$base" | tr -cd '[:alnum:]') for item in "${all_items[@]}"; do fuzzy_item=$(echo "$item" | tr -cd '[:alnum:]') if [[ "$fuzzy_item" == *"$fuzzy_base"* ]]; then matched_items+=("$item") fi done fi } apply_nearest_matching() { local base="$1" local -n all_items="$2" local -n matched_items="$3" if [[ ${#matched_items[@]} -eq 0 && $NEAREST -eq 1 ]]; then >&2 echo "Applying nearest matching for: $base" local norm_base norm_item norm_base=$(echo "$base" | tr -d '_-') for item in "${all_items[@]}"; do norm_item=$(echo "$item" | tr -d '_-') if [[ "$norm_item" == *"$norm_base"* ]]; then matched_items+=("$item") fi done fi } find_matching_pvc() { local base_name="$1" readarray -t ALL_PVCS < <(kubectl get pvc -n "$NAMESPACE" -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}') MATCHED_PVCS=() for pvc in "${ALL_PVCS[@]}"; do if [[ "$pvc" == *"$base_name"* ]]; then MATCHED_PVCS+=("$pvc") fi done apply_nearest_matching "$base_name" ALL_PVCS MATCHED_PVCS apply_fuzzy_matching "$base_name" ALL_PVCS MATCHED_PVCS if [[ ${#MATCHED_PVCS[@]} -eq 0 ]]; then >&2 echo "No PVC found matching $base_name" return 1 fi echo "${MATCHED_PVCS[0]}" } find_pod_using_pvc() { local pvc_name="$1" readarray -t ALL_PODS < <( kubectl get pods -n "$NAMESPACE" -o jsonpath='{range .items[*]}{.metadata.name}{"|"}{.status.phase}{"|"}{range .spec.volumes[*]}{.persistentVolumeClaim.claimName}{" "} {end}{"\n"}{end}' ) for podinfo in "${ALL_PODS[@]}"; do IFS='|' read -r pod_name pod_phase volumes <<< "$podinfo" if [[ " $volumes " == *" $pvc_name "* && "$pod_phase" == "Running" ]]; then echo "$pod_name" return 0 fi done return 1 } get_db_config() { local dir_path="$1" local db_name db_name=$(basename "$dir_path") case "$db_name" in "immich-db") echo "postgres|immich-postgres|immich|immichpassword|5432|immich" ;; "wp_db") echo "mysql|wp-db|wordpress|your-secure-root-password|3306|wordpress" ;; "gitea-db") echo "mysql|gitea-mysql|gitea|gitea|3306|gitea" ;; *) echo "mysql|$db_name|root|password|3306|$db_name" ;; esac } create_temp_db_client_pod() { local db_type="$1" local pvc_name="$2" local db_identifier="$3" # Original DB identifier (like "immich-db") local temp_pod_name="temp-db-client-$(date +%s)" >&2 echo "Creating temporary DB client pod: $temp_pod_name" if [[ "$db_type" == "mysql" ]]; then # MySQL configuration local image="mariadb:10.11" local data_path="/var/lib/mysql" kubectl apply -f - </dev/null 2>&1 apiVersion: v1 kind: Pod metadata: name: $temp_pod_name namespace: $NAMESPACE spec: containers: - name: db-client image: $image env: - name: MYSQL_ROOT_PASSWORD value: "temp_root_pass" ports: - containerPort: 3306 volumeMounts: - name: db-data mountPath: /var/lib/mysql command: - "bash" - "-c" - | # Wait for the data directory to be mounted while [ ! -d /var/lib/mysql ]; do echo "Waiting for /var/lib/mysql to be available..." sleep 5 done # Check if MySQL data already exists and is valid if [ -f /var/lib/mysql/ibdata1 ] && [ -f /var/lib/mysql/mysql.ibd ]; then echo "Existing MySQL data found, starting MySQL normally..." # Ensure proper ownership chown -R mysql:mysql /var/lib/mysql exec docker-entrypoint.sh mysqld else echo "No valid MySQL data found, cleaning directory and initializing..." # Clean up any partial/corrupted files rm -rf /var/lib/mysql/* exec docker-entrypoint.sh mysqld fi readinessProbe: exec: command: - bash - "-c" - "mysqladmin ping -uroot -p\$MYSQL_ROOT_PASSWORD" initialDelaySeconds: 30 timeoutSeconds: 10 periodSeconds: 15 successThreshold: 1 failureThreshold: 10 livenessProbe: exec: command: - bash - "-c" - "mysqladmin ping -uroot -p\$MYSQL_ROOT_PASSWORD" initialDelaySeconds: 60 timeoutSeconds: 10 periodSeconds: 30 failureThreshold: 3 volumes: - name: db-data persistentVolumeClaim: claimName: $pvc_name restartPolicy: Never EOF else # PostgreSQL configuration - use the original DB identifier to get correct config IFS="|" read -r _ _ db_user db_pass _ db_name <<< "$(get_db_config "$db_identifier")" local pg_user="${db_user:-postgres}" local pg_password="${db_pass:-immichpassword}" local pg_db="${db_name:-immich}" >&2 echo "Using PostgreSQL configuration - User: $pg_user, Database: $pg_db" kubectl apply -f - </dev/null 2>&1 apiVersion: v1 kind: Pod metadata: name: $temp_pod_name namespace: $NAMESPACE spec: containers: - name: db-client image: postgres:15 env: - name: POSTGRES_DB value: "postgres" - name: POSTGRES_USER value: "$pg_user" - name: POSTGRES_PASSWORD value: "$pg_password" - name: PGDATA value: "/var/lib/postgresql/data/pgdata" ports: - containerPort: 5432 volumeMounts: - name: db-data mountPath: /var/lib/postgresql/data command: - "bash" - "-c" - | # Wait for data directory to be mounted while [ ! -d /var/lib/postgresql/data ]; do echo "Waiting for /var/lib/postgresql/data to be available..." sleep 5 done # Set proper permissions chown -R postgres:postgres /var/lib/postgresql/data chmod 700 /var/lib/postgresql/data # Check if database already exists if [ -f /var/lib/postgresql/data/pgdata/PG_VERSION ]; then echo "Existing PostgreSQL database found, starting PostgreSQL..." # Ensure proper ownership chown -R postgres:postgres /var/lib/postgresql/data chmod 700 /var/lib/postgresql/data/pgdata else echo "Initializing new PostgreSQL database..." gosu postgres initdb -D /var/lib/postgresql/data/pgdata echo "host all all 0.0.0.0/0 md5" >> /var/lib/postgresql/data/pgdata/pg_hba.conf echo "local all all md5" >> /var/lib/postgresql/data/pgdata/pg_hba.conf fi echo "Starting PostgreSQL with user: $pg_user..." # Start PostgreSQL in the background gosu postgres postgres -D /var/lib/postgresql/data/pgdata -c listen_addresses=* -c port=5432 & PG_PID=\$! # Wait for PostgreSQL to start for i in {1..30}; do if gosu postgres pg_isready -q; then break fi echo "Waiting for PostgreSQL to start... (\$i/30)" sleep 2 done # If this is a new database, ensure our user exists with proper privileges if ! gosu postgres psql -t -c "SELECT 1 FROM pg_roles WHERE rolname='$pg_user';" | grep -q 1; then echo "Creating user: $pg_user" gosu postgres psql -c "CREATE USER $pg_user WITH PASSWORD '$pg_password';" gosu postgres psql -c "ALTER USER $pg_user WITH SUPERUSER;" fi # Ensure the target database exists (we'll use postgres as the default connection DB) if ! gosu postgres psql -lqt | cut -d \| -f 1 | grep -qw "$pg_db"; then echo "Creating database: $pg_db using template0 to avoid collation issues" gosu postgres createdb -O $pg_user -T template0 "$pg_db" fi # Grant privileges gosu postgres psql -c "GRANT ALL PRIVILEGES ON DATABASE \"$pg_db\" TO $pg_user;" echo "PostgreSQL is ready for connections" wait \$PG_PID readinessProbe: exec: command: - bash - "-c" - "pg_isready -U $pg_user -d postgres" initialDelaySeconds: 15 timeoutSeconds: 10 periodSeconds: 15 successThreshold: 1 failureThreshold: 10 livenessProbe: exec: command: - bash - "-c" - "pg_isready -U $pg_user -d postgres" initialDelaySeconds: 30 timeoutSeconds: 10 periodSeconds: 30 failureThreshold: 3 volumes: - name: db-data persistentVolumeClaim: claimName: $pvc_name restartPolicy: Never EOF fi >&2 echo "Waiting for temporary DB client pod to start..." # Wait for pod to be running (not necessarily ready) local wait_time=0 while true; do local phase=$(kubectl get pod "$temp_pod_name" -n "$NAMESPACE" -o jsonpath='{.status.phase}' 2>/dev/null || echo "Unknown") if [[ "$phase" == "Running" ]]; then break elif [[ "$phase" == "Failed" || "$phase" == "Error" ]]; then >&2 echo "❌ Pod failed to start. Check pod logs:" kubectl logs "$temp_pod_name" -n "$NAMESPACE" return 1 fi sleep 10 ((wait_time+=10)) if [[ $wait_time -gt 300 ]]; then >&2 echo "❌ Timeout waiting for pod to start" kubectl describe pod "$temp_pod_name" -n "$NAMESPACE" return 1 fi done >&2 echo "Pod is running, waiting for database to be ready..." # Wait for database to be ready with more attempts local db_ready_cmd if [[ "$db_type" == "mysql" ]]; then db_ready_cmd="timeout 10 mysqladmin ping -uroot -ptemp_root_pass" else IFS="|" read -r _ _ db_user _ _ db_name <<< "$(get_db_config "$db_identifier")" db_ready_cmd="pg_isready -U ${db_user:-postgres} -d postgres" fi for i in {1..30}; do if kubectl exec "$temp_pod_name" -c db-client -n "$NAMESPACE" -- \ bash -c "$db_ready_cmd" >/dev/null 2>&1; then >&2 echo "✅ Database is ready after $((i*5)) seconds" # Additional safety wait sleep 5 echo "$temp_pod_name" return 0 fi >&2 echo "Database not ready yet, waiting... (attempt $i/30)" sleep 5 done >&2 echo "❌ Database failed to become ready" >&2 echo "📋 Pod description:" kubectl describe pod "$temp_pod_name" -n "$NAMESPACE" >&2 echo "📋 Pod logs:" kubectl logs "$temp_pod_name" -n "$NAMESPACE" -c db-client return 1 } # Cleanup temp pod cleanup_temp_pod() { local pod_name="$1" if [[ -n "$pod_name" ]]; then echo "Deleting temporary pod: $pod_name" #kubectl delete pod "$pod_name" -n "$NAMESPACE" --ignore-not-found=true >/dev/null 2>&1 fi } restore_backup() { local pod="$1" local db_type="$2" local backup_file="$3" local pvc_name="$4" local db_name="$5" local db_identifier="$6" # Original DB identifier (like "immich-db") local temp_pod="" local restore_container="db-client" # Get database configuration using the original DB identifier IFS="|" read -r _ _ db_user db_pass _ configured_db_name <<< "$(get_db_config "$db_identifier")" local pg_user="${db_user:-postgres}" local pg_password="${db_pass:-immichpassword}" local pg_db="${configured_db_name:-immich}" if [[ ! -f "$backup_file" ]]; then >&2 echo "❌ Backup file not found or not readable: $backup_file" return 1 fi if [[ ! -r "$backup_file" ]]; then >&2 echo "Fixing file permissions for: $backup_file" chmod +r "$backup_file" || { >&2 echo "❌ Cannot fix permissions for $backup_file"; return 1; } fi if [[ -n "$pod" ]]; then if ! kubectl exec "$pod" -c "$restore_container" -n "$NAMESPACE" -- echo "Pod is accessible" >/dev/null 2>&1; then >&2 echo "Existing pod $pod is not accessible (probably completed/failed). Creating new temporary pod." pod="" fi fi if [[ -z "$pod" ]]; then >&2 echo "Creating temporary pod for restore operation..." temp_pod=$(create_temp_db_client_pod "$db_type" "$pvc_name" "$db_identifier") || { >&2 echo "❌ Failed to create temporary pod"; return 1; } pod="$temp_pod" >&2 echo "Using temporary pod: $pod" fi >&2 echo "Restoring backup: $backup_file into pod: $pod" >&2 echo "Using PostgreSQL user: $pg_user, database: $db_name" if [[ "$db_type" == "mysql" ]]; then # MySQL restore logic >&2 echo "Testing MySQL connectivity..." local mysql_ready=false for i in {1..10}; do if kubectl exec "$pod" -c "$restore_container" -n "$NAMESPACE" -- \ timeout 10 mysqladmin ping -uroot -ptemp_root_pass >/dev/null 2>&1; then mysql_ready=true break fi >&2 echo "MySQL not ready yet, waiting... (attempt $i/10)" sleep 5 done if [[ "$mysql_ready" != "true" ]]; then >&2 echo "❌ MySQL is not responding after multiple attempts" return 1 fi >&2 echo "MySQL is ready! Proceeding with restore..." >&2 echo "Preparing database: $db_name" kubectl exec "$pod" -c "$restore_container" -n "$NAMESPACE" -- \ mysql -uroot -ptemp_root_pass -e "DROP DATABASE IF EXISTS \`$db_name\`; CREATE DATABASE \`$db_name\`;" || { >&2 echo "❌ Failed to prepare database $db_name" return 1 } >&2 echo "Importing backup data..." if gunzip -c "$backup_file" | \ sed 's/utf8mb4_0900_as_cs/utf8mb4_general_ci/g' | \ tee /tmp/current_restore.sql | \ kubectl exec -i "$pod" -c "$restore_container" -n "$NAMESPACE" -- \ mysql -uroot -ptemp_root_pass "$db_name"; then >&2 echo "✅ Restore completed successfully" else >&2 echo "❌ Failed to import backup" >&2 echo "Last few lines of the SQL being applied:" tail -n 20 /tmp/current_restore.sql return 1 fi else # PostgreSQL restore logic >&2 echo "Testing PostgreSQL connectivity..." local postgres_ready=false for i in {1..10}; do if kubectl exec "$pod" -c "$restore_container" -n "$NAMESPACE" -- \ timeout 10 pg_isready -U "$pg_user" -d "postgres" >/dev/null 2>&1; then postgres_ready=true break fi >&2 echo "PostgreSQL not ready yet, waiting... (attempt $i/10)" sleep 5 done if [[ "$postgres_ready" != "true" ]]; then >&2 echo "❌ PostgreSQL is not responding after multiple attempts" return 1 fi >&2 echo "PostgreSQL is ready! Proceeding with restore..." >&2 echo "Preparing database: $db_name" # First, try to connect as the configured user to postgres database if kubectl exec "$pod" -c "$restore_container" -n "$NAMESPACE" -- \ psql -U "$pg_user" -d "postgres" -c "SELECT 1;" >/dev/null 2>&1; then >&2 echo "✅ Successfully connected as user: $pg_user" # Check if user has superuser privileges if kubectl exec "$pod" -c "$restore_container" -n "$NAMESPACE" -- \ psql -U "$pg_user" -d "postgres" -t -c "SELECT rolsuper FROM pg_roles WHERE rolname='$pg_user';" | grep -q "t"; then >&2 echo "✅ User $pg_user has superuser privileges" # Terminate connections to the target database >&2 echo "Terminating connections to database: $db_name" kubectl exec "$pod" -c "$restore_container" -n "$NAMESPACE" -- \ psql -U "$pg_user" -d "postgres" -c "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = '$db_name';" 2>/dev/null || true # Drop and recreate the database using template0 to avoid collation issues >&2 echo "Dropping database: $db_name" kubectl exec "$pod" -c "$restore_container" -n "$NAMESPACE" -- \ psql -U "$pg_user" -d "postgres" -c "DROP DATABASE IF EXISTS \"$db_name\";" || { >&2 echo "❌ Failed to drop database $db_name" return 1 } >&2 echo "Creating database: $db_name using template0 to avoid collation issues" kubectl exec "$pod" -c "$restore_container" -n "$NAMESPACE" -- \ psql -U "$pg_user" -d "postgres" -c "CREATE DATABASE \"$db_name\" TEMPLATE template0;" || { >&2 echo "❌ Failed to create database $db_name" return 1 } else >&2 echo "⚠ User $pg_user does not have superuser privileges, trying alternative approach" # Try to drop and recreate without terminating connections first, using template0 kubectl exec "$pod" -c "$restore_container" -n "$NAMESPACE" -- \ psql -U "$pg_user" -d "postgres" -c "DROP DATABASE IF EXISTS \"$db_name\";" 2>/dev/null || true kubectl exec "$pod" -c "$restore_container" -n "$NAMESPACE" -- \ psql -U "$pg_user" -d "postgres" -c "CREATE DATABASE \"$db_name\" TEMPLATE template0;" || { >&2 echo "❌ Failed to prepare PostgreSQL database $db_name" return 1 } fi else >&2 echo "❌ Cannot connect as user: $pg_user" >&2 echo "📋 Attempting to list available users and databases:" kubectl exec "$pod" -c "$restore_container" -n "$NAMESPACE" -- \ psql -U postgres -d postgres -c "\\du" 2>/dev/null || \ kubectl exec "$pod" -c "$restore_container" -n "$NAMESPACE" -- \ psql -d postgres -c "\\du" 2>/dev/null || true kubectl exec "$pod" -c "$restore_container" -n "$NAMESPACE" -- \ psql -U postgres -d postgres -c "\\l" 2>/dev/null || \ kubectl exec "$pod" -c "$restore_container" -n "$NAMESPACE" -- \ psql -d postgres -c "\\l" 2>/dev/null || true return 1 fi >&2 echo "Importing backup data..." if gunzip -c "$backup_file" | \ kubectl exec -i "$pod" -c "$restore_container" -n "$NAMESPACE" -- \ psql -U "$pg_user" -d "$db_name"; then >&2 echo "✅ PostgreSQL restore completed successfully as user: $pg_user" else >&2 echo "❌ Failed to import PostgreSQL backup as user: $pg_user" >&2 echo "Checking if backup file is valid PostgreSQL..." if gunzip -c "$backup_file" | head -n 5 | grep -q -E "(CREATE|DROP|INSERT|UPDATE|DELETE|ALTER)"; then >&2 echo "Backup appears to contain valid SQL, but restore failed" else >&2 echo "Backup file may not be valid PostgreSQL format" fi return 1 fi fi if [[ -n "$temp_pod" ]]; then cleanup_temp_pod "$temp_pod" fi return 0 } # ------------------------------- # MAIN LOOP # ------------------------------- for DIR in "${DB_DIRS[@]}"; do echo "=========================================" echo "Processing directory: $DIR" echo "=========================================" DB_NAME=$(basename "$DIR") # Determine PVC to use for pod discovery if [[ -n "${PVC_OVERRIDE_DB[$DB_NAME]}" ]]; then TARGET_PVC_LOOKUP="${PVC_OVERRIDE_DB[$DB_NAME]}" >&2 echo "Using PVC override for lookup (per-DB): $TARGET_PVC_LOOKUP" else TARGET_PVC_LOOKUP=$(find_matching_pvc "$DB_NAME") || { >&2 echo "❌ Could not resolve PVC for $DB_NAME"; continue; } fi POD_NAME=$(find_pod_using_pvc "$TARGET_PVC_LOOKUP" || true) # Determine PVC to restore into if [[ -n "${OVERRIDE_DEST_DB[$DB_NAME]}" ]]; then TARGET_PVC="${OVERRIDE_DEST_DB[$DB_NAME]}" >&2 echo "Using override destination PVC (per-DB): $TARGET_PVC" else TARGET_PVC="$TARGET_PVC_LOOKUP" fi IFS="|" read -r DB_TYPE DEPLOYMENT_NAME DB_USER DB_PASS DB_PORT ACTUAL_DB_NAME <<< "$(get_db_config "$DIR")" BACKUP_SUBDIR=$(basename "$DIR") BACKUP_PATH="$BACKUP_BASE_DIR/$BACKUP_SUBDIR" if [[ ! -d "$BACKUP_PATH" ]]; then >&2 echo "❌ Backup folder not found: $BACKUP_PATH" continue fi BACKUP_FILE=$(ls -t "$BACKUP_PATH"/*.sql.gz 2>/dev/null | head -n1) if [[ -z "$BACKUP_FILE" ]]; then >&2 echo "❌ No backup file found for $DB_NAME" continue fi >&2 echo "Found backup file: $BACKUP_FILE" >&2 echo "Target PVC for restore: $TARGET_PVC" >&2 echo "Pod using PVC: ${POD_NAME:-""}" # Pass the original DB_NAME as the 6th parameter to preserve the correct configuration if restore_backup "$POD_NAME" "$DB_TYPE" "$BACKUP_FILE" "$TARGET_PVC" "$ACTUAL_DB_NAME" "$DB_NAME"; then >&2 echo "✅ Successfully restored $BACKUP_FILE" else >&2 echo "❌ Failed to restore $BACKUP_FILE" fi done echo "=========================================" echo "All restores completed!" echo "=========================================" spiderunderurbed@raspberrypi:~ $