From 883b7820ed7a51c8a136d0230110b28cfe4d9fd9 Mon Sep 17 00:00:00 2001 From: Paul R Kartchner Date: Tue, 20 Jan 2026 15:29:35 -0700 Subject: [PATCH] docs: add comprehensive database migration and backup documentation Added complete guide for migrating from containerized PostgreSQL to standalone server with production-grade backup strategies. New files: - docs/DATABASE-MIGRATION-GUIDE.md - Complete migration guide with step-by-step instructions, troubleshooting, and rollback procedures - scripts/backup-standalone-postgres.sh - Automated backup script with daily, weekly, and monthly retention policies - scripts/restore-standalone-postgres.sh - Safe restore script with verification and pre-restore safety backup Features: - Hybrid backup strategy (PostgreSQL native + Basil API) - Automated retention policy (30/90/365 days) - Integrity verification - Safety backups before restore - Complete troubleshooting guide - Rollback procedures Co-Authored-By: Claude Opus 4.5 --- docs/DATABASE-MIGRATION-GUIDE.md | 465 +++++++++++++++++++++++++ scripts/backup-standalone-postgres.sh | 74 ++++ scripts/restore-standalone-postgres.sh | 88 +++++ 3 files changed, 627 insertions(+) create mode 100644 docs/DATABASE-MIGRATION-GUIDE.md create mode 100755 scripts/backup-standalone-postgres.sh create mode 100755 scripts/restore-standalone-postgres.sh diff --git a/docs/DATABASE-MIGRATION-GUIDE.md b/docs/DATABASE-MIGRATION-GUIDE.md new file mode 100644 index 0000000..71085dc --- /dev/null +++ b/docs/DATABASE-MIGRATION-GUIDE.md @@ -0,0 +1,465 @@ +# Database Migration Guide: Container → Standalone PostgreSQL + +This guide covers migrating Basil from containerized PostgreSQL to a standalone PostgreSQL server and setting up production-grade backups. + +## Table of Contents +1. [Why Migrate?](#why-migrate) +2. [Pre-Migration Checklist](#pre-migration-checklist) +3. [Migration Steps](#migration-steps) +4. [Backup Strategy](#backup-strategy) +5. [Testing & Verification](#testing--verification) +6. [Rollback Plan](#rollback-plan) + +--- + +## Why Migrate? + +### Standalone PostgreSQL Advantages +- ✅ Dedicated database resources (no competition with app containers) +- ✅ Standard PostgreSQL backup/restore tools +- ✅ Point-in-time recovery (PITR) capabilities +- ✅ Better monitoring and administration +- ✅ Industry best practice for production +- ✅ Easier to scale independently + +### When to Keep Containerized +- Local development environments +- Staging/test environments +- Simple single-server deployments +- Environments where simplicity > resilience + +--- + +## Pre-Migration Checklist + +- [ ] Standalone PostgreSQL server is installed and accessible +- [ ] PostgreSQL version is 13 or higher (check: `psql --version`) +- [ ] Network connectivity from app server to DB server +- [ ] Firewall rules allow PostgreSQL port (default: 5432) +- [ ] You have PostgreSQL superuser credentials +- [ ] Current Basil data is backed up +- [ ] Maintenance window scheduled (expect ~15-30 min downtime) + +--- + +## Migration Steps + +### Step 1: Create Backup of Current Data + +**Option A: Use Basil's Built-in API (Recommended)** + +```bash +# Create full backup (database + uploaded images) +curl -X POST http://localhost:3001/api/backup + +# List available backups +curl http://localhost:3001/api/backup + +# Download the latest backup +curl -O http://localhost:3001/api/backup/basil-backup-YYYY-MM-DDTHH-MM-SS.zip +``` + +**Option B: Direct PostgreSQL Dump** + +```bash +# From container +docker exec basil-postgres pg_dump -U basil basil > /tmp/basil_migration.sql + +# Verify backup +head -20 /tmp/basil_migration.sql +``` + +### Step 2: Prepare Standalone PostgreSQL Server + +SSH into your PostgreSQL server: + +```bash +ssh your-postgres-server + +# Switch to postgres user +sudo -u postgres psql +``` + +Create database and user: + +```sql +-- Create database +CREATE DATABASE basil; + +-- Create user with password +CREATE USER basil WITH ENCRYPTED PASSWORD 'your-secure-password-here'; + +-- Grant privileges +GRANT ALL PRIVILEGES ON DATABASE basil TO basil; + +-- Connect to basil database +\c basil + +-- Grant schema permissions +GRANT ALL ON SCHEMA public TO basil; + +-- Exit +\q +``` + +**Security Best Practices:** +```bash +# Generate strong password +openssl rand -base64 32 + +# Store in password manager or .pgpass file +echo "your-postgres-server:5432:basil:basil:your-password" >> ~/.pgpass +chmod 600 ~/.pgpass +``` + +### Step 3: Update Firewall Rules + +On PostgreSQL server: + +```bash +# Allow app server to connect +sudo ufw allow from to any port 5432 + +# Or edit pg_hba.conf +sudo nano /etc/postgresql/15/main/pg_hba.conf +``` + +Add line: +``` +host basil basil /32 scram-sha-256 +``` + +Reload PostgreSQL: +```bash +sudo systemctl reload postgresql +``` + +### Step 4: Test Connectivity + +From app server: + +```bash +# Test connection +psql -h your-postgres-server -U basil -d basil -c "SELECT version();" + +# Should show PostgreSQL version +``` + +### Step 5: Update Basil Configuration + +**On app server**, update environment configuration: + +```bash +# Edit .env file +cd /srv/docker-compose/basil +nano .env +``` + +Add or update: +```bash +DATABASE_URL=postgresql://basil:your-password@your-postgres-server-ip:5432/basil?schema=public +``` + +**Update docker-compose.yml:** + +```yaml +services: + api: + environment: + - DATABASE_URL=${DATABASE_URL} + # ... other variables + + # Comment out postgres service + # postgres: + # image: postgres:15 + # ... +``` + +### Step 6: Run Prisma Migrations + +This creates the schema on your new database: + +```bash +cd /home/pkartch/development/basil/packages/api + +# Generate Prisma client +npm run prisma:generate + +# Deploy migrations +npm run prisma:migrate deploy +``` + +### Step 7: Restore Data + +**Option A: Use Basil's Restore API** + +```bash +# Copy backup to server (if needed) +scp basil-backup-*.zip app-server:/tmp/ + +# Restore via API +curl -X POST http://localhost:3001/api/backup/restore \ + -F "backup=@/tmp/basil-backup-YYYY-MM-DDTHH-MM-SS.zip" +``` + +**Option B: Direct PostgreSQL Restore** + +```bash +# Copy SQL dump to DB server +scp /tmp/basil_migration.sql your-postgres-server:/tmp/ + +# On PostgreSQL server +psql -h localhost -U basil basil < /tmp/basil_migration.sql +``` + +### Step 8: Restart Application + +```bash +cd /srv/docker-compose/basil +./dev-rebuild.sh + +# Or +docker-compose down +docker-compose up -d +``` + +### Step 9: Verify Migration + +```bash +# Check API logs +docker-compose logs api | grep -i "database\|connected" + +# Test API +curl http://localhost:3001/api/recipes +curl http://localhost:3001/api/cookbooks + +# Check database directly +psql -h your-postgres-server -U basil basil -c "SELECT COUNT(*) FROM \"Recipe\";" +psql -h your-postgres-server -U basil basil -c "SELECT COUNT(*) FROM \"Cookbook\";" +``` + +--- + +## Backup Strategy + +### Daily Automated Backups + +**On PostgreSQL server:** + +```bash +# Copy backup script to server +scp scripts/backup-standalone-postgres.sh your-postgres-server:/usr/local/bin/ +ssh your-postgres-server chmod +x /usr/local/bin/backup-standalone-postgres.sh + +# Set up cron job +ssh your-postgres-server +sudo crontab -e +``` + +Add: +```cron +# Daily backup at 2 AM +0 2 * * * /usr/local/bin/backup-standalone-postgres.sh >> /var/log/basil-backup.log 2>&1 +``` + +### Weekly Application Backups + +**On app server:** + +```bash +sudo crontab -e +``` + +Add: +```cron +# Weekly full backup (DB + images) on Sundays at 3 AM +0 3 * * 0 curl -X POST http://localhost:3001/api/backup >> /var/log/basil-api-backup.log 2>&1 +``` + +### Off-Site Backup Sync + +**Set up rsync to NAS or remote server:** + +```bash +# On PostgreSQL server +sudo crontab -e +``` + +Add: +```cron +# Sync backups to NAS at 4 AM +0 4 * * * rsync -av /var/backups/basil/ /mnt/nas/backups/basil/ >> /var/log/basil-sync.log 2>&1 + +# Optional: Upload to S3 +0 5 * * * aws s3 sync /var/backups/basil/ s3://your-bucket/basil-backups/ --storage-class GLACIER >> /var/log/basil-s3.log 2>&1 +``` + +### Backup Retention + +The backup script automatically maintains: +- **Daily backups:** 30 days +- **Weekly backups:** 90 days (12 weeks) +- **Monthly backups:** 365 days (12 months) + +--- + +## Testing & Verification + +### Test Backup Process + +```bash +# Run backup manually +/usr/local/bin/backup-standalone-postgres.sh + +# Verify backup exists +ls -lh /var/backups/basil/daily/ + +# Test backup integrity +gzip -t /var/backups/basil/daily/basil-*.sql.gz +``` + +### Test Restore Process + +**On a test server (NOT production!):** + +```bash +# Copy restore script +scp scripts/restore-standalone-postgres.sh test-server:/tmp/ + +# Run restore +/tmp/restore-standalone-postgres.sh /var/backups/basil/daily/basil-YYYYMMDD.sql.gz +``` + +### Monitoring + +**Set up monitoring checks:** + +```bash +# Check backup file age (should be < 24 hours) +find /var/backups/basil/daily/ -name "basil-*.sql.gz" -mtime -1 | grep -q . || echo "ALERT: No recent backup!" + +# Check backup size (should be reasonable) +BACKUP_SIZE=$(du -sb /var/backups/basil/daily/basil-$(date +%Y%m%d).sql.gz 2>/dev/null | cut -f1) +if [ "$BACKUP_SIZE" -lt 1000000 ]; then + echo "ALERT: Backup size suspiciously small!" +fi +``` + +--- + +## Rollback Plan + +If migration fails, you can quickly rollback: + +### Quick Rollback to Containerized PostgreSQL + +```bash +cd /srv/docker-compose/basil + +# 1. Restore old docker-compose.yml (uncomment postgres service) +nano docker-compose.yml + +# 2. Remove DATABASE_URL override +nano .env # Comment out or remove DATABASE_URL + +# 3. Restart with containerized database +docker-compose down +docker-compose up -d + +# 4. Restore from backup +curl -X POST http://localhost:3001/api/backup/restore \ + -F "backup=@basil-backup-YYYY-MM-DDTHH-MM-SS.zip" +``` + +### Data Recovery + +If you need to recover data from standalone server after rollback: + +```bash +# Dump from standalone server +ssh your-postgres-server +pg_dump -U basil basil > /tmp/basil_recovery.sql + +# Import to containerized database +docker exec -i basil-postgres psql -U basil basil < /tmp/basil_recovery.sql +``` + +--- + +## Troubleshooting + +### Connection Issues + +**Error: "Connection refused"** +```bash +# Check PostgreSQL is listening on network +sudo netstat -tlnp | grep 5432 + +# Verify postgresql.conf +grep "listen_addresses" /etc/postgresql/*/main/postgresql.conf +# Should be: listen_addresses = '*' + +# Restart PostgreSQL +sudo systemctl restart postgresql +``` + +**Error: "Authentication failed"** +```bash +# Verify user exists +psql -U postgres -c "\du basil" + +# Reset password +psql -U postgres -c "ALTER USER basil WITH PASSWORD 'new-password';" + +# Check pg_hba.conf authentication method +sudo cat /etc/postgresql/*/main/pg_hba.conf | grep basil +``` + +### Migration Issues + +**Error: "Relation already exists"** +```bash +# Drop and recreate database +psql -U postgres -c "DROP DATABASE basil;" +psql -U postgres -c "CREATE DATABASE basil;" +psql -U postgres -c "GRANT ALL PRIVILEGES ON DATABASE basil TO basil;" + +# Re-run migrations +cd packages/api +npm run prisma:migrate deploy +``` + +**Error: "Foreign key constraint violation"** +```bash +# Restore with --no-owner --no-privileges flags +pg_restore --no-owner --no-privileges -U basil -d basil backup.sql +``` + +--- + +## Additional Resources + +- [PostgreSQL Backup Documentation](https://www.postgresql.org/docs/current/backup.html) +- [Prisma Migration Guide](https://www.prisma.io/docs/concepts/components/prisma-migrate) +- [Docker PostgreSQL Volume Management](https://docs.docker.com/storage/volumes/) + +--- + +## Summary Checklist + +Post-migration verification: + +- [ ] Application connects to standalone PostgreSQL +- [ ] All recipes visible in UI +- [ ] All cookbooks visible in UI +- [ ] Recipe import works +- [ ] Image uploads work +- [ ] Daily backups running +- [ ] Weekly API backups running +- [ ] Backup integrity verified +- [ ] Restore process tested (on test server) +- [ ] Monitoring alerts configured +- [ ] Old containerized database backed up (for safety) +- [ ] Documentation updated with new DATABASE_URL + +**Congratulations! You've successfully migrated to standalone PostgreSQL! 🎉** diff --git a/scripts/backup-standalone-postgres.sh b/scripts/backup-standalone-postgres.sh new file mode 100755 index 0000000..858c858 --- /dev/null +++ b/scripts/backup-standalone-postgres.sh @@ -0,0 +1,74 @@ +#!/bin/bash +# +# Basil Backup Script for Standalone PostgreSQL +# Place on database server and run via cron +# +# Cron example (daily at 2 AM): +# 0 2 * * * /path/to/backup-standalone-postgres.sh + +set -e + +# Configuration +DB_HOST="localhost" +DB_PORT="5432" +DB_NAME="basil" +DB_USER="basil" +BACKUP_DIR="/var/backups/basil" +RETENTION_DAYS=30 + +# Create backup directories +mkdir -p "$BACKUP_DIR/daily" +mkdir -p "$BACKUP_DIR/weekly" +mkdir -p "$BACKUP_DIR/monthly" + +# Timestamp +TIMESTAMP=$(date +%Y%m%d_%H%M%S) +DATE=$(date +%Y%m%d) +DAY_OF_WEEK=$(date +%u) # 1=Monday, 7=Sunday +DAY_OF_MONTH=$(date +%d) + +# Daily backup +echo "Starting daily backup: $TIMESTAMP" +DAILY_BACKUP="$BACKUP_DIR/daily/basil-$DATE.sql.gz" +pg_dump -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" "$DB_NAME" | gzip > "$DAILY_BACKUP" +echo "Daily backup completed: $DAILY_BACKUP" + +# Weekly backup (on Sundays) +if [ "$DAY_OF_WEEK" -eq 7 ]; then + echo "Creating weekly backup" + WEEK=$(date +%V) + WEEKLY_BACKUP="$BACKUP_DIR/weekly/basil-week$WEEK-$DATE.sql.gz" + cp "$DAILY_BACKUP" "$WEEKLY_BACKUP" + echo "Weekly backup completed: $WEEKLY_BACKUP" +fi + +# Monthly backup (on 1st of month) +if [ "$DAY_OF_MONTH" -eq 01 ]; then + echo "Creating monthly backup" + MONTH=$(date +%Y%m) + MONTHLY_BACKUP="$BACKUP_DIR/monthly/basil-$MONTH.sql.gz" + cp "$DAILY_BACKUP" "$MONTHLY_BACKUP" + echo "Monthly backup completed: $MONTHLY_BACKUP" +fi + +# Cleanup old backups +echo "Cleaning up old backups..." +find "$BACKUP_DIR/daily" -name "basil-*.sql.gz" -mtime +$RETENTION_DAYS -delete +find "$BACKUP_DIR/weekly" -name "basil-*.sql.gz" -mtime +90 -delete +find "$BACKUP_DIR/monthly" -name "basil-*.sql.gz" -mtime +365 -delete + +# Verify backup integrity +echo "Verifying backup integrity..." +if gzip -t "$DAILY_BACKUP"; then + BACKUP_SIZE=$(du -h "$DAILY_BACKUP" | cut -f1) + echo "Backup verification successful. Size: $BACKUP_SIZE" +else + echo "ERROR: Backup verification failed!" >&2 + exit 1 +fi + +# Optional: Send notification (uncomment to enable) +# echo "Basil backup completed successfully on $(hostname) at $(date)" | \ +# mail -s "Basil Backup Success" your-email@example.com + +echo "Backup process completed successfully" diff --git a/scripts/restore-standalone-postgres.sh b/scripts/restore-standalone-postgres.sh new file mode 100755 index 0000000..1ad60a4 --- /dev/null +++ b/scripts/restore-standalone-postgres.sh @@ -0,0 +1,88 @@ +#!/bin/bash +# +# Basil Restore Script for Standalone PostgreSQL +# Run manually when you need to restore from backup +# +# Usage: ./restore-standalone-postgres.sh /path/to/backup.sql.gz + +set -e + +# Configuration +DB_HOST="localhost" +DB_PORT="5432" +DB_NAME="basil" +DB_USER="basil" + +# Check arguments +if [ $# -eq 0 ]; then + echo "Usage: $0 /path/to/backup.sql.gz" + echo "" + echo "Available backups:" + echo "Daily:" + ls -lh /var/backups/basil/daily/ 2>/dev/null | tail -5 + echo "" + echo "Weekly:" + ls -lh /var/backups/basil/weekly/ 2>/dev/null | tail -5 + exit 1 +fi + +BACKUP_FILE="$1" + +# Verify backup file exists +if [ ! -f "$BACKUP_FILE" ]; then + echo "ERROR: Backup file not found: $BACKUP_FILE" + exit 1 +fi + +# Verify backup integrity +echo "Verifying backup integrity..." +if ! gzip -t "$BACKUP_FILE"; then + echo "ERROR: Backup file is corrupted!" + exit 1 +fi + +# Confirm restore +echo "===== WARNING =====" +echo "This will DESTROY all current data in database: $DB_NAME" +echo "Backup file: $BACKUP_FILE" +echo "Database: $DB_USER@$DB_HOST:$DB_PORT/$DB_NAME" +echo "" +read -p "Are you sure you want to continue? (type 'yes' to confirm): " CONFIRM + +if [ "$CONFIRM" != "yes" ]; then + echo "Restore cancelled." + exit 0 +fi + +# Create backup of current database before restore +TIMESTAMP=$(date +%Y%m%d_%H%M%S) +PRE_RESTORE_BACKUP="/tmp/basil-pre-restore-$TIMESTAMP.sql.gz" +echo "Creating safety backup of current database..." +pg_dump -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" "$DB_NAME" | gzip > "$PRE_RESTORE_BACKUP" +echo "Safety backup created: $PRE_RESTORE_BACKUP" + +# Drop and recreate database +echo "Dropping existing database..." +psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" postgres <