From 6170c6e43bb17de0f48e09eb076b16b4315febc3 Mon Sep 17 00:00:00 2001 From: uprightbass360 Date: Thu, 29 Jan 2026 18:17:53 -0500 Subject: [PATCH] feat: db migration support --- README.md | 2 + deploy.sh | 9 +++ docs/GETTING_STARTED.md | 27 +++++++- docs/TROUBLESHOOTING.md | 15 +++++ scripts/bash/db-import-conditional.sh | 90 ++++++++++++++++++++++++++- scripts/bash/migrate-stack.sh | 76 ++++++++++++++++++++++ 6 files changed, 216 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 3861b24..de25742 100644 --- a/README.md +++ b/README.md @@ -66,6 +66,8 @@ cp .env.prebuilt .env Pre-built images include the **RealmMaster profile** (32 modules) and are automatically built nightly. See **[docs/PREBUILT_IMAGES.md](docs/PREBUILT_IMAGES.md)** for details. +**Note:** Remote deployments require one additional step after migration - see [Remote Deployment Guide](docs/GETTING_STARTED.md#remote-deployment). + See [Getting Started](#getting-started) for detailed walkthrough. ## What You Get diff --git a/deploy.sh b/deploy.sh index 739326e..35a6469 100755 --- a/deploy.sh +++ b/deploy.sh @@ -32,6 +32,7 @@ REMOTE_IDENTITY="" REMOTE_PROJECT_DIR="" REMOTE_SKIP_STORAGE=0 REMOTE_COPY_SOURCE=0 +REMOTE_SETUP_SOURCE="" REMOTE_ARGS_PROVIDED=0 REMOTE_AUTO_DEPLOY=0 REMOTE_CLEAN_CONTAINERS=0 @@ -261,6 +262,8 @@ Options: --remote-project-dir DIR Remote project directory (default: ~/) --remote-skip-storage Skip syncing the storage directory during migration --remote-copy-source Copy the local project directory to remote instead of relying on git + --remote-setup-source Automatically run setup-source.sh on remote (clone AzerothCore SQL) + --remote-skip-source-setup Skip source repository setup (you'll run manually later) --remote-auto-deploy Run './deploy.sh --yes --no-watch' on the remote host after migration --remote-clean-containers Stop/remove remote containers & project images during migration --remote-storage-path PATH Override STORAGE_PATH/STORAGE_PATH_LOCAL in the remote .env @@ -294,6 +297,8 @@ while [[ $# -gt 0 ]]; do --remote-project-dir) REMOTE_PROJECT_DIR="$2"; REMOTE_MODE=1; REMOTE_ARGS_PROVIDED=1; shift 2;; --remote-skip-storage) REMOTE_SKIP_STORAGE=1; REMOTE_MODE=1; REMOTE_ARGS_PROVIDED=1; shift;; --remote-copy-source) REMOTE_COPY_SOURCE=1; REMOTE_MODE=1; REMOTE_ARGS_PROVIDED=1; shift;; + --remote-setup-source) REMOTE_SETUP_SOURCE=1; REMOTE_MODE=1; REMOTE_ARGS_PROVIDED=1; shift;; + --remote-skip-source-setup) REMOTE_SETUP_SOURCE=0; REMOTE_MODE=1; REMOTE_ARGS_PROVIDED=1; shift;; --remote-auto-deploy) REMOTE_AUTO_DEPLOY=1; REMOTE_MODE=1; REMOTE_ARGS_PROVIDED=1; shift;; --remote-clean-containers) REMOTE_CLEAN_CONTAINERS=1; REMOTE_MODE=1; REMOTE_ARGS_PROVIDED=1; shift;; --remote-storage-path) REMOTE_STORAGE_OVERRIDE="$2"; REMOTE_MODE=1; REMOTE_ARGS_PROVIDED=1; shift 2;; @@ -754,6 +759,10 @@ run_remote_migration(){ args+=(--env-file "$REMOTE_ENV_FILE") fi + if [ -n "$REMOTE_SETUP_SOURCE" ]; then + args+=(--setup-source "$REMOTE_SETUP_SOURCE") + fi + (cd "$ROOT_DIR" && ./scripts/bash/migrate-stack.sh "${args[@]}") } diff --git a/docs/GETTING_STARTED.md b/docs/GETTING_STARTED.md index 6531cb7..9117baf 100644 --- a/docs/GETTING_STARTED.md +++ b/docs/GETTING_STARTED.md @@ -199,7 +199,32 @@ The remote deployment process transfers: - ✅ Docker images (exported to `local-storage/images/`) - ✅ Project files (scripts, configs, docker-compose.yml, .env) - ✅ Storage directory (unless `--remote-skip-storage` is used) -- ❌ Build artifacts (source code, compilation files stay local) +- ❌ AzerothCore source repository (must be set up separately - see below) + +**IMPORTANT: AzerothCore Source Setup** + +The AzerothCore source repository (~2GB) is NOT synced during migration to avoid slow transfers. However, it's **required** for database initialization. + +**Option 1: Automatic Setup (Recommended)** + +Use the `--remote-setup-source` flag to automatically clone the source on the remote host: + +```bash +./deploy.sh --remote-host your-server --remote-user youruser --remote-setup-source +``` + +**Option 2: Manual Setup** + +After migration completes, SSH to the remote host and run: + +```bash +ssh your-server +cd ~/AzerothCore-RealmMaster # or your custom project directory +./scripts/bash/setup-source.sh +``` + +**Without this step, database initialization will fail with:** +`❌ FATAL: SQL source directory not found` ### Module Presets diff --git a/docs/TROUBLESHOOTING.md b/docs/TROUBLESHOOTING.md index 166936f..3b0bcdf 100644 --- a/docs/TROUBLESHOOTING.md +++ b/docs/TROUBLESHOOTING.md @@ -38,6 +38,21 @@ ls storage/config/mod_*.conf* **Database connection issues** ```bash +# Check if source repository is set up (common issue after remote deployment) +ls -la local-storage/source/azerothcore*/data/sql/base/db_world/ + +# If empty or missing, set up source: +./scripts/bash/setup-source.sh + +# Then restart database import: +docker compose run --rm ac-db-import + +# Error: "SQL source directory not found" +# This means the AzerothCore source repository hasn't been cloned. +# Solution: Run ./scripts/bash/setup-source.sh +# See docs/GETTING_STARTED.md for details + +# Legacy database issues: # Verify MySQL is running and responsive docker exec ac-mysql mysql -u root -p -e "SELECT 1;" diff --git a/scripts/bash/db-import-conditional.sh b/scripts/bash/db-import-conditional.sh index 0e35f8d..3313ac5 100755 --- a/scripts/bash/db-import-conditional.sh +++ b/scripts/bash/db-import-conditional.sh @@ -153,8 +153,33 @@ if [ -f "$RESTORE_SUCCESS_MARKER" ]; then if verify_databases_populated; then echo "✅ Backup restoration completed successfully" cat "$RESTORE_SUCCESS_MARKER" || true - echo "🚫 Skipping database import - data already restored from backup" - exit 0 + + # Check if there are pending module SQL updates to apply + echo "🔍 Checking for pending module SQL updates..." + local has_pending_updates=0 + + # Check if module SQL staging directory has files + if [ -d "/azerothcore/data/sql/updates/db_world" ] && [ -n "$(find /azerothcore/data/sql/updates/db_world -name 'MODULE_*.sql' -type f 2>/dev/null)" ]; then + echo " ⚠️ Found staged module SQL updates that may need application" + has_pending_updates=1 + fi + + if [ "$has_pending_updates" -eq 0 ]; then + echo "🚫 Skipping database import - data already restored and no pending updates" + exit 0 + fi + + echo "📦 Running dbimport to apply pending module SQL updates..." + cd /azerothcore/env/dist/bin + seed_dbimport_conf + + if ./dbimport; then + echo "✅ Module SQL updates applied successfully!" + exit 0 + else + echo "⚠️ dbimport reported issues - check logs for details" + exit 1 + fi fi echo "⚠️ Restoration marker found, but databases are empty - forcing re-import" @@ -470,6 +495,65 @@ echo "🚀 Running database import..." cd /azerothcore/env/dist/bin seed_dbimport_conf +validate_sql_source(){ + local sql_base_dir="/azerothcore/data/sql/base" + local required_dirs=("db_auth" "db_world" "db_characters") + local missing_dirs=() + + echo "🔍 Validating SQL source availability..." + + if [ ! -d "$sql_base_dir" ]; then + cat </dev/null)" ]; then + missing_dirs+=("$dir") + fi + done + + if [ ${#missing_dirs[@]} -gt 0 ]; then + echo "" + echo "❌ FATAL: SQL source directories are empty or missing:" + printf ' - %s\n' "${missing_dirs[@]}" + echo "" + echo "The AzerothCore source directory exists but hasn't been populated with SQL files." + echo "Run './scripts/bash/setup-source.sh' on the host to clone and populate the repository." + echo "" + exit 1 + fi + + echo "✅ SQL source validation passed - all required schemas present" +} + maybe_run_base_import(){ local mysql_host="${CONTAINER_MYSQL:-ac-mysql}" local mysql_port="${MYSQL_PORT:-3306}" @@ -506,6 +590,8 @@ maybe_run_base_import(){ fi } +# Validate SQL source is available before attempting import +validate_sql_source maybe_run_base_import if ./dbimport; then echo "✅ Database import completed successfully!" diff --git a/scripts/bash/migrate-stack.sh b/scripts/bash/migrate-stack.sh index 9089db5..4955ab7 100755 --- a/scripts/bash/migrate-stack.sh +++ b/scripts/bash/migrate-stack.sh @@ -144,6 +144,7 @@ Options: --port PORT SSH port (default: 22) --identity PATH SSH private key (passed to scp/ssh) --project-dir DIR Remote project directory (default: ~/) + --setup-source 0|1 Auto-setup AzerothCore source on remote (0=skip, 1=setup, unset=prompt) --env-file PATH Use this env file for image lookup and upload (default: ./.env) --tarball PATH Output path for the image tar (default: ./local-storage/images/acore-modules-images.tar) --storage PATH Remote storage directory (default: /storage) @@ -168,6 +169,7 @@ SKIP_STORAGE=0 ASSUME_YES=0 COPY_SOURCE=0 SKIP_ENV=0 +REMOTE_SETUP_SOURCE="" PRESERVE_CONTAINERS=0 CLEAN_CONTAINERS=0 @@ -181,6 +183,7 @@ while [[ $# -gt 0 ]]; do --env-file) ENV_FILE="$2"; shift 2;; --tarball) TARBALL="$2"; shift 2;; --storage) REMOTE_STORAGE="$2"; shift 2;; + --setup-source) REMOTE_SETUP_SOURCE="$2"; shift 2;; --skip-storage) SKIP_STORAGE=1; shift;; --skip-env) SKIP_ENV=1; shift;; --preserve-containers) PRESERVE_CONTAINERS=1; shift;; @@ -446,6 +449,76 @@ setup_remote_repository(){ echo " • Repository synchronized ✓" } +setup_source_if_needed(){ + local should_setup="${REMOTE_SETUP_SOURCE:-}" + + # Check if source already exists and is populated + echo " • Checking for existing AzerothCore source repository..." + if run_ssh "[ -d '$PROJECT_DIR/local-storage/source/azerothcore-playerbots/data/sql/base/db_world' ] && [ -n \"\$(ls -A '$PROJECT_DIR/local-storage/source/azerothcore-playerbots/data/sql/base/db_world' 2>/dev/null)\" ]" 2>/dev/null; then + echo " ✅ Source repository already populated on remote" + return 0 + elif run_ssh "[ -d '$PROJECT_DIR/local-storage/source/azerothcore/data/sql/base/db_world' ] && [ -n \"\$(ls -A '$PROJECT_DIR/local-storage/source/azerothcore/data/sql/base/db_world' 2>/dev/null)\" ]" 2>/dev/null; then + echo " ✅ Source repository already populated on remote" + return 0 + fi + + echo " ⚠️ Source repository not found or empty on remote" + + # If not set, ask user (unless --yes) + if [ -z "$should_setup" ]; then + if [ "$ASSUME_YES" = "1" ]; then + # Auto-yes in non-interactive: default to YES for safety + echo " ℹ️ Auto-confirming source setup (--yes flag)" + should_setup=1 + else + echo "" + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + echo "📦 AzerothCore Source Repository Setup" + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + echo "" + echo "The remote server needs AzerothCore source code for database schemas." + echo "This will clone ~2GB repository (one-time operation, takes 2-5 minutes)." + echo "" + echo "Without this, database initialization will FAIL." + echo "" + read -rp "Set up source repository now? [Y/n]: " answer + answer="${answer:-Y}" + case "${answer,,}" in + y|yes) should_setup=1 ;; + *) should_setup=0 ;; + esac + fi + fi + + if [ "$should_setup" != "1" ]; then + echo "" + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + echo "⚠️ WARNING: Source setup skipped" + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + echo "" + echo "You MUST run this manually on the remote host BEFORE starting services:" + echo "" + echo " ssh $USER@$HOST" + echo " cd $PROJECT_DIR" + echo " ./scripts/bash/setup-source.sh" + echo "" + return 0 + fi + + echo " 🔧 Setting up AzerothCore source repository on remote..." + echo " ⏳ Cloning AzerothCore (this may take 2-5 minutes)..." + + # Run setup-source.sh on remote, capturing output + if run_ssh "cd '$PROJECT_DIR' && ./scripts/bash/setup-source.sh" 2>&1 | sed 's/^/ /'; then + echo " ✅ Source repository setup complete" + return 0 + else + echo " ❌ Source setup failed (check output above for details)" + echo " ⚠️ Run manually: ssh $USER@$HOST 'cd $PROJECT_DIR && ./scripts/bash/setup-source.sh'" + return 1 + fi +} + cleanup_stale_docker_resources(){ if [ "$PRESERVE_CONTAINERS" -eq 1 ]; then echo "⋅ Skipping remote container/image cleanup (--preserve-containers)" @@ -477,6 +550,9 @@ cleanup_stale_docker_resources(){ validate_remote_environment +# Set up source repository if needed (after project files are synced) +setup_source_if_needed || true # Don't fail entire deployment if source setup fails + collect_deploy_image_refs echo "⋅ Exporting deployment images to $TARBALL"