From ce76769f7958b01fd3ee64994a0fc7621b23531f Mon Sep 17 00:00:00 2001 From: uprightbass360 Date: Thu, 8 Jan 2026 02:39:08 -0500 Subject: [PATCH] Add graceful MySQL tmpfs sync on shutdown --- .env.template | 1 + docker-compose.yml | 2 + docs/DATABASE_MANAGEMENT.md | 2 + docs/SCRIPTS.md | 2 +- docs/installing-azerothcore-with-docker.md | 2 +- scripts/bash/db-import-conditional.sh | 2 +- scripts/bash/mysql-entrypoint.sh | 201 +++++++++++++++------ 7 files changed, 152 insertions(+), 60 deletions(-) diff --git a/.env.template b/.env.template index 1f43ac0..b97555b 100644 --- a/.env.template +++ b/.env.template @@ -155,6 +155,7 @@ MYSQL_MAX_CONNECTIONS=1000 MYSQL_INNODB_BUFFER_POOL_SIZE=256M MYSQL_INNODB_LOG_FILE_SIZE=64M MYSQL_INNODB_REDO_LOG_CAPACITY=512M +# MySQL runs on tmpfs (RAM) for performance, with sync to persistent storage on shutdown MYSQL_RUNTIME_TMPFS_SIZE=8G MYSQL_DISABLE_BINLOG=1 MYSQL_CONFIG_DIR=${STORAGE_CONFIG_PATH}/mysql/conf.d diff --git a/docker-compose.yml b/docker-compose.yml index d08dd8e..e068107 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -26,6 +26,7 @@ services: MYSQL_INNODB_BUFFER_POOL_SIZE: ${MYSQL_INNODB_BUFFER_POOL_SIZE} MYSQL_INNODB_LOG_FILE_SIZE: ${MYSQL_INNODB_LOG_FILE_SIZE} MYSQL_BINLOG_EXPIRE_LOGS_SECONDS: 86400 + MYSQL_DISABLE_BINLOG: ${MYSQL_DISABLE_BINLOG} TZ: "${TZ}" entrypoint: - /usr/local/bin/mysql-entrypoint.sh @@ -50,6 +51,7 @@ services: - --expire_logs_days=0 - --binlog_expire_logs_seconds=86400 - --binlog_expire_logs_auto_purge=ON + stop_grace_period: 2m restart: unless-stopped logging: *logging-default healthcheck: diff --git a/docs/DATABASE_MANAGEMENT.md b/docs/DATABASE_MANAGEMENT.md index 823e623..79c94b8 100644 --- a/docs/DATABASE_MANAGEMENT.md +++ b/docs/DATABASE_MANAGEMENT.md @@ -187,6 +187,8 @@ Because MySQL stores its hot data in a tmpfs (`/var/lib/mysql-runtime`) while pe - If **any tables exist**, the script logs `Backup restoration completed successfully` and skips the expensive restore just as before. - If **no tables are found or the query fails**, the script logs `Restoration marker found, but databases are empty - forcing re-import`, automatically clears the stale marker, and reruns the backup restore + `dbimport` pipeline so services always start with real data. +On graceful shutdown, the MySQL container now syncs the tmpfs datadir back into `/var/lib/mysql-persistent` so a normal restart keeps the latest state. Unclean shutdowns (host reboot, OOM kill) can still lose recent changes, so the backup restore path remains the safety net. + To complement that one-shot safety net, the long-running `ac-db-guard` service now watches the runtime tmpfs. It polls MySQL, and if it ever finds those schemas empty (the usual symptom after a daemon restart), it automatically reruns `db-import-conditional.sh` to rehydrate from the most recent backup before marking itself healthy. All auth/world services now depend on `ac-db-guard`'s health check, guaranteeing that AzerothCore never boots without real tables in memory. The guard also mounts the working SQL tree from `local-storage/source/azerothcore-playerbots/data/sql` into the db containers so that every `dbimport` run uses the exact SQL that matches your checked-out source, even if the Docker image was built earlier. Because new features sometimes require schema changes even when the databases already contain data, `ac-db-guard` now performs a `dbimport` verification sweep (configurable via `DB_GUARD_VERIFY_INTERVAL_SECONDS`) to proactively apply any outstanding updates from the mounted SQL tree. By default it runs once per bootstrap and then every 24 hours, so the auth/world servers always see the columns/tables expected by their binaries without anyone having to run host scripts manually. diff --git a/docs/SCRIPTS.md b/docs/SCRIPTS.md index 7168085..2483666 100644 --- a/docs/SCRIPTS.md +++ b/docs/SCRIPTS.md @@ -96,7 +96,7 @@ Comprehensive cleanup with multiple destruction levels and safety checks. Starts all configured containers using appropriate profiles. #### `scripts/bash/stop-containers.sh` - Graceful Shutdown -Stops all containers with proper cleanup and data protection. +Stops all containers with proper cleanup and data protection. The MySQL container performs a shutdown-time sync from tmpfs to persistent storage. #### `status.sh` - Service Health Monitoring ```bash diff --git a/docs/installing-azerothcore-with-docker.md b/docs/installing-azerothcore-with-docker.md index 0dc3b9d..ae4adcc 100644 --- a/docs/installing-azerothcore-with-docker.md +++ b/docs/installing-azerothcore-with-docker.md @@ -75,7 +75,7 @@ services: | Upstream Concept | RealmMaster Equivalent | Notes | | ---------------- | ---------------------- | ----- | -| MySQL container with bind-mounted storage | `ac-mysql` + `ac-storage-init` | Bind mounts live under `storage/` and `local-storage/`; tmpfs keeps runtime data fast and is checkpointed to disk automatically. | +| MySQL container with bind-mounted storage | `ac-mysql` + `ac-storage-init` | Bind mounts live under `storage/` and `local-storage/`; tmpfs keeps runtime data fast and is synced to disk on graceful shutdown. | | Manual DB import container | `ac-db-import` & `ac-db-init` | Automatically imports schemas or restores from backups; disable by skipping the `db` profile if you truly want manual control. | | World/Auth servers with optional DBC overrides | `ac-authserver-*` / `ac-worldserver-*` | Profile-based builds cover vanilla, playerbots, and custom module binaries. DBC overrides go into the shared client data mount just like upstream. | | Client data bind mounts | `ac-client-data-standard` (or `-playerbots`) | Runs `scripts/bash/download-client-data.sh`, caches releases, and mounts them read-only into the worldserver. | diff --git a/scripts/bash/db-import-conditional.sh b/scripts/bash/db-import-conditional.sh index e800a5e..0e35f8d 100755 --- a/scripts/bash/db-import-conditional.sh +++ b/scripts/bash/db-import-conditional.sh @@ -449,7 +449,7 @@ if [ -n "$backup_path" ]; then echo "⚠️ Backup restoration failed, will proceed with fresh database setup" fi else - echo "ℹ️ No valid backups found - proceeding with fresh setup" + echo "ℹ️ No valid SQL backups found - proceeding with fresh setup" echo "$(date): No backup found - fresh setup needed" > "$RESTORE_FAILED_MARKER" fi diff --git a/scripts/bash/mysql-entrypoint.sh b/scripts/bash/mysql-entrypoint.sh index 5e9bb58..5e613a2 100755 --- a/scripts/bash/mysql-entrypoint.sh +++ b/scripts/bash/mysql-entrypoint.sh @@ -11,66 +11,66 @@ if ! command -v "$ORIGINAL_ENTRYPOINT" >/dev/null 2>&1; then fi TARGET_SPEC="${MYSQL_RUNTIME_USER:-${CONTAINER_USER:-}}" -if [ -z "${TARGET_SPEC:-}" ] || [ "${TARGET_SPEC}" = "0:0" ]; then - exec "$ORIGINAL_ENTRYPOINT" "$@" -fi - -if [[ "$TARGET_SPEC" != *:* ]]; then - echo "mysql-entrypoint: Expected MYSQL_RUNTIME_USER/CONTAINER_USER in uid:gid form, got '${TARGET_SPEC}'" >&2 - exit 1 -fi - -IFS=':' read -r TARGET_UID TARGET_GID <<< "$TARGET_SPEC" - -if ! [[ "$TARGET_UID" =~ ^[0-9]+$ ]] || ! [[ "$TARGET_GID" =~ ^[0-9]+$ ]]; then - echo "mysql-entrypoint: UID/GID must be numeric (received uid='${TARGET_UID}' gid='${TARGET_GID}')" >&2 - exit 1 -fi - -if ! id mysql >/dev/null 2>&1; then - echo "mysql-entrypoint: mysql user not found in container" >&2 - exit 1 -fi - -current_uid="$(id -u mysql)" -current_gid="$(id -g mysql)" - -# Adjust group if needed target_group_name="" -if [ "$current_gid" != "$TARGET_GID" ]; then - if groupmod -g "$TARGET_GID" mysql 2>/dev/null; then - target_group_name="mysql" - else - existing_group="$(getent group "$TARGET_GID" | cut -d: -f1 || true)" - if [ -z "$existing_group" ]; then - existing_group="mysql-host" - if ! getent group "$existing_group" >/dev/null 2>&1; then - groupadd -g "$TARGET_GID" "$existing_group" - fi - fi - usermod -g "$existing_group" mysql - target_group_name="$existing_group" - fi -else - target_group_name="$(getent group mysql | cut -d: -f1)" -fi - -if [ -z "$target_group_name" ]; then - target_group_name="$(getent group "$TARGET_GID" | cut -d: -f1 || true)" -fi - -# Adjust user UID if needed -if [ "$current_uid" != "$TARGET_UID" ]; then - if getent passwd "$TARGET_UID" >/dev/null 2>&1 && [ "$(getent passwd "$TARGET_UID" | cut -d: -f1)" != "mysql" ]; then - echo "mysql-entrypoint: UID ${TARGET_UID} already in use by $(getent passwd "$TARGET_UID" | cut -d: -f1)." >&2 - echo "mysql-entrypoint: Please choose a different CONTAINER_USER or adjust the image." >&2 +if [ -n "${TARGET_SPEC:-}" ] && [ "${TARGET_SPEC}" != "0:0" ]; then + if [[ "$TARGET_SPEC" != *:* ]]; then + echo "mysql-entrypoint: Expected MYSQL_RUNTIME_USER/CONTAINER_USER in uid:gid form, got '${TARGET_SPEC}'" >&2 exit 1 fi - usermod -u "$TARGET_UID" mysql -fi -# Ensure group lookup after potential changes -target_group_name="$(getent group "$TARGET_GID" | cut -d: -f1 || echo "$target_group_name")" + IFS=':' read -r TARGET_UID TARGET_GID <<< "$TARGET_SPEC" + + if ! [[ "$TARGET_UID" =~ ^[0-9]+$ ]] || ! [[ "$TARGET_GID" =~ ^[0-9]+$ ]]; then + echo "mysql-entrypoint: UID/GID must be numeric (received uid='${TARGET_UID}' gid='${TARGET_GID}')" >&2 + exit 1 + fi + + if ! id mysql >/dev/null 2>&1; then + echo "mysql-entrypoint: mysql user not found in container" >&2 + exit 1 + fi + + current_uid="$(id -u mysql)" + current_gid="$(id -g mysql)" + + # Adjust group if needed + if [ "$current_gid" != "$TARGET_GID" ]; then + if groupmod -g "$TARGET_GID" mysql 2>/dev/null; then + target_group_name="mysql" + else + existing_group="$(getent group "$TARGET_GID" | cut -d: -f1 || true)" + if [ -z "$existing_group" ]; then + existing_group="mysql-host" + if ! getent group "$existing_group" >/dev/null 2>&1; then + groupadd -g "$TARGET_GID" "$existing_group" + fi + fi + usermod -g "$existing_group" mysql + target_group_name="$existing_group" + fi + else + target_group_name="$(getent group mysql | cut -d: -f1)" + fi + + if [ -z "$target_group_name" ]; then + target_group_name="$(getent group "$TARGET_GID" | cut -d: -f1 || true)" + fi + + # Adjust user UID if needed + if [ "$current_uid" != "$TARGET_UID" ]; then + if getent passwd "$TARGET_UID" >/dev/null 2>&1 && [ "$(getent passwd "$TARGET_UID" | cut -d: -f1)" != "mysql" ]; then + echo "mysql-entrypoint: UID ${TARGET_UID} already in use by $(getent passwd "$TARGET_UID" | cut -d: -f1)." >&2 + echo "mysql-entrypoint: Please choose a different CONTAINER_USER or adjust the image." >&2 + exit 1 + fi + usermod -u "$TARGET_UID" mysql + fi + + # Ensure group lookup after potential changes + target_group_name="$(getent group "$TARGET_GID" | cut -d: -f1 || echo "$target_group_name")" +else + target_group_name="$(getent group mysql | cut -d: -f1 || echo mysql)" +fi # Update ownership on relevant directories if they exist for path in /var/lib/mysql-runtime /var/lib/mysql /var/lib/mysql-persistent /backups; do @@ -79,6 +79,91 @@ for path in /var/lib/mysql-runtime /var/lib/mysql /var/lib/mysql-persistent /bac fi done +# Minimal fix: Restore data from persistent storage on startup and sync on shutdown only +RUNTIME_DIR="/var/lib/mysql-runtime" +PERSISTENT_DIR="/var/lib/mysql-persistent" + +sync_datadir() { + if [ ! -d "$RUNTIME_DIR" ]; then + echo "⚠️ Runtime directory not found: $RUNTIME_DIR" + return 1 + fi + if [ ! -d "$PERSISTENT_DIR" ]; then + echo "⚠️ Persistent directory not found: $PERSISTENT_DIR" + return 1 + fi + + user_schema_count="$(find "$RUNTIME_DIR" -mindepth 1 -maxdepth 1 -type d \ + ! -name mysql \ + ! -name performance_schema \ + ! -name information_schema \ + ! -name sys \ + ! -name "#innodb_temp" \ + ! -name "#innodb_redo" 2>/dev/null | wc -l | tr -d ' ')" + if [ "${user_schema_count:-0}" -eq 0 ]; then + echo "⚠️ Runtime data appears empty (system schemas only); skipping sync" + return 0 + fi + + echo "📦 Syncing MySQL data to persistent storage..." + if command -v rsync >/dev/null 2>&1; then + rsync -a --delete \ + --exclude='.restore-completed' \ + --exclude='.restore-failed' \ + --exclude='.import-completed' \ + --exclude='backup.sql' \ + "$RUNTIME_DIR"/ "$PERSISTENT_DIR"/ + else + # Mirror the runtime state while preserving marker files. + find "$PERSISTENT_DIR" -mindepth 1 -maxdepth 1 \ + ! -name ".restore-completed" \ + ! -name ".restore-failed" \ + ! -name ".import-completed" \ + ! -name "backup.sql" \ + -exec rm -rf {} + 2>/dev/null || true + cp -a "$RUNTIME_DIR"/. "$PERSISTENT_DIR"/ + fi + chown -R mysql:"$target_group_name" "$PERSISTENT_DIR" + echo "✅ Sync completed" +} + +handle_shutdown() { + echo "🔻 Shutdown signal received" + if command -v mysqladmin >/dev/null 2>&1; then + if mysqladmin -h localhost -u root -p"${MYSQL_ROOT_PASSWORD:-}" shutdown 2>/dev/null; then + echo "✅ MySQL shutdown complete" + sync_datadir || true + else + echo "⚠️ mysqladmin shutdown failed; skipping sync to avoid corruption" + fi + else + echo "⚠️ mysqladmin not found; skipping sync" + fi + + if [ -n "${child_pid:-}" ] && kill -0 "$child_pid" 2>/dev/null; then + wait "$child_pid" || true + fi + exit 0 +} + +# Simple startup restoration +if [ -d "$PERSISTENT_DIR" ]; then + # Check for MySQL data files (exclude marker files starting with .) + if find "$PERSISTENT_DIR" -maxdepth 1 -name "*" ! -name ".*" ! -path "$PERSISTENT_DIR" | grep -q .; then + if [ -d "$RUNTIME_DIR" ] && [ -z "$(ls -A "$RUNTIME_DIR" 2>/dev/null)" ]; then + echo "🔄 Restoring MySQL data from persistent storage..." + cp -a "$PERSISTENT_DIR"/* "$RUNTIME_DIR/" 2>/dev/null || true + chown -R mysql:"$target_group_name" "$RUNTIME_DIR" + echo "✅ Data restored from persistent storage" + fi + fi +fi + +# Simple approach: restore on startup only +# Data loss window exists but prevents complete loss on restart + +trap handle_shutdown TERM INT + disable_binlog="${MYSQL_DISABLE_BINLOG:-}" if [ "${disable_binlog}" = "1" ]; then add_skip_flag=1 @@ -93,4 +178,6 @@ if [ "${disable_binlog}" = "1" ]; then fi fi -exec "$ORIGINAL_ENTRYPOINT" "$@" +"$ORIGINAL_ENTRYPOINT" "$@" & +child_pid=$! +wait "$child_pid"