feat(CORE): docker permissions and rework (#13454)

- fix docker permissions with mounted volumes on new docker versions (https://github.com/docker/desktop-linux/issues/31)
- fix ac-tools image
- cleanup unused env
- add  `cap_add: SYS_NICE` to allow setting process high priority
- fix ccache in docker
- allow to switch between root user (default) to acore user in any container
- fix cache of the downloaded client data
- split docker github actions in 2 parts
- versioning mysql images
- performance improvements

apps/bash_shared/common.sh

@@ -9,7 +9,7 @@ USER_CONF_PATH=${USER_CONF_PATH:-"$AC_PATH_CONF/config.sh"}
 if [ -f  "$USER_CONF_PATH" ]; then
     source "$USER_CONF_PATH" # should overwrite previous
 else
-    echo "NOTICE: file <$USER_CONF_PATH> has not been found, you should create and configure it."
+    echo "NOTICE: file <$USER_CONF_PATH> not found, we use default configuration only."
 fi
 
 #

apps/ci/ci-conf-core.sh

@@ -14,9 +14,6 @@ CBUILD_TESTING=ON
 CSCRIPTPCH=OFF
 CCOREPCH=OFF
 CCUSTOMOPTIONS='-DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_C_FLAGS="-Werror" -DCMAKE_CXX_FLAGS="-Werror"'
-DB_CHARACTERS_CONF="MYSQL_USER='root'; MYSQL_PASS='root'; MYSQL_HOST='localhost';"
-DB_AUTH_CONF="MYSQL_USER='root'; MYSQL_PASS='root'; MYSQL_HOST='localhost';"
-DB_WORLD_CONF="MYSQL_USER='root'; MYSQL_PASS='root'; MYSQL_HOST='localhost';"
 CONFIG_SH
 
 case $COMPILER in

apps/ci/ci-conf-db.sh

@@ -12,9 +12,6 @@ CTOOLS_BUILD=db-only
 CSCRIPTPCH=OFF
 CCOREPCH=OFF
 CCUSTOMOPTIONS='-DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_C_FLAGS="-Werror" -DCMAKE_CXX_FLAGS="-Werror"'
-DB_CHARACTERS_CONF="MYSQL_USER='root'; MYSQL_PASS='root'; MYSQL_HOST='localhost';"
-DB_AUTH_CONF="MYSQL_USER='root'; MYSQL_PASS='root'; MYSQL_HOST='localhost';"
-DB_WORLD_CONF="MYSQL_USER='root'; MYSQL_PASS='root'; MYSQL_HOST='localhost';"
 CONFIG_SH
 
 case $COMPILER in

apps/ci/ci-install.sh

@@ -14,9 +14,6 @@ CTOOLS=ON
 CSCRIPTPCH=OFF
 CCOREPCH=OFF
 CCUSTOMOPTIONS='-DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_C_FLAGS="-Werror" -DCMAKE_CXX_FLAGS="-Werror"'
-DB_CHARACTERS_CONF="MYSQL_USER='root'; MYSQL_PASS='root'; MYSQL_HOST='localhost';"
-DB_AUTH_CONF="MYSQL_USER='root'; MYSQL_PASS='root'; MYSQL_HOST='localhost';"
-DB_WORLD_CONF="MYSQL_USER='root'; MYSQL_PASS='root'; MYSQL_HOST='localhost';"
 CONFIG_SH
 
 time sudo apt-get update -y

apps/compiler/compiler.sh

@@ -29,6 +29,7 @@ comp_options=(
     "compile: Compile only"
     "all: clean, configure and compile"
     "ccacheClean: Clean ccache files, normally not needed"
+    "ccacheShowStats: show ccache statistics"
     "quit: Close this menu")
 comp_functions=(
     "comp_build"
@@ -37,6 +38,7 @@ comp_functions=(
     "comp_compile"
     "comp_all"
     "comp_ccacheClean"
+    "comp_ccacheShowStats"
     "comp_quit")
 
 PS3='[ Please enter your choice ]: '

apps/compiler/includes/functions.sh

@@ -17,6 +17,8 @@ function comp_ccacheEnable() {
     export CCACHE_CPP2=${CCACHE_CPP2:-true} # optimization for clang
     export CCACHE_COMPRESS=${CCACHE_COMPRESS:-1}
     export CCACHE_COMPRESSLEVEL=${CCACHE_COMPRESSLEVEL:-9}
+    export CCACHE_COMPILERCHECK=${CCACHE_COMPILERCHECK:-content}
+    export CCACHE_LOGFILE=${CCACHE_LOGFILE:-"$CCACHE_DIR/cache.debug"}
     #export CCACHE_NODIRECT=true
 
     export CCUSTOMOPTIONS="$CCUSTOMOPTIONS -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache"
@@ -106,6 +108,8 @@ function comp_compile() {
 
   cd $BUILDPATH
 
+  comp_ccacheEnable
+
   comp_ccacheResetStats
 
   time cmake --build . --config $CTYPE  -j $MTHREADS

apps/docker/Dockerfile

@@ -45,7 +45,11 @@ RUN addgroup --gid $GROUP_ID acore && \
 
 # must be created to set the correct permissions on them
 RUN mkdir -p /azerothcore/env/dist/bin
-RUN mkdir -p /azerothcore/env/dist/data
+RUN mkdir -p /azerothcore/env/dist/data/Cameras
+RUN mkdir -p /azerothcore/env/dist/data/dbc
+RUN mkdir -p /azerothcore/env/dist/data/maps
+RUN mkdir -p /azerothcore/env/dist/data/mmaps
+RUN mkdir -p /azerothcore/env/dist/data/vmaps
 RUN mkdir -p /azerothcore/env/dist/logs
 RUN mkdir -p /azerothcore/env/dist/temp
 RUN mkdir -p /azerothcore/env/dist/etc
@@ -59,10 +63,14 @@ RUN chown -R $DOCKER_USER:$DOCKER_USER /azerothcore
 
 USER $DOCKER_USER
 
-# copy everything so we can work directly within the container
-# using tools such as vscode dev-container
-# NOTE: this folder is different by the /azerothcore (which is binded instead)
-COPY --chown=$DOCKER_USER:$DOCKER_USER . /azerothcore
+# copy only necessary files for the acore dashboard
+COPY --chown=$DOCKER_USER:$DOCKER_USER apps /azerothcore/apps
+COPY --chown=$DOCKER_USER:$DOCKER_USER bin /azerothcore/bin
+COPY --chown=$DOCKER_USER:$DOCKER_USER conf /azerothcore/conf
+COPY --chown=$DOCKER_USER:$DOCKER_USER data /azerothcore/data
+COPY --chown=$DOCKER_USER:$DOCKER_USER deps /azerothcore/deps
+COPY --chown=$DOCKER_USER:$DOCKER_USER acore.json /azerothcore/acore.json
+COPY --chown=$DOCKER_USER:$DOCKER_USER acore.sh /azerothcore/acore.sh
 
 # Download deno and make sure the dashboard works
 RUN bash /azerothcore/acore.sh quit
@@ -79,6 +87,13 @@ FROM base as dev
 
 LABEL description="AC dev image for dev containers"
 
+USER $DOCKER_USER
+
+# copy everything so we can work directly within the container
+# using tools such as vscode dev-container
+# NOTE: this folder is different by the /azerothcore (which is binded instead)
+COPY --chown=$DOCKER_USER:$DOCKER_USER . /azerothcore
+
 #================================================================
 #
 # SERVICE BASE: prepare the OS for the production-ready services
@@ -115,26 +130,21 @@ RUN apt-get update && apt-get install -y dos2unix gdb gdbserver google-perftools
 # change timezone in container
 RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone && dpkg-reconfigure --frontend noninteractive tzdata
 
-# copy the sources from the host machine
-COPY --chown=$DOCKER_USER:$DOCKER_USER apps /azerothcore/apps
-COPY --chown=$DOCKER_USER:$DOCKER_USER bin /azerothcore/bin
-COPY --chown=$DOCKER_USER:$DOCKER_USER conf /azerothcore/conf
-COPY --chown=$DOCKER_USER:$DOCKER_USER data /azerothcore/data
-COPY --chown=$DOCKER_USER:$DOCKER_USER deps /azerothcore/deps
-COPY --chown=$DOCKER_USER:$DOCKER_USER acore.json /azerothcore/acore.json
-COPY --chown=$DOCKER_USER:$DOCKER_USER acore.sh /azerothcore/acore.sh
-
 # Correct permissions for non-root operations
 RUN chown -R $DOCKER_USER:$DOCKER_USER /home/acore
 RUN chown -R $DOCKER_USER:$DOCKER_USER /run
 RUN chown -R $DOCKER_USER:$DOCKER_USER /opt
 
-RUN chown $DOCKER_USER:$DOCKER_USER /azerothcore
+COPY --chown=$DOCKER_USER:$DOCKER_USER --from=base /azerothcore /azerothcore
 
 USER $DOCKER_USER
 
 # must be created to avoid permissions errors
-RUN mkdir -p /azerothcore/env/dist/data
+RUN mkdir -p /azerothcore/env/dist/data/Cameras
+RUN mkdir -p /azerothcore/env/dist/data/dbc
+RUN mkdir -p /azerothcore/env/dist/data/maps
+RUN mkdir -p /azerothcore/env/dist/data/mmaps
+RUN mkdir -p /azerothcore/env/dist/data/vmaps
 RUN mkdir -p /azerothcore/env/dist/logs
 RUN mkdir -p /azerothcore/env/dist/etc
 RUN mkdir -p /azerothcore/env/dist/bin
@@ -157,12 +167,16 @@ LABEL description="AC authserver image for local environment"
 
 CMD ./acore.sh run-authserver
 
+USER $DOCKER_USER
+
 FROM servicebase as worldserver-local
 
 LABEL description="AC worldserver image for local environment"
 
 CMD ./acore.sh run-worldserver
 
+USER $DOCKER_USER
+
 #================================================================
 #
 # BUILD: compile sources
@@ -171,12 +185,20 @@ CMD ./acore.sh run-worldserver
 FROM base as build
 
 ARG DOCKER_USER=acore
+USER $DOCKER_USER
 
 LABEL description="AC Image used by the build stage to generate production images"
 
 RUN mkdir -p /azerothcore/env/etc/
 
+# .git is needed by the compiler
+COPY --chown=$DOCKER_USER:$DOCKER_USER ./.git /azerothcore/.git
+COPY --chown=$DOCKER_USER:$DOCKER_USER ./CMakeLists.txt /azerothcore/CMakeLists.txt
+COPY --chown=$DOCKER_USER:$DOCKER_USER ./deps /azerothcore/deps
+COPY --chown=$DOCKER_USER:$DOCKER_USER ./src /azerothcore/src
+COPY --chown=$DOCKER_USER:$DOCKER_USER ./modules /azerothcore/modules
 # check if we have ccache files available outside
+RUN rm -rf /azerothcore/var/ccache/*
 COPY --chown=$DOCKER_USER:$DOCKER_USER var/docker/ccache /azerothcore/var/ccache
 COPY --chown=$DOCKER_USER:$DOCKER_USER env/docker/etc/authserver.conf.dockerdist /azerothcore/env/dist/etc/authserver.conf.dockerdist
 COPY --chown=$DOCKER_USER:$DOCKER_USER env/docker/etc/worldserver.conf.dockerdist /azerothcore/env/dist/etc/worldserver.conf.dockerdist
@@ -207,6 +229,7 @@ FROM authserver-local as authserver
 LABEL description="AC Production: authserver"
 
 ARG DOCKER_USER=acore
+USER $DOCKER_USER
 
 COPY --chown=$DOCKER_USER:$DOCKER_USER --from=build /azerothcore/env/dist/etc /azerothcore/env/dist/etc
 COPY --chown=$DOCKER_USER:$DOCKER_USER --from=build /azerothcore/env/dist/bin/authserver /azerothcore/env/dist/bin/authserver
@@ -222,6 +245,7 @@ FROM worldserver-local as worldserver
 LABEL description="AC Production: worldserver"
 
 ARG DOCKER_USER=acore
+USER $DOCKER_USER
 
 RUN mkdir -p /azerothcore/env/dist/bin/lua_scripts
 COPY --chown=$DOCKER_USER:$DOCKER_USER --from=build /azerothcore/env/dist/etc /azerothcore/env/dist/etc
@@ -258,9 +282,8 @@ RUN addgroup --gid $GROUP_ID acore && \
 ENV DATAPATH=/azerothcore/env/dist/data
 ENV DATAPATH_ZIP=/tmp/data.zip
 
-RUN --mount=type=bind,target=/azerothcore-temp,readwrite /azerothcore-temp/acore.sh client-data && chown -R $DOCKER_USER:$DOCKER_USER /azerothcore
-
-RUN apt-get remove --purge -y tzdata curl unzip && apt-get autoremove -y
+RUN mkdir -p "$DATAPATH"
+RUN --mount=type=bind,target=/azerothcore-temp,readwrite --mount=type=cache,target=/azerothcore/env/dist/data /azerothcore-temp/acore.sh client-data && chown -R $DOCKER_USER:$DOCKER_USER /azerothcore
 
 USER $DOCKER_USER
 
@@ -302,14 +325,14 @@ USER $DOCKER_USER
 
 WORKDIR /azerothcore/env/client/
 
+RUN mkdir -p /azerothcore/env/client/Cameras
 RUN mkdir -p /azerothcore/env/client/dbc
 RUN mkdir -p /azerothcore/env/client/maps
 RUN mkdir -p /azerothcore/env/client/mmaps
 RUN mkdir -p /azerothcore/env/client/vmaps
 
-# Need fix
-# COPY --chown=$DOCKER_USER:$DOCKER_USER --from=build /azerothcore/env/dist/bin/mapextractor /azerothcore/env/client/mapextractor
-# COPY --chown=$DOCKER_USER:$DOCKER_USER --from=build /azerothcore/env/dist/bin/mmaps_generator /azerothcore/env/client/mmaps_generator
-# COPY --chown=$DOCKER_USER:$DOCKER_USER --from=build /azerothcore/env/dist/bin/vmap4assembler /azerothcore/env/client/vmap4assembler
-# COPY --chown=$DOCKER_USER:$DOCKER_USER --from=build /azerothcore/env/dist/bin/vmap4extractor /azerothcore/env/client/vmap4extractor
+COPY --chown=$DOCKER_USER:$DOCKER_USER --from=build /azerothcore/env/dist/bin/map_extractor /azerothcore/env/client/map_extractor
+COPY --chown=$DOCKER_USER:$DOCKER_USER --from=build /azerothcore/env/dist/bin/mmaps_generator /azerothcore/env/client/mmaps_generator
+COPY --chown=$DOCKER_USER:$DOCKER_USER --from=build /azerothcore/env/dist/bin/vmap4_assembler /azerothcore/env/client/vmap4_assembler
+COPY --chown=$DOCKER_USER:$DOCKER_USER --from=build /azerothcore/env/dist/bin/vmap4_extractor /azerothcore/env/client/vmap4_extractor
 

apps/docker/config-docker.sh

@@ -1,5 +1,7 @@
 CUR_PATH="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 
+CTOOLS_BUILD=all
+
 # allow the user to override configs
 if [ -f  "$AC_PATH_CONF/config.sh" ]; then
     source "$AC_PATH_CONF/config.sh" # should overwrite previous

apps/docker/docker-cmd.ts

@@ -39,9 +39,19 @@ shellCommandFactory(
   "build",
   "Build the authserver and worldserver",
   [
-    "docker compose --profile local build --parallel",
+    "docker compose --profile local --profile dev --profile dev-build build --parallel",
+    "docker image prune -f",
+    "docker compose run --rm --no-deps ac-dev-build bash apps/docker/docker-build-dev.sh",
+  ],
+  env
+);
+
+shellCommandFactory(
+  "pull",
+  "Pull build and local images",
+  [
+    "docker compose --profile local --profile dev --profile dev-build pull --parallel",
     "docker image prune -f",
-    "docker compose run --rm ac-build bash apps/docker/docker-build-dev.sh",
   ],
   env
 );
@@ -50,20 +60,9 @@ shellCommandFactory(
   "build:nocache",
   "Build the authserver and worldserver without docker cache",
   [
-    "docker compose --profile local build --no-cache --parallel",
+    "docker compose --profile local --profile dev --profile dev-build build --no-cache --parallel",
     "docker image prune -f",
-    "docker compose run --rm ac-build bash apps/docker/docker-build-dev.sh",
-  ],
-  env
-);
-
-shellCommandFactory(
-  "build:compile",
-  "Run the compilation process only, without rebuilding all docker images",
-  [
-    "docker compose build --parallel ac-build",
-    "docker image prune -f",
-    "docker compose run --rm ac-build bash apps/docker/docker-build-dev.sh",
+    "docker compose run --rm --no-deps ac-dev-build bash apps/docker/docker-build-dev.sh",
   ],
   env
 );
@@ -73,7 +72,8 @@ shellCommandFactory(
   "Clean build files",
   [
     "docker image prune -f",
-    `docker compose run --rm ac-build bash acore.sh compiler clean`,
+    `docker compose run --rm --no-deps ac-dev-server bash acore.sh compiler clean`,
+    `docker compose run --rm --no-deps ac-dev-server bash acore.sh compiler ccacheClean`,
   ],
   env
 );
@@ -81,7 +81,7 @@ shellCommandFactory(
 shellCommandFactory(
   "client-data",
   "Download client data inside the ac-data volume",
-  ["docker compose run --rm ac-build bash acore.sh client-data"],
+  ["docker compose run --rm --no-deps ac-dev-server bash acore.sh client-data"],
   env
 );
 
@@ -94,7 +94,7 @@ shellCommandFactory(
 
 shellCommandFactory(
   "dev:build",
-  "Build using the dev server, it uses volumes to compile which can be faster on linux & WSL",
+  "Build using the dev server",
   ["docker compose run --rm ac-dev-server bash acore.sh compiler build"],
   env
 );
@@ -118,28 +118,28 @@ shellCommandFactory(
 
 shellCommandFactory(
   "prod:build",
-  "Build producion services",
+  "[TEST ONLY] Build producion services",
   ["docker compose --profile prod build --parallel", "docker image prune -f"],
   env
 );
 
 shellCommandFactory(
   "prod:pull",
-  "Pull production services from the remote registry",
+  "[TEST ONLY] Pull production services from the remote registry",
   ["docker compose --profile prod pull"],
   env
 );
 
 shellCommandFactory(
   "prod:up",
-  "Start production services (foreground)",
+  "[TEST ONLY] Start production services (foreground)",
   ["docker compose --profile prod-app up"],
   env
 );
 
 shellCommandFactory(
   "prod:up:d",
-  "Start production services (background)",
+  "[TEST ONLY] Start production services (background)",
   ["docker compose --profile prod-app up -d"],
   env
 );
@@ -176,7 +176,7 @@ program
     }
 
     services.pop();
-    services = services.slice(2);
+    services = services.slice(1);
 
     res.close(); // Don't forget to close it
 
@@ -304,8 +304,6 @@ async function checkDockerVersion() {
 }
 
 async function main() {
-  // Handle it however you like
-  // e.g. display usage
   while (true) {
     const version = await checkDockerVersion();
     if (version !== true) {

apps/installer/main.sh

@@ -67,7 +67,11 @@ function _switch() {
             DOCKER=1 denoRunFile "$AC_PATH_APPS/docker/docker-cmd.ts" "${@:2}"
             exit
             ;;
-        ""|"quit"|"14")
+        ""|"v"|"version"|"14")
+            denoRunFile "$AC_PATH_APPS/installer/main.ts" "version"
+            exit
+            ;;
+        ""|"quit"|"15")
             echo "Goodbye!"
             exit
             ;;

apps/installer/main.ts

@@ -0,0 +1,43 @@
+import { Command } from "https://cdn.deno.land/cmd/versions/v1.2.0/raw/mod.ts";
+import { getAcoreReleaseVersion } from "./utils.ts";
+import { Input } from "https://deno.land/x/cliffy@v0.25.2/prompt/mod.ts";
+
+const program = new Command();
+
+program
+  .name("acore.sh")
+  .description("Shell scripts for docker")
+  .version("1.0.0");
+
+// program
+//   .command("quit")
+//   .description("Close docker command")
+//   .action(() => {
+//     process.exit(0);
+//   });
+
+program
+  .command("version")
+  .description("Get the version of the current AzerothCore revision")
+  .action(async () => {
+    console.log(await getAcoreReleaseVersion());
+  });
+
+async function main() {
+  let exit = false;
+  do {
+    if (Deno.args.length === 0) {
+      program.outputHelp();
+      const command = await Input.prompt({
+        message: "Enter the command:",
+      });
+      console.log(command);
+      await program.parseAsync(command.split(" "));
+    } else {
+      exit = true;
+      await program.parseAsync(Deno.args);
+      process.exit(0);
+    }
+  } while (!exit);
+}
+main();

apps/installer/utils.ts

@@ -0,0 +1,17 @@
+import * as path from "https://deno.land/std/path/mod.ts";
+import makeloc from "https://deno.land/x/dirname@1.1.2/mod.ts";
+
+const { __dirname } = makeloc(import.meta);
+
+// specify the needed paths here
+const ACORE_JSON = path.resolve(__dirname + "/../../acore.json");
+
+export async function getAcoreReleaseVersion() {
+  // read the acore.json file to work with the versioning
+  const decoder = new TextDecoder("utf-8");
+  //console.debug(`Open ${ACORE_JSON}`)
+  const data = await Deno.readFile(ACORE_JSON);
+  const acoreInfo = JSON.parse(decoder.decode(data));
+
+  return `AzerothCore Rev. ${acoreInfo.version}`;
+}