From 3ef1a4a22a8214fdfcf99fc969ed4cec0d3adbb9 Mon Sep 17 00:00:00 2001
From: Mike Delago <32778141+michaeldelago@users.noreply.github.com>
Date: Mon, 14 Oct 2024 17:42:04 -0700
Subject: [PATCH] fix(CI): simplify ci-pending-sql.sh and make it safer
 (#20209)

* apps: ci: ci-pending-sql.sh: simplify

* ensure proper file header is used
---
 apps/ci/ci-pending-sql.sh | 98 ++++++++++++++++-----------------------
 1 file changed, 39 insertions(+), 59 deletions(-)

diff --git a/apps/ci/ci-pending-sql.sh b/apps/ci/ci-pending-sql.sh
index dcfb74df3..e22b742b0 100644
--- a/apps/ci/ci-pending-sql.sh
+++ b/apps/ci/ci-pending-sql.sh
@@ -1,77 +1,57 @@
 #!/usr/bin/env bash
 
+set -euo pipefail
+
 CURRENT_PATH="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 
 source "$CURRENT_PATH/../bash_shared/includes.sh"
 
-UPDATES_PATH="$AC_PATH_ROOT/data/sql/updates/"
+UPDATES_PATH="$AC_PATH_ROOT/data/sql/updates"
 
-COMMIT_HASH=
 
-function import() {
-    db=$1
-    folder="db_"$db
-    pendingPath="$AC_PATH_ROOT/data/sql/updates/pending_$folder"
-    updPath="$UPDATES_PATH/$folder"
-    archivedPath="$AC_PATH_ROOT/data/sql/archive/$folder/6.x"
-
-    latestUpd=$(ls -1 $updPath/ | tail -n 1)
-
-    if [ -z $latestUpd ]; then
-        latestUpd=$(ls -1 $archivedPath/ | tail -n 1)
-        echo "> Last update file for db $db is missing! Using archived file" $latestUpd
+# get_next_index "data/sql/updates/db_world/2024_10_14_22.sql"
+#   => 23
+# get_next_index ""
+#   => 00
+function get_next_index() {
+    if [[ -n "$1" ]]; then
+        PREV_COUNT="$(basename "$1" | cut -f4 -d_ | cut -f1 -d\.)"
+        printf '%02d' "$((PREV_COUNT + 1))"
+    else
+        echo "00"
     fi
+}
 
-    dateToday=$(date +%Y_%m_%d)
-    counter=0
+# lists all SQL files in the appropriate data/sql/updates/db_$1, and then moves them to a standard format, ordered by date and how many imports have happened that day. The name should be in this format:
+#
+# /path/to/data/sql/updates/db_NAME/YYYY_MM_DD_INDEX.sql
+#
+# Where INDEX is a number with a minimum with a minimum width (0-padded) of 2
+#
+# for example, "data/sql/updates/db_world/2024_10_01_03.sql" translates to "the third update in the world database from October 01, 2024"
 
-    dateLast=$latestUpd
-    tmp=${dateLast#*_*_*_}
-    oldCnt=${tmp%.sql}
-    oldDate=${dateLast%_$tmp}
+TODAY="$(date +%Y_%m_%d)"
+function import() {
+    PENDING_PATH="$AC_PATH_ROOT/data/sql/updates/pending_db_$1"
+    UPDATES_DIR="$UPDATES_PATH/db_$1"
 
-    if [ "$oldDate" = "$dateToday" ]; then
-        ((counter=10#$oldCnt+1)) # 10 # is needed to explictly add to a base 10 number
-    fi;
+    # Get latest SQL file applied to this database
+    LATEST_UPDATE="$(find "$UPDATES_DIR" -iname "*.sql" | sort -h | tail -n 1)"
 
-    for entry in "$pendingPath"/*.sql
+    for entry in "$PENDING_PATH"/*.sql
     do
-        if [[ -e $entry ]]; then
-            oldVer=$oldDate"_"$oldCnt
+        if [[ -f "$entry" ]]; then
+            INDEX="$(get_next_index "$LATEST_UPDATE")"
+            OUTPUT_FILE="${UPDATES_DIR}/${TODAY}_${INDEX}.sql"
 
-            cnt=$(printf -v counter "%02d" $counter ; echo $counter)
-
-            newVer=$dateToday"_"$cnt
-
-            newFile="$updPath/"$dateToday"_"$cnt".sql"
-
-            oldFile=$(basename "$entry")
-            prefix=${oldFile%_*.sql}
-            suffix=${oldFile#rev_}
-            rev=${suffix%.sql}
-
-            isRev=0
-            if [[ $prefix = "rev" && $rev =~ ^-?[0-9]+$ ]]; then
-                isRev=1
-            fi
-
-            echo "-- DB update $oldVer -> $newVer" > "$newFile";
-
-            cat $entry >> "$newFile";
-
-            currentHash="$(git log --diff-filter=A "$entry" | grep "^commit " | sed -e 's/commit //')"
-
-            if [[ "$COMMIT_HASH" != *"$currentHash"* ]]
-            then
-              COMMIT_HASH="$COMMIT_HASH $currentHash"
-            fi
-
-            rm $entry;
-
-            oldDate=$dateToday
-            oldCnt=$cnt
-
-            ((counter+=1))
+            # ensure a note is added as a header comment
+            echo "-- DB update $(basename "$LATEST_UPDATE" .sql) -> $(basename "$OUTPUT_FILE" .sql)" > "$OUTPUT_FILE"
+            # fill in the SQL contents under that
+            cat "$entry" >> "$OUTPUT_FILE"
+            # remove the unneeded file
+            rm -f "$entry"
+            # set the newest file to the file we just moved
+            LATEST_UPDATE="$OUTPUT_FILE"
         fi
     done