nchebrov/azerothcore-wotlk
1
0
Fork 0
mirror of https://github.com/mod-playerbots/azerothcore-wotlk.git synced 2026-01-18 03:15:41 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(Core/DBUpdater): implement db auto update (#6576)

Browse Source 
* feat(Core/DBUpdater): implement db auto update

* 1

* 2

* 3

* Some minor improvements

* add find bin for mysql 8.0

* lic

Co-authored-by: Kitzunu <24550914+Kitzunu@users.noreply.github.com>
This commit is contained in:
Kargatum
2021-06-27 23:59:44 +07:00
committed by GitHub
parent ccd73918ae
commit 2d2857ce81
25 changed files with 929 additions and 79 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/server/database/CMakeLists.txt
View File

@@ -6,7 +6,6 @@ CollectSourceFiles(
${CMAKE_CURRENT_SOURCE_DIR}
PRIVATE_SOURCES
# Exclude
${CMAKE_CURRENT_SOURCE_DIR}/Updater
${CMAKE_CURRENT_SOURCE_DIR}/PrecompiledHeaders)
if(USE_COREPCH)
@@ -47,6 +46,9 @@ target_link_libraries(database
PUBLIC
common)
# Add support auto update db for modules
GetModuleList()
set_target_properties(database
PROPERTIES
FOLDER

62
src/server/database/Database/DatabaseLoader.cpp
View File

@@ -5,22 +5,26 @@
#include "DatabaseLoader.h"
#include "Config.h"
// #include "DBUpdater.h" not implement
#include "DBUpdater.h"
#include "DatabaseEnv.h"
#include "Duration.h"
#include "Log.h"
#include "Duration.h"
#include <errmsg.h>
#include <mysqld_error.h>
#include <thread>
DatabaseLoader::DatabaseLoader(std::string const& logger)
: _logger(logger) { }
DatabaseLoader::DatabaseLoader(std::string const& logger, uint32 const defaultUpdateMask)
: _logger(logger), _autoSetup(sConfigMgr->GetOption<bool>("Updates.AutoSetup", true)),
_updateFlags(sConfigMgr->GetOption<uint32>("Updates.EnableDatabases", defaultUpdateMask))
{
}
template <class T>
DatabaseLoader& DatabaseLoader::AddDatabase(DatabaseWorkerPool<T>& pool, std::string const& name)
{
_open.push([this, name, &pool]() -> bool
bool const updatesEnabledForThis = DBUpdater<T>::IsEnabled(_updateFlags);
_open.push([this, name, updatesEnabledForThis, &pool]() -> bool
{
std::string const dbString = sConfigMgr->GetOption<std::string>(name + "DatabaseInfo", "");
if (dbString.empty())
@@ -67,6 +71,16 @@ DatabaseLoader& DatabaseLoader::AddDatabase(DatabaseWorkerPool<T>& pool, std::st
}
}
// Database does not exist
if ((error == ER_BAD_DB_ERROR) && updatesEnabledForThis && _autoSetup && !sConfigMgr->isDryRun())
{
// Try to create the database and connect again if auto setup is enabled
if (DBUpdater<T>::Create(pool) && (!pool.Open()))
{
error = 0;
}
}
// If the error wasn't handled quit
if (error)
{
@@ -85,6 +99,32 @@ DatabaseLoader& DatabaseLoader::AddDatabase(DatabaseWorkerPool<T>& pool, std::st
return true;
});
// Populate and update only if updates are enabled for this pool
if (updatesEnabledForThis && !sConfigMgr->isDryRun())
{
_populate.push([this, name, &pool]() -> bool
{
if (!DBUpdater<T>::Populate(pool))
{
LOG_ERROR(_logger, "Could not populate the %s database, see log for details.", name.c_str());
return false;
}
return true;
});
_update.push([this, name, &pool]() -> bool
{
if (!DBUpdater<T>::Update(pool))
{
LOG_ERROR(_logger, "Could not update the %s database, see log for details.", name.c_str());
return false;
}
return true;
});
}
_prepare.push([this, name, &pool]() -> bool
{
if (!pool.PrepareStatements())
@@ -101,7 +141,7 @@ DatabaseLoader& DatabaseLoader::AddDatabase(DatabaseWorkerPool<T>& pool, std::st
bool DatabaseLoader::Load()
{
return OpenDatabases() && PrepareStatements();
return OpenDatabases() && PopulateDatabases() && UpdateDatabases() && PrepareStatements();
}
bool DatabaseLoader::OpenDatabases()
@@ -109,6 +149,16 @@ bool DatabaseLoader::OpenDatabases()
return Process(_open);
}
bool DatabaseLoader::PopulateDatabases()
{
return Process(_populate);
}
bool DatabaseLoader::UpdateDatabases()
{
return Process(_update);
}
bool DatabaseLoader::PrepareStatements()
{
return Process(_prepare);

8
src/server/database/Database/DatabaseLoader.h
View File

@@ -20,7 +20,7 @@ class DatabaseWorkerPool;
class AC_DATABASE_API DatabaseLoader
{
public:
DatabaseLoader(std::string const& logger);
DatabaseLoader(std::string const& logger, uint32 const defaultUpdateMask = 0);
// Register a database to the loader (lazy implemented)
template <class T>
@@ -42,6 +42,8 @@ public:
private:
bool OpenDatabases();
bool PopulateDatabases();
bool UpdateDatabases();
bool PrepareStatements();
using Predicate = std::function<bool()>;
@@ -52,8 +54,10 @@ private:
bool Process(std::queue<Predicate>& queue);
std::string const _logger;
bool const _autoSetup;
uint32 const _updateFlags;
std::queue<Predicate> _open, _prepare;
std::queue<Predicate> _open, _populate, _update, _prepare;
std::stack<Closer> _close;
};

27
src/server/database/Updater/DBUpdater.cpp
View File

@@ -2,6 +2,7 @@
* Copyright (C) 2016+ AzerothCore <www.azerothcore.org>, released under GNU AGPL v3 license: https://github.com/azerothcore/azerothcore-wotlk/blob/master/LICENSE-AGPL3
* Copyright (C) 2021+ WarheadCore <https://github.com/WarheadCore>
*/
#include "DBUpdater.h"
#include "BuiltInConfig.h"
#include "Config.h"
@@ -29,7 +30,7 @@ bool DBUpdaterUtil::CheckExecutable()
boost::filesystem::path exe(GetCorrectedMySQLExecutable());
if (!is_regular_file(exe))
{
exe = Warhead::SearchExecutableInPath("mysql");
exe = Acore::SearchExecutableInPath("mysql");
if (!exe.empty() && is_regular_file(exe))
{
// Correct the path to the cli
@@ -80,7 +81,7 @@ bool DBUpdater<LoginDatabaseConnection>::IsEnabled(uint32 const updateMask)
template<>
std::string DBUpdater<LoginDatabaseConnection>::GetDBModuleName()
{
return "db_auth";
return "db-auth";
}
// World Database
@@ -112,7 +113,7 @@ bool DBUpdater<WorldDatabaseConnection>::IsEnabled(uint32 const updateMask)
template<>
std::string DBUpdater<WorldDatabaseConnection>::GetDBModuleName()
{
return "db_world";
return "db-world";
}
// Character Database
@@ -144,7 +145,7 @@ bool DBUpdater<CharacterDatabaseConnection>::IsEnabled(uint32 const updateMask)
template<>
std::string DBUpdater<CharacterDatabaseConnection>::GetDBModuleName()
{
return "db_characters";
return "db-characters";
}
// All
@@ -162,7 +163,7 @@ bool DBUpdater<T>::Create(DatabaseWorkerPool<T>& pool)
std::string answer;
std::getline(std::cin, answer);
if (!sConfigMgr->isDryRun() && !answer.empty() && !(answer.substr(0, 1) == "y"))
if (!answer.empty() && !(answer.substr(0, 1) == "y"))
return false;
LOG_INFO("sql.updates", "Creating database \"%s\"...", pool.GetConnectionInfo()->database.c_str());
@@ -219,7 +220,7 @@ bool DBUpdater<T>::Update(DatabaseWorkerPool<T>& pool)
auto CheckUpdateTable = [&](std::string const& tableName)
{
auto checkTable = DBUpdater<T>::Retrieve(pool, Warhead::StringFormat("SHOW TABLES LIKE '%s'", tableName.c_str()));
auto checkTable = DBUpdater<T>::Retrieve(pool, Acore::StringFormat("SHOW TABLES LIKE '%s'", tableName.c_str()));
if (!checkTable)
{
LOG_WARN("sql.updates", "> Table '%s' not exist! Try add based table", tableName.c_str());
@@ -253,17 +254,17 @@ bool DBUpdater<T>::Update(DatabaseWorkerPool<T>& pool)
try
{
result = updateFetcher.Update(
sConfigMgr->GetBoolDefault("Updates.Redundancy", true),
sConfigMgr->GetBoolDefault("Updates.AllowRehash", true),
sConfigMgr->GetBoolDefault("Updates.ArchivedRedundancy", false),
sConfigMgr->GetIntDefault("Updates.CleanDeadRefMaxCount", 3));
sConfigMgr->GetOption<bool>("Updates.Redundancy", true),
sConfigMgr->GetOption<bool>("Updates.AllowRehash", true),
sConfigMgr->GetOption<bool>("Updates.ArchivedRedundancy", false),
sConfigMgr->GetOption<int32>("Updates.CleanDeadRefMaxCount", 3));
}
catch (UpdateException&)
{
return false;
}
std::string const info = Warhead::StringFormat("Containing " SZFMTD " new and " SZFMTD " archived updates.",
std::string const info = Acore::StringFormat("Containing " SZFMTD " new and " SZFMTD " archived updates.",
result.recent, result.archived);
if (!result.updated)
@@ -412,7 +413,7 @@ void DBUpdater<T>::ApplyFile(DatabaseWorkerPool<T>& pool, std::string const& hos
args.emplace_back(database);
// Invokes a mysql process which doesn't leak credentials to logs
int const ret = Warhead::StartProcess(DBUpdaterUtil::GetCorrectedMySQLExecutable(), args,
int const ret = Acore::StartProcess(DBUpdaterUtil::GetCorrectedMySQLExecutable(), args,
"sql.updates", path.generic_string(), true);
if (ret != EXIT_SUCCESS)
@@ -420,7 +421,7 @@ void DBUpdater<T>::ApplyFile(DatabaseWorkerPool<T>& pool, std::string const& hos
LOG_FATAL("sql.updates", "Applying of file \'%s\' to database \'%s\' failed!" \
" If you are a user, please pull the latest revision from the repository. "
"Also make sure you have not applied any of the databases with your sql client. "
"You cannot use auto-update system and import sql files from WarheadCore repository with your sql client. "
"You cannot use auto-update system and import sql files from AzerothCore repository with your sql client. "
"If you are a developer, please fix your sql query.",
path.generic_string().c_str(), pool.GetConnectionInfo()->database.c_str());

90
src/server/database/Updater/UpdateFetcher.cpp
View File

@@ -63,7 +63,7 @@ void UpdateFetcher::FillFileListRecursively(Path const& path, LocaleFileStorage&
}
else if (itr->path().extension() == ".sql")
{
LOG_TRACE("sql.updates", "Added locale file \"%s\".", itr->path().filename().generic_string().c_str());
LOG_TRACE("sql.updates", "Added locale file \"%s\" state '%s'.", itr->path().filename().generic_string().c_str(), AppliedFileEntry::StateConvert(state).c_str());
LocaleFileEntry const entry = { itr->path(), state };
@@ -95,6 +95,7 @@ UpdateFetcher::DirectoryStorage UpdateFetcher::ReceiveIncludedDirectories() cons
Field* fields = result->Fetch();
std::string path = fields[0].GetString();
std::string state = fields[1].GetString();
if (path.substr(0, 1) == "$")
path = _sourceDirectory->generic_string() + path.substr(1);
@@ -106,28 +107,29 @@ UpdateFetcher::DirectoryStorage UpdateFetcher::ReceiveIncludedDirectories() cons
continue;
}
DirectoryEntry const entry = { p, AppliedFileEntry::StateConvert(fields[1].GetString()) };
DirectoryEntry const entry = { p, AppliedFileEntry::StateConvert(state) };
directories.push_back(entry);
LOG_TRACE("sql.updates", "Added applied file \"%s\" from remote.", p.filename().generic_string().c_str());
LOG_TRACE("sql.updates", "Added applied file \"%s\" '%s' state from remote.", p.filename().generic_string().c_str(), state.c_str());
} while (result->NextRow());
std::vector<std::string> moduleList;
auto const& _modulesTokens = Warhead::Tokenize(WH_MODULES_LIST, ',', true);
auto const& _modulesTokens = Acore::Tokenize(AC_MODULES_LIST, ',', true);
for (auto const& itr : _modulesTokens)
moduleList.push_back(std::string(itr));
moduleList.emplace_back(itr);
// data/sql
for (auto const& itr : moduleList)
{
std::string path = _sourceDirectory->generic_string() + "/modules/" + itr + "/sql/" + _dbModuleName; // module/mod-name/sql/db_world
std::string path = _sourceDirectory->generic_string() + "/modules/" + itr + "/data/sql/" + _dbModuleName; // modules/mod-name/data/sql/db-world
Path const p(path);
if (!is_directory(p))
continue;
DirectoryEntry const entry = { p, AppliedFileEntry::StateConvert("RELEASED") };
DirectoryEntry const entry = { p, AppliedFileEntry::StateConvert("CUSTOM") };
directories.push_back(entry);
LOG_TRACE("sql.updates", "Added applied modules file \"%s\" from remote.", p.filename().generic_string().c_str());
@@ -148,11 +150,12 @@ UpdateFetcher::AppliedFileStorage UpdateFetcher::ReceiveAppliedFiles() const
{
Field* fields = result->Fetch();
AppliedFileEntry const entry = { fields[0].GetString(), fields[1].GetString(),
AppliedFileEntry::StateConvert(fields[2].GetString()), fields[3].GetUInt64()
};
AppliedFileEntry const entry =
{
fields[0].GetString(), fields[1].GetString(), AppliedFileEntry::StateConvert(fields[2].GetString()), fields[3].GetUInt64()
};
map.insert(std::make_pair(entry.name, entry));
map.emplace(entry.name, entry);
} while (result->NextRow());
return map;
@@ -207,11 +210,14 @@ UpdateResult UpdateFetcher::Update(bool const redundancyChecks,
size_t importedUpdates = 0;
for (auto const& availableQuery : available)
auto ApplyUpdateFile = [&](LocaleFileEntry const& sqlFile)
{
LOG_DEBUG("sql.updates", "Checking update \"%s\"...", availableQuery.first.filename().generic_string().c_str());
auto filePath = sqlFile.first;
auto fileState = sqlFile.second;
AppliedFileStorage::const_iterator iter = applied.find(availableQuery.first.filename().string());
LOG_DEBUG("sql.updates", "Checking update \"%s\"...", filePath.filename().generic_string().c_str());
AppliedFileStorage::const_iterator iter = applied.find(filePath.filename().string());
if (iter != applied.end())
{
// If redundancy is disabled, skip it, because the update is already applied.
@@ -219,19 +225,19 @@ UpdateResult UpdateFetcher::Update(bool const redundancyChecks,
{
LOG_DEBUG("sql.updates", ">> Update is already applied, skipping redundancy checks.");
applied.erase(iter);
continue;
return;
}
// If the update is in an archived directory and is marked as archived in our database, skip redundancy checks (archived updates never change).
if (!archivedRedundancy && (iter->second.state == ARCHIVED) && (availableQuery.second == ARCHIVED))
if (!archivedRedundancy && (iter->second.state == ARCHIVED) && (sqlFile.second == ARCHIVED))
{
LOG_DEBUG("sql.updates", ">> Update is archived and marked as archived in database, skipping redundancy checks.");
applied.erase(iter);
continue;
return;
}
}
std::string const hash = ByteArrayToHexStr(Warhead::Crypto::SHA1::GetDigestOf(ReadSQLUpdate(availableQuery.first)));
std::string const hash = ByteArrayToHexStr(Acore::Crypto::SHA1::GetDigestOf(ReadSQLUpdate(filePath)));
UpdateMode mode = MODE_APPLY;
@@ -244,6 +250,7 @@ UpdateResult UpdateFetcher::Update(bool const redundancyChecks,
{
// Check if the original file was removed. If not, we've got a problem.
LocaleFileStorage::const_iterator localeIter;
// Push localeIter forward
for (localeIter = available.begin(); (localeIter != available.end()) &&
(localeIter->first.filename().string() != hashIter->second); ++localeIter);
@@ -253,25 +260,24 @@ UpdateResult UpdateFetcher::Update(bool const redundancyChecks,
{
LOG_WARN("sql.updates", ">> It seems like the update \"%s\" \'%s\' was renamed, but the old file is still there! " \
"Treating it as a new file! (It is probably an unmodified copy of the file \"%s\")",
availableQuery.first.filename().string().c_str(), hash.substr(0, 7).c_str(),
filePath.filename().string().c_str(), hash.substr(0, 7).c_str(),
localeIter->first.filename().string().c_str());
}
// It is safe to treat the file as renamed here
else
else // It is safe to treat the file as renamed here
{
LOG_INFO("sql.updates", ">> Renaming update \"%s\" to \"%s\" \'%s\'.",
hashIter->second.c_str(), availableQuery.first.filename().string().c_str(), hash.substr(0, 7).c_str());
hashIter->second.c_str(), filePath.filename().string().c_str(), hash.substr(0, 7).c_str());
RenameEntry(hashIter->second, availableQuery.first.filename().string());
RenameEntry(hashIter->second, filePath.filename().string());
applied.erase(hashIter->second);
continue;
return;
}
}
// Apply the update if it was never seen before.
else
{
LOG_INFO("sql.updates", ">> Applying update \"%s\" \'%s\'...",
availableQuery.first.filename().string().c_str(), hash.substr(0, 7).c_str());
filePath.filename().string().c_str(), hash.substr(0, 7).c_str());
}
}
// Rehash the update entry if it exists in our database with an empty hash.
@@ -279,42 +285,42 @@ UpdateResult UpdateFetcher::Update(bool const redundancyChecks,
{
mode = MODE_REHASH;
LOG_INFO("sql.updates", ">> Re-hashing update \"%s\" \'%s\'...", availableQuery.first.filename().string().c_str(),
hash.substr(0, 7).c_str());
LOG_INFO("sql.updates", ">> Re-hashing update \"%s\" \'%s\'...", filePath.filename().string().c_str(),
hash.substr(0, 7).c_str());
}
else
{
// If the hash of the files differs from the one stored in our database, reapply the update (because it changed).
if (iter->second.hash != hash)
{
LOG_INFO("sql.updates", ">> Reapplying update \"%s\" \'%s\' -> \'%s\' (it changed)...", availableQuery.first.filename().string().c_str(),
iter->second.hash.substr(0, 7).c_str(), hash.substr(0, 7).c_str());
LOG_INFO("sql.updates", ">> Reapplying update \"%s\" \'%s\' -> \'%s\' (it changed)...", filePath.filename().string().c_str(),
iter->second.hash.substr(0, 7).c_str(), hash.substr(0, 7).c_str());
}
else
{
// If the file wasn't changed and just moved, update its state (if necessary).
if (iter->second.state != availableQuery.second)
if (iter->second.state != fileState)
{
LOG_DEBUG("sql.updates", ">> Updating the state of \"%s\" to \'%s\'...",
availableQuery.first.filename().string().c_str(), AppliedFileEntry::StateConvert(availableQuery.second).c_str());
filePath.filename().string().c_str(), AppliedFileEntry::StateConvert(fileState).c_str());
UpdateState(availableQuery.first.filename().string(), availableQuery.second);
UpdateState(filePath.filename().string(), fileState);
}
LOG_DEBUG("sql.updates", ">> Update is already applied and matches the hash \'%s\'.", hash.substr(0, 7).c_str());
applied.erase(iter);
continue;
return;
}
}
uint32 speed = 0;
AppliedFileEntry const file = { availableQuery.first.filename().string(), hash, availableQuery.second, 0 };
AppliedFileEntry const file = { filePath.filename().string(), hash, fileState, 0 };
switch (mode)
{
case MODE_APPLY:
speed = Apply(availableQuery.first);
speed = Apply(filePath);
/* fallthrough */
case MODE_REHASH:
UpdateEntry(file, speed);
@@ -326,6 +332,20 @@ UpdateResult UpdateFetcher::Update(bool const redundancyChecks,
if (mode == MODE_APPLY)
++importedUpdates;
};
// Apply default updates
for (auto const& availableQuery : available)
{
if (availableQuery.second != CUSTOM)
ApplyUpdateFile(availableQuery);
}
// Apply only custom updates
for (auto const& availableQuery : available)
{
if (availableQuery.second == CUSTOM)
ApplyUpdateFile(availableQuery);
}
// Cleanup up orphaned entries (if enabled)

23
src/server/database/Updater/UpdateFetcher.h
View File

@@ -55,6 +55,7 @@ private:
enum State
{
RELEASED,
CUSTOM,
ARCHIVED
};
@@ -64,21 +65,33 @@ private:
: name(name_), hash(hash_), state(state_), timestamp(timestamp_) { }
std::string const name;
std::string const hash;
State const state;
uint64 const timestamp;
static inline State StateConvert(std::string const& state)
{
return (state == "RELEASED") ? RELEASED : ARCHIVED;
if (state == "RELEASED")
return RELEASED;
else if (state == "CUSTOM")
return CUSTOM;
return ARCHIVED;
}
static inline std::string StateConvert(State const state)
{
return (state == RELEASED) ? "RELEASED" : "ARCHIVED";
switch (state)
{
case RELEASED:
return "RELEASED";
case CUSTOM:
return "CUSTOM";
case ARCHIVED:
return "ARCHIVED";
default:
return "";
}
}
std::string GetStateAsString() const