mirror of
https://github.com/mod-playerbots/azerothcore-wotlk.git
synced 2026-01-21 20:56:23 +00:00
feat(Core/Database): port TrinityCore database API (#5611)
This commit is contained in:
Kargatum
433
src/server/database/Updater/DBUpdater.cpp
Normal file
433
src/server/database/Updater/DBUpdater.cpp
Normal file
@@ -0,0 +1,433 @@
|
||||
/*
|
||||
* Copyright (C) 2016+ AzerothCore <www.azerothcore.org>, released under GNU AGPL v3 license: https://github.com/azerothcore/azerothcore-wotlk/blob/master/LICENSE-AGPL3
|
||||
* Copyright (C) 2021+ WarheadCore <https://github.com/WarheadCore>
|
||||
*/
|
||||
#include "DBUpdater.h"
|
||||
#include "BuiltInConfig.h"
|
||||
#include "Config.h"
|
||||
#include "DatabaseEnv.h"
|
||||
#include "DatabaseLoader.h"
|
||||
#include "GitRevision.h"
|
||||
#include "Log.h"
|
||||
#include "QueryResult.h"
|
||||
#include "StartProcess.h"
|
||||
#include "UpdateFetcher.h"
|
||||
#include <boost/filesystem/operations.hpp>
|
||||
#include <fstream>
|
||||
#include <iostream>
|
||||
|
||||
std::string DBUpdaterUtil::GetCorrectedMySQLExecutable()
|
||||
{
|
||||
if (!corrected_path().empty())
|
||||
return corrected_path();
|
||||
else
|
||||
return BuiltInConfig::GetMySQLExecutable();
|
||||
}
|
||||
|
||||
bool DBUpdaterUtil::CheckExecutable()
|
||||
{
|
||||
boost::filesystem::path exe(GetCorrectedMySQLExecutable());
|
||||
if (!is_regular_file(exe))
|
||||
{
|
||||
exe = Warhead::SearchExecutableInPath("mysql");
|
||||
if (!exe.empty() && is_regular_file(exe))
|
||||
{
|
||||
// Correct the path to the cli
|
||||
corrected_path() = absolute(exe).generic_string();
|
||||
return true;
|
||||
}
|
||||
|
||||
LOG_FATAL("sql.updates", "Didn't find any executable MySQL binary at \'%s\' or in path, correct the path in the *.conf (\"MySQLExecutable\").",
|
||||
absolute(exe).generic_string().c_str());
|
||||
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
std::string& DBUpdaterUtil::corrected_path()
|
||||
{
|
||||
static std::string path;
|
||||
return path;
|
||||
}
|
||||
|
||||
// Auth Database
|
||||
template<>
|
||||
std::string DBUpdater<LoginDatabaseConnection>::GetConfigEntry()
|
||||
{
|
||||
return "Updates.Auth";
|
||||
}
|
||||
|
||||
template<>
|
||||
std::string DBUpdater<LoginDatabaseConnection>::GetTableName()
|
||||
{
|
||||
return "Auth";
|
||||
}
|
||||
|
||||
template<>
|
||||
std::string DBUpdater<LoginDatabaseConnection>::GetBaseFilesDirectory()
|
||||
{
|
||||
return BuiltInConfig::GetSourceDirectory() + "/data/sql/base/db_auth/";
|
||||
}
|
||||
|
||||
template<>
|
||||
bool DBUpdater<LoginDatabaseConnection>::IsEnabled(uint32 const updateMask)
|
||||
{
|
||||
// This way silences warnings under msvc
|
||||
return (updateMask & DatabaseLoader::DATABASE_LOGIN) ? true : false;
|
||||
}
|
||||
|
||||
template<>
|
||||
std::string DBUpdater<LoginDatabaseConnection>::GetDBModuleName()
|
||||
{
|
||||
return "db_auth";
|
||||
}
|
||||
|
||||
// World Database
|
||||
template<>
|
||||
std::string DBUpdater<WorldDatabaseConnection>::GetConfigEntry()
|
||||
{
|
||||
return "Updates.World";
|
||||
}
|
||||
|
||||
template<>
|
||||
std::string DBUpdater<WorldDatabaseConnection>::GetTableName()
|
||||
{
|
||||
return "World";
|
||||
}
|
||||
|
||||
template<>
|
||||
std::string DBUpdater<WorldDatabaseConnection>::GetBaseFilesDirectory()
|
||||
{
|
||||
return BuiltInConfig::GetSourceDirectory() + "/data/sql/base/db_world/";
|
||||
}
|
||||
|
||||
template<>
|
||||
bool DBUpdater<WorldDatabaseConnection>::IsEnabled(uint32 const updateMask)
|
||||
{
|
||||
// This way silences warnings under msvc
|
||||
return (updateMask & DatabaseLoader::DATABASE_WORLD) ? true : false;
|
||||
}
|
||||
|
||||
template<>
|
||||
std::string DBUpdater<WorldDatabaseConnection>::GetDBModuleName()
|
||||
{
|
||||
return "db_world";
|
||||
}
|
||||
|
||||
// Character Database
|
||||
template<>
|
||||
std::string DBUpdater<CharacterDatabaseConnection>::GetConfigEntry()
|
||||
{
|
||||
return "Updates.Character";
|
||||
}
|
||||
|
||||
template<>
|
||||
std::string DBUpdater<CharacterDatabaseConnection>::GetTableName()
|
||||
{
|
||||
return "Character";
|
||||
}
|
||||
|
||||
template<>
|
||||
std::string DBUpdater<CharacterDatabaseConnection>::GetBaseFilesDirectory()
|
||||
{
|
||||
return BuiltInConfig::GetSourceDirectory() + "/data/sql/base/db_characters/";
|
||||
}
|
||||
|
||||
template<>
|
||||
bool DBUpdater<CharacterDatabaseConnection>::IsEnabled(uint32 const updateMask)
|
||||
{
|
||||
// This way silences warnings under msvc
|
||||
return (updateMask & DatabaseLoader::DATABASE_CHARACTER) ? true : false;
|
||||
}
|
||||
|
||||
template<>
|
||||
std::string DBUpdater<CharacterDatabaseConnection>::GetDBModuleName()
|
||||
{
|
||||
return "db_characters";
|
||||
}
|
||||
|
||||
// All
|
||||
template<class T>
|
||||
BaseLocation DBUpdater<T>::GetBaseLocationType()
|
||||
{
|
||||
return LOCATION_REPOSITORY;
|
||||
}
|
||||
|
||||
template<class T>
|
||||
bool DBUpdater<T>::Create(DatabaseWorkerPool<T>& pool)
|
||||
{
|
||||
LOG_WARN("sql.updates", "Database \"%s\" does not exist, do you want to create it? [yes (default) / no]: ",
|
||||
pool.GetConnectionInfo()->database.c_str());
|
||||
|
||||
std::string answer;
|
||||
std::getline(std::cin, answer);
|
||||
if (!sConfigMgr->isDryRun() && !answer.empty() && !(answer.substr(0, 1) == "y"))
|
||||
return false;
|
||||
|
||||
LOG_INFO("sql.updates", "Creating database \"%s\"...", pool.GetConnectionInfo()->database.c_str());
|
||||
|
||||
// Path of temp file
|
||||
static Path const temp("create_table.sql");
|
||||
|
||||
// Create temporary query to use external MySQL CLi
|
||||
std::ofstream file(temp.generic_string());
|
||||
if (!file.is_open())
|
||||
{
|
||||
LOG_FATAL("sql.updates", "Failed to create temporary query file \"%s\"!", temp.generic_string().c_str());
|
||||
return false;
|
||||
}
|
||||
|
||||
file << "CREATE DATABASE `" << pool.GetConnectionInfo()->database << "` DEFAULT CHARACTER SET UTF8MB4 COLLATE utf8mb4_general_ci;\n\n";
|
||||
|
||||
file.close();
|
||||
|
||||
try
|
||||
{
|
||||
DBUpdater<T>::ApplyFile(pool, pool.GetConnectionInfo()->host, pool.GetConnectionInfo()->user, pool.GetConnectionInfo()->password,
|
||||
pool.GetConnectionInfo()->port_or_socket, "", pool.GetConnectionInfo()->ssl, temp);
|
||||
}
|
||||
catch (UpdateException&)
|
||||
{
|
||||
LOG_FATAL("sql.updates", "Failed to create database %s! Does the user (named in *.conf) have `CREATE`, `ALTER`, `DROP`, `INSERT` and `DELETE` privileges on the MySQL server?", pool.GetConnectionInfo()->database.c_str());
|
||||
boost::filesystem::remove(temp);
|
||||
return false;
|
||||
}
|
||||
|
||||
LOG_INFO("sql.updates", "Done.");
|
||||
LOG_INFO("sql.updates", " ");
|
||||
boost::filesystem::remove(temp);
|
||||
return true;
|
||||
}
|
||||
|
||||
template<class T>
|
||||
bool DBUpdater<T>::Update(DatabaseWorkerPool<T>& pool)
|
||||
{
|
||||
if (!DBUpdaterUtil::CheckExecutable())
|
||||
return false;
|
||||
|
||||
LOG_INFO("sql.updates", "Updating %s database...", DBUpdater<T>::GetTableName().c_str());
|
||||
|
||||
Path const sourceDirectory(BuiltInConfig::GetSourceDirectory());
|
||||
|
||||
if (!is_directory(sourceDirectory))
|
||||
{
|
||||
LOG_ERROR("sql.updates", "DBUpdater: The given source directory %s does not exist, change the path to the directory where your sql directory exists (for example c:\\source\\trinitycore). Shutting down.",
|
||||
sourceDirectory.generic_string().c_str());
|
||||
return false;
|
||||
}
|
||||
|
||||
auto CheckUpdateTable = [&](std::string const& tableName)
|
||||
{
|
||||
auto checkTable = DBUpdater<T>::Retrieve(pool, Warhead::StringFormat("SHOW TABLES LIKE '%s'", tableName.c_str()));
|
||||
if (!checkTable)
|
||||
{
|
||||
LOG_WARN("sql.updates", "> Table '%s' not exist! Try add based table", tableName.c_str());
|
||||
|
||||
Path const temp(GetBaseFilesDirectory() + tableName + ".sql");
|
||||
|
||||
try
|
||||
{
|
||||
DBUpdater<T>::ApplyFile(pool, temp);
|
||||
}
|
||||
catch (UpdateException&)
|
||||
{
|
||||
LOG_FATAL("sql.updates", "Failed apply file to database %s! Does the user (named in *.conf) have `INSERT` and `DELETE` privileges on the MySQL server?", pool.GetConnectionInfo()->database.c_str());
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
return true;
|
||||
};
|
||||
|
||||
if (!CheckUpdateTable("updates") || !CheckUpdateTable("updates_include"))
|
||||
return false;
|
||||
|
||||
UpdateFetcher updateFetcher(sourceDirectory, [&](std::string const & query) { DBUpdater<T>::Apply(pool, query); },
|
||||
[&](Path const & file) { DBUpdater<T>::ApplyFile(pool, file); },
|
||||
[&](std::string const & query) -> QueryResult { return DBUpdater<T>::Retrieve(pool, query); }, DBUpdater<T>::GetDBModuleName());
|
||||
|
||||
UpdateResult result;
|
||||
try
|
||||
{
|
||||
result = updateFetcher.Update(
|
||||
sConfigMgr->GetBoolDefault("Updates.Redundancy", true),
|
||||
sConfigMgr->GetBoolDefault("Updates.AllowRehash", true),
|
||||
sConfigMgr->GetBoolDefault("Updates.ArchivedRedundancy", false),
|
||||
sConfigMgr->GetIntDefault("Updates.CleanDeadRefMaxCount", 3));
|
||||
}
|
||||
catch (UpdateException&)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
std::string const info = Warhead::StringFormat("Containing " SZFMTD " new and " SZFMTD " archived updates.",
|
||||
result.recent, result.archived);
|
||||
|
||||
if (!result.updated)
|
||||
LOG_INFO("sql.updates", ">> %s database is up-to-date! %s", DBUpdater<T>::GetTableName().c_str(), info.c_str());
|
||||
else
|
||||
LOG_INFO("sql.updates", ">> Applied " SZFMTD " %s. %s", result.updated, result.updated == 1 ? "query" : "queries", info.c_str());
|
||||
|
||||
LOG_INFO("sql.updates", " ");
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
template<class T>
|
||||
bool DBUpdater<T>::Populate(DatabaseWorkerPool<T>& pool)
|
||||
{
|
||||
{
|
||||
QueryResult const result = Retrieve(pool, "SHOW TABLES");
|
||||
if (result && (result->GetRowCount() > 0))
|
||||
return true;
|
||||
}
|
||||
|
||||
if (!DBUpdaterUtil::CheckExecutable())
|
||||
return false;
|
||||
|
||||
LOG_INFO("sql.updates", "Database %s is empty, auto populating it...", DBUpdater<T>::GetTableName().c_str());
|
||||
|
||||
std::string const DirPathStr = DBUpdater<T>::GetBaseFilesDirectory();
|
||||
|
||||
Path const DirPath(DirPathStr);
|
||||
if (!boost::filesystem::is_directory(DirPath))
|
||||
{
|
||||
LOG_ERROR("sql.updates", ">> Directory \"%s\" not exist", DirPath.generic_string().c_str());
|
||||
return false;
|
||||
}
|
||||
|
||||
if (DirPath.empty())
|
||||
{
|
||||
LOG_ERROR("sql.updates", ">> Directory \"%s\" is empty", DirPath.generic_string().c_str());
|
||||
return false;
|
||||
}
|
||||
|
||||
boost::filesystem::directory_iterator const DirItr;
|
||||
uint32 FilesCount = 0;
|
||||
|
||||
for (boost::filesystem::directory_iterator itr(DirPath); itr != DirItr; ++itr)
|
||||
{
|
||||
if (itr->path().extension() == ".sql")
|
||||
FilesCount++;
|
||||
}
|
||||
|
||||
if (!FilesCount)
|
||||
{
|
||||
LOG_ERROR("sql.updates", ">> In directory \"%s\" not exist '*.sql' files", DirPath.generic_string().c_str());
|
||||
return false;
|
||||
}
|
||||
|
||||
for (boost::filesystem::directory_iterator itr(DirPath); itr != DirItr; ++itr)
|
||||
{
|
||||
if (itr->path().extension() != ".sql")
|
||||
continue;
|
||||
|
||||
LOG_INFO("sql.updates", ">> Applying \'%s\'...", itr->path().filename().generic_string().c_str());
|
||||
|
||||
try
|
||||
{
|
||||
ApplyFile(pool, itr->path());
|
||||
}
|
||||
catch (UpdateException&)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
LOG_INFO("sql.updates", ">> Done!");
|
||||
LOG_INFO("sql.updates", " ");
|
||||
return true;
|
||||
}
|
||||
|
||||
template<class T>
|
||||
QueryResult DBUpdater<T>::Retrieve(DatabaseWorkerPool<T>& pool, std::string const& query)
|
||||
{
|
||||
return pool.Query(query.c_str());
|
||||
}
|
||||
|
||||
template<class T>
|
||||
void DBUpdater<T>::Apply(DatabaseWorkerPool<T>& pool, std::string const& query)
|
||||
{
|
||||
pool.DirectExecute(query.c_str());
|
||||
}
|
||||
|
||||
template<class T>
|
||||
void DBUpdater<T>::ApplyFile(DatabaseWorkerPool<T>& pool, Path const& path)
|
||||
{
|
||||
DBUpdater<T>::ApplyFile(pool, pool.GetConnectionInfo()->host, pool.GetConnectionInfo()->user, pool.GetConnectionInfo()->password,
|
||||
pool.GetConnectionInfo()->port_or_socket, pool.GetConnectionInfo()->database, pool.GetConnectionInfo()->ssl, path);
|
||||
}
|
||||
|
||||
template<class T>
|
||||
void DBUpdater<T>::ApplyFile(DatabaseWorkerPool<T>& pool, std::string const& host, std::string const& user,
|
||||
std::string const& password, std::string const& port_or_socket, std::string const& database, std::string const& ssl, Path const& path)
|
||||
{
|
||||
std::vector<std::string> args;
|
||||
args.reserve(7);
|
||||
|
||||
// CLI Client connection info
|
||||
args.emplace_back("-h" + host);
|
||||
args.emplace_back("-u" + user);
|
||||
|
||||
if (!password.empty())
|
||||
args.emplace_back("-p" + password);
|
||||
|
||||
// Check if we want to connect through ip or socket (Unix only)
|
||||
#ifdef _WIN32
|
||||
|
||||
if (host == ".")
|
||||
args.emplace_back("--protocol=PIPE");
|
||||
else
|
||||
args.emplace_back("-P" + port_or_socket);
|
||||
|
||||
#else
|
||||
|
||||
if (!std::isdigit(port_or_socket[0]))
|
||||
{
|
||||
// We can't check if host == "." here, because it is named localhost if socket option is enabled
|
||||
args.emplace_back("-P0");
|
||||
args.emplace_back("--protocol=SOCKET");
|
||||
args.emplace_back("-S" + port_or_socket);
|
||||
}
|
||||
else
|
||||
// generic case
|
||||
args.emplace_back("-P" + port_or_socket);
|
||||
|
||||
#endif
|
||||
|
||||
// Set the default charset to utf8
|
||||
args.emplace_back("--default-character-set=utf8");
|
||||
|
||||
// Set max allowed packet to 1 GB
|
||||
args.emplace_back("--max-allowed-packet=1GB");
|
||||
|
||||
if (ssl == "ssl")
|
||||
args.emplace_back("--ssl");
|
||||
|
||||
// Database
|
||||
if (!database.empty())
|
||||
args.emplace_back(database);
|
||||
|
||||
// Invokes a mysql process which doesn't leak credentials to logs
|
||||
int const ret = Warhead::StartProcess(DBUpdaterUtil::GetCorrectedMySQLExecutable(), args,
|
||||
"sql.updates", path.generic_string(), true);
|
||||
|
||||
if (ret != EXIT_SUCCESS)
|
||||
{
|
||||
LOG_FATAL("sql.updates", "Applying of file \'%s\' to database \'%s\' failed!" \
|
||||
" If you are a user, please pull the latest revision from the repository. "
|
||||
"Also make sure you have not applied any of the databases with your sql client. "
|
||||
"You cannot use auto-update system and import sql files from WarheadCore repository with your sql client. "
|
||||
"If you are a developer, please fix your sql query.",
|
||||
path.generic_string().c_str(), pool.GetConnectionInfo()->database.c_str());
|
||||
|
||||
throw UpdateException("update failed");
|
||||
}
|
||||
}
|
||||
|
||||
template class AC_DATABASE_API DBUpdater<LoginDatabaseConnection>;
|
||||
template class AC_DATABASE_API DBUpdater<WorldDatabaseConnection>;
|
||||
template class AC_DATABASE_API DBUpdater<CharacterDatabaseConnection>;
|
||||
79
src/server/database/Updater/DBUpdater.h
Normal file
79
src/server/database/Updater/DBUpdater.h
Normal file
@@ -0,0 +1,79 @@
|
||||
/*
|
||||
* Copyright (C) 2016+ AzerothCore <www.azerothcore.org>, released under GNU AGPL v3 license: https://github.com/azerothcore/azerothcore-wotlk/blob/master/LICENSE-AGPL3
|
||||
* Copyright (C) 2021+ WarheadCore <https://github.com/WarheadCore>
|
||||
*/
|
||||
|
||||
#ifndef DBUpdater_h__
|
||||
#define DBUpdater_h__
|
||||
|
||||
#include "DatabaseEnv.h"
|
||||
#include "Define.h"
|
||||
#include <string>
|
||||
|
||||
template <class T>
|
||||
class DatabaseWorkerPool;
|
||||
|
||||
namespace boost
|
||||
{
|
||||
namespace filesystem
|
||||
{
|
||||
class path;
|
||||
}
|
||||
}
|
||||
|
||||
class AC_DATABASE_API UpdateException : public std::exception
|
||||
{
|
||||
public:
|
||||
UpdateException(std::string const& msg) : _msg(msg) { }
|
||||
~UpdateException() throw() { }
|
||||
|
||||
char const* what() const throw() override { return _msg.c_str(); }
|
||||
|
||||
private:
|
||||
std::string const _msg;
|
||||
};
|
||||
|
||||
enum BaseLocation
|
||||
{
|
||||
LOCATION_REPOSITORY,
|
||||
LOCATION_DOWNLOAD
|
||||
};
|
||||
|
||||
class AC_DATABASE_API DBUpdaterUtil
|
||||
{
|
||||
public:
|
||||
static std::string GetCorrectedMySQLExecutable();
|
||||
|
||||
static bool CheckExecutable();
|
||||
|
||||
private:
|
||||
static std::string& corrected_path();
|
||||
};
|
||||
|
||||
template <class T>
|
||||
class AC_DATABASE_API DBUpdater
|
||||
{
|
||||
public:
|
||||
using Path = boost::filesystem::path;
|
||||
|
||||
static inline std::string GetConfigEntry();
|
||||
static inline std::string GetTableName();
|
||||
static std::string GetBaseFilesDirectory();
|
||||
static bool IsEnabled(uint32 const updateMask);
|
||||
static BaseLocation GetBaseLocationType();
|
||||
static bool Create(DatabaseWorkerPool<T>& pool);
|
||||
static bool Update(DatabaseWorkerPool<T>& pool);
|
||||
static bool Populate(DatabaseWorkerPool<T>& pool);
|
||||
|
||||
// module
|
||||
static std::string GetDBModuleName();
|
||||
|
||||
private:
|
||||
static QueryResult Retrieve(DatabaseWorkerPool<T>& pool, std::string const& query);
|
||||
static void Apply(DatabaseWorkerPool<T>& pool, std::string const& query);
|
||||
static void ApplyFile(DatabaseWorkerPool<T>& pool, Path const& path);
|
||||
static void ApplyFile(DatabaseWorkerPool<T>& pool, std::string const& host, std::string const& user,
|
||||
std::string const& password, std::string const& port_or_socket, std::string const& database, std::string const& ssl, Path const& path);
|
||||
};
|
||||
|
||||
#endif // DBUpdater_h__
|
||||
433
src/server/database/Updater/UpdateFetcher.cpp
Normal file
433
src/server/database/Updater/UpdateFetcher.cpp
Normal file
@@ -0,0 +1,433 @@
|
||||
/*
|
||||
* Copyright (C) 2016+ AzerothCore <www.azerothcore.org>, released under GNU AGPL v3 license: https://github.com/azerothcore/azerothcore-wotlk/blob/master/LICENSE-AGPL3
|
||||
* Copyright (C) 2021+ WarheadCore <https://github.com/WarheadCore>
|
||||
*/
|
||||
|
||||
#include "UpdateFetcher.h"
|
||||
#include "Common.h"
|
||||
#include "CryptoHash.h"
|
||||
#include "DBUpdater.h"
|
||||
#include "Field.h"
|
||||
#include "Log.h"
|
||||
#include "QueryResult.h"
|
||||
#include "Tokenize.h"
|
||||
#include "Util.h"
|
||||
#include <boost/filesystem/operations.hpp>
|
||||
#include <fstream>
|
||||
#include <sstream>
|
||||
|
||||
using namespace boost::filesystem;
|
||||
|
||||
struct UpdateFetcher::DirectoryEntry
|
||||
{
|
||||
DirectoryEntry(Path const& path_, State state_) : path(path_), state(state_) { }
|
||||
|
||||
Path const path;
|
||||
State const state;
|
||||
};
|
||||
|
||||
UpdateFetcher::UpdateFetcher(Path const& sourceDirectory,
|
||||
std::function<void(std::string const&)> const& apply,
|
||||
std::function<void(Path const& path)> const& applyFile,
|
||||
std::function<QueryResult(std::string const&)> const& retrieve, std::string const& dbModuleName_) :
|
||||
_sourceDirectory(std::make_unique<Path>(sourceDirectory)), _apply(apply), _applyFile(applyFile),
|
||||
_retrieve(retrieve), _dbModuleName(dbModuleName_)
|
||||
{
|
||||
}
|
||||
|
||||
UpdateFetcher::~UpdateFetcher()
|
||||
{
|
||||
}
|
||||
|
||||
UpdateFetcher::LocaleFileStorage UpdateFetcher::GetFileList() const
|
||||
{
|
||||
LocaleFileStorage files;
|
||||
DirectoryStorage directories = ReceiveIncludedDirectories();
|
||||
for (auto const& entry : directories)
|
||||
FillFileListRecursively(entry.path, files, entry.state, 1);
|
||||
|
||||
return files;
|
||||
}
|
||||
|
||||
void UpdateFetcher::FillFileListRecursively(Path const& path, LocaleFileStorage& storage, State const state, uint32 const depth) const
|
||||
{
|
||||
static uint32 const MAX_DEPTH = 10;
|
||||
static directory_iterator const end;
|
||||
|
||||
for (directory_iterator itr(path); itr != end; ++itr)
|
||||
{
|
||||
if (is_directory(itr->path()))
|
||||
{
|
||||
if (depth < MAX_DEPTH)
|
||||
FillFileListRecursively(itr->path(), storage, state, depth + 1);
|
||||
}
|
||||
else if (itr->path().extension() == ".sql")
|
||||
{
|
||||
LOG_TRACE("sql.updates", "Added locale file \"%s\".", itr->path().filename().generic_string().c_str());
|
||||
|
||||
LocaleFileEntry const entry = { itr->path(), state };
|
||||
|
||||
// Check for doubled filenames
|
||||
// Because elements are only compared by their filenames, this is ok
|
||||
if (storage.find(entry) != storage.end())
|
||||
{
|
||||
LOG_FATAL("sql.updates", "Duplicate filename \"%s\" occurred. Because updates are ordered " \
|
||||
"by their filenames, every name needs to be unique!", itr->path().generic_string().c_str());
|
||||
|
||||
throw UpdateException("Updating failed, see the log for details.");
|
||||
}
|
||||
|
||||
storage.insert(entry);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
UpdateFetcher::DirectoryStorage UpdateFetcher::ReceiveIncludedDirectories() const
|
||||
{
|
||||
DirectoryStorage directories;
|
||||
|
||||
QueryResult const result = _retrieve("SELECT `path`, `state` FROM `updates_include`");
|
||||
if (!result)
|
||||
return directories;
|
||||
|
||||
do
|
||||
{
|
||||
Field* fields = result->Fetch();
|
||||
|
||||
std::string path = fields[0].GetString();
|
||||
if (path.substr(0, 1) == "$")
|
||||
path = _sourceDirectory->generic_string() + path.substr(1);
|
||||
|
||||
Path const p(path);
|
||||
|
||||
if (!is_directory(p))
|
||||
{
|
||||
LOG_WARN("sql.updates", "DBUpdater: Given update include directory \"%s\" does not exist, skipped!", p.generic_string().c_str());
|
||||
continue;
|
||||
}
|
||||
|
||||
DirectoryEntry const entry = { p, AppliedFileEntry::StateConvert(fields[1].GetString()) };
|
||||
directories.push_back(entry);
|
||||
|
||||
LOG_TRACE("sql.updates", "Added applied file \"%s\" from remote.", p.filename().generic_string().c_str());
|
||||
|
||||
} while (result->NextRow());
|
||||
|
||||
std::vector<std::string> moduleList;
|
||||
|
||||
auto const& _modulesTokens = Warhead::Tokenize(WH_MODULES_LIST, ',', true);
|
||||
for (auto const& itr : _modulesTokens)
|
||||
moduleList.push_back(std::string(itr));
|
||||
|
||||
for (auto const& itr : moduleList)
|
||||
{
|
||||
std::string path = _sourceDirectory->generic_string() + "/modules/" + itr + "/sql/" + _dbModuleName; // module/mod-name/sql/db_world
|
||||
|
||||
Path const p(path);
|
||||
if (!is_directory(p))
|
||||
continue;
|
||||
|
||||
DirectoryEntry const entry = { p, AppliedFileEntry::StateConvert("RELEASED") };
|
||||
directories.push_back(entry);
|
||||
|
||||
LOG_TRACE("sql.updates", "Added applied modules file \"%s\" from remote.", p.filename().generic_string().c_str());
|
||||
}
|
||||
|
||||
return directories;
|
||||
}
|
||||
|
||||
UpdateFetcher::AppliedFileStorage UpdateFetcher::ReceiveAppliedFiles() const
|
||||
{
|
||||
AppliedFileStorage map;
|
||||
|
||||
QueryResult result = _retrieve("SELECT `name`, `hash`, `state`, UNIX_TIMESTAMP(`timestamp`) FROM `updates` ORDER BY `name` ASC");
|
||||
if (!result)
|
||||
return map;
|
||||
|
||||
do
|
||||
{
|
||||
Field* fields = result->Fetch();
|
||||
|
||||
AppliedFileEntry const entry = { fields[0].GetString(), fields[1].GetString(),
|
||||
AppliedFileEntry::StateConvert(fields[2].GetString()), fields[3].GetUInt64()
|
||||
};
|
||||
|
||||
map.insert(std::make_pair(entry.name, entry));
|
||||
} while (result->NextRow());
|
||||
|
||||
return map;
|
||||
}
|
||||
|
||||
std::string UpdateFetcher::ReadSQLUpdate(Path const& file) const
|
||||
{
|
||||
std::ifstream in(file.c_str());
|
||||
if (!in.is_open())
|
||||
{
|
||||
LOG_FATAL("sql.updates", "Failed to open the sql update \"%s\" for reading! "
|
||||
"Stopping the server to keep the database integrity, "
|
||||
"try to identify and solve the issue or disable the database updater.",
|
||||
file.generic_string().c_str());
|
||||
|
||||
throw UpdateException("Opening the sql update failed!");
|
||||
}
|
||||
|
||||
auto update = [&in]
|
||||
{
|
||||
std::ostringstream ss;
|
||||
ss << in.rdbuf();
|
||||
return ss.str();
|
||||
}();
|
||||
|
||||
in.close();
|
||||
return update;
|
||||
}
|
||||
|
||||
UpdateResult UpdateFetcher::Update(bool const redundancyChecks,
|
||||
bool const allowRehash,
|
||||
bool const archivedRedundancy,
|
||||
int32 const cleanDeadReferencesMaxCount) const
|
||||
{
|
||||
LocaleFileStorage const available = GetFileList();
|
||||
AppliedFileStorage applied = ReceiveAppliedFiles();
|
||||
|
||||
size_t countRecentUpdates = 0;
|
||||
size_t countArchivedUpdates = 0;
|
||||
|
||||
// Count updates
|
||||
for (auto const& entry : applied)
|
||||
if (entry.second.state == RELEASED)
|
||||
++countRecentUpdates;
|
||||
else
|
||||
++countArchivedUpdates;
|
||||
|
||||
// Fill hash to name cache
|
||||
HashToFileNameStorage hashToName;
|
||||
for (auto entry : applied)
|
||||
hashToName.insert(std::make_pair(entry.second.hash, entry.first));
|
||||
|
||||
size_t importedUpdates = 0;
|
||||
|
||||
for (auto const& availableQuery : available)
|
||||
{
|
||||
LOG_DEBUG("sql.updates", "Checking update \"%s\"...", availableQuery.first.filename().generic_string().c_str());
|
||||
|
||||
AppliedFileStorage::const_iterator iter = applied.find(availableQuery.first.filename().string());
|
||||
if (iter != applied.end())
|
||||
{
|
||||
// If redundancy is disabled, skip it, because the update is already applied.
|
||||
if (!redundancyChecks)
|
||||
{
|
||||
LOG_DEBUG("sql.updates", ">> Update is already applied, skipping redundancy checks.");
|
||||
applied.erase(iter);
|
||||
continue;
|
||||
}
|
||||
|
||||
// If the update is in an archived directory and is marked as archived in our database, skip redundancy checks (archived updates never change).
|
||||
if (!archivedRedundancy && (iter->second.state == ARCHIVED) && (availableQuery.second == ARCHIVED))
|
||||
{
|
||||
LOG_DEBUG("sql.updates", ">> Update is archived and marked as archived in database, skipping redundancy checks.");
|
||||
applied.erase(iter);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
std::string const hash = ByteArrayToHexStr(Warhead::Crypto::SHA1::GetDigestOf(ReadSQLUpdate(availableQuery.first)));
|
||||
|
||||
UpdateMode mode = MODE_APPLY;
|
||||
|
||||
// Update is not in our applied list
|
||||
if (iter == applied.end())
|
||||
{
|
||||
// Catch renames (different filename, but same hash)
|
||||
HashToFileNameStorage::const_iterator const hashIter = hashToName.find(hash);
|
||||
if (hashIter != hashToName.end())
|
||||
{
|
||||
// Check if the original file was removed. If not, we've got a problem.
|
||||
LocaleFileStorage::const_iterator localeIter;
|
||||
// Push localeIter forward
|
||||
for (localeIter = available.begin(); (localeIter != available.end()) &&
|
||||
(localeIter->first.filename().string() != hashIter->second); ++localeIter);
|
||||
|
||||
// Conflict!
|
||||
if (localeIter != available.end())
|
||||
{
|
||||
LOG_WARN("sql.updates", ">> It seems like the update \"%s\" \'%s\' was renamed, but the old file is still there! " \
|
||||
"Treating it as a new file! (It is probably an unmodified copy of the file \"%s\")",
|
||||
availableQuery.first.filename().string().c_str(), hash.substr(0, 7).c_str(),
|
||||
localeIter->first.filename().string().c_str());
|
||||
}
|
||||
// It is safe to treat the file as renamed here
|
||||
else
|
||||
{
|
||||
LOG_INFO("sql.updates", ">> Renaming update \"%s\" to \"%s\" \'%s\'.",
|
||||
hashIter->second.c_str(), availableQuery.first.filename().string().c_str(), hash.substr(0, 7).c_str());
|
||||
|
||||
RenameEntry(hashIter->second, availableQuery.first.filename().string());
|
||||
applied.erase(hashIter->second);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
// Apply the update if it was never seen before.
|
||||
else
|
||||
{
|
||||
LOG_INFO("sql.updates", ">> Applying update \"%s\" \'%s\'...",
|
||||
availableQuery.first.filename().string().c_str(), hash.substr(0, 7).c_str());
|
||||
}
|
||||
}
|
||||
// Rehash the update entry if it exists in our database with an empty hash.
|
||||
else if (allowRehash && iter->second.hash.empty())
|
||||
{
|
||||
mode = MODE_REHASH;
|
||||
|
||||
LOG_INFO("sql.updates", ">> Re-hashing update \"%s\" \'%s\'...", availableQuery.first.filename().string().c_str(),
|
||||
hash.substr(0, 7).c_str());
|
||||
}
|
||||
else
|
||||
{
|
||||
// If the hash of the files differs from the one stored in our database, reapply the update (because it changed).
|
||||
if (iter->second.hash != hash)
|
||||
{
|
||||
LOG_INFO("sql.updates", ">> Reapplying update \"%s\" \'%s\' -> \'%s\' (it changed)...", availableQuery.first.filename().string().c_str(),
|
||||
iter->second.hash.substr(0, 7).c_str(), hash.substr(0, 7).c_str());
|
||||
}
|
||||
else
|
||||
{
|
||||
// If the file wasn't changed and just moved, update its state (if necessary).
|
||||
if (iter->second.state != availableQuery.second)
|
||||
{
|
||||
LOG_DEBUG("sql.updates", ">> Updating the state of \"%s\" to \'%s\'...",
|
||||
availableQuery.first.filename().string().c_str(), AppliedFileEntry::StateConvert(availableQuery.second).c_str());
|
||||
|
||||
UpdateState(availableQuery.first.filename().string(), availableQuery.second);
|
||||
}
|
||||
|
||||
LOG_DEBUG("sql.updates", ">> Update is already applied and matches the hash \'%s\'.", hash.substr(0, 7).c_str());
|
||||
|
||||
applied.erase(iter);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
uint32 speed = 0;
|
||||
AppliedFileEntry const file = { availableQuery.first.filename().string(), hash, availableQuery.second, 0 };
|
||||
|
||||
switch (mode)
|
||||
{
|
||||
case MODE_APPLY:
|
||||
speed = Apply(availableQuery.first);
|
||||
/* fallthrough */
|
||||
case MODE_REHASH:
|
||||
UpdateEntry(file, speed);
|
||||
break;
|
||||
}
|
||||
|
||||
if (iter != applied.end())
|
||||
applied.erase(iter);
|
||||
|
||||
if (mode == MODE_APPLY)
|
||||
++importedUpdates;
|
||||
}
|
||||
|
||||
// Cleanup up orphaned entries (if enabled)
|
||||
if (!applied.empty())
|
||||
{
|
||||
bool const doCleanup = (cleanDeadReferencesMaxCount < 0) || (applied.size() <= static_cast<size_t>(cleanDeadReferencesMaxCount));
|
||||
|
||||
for (auto const& entry : applied)
|
||||
{
|
||||
LOG_WARN("sql.updates", ">> The file \'%s\' was applied to the database, but is missing in" \
|
||||
" your update directory now!", entry.first.c_str());
|
||||
|
||||
if (doCleanup)
|
||||
LOG_INFO("sql.updates", "Deleting orphaned entry \'%s\'...", entry.first.c_str());
|
||||
}
|
||||
|
||||
if (doCleanup)
|
||||
CleanUp(applied);
|
||||
else
|
||||
{
|
||||
LOG_ERROR("sql.updates", "Cleanup is disabled! There were " SZFMTD " dirty files applied to your database, " \
|
||||
"but they are now missing in your source directory!", applied.size());
|
||||
}
|
||||
}
|
||||
|
||||
return UpdateResult(importedUpdates, countRecentUpdates, countArchivedUpdates);
|
||||
}
|
||||
|
||||
uint32 UpdateFetcher::Apply(Path const& path) const
|
||||
{
|
||||
using Time = std::chrono::high_resolution_clock;
|
||||
|
||||
// Benchmark query speed
|
||||
auto const begin = Time::now();
|
||||
|
||||
// Update database
|
||||
_applyFile(path);
|
||||
|
||||
// Return the time it took the query to apply
|
||||
return uint32(std::chrono::duration_cast<std::chrono::milliseconds>(Time::now() - begin).count());
|
||||
}
|
||||
|
||||
void UpdateFetcher::UpdateEntry(AppliedFileEntry const& entry, uint32 const speed) const
|
||||
{
|
||||
std::string const update = "REPLACE INTO `updates` (`name`, `hash`, `state`, `speed`) VALUES (\"" +
|
||||
entry.name + "\", \"" + entry.hash + "\", \'" + entry.GetStateAsString() + "\', " + std::to_string(speed) + ")";
|
||||
|
||||
// Update database
|
||||
_apply(update);
|
||||
}
|
||||
|
||||
void UpdateFetcher::RenameEntry(std::string const& from, std::string const& to) const
|
||||
{
|
||||
// Delete the target if it exists
|
||||
{
|
||||
std::string const update = "DELETE FROM `updates` WHERE `name`=\"" + to + "\"";
|
||||
|
||||
// Update database
|
||||
_apply(update);
|
||||
}
|
||||
|
||||
// Rename
|
||||
{
|
||||
std::string const update = "UPDATE `updates` SET `name`=\"" + to + "\" WHERE `name`=\"" + from + "\"";
|
||||
|
||||
// Update database
|
||||
_apply(update);
|
||||
}
|
||||
}
|
||||
|
||||
void UpdateFetcher::CleanUp(AppliedFileStorage const& storage) const
|
||||
{
|
||||
if (storage.empty())
|
||||
return;
|
||||
|
||||
std::stringstream update;
|
||||
size_t remaining = storage.size();
|
||||
|
||||
update << "DELETE FROM `updates` WHERE `name` IN(";
|
||||
|
||||
for (auto const& entry : storage)
|
||||
{
|
||||
update << "\"" << entry.first << "\"";
|
||||
if ((--remaining) > 0)
|
||||
update << ", ";
|
||||
}
|
||||
|
||||
update << ")";
|
||||
|
||||
// Update database
|
||||
_apply(update.str());
|
||||
}
|
||||
|
||||
void UpdateFetcher::UpdateState(std::string const& name, State const state) const
|
||||
{
|
||||
std::string const update = "UPDATE `updates` SET `state`=\'" + AppliedFileEntry::StateConvert(state) + "\' WHERE `name`=\"" + name + "\"";
|
||||
|
||||
// Update database
|
||||
_apply(update);
|
||||
}
|
||||
|
||||
bool UpdateFetcher::PathCompare::operator()(LocaleFileEntry const& left, LocaleFileEntry const& right) const
|
||||
{
|
||||
return left.first.filename().string() < right.first.filename().string();
|
||||
}
|
||||
131
src/server/database/Updater/UpdateFetcher.h
Normal file
131
src/server/database/Updater/UpdateFetcher.h
Normal file
@@ -0,0 +1,131 @@
|
||||
/*
|
||||
* Copyright (C) 2016+ AzerothCore <www.azerothcore.org>, released under GNU AGPL v3 license: https://github.com/azerothcore/azerothcore-wotlk/blob/master/LICENSE-AGPL3
|
||||
* Copyright (C) 2021+ WarheadCore <https://github.com/WarheadCore>
|
||||
*/
|
||||
|
||||
#ifndef UpdateFetcher_h__
|
||||
#define UpdateFetcher_h__
|
||||
|
||||
#include "DatabaseEnv.h"
|
||||
#include "Define.h"
|
||||
#include <set>
|
||||
#include <string>
|
||||
#include <unordered_map>
|
||||
#include <vector>
|
||||
|
||||
namespace boost::filesystem
|
||||
{
|
||||
class path;
|
||||
}
|
||||
|
||||
struct AC_DATABASE_API UpdateResult
|
||||
{
|
||||
UpdateResult()
|
||||
: updated(0), recent(0), archived(0) { }
|
||||
|
||||
UpdateResult(size_t const updated_, size_t const recent_, size_t const archived_)
|
||||
: updated(updated_), recent(recent_), archived(archived_) { }
|
||||
|
||||
size_t updated;
|
||||
size_t recent;
|
||||
size_t archived;
|
||||
};
|
||||
|
||||
class AC_DATABASE_API UpdateFetcher
|
||||
{
|
||||
typedef boost::filesystem::path Path;
|
||||
|
||||
public:
|
||||
UpdateFetcher(Path const& updateDirectory,
|
||||
std::function<void(std::string const&)> const& apply,
|
||||
std::function<void(Path const& path)> const& applyFile,
|
||||
std::function<QueryResult(std::string const&)> const& retrieve, std::string const& dbModuleName);
|
||||
~UpdateFetcher();
|
||||
|
||||
UpdateResult Update(bool const redundancyChecks, bool const allowRehash,
|
||||
bool const archivedRedundancy, int32 const cleanDeadReferencesMaxCount) const;
|
||||
|
||||
private:
|
||||
enum UpdateMode
|
||||
{
|
||||
MODE_APPLY,
|
||||
MODE_REHASH
|
||||
};
|
||||
|
||||
enum State
|
||||
{
|
||||
RELEASED,
|
||||
ARCHIVED
|
||||
};
|
||||
|
||||
struct AppliedFileEntry
|
||||
{
|
||||
AppliedFileEntry(std::string const& name_, std::string const& hash_, State state_, uint64 timestamp_)
|
||||
: name(name_), hash(hash_), state(state_), timestamp(timestamp_) { }
|
||||
|
||||
std::string const name;
|
||||
|
||||
std::string const hash;
|
||||
|
||||
State const state;
|
||||
|
||||
uint64 const timestamp;
|
||||
|
||||
static inline State StateConvert(std::string const& state)
|
||||
{
|
||||
return (state == "RELEASED") ? RELEASED : ARCHIVED;
|
||||
}
|
||||
|
||||
static inline std::string StateConvert(State const state)
|
||||
{
|
||||
return (state == RELEASED) ? "RELEASED" : "ARCHIVED";
|
||||
}
|
||||
|
||||
std::string GetStateAsString() const
|
||||
{
|
||||
return StateConvert(state);
|
||||
}
|
||||
};
|
||||
|
||||
struct DirectoryEntry;
|
||||
|
||||
typedef std::pair<Path, State> LocaleFileEntry;
|
||||
|
||||
struct PathCompare
|
||||
{
|
||||
bool operator()(LocaleFileEntry const& left, LocaleFileEntry const& right) const;
|
||||
};
|
||||
|
||||
typedef std::set<LocaleFileEntry, PathCompare> LocaleFileStorage;
|
||||
typedef std::unordered_map<std::string, std::string> HashToFileNameStorage;
|
||||
typedef std::unordered_map<std::string, AppliedFileEntry> AppliedFileStorage;
|
||||
typedef std::vector<UpdateFetcher::DirectoryEntry> DirectoryStorage;
|
||||
|
||||
LocaleFileStorage GetFileList() const;
|
||||
void FillFileListRecursively(Path const& path, LocaleFileStorage& storage,
|
||||
State const state, uint32 const depth) const;
|
||||
|
||||
DirectoryStorage ReceiveIncludedDirectories() const;
|
||||
AppliedFileStorage ReceiveAppliedFiles() const;
|
||||
|
||||
std::string ReadSQLUpdate(Path const& file) const;
|
||||
|
||||
uint32 Apply(Path const& path) const;
|
||||
|
||||
void UpdateEntry(AppliedFileEntry const& entry, uint32 const speed = 0) const;
|
||||
void RenameEntry(std::string const& from, std::string const& to) const;
|
||||
void CleanUp(AppliedFileStorage const& storage) const;
|
||||
|
||||
void UpdateState(std::string const& name, State const state) const;
|
||||
|
||||
std::unique_ptr<Path> const _sourceDirectory;
|
||||
|
||||
std::function<void(std::string const&)> const _apply;
|
||||
std::function<void(Path const& path)> const _applyFile;
|
||||
std::function<QueryResult(std::string const&)> const _retrieve;
|
||||
|
||||
// modules
|
||||
std::string const _dbModuleName;
|
||||
};
|
||||
|
||||
#endif // UpdateFetcher_h__
|
||||
Reference in New Issue
Block a user