diff --git a/.clang-format b/.clang-format new file mode 100644 index 0000000..fa684b2 --- /dev/null +++ b/.clang-format @@ -0,0 +1,26 @@ +--- +# We'll use defaults from the LLVM style, but with 4 columns indentation. +BasedOnStyle: LLVM +IndentWidth: 4 +--- +Language: Cpp +# Force pointers to the type for C++. +DerivePointerAlignment: false +PointerAlignment: Left +AlignConsecutiveAssignments: true +AllowShortFunctionsOnASingleLine: Inline +AllowShortIfStatementsOnASingleLine: Never +AllowShortLambdasOnASingleLine: Empty +AlwaysBreakTemplateDeclarations: Yes +AccessModifierOffset: -4 +AlignTrailingComments: true +SpacesBeforeTrailingComments: 2 +NamespaceIndentation: All +MaxEmptyLinesToKeep: 1 +BreakBeforeBraces: Stroustrup +ColumnLimit: 88 +IncludeBlocks: Preserve +IncludeCategories: + - Regex: '^"pch.h"$' + Priority: -1 + SortPriority: -1 diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs new file mode 100644 index 0000000..346c2d2 --- /dev/null +++ b/.git-blame-ignore-revs @@ -0,0 +1 @@ +d0913d07d33929d7b753a3a09a0dac70e5befbc1 diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..f869712 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,7 @@ +# Set the default behavior, in case people don't have core.autocrlf set. +* text=auto + +# Explicitly declare text files you want to always be normalized and converted +# to native line endings on checkout. +*.cpp text eol=crlf +*.h text eol=crlf diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..935b9e1 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,16 @@ +name: Build Mob + +on: + push: + branches: master + pull_request: + types: [opened, synchronize, reopened] + +jobs: + build: + runs-on: windows-2022 + steps: + - uses: actions/checkout@v4 + - name: Build Mob + shell: pwsh + run: ./bootstrap.ps1 -Verbose diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml new file mode 100644 index 0000000..7a3ae42 --- /dev/null +++ b/.github/workflows/linting.yml @@ -0,0 +1,18 @@ +name: Lint Mob + +on: + push: + pull_request: + types: [opened, synchronize, reopened] + +jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Run clang-format + uses: jidicula/clang-format-action@v4.11.0 + with: + clang-format-version: "16" + check-path: "." + exclude-regex: "third-party" diff --git a/src/cmd/build.cpp b/src/cmd/build.cpp index 684abf0..8511af3 100644 --- a/src/cmd/build.cpp +++ b/src/cmd/build.cpp @@ -1,200 +1,195 @@ #include "pch.h" -#include "commands.h" -#include "../core/ini.h" #include "../core/conf.h" #include "../core/context.h" +#include "../core/ini.h" #include "../core/op.h" #include "../tasks/task_manager.h" +#include "commands.h" -namespace mob -{ - -build_command::build_command() - : command(requires_options | handle_sigint) -{ -} - -command::meta_t build_command::meta() const -{ - return - { - "build", - "builds tasks" - }; -} - -clipp::group build_command::do_group() -{ - return - (clipp::command("build")).set(picked_), - - (clipp::option("-h", "--help") >> help_) - % ("shows this message"), - - (clipp::option("-g", "--redownload") >> redownload_) - % "redownloads archives, see --reextract", - - (clipp::option("-e", "--reextract") >> reextract_) - % "deletes source directories and re-extracts archives", - - (clipp::option("-c", "--reconfigure") >> reconfigure_) - % "reconfigures the task by running cmake, configure scripts, " - "etc.; some tasks might have to delete the whole source " - "directory", - - (clipp::option("-b", "--rebuild") >> rebuild_) - % "cleans and rebuilds projects; some tasks might have to " - "delete the whole source directory", - - (clipp::option("-n", "--new") >> new_) - % "deletes everything and starts from scratch", - - ( - clipp::option("--clean-task").call([&]{ clean_ = true; }) | - clipp::option("--no-clean-task").call([&]{ clean_ = false; }) - ) % "sets whether tasks are cleaned", - - ( - clipp::option("--fetch-task").call([&]{ fetch_ = true; }) | - clipp::option("--no-fetch-task").call([&]{ fetch_ = false; }) - ) % "sets whether tasks are fetched", - - ( - clipp::option("--build-task").call([&]{ build_ = true; }) | - clipp::option("--no-build-task").call([&]{ build_ = false; }) - ) % "sets whether tasks are built", - - ( - clipp::option("--pull").call([&]{ nopull_ = false; }) | - clipp::option("--no-pull").call([&]{ nopull_ = true; }) - ) % "whether to pull repos that are already cloned; global override", - - ( - clipp::option("--revert-ts").call([&]{ revert_ts_ = true; }) | - clipp::option("--no-revert-ts").call([&]{ revert_ts_ = false; }) - ) % "whether to revert all the .ts files in a repo before pulling to " - "avoid merge errors; global override", - - (clipp::option("--ignore-uncommitted-changes") >> ignore_uncommitted_) - % "when --reextract is given, directories controlled by git will " - "be deleted even if they contain uncommitted changes", - - (clipp::option("--keep-msbuild") >> keep_msbuild_) - % "don't terminate msbuild.exe instances after building", - - (clipp::opt_values( - clipp::match::prefix_not("-"), "task", tasks_)) - % "tasks to run; specify 'super' to only build modorganizer " - "projects"; -} - -void build_command::convert_cl_to_conf() -{ - command::convert_cl_to_conf(); - - if (redownload_ || new_) - common.options.push_back("global/redownload=true"); - - if (reextract_ || new_) - common.options.push_back("global/reextract=true"); - - if (reconfigure_ || new_) - common.options.push_back("global/reconfigure=true"); - - if (rebuild_ || new_) - common.options.push_back("global/rebuild=true"); - - if (ignore_uncommitted_) - common.options.push_back("global/ignore_uncommitted=true"); - - if (clean_) - { - if (*clean_) - common.options.push_back("global/clean_task=true"); - else - common.options.push_back("global/clean_task=false"); - } - - if (fetch_) - { - if (*fetch_) - common.options.push_back("global/fetch_task=true"); - else - common.options.push_back("global/fetch_task=false"); - } - - if (build_) - { - if (*build_) - common.options.push_back("global/build_task=true"); - else - common.options.push_back("global/build_task=false"); - } - - if (nopull_) - { - if (*nopull_) - common.options.push_back("_override:task/no_pull=true"); - else - common.options.push_back("_override:task/no_pull=false"); - } - - if (revert_ts_) - { - if (*revert_ts_) - common.options.push_back("_override:task/revert_ts=true"); - else - common.options.push_back("_override:task/revert_ts=false"); - } - - if (!tasks_.empty()) - set_task_enabled_flags(tasks_); -} - -int build_command::do_run() -{ - try - { - create_prefix_ini(); - - task_manager::instance().run_all(); - - if (!keep_msbuild_) - terminate_msbuild(); - - mob::gcx().info(mob::context::generic, "mob done"); - return 0; - } - catch(bailed&) - { - gcx().error(context::generic, "bailing out"); - return 1; - } -} - -void build_command::create_prefix_ini() -{ - const auto prefix = conf().path().prefix(); - - // creating prefix - if (!exists(prefix)) - op::create_directories(gcx(), prefix); - - const auto ini = prefix / default_ini_filename(); - if (!exists(ini)) - { - std::ofstream(ini) - << "[paths]\n" - << "prefix = .\n"; - } -} - -void build_command::terminate_msbuild() -{ - if (conf().global().dry()) - return; - - system("taskkill /im msbuild.exe /f > NUL 2>&1"); -} - -} // namespace +namespace mob { + + build_command::build_command() : command(requires_options | handle_sigint) {} + + command::meta_t build_command::meta() const + { + return {"build", "builds tasks"}; + } + + clipp::group build_command::do_group() + { + return (clipp::command("build")).set(picked_), + + (clipp::option("-h", "--help") >> help_) % ("shows this message"), + + (clipp::option("-g", "--redownload") >> redownload_) % + "redownloads archives, see --reextract", + + (clipp::option("-e", "--reextract") >> reextract_) % + "deletes source directories and re-extracts archives", + + (clipp::option("-c", "--reconfigure") >> reconfigure_) % + "reconfigures the task by running cmake, configure scripts, " + "etc.; some tasks might have to delete the whole source " + "directory", + + (clipp::option("-b", "--rebuild") >> rebuild_) % + "cleans and rebuilds projects; some tasks might have to " + "delete the whole source directory", + + (clipp::option("-n", "--new") >> new_) % + "deletes everything and starts from scratch", + + (clipp::option("--clean-task").call([&] { + clean_ = true; + }) | + clipp::option("--no-clean-task").call([&] { + clean_ = false; + })) % + "sets whether tasks are cleaned", + + (clipp::option("--fetch-task").call([&] { + fetch_ = true; + }) | + clipp::option("--no-fetch-task").call([&] { + fetch_ = false; + })) % + "sets whether tasks are fetched", + + (clipp::option("--build-task").call([&] { + build_ = true; + }) | + clipp::option("--no-build-task").call([&] { + build_ = false; + })) % + "sets whether tasks are built", + + (clipp::option("--pull").call([&] { + nopull_ = false; + }) | + clipp::option("--no-pull").call([&] { + nopull_ = true; + })) % + "whether to pull repos that are already cloned; global override", + + (clipp::option("--revert-ts").call([&] { + revert_ts_ = true; + }) | + clipp::option("--no-revert-ts").call([&] { + revert_ts_ = false; + })) % + "whether to revert all the .ts files in a repo before pulling to " + "avoid merge errors; global override", + + (clipp::option("--ignore-uncommitted-changes") >> ignore_uncommitted_) % + "when --reextract is given, directories controlled by git will " + "be deleted even if they contain uncommitted changes", + + (clipp::option("--keep-msbuild") >> keep_msbuild_) % + "don't terminate msbuild.exe instances after building", + + (clipp::opt_values(clipp::match::prefix_not("-"), "task", tasks_)) % + "tasks to run; specify 'super' to only build modorganizer " + "projects"; + } + + void build_command::convert_cl_to_conf() + { + command::convert_cl_to_conf(); + + if (redownload_ || new_) + common.options.push_back("global/redownload=true"); + + if (reextract_ || new_) + common.options.push_back("global/reextract=true"); + + if (reconfigure_ || new_) + common.options.push_back("global/reconfigure=true"); + + if (rebuild_ || new_) + common.options.push_back("global/rebuild=true"); + + if (ignore_uncommitted_) + common.options.push_back("global/ignore_uncommitted=true"); + + if (clean_) { + if (*clean_) + common.options.push_back("global/clean_task=true"); + else + common.options.push_back("global/clean_task=false"); + } + + if (fetch_) { + if (*fetch_) + common.options.push_back("global/fetch_task=true"); + else + common.options.push_back("global/fetch_task=false"); + } + + if (build_) { + if (*build_) + common.options.push_back("global/build_task=true"); + else + common.options.push_back("global/build_task=false"); + } + + if (nopull_) { + if (*nopull_) + common.options.push_back("_override:task/no_pull=true"); + else + common.options.push_back("_override:task/no_pull=false"); + } + + if (revert_ts_) { + if (*revert_ts_) + common.options.push_back("_override:task/revert_ts=true"); + else + common.options.push_back("_override:task/revert_ts=false"); + } + + if (!tasks_.empty()) + set_task_enabled_flags(tasks_); + } + + int build_command::do_run() + { + try { + create_prefix_ini(); + + task_manager::instance().run_all(); + + if (!keep_msbuild_) + terminate_msbuild(); + + mob::gcx().info(mob::context::generic, "mob done"); + return 0; + } + catch (bailed&) { + gcx().error(context::generic, "bailing out"); + return 1; + } + } + + void build_command::create_prefix_ini() + { + const auto prefix = conf().path().prefix(); + + // creating prefix + if (!exists(prefix)) + op::create_directories(gcx(), prefix); + + const auto ini = prefix / default_ini_filename(); + if (!exists(ini)) { + std::ofstream(ini) << "[paths]\n" + << "prefix = .\n"; + } + } + + void build_command::terminate_msbuild() + { + if (conf().global().dry()) + return; + + system("taskkill /im msbuild.exe /f > NUL 2>&1"); + } + +} // namespace mob diff --git a/src/cmd/cmake.cpp b/src/cmd/cmake.cpp index eea491e..8e63d76 100644 --- a/src/cmd/cmake.cpp +++ b/src/cmd/cmake.cpp @@ -1,83 +1,65 @@ #include "pch.h" -#include "commands.h" #include "../tasks/tasks.h" +#include "commands.h" + +namespace mob { + + cmake_command::cmake_command() : command(requires_options) {} + + command::meta_t cmake_command::meta() const + { + return {"cmake", "runs cmake in a directory"}; + } + + clipp::group cmake_command::do_group() + { + return clipp::group( + clipp::command("cmake").set(picked_), + + (clipp::option("-h", "--help") >> help_) % "shows this message", + + (clipp::option("-G", "--generator") & clipp::value("GEN") >> gen_) % + ("sets the -G option for cmake [default: VS]"), + + (clipp::option("-c", "--cmd") & clipp::value("CMD") >> cmd_) % + "overrides the cmake command line [default: \"..\"]", + + (clipp::option("--x64").set(x64_, true) | + clipp::option("--x86").set(x64_, false)) % + "whether to use the x64 or x86 vcvars; if -G is not set, " + "whether to pass \"-A Win32\" or \"-A x64\" for the default " + "VS generator [default: x64]", + + (clipp::option("--install-prefix") & clipp::value("PATH") >> prefix_) % + "sets CMAKE_INSTALL_PREFIX [default: empty]", + + (clipp::value("PATH") >> path_) % "path from which to run `cmake`"); + } + + int cmake_command::do_run() + { + auto t = tasks::modorganizer::create_cmake_tool(fs::path(utf8_to_utf16(path_))); + + t.generator(gen_); + t.cmd(cmd_); + t.prefix(prefix_); + t.output(path_); + + if (!x64_) + t.architecture(arch::x86); + + // copy the global context, the tool will modify it + context cxcopy(gcx()); + + t.run(cxcopy); + + return 0; + } + + std::string cmake_command::do_doc() + { + return "Runs `cmake ..` in the given directory with the same command line\n" + "as the one used for modorganizer projects."; + } -namespace mob -{ - -cmake_command::cmake_command() - : command(requires_options) -{ -} - -command::meta_t cmake_command::meta() const -{ - return - { - "cmake", - "runs cmake in a directory" - }; -} - -clipp::group cmake_command::do_group() -{ - return clipp::group( - clipp::command("cmake").set(picked_), - - (clipp::option("-h", "--help") >> help_) - % "shows this message", - - (clipp::option("-G", "--generator") - & clipp::value("GEN") >> gen_) - % ("sets the -G option for cmake [default: VS]"), - - (clipp::option("-c", "--cmd") - & clipp::value("CMD") >> cmd_) - % "overrides the cmake command line [default: \"..\"]", - - ( - clipp::option("--x64").set(x64_, true) | - clipp::option("--x86").set(x64_, false) - ) - % "whether to use the x64 or x86 vcvars; if -G is not set, " - "whether to pass \"-A Win32\" or \"-A x64\" for the default " - "VS generator [default: x64]", - - (clipp::option("--install-prefix") - & clipp::value("PATH") >> prefix_) - % "sets CMAKE_INSTALL_PREFIX [default: empty]", - - (clipp::value("PATH") >> path_) - % "path from which to run `cmake`" - ); -} - -int cmake_command::do_run() -{ - auto t = tasks::modorganizer::create_cmake_tool( - fs::path(utf8_to_utf16(path_))); - - t.generator(gen_); - t.cmd(cmd_); - t.prefix(prefix_); - t.output(path_); - - if (!x64_) - t.architecture(arch::x86); - - // copy the global context, the tool will modify it - context cxcopy(gcx()); - - t.run(cxcopy); - - return 0; -} - -std::string cmake_command::do_doc() -{ - return - "Runs `cmake ..` in the given directory with the same command line\n" - "as the one used for modorganizer projects."; -} - -} // namespace +} // namespace mob diff --git a/src/cmd/commands.cpp b/src/cmd/commands.cpp index 4b543d8..303b797 100644 --- a/src/cmd/commands.cpp +++ b/src/cmd/commands.cpp @@ -1,380 +1,327 @@ #include "pch.h" #include "commands.h" -#include "../utility.h" -#include "../net.h" #include "../core/conf.h" #include "../core/ini.h" +#include "../net.h" #include "../tasks/task_manager.h" #include "../tools/tools.h" +#include "../utility.h" #include "../utility/threading.h" -namespace mob -{ - -BOOL WINAPI signal_handler(DWORD) noexcept -{ - // don't use u8cout, this would lock the global mutex, but the handler - // can be called while stuff is being output and the mutex is locked - std::wcout << L"sigint, interrupting...\n"; - task_manager::instance().interrupt_all(); - return TRUE; -} - -void set_sigint_handler() -{ - ::SetConsoleCtrlHandler(mob::signal_handler, TRUE); -} - - -void help(const clipp::group& g, const std::string& more) -{ - auto usage_df = clipp::doc_formatting() - .first_column(4) - .doc_column(30); - - auto options_df = clipp::doc_formatting() - .first_column(4) - .doc_column(30); - - u8cout - << "Usage:\n" << clipp::usage_lines(g, "mob", usage_df) - << "\n\n" - << "Options:\n" - << clipp::documentation(g, options_df) - << "\n\n" - << "To use global options with command options, make sure command \n" - << "options are together, with no global options in the middle.\n"; - - if (!more.empty()) - u8cout << "\n" << more << "\n"; -} - - -command::common_options command::common; - -command::command(flags f) - : picked_(false), help_(false), flags_(f), code_() -{ -} - -clipp::group command::common_options_group() -{ - auto& o = common; - const auto master = default_ini_filename(); - - return - (clipp::repeatable(clipp::option("-i", "--ini") - & clipp::value("FILE") >> o.inis)) - % "path to the ini file", - - (clipp::option("--dry") >> o.dry) - % "simulates filesystem operations", - - (clipp::option("-l", "--log-level") - & clipp::value("LEVEL") >> o.output_log_level) - % "0 is silent, 6 is max", - - (clipp::option("--file-log-level") - & clipp::value("LEVEL") >> o.file_log_level) - % "overrides --log-level for the log file", - - (clipp::option("--log-file") - & clipp::value("FILE") >> o.log_file) - % "path to log file", - - (clipp::option("-d", "--destination") - & clipp::value("DIR") >> o.prefix) - % ("base output directory, will contain build/, install/, etc."), - - (clipp::repeatable(clipp::option("-s", "--set") - & clipp::value("OPTION", o.options))) - % "sets an option, such as 'versions/openssl=1.2'", - - (clipp::option("--no-default-inis") >> o.no_default_inis) - % "disables auto loading of ini files, only uses --ini; the first " - "--ini must be the master ini file"; -} - -void command::force_exit_code(int code) -{ - code_ = code; -} - -void command::force_help() -{ - help_ = true; -} - -bool command::picked() const -{ - return picked_; -} - -clipp::group command::group() -{ - return do_group(); -} - -void command::convert_cl_to_conf() -{ - auto& o = common; - - if (o.file_log_level == -1) - o.file_log_level = o.output_log_level; - - if (o.output_log_level >= 0) - { - o.options.push_back( - "global/output_log_level=" + - std::to_string(o.output_log_level)); - } - - if (o.file_log_level > 0) - { - o.options.push_back( - "global/file_log_level=" + - std::to_string(o.file_log_level)); - } - - if (!o.log_file.empty()) - o.options.push_back("global/log_file=" + o.log_file); - - if (o.dry) - o.options.push_back("global/dry=true"); - - if (!o.prefix.empty()) - o.options.push_back("paths/prefix=" + o.prefix); -} - -int command::gather_inis(bool verbose) -{ - auto& o = common; - - if (o.no_default_inis && o.inis.empty()) - { - u8cerr - << "--no-default-inis requires at least one --ini for the " - << "master ini file\n"; - - return 1; - } - - try - { - inis_ = find_inis(!o.no_default_inis, o.inis, verbose); - return 0; - } - catch(bailed&) - { - return 1; - } -} - -void command::set_task_enabled_flags(const std::vector& names) -{ - common.options.push_back("task/enabled=false"); - - bool failed = false; - - for (auto&& pattern : names) - common.options.push_back(pattern + ":task/enabled=true"); - - if (failed) - throw bailed(); -} - -int command::prepare_options(bool verbose) -{ - convert_cl_to_conf(); - return gather_inis(verbose); -} - -int command::run() -{ - if (help_) - { - help(group(), do_doc()); - return 0; - } - - if (flags_ & requires_options) - { - const auto r = load_options(); - if (r != 0) - return r; - } - - if (flags_ & handle_sigint) - set_sigint_handler(); - - const auto r = do_run(); - - if (code_) - return *code_; - - return r; -} - -int command::load_options() -{ - const int r = prepare_options(false); - if (r != 0) - return r; - - init_options(inis_, common.options); - - for (auto&& line : format_options()) - gcx().trace(context::conf, "{}", line); - - if (!verify_options()) - return 1; - - return 0; -} - - -command::meta_t version_command::meta() const -{ - return - { - "version", - "shows the version" - }; -} - -clipp::group version_command::do_group() -{ - return clipp::group( - clipp::command("version", "-v", "--version").set(picked_)); -} - -int version_command::do_run() -{ - u8cout << mob_version() << "\n"; - return 0; -} - - -command::meta_t help_command::meta() const -{ - return - { - "help", - "shows this message" - }; -} - -void help_command::set_commands(const std::vector>& v) -{ - std::vector> s; - - for (auto&& c : v) - s.push_back({c->meta().name, c->meta().description}); - - commands_ = table(s, 4, 3); -} - -clipp::group help_command::do_group() -{ - return clipp::group( - clipp::command("-h", "--help").set(picked_)); -} - -int help_command::do_run() -{ -#pragma warning(suppress: 4548) - auto doc = (command::common_options_group(), (clipp::value("command"))); - - const auto master = default_ini_filename(); - - help(doc, - "Commands:\n" - + commands_ + - "\n\n" - "Invoking `mob -d some/prefix build` builds everything. Do \n" - "`mob build ...` to build specific tasks. See\n" - "`mob command --help` for more information about a command.\n" - "\n" - "INI files\n" - "\n" - "By default, mob will look for a master INI `" + master + "` in the \n" - "root directory (typically where mob.exe resides). Once mob has\n" - "found the master INI, it will look for the same filename in the\n" - "current directory, if different from the root. If found, both will\n" - "be loaded, but the one in the current directory will override the\n" - "the other. Additional INIs can be specified with --ini, those will\n" - "be loaded after the two mentioned above. Use --no-default-inis to\n" - "disable auto detection and only use --ini."); - - return 0; -} - - -options_command::options_command() - : command(requires_options) -{ -} - -command::meta_t options_command::meta() const -{ - return - { - "options", - "lists all options and their values from the inis" - }; -} - -clipp::group options_command::do_group() -{ - return clipp::group( - clipp::command("options").set(picked_), - - (clipp::option("-h", "--help") >> help_) - % ("shows this message") - ); -} - -int options_command::do_run() -{ - for (auto&& line : format_options()) - u8cout << line << "\n"; - - return 0; -} - -std::string options_command::do_doc() -{ - return "Lists the final value of all options found by loading the INIs."; -} - - -command::meta_t inis_command::meta() const -{ - return - { - "inis", - "lists the INIs used by mob" - }; -} - -clipp::group inis_command::do_group() -{ - return clipp::group( - clipp::command("inis").set(picked_), - - (clipp::option("-h", "--help") >> help_) - % ("shows this message") - ); -} - -int inis_command::do_run() -{ - return prepare_options(true); -} - -std::string inis_command::do_doc() -{ - return "Shows which INIs are found."; -} - -} // namespace +namespace mob { + + BOOL WINAPI signal_handler(DWORD) noexcept + { + // don't use u8cout, this would lock the global mutex, but the handler + // can be called while stuff is being output and the mutex is locked + std::wcout << L"sigint, interrupting...\n"; + task_manager::instance().interrupt_all(); + return TRUE; + } + + void set_sigint_handler() + { + ::SetConsoleCtrlHandler(mob::signal_handler, TRUE); + } + + void help(const clipp::group& g, const std::string& more) + { + auto usage_df = clipp::doc_formatting().first_column(4).doc_column(30); + + auto options_df = clipp::doc_formatting().first_column(4).doc_column(30); + + u8cout << "Usage:\n" + << clipp::usage_lines(g, "mob", usage_df) << "\n\n" + << "Options:\n" + << clipp::documentation(g, options_df) << "\n\n" + << "To use global options with command options, make sure command \n" + << "options are together, with no global options in the middle.\n"; + + if (!more.empty()) + u8cout << "\n" << more << "\n"; + } + + command::common_options command::common; + + command::command(flags f) : picked_(false), help_(false), flags_(f), code_() {} + + clipp::group command::common_options_group() + { + auto& o = common; + const auto master = default_ini_filename(); + + return (clipp::repeatable(clipp::option("-i", "--ini") & + clipp::value("FILE") >> o.inis)) % + "path to the ini file", + + (clipp::option("--dry") >> o.dry) % "simulates filesystem operations", + + (clipp::option("-l", "--log-level") & + clipp::value("LEVEL") >> o.output_log_level) % + "0 is silent, 6 is max", + + (clipp::option("--file-log-level") & + clipp::value("LEVEL") >> o.file_log_level) % + "overrides --log-level for the log file", + + (clipp::option("--log-file") & clipp::value("FILE") >> o.log_file) % + "path to log file", + + (clipp::option("-d", "--destination") & + clipp::value("DIR") >> o.prefix) % + ("base output directory, will contain build/, install/, etc."), + + (clipp::repeatable(clipp::option("-s", "--set") & + clipp::value("OPTION", o.options))) % + "sets an option, such as 'versions/openssl=1.2'", + + (clipp::option("--no-default-inis") >> o.no_default_inis) % + "disables auto loading of ini files, only uses --ini; the first " + "--ini must be the master ini file"; + } + + void command::force_exit_code(int code) + { + code_ = code; + } + + void command::force_help() + { + help_ = true; + } + + bool command::picked() const + { + return picked_; + } + + clipp::group command::group() + { + return do_group(); + } + + void command::convert_cl_to_conf() + { + auto& o = common; + + if (o.file_log_level == -1) + o.file_log_level = o.output_log_level; + + if (o.output_log_level >= 0) { + o.options.push_back("global/output_log_level=" + + std::to_string(o.output_log_level)); + } + + if (o.file_log_level > 0) { + o.options.push_back("global/file_log_level=" + + std::to_string(o.file_log_level)); + } + + if (!o.log_file.empty()) + o.options.push_back("global/log_file=" + o.log_file); + + if (o.dry) + o.options.push_back("global/dry=true"); + + if (!o.prefix.empty()) + o.options.push_back("paths/prefix=" + o.prefix); + } + + int command::gather_inis(bool verbose) + { + auto& o = common; + + if (o.no_default_inis && o.inis.empty()) { + u8cerr << "--no-default-inis requires at least one --ini for the " + << "master ini file\n"; + + return 1; + } + + try { + inis_ = find_inis(!o.no_default_inis, o.inis, verbose); + return 0; + } + catch (bailed&) { + return 1; + } + } + + void command::set_task_enabled_flags(const std::vector& names) + { + common.options.push_back("task/enabled=false"); + + bool failed = false; + + for (auto&& pattern : names) + common.options.push_back(pattern + ":task/enabled=true"); + + if (failed) + throw bailed(); + } + + int command::prepare_options(bool verbose) + { + convert_cl_to_conf(); + return gather_inis(verbose); + } + + int command::run() + { + if (help_) { + help(group(), do_doc()); + return 0; + } + + if (flags_ & requires_options) { + const auto r = load_options(); + if (r != 0) + return r; + } + + if (flags_ & handle_sigint) + set_sigint_handler(); + + const auto r = do_run(); + + if (code_) + return *code_; + + return r; + } + + int command::load_options() + { + const int r = prepare_options(false); + if (r != 0) + return r; + + init_options(inis_, common.options); + + for (auto&& line : format_options()) + gcx().trace(context::conf, "{}", line); + + if (!verify_options()) + return 1; + + return 0; + } + + command::meta_t version_command::meta() const + { + return {"version", "shows the version"}; + } + + clipp::group version_command::do_group() + { + return clipp::group(clipp::command("version", "-v", "--version").set(picked_)); + } + + int version_command::do_run() + { + u8cout << mob_version() << "\n"; + return 0; + } + + command::meta_t help_command::meta() const + { + return {"help", "shows this message"}; + } + + void help_command::set_commands(const std::vector>& v) + { + std::vector> s; + + for (auto&& c : v) + s.push_back({c->meta().name, c->meta().description}); + + commands_ = table(s, 4, 3); + } + + clipp::group help_command::do_group() + { + return clipp::group(clipp::command("-h", "--help").set(picked_)); + } + + int help_command::do_run() + { +#pragma warning(suppress : 4548) + auto doc = (command::common_options_group(), (clipp::value("command"))); + + const auto master = default_ini_filename(); + + help(doc, + "Commands:\n" + commands_ + + "\n\n" + "Invoking `mob -d some/prefix build` builds everything. Do \n" + "`mob build ...` to build specific tasks. See\n" + "`mob command --help` for more information about a command.\n" + "\n" + "INI files\n" + "\n" + "By default, mob will look for a master INI `" + + master + + "` in the \n" + "root directory (typically where mob.exe resides). Once mob has\n" + "found the master INI, it will look for the same filename in the\n" + "current directory, if different from the root. If found, both will\n" + "be loaded, but the one in the current directory will override the\n" + "the other. Additional INIs can be specified with --ini, those will\n" + "be loaded after the two mentioned above. Use --no-default-inis to\n" + "disable auto detection and only use --ini."); + + return 0; + } + + options_command::options_command() : command(requires_options) {} + + command::meta_t options_command::meta() const + { + return {"options", "lists all options and their values from the inis"}; + } + + clipp::group options_command::do_group() + { + return clipp::group(clipp::command("options").set(picked_), + + (clipp::option("-h", "--help") >> help_) % + ("shows this message")); + } + + int options_command::do_run() + { + for (auto&& line : format_options()) + u8cout << line << "\n"; + + return 0; + } + + std::string options_command::do_doc() + { + return "Lists the final value of all options found by loading the INIs."; + } + + command::meta_t inis_command::meta() const + { + return {"inis", "lists the INIs used by mob"}; + } + + clipp::group inis_command::do_group() + { + return clipp::group(clipp::command("inis").set(picked_), + + (clipp::option("-h", "--help") >> help_) % + ("shows this message")); + } + + int inis_command::do_run() + { + return prepare_options(true); + } + + std::string inis_command::do_doc() + { + return "Shows which INIs are found."; + } + +} // namespace mob diff --git a/src/cmd/commands.h b/src/cmd/commands.h index 7efa2de..59d975e 100644 --- a/src/cmd/commands.h +++ b/src/cmd/commands.h @@ -2,484 +2,428 @@ #include "../utility/enum.h" -namespace mob::tasks -{ - class modorganizer; +namespace mob::tasks { + class modorganizer; } -namespace mob -{ - -class task; -class url; - -// base class for all commands -// -class command -{ -public: - // values of options available for all commands - // - struct common_options - { - bool dry = false; - int output_log_level = -1; - int file_log_level = -1; - std::string log_file; - std::vector options; - std::vector inis; - bool no_default_inis = false; - bool dump_inis = false; - std::string prefix; - }; - - // returned by meta() by each command - // - struct meta_t - { - std::string name, description; - }; - - - static common_options common; - static clipp::group common_options_group(); - - virtual ~command() = default; - - // overrides the exit code returned by do_run() - // - void force_exit_code(int code); - - // forces this command to show help in run() as if --help had been given - // - void force_help(); - - // whether this command was entered by the user - // - bool picked() const; - - // command line options for this command - // - clipp::group group(); - - // executes this command - // - int run(); - - - // returns meta information about this command - // - virtual meta_t meta() const = 0; - -protected: - // passed by derived classes in the constructor - // - enum flags - { - noflags = 0x00, - - // this command needs the ini loaded before running - requires_options = 0x01, - - // this command does not handle sigint by itself, run() will hook it - handle_sigint = 0x02 - }; - - MOB_ENUM_FRIEND_OPERATORS(flags); - - - // set to true when this command is entered by the user - bool picked_; - - // set to true with --help or by force_help() - bool help_; - - - command(flags f=noflags); - - // some options have a unique version on the command line because they're - // used often, such as --dry or -l; those are converted to their ini - // equivalent here (`-l 3` becomes `global/output_log_level=3` in - // common.options) - // - virtual void convert_cl_to_conf(); - - // disables all tasks by setting task/enabled=false, but sets - // name:task/enabled=true for the given tasks - // - void set_task_enabled_flags(const std::vector& tasks); - - // calls prepare_options() and loads inis - // - int load_options(); - - // calls convert_cl_to_conf() and populates inis_ - // - int prepare_options(bool verbose); - - // called by group(), returns the clipp group for this command - // - virtual clipp::group do_group() = 0; - - // called by run(), executes this command - // - virtual int do_run() = 0; - - // called by run() when --help is given; should return additional text that - // is output at the end of the usage info - // - virtual std::string do_doc() { return {}; } - -private: - // flags requested by this command - flags flags_; - - // exit code set by force_exit_code() - std::optional code_; - - // list of inis found by gather_inis() - std::vector inis_; - - - // finds all the inis - // - int gather_inis(bool verbose); -}; - - -// displays mob's version -// -class version_command : public command -{ -public: - meta_t meta() const override; - -protected: - clipp::group do_group() override; - int do_run() override; -}; - - -// displays the usage, list of commands and some additional text -// -class help_command : public command -{ -public: - meta_t meta() const override; - void set_commands(const std::vector>& v); - -protected: - clipp::group do_group() override; - int do_run() override; - -private: - std::string commands_; -}; - - -// lists all options and their values -// -class options_command : public command -{ -public: - options_command(); - meta_t meta() const override; - -protected: - clipp::group do_group() override; - int do_run() override; - std::string do_doc() override; -}; - - -// builds stuff -// -class build_command : public command -{ -public: - build_command(); - - meta_t meta() const override; - - // kills any msbuild.exe process, they like to linger and keep file locks - // - static void terminate_msbuild(); - -protected: - void convert_cl_to_conf() override; - clipp::group do_group() override; - int do_run() override; - -private: - std::vector tasks_; - bool redownload_ = false; - bool reextract_ = false; - bool rebuild_ = false; - bool reconfigure_ = false; - bool new_ = false; - std::optional clean_; - std::optional fetch_; - std::optional build_; - std::optional nopull_; - bool ignore_uncommitted_ = false; - bool keep_msbuild_ = false; - std::optional revert_ts_; - - - // creates a bare bones ini file in the prefix so mob can be invoked in any - // directory below it - // - void create_prefix_ini(); -}; - - -// applies a pr -// -class pr_command : public command -{ -public: - pr_command(); - - meta_t meta() const override; - -protected: - clipp::group do_group() override; - std::string do_doc() override; - int do_run() override; - -private: - struct pr_info - { - std::string repo, author, branch, title, number; - }; - - - std::string op_; - std::string pr_; - std::string github_token_; - - std::pair parse_pr( - const std::string& pr) const; - - pr_info get_pr_info(const tasks::modorganizer* task, const std::string& pr); - - std::vector get_matching_prs( - const std::string& repo_pr); - - std::vector search_prs( - const std::string& org, - const std::string& author, const std::string& branch); - - std::vector validate_prs(const std::vector& prs); - - int pull(); - int find(); - int revert(); -}; - - -// lists available tasks -// -class list_command : public command -{ -public: - meta_t meta() const override; - -protected: - clipp::group do_group() override; - int do_run() override; - -private: - bool all_ = false; - bool aliases_ = false; - std::vector tasks_; - - void dump(const std::vector& v, std::size_t indent) const; - void dump_aliases() const; -}; - - -// creates a devbuild or an official release -// -class release_command : public command -{ -public: - release_command(); - meta_t meta() const override; - - void make_bin(); - void make_pdbs(); - void make_src(); - void make_uibase(); - void make_installer(); - -protected: - clipp::group do_group() override; - int do_run() override; - std::string do_doc() override; - void convert_cl_to_conf() override; - -private: - enum class modes - { - none = 0, - devbuild, - official - }; - - - modes mode_= modes::none; - bool bin_ = true; - bool src_ = true; - bool pdbs_ = true; - bool uibase_ = true; - bool installer_ = false; - std::string utf8out_; - fs::path out_; - std::string version_; - bool version_exe_ = false; - bool version_rc_ = false; - std::string utf8_rc_path_; - fs::path rc_path_; - bool force_ = false; - std::string suffix_; - std::string branch_; - - - int do_devbuild(); - int do_official(); - - void prepare(); - void check_repos_for_branch(); - bool check_clean_prefix(); - - fs::path make_filename(const std::string& what) const; - - void walk_dir( - const fs::path& dir, std::vector& files, - const std::vector& ignore_re, std::size_t& total_size); - - std::string version_from_exe() const; - std::string version_from_rc() const; -}; - - -// manages git repos -// -class git_command : public command -{ -public: - git_command(); - meta_t meta() const override; - -protected: - clipp::group do_group() override; - int do_run() override; - std::string do_doc() override; - -private: - enum class modes - { - none = 0, - set_remotes, - add_remote, - ignore_ts, - branches - }; - - modes mode_ = modes::none; - std::string username_; - std::string email_; - std::string key_; - std::string remote_; - std::string path_; - bool tson_ = false; - bool nopush_ = false; - bool push_default_ = false; - bool all_branches_ = false; - - void do_set_remotes(); - void do_set_remotes(const fs::path& r); - - void do_add_remote(); - void do_add_remote(const fs::path& r); - - void do_ignore_ts(); - void do_ignore_ts(const fs::path& r); - - void do_branches(); - - std::vector get_repos() const; -}; - - -// runs cmake in a directory with the same parameters as `build` would -// -class cmake_command : public command -{ -public: - cmake_command(); - meta_t meta() const override; - -protected: - clipp::group do_group() override; - int do_run() override; - std::string do_doc() override; - -private: - std::string gen_; - std::string cmd_; - bool x64_ = true; - std::string prefix_; - std::string path_; -}; - - -// lists the inis found by mob -// -class inis_command : public command -{ -public: - meta_t meta() const override; - -protected: - clipp::group do_group() override; - int do_run() override; - std::string do_doc() override; -}; - - -// manages transifex -// -class tx_command : public command -{ -public: - tx_command(); - meta_t meta() const override; - -protected: - clipp::group do_group() override; - void convert_cl_to_conf() override; - int do_run() override; - std::string do_doc() override; - -private: - enum class modes - { - none = 0, - get, - build - }; - - modes mode_ = modes::none; - std::string key_, team_, project_, url_; - int min_ = -1; - std::optional force_; - std::string path_; - std::string dest_; - - void do_get(); - void do_build(); -}; - -} // namespace +namespace mob { + + class task; + class url; + + // base class for all commands + // + class command { + public: + // values of options available for all commands + // + struct common_options { + bool dry = false; + int output_log_level = -1; + int file_log_level = -1; + std::string log_file; + std::vector options; + std::vector inis; + bool no_default_inis = false; + bool dump_inis = false; + std::string prefix; + }; + + // returned by meta() by each command + // + struct meta_t { + std::string name, description; + }; + + static common_options common; + static clipp::group common_options_group(); + + virtual ~command() = default; + + // overrides the exit code returned by do_run() + // + void force_exit_code(int code); + + // forces this command to show help in run() as if --help had been given + // + void force_help(); + + // whether this command was entered by the user + // + bool picked() const; + + // command line options for this command + // + clipp::group group(); + + // executes this command + // + int run(); + + // returns meta information about this command + // + virtual meta_t meta() const = 0; + + protected: + // passed by derived classes in the constructor + // + enum flags { + noflags = 0x00, + + // this command needs the ini loaded before running + requires_options = 0x01, + + // this command does not handle sigint by itself, run() will hook it + handle_sigint = 0x02 + }; + + MOB_ENUM_FRIEND_OPERATORS(flags); + + // set to true when this command is entered by the user + bool picked_; + + // set to true with --help or by force_help() + bool help_; + + command(flags f = noflags); + + // some options have a unique version on the command line because they're + // used often, such as --dry or -l; those are converted to their ini + // equivalent here (`-l 3` becomes `global/output_log_level=3` in + // common.options) + // + virtual void convert_cl_to_conf(); + + // disables all tasks by setting task/enabled=false, but sets + // name:task/enabled=true for the given tasks + // + void set_task_enabled_flags(const std::vector& tasks); + + // calls prepare_options() and loads inis + // + int load_options(); + + // calls convert_cl_to_conf() and populates inis_ + // + int prepare_options(bool verbose); + + // called by group(), returns the clipp group for this command + // + virtual clipp::group do_group() = 0; + + // called by run(), executes this command + // + virtual int do_run() = 0; + + // called by run() when --help is given; should return additional text that + // is output at the end of the usage info + // + virtual std::string do_doc() { return {}; } + + private: + // flags requested by this command + flags flags_; + + // exit code set by force_exit_code() + std::optional code_; + + // list of inis found by gather_inis() + std::vector inis_; + + // finds all the inis + // + int gather_inis(bool verbose); + }; + + // displays mob's version + // + class version_command : public command { + public: + meta_t meta() const override; + + protected: + clipp::group do_group() override; + int do_run() override; + }; + + // displays the usage, list of commands and some additional text + // + class help_command : public command { + public: + meta_t meta() const override; + void set_commands(const std::vector>& v); + + protected: + clipp::group do_group() override; + int do_run() override; + + private: + std::string commands_; + }; + + // lists all options and their values + // + class options_command : public command { + public: + options_command(); + meta_t meta() const override; + + protected: + clipp::group do_group() override; + int do_run() override; + std::string do_doc() override; + }; + + // builds stuff + // + class build_command : public command { + public: + build_command(); + + meta_t meta() const override; + + // kills any msbuild.exe process, they like to linger and keep file locks + // + static void terminate_msbuild(); + + protected: + void convert_cl_to_conf() override; + clipp::group do_group() override; + int do_run() override; + + private: + std::vector tasks_; + bool redownload_ = false; + bool reextract_ = false; + bool rebuild_ = false; + bool reconfigure_ = false; + bool new_ = false; + std::optional clean_; + std::optional fetch_; + std::optional build_; + std::optional nopull_; + bool ignore_uncommitted_ = false; + bool keep_msbuild_ = false; + std::optional revert_ts_; + + // creates a bare bones ini file in the prefix so mob can be invoked in any + // directory below it + // + void create_prefix_ini(); + }; + + // applies a pr + // + class pr_command : public command { + public: + pr_command(); + + meta_t meta() const override; + + protected: + clipp::group do_group() override; + std::string do_doc() override; + int do_run() override; + + private: + struct pr_info { + std::string repo, author, branch, title, number; + }; + + std::string op_; + std::string pr_; + std::string github_token_; + + std::pair + parse_pr(const std::string& pr) const; + + pr_info get_pr_info(const tasks::modorganizer* task, const std::string& pr); + + std::vector get_matching_prs(const std::string& repo_pr); + + std::vector search_prs(const std::string& org, + const std::string& author, + const std::string& branch); + + std::vector validate_prs(const std::vector& prs); + + int pull(); + int find(); + int revert(); + }; + + // lists available tasks + // + class list_command : public command { + public: + meta_t meta() const override; + + protected: + clipp::group do_group() override; + int do_run() override; + + private: + bool all_ = false; + bool aliases_ = false; + std::vector tasks_; + + void dump(const std::vector& v, std::size_t indent) const; + void dump_aliases() const; + }; + + // creates a devbuild or an official release + // + class release_command : public command { + public: + release_command(); + meta_t meta() const override; + + void make_bin(); + void make_pdbs(); + void make_src(); + void make_uibase(); + void make_installer(); + + protected: + clipp::group do_group() override; + int do_run() override; + std::string do_doc() override; + void convert_cl_to_conf() override; + + private: + enum class modes { none = 0, devbuild, official }; + + modes mode_ = modes::none; + bool bin_ = true; + bool src_ = true; + bool pdbs_ = true; + bool uibase_ = true; + bool installer_ = false; + std::string utf8out_; + fs::path out_; + std::string version_; + bool version_exe_ = false; + bool version_rc_ = false; + std::string utf8_rc_path_; + fs::path rc_path_; + bool force_ = false; + std::string suffix_; + std::string branch_; + + int do_devbuild(); + int do_official(); + + void prepare(); + void check_repos_for_branch(); + bool check_clean_prefix(); + + fs::path make_filename(const std::string& what) const; + + void walk_dir(const fs::path& dir, std::vector& files, + const std::vector& ignore_re, + std::size_t& total_size); + + std::string version_from_exe() const; + std::string version_from_rc() const; + }; + + // manages git repos + // + class git_command : public command { + public: + git_command(); + meta_t meta() const override; + + protected: + clipp::group do_group() override; + int do_run() override; + std::string do_doc() override; + + private: + enum class modes { none = 0, set_remotes, add_remote, ignore_ts, branches }; + + modes mode_ = modes::none; + std::string username_; + std::string email_; + std::string key_; + std::string remote_; + std::string path_; + bool tson_ = false; + bool nopush_ = false; + bool push_default_ = false; + bool all_branches_ = false; + + void do_set_remotes(); + void do_set_remotes(const fs::path& r); + + void do_add_remote(); + void do_add_remote(const fs::path& r); + + void do_ignore_ts(); + void do_ignore_ts(const fs::path& r); + + void do_branches(); + + std::vector get_repos() const; + }; + + // runs cmake in a directory with the same parameters as `build` would + // + class cmake_command : public command { + public: + cmake_command(); + meta_t meta() const override; + + protected: + clipp::group do_group() override; + int do_run() override; + std::string do_doc() override; + + private: + std::string gen_; + std::string cmd_; + bool x64_ = true; + std::string prefix_; + std::string path_; + }; + + // lists the inis found by mob + // + class inis_command : public command { + public: + meta_t meta() const override; + + protected: + clipp::group do_group() override; + int do_run() override; + std::string do_doc() override; + }; + + // manages transifex + // + class tx_command : public command { + public: + tx_command(); + meta_t meta() const override; + + protected: + clipp::group do_group() override; + void convert_cl_to_conf() override; + int do_run() override; + std::string do_doc() override; + + private: + enum class modes { none = 0, get, build }; + + modes mode_ = modes::none; + std::string key_, team_, project_, url_; + int min_ = -1; + std::optional force_; + std::string path_; + std::string dest_; + + void do_get(); + void do_build(); + }; + +} // namespace mob diff --git a/src/cmd/git.cpp b/src/cmd/git.cpp index 833c106..c857bc0 100644 --- a/src/cmd/git.cpp +++ b/src/cmd/git.cpp @@ -1,285 +1,247 @@ #include "pch.h" -#include "commands.h" #include "../tasks/tasks.h" #include "../tools/tools.h" +#include "commands.h" + +namespace mob { + + git_command::git_command() : command(requires_options) {} + + command::meta_t git_command::meta() const + { + return {"git", "manages the git repos"}; + } + + clipp::group git_command::do_group() + { + return clipp::group( + clipp::command("git").set(picked_), -namespace mob -{ - -git_command::git_command() - : command(requires_options) -{ -} - -command::meta_t git_command::meta() const -{ - return - { - "git", - "manages the git repos" - }; -} - -clipp::group git_command::do_group() -{ - return clipp::group( - clipp::command("git").set(picked_), - - (clipp::option("-h", "--help") >> help_) - % ("shows this message"), - - "set-remotes" % - (clipp::command("set-remotes").set(mode_, modes::set_remotes), - (clipp::required("-u", "--username") - & clipp::value("USERNAME") >> username_) - % "git username", - - (clipp::required("-e", "--email") - & clipp::value("EMAIL") >> email_) - % "git email", - - (clipp::option("-k", "--key") - & clipp::value("PATH") >> key_) - % "path to putty key", - - (clipp::option("-s", "--no-push").set(nopush_) - % "disables pushing to 'upstream' by changing the push url " - "to 'nopushurl' to avoid accidental pushes"), - - (clipp::option("-p", "--push-origin").set(push_default_) - % "sets the new 'origin' remote as the default push target"), - - (clipp::opt_value("path") >> path_) - % "only use this repo" - ) - - | - - "add-remote" % - (clipp::command("add-remote").set(mode_, modes::add_remote), - (clipp::required("-n", "--name") - & clipp::value("NAME") >> remote_) - % "name of new remote", - - (clipp::required("-u", "--username") - & clipp::value("USERNAME") >> username_) - % "git username", - - (clipp::option("-k", "--key") - & clipp::value("PATH") >> key_) - % "path to putty key", - - (clipp::option("-p", "--push-origin").set(push_default_) - % "sets this new remote as the default push target"), - - (clipp::opt_value("path") >> path_) - % "only use this repo" - ) - - | - - "ignore-ts" % - (clipp::command("ignore-ts").set(mode_, modes::ignore_ts), - ( - clipp::command("on").set(tson_, true) | - clipp::command("off").set(tson_, false) - ) - ) - - | - - "branches" % - (clipp::command("branches").set(mode_, modes::branches), - clipp::option("-a", "--all").set(all_branches_) - % "shows all branches, including those on master" - ) - ); -} - -int git_command::do_run() -{ - switch (mode_) - { - case modes::set_remotes: - { - do_set_remotes(); - break; - } - - case modes::add_remote: - { - do_add_remote(); - break; - } - - case modes::ignore_ts: - { - do_ignore_ts(); - break; - } - - case modes::branches: - { - do_branches(); - break; - } - - case modes::none: - default: - u8cerr << "bad git mode " << static_cast(mode_) << "\n"; - throw bailed(); - } - - return 0; -} - -std::string git_command::do_doc() -{ - return - "All the commands will go through all modorganizer repos, plus usvfs\n" - "and NCC.\n" - "\n" - "Commands:\n" - "set-remotes\n" - " For each repo, this first sets the username and email. Then, it\n" - " will rename the remote 'origin' to 'upstream' and create a new\n" - " remote 'origin' with the given information. If the remote\n" - " 'upstream' already exists in a repo, nothing happens.\n" - "\n" - "add-remote\n" - " For each repo, adds a new remote with the given information. If a\n" - " remote with the same name already exists, nothing happens.\n" - "\n" - "ignore-ts\n" - " Toggles the --assume-changed status of all .ts files in all repos.\n" - "\n" - "branches\n" - " Lists all git repos that are not on master. With -a, show all \n" - " repos and their current branch."; -} - -void git_command::do_set_remotes() -{ - if (path_.empty()) - { - const auto repos = get_repos(); - - for (auto&& r : repos) - do_set_remotes(r); - } - else - { - do_set_remotes(path_); - } -} - -void git_command::do_set_remotes(const fs::path& r) -{ - u8cout << "setting up " << path_to_utf8(r.filename()) << "\n"; - - git_wrap(r).set_credentials(username_, email_); - - git_wrap(r).set_origin_and_upstream_remotes( - username_, key_, nopush_, push_default_); -} - -void git_command::do_add_remote() -{ - u8cout - << "adding remote '" << remote_ << "' " - << "from '" << username_ << "' to repos\n"; - - if (path_.empty()) - { - const auto repos = get_repos(); - - for (auto&& r : repos) - do_add_remote(r); - } - else - { - do_add_remote(path_); - } -} - -void git_command::do_add_remote(const fs::path& r) -{ - u8cout << path_to_utf8(r.filename()) << "\n"; - git_wrap(r).add_remote(remote_, username_, key_, push_default_); -} - -void git_command::do_ignore_ts() -{ - if (tson_) - u8cout << "ignoring .ts files\n"; - else - u8cout << "un-ignoring .ts files\n"; - - if (path_.empty()) - { - const auto repos = get_repos(); - - for (auto&& r : repos) - do_ignore_ts(r); - } - else - { - do_ignore_ts(path_); - } -} - -void git_command::do_ignore_ts(const fs::path& r) -{ - u8cout << path_to_utf8(r.filename()) << "\n"; - git_wrap(r).ignore_ts(tson_); -} - -void git_command::do_branches() -{ - std::vector> v; - - for (auto&& r : get_repos()) - { - const auto b = git_wrap(r).current_branch(); - if (b == "master" && !all_branches_) - continue; - - if (b.empty()) - v.push_back({r.filename().string(), "detached head"}); - else - v.push_back({r.filename().string(), b}); - } - - u8cout << table(v, 0, 3) << "\n"; -} - -std::vector git_command::get_repos() const -{ - std::vector v; - - // usvfs - if (fs::exists(tasks::usvfs::source_path())) - v.push_back(tasks::usvfs::source_path()); - - const auto super = tasks::modorganizer::super_path(); - - // all directories in super except for those starting with a dot - if (fs::exists(super)) - { - for (auto e : fs::directory_iterator(super)) - { - if (!e.is_directory()) - continue; - - const auto p = e.path(); - if (path_to_utf8(p.filename()).starts_with(".")) - continue; - - v.push_back(p); - } - } - - return v; -} - -} // namespace + (clipp::option("-h", "--help") >> help_) % ("shows this message"), + + "set-remotes" % + (clipp::command("set-remotes").set(mode_, modes::set_remotes), + (clipp::required("-u", "--username") & + clipp::value("USERNAME") >> username_) % + "git username", + + (clipp::required("-e", "--email") & + clipp::value("EMAIL") >> email_) % + "git email", + + (clipp::option("-k", "--key") & clipp::value("PATH") >> key_) % + "path to putty key", + + (clipp::option("-s", "--no-push").set(nopush_) % + "disables pushing to 'upstream' by changing the push url " + "to 'nopushurl' to avoid accidental pushes"), + + (clipp::option("-p", "--push-origin").set(push_default_) % + "sets the new 'origin' remote as the default push target"), + + (clipp::opt_value("path") >> path_) % "only use this repo") + + | + + "add-remote" % + (clipp::command("add-remote").set(mode_, modes::add_remote), + (clipp::required("-n", "--name") & + clipp::value("NAME") >> remote_) % + "name of new remote", + + (clipp::required("-u", "--username") & + clipp::value("USERNAME") >> username_) % + "git username", + + (clipp::option("-k", "--key") & clipp::value("PATH") >> key_) % + "path to putty key", + + (clipp::option("-p", "--push-origin").set(push_default_) % + "sets this new remote as the default push target"), + + (clipp::opt_value("path") >> path_) % "only use this repo") + + | + + "ignore-ts" % (clipp::command("ignore-ts").set(mode_, modes::ignore_ts), + (clipp::command("on").set(tson_, true) | + clipp::command("off").set(tson_, false))) + + | + + "branches" % (clipp::command("branches").set(mode_, modes::branches), + clipp::option("-a", "--all").set(all_branches_) % + "shows all branches, including those on master")); + } + + int git_command::do_run() + { + switch (mode_) { + case modes::set_remotes: { + do_set_remotes(); + break; + } + + case modes::add_remote: { + do_add_remote(); + break; + } + + case modes::ignore_ts: { + do_ignore_ts(); + break; + } + + case modes::branches: { + do_branches(); + break; + } + + case modes::none: + default: + u8cerr << "bad git mode " << static_cast(mode_) << "\n"; + throw bailed(); + } + + return 0; + } + + std::string git_command::do_doc() + { + return "All the commands will go through all modorganizer repos, plus usvfs\n" + "and NCC.\n" + "\n" + "Commands:\n" + "set-remotes\n" + " For each repo, this first sets the username and email. Then, it\n" + " will rename the remote 'origin' to 'upstream' and create a new\n" + " remote 'origin' with the given information. If the remote\n" + " 'upstream' already exists in a repo, nothing happens.\n" + "\n" + "add-remote\n" + " For each repo, adds a new remote with the given information. If a\n" + " remote with the same name already exists, nothing happens.\n" + "\n" + "ignore-ts\n" + " Toggles the --assume-changed status of all .ts files in all repos.\n" + "\n" + "branches\n" + " Lists all git repos that are not on master. With -a, show all \n" + " repos and their current branch."; + } + + void git_command::do_set_remotes() + { + if (path_.empty()) { + const auto repos = get_repos(); + + for (auto&& r : repos) + do_set_remotes(r); + } + else { + do_set_remotes(path_); + } + } + + void git_command::do_set_remotes(const fs::path& r) + { + u8cout << "setting up " << path_to_utf8(r.filename()) << "\n"; + + git_wrap(r).set_credentials(username_, email_); + + git_wrap(r).set_origin_and_upstream_remotes(username_, key_, nopush_, + push_default_); + } + + void git_command::do_add_remote() + { + u8cout << "adding remote '" << remote_ << "' " + << "from '" << username_ << "' to repos\n"; + + if (path_.empty()) { + const auto repos = get_repos(); + + for (auto&& r : repos) + do_add_remote(r); + } + else { + do_add_remote(path_); + } + } + + void git_command::do_add_remote(const fs::path& r) + { + u8cout << path_to_utf8(r.filename()) << "\n"; + git_wrap(r).add_remote(remote_, username_, key_, push_default_); + } + + void git_command::do_ignore_ts() + { + if (tson_) + u8cout << "ignoring .ts files\n"; + else + u8cout << "un-ignoring .ts files\n"; + + if (path_.empty()) { + const auto repos = get_repos(); + + for (auto&& r : repos) + do_ignore_ts(r); + } + else { + do_ignore_ts(path_); + } + } + + void git_command::do_ignore_ts(const fs::path& r) + { + u8cout << path_to_utf8(r.filename()) << "\n"; + git_wrap(r).ignore_ts(tson_); + } + + void git_command::do_branches() + { + std::vector> v; + + for (auto&& r : get_repos()) { + const auto b = git_wrap(r).current_branch(); + if (b == "master" && !all_branches_) + continue; + + if (b.empty()) + v.push_back({r.filename().string(), "detached head"}); + else + v.push_back({r.filename().string(), b}); + } + + u8cout << table(v, 0, 3) << "\n"; + } + + std::vector git_command::get_repos() const + { + std::vector v; + + // usvfs + if (fs::exists(tasks::usvfs::source_path())) + v.push_back(tasks::usvfs::source_path()); + + const auto super = tasks::modorganizer::super_path(); + + // all directories in super except for those starting with a dot + if (fs::exists(super)) { + for (auto e : fs::directory_iterator(super)) { + if (!e.is_directory()) + continue; + + const auto p = e.path(); + if (path_to_utf8(p.filename()).starts_with(".")) + continue; + + v.push_back(p); + } + } + + return v; + } + +} // namespace mob diff --git a/src/cmd/list.cpp b/src/cmd/list.cpp index f98964d..f0995a9 100644 --- a/src/cmd/list.cpp +++ b/src/cmd/list.cpp @@ -1,100 +1,83 @@ #include "pch.h" -#include "commands.h" -#include "../tasks/task_manager.h" #include "../tasks/task.h" +#include "../tasks/task_manager.h" #include "../utility/io.h" +#include "commands.h" -namespace mob -{ - -command::meta_t list_command::meta() const -{ - return - { - "list", - "lists available tasks" - }; -} - -clipp::group list_command::do_group() -{ - return clipp::group( - clipp::command("list").set(picked_), - - (clipp::option("-h", "--help") >> help_) - % "shows this message", - - (clipp::option("-a", "--all") >> all_) - % "shows all the tasks, including pseudo parallel tasks", - - (clipp::option("-i", "--aliases") >> aliases_) - % "shows only aliases", - - (clipp::opt_values( - clipp::match::prefix_not("-"), "task", tasks_)) - % "with -a; when given, acts like the tasks given to `build` and " - "shows only the tasks that would run" - ); -} - -int list_command::do_run() -{ - auto& tm = task_manager::instance(); - - if (aliases_) - { - load_options(); - dump_aliases(); - } - else - { - if (all_) - { - if (!tasks_.empty()) - set_task_enabled_flags(tasks_); - - load_options(); - dump(tm.top_level(), 0); - - u8cout << "\n\naliases:\n"; - dump_aliases(); - } - else - { - for (auto&& t : tm.all()) - u8cout << " - " << join(t->names(), ", ") << "\n"; - } - } - - - return 0; -} - -void list_command::dump(const std::vector& v, std::size_t indent) const -{ - for (auto&& t : v) - { - if (!t->enabled()) - continue; - - u8cout - << std::string(indent*4, ' ') - << " - " << join(t->names(), ",") - << "\n"; - - if (auto* pt=dynamic_cast(t)) - dump(pt->children(), indent + 1); - } -} - -void list_command::dump_aliases() const -{ - const auto v = task_manager::instance().aliases(); - if (v.empty()) - return; - - for (auto&& [k, patterns] : v) - u8cout << " - " << k << ": " << join(patterns, ", ") << "\n"; -} - -} // namespace +namespace mob { + + command::meta_t list_command::meta() const + { + return {"list", "lists available tasks"}; + } + + clipp::group list_command::do_group() + { + return clipp::group( + clipp::command("list").set(picked_), + + (clipp::option("-h", "--help") >> help_) % "shows this message", + + (clipp::option("-a", "--all") >> all_) % + "shows all the tasks, including pseudo parallel tasks", + + (clipp::option("-i", "--aliases") >> aliases_) % "shows only aliases", + + (clipp::opt_values(clipp::match::prefix_not("-"), "task", tasks_)) % + "with -a; when given, acts like the tasks given to `build` and " + "shows only the tasks that would run"); + } + + int list_command::do_run() + { + auto& tm = task_manager::instance(); + + if (aliases_) { + load_options(); + dump_aliases(); + } + else { + if (all_) { + if (!tasks_.empty()) + set_task_enabled_flags(tasks_); + + load_options(); + dump(tm.top_level(), 0); + + u8cout << "\n\naliases:\n"; + dump_aliases(); + } + else { + for (auto&& t : tm.all()) + u8cout << " - " << join(t->names(), ", ") << "\n"; + } + } + + return 0; + } + + void list_command::dump(const std::vector& v, std::size_t indent) const + { + for (auto&& t : v) { + if (!t->enabled()) + continue; + + u8cout << std::string(indent * 4, ' ') << " - " << join(t->names(), ",") + << "\n"; + + if (auto* pt = dynamic_cast(t)) + dump(pt->children(), indent + 1); + } + } + + void list_command::dump_aliases() const + { + const auto v = task_manager::instance().aliases(); + if (v.empty()) + return; + + for (auto&& [k, patterns] : v) + u8cout << " - " << k << ": " << join(patterns, ", ") << "\n"; + } + +} // namespace mob diff --git a/src/cmd/pr.cpp b/src/cmd/pr.cpp index 5b73fd8..4754875 100644 --- a/src/cmd/pr.cpp +++ b/src/cmd/pr.cpp @@ -1,409 +1,364 @@ #include "pch.h" -#include "commands.h" #include "../core/conf.h" #include "../tasks/task_manager.h" #include "../tasks/tasks.h" #include "../utility.h" +#include "commands.h" + +namespace mob { + + std::string read_file(const fs::path& p) + { + std::ifstream t(p); + return {std::istreambuf_iterator(t), std::istreambuf_iterator()}; + } + + pr_command::pr_command() : command(requires_options | handle_sigint) {} + + command::meta_t pr_command::meta() const + { + return {"pr", "applies changes from PRs"}; + } + + std::string pr_command::do_doc() + { + return "Operations:\n" + " - find: lists all the repos that would affected by `pull` or\n" + " `revert`\n" + " - pull: fetches the pr's branch and checks it out; all repos\n" + " will be in detached HEAD state\n" + " - revert: checks out branch `master` for every affected repo\n" + "\n" + "Repos that are not handled:\n" + " - mob itself\n" + " - umbrella\n" + " - any repo that's not in modorganizer_super\n" + " - modorganizer_installer"; + } + + clipp::group pr_command::do_group() + { + return (clipp::command("pr")).set(picked_), + + (clipp::option("-h", "--help") >> help_) % ("shows this message"), + + (clipp::option("--github-token") & + clipp::value("TOKEN") >> github_token_) % + "github api key", + + (clipp::value("OP") >> op_) % + "one of `find`, `pull` or `revert`; see below", + + (clipp::value("PR") >> pr_) % + "PR to apply, must be `task/pr`, such as `modorganizer/123`"; + } + + int pr_command::do_run() + { + if (github_token_.empty()) + github_token_ = conf().global().get("github_key"); + + if (op_ == "pull") + return pull(); + else if (op_ == "find") + return find(); + else if (op_ == "revert") + return revert(); + else + u8cerr << "bad operation '" << op_ << "'\n"; + + return 1; + } + + std::pair + pr_command::parse_pr(const std::string& pr) const + { + if (pr.empty()) + return {}; + + const auto cs = split(pr, "/"); + if (cs.size() != 2) { + u8cerr << "--pr must be task/pr, such as modorganizer/123\n"; + return {}; + } + + const std::string pattern = cs[0]; + const std::string pr_number = cs[1]; + + const auto* task = task_manager::instance().find_one(pattern); + if (!task) + return {}; + + const auto* mo_task = dynamic_cast(task); + if (!mo_task) { + u8cerr << "only modorganizer tasks are supported\n"; + return {}; + } + + return {mo_task, pr_number}; + } + + int pr_command::pull() + { + const auto prs = get_matching_prs(pr_); + if (prs.empty()) + return 1; + + const auto okay_prs = validate_prs(prs); + if (okay_prs.empty()) + return 1; + + try { + for (auto&& pr : okay_prs) { + const auto* task = dynamic_cast( + task_manager::instance().find_one(pr.repo)); + + if (!task) + return 1; + + u8cout << "checking out pr " << pr.number << " " + << "in " << task->name() << "\n"; + + git_wrap g(task->source_path()); + + g.fetch(task->git_url().string(), + fmt::format("pull/{}/head", pr.number)); + + g.checkout("FETCH_HEAD"); + } + + u8cout << "note: all these repos are now in detached HEAD state\n"; + + return 0; + } + catch (std::exception& e) { + u8cerr << e.what() << "\n"; + return 1; + } + } + + int pr_command::find() + { + return !get_matching_prs(pr_).empty(); + } + + int pr_command::revert() + { + const auto prs = get_matching_prs(pr_); + if (prs.empty()) + return 1; + + const auto okay_prs = validate_prs(prs); + if (okay_prs.empty()) + return 1; + + try { + for (auto&& pr : okay_prs) { + const auto* task = dynamic_cast( + task_manager::instance().find_one(pr.repo)); + + if (!task) + return 1; + + u8cout << "reverting " << task->name() << " to master\n"; + + git_wrap(task->source_path()).checkout("master"); + } + + return 0; + } + catch (std::exception& e) { + u8cerr << e.what() << "\n"; + return 1; + } + } + + std::vector + pr_command::get_matching_prs(const std::string& repo_pr) + { + auto&& [task, src_pr] = parse_pr(repo_pr); + if (!task) + return {}; + + u8cout << "getting info for pr " << src_pr << " in " << task->name() << "\n"; + const auto info = get_pr_info(task, src_pr); + if (info.repo.empty()) + return {}; + + u8cout << "found pr from " << info.author << ":" << info.branch << "\n"; + + u8cout << "searching\n"; + const auto prs = search_prs(task->org(), info.author, info.branch); + + u8cout << "found matching prs in " << prs.size() << " repos:\n"; + + u8cout << table(map(prs, + [&](auto&& pr) { + return std::pair(pr.repo + "/" + pr.number, pr.title); + }), + 2, 5) + << "\n"; + + return prs; + } + + std::vector pr_command::search_prs(const std::string& org, + const std::string& author, + const std::string& branch) + { + nlohmann::json json; + + constexpr auto* pattern = + "https://api.github.com/search/issues?per_page=100&q=" + "is:pr+org:{org:}+author:{author:}+is:open+head:{branch:}"; + + const auto search_url = + fmt::format(pattern, fmt::arg("org", org), fmt::arg("author", author), + fmt::arg("branch", branch)); + + u8cout << "search url is " << search_url << "\n"; + + u8cout << "searching for matching prs\n"; + + curl_downloader dl; + + dl.url(search_url) + .header("Authorization", "token " + github_token_) + .start() + .join(); + + if (!dl.ok()) { + u8cerr << "failed to search github\n"; + return {}; + } + + const auto output = dl.steal_output(); + json = nlohmann::json::parse(output); + + std::map repos; + + for (auto&& item : json["items"]) { + // ex: https://api.github.com/repos/ModOrganizer2/modorganizer-Installer + const std::string url = item["repository_url"]; + + const auto last_slash = url.find_last_of("/"); + if (last_slash == std::string::npos) { + u8cerr << "bad repo url in search: '" << url << "'\n"; + return {}; + } + + const auto repo = url.substr(last_slash + 1); + + pr_info info = {repo, author, branch, item["title"], + std::to_string(item["number"].get())}; + + if (!repos.emplace(repo, info).second) { + u8cerr << "multiple prs found in repo " << repo << ", " + << "not supported\n"; + + return {}; + } + } + + return map(repos, [&](auto&& pair) { + return pair.second; + }); + } + + pr_command::pr_info pr_command::get_pr_info(const tasks::modorganizer* task, + const std::string& pr) + { + nlohmann::json json; + + if (github_token_.empty()) { + u8cerr << "missing --github-token\n"; + return {}; + } + + const url u(fmt::format("https://api.github.com/repos/{}/{}/pulls/{}", + task->org(), task->repo(), pr)); + + curl_downloader dl; + + dl.url(u).header("Authorization", "token " + github_token_).start().join(); + + if (!dl.ok()) { + u8cerr << "failed to get pr info from github\n"; + return {}; + } + + const auto output = dl.steal_output(); + json = nlohmann::json::parse(output); + + const std::string repo = json["head"]["repo"]["name"]; + const std::string author = json["head"]["repo"]["user"]["login"]; + const std::string branch = json["head"]["ref"]; + + return {repo, author, branch}; + } + + std::vector + pr_command::validate_prs(const std::vector& prs) + { + std::vector problems; + std::vector okay_prs; + + for (auto&& pr : prs) { + if (pr.repo == "mob") { + problems.push_back("there's a pr for mob itself"); + continue; + } + else { + const auto tasks = task_manager::instance().find(pr.repo); + + if (tasks.empty()) { + problems.push_back("task " + pr.repo + " does not exist"); + continue; + } + else if (tasks.size() > 1) { + problems.push_back("found more than one task for repo " + pr.repo); + continue; + } + else { + const auto* mo_task = + dynamic_cast(tasks[0]); + + if (!mo_task) { + problems.push_back("task " + pr.repo + + " is not a modorganizer repo"); + + continue; + } + } + } -namespace mob -{ - -std::string read_file(const fs::path& p) -{ - std::ifstream t(p); - return {std::istreambuf_iterator(t), std::istreambuf_iterator()}; -} - - -pr_command::pr_command() - : command(requires_options | handle_sigint) -{ -} - -command::meta_t pr_command::meta() const -{ - return - { - "pr", - "applies changes from PRs" - }; -} - -std::string pr_command::do_doc() -{ - return - "Operations:\n" - " - find: lists all the repos that would affected by `pull` or\n" - " `revert`\n" - " - pull: fetches the pr's branch and checks it out; all repos\n" - " will be in detached HEAD state\n" - " - revert: checks out branch `master` for every affected repo\n" - "\n" - "Repos that are not handled:\n" - " - mob itself\n" - " - umbrella\n" - " - any repo that's not in modorganizer_super\n" - " - modorganizer_installer"; -} - -clipp::group pr_command::do_group() -{ - return - (clipp::command("pr")).set(picked_), - - (clipp::option("-h", "--help") >> help_) - % ("shows this message"), - - (clipp::option("--github-token") - & clipp::value("TOKEN") >> github_token_) - % "github api key", - - (clipp::value("OP") >> op_) - % "one of `find`, `pull` or `revert`; see below", - - (clipp::value("PR") >> pr_) - % "PR to apply, must be `task/pr`, such as `modorganizer/123`"; -} - -int pr_command::do_run() -{ - if (github_token_.empty()) - github_token_ = conf().global().get("github_key"); - - if (op_ == "pull") - return pull(); - else if (op_ == "find") - return find(); - else if (op_ == "revert") - return revert(); - else - u8cerr << "bad operation '" << op_ << "'\n"; - - return 1; -} - -std::pair pr_command::parse_pr( - const std::string& pr) const -{ - if (pr.empty()) - return {}; - - const auto cs = split(pr, "/"); - if (cs.size() != 2) - { - u8cerr << "--pr must be task/pr, such as modorganizer/123\n"; - return {}; - } - - const std::string pattern = cs[0]; - const std::string pr_number = cs[1]; - - const auto* task = task_manager::instance().find_one(pattern); - if (!task) - return {}; - - const auto* mo_task = dynamic_cast(task); - if (!mo_task) - { - u8cerr << "only modorganizer tasks are supported\n"; - return {}; - } - - return {mo_task, pr_number}; -} - -int pr_command::pull() -{ - const auto prs = get_matching_prs(pr_); - if (prs.empty()) - return 1; - - const auto okay_prs = validate_prs(prs); - if (okay_prs.empty()) - return 1; - - try - { - for (auto&& pr : okay_prs) - { - const auto* task = dynamic_cast( - task_manager::instance().find_one(pr.repo)); - - if (!task) - return 1; - - u8cout - << "checking out pr " << pr.number << " " - << "in " << task->name() << "\n"; - - git_wrap g(task->source_path()); - - g.fetch( - task->git_url().string(), - fmt::format("pull/{}/head", pr.number)); - - g.checkout("FETCH_HEAD"); - } - - u8cout << "note: all these repos are now in detached HEAD state\n"; - - return 0; - } - catch(std::exception& e) - { - u8cerr << e.what() << "\n"; - return 1; - } -} - -int pr_command::find() -{ - return !get_matching_prs(pr_).empty(); -} - -int pr_command::revert() -{ - const auto prs = get_matching_prs(pr_); - if (prs.empty()) - return 1; - - const auto okay_prs = validate_prs(prs); - if (okay_prs.empty()) - return 1; - - try - { - for (auto&& pr : okay_prs) - { - const auto* task = dynamic_cast( - task_manager::instance().find_one(pr.repo)); - - if (!task) - return 1; - - u8cout << "reverting " << task->name() << " to master\n"; - - git_wrap(task->source_path()).checkout("master"); - } - - return 0; - } - catch(std::exception& e) - { - u8cerr << e.what() << "\n"; - return 1; - } -} - -std::vector pr_command::get_matching_prs( - const std::string& repo_pr) -{ - auto&& [task, src_pr] = parse_pr(repo_pr); - if (!task) - return {}; - - u8cout << "getting info for pr " << src_pr << " in " << task->name() << "\n"; - const auto info = get_pr_info(task, src_pr); - if (info.repo.empty()) - return {}; - - u8cout << "found pr from " << info.author << ":" << info.branch << "\n"; - - u8cout << "searching\n"; - const auto prs = search_prs(task->org(), info.author, info.branch); - - u8cout << "found matching prs in " << prs.size() << " repos:\n"; - - u8cout - << table(map(prs, [&](auto&& pr) - { - return std::pair(pr.repo + "/" + pr.number, pr.title); - }), 2, 5) - << "\n"; - - return prs; -} - -std::vector pr_command::search_prs( - const std::string& org, const std::string& author, const std::string& branch) -{ - nlohmann::json json; - - constexpr auto* pattern = - "https://api.github.com/search/issues?per_page=100&q=" - "is:pr+org:{org:}+author:{author:}+is:open+head:{branch:}"; - - const auto search_url = fmt::format( - pattern, - fmt::arg("org", org), - fmt::arg("author", author), - fmt::arg("branch", branch)); - - u8cout << "search url is " << search_url << "\n"; - - u8cout << "searching for matching prs\n"; - - curl_downloader dl; - - dl - .url(search_url) - .header("Authorization", "token " + github_token_) - .start() - .join(); - - if (!dl.ok()) - { - u8cerr << "failed to search github\n"; - return {}; - } - - const auto output = dl.steal_output(); - json = nlohmann::json::parse(output); - - - std::map repos; - - for (auto&& item : json["items"]) - { - // ex: https://api.github.com/repos/ModOrganizer2/modorganizer-Installer - const std::string url = item["repository_url"]; - - const auto last_slash = url.find_last_of("/"); - if (last_slash == std::string::npos) - { - u8cerr << "bad repo url in search: '" << url << "'\n"; - return {}; - } - - const auto repo = url.substr(last_slash + 1); - - pr_info info = { - repo, author, branch, item["title"], - std::to_string(item["number"].get()) - }; - - if (!repos.emplace(repo, info).second) - { - u8cerr - << "multiple prs found in repo " << repo << ", " - << "not supported\n"; - - return {}; - } - } - - return map(repos, [&](auto&& pair){ return pair.second; }); -} - -pr_command::pr_info pr_command::get_pr_info( - const tasks::modorganizer* task, const std::string& pr) -{ - nlohmann::json json; - - if (github_token_.empty()) - { - u8cerr << "missing --github-token\n"; - return {}; - } - - const url u(fmt::format( - "https://api.github.com/repos/{}/{}/pulls/{}", - task->org(), task->repo(), pr)); - - curl_downloader dl; - - dl - .url(u) - .header("Authorization", "token " + github_token_) - .start() - .join(); - - if (!dl.ok()) - { - u8cerr << "failed to get pr info from github\n"; - return {}; - } - - const auto output = dl.steal_output(); - json = nlohmann::json::parse(output); - - const std::string repo = json["head"]["repo"]["name"]; - const std::string author = json["head"]["repo"]["user"]["login"]; - const std::string branch = json["head"]["ref"]; - - return {repo, author, branch}; -} - -std::vector pr_command::validate_prs( - const std::vector& prs) -{ - std::vector problems; - std::vector okay_prs; - - for (auto&& pr : prs) - { - if (pr.repo == "mob") - { - problems.push_back("there's a pr for mob itself"); - continue; - } - else - { - const auto tasks = task_manager::instance().find(pr.repo); - - if (tasks.empty()) - { - problems.push_back("task " + pr.repo + " does not exist"); - continue; - } - else if (tasks.size() > 1) - { - problems.push_back("found more than one task for repo " + pr.repo); - continue; - } - else - { - const auto* mo_task = - dynamic_cast(tasks[0]); - - if (!mo_task) - { - problems.push_back( - "task " + pr.repo + " is not a modorganizer repo"); - - continue; - } - } - } - - okay_prs.push_back(pr); - } - - if (!problems.empty()) - { - { - console_color cc(console_color::yellow); - - u8cout << "\nproblems:\n"; - for (auto&& p : problems) - u8cout << " - " << p << "\n"; - } - - u8cout << "\n"; - - if (okay_prs.empty()) - { - u8cout << "all prs would be ignored, bailing out\n"; - return {}; - } - - if (ask_yes_no("these prs will be ignored; proceed anyway?", yn::no) != yn::yes) - return {}; - - u8cout << "\n"; - } - - return okay_prs; -} - -} // namespace + okay_prs.push_back(pr); + } + + if (!problems.empty()) { + { + console_color cc(console_color::yellow); + + u8cout << "\nproblems:\n"; + for (auto&& p : problems) + u8cout << " - " << p << "\n"; + } + + u8cout << "\n"; + + if (okay_prs.empty()) { + u8cout << "all prs would be ignored, bailing out\n"; + return {}; + } + + if (ask_yes_no("these prs will be ignored; proceed anyway?", yn::no) != + yn::yes) + return {}; + + u8cout << "\n"; + } + + return okay_prs; + } + +} // namespace mob diff --git a/src/cmd/release.cpp b/src/cmd/release.cpp index 2d8a2f7..0086755 100644 --- a/src/cmd/release.cpp +++ b/src/cmd/release.cpp @@ -1,594 +1,532 @@ #include "pch.h" -#include "commands.h" -#include "../utility.h" #include "../core/conf.h" #include "../core/context.h" -#include "../core/op.h" #include "../core/ini.h" -#include "../tasks/tasks.h" +#include "../core/op.h" #include "../tasks/task_manager.h" +#include "../tasks/tasks.h" +#include "../utility.h" #include "../utility/threading.h" +#include "commands.h" + +namespace mob { + + void set_sigint_handler(); + + release_command::release_command() : command(requires_options) {} + + command::meta_t release_command::meta() const + { + return {"release", "creates a release"}; + } + + void release_command::make_bin() + { + const auto out = out_ / make_filename(""); + u8cout << "making binary archive " << path_to_utf8(out) << "\n"; + + op::archive_from_glob(gcx(), conf().path().install_bin() / "*", out, + {"__pycache__"}); + } + + void release_command::make_pdbs() + { + const auto out = out_ / make_filename("pdbs"); + u8cout << "making pdbs archive " << path_to_utf8(out) << "\n"; + + op::archive_from_glob(gcx(), conf().path().install_pdbs() / "*", out, + {"__pycache__"}); + } + + void release_command::make_src() + { + const auto out = out_ / make_filename("src"); + u8cout << "making src archive " << path_to_utf8(out) << "\n"; + + const std::vector ignore = {"\\..+", // dot files + ".*\\.log", ".*\\.tlog", ".*\\.dll", + ".*\\.exe", ".*\\.lib", ".*\\.obj", + ".*\\.ts", ".*\\.aps", "vsbuild"}; + + const std::vector ignore_re(ignore.begin(), ignore.end()); + + std::vector files; + std::size_t total_size = 0; + + if (!fs::exists(tasks::modorganizer::super_path())) { + gcx().bail_out(context::generic, "modorganizer super path not found: {}", + tasks::modorganizer::super_path()); + } + + // build list list + walk_dir(tasks::modorganizer::super_path(), files, ignore_re, total_size); + + // should be below 20MB + const std::size_t max_expected_size = 20 * 1024 * 1024; + if (total_size >= max_expected_size) { + gcx().warning(context::generic, + "total size of source files would be {}, expected something " + "below {}, something might be wrong", + total_size, max_expected_size); + + if (!force_) { + gcx().bail_out(context::generic, "bailing out, use --force to ignore"); + } + } + + op::archive_from_files(gcx(), files, tasks::modorganizer::super_path(), out); + } + + void release_command::make_uibase() + { + const auto out = out_ / make_filename("uibase"); + u8cout << "making uibase archive " << path_to_utf8(out) << "\n"; + + std::vector files; + + if (!fs::exists(tasks::modorganizer::super_path())) { + gcx().bail_out(context::generic, "modorganizer super path not found: {}", + tasks::modorganizer::super_path()); + } + + op::archive_from_glob( + gcx(), tasks::modorganizer::super_path() / "uibase" / "src" / "*.h", out, + {}); + op::archive_from_files(gcx(), {conf().path().install_libs() / "uibase.lib"}, + conf().path().install_libs(), out); + } + + void release_command::make_installer() + { + const auto file = "Mod.Organizer-" + version_ + ".exe"; + const auto src = conf().path().install_installer() / file; + const auto dest = out_; + + u8cout << "copying installer " << file << "\n"; + + op::copy_file_to_dir_if_better(gcx(), src, dest); + } + + void release_command::walk_dir(const fs::path& dir, std::vector& files, + const std::vector& ignore_re, + std::size_t& total_size) + { + // adds all files that are not in the ignore list to `files`, recursive + + for (auto e : fs::directory_iterator(dir)) { + const auto p = e.path(); + const auto filename = path_to_utf8(p.filename()); + + bool ignored = false; + + for (auto&& re : ignore_re) { + if (std::regex_match(filename, re)) { + ignored = true; + break; + } + } + + if (ignored) + continue; + + if (e.is_directory()) { + walk_dir(e.path(), files, ignore_re, total_size); + } + else if (e.is_regular_file()) { + total_size += fs::file_size(p); + files.push_back(p); + } + } + } + + fs::path release_command::make_filename(const std::string& what) const + { + std::string filename = "Mod.Organizer"; + + if (!version_.empty()) + filename += "-" + version_; + + if (!suffix_.empty()) + filename += "-" + suffix_; + + if (!what.empty()) + filename += "-" + what; + + filename += ".7z"; + + return filename; + } + + clipp::group release_command::do_group() + { + return clipp::group( + clipp::command("release").set(picked_), + + (clipp::option("-h", "--help") >> help_) % ("shows this message"), + + "devbuild" % + (clipp::command("devbuild").set(mode_, modes::devbuild), + (clipp::option("--bin").set(bin_, true) | + clipp::option("--no-bin").set(bin_, false)) % + "sets whether the binary archive is created [default: yes]", + + (clipp::option("--pdbs").set(pdbs_, true) | + clipp::option("--no-pdbs").set(pdbs_, false)) % + "sets whether the PDBs archive is created [default: yes]", + + (clipp::option("--src").set(src_, true) | + clipp::option("--no-src").set(src_, false)) % + "sets whether the source archive is created [default: yes]", + + //( + // clipp::option("--uibase").set(uibase_, true) | + // clipp::option("--no-uibase").set(uibase_, false) + //) % "sets whether the uibase archive is created [default: yes]", + + (clipp::option("--inst").set(installer_, true) | + clipp::option("--no-inst").set(installer_, false)) % + "sets whether the installer is copied [default: no]", + + clipp::option("--version-from-exe").set(version_exe_) % + "retrieves version information from ModOrganizer.exe " + "[default]", + + clipp::option("--version-from-rc").set(version_rc_) % + "retrieves version information from " + "modorganizer/src/version.rc", + + (clipp::option("--rc") & clipp::value("PATH") >> utf8_rc_path_) % + "overrides the path to version.rc", + + (clipp::option("--version") & + clipp::value("VERSION") >> version_) % + "overrides the version string", + + (clipp::option("--output-dir") & + clipp::value("PATH") >> utf8out_) % + "sets the output directory to use instead of " + "`$prefix/releases`", + + (clipp::option("--suffix") & clipp::value("SUFFIX") >> suffix_) % + "optional suffix to add to the archive filenames", + + clipp::option("--force").set(force_) % + "ignores file size warnings and existing release directories") + + | + + "official" % (clipp::command("official").set(mode_, modes::official), + (clipp::value("branch") >> branch_) % + "use this branch in the super repos")); + } + + void release_command::convert_cl_to_conf() + { + command::convert_cl_to_conf(); + + if (mode_ == modes::official) { + // force enable translations, installer and tx + + common.options.push_back("task/mo_branch=" + branch_); + common.options.push_back("translations:task/enabled=true"); + common.options.push_back("installer:task/enabled=true"); + + common.options.push_back("transifex/force=true"); + common.options.push_back("transifex/configure=true"); + common.options.push_back("transifex/pull=true"); + } + } + + int release_command::do_run() + { + switch (mode_) { + case modes::devbuild: + return do_devbuild(); + + case modes::official: + return do_official(); + + case modes::none: + default: + u8cerr << "bad release mode " << static_cast(mode_) << "\n"; + throw bailed(); + } + } + + int release_command::do_devbuild() + { + prepare(); + + u8cout + << ">> don't forget to update the version number before making a release\n" + << "\n" + << "creating release for " << version_ << "\n"; + + if (bin_) + make_bin(); + + if (pdbs_) + make_pdbs(); + + if (src_) + make_src(); + + if (uibase_) + make_uibase(); + + if (installer_) + make_installer(); + + return 0; + } + + int release_command::do_official() + { + set_sigint_handler(); + + // make sure the given branch exists in all repos, this avoids failure + // much later on in the process; throws on failure + check_repos_for_branch(); + + // if the prefix exists, asks the user to delete it + if (!check_clean_prefix()) + return 1; + + task_manager::instance().run_all(); + build_command::terminate_msbuild(); + + prepare(); + make_bin(); + make_pdbs(); + make_src(); + make_uibase(); + make_installer(); -namespace mob -{ - -void set_sigint_handler(); - -release_command::release_command() - : command(requires_options) -{ -} - -command::meta_t release_command::meta() const -{ - return - { - "release", - "creates a release" - }; -} - -void release_command::make_bin() -{ - const auto out = out_ / make_filename(""); - u8cout << "making binary archive " << path_to_utf8(out) << "\n"; - - op::archive_from_glob(gcx(), - conf().path().install_bin() / "*", out, {"__pycache__"}); -} - -void release_command::make_pdbs() -{ - const auto out = out_ / make_filename("pdbs"); - u8cout << "making pdbs archive " << path_to_utf8(out) << "\n"; - - op::archive_from_glob(gcx(), - conf().path().install_pdbs() / "*", out, {"__pycache__"}); -} - -void release_command::make_src() -{ - const auto out = out_ / make_filename("src"); - u8cout << "making src archive " << path_to_utf8(out) << "\n"; - - const std::vector ignore = - { - "\\..+", // dot files - ".*\\.log", - ".*\\.tlog", - ".*\\.dll", - ".*\\.exe", - ".*\\.lib", - ".*\\.obj", - ".*\\.ts", - ".*\\.aps", - "vsbuild" - }; - - const std::vector ignore_re(ignore.begin(), ignore.end()); - - std::vector files; - std::size_t total_size = 0; - - if (!fs::exists(tasks::modorganizer::super_path())) - { - gcx().bail_out(context::generic, - "modorganizer super path not found: {}", - tasks::modorganizer::super_path()); - } - - // build list list - walk_dir(tasks::modorganizer::super_path(), files, ignore_re, total_size); - - // should be below 20MB - const std::size_t max_expected_size = 20 * 1024 * 1024; - if (total_size >= max_expected_size) - { - gcx().warning(context::generic, - "total size of source files would be {}, expected something " - "below {}, something might be wrong", - total_size, max_expected_size); - - if (!force_) - { - gcx().bail_out(context::generic, - "bailing out, use --force to ignore"); - } - } - - op::archive_from_files(gcx(), - files, tasks::modorganizer::super_path(), out); -} - -void release_command::make_uibase() -{ - const auto out = out_ / make_filename("uibase"); - u8cout << "making uibase archive " << path_to_utf8(out) << "\n"; - - std::vector files; - - if (!fs::exists(tasks::modorganizer::super_path())) - { - gcx().bail_out(context::generic, - "modorganizer super path not found: {}", - tasks::modorganizer::super_path()); - } - - op::archive_from_glob(gcx(), - tasks::modorganizer::super_path() / "uibase" / "src" / "*.h", out, {}); - op::archive_from_files(gcx(), { conf().path().install_libs() / "uibase.lib" }, conf().path().install_libs(), out); -} - -void release_command::make_installer() -{ - const auto file = "Mod.Organizer-" + version_ + ".exe"; - const auto src = conf().path().install_installer() / file; - const auto dest = out_; - - u8cout << "copying installer " << file << "\n"; - - op::copy_file_to_dir_if_better(gcx(), src, dest); -} - -void release_command::walk_dir( - const fs::path& dir, std::vector& files, - const std::vector& ignore_re, std::size_t& total_size) -{ - // adds all files that are not in the ignore list to `files`, recursive - - for (auto e : fs::directory_iterator(dir)) - { - const auto p = e.path(); - const auto filename = path_to_utf8(p.filename()); - - bool ignored = false; - - for (auto&& re : ignore_re) - { - if (std::regex_match(filename, re)) - { - ignored = true; - break; - } - } - - if (ignored) - continue; - - if (e.is_directory()) - { - walk_dir(e.path(), files, ignore_re, total_size); - } - else if (e.is_regular_file()) - { - total_size += fs::file_size(p); - files.push_back(p); - } - } -} - -fs::path release_command::make_filename(const std::string& what) const -{ - std::string filename = "Mod.Organizer"; - - if (!version_.empty()) - filename += "-" + version_; - - if (!suffix_.empty()) - filename += "-" + suffix_; - - if (!what.empty()) - filename += "-" + what; - - filename += ".7z"; - - return filename; -} - -clipp::group release_command::do_group() -{ - return clipp::group( - clipp::command("release").set(picked_), - - (clipp::option("-h", "--help") >> help_) - % ("shows this message"), - - "devbuild" % - (clipp::command("devbuild").set(mode_, modes::devbuild), - ( - clipp::option("--bin").set(bin_, true) | - clipp::option("--no-bin").set(bin_, false) - ) % "sets whether the binary archive is created [default: yes]", - - ( - clipp::option("--pdbs").set(pdbs_, true) | - clipp::option("--no-pdbs").set(pdbs_, false) - ) % "sets whether the PDBs archive is created [default: yes]", - - ( - clipp::option("--src").set(src_, true) | - clipp::option("--no-src").set(src_, false) - ) % "sets whether the source archive is created [default: yes]", - - //( - // clipp::option("--uibase").set(uibase_, true) | - // clipp::option("--no-uibase").set(uibase_, false) - //) % "sets whether the uibase archive is created [default: yes]", - - ( - clipp::option("--inst").set(installer_, true) | - clipp::option("--no-inst").set(installer_, false) - ) % "sets whether the installer is copied [default: no]", - - clipp::option("--version-from-exe").set(version_exe_) - % "retrieves version information from ModOrganizer.exe " - "[default]", - - clipp::option("--version-from-rc").set(version_rc_) - % "retrieves version information from modorganizer/src/version.rc", - - (clipp::option("--rc") - & clipp::value("PATH") >> utf8_rc_path_) - % "overrides the path to version.rc", - - (clipp::option("--version") - & clipp::value("VERSION") >> version_) - % "overrides the version string", - - (clipp::option("--output-dir") - & clipp::value("PATH") >> utf8out_) - % "sets the output directory to use instead of `$prefix/releases`", - - (clipp::option("--suffix") - & clipp::value("SUFFIX") >> suffix_) - % "optional suffix to add to the archive filenames", - - clipp::option("--force").set(force_) - % "ignores file size warnings and existing release directories" - ) - - | - - "official" % - (clipp::command("official").set(mode_, modes::official), - (clipp::value("branch") >> branch_) - % "use this branch in the super repos" - ) - ); -} - -void release_command::convert_cl_to_conf() -{ - command::convert_cl_to_conf(); - - if (mode_ == modes::official) - { - // force enable translations, installer and tx - - common.options.push_back("task/mo_branch=" + branch_); - common.options.push_back("translations:task/enabled=true"); - common.options.push_back("installer:task/enabled=true"); - - common.options.push_back("transifex/force=true"); - common.options.push_back("transifex/configure=true"); - common.options.push_back("transifex/pull=true"); - } -} - -int release_command::do_run() -{ - switch (mode_) - { - case modes::devbuild: - return do_devbuild(); - - case modes::official: - return do_official(); - - case modes::none: - default: - u8cerr << "bad release mode " << static_cast(mode_) << "\n"; - throw bailed(); - } -} - -int release_command::do_devbuild() -{ - prepare(); - - u8cout - << ">> don't forget to update the version number before making a release\n" - << "\n" - << "creating release for " << version_ << "\n"; - - if (bin_) - make_bin(); - - if (pdbs_) - make_pdbs(); - - if (src_) - make_src(); - - if (uibase_) - make_uibase(); - - if (installer_) - make_installer(); - - return 0; -} - -int release_command::do_official() -{ - set_sigint_handler(); - - // make sure the given branch exists in all repos, this avoids failure - // much later on in the process; throws on failure - check_repos_for_branch(); - - // if the prefix exists, asks the user to delete it - if (!check_clean_prefix()) - return 1; - - task_manager::instance().run_all(); - build_command::terminate_msbuild(); - - prepare(); - make_bin(); - make_pdbs(); - make_src(); - make_uibase(); - make_installer(); - - return 0; -} - -void release_command::check_repos_for_branch() -{ - u8cout << "checking repos for branch " << branch_ << "...\n"; - - thread_pool tp; - std::atomic failed = false; - - for (const auto* t : task_manager::instance().find("super")) - { - if (!t->enabled()) - continue; - - tp.add([this, t, &failed] - { - const auto* o = dynamic_cast(t); - - if (!git_wrap::remote_branch_exists(o->git_url(), branch_)) - { - gcx().error(context::generic, - "branch {} doesn't exist in the {} repo", - branch_, o->name()); - - failed = true; - } - }); - } - - tp.join(); - - if (failed) - { - gcx().bail_out(context::generic, - "either fix the branch name, create a remote branch for the " - "repos that don't have it, or disable tasks with " - "`-s TASKNAME:task/enabled=false`"); - } -} - -bool release_command::check_clean_prefix() -{ - const auto prefix = conf().path().prefix(); - - if (!fs::exists(prefix)) - return true; - - bool saw_file = false; - const fs::path log_file = conf().global().get("log_file"); - const std::string ini_file = default_ini_filename(); - - for (auto itor : fs::directory_iterator(prefix)) - { - const auto name = itor.path().filename(); - - // ignore ini and logs - if (name == log_file.filename() || name == ini_file) - continue; - - saw_file = true; - break; - } - - if (!saw_file) - { - // empty directory, that's fine - return true; - } - - const auto q = fmt::format( - "prefix {} already exists, delete?", path_to_utf8(prefix)); - - if (ask_yes_no(q, yn::no) != yn::yes) - return false; - - // the log file might be in this directory, close it now and reopen it - // when deletion is finished - context::close_log_file(); - - build_command::terminate_msbuild(); - op::delete_directory(gcx(), prefix); - - // reopen log file - conf().set_log_file(); - - return true; -} - -void release_command::prepare() -{ - // finding rc file - rc_path_ = fs::path(utf8_to_utf16(utf8_rc_path_)); - if (rc_path_.empty()) - { - rc_path_ = - tasks::modorganizer::super_path() - / "modorganizer" - / "src" - / "version.rc"; - } - - // getting version from rc or exe - if (version_.empty()) - { - if (version_rc_) - version_ = version_from_rc(); - else - version_ = version_from_exe(); - } - - // finding output path - const auto prefix = conf().path().prefix(); - out_ = fs::path(utf8_to_utf16(utf8out_)); - - if (out_.empty()) - out_ = prefix / "releases" / version_; - else if (out_.is_relative()) - out_ = prefix / out_; -} - -std::string release_command::do_doc() -{ - return - "Creates archives for an MO installation, PDBs and sources.\n" - "\n" - "Commands:\n" - "devbuild\n" - " Can creates three archives in `$prefix/releases/version`: one from\n" - " `install/bin/*`, one from `install/pdbs/*` and another with the\n" - " sources of projects from modorganizer_super.\n" - " \n" - " The archive filename is `Mod.Organizer-version-suffix-what.7z`,\n" - " where:\n" - " - `version` is taken from `ModOrganizer.exe`, `version.rc`\n" - " or from --version;\n" - " - `suffix` is the optional `--suffix` argument;\n" - " - `what` is either nothing, `src` or `pdbs`.\n" - "\n" - "official\n" - " Creates a new full build in the prefix. Requires that directory\n" - " to be empty. Puts the binary archive, source, PDBs and installer\n" - " in `$prefix/releases/version`. Forces all tasks to be enabled,\n" - " including translations and installer. Make sure the transifex API\n" - " key is in the INI or TX_TOKEN is set."; -} - -std::string release_command::version_from_exe() const -{ - const auto exe = conf().path().install_bin() / "ModOrganizer.exe"; - - // getting version info size - DWORD dummy = 0; - const DWORD size = GetFileVersionInfoSizeW(exe.native().c_str(), &dummy); - - if (size == 0) - { - const auto e = GetLastError(); - gcx().bail_out(context::generic, - "can't get file version info size from {}, {}", - exe, error_message(e)); - } - - // getting version info - auto buffer = std::make_unique(size); - - if (!GetFileVersionInfoW(exe.native().c_str(), 0, size, buffer.get())) - { - const auto e = GetLastError(); - gcx().bail_out(context::generic, - "can't get file version info from {}, {}", - exe, error_message(e)); - } - - struct LANGANDCODEPAGE - { - WORD wLanguage; - WORD wCodePage; - }; - - void* value_pointer = nullptr; - unsigned int value_size = 0; - - // getting list of available languages - auto ret = VerQueryValueW( - buffer.get(), L"\\VarFileInfo\\Translation", - &value_pointer, &value_size); - - if (!ret || !value_pointer || value_size == 0) - { - const auto e = GetLastError(); - gcx().bail_out(context::generic, - "VerQueryValueW() for translations failed on {}, {}", - exe, error_message(e)); - } - - // number of languages - const auto count = value_size / sizeof(LANGANDCODEPAGE); - if (count == 0) - gcx().bail_out(context::generic, "no languages found in {}", exe); - - // using the first language in the list to get FileVersion - const auto* lcp = static_cast(value_pointer); - - const auto sub_block = fmt::format( - L"\\StringFileInfo\\{:04x}{:04x}\\FileVersion", - lcp->wLanguage, lcp->wCodePage); - - ret = VerQueryValueW( - buffer.get(), sub_block.c_str(), &value_pointer, &value_size); - - if (!ret || !value_pointer || value_size == 0) - { - gcx().bail_out(context::generic, - "language {} not found in {}", sub_block, exe); - } - - // value_size includes the null terminator - return utf16_to_utf8(std::wstring( - static_cast(value_pointer), - value_size - 1)); -} - -std::string release_command::version_from_rc() const -{ - // matching: #define VER_FILEVERSION_STR "2.2.1\0" - std::regex re(R"(#define VER_FILEVERSION_STR "(.+)\\0")"); - - const std::string rc = op::read_text_file( - gcx(), encodings::utf8, rc_path_); - - std::smatch m; - std::string v; - - for_each_line(rc, [&](std::string_view line) - { - std::string line_s(line); - if (std::regex_match(line_s, m, re)) - v = m[1]; - }); - - if (v.empty()) - { - gcx().bail_out(context::generic, - "can't find version string in {}", rc_path_); - } - - return v; -} - -} // namespace + return 0; + } + + void release_command::check_repos_for_branch() + { + u8cout << "checking repos for branch " << branch_ << "...\n"; + + thread_pool tp; + std::atomic failed = false; + + for (const auto* t : task_manager::instance().find("super")) { + if (!t->enabled()) + continue; + + tp.add([this, t, &failed] { + const auto* o = dynamic_cast(t); + + if (!git_wrap::remote_branch_exists(o->git_url(), branch_)) { + gcx().error(context::generic, + "branch {} doesn't exist in the {} repo", branch_, + o->name()); + + failed = true; + } + }); + } + + tp.join(); + + if (failed) { + gcx().bail_out(context::generic, + "either fix the branch name, create a remote branch for the " + "repos that don't have it, or disable tasks with " + "`-s TASKNAME:task/enabled=false`"); + } + } + + bool release_command::check_clean_prefix() + { + const auto prefix = conf().path().prefix(); + + if (!fs::exists(prefix)) + return true; + + bool saw_file = false; + const fs::path log_file = conf().global().get("log_file"); + const std::string ini_file = default_ini_filename(); + + for (auto itor : fs::directory_iterator(prefix)) { + const auto name = itor.path().filename(); + + // ignore ini and logs + if (name == log_file.filename() || name == ini_file) + continue; + + saw_file = true; + break; + } + + if (!saw_file) { + // empty directory, that's fine + return true; + } + + const auto q = + fmt::format("prefix {} already exists, delete?", path_to_utf8(prefix)); + + if (ask_yes_no(q, yn::no) != yn::yes) + return false; + + // the log file might be in this directory, close it now and reopen it + // when deletion is finished + context::close_log_file(); + + build_command::terminate_msbuild(); + op::delete_directory(gcx(), prefix); + + // reopen log file + conf().set_log_file(); + + return true; + } + + void release_command::prepare() + { + // finding rc file + rc_path_ = fs::path(utf8_to_utf16(utf8_rc_path_)); + if (rc_path_.empty()) { + rc_path_ = tasks::modorganizer::super_path() / "modorganizer" / "src" / + "version.rc"; + } + + // getting version from rc or exe + if (version_.empty()) { + if (version_rc_) + version_ = version_from_rc(); + else + version_ = version_from_exe(); + } + + // finding output path + const auto prefix = conf().path().prefix(); + out_ = fs::path(utf8_to_utf16(utf8out_)); + + if (out_.empty()) + out_ = prefix / "releases" / version_; + else if (out_.is_relative()) + out_ = prefix / out_; + } + + std::string release_command::do_doc() + { + return "Creates archives for an MO installation, PDBs and sources.\n" + "\n" + "Commands:\n" + "devbuild\n" + " Can creates three archives in `$prefix/releases/version`: one from\n" + " `install/bin/*`, one from `install/pdbs/*` and another with the\n" + " sources of projects from modorganizer_super.\n" + " \n" + " The archive filename is `Mod.Organizer-version-suffix-what.7z`,\n" + " where:\n" + " - `version` is taken from `ModOrganizer.exe`, `version.rc`\n" + " or from --version;\n" + " - `suffix` is the optional `--suffix` argument;\n" + " - `what` is either nothing, `src` or `pdbs`.\n" + "\n" + "official\n" + " Creates a new full build in the prefix. Requires that directory\n" + " to be empty. Puts the binary archive, source, PDBs and installer\n" + " in `$prefix/releases/version`. Forces all tasks to be enabled,\n" + " including translations and installer. Make sure the transifex API\n" + " key is in the INI or TX_TOKEN is set."; + } + + std::string release_command::version_from_exe() const + { + const auto exe = conf().path().install_bin() / "ModOrganizer.exe"; + + // getting version info size + DWORD dummy = 0; + const DWORD size = GetFileVersionInfoSizeW(exe.native().c_str(), &dummy); + + if (size == 0) { + const auto e = GetLastError(); + gcx().bail_out(context::generic, + "can't get file version info size from {}, {}", exe, + error_message(e)); + } + + // getting version info + auto buffer = std::make_unique(size); + + if (!GetFileVersionInfoW(exe.native().c_str(), 0, size, buffer.get())) { + const auto e = GetLastError(); + gcx().bail_out(context::generic, "can't get file version info from {}, {}", + exe, error_message(e)); + } + + struct LANGANDCODEPAGE { + WORD wLanguage; + WORD wCodePage; + }; + + void* value_pointer = nullptr; + unsigned int value_size = 0; + + // getting list of available languages + auto ret = VerQueryValueW(buffer.get(), L"\\VarFileInfo\\Translation", + &value_pointer, &value_size); + + if (!ret || !value_pointer || value_size == 0) { + const auto e = GetLastError(); + gcx().bail_out(context::generic, + "VerQueryValueW() for translations failed on {}, {}", exe, + error_message(e)); + } + + // number of languages + const auto count = value_size / sizeof(LANGANDCODEPAGE); + if (count == 0) + gcx().bail_out(context::generic, "no languages found in {}", exe); + + // using the first language in the list to get FileVersion + const auto* lcp = static_cast(value_pointer); + + const auto sub_block = + fmt::format(L"\\StringFileInfo\\{:04x}{:04x}\\FileVersion", lcp->wLanguage, + lcp->wCodePage); + + ret = VerQueryValueW(buffer.get(), sub_block.c_str(), &value_pointer, + &value_size); + + if (!ret || !value_pointer || value_size == 0) { + gcx().bail_out(context::generic, "language {} not found in {}", sub_block, + exe); + } + + // value_size includes the null terminator + return utf16_to_utf8( + std::wstring(static_cast(value_pointer), value_size - 1)); + } + + std::string release_command::version_from_rc() const + { + // matching: #define VER_FILEVERSION_STR "2.2.1\0" + std::regex re(R"(#define VER_FILEVERSION_STR "(.+)\\0")"); + + const std::string rc = op::read_text_file(gcx(), encodings::utf8, rc_path_); + + std::smatch m; + std::string v; + + for_each_line(rc, [&](std::string_view line) { + std::string line_s(line); + if (std::regex_match(line_s, m, re)) + v = m[1]; + }); + + if (v.empty()) { + gcx().bail_out(context::generic, "can't find version string in {}", + rc_path_); + } + + return v; + } + +} // namespace mob diff --git a/src/cmd/tx.cpp b/src/cmd/tx.cpp index a4f1be9..1ab97d3 100644 --- a/src/cmd/tx.cpp +++ b/src/cmd/tx.cpp @@ -1,213 +1,188 @@ #include "pch.h" -#include "commands.h" -#include "../net.h" -#include "../utility.h" #include "../core/conf.h" #include "../core/context.h" #include "../core/env.h" +#include "../net.h" #include "../tasks/tasks.h" +#include "../utility.h" +#include "commands.h" + +namespace mob { + + tx_command::tx_command() : command(requires_options) {} + + command::meta_t tx_command::meta() const + { + return {"tx", "manages transifex translations"}; + } + + clipp::group tx_command::do_group() + { + return clipp::group( + clipp::command("tx").set(picked_), + + (clipp::option("-h", "--help") >> help_) % ("shows this message"), + + "get" % (clipp::command("get").set(mode_, modes::get), + (clipp::option("-k", "--key") & clipp::value("APIKEY") >> key_) % + "API key", + + (clipp::option("-t", "--team") & clipp::value("TEAM") >> team_) % + "team name", + + (clipp::option("-p", "--project") & + clipp::value("PROJECT") >> project_) % + "project name", + + (clipp::option("-u", "--url") & clipp::value("URL") >> url_) % + "project URL", + + (clipp::option("-m", "--minimum") & + clipp::value("PERCENT").set(min_)) % + "minimum translation threshold to download [0-100]", + + (clipp::option("-f", "--force").call([&] { + force_ = true; + })) % + "don't check timestamps, re-download all translation files", + + (clipp::value("path") >> path_) % + "path that will contain the .tx directory") -namespace mob -{ + | + + "build" % (clipp::command("build").set(mode_, modes::build), + + (clipp::value("source") >> path_) % + "path that contains the translation directories", + + (clipp::value("destination") >> dest_) % + "path that will contain the .qm files")); + } + + void tx_command::convert_cl_to_conf() + { + command::convert_cl_to_conf(); + + if (!key_.empty()) + common.options.push_back("transifex/key=" + key_); + + if (!team_.empty()) + common.options.push_back("transifex/team=" + team_); + + if (!project_.empty()) + common.options.push_back("transifex/project=" + project_); -tx_command::tx_command() - : command(requires_options) -{ -} - -command::meta_t tx_command::meta() const -{ - return - { - "tx", - "manages transifex translations" - }; -} - -clipp::group tx_command::do_group() -{ - return clipp::group( - clipp::command("tx").set(picked_), - - (clipp::option("-h", "--help") >> help_) - % ("shows this message"), - - "get" % - (clipp::command("get").set(mode_, modes::get), - (clipp::option("-k", "--key") - & clipp::value("APIKEY") >> key_) - % "API key", - - (clipp::option("-t", "--team") - & clipp::value("TEAM") >> team_) - % "team name", - - (clipp::option("-p", "--project") - & clipp::value("PROJECT") >> project_) - % "project name", - - (clipp::option("-u", "--url") - & clipp::value("URL") >> url_) - % "project URL", - - (clipp::option("-m", "--minimum") - & clipp::value("PERCENT").set(min_)) - % "minimum translation threshold to download [0-100]", - - (clipp::option("-f", "--force").call([&]{ force_ = true; })) - % "don't check timestamps, re-download all translation files", - - (clipp::value("path") >> path_) - % "path that will contain the .tx directory" - ) - - | - - "build" % - (clipp::command("build").set(mode_, modes::build), - - (clipp::value("source") >> path_) - % "path that contains the translation directories", - - (clipp::value("destination") >> dest_) - % "path that will contain the .qm files" - ) - ); -} - -void tx_command::convert_cl_to_conf() -{ - command::convert_cl_to_conf(); - - if (!key_.empty()) - common.options.push_back("transifex/key=" + key_); - - if (!team_.empty()) - common.options.push_back("transifex/team=" + team_); - - if (!project_.empty()) - common.options.push_back("transifex/project=" + project_); - - if (!url_.empty()) - common.options.push_back("transifex/url=" + url_); - - if (min_ >= 0) - common.options.push_back("transifex/minimum=" + std::to_string(min_)); - - if (force_) - common.options.push_back("transifex/force=" + std::to_string(*force_)); -} - -int tx_command::do_run() -{ - switch (mode_) - { - case modes::get: - do_get(); - break; - - case modes::build: - do_build(); - break; - - case modes::none: - default: - u8cerr << "bad tx mode " << static_cast(mode_) << "\n"; - throw bailed(); - } - - return 0; -} - -std::string tx_command::do_doc() -{ - return - "Some values will be taken from the INI file if not specified.\n" - "\n" - "Commands:\n" - "get\n" - " Initializes a Transifex project in the given directory if\n" - " necessary and pulls all the translation files.\n" - "\n" - "build\n" - " Builds all .qm files. The path can either be the transifex\n" - " project (where .tx is) or the `translations` directory (where the\n" - " individual translation directories are)."; -} - -void tx_command::do_get() -{ - const url u = - conf().transifex().get("url") + "/" + - conf().transifex().get("team") + "/" + - conf().transifex().get("project"); - - const std::string key = conf().transifex().get("key"); - - if (key.empty() && !this_env::get_opt("TX_TOKEN")) - { - u8cout << - "(no key was in the INI, --key wasn't given and TX_TOKEN env\n" - "variable doesn't exist, this will probably fail)\n\n"; - } - - // copy the global context, the tools will modify it - context cxcopy = gcx(); - - u8cout << "initializing\n"; - transifex(transifex::init) - .root(path_) - .run(cxcopy); - - u8cout << "configuring\n"; - transifex(transifex::config) - .stdout_level(context::level::info) - .root(path_) - .api_key(key) - .url(u) - .run(cxcopy); - - u8cout << "pulling\n"; - transifex(transifex::pull) - .stdout_level(context::level::info) - .root(path_) - .api_key(key) - .minimum(conf().transifex().get("minimum")) - .force(conf().transifex().get("force")) - .run(cxcopy); -} - -void tx_command::do_build() -{ - fs::path root = path_; - if (fs::exists(root / ".tx") && fs::exists(root / "translations")) - root = root / "translations"; - - tasks::translations::projects ps(root); - - fs::path dest = dest_; - op::create_directories(gcx(), dest, op::unsafe); - - for (auto&& w : ps.warnings()) - u8cerr << w << "\n"; - - thread_pool tp; - - for (auto& p : ps.get()) - { - for (auto& lg : p.langs) - { - // copy the global context, each thread must have its own - tp.add([&, cxcopy=gcx()]() mutable - { - lrelease() - .project(p.name) - .sources(lg.ts_files) - .out(dest) - .run(cxcopy); - }); - } - } -} - -} // namespace + if (!url_.empty()) + common.options.push_back("transifex/url=" + url_); + + if (min_ >= 0) + common.options.push_back("transifex/minimum=" + std::to_string(min_)); + + if (force_) + common.options.push_back("transifex/force=" + std::to_string(*force_)); + } + + int tx_command::do_run() + { + switch (mode_) { + case modes::get: + do_get(); + break; + + case modes::build: + do_build(); + break; + + case modes::none: + default: + u8cerr << "bad tx mode " << static_cast(mode_) << "\n"; + throw bailed(); + } + + return 0; + } + + std::string tx_command::do_doc() + { + return "Some values will be taken from the INI file if not specified.\n" + "\n" + "Commands:\n" + "get\n" + " Initializes a Transifex project in the given directory if\n" + " necessary and pulls all the translation files.\n" + "\n" + "build\n" + " Builds all .qm files. The path can either be the transifex\n" + " project (where .tx is) or the `translations` directory (where the\n" + " individual translation directories are)."; + } + + void tx_command::do_get() + { + const url u = conf().transifex().get("url") + "/" + + conf().transifex().get("team") + "/" + + conf().transifex().get("project"); + + const std::string key = conf().transifex().get("key"); + + if (key.empty() && !this_env::get_opt("TX_TOKEN")) { + u8cout << "(no key was in the INI, --key wasn't given and TX_TOKEN env\n" + "variable doesn't exist, this will probably fail)\n\n"; + } + + // copy the global context, the tools will modify it + context cxcopy = gcx(); + + u8cout << "initializing\n"; + transifex(transifex::init).root(path_).run(cxcopy); + + u8cout << "configuring\n"; + transifex(transifex::config) + .stdout_level(context::level::info) + .root(path_) + .api_key(key) + .url(u) + .run(cxcopy); + + u8cout << "pulling\n"; + transifex(transifex::pull) + .stdout_level(context::level::info) + .root(path_) + .api_key(key) + .minimum(conf().transifex().get("minimum")) + .force(conf().transifex().get("force")) + .run(cxcopy); + } + + void tx_command::do_build() + { + fs::path root = path_; + if (fs::exists(root / ".tx") && fs::exists(root / "translations")) + root = root / "translations"; + + tasks::translations::projects ps(root); + + fs::path dest = dest_; + op::create_directories(gcx(), dest, op::unsafe); + + for (auto&& w : ps.warnings()) + u8cerr << w << "\n"; + + thread_pool tp; + + for (auto& p : ps.get()) { + for (auto& lg : p.langs) { + // copy the global context, each thread must have its own + tp.add([&, cxcopy = gcx()]() mutable { + lrelease() + .project(p.name) + .sources(lg.ts_files) + .out(dest) + .run(cxcopy); + }); + } + } + } + +} // namespace mob diff --git a/src/core/conf.cpp b/src/core/conf.cpp index be9bc4e..1f99b02 100644 --- a/src/core/conf.cpp +++ b/src/core/conf.cpp @@ -1,805 +1,727 @@ #include "pch.h" #include "conf.h" +#include "../tasks/task.h" +#include "../tasks/task_manager.h" +#include "../tools/tools.h" +#include "../utility.h" #include "context.h" #include "env.h" #include "ini.h" #include "paths.h" -#include "../utility.h" -#include "../tasks/task.h" -#include "../tasks/task_manager.h" -#include "../tools/tools.h" -namespace mob::details -{ - -using key_value_map = std::map>; -using section_map = std::map>; - -// holds all the options not related to tasks (global, tools, paths, etc.) -static section_map g_conf; - -// holds all the task options; has a special map element with an empty string -// for options that apply to all tasks, and elements with specific task names -// for overrides -static section_map g_tasks; - -// special cases to avoid string manipulations -static int g_output_log_level = 3; -static int g_file_log_level = 5; -static bool g_dry = false; - -bool bool_from_string(std::string_view s) -{ - return (s == "true" || s == "yes" || s == "1"); -} - -// returns a string from conf, bails out if it doesn't exist -// -std::string get_string(std::string_view section, std::string_view key) -{ - auto sitor = g_conf.find(section); - if (sitor == g_conf.end()) - gcx().bail_out(context::conf, "[{}] doesn't exist", section); - - auto kitor = sitor->second.find(key); - if (kitor == sitor->second.end()) - gcx().bail_out(context::conf, "no key '{}' in [{}]", key, section); - - return kitor->second; -} - -// calls get_string(), converts to int -// -int get_int(std::string_view section, std::string_view key) -{ - const auto s = get_string(section, key); - - try - { - return std::stoi(s); - } - catch(std::exception&) - { - gcx().bail_out(context::conf, "bad int for {}/{}", section, key); - } -} - -// calls get_string(), converts to bool -// -bool get_bool(std::string_view section, std::string_view key) -{ - const auto s = get_string(section, key); - return bool_from_string(s); -} - -// sets the given option, bails out if the option doesn't exist -// -void set_string(std::string_view section, std::string_view key, std::string_view value) -{ - auto sitor = g_conf.find(section); - if (sitor == g_conf.end()) - gcx().bail_out(context::conf, "[{}] doesn't exist", section); - - auto kitor = sitor->second.find(key); - if (kitor == sitor->second.end()) - gcx().bail_out(context::conf, "no key '{}' [{}]", key, section); - - kitor->second = value; -} - -// sets the given option, adds it if it doesn't exist; used when setting options -// from the master ini -// -void add_string(const std::string& section, const std::string& key, std::string value) -{ - g_conf[section][key] = value; -} - - -// finds an option for the given task, returns empty if not found -// -std::optional find_string_for_task( - std::string_view task_name, std::string_view key) -{ - // find task - auto titor = g_tasks.find(task_name); - if (titor == g_tasks.end()) - return {}; - - const auto& task = titor->second; - - // find key - auto itor = task.find(key); - if (itor == task.end()) - return {}; - - return itor->second; -} - -// gets an option for any of the given task names, typically what task::names() -// returns, which contains the main task name plus some alternate names -// -// there's a hierarchy for task options: -// -// 1) there's a special "_override" entry in g_tasks, for options set from -// the command line that should override everything, like --no-pull -// should override all pull settings for all tasks -// -// 2) if the key is not found in "_override", then there can be an entry -// in g_tasks with any of given task names -// -// 3) if the key doesn't exist, then use the generic task option for it, stored -// in an element with an empty string in g_tasks -// -std::string get_string_for_task( - const std::vector& task_names, std::string_view key) -{ - // some command line options will override any user settings, like - // --no-pull, those are stored in a special _override task name - auto v = find_string_for_task("_override", key); - if (v) - return *v; - - // look for an option for this task by name - for (auto&& tn : task_names) - { - v = find_string_for_task(tn, key); - if (v) - return *v; - } - - // default task options are in a special empty string entry in g_tasks - v = find_string_for_task("", key); - if (v) - return *v; - - // doesn't exist anywhere - gcx().bail_out(context::conf, - "no task option '{}' found for any of {}", - key, join(task_names, ",")); -} - -// calls get_string_for_task(), converts to bool -// -bool get_bool_for_task( - const std::vector& task_names, std::string_view key) -{ - const std::string s = get_string_for_task(task_names, key); - return bool_from_string(s); -} - -// sets the given task option, bails out if the option doesn't exist -// -void set_string_for_task( - const std::string& task_name, const std::string& key, std::string value) -{ - // make sure the key exists, will throw if it doesn't - get_string_for_task({task_name}, key); - - g_tasks[task_name][key] = std::move(value); -} - -// sets the given task option, adds it if it doesn't exist; used when setting -// options from the master ini -// -void add_string_for_task( - const std::string& task_name, const std::string& key, std::string value) -{ - g_tasks[task_name][key] = std::move(value); -} - -} // namespace - - -namespace mob -{ - -std::vector format_options() -{ - // don't log private stuff - auto hide = [](std::string_view section, std::string_view key) - { - if (key == "github_key") - return true; - - if (section == "transifex" && key == "key") - return true; - - if (key == "git_email") - return true; - - return false; - }; - - auto make_value = [&](auto&& s, auto&& k, auto&& v) -> std::string - { - if (hide(s, k)) - return v.empty() ? "" : "(hidden)"; - else - return v; - }; - - - auto& tm = task_manager::instance(); - - std::size_t longest_what = 0; - std::size_t longest_key = 0; - - for (auto&& [section, kvs] : details::g_conf) - { - longest_what = std::max(longest_what, section.size()); - - for (auto&& [k, v] : kvs) - longest_key = std::max(longest_key, k.size()); - } - - for (auto&& [k, v] : details::g_tasks[""]) - longest_key = std::max(longest_key, k.size()); - - for (const auto* task : tm.all()) - longest_what = std::max(longest_what, task->name().size()); - - std::vector lines; - - lines.push_back( - pad_right("what", longest_what) + " " + - pad_right("key",longest_key) + " " + - "value"); - - lines.push_back( - pad_right("-", longest_what, '-') + " " + - pad_right("-",longest_key, '-') + " " + - "-----"); - - for (auto&& [section, kvs] : details::g_conf) - { - for (auto&& [k, v] : kvs) - { - lines.push_back( - pad_right(section, longest_what) + " " + - pad_right(k, longest_key) + " = " + make_value(section, k, v)); - } - } - - for (auto&& [k, v] : details::g_tasks[""]) - { - lines.push_back( - pad_right("task", longest_what) + " " + - pad_right(k, longest_key) + " = " + make_value("", k, v)); - } - - for (const auto* t : tm.all()) - { - for (auto&& [k, unused] : details::g_tasks[""]) - { - lines.push_back( - pad_right(t->name(), longest_what) + " " + - pad_right(k, longest_key) + " = " + - make_value("", k, details::get_string_for_task({t->name()}, k))); - } - } - - return lines; -} - -// sets commonly used options that need to be converted to int/bool, for -// performance -// -void set_special_options() -{ - details::g_output_log_level = details::get_int("global", "output_log_level"); - details::g_file_log_level = details::get_int("global", "file_log_level"); - details::g_dry = details::get_bool("global", "dry"); -} - -// sets an option `key` in the `paths` section; if the path is currently empty, -// sets it using `f` (which is either a callable or a string) -// -// in any case, makes it absolute and canonical, bails out if the path does not -// exist -// -// this is used for paths that should already exist (qt, vs, etc.) -// -template -void set_path_if_empty(std::string_view key, F&& f) -{ - // current value - fs::path p = conf().path().get(key); - - if (p.empty()) - { - // empty, set it from `f` - if constexpr (std::is_same_v>) - p = f; - else - p = f(); - } - - p = fs::absolute(p); - - if (!conf().global().dry()) - { - if (!fs::exists(p)) - gcx().bail_out(context::conf, "path {} not found", p); - - p = fs::canonical(p); - } - - // new value - details::set_string("paths", key, path_to_utf8(p)); -} - -// sets an option `key` in the `paths` section: -// - if the path is empty, sets it as default_parent/default_dir, -// - if the path is not empty but is relative, resolves it against -// default_parent -// -// in any case, makes it absolute but weakly canonical since it might not exist -// at that point (this is used for build, install, etc.) -// -void resolve_path( - std::string_view key, - const fs::path& default_parent, std::string_view default_dir) -{ - // current value - fs::path p = conf().path().get(key); - - if (p.empty()) - { - p = default_parent / default_dir; - } - else - { - if (p.is_relative()) - p = default_parent / p; - } - - if (!conf().global().dry()) - p = fs::weakly_canonical(fs::absolute(p)); - - details::set_string("paths", key, path_to_utf8(p)); -} - -// `section_string` can be something like "global" or "paths", but also "task" -// or a task-specific name like "uibase:task" -// -// `master` is true if the ini being processed is the master ini so options are -// added to the maps instead of set, which throws if they're not found -// -void process_option( - const std::string& section_string, - const std::string& key, const std::string& value, bool master) -{ - // split section string on ":" - const auto col = section_string.find(":"); - std::string task, section; - - if (col == std::string::npos) - { - // not a "task_name:task" section - section = section_string; - } - else - { - // that's a "task_name:task" section - task = section_string.substr(0, col); - section = section_string.substr(col + 1); - } - - if (section == "task") - { - // task options go in g_tasks - - if (task == "_override") - { - // special case, comes from options on the command line - details::set_string_for_task("_override", key, value); - } - else if (task != "") - { - // task specific - - // task must exist - const auto& tasks = task_manager::instance().find(task); - - if (tasks.empty()) - { - gcx().bail_out(context::conf, - "bad option {}, task '{}' not found", section_string, task); - } - - MOB_ASSERT(!tasks.empty()); - - for (auto& t : tasks) - details::set_string_for_task(t->name(), key, value); - } - else - { - // global task option - - if (master) - details::add_string_for_task("", key, value); - else - details::set_string_for_task("", key, value); - } - } - else - { - // not a task option, goes into g_conf - - if (master) - details::add_string(section, key, value); - else - details::set_string(section, key, value); - } -} - -// reads the given ini and adds all of its content to the options -// -void process_ini(const fs::path& ini, bool master) -{ - const auto data = parse_ini(ini); - - for (auto&& a : data.aliases) - task_manager::instance().add_alias(a.first, a.second); - - for (auto&& [section_string, kvs] : data.sections) - { - for (auto&& [k, v] : kvs) - process_option(section_string, k, v, master); - } -} - -// parses the given option strings and adds them as options -// -void process_cmd_options(const std::vector& opts) -{ - // parses "section/key=value" - static std::regex re(R"((.+)/(.+)=(.*))"); - - gcx().debug(context::conf, "overriding from command line:"); - - for (auto&& o : opts) - { - std::smatch m; - if (!std::regex_match(o, m, re)) - { - gcx().bail_out(context::conf, - "bad option {}, must be [task:]section/key=value", o); - } - - process_option(m[1], m[2], m[3], false); - } -} - -// goes through all the options that have to do with paths, checks them and -// resolves them if necessary -// -void resolve_paths() -{ - // first, if any of these paths are empty, they are set using the second - // argument, which can be callable or a path - // - // the resulting path is made absolute and canonical and will bail out if it - // doesn't exist - - // make sure third-party is in PATH before the other paths are checked - // because some of these paths will need to look in there to find stuff - set_path_if_empty("third_party", find_third_party_directory); - this_env::prepend_to_path(conf().path().third_party() / "bin"); - - set_path_if_empty("pf_x86", find_program_files_x86); - set_path_if_empty("pf_x64", find_program_files_x64); - set_path_if_empty("vs", find_vs); - set_path_if_empty("qt_install", find_qt); - set_path_if_empty("temp_dir", find_temp_dir); - set_path_if_empty("patches", find_in_root("patches")); - set_path_if_empty("licenses", find_in_root("licenses")); - set_path_if_empty("qt_bin", qt::installation_path() / "bin"); - set_path_if_empty("qt_translations", qt::installation_path() / "translations"); - - // second, if any of these paths are relative, they use the second argument - // as the root; if they're empty, they combine the second and third - // arguments - // - // these paths might not exist yet, so they're only made weakly canonical, - // they'll be created as needed during the build process - - const auto p = conf().path(); - - resolve_path("cache", p.prefix(), "downloads"); - resolve_path("build", p.prefix(), "build"); - resolve_path("install", p.prefix(), "install"); - resolve_path("install_installer", p.install(), "installer"); - resolve_path("install_bin", p.install(), "bin"); - resolve_path("install_libs", p.install(), "libs"); - resolve_path("install_pdbs", p.install(), "pdb"); - resolve_path("install_dlls", p.install_bin(), "dlls"); - resolve_path("install_loot", p.install_bin(), "loot"); - resolve_path("install_plugins", p.install_bin(), "plugins"); - resolve_path("install_licenses", p.install_bin(), "licenses"); - resolve_path("install_pythoncore", p.install_bin(), "pythoncore"); - resolve_path("install_stylesheets", p.install_bin(), "stylesheets"); - resolve_path("install_translations", p.install_bin(), "translations"); - - // finally, resolve the tools that are unlikely to be in PATH; all the - // other tools (7z, jom, patch, etc.) are assumed to be in PATH (which - // now contains third-party) or have valid absolute paths in the ini - - details::set_string("tools", "vcvars", path_to_utf8(find_vcvars())); - details::set_string("tools", "iscc", path_to_utf8(find_iscc())); -} - -void conf::set_log_file() -{ - // set up the log file, resolve against prefix if relative - fs::path log_file = conf().global().get("log_file"); - if (log_file.is_relative()) - log_file = conf().path().prefix() / log_file; - - context::set_log_file(log_file); -} - -void init_options( - const std::vector& inis, const std::vector& opts) -{ - MOB_ASSERT(!inis.empty()); - - // some logging - gcx().debug(context::conf, "cl: {}", std::wstring(GetCommandLineW())); - gcx().debug(context::conf, "using inis in order:"); - for (auto&& ini : inis) - gcx().debug(context::conf, " . {}", ini); - - - // used to resolve a relative prefix; by default, it's resolved against cwd, - // but if an ini other than the master contains a prefix, use the ini's - // parent directory instead - fs::path prefix_root = fs::current_path(); - - // true for the first ini, will add values to the configuration maps instead - // of setting them, which throws if the option doesn't exist - // - // the goal is that the first, master ini contains all existing options and - // if an option set in another ini or on the command line doesn't exist in - // the master, it's an error - bool master = true; - - for (auto&& ini : inis) - { - fs::path prefix_before; - - // if this is the master ini, the prefix doesn't exist in the config - // yet because no inis have been loaded - if (!master) - prefix_before = conf().path().prefix(); - - process_ini(ini, master); - - // check if the prefix was changed by this ini - if (!master && conf().path().prefix() != prefix_before) - { - // remember its path - prefix_root = ini.parent_path(); - } - - // further inis should only contain options that already exist - master = false; - } - - - if (!opts.empty()) - { - const fs::path prefix_before = conf().path().prefix(); - - process_cmd_options(opts); - - // check if the prefix was changed on the command line - if (conf().path().prefix() != prefix_before) - { - // use cwd as the parent of a relative prefix - prefix_root = fs::current_path(); - } - } - - // converts some options to ints or bools, these are used everywhere, like - // the log levels - set_special_options(); - - // an empty prefix is an error and will fail in validate_options(), but - // don't check it here to allow some commands to run, like `mob options`, - // and make sure it's not set to something that's not empty to make sure it - // _does_ fail later on - if (!conf().path().prefix().empty()) - resolve_path("prefix", prefix_root, ""); - - // set up the log file, resolve against prefix if relative - conf().set_log_file(); - - // goes through all paths and tools, finds missing or relative stuff, bails - // out of stuff can't be found - resolve_paths(); - - // make sure qt's bin directory is in the path - this_env::append_to_path(conf().path().get("qt_bin")); -} - -bool verify_options() -{ - // can't have an empty prefix - if (conf().path().prefix().empty()) - { - u8cerr - << "missing prefix; either specify it the [paths] section of " - << "the ini or pass '-d path'\n"; - - return false; - } - - // don't build mo inside mob - if (fs::exists(conf().path().prefix())) - { - if (fs::equivalent(conf().path().prefix(), mob_exe_path().parent_path())) - { - u8cerr - << "the prefix cannot be where mob.exe is, there's already a " - << "build directory in there\n"; - - return false; - } - } - - return true; -} - - -conf_global conf::global() -{ - return {}; -} - -conf_task conf::task(const std::vector& names) -{ - return {names}; -} - -conf_cmake conf::cmake() -{ - return {}; -} - -conf_tools conf::tool() -{ - return {}; -} - -conf_transifex conf::transifex() -{ - return {}; -} - -conf_prebuilt conf::prebuilt() -{ - return {}; -} - -conf_versions conf::version() -{ - return {}; -} - -conf_paths conf::path() -{ - return {}; -} - - -conf_global::conf_global() - : conf_section("global") -{ -} - -int conf_global::output_log_level() const -{ - return details::g_output_log_level; -} - -int conf_global::file_log_level() const -{ - return details::g_file_log_level; -} - -bool conf_global::dry() const -{ - return details::g_dry; -} - - -conf_task::conf_task(std::vector names) - : names_(std::move(names)) -{ -} - -std::string conf_task::get(std::string_view key) const -{ - return details::get_string_for_task(names_, key); -} - -bool conf_task::get_bool(std::string_view key) const -{ - return details::get_bool_for_task(names_, key); -} - - -bool conf_cmake::cmake_constant::is_equivalent(std::string_view other) const -{ - // _strcmpi does not have a n-overload, and since string_view is not - // necessarily null-terminated, _strmcpi cannot be safely used - return std::equal( - std::begin(value_), std::end(value_), - std::begin(other), std::end(other), - [](auto&& c1, auto&& c2) { return ::tolower(c1) == ::tolower(c2); } - ); -} - -conf_cmake::conf_cmake() - : conf_section("cmake") -{ -} - -const conf_cmake::cmake_constant conf_cmake::ALWAYS{"always"}; -const conf_cmake::cmake_constant conf_cmake::LAZY{"lazy"}; -const conf_cmake::cmake_constant conf_cmake::NEVER{"never"}; - -conf_cmake::cmake_constant conf_cmake::install_message() const -{ - return read_cmake_constant("install_message", { ALWAYS, LAZY, NEVER }); -} - -std::string conf_cmake::host() const -{ - return details::get_string(name(), "host"); -} - -conf_cmake::cmake_constant conf_cmake::read_cmake_constant( - std::string_view key, std::vector const& allowed) const -{ - const auto value = details::get_string(name(), key); - for (const auto& constant : allowed) - { - if (constant.is_equivalent(value)) { - return constant; - } - } - - gcx().bail_out(context::conf, - "bad value '{}' for {}/{} (expected one of {})", - value, name(), key, join(allowed, ", ", std::string{})); -} - -conf_tools::conf_tools() - : conf_section("tools") -{ -} - -conf_transifex::conf_transifex() - : conf_section("transifex") -{ -} - -conf_versions::conf_versions() - : conf_section("versions") -{ -} - -conf_prebuilt::conf_prebuilt() - : conf_section("prebuilt") -{ -} - -conf_paths::conf_paths() - : conf_section("paths") -{ -} - -} // namespace +namespace mob::details { + + using key_value_map = std::map>; + using section_map = std::map>; + + // holds all the options not related to tasks (global, tools, paths, etc.) + static section_map g_conf; + + // holds all the task options; has a special map element with an empty string + // for options that apply to all tasks, and elements with specific task names + // for overrides + static section_map g_tasks; + + // special cases to avoid string manipulations + static int g_output_log_level = 3; + static int g_file_log_level = 5; + static bool g_dry = false; + + bool bool_from_string(std::string_view s) + { + return (s == "true" || s == "yes" || s == "1"); + } + + // returns a string from conf, bails out if it doesn't exist + // + std::string get_string(std::string_view section, std::string_view key) + { + auto sitor = g_conf.find(section); + if (sitor == g_conf.end()) + gcx().bail_out(context::conf, "[{}] doesn't exist", section); + + auto kitor = sitor->second.find(key); + if (kitor == sitor->second.end()) + gcx().bail_out(context::conf, "no key '{}' in [{}]", key, section); + + return kitor->second; + } + + // calls get_string(), converts to int + // + int get_int(std::string_view section, std::string_view key) + { + const auto s = get_string(section, key); + + try { + return std::stoi(s); + } + catch (std::exception&) { + gcx().bail_out(context::conf, "bad int for {}/{}", section, key); + } + } + + // calls get_string(), converts to bool + // + bool get_bool(std::string_view section, std::string_view key) + { + const auto s = get_string(section, key); + return bool_from_string(s); + } + + // sets the given option, bails out if the option doesn't exist + // + void set_string(std::string_view section, std::string_view key, + std::string_view value) + { + auto sitor = g_conf.find(section); + if (sitor == g_conf.end()) + gcx().bail_out(context::conf, "[{}] doesn't exist", section); + + auto kitor = sitor->second.find(key); + if (kitor == sitor->second.end()) + gcx().bail_out(context::conf, "no key '{}' [{}]", key, section); + + kitor->second = value; + } + + // sets the given option, adds it if it doesn't exist; used when setting options + // from the master ini + // + void add_string(const std::string& section, const std::string& key, + std::string value) + { + g_conf[section][key] = value; + } + + // finds an option for the given task, returns empty if not found + // + std::optional find_string_for_task(std::string_view task_name, + std::string_view key) + { + // find task + auto titor = g_tasks.find(task_name); + if (titor == g_tasks.end()) + return {}; + + const auto& task = titor->second; + + // find key + auto itor = task.find(key); + if (itor == task.end()) + return {}; + + return itor->second; + } + + // gets an option for any of the given task names, typically what task::names() + // returns, which contains the main task name plus some alternate names + // + // there's a hierarchy for task options: + // + // 1) there's a special "_override" entry in g_tasks, for options set from + // the command line that should override everything, like --no-pull + // should override all pull settings for all tasks + // + // 2) if the key is not found in "_override", then there can be an entry + // in g_tasks with any of given task names + // + // 3) if the key doesn't exist, then use the generic task option for it, stored + // in an element with an empty string in g_tasks + // + std::string get_string_for_task(const std::vector& task_names, + std::string_view key) + { + // some command line options will override any user settings, like + // --no-pull, those are stored in a special _override task name + auto v = find_string_for_task("_override", key); + if (v) + return *v; + + // look for an option for this task by name + for (auto&& tn : task_names) { + v = find_string_for_task(tn, key); + if (v) + return *v; + } + + // default task options are in a special empty string entry in g_tasks + v = find_string_for_task("", key); + if (v) + return *v; + + // doesn't exist anywhere + gcx().bail_out(context::conf, "no task option '{}' found for any of {}", key, + join(task_names, ",")); + } + + // calls get_string_for_task(), converts to bool + // + bool get_bool_for_task(const std::vector& task_names, + std::string_view key) + { + const std::string s = get_string_for_task(task_names, key); + return bool_from_string(s); + } + + // sets the given task option, bails out if the option doesn't exist + // + void set_string_for_task(const std::string& task_name, const std::string& key, + std::string value) + { + // make sure the key exists, will throw if it doesn't + get_string_for_task({task_name}, key); + + g_tasks[task_name][key] = std::move(value); + } + + // sets the given task option, adds it if it doesn't exist; used when setting + // options from the master ini + // + void add_string_for_task(const std::string& task_name, const std::string& key, + std::string value) + { + g_tasks[task_name][key] = std::move(value); + } + +} // namespace mob::details + +namespace mob { + + std::vector format_options() + { + // don't log private stuff + auto hide = [](std::string_view section, std::string_view key) { + if (key == "github_key") + return true; + + if (section == "transifex" && key == "key") + return true; + + if (key == "git_email") + return true; + + return false; + }; + + auto make_value = [&](auto&& s, auto&& k, auto&& v) -> std::string { + if (hide(s, k)) + return v.empty() ? "" : "(hidden)"; + else + return v; + }; + + auto& tm = task_manager::instance(); + + std::size_t longest_what = 0; + std::size_t longest_key = 0; + + for (auto&& [section, kvs] : details::g_conf) { + longest_what = std::max(longest_what, section.size()); + + for (auto&& [k, v] : kvs) + longest_key = std::max(longest_key, k.size()); + } + + for (auto&& [k, v] : details::g_tasks[""]) + longest_key = std::max(longest_key, k.size()); + + for (const auto* task : tm.all()) + longest_what = std::max(longest_what, task->name().size()); + + std::vector lines; + + lines.push_back(pad_right("what", longest_what) + " " + + pad_right("key", longest_key) + " " + "value"); + + lines.push_back(pad_right("-", longest_what, '-') + " " + + pad_right("-", longest_key, '-') + " " + "-----"); + + for (auto&& [section, kvs] : details::g_conf) { + for (auto&& [k, v] : kvs) { + lines.push_back(pad_right(section, longest_what) + " " + + pad_right(k, longest_key) + " = " + + make_value(section, k, v)); + } + } + + for (auto&& [k, v] : details::g_tasks[""]) { + lines.push_back(pad_right("task", longest_what) + " " + + pad_right(k, longest_key) + " = " + make_value("", k, v)); + } + + for (const auto* t : tm.all()) { + for (auto&& [k, unused] : details::g_tasks[""]) { + lines.push_back( + pad_right(t->name(), longest_what) + " " + + pad_right(k, longest_key) + " = " + + make_value("", k, details::get_string_for_task({t->name()}, k))); + } + } + + return lines; + } + + // sets commonly used options that need to be converted to int/bool, for + // performance + // + void set_special_options() + { + details::g_output_log_level = details::get_int("global", "output_log_level"); + details::g_file_log_level = details::get_int("global", "file_log_level"); + details::g_dry = details::get_bool("global", "dry"); + } + + // sets an option `key` in the `paths` section; if the path is currently empty, + // sets it using `f` (which is either a callable or a string) + // + // in any case, makes it absolute and canonical, bails out if the path does not + // exist + // + // this is used for paths that should already exist (qt, vs, etc.) + // + template + void set_path_if_empty(std::string_view key, F&& f) + { + // current value + fs::path p = conf().path().get(key); + + if (p.empty()) { + // empty, set it from `f` + if constexpr (std::is_same_v>) + p = f; + else + p = f(); + } + + p = fs::absolute(p); + + if (!conf().global().dry()) { + if (!fs::exists(p)) + gcx().bail_out(context::conf, "path {} not found", p); + + p = fs::canonical(p); + } + + // new value + details::set_string("paths", key, path_to_utf8(p)); + } + + // sets an option `key` in the `paths` section: + // - if the path is empty, sets it as default_parent/default_dir, + // - if the path is not empty but is relative, resolves it against + // default_parent + // + // in any case, makes it absolute but weakly canonical since it might not exist + // at that point (this is used for build, install, etc.) + // + void resolve_path(std::string_view key, const fs::path& default_parent, + std::string_view default_dir) + { + // current value + fs::path p = conf().path().get(key); + + if (p.empty()) { + p = default_parent / default_dir; + } + else { + if (p.is_relative()) + p = default_parent / p; + } + + if (!conf().global().dry()) + p = fs::weakly_canonical(fs::absolute(p)); + + details::set_string("paths", key, path_to_utf8(p)); + } + + // `section_string` can be something like "global" or "paths", but also "task" + // or a task-specific name like "uibase:task" + // + // `master` is true if the ini being processed is the master ini so options are + // added to the maps instead of set, which throws if they're not found + // + void process_option(const std::string& section_string, const std::string& key, + const std::string& value, bool master) + { + // split section string on ":" + const auto col = section_string.find(":"); + std::string task, section; + + if (col == std::string::npos) { + // not a "task_name:task" section + section = section_string; + } + else { + // that's a "task_name:task" section + task = section_string.substr(0, col); + section = section_string.substr(col + 1); + } + + if (section == "task") { + // task options go in g_tasks + + if (task == "_override") { + // special case, comes from options on the command line + details::set_string_for_task("_override", key, value); + } + else if (task != "") { + // task specific + + // task must exist + const auto& tasks = task_manager::instance().find(task); + + if (tasks.empty()) { + gcx().bail_out(context::conf, "bad option {}, task '{}' not found", + section_string, task); + } + + MOB_ASSERT(!tasks.empty()); + + for (auto& t : tasks) + details::set_string_for_task(t->name(), key, value); + } + else { + // global task option + + if (master) + details::add_string_for_task("", key, value); + else + details::set_string_for_task("", key, value); + } + } + else { + // not a task option, goes into g_conf + + if (master) + details::add_string(section, key, value); + else + details::set_string(section, key, value); + } + } + + // reads the given ini and adds all of its content to the options + // + void process_ini(const fs::path& ini, bool master) + { + const auto data = parse_ini(ini); + + for (auto&& a : data.aliases) + task_manager::instance().add_alias(a.first, a.second); + + for (auto&& [section_string, kvs] : data.sections) { + for (auto&& [k, v] : kvs) + process_option(section_string, k, v, master); + } + } + + // parses the given option strings and adds them as options + // + void process_cmd_options(const std::vector& opts) + { + // parses "section/key=value" + static std::regex re(R"((.+)/(.+)=(.*))"); + + gcx().debug(context::conf, "overriding from command line:"); + + for (auto&& o : opts) { + std::smatch m; + if (!std::regex_match(o, m, re)) { + gcx().bail_out(context::conf, + "bad option {}, must be [task:]section/key=value", o); + } + + process_option(m[1], m[2], m[3], false); + } + } + + // goes through all the options that have to do with paths, checks them and + // resolves them if necessary + // + void resolve_paths() + { + // first, if any of these paths are empty, they are set using the second + // argument, which can be callable or a path + // + // the resulting path is made absolute and canonical and will bail out if it + // doesn't exist + + // make sure third-party is in PATH before the other paths are checked + // because some of these paths will need to look in there to find stuff + set_path_if_empty("third_party", find_third_party_directory); + this_env::prepend_to_path(conf().path().third_party() / "bin"); + + set_path_if_empty("pf_x86", find_program_files_x86); + set_path_if_empty("pf_x64", find_program_files_x64); + set_path_if_empty("vs", find_vs); + set_path_if_empty("qt_install", find_qt); + set_path_if_empty("temp_dir", find_temp_dir); + set_path_if_empty("patches", find_in_root("patches")); + set_path_if_empty("licenses", find_in_root("licenses")); + set_path_if_empty("qt_bin", qt::installation_path() / "bin"); + set_path_if_empty("qt_translations", qt::installation_path() / "translations"); + + // second, if any of these paths are relative, they use the second argument + // as the root; if they're empty, they combine the second and third + // arguments + // + // these paths might not exist yet, so they're only made weakly canonical, + // they'll be created as needed during the build process + + const auto p = conf().path(); + + resolve_path("cache", p.prefix(), "downloads"); + resolve_path("build", p.prefix(), "build"); + resolve_path("install", p.prefix(), "install"); + resolve_path("install_installer", p.install(), "installer"); + resolve_path("install_bin", p.install(), "bin"); + resolve_path("install_libs", p.install(), "libs"); + resolve_path("install_pdbs", p.install(), "pdb"); + resolve_path("install_dlls", p.install_bin(), "dlls"); + resolve_path("install_loot", p.install_bin(), "loot"); + resolve_path("install_plugins", p.install_bin(), "plugins"); + resolve_path("install_licenses", p.install_bin(), "licenses"); + resolve_path("install_pythoncore", p.install_bin(), "pythoncore"); + resolve_path("install_stylesheets", p.install_bin(), "stylesheets"); + resolve_path("install_translations", p.install_bin(), "translations"); + + // finally, resolve the tools that are unlikely to be in PATH; all the + // other tools (7z, jom, patch, etc.) are assumed to be in PATH (which + // now contains third-party) or have valid absolute paths in the ini + + details::set_string("tools", "vcvars", path_to_utf8(find_vcvars())); + details::set_string("tools", "iscc", path_to_utf8(find_iscc())); + } + + void conf::set_log_file() + { + // set up the log file, resolve against prefix if relative + fs::path log_file = conf().global().get("log_file"); + if (log_file.is_relative()) + log_file = conf().path().prefix() / log_file; + + context::set_log_file(log_file); + } + + void init_options(const std::vector& inis, + const std::vector& opts) + { + MOB_ASSERT(!inis.empty()); + + // some logging + gcx().debug(context::conf, "cl: {}", std::wstring(GetCommandLineW())); + gcx().debug(context::conf, "using inis in order:"); + for (auto&& ini : inis) + gcx().debug(context::conf, " . {}", ini); + + // used to resolve a relative prefix; by default, it's resolved against cwd, + // but if an ini other than the master contains a prefix, use the ini's + // parent directory instead + fs::path prefix_root = fs::current_path(); + + // true for the first ini, will add values to the configuration maps instead + // of setting them, which throws if the option doesn't exist + // + // the goal is that the first, master ini contains all existing options and + // if an option set in another ini or on the command line doesn't exist in + // the master, it's an error + bool master = true; + + for (auto&& ini : inis) { + fs::path prefix_before; + + // if this is the master ini, the prefix doesn't exist in the config + // yet because no inis have been loaded + if (!master) + prefix_before = conf().path().prefix(); + + process_ini(ini, master); + + // check if the prefix was changed by this ini + if (!master && conf().path().prefix() != prefix_before) { + // remember its path + prefix_root = ini.parent_path(); + } + + // further inis should only contain options that already exist + master = false; + } + + if (!opts.empty()) { + const fs::path prefix_before = conf().path().prefix(); + + process_cmd_options(opts); + + // check if the prefix was changed on the command line + if (conf().path().prefix() != prefix_before) { + // use cwd as the parent of a relative prefix + prefix_root = fs::current_path(); + } + } + + // converts some options to ints or bools, these are used everywhere, like + // the log levels + set_special_options(); + + // an empty prefix is an error and will fail in validate_options(), but + // don't check it here to allow some commands to run, like `mob options`, + // and make sure it's not set to something that's not empty to make sure it + // _does_ fail later on + if (!conf().path().prefix().empty()) + resolve_path("prefix", prefix_root, ""); + + // set up the log file, resolve against prefix if relative + conf().set_log_file(); + + // goes through all paths and tools, finds missing or relative stuff, bails + // out of stuff can't be found + resolve_paths(); + + // make sure qt's bin directory is in the path + this_env::append_to_path(conf().path().get("qt_bin")); + } + + bool verify_options() + { + // can't have an empty prefix + if (conf().path().prefix().empty()) { + u8cerr << "missing prefix; either specify it the [paths] section of " + << "the ini or pass '-d path'\n"; + + return false; + } + + // don't build mo inside mob + if (fs::exists(conf().path().prefix())) { + if (fs::equivalent(conf().path().prefix(), mob_exe_path().parent_path())) { + u8cerr << "the prefix cannot be where mob.exe is, there's already a " + << "build directory in there\n"; + + return false; + } + } + + return true; + } + + conf_global conf::global() + { + return {}; + } + + conf_task conf::task(const std::vector& names) + { + return {names}; + } + + conf_cmake conf::cmake() + { + return {}; + } + + conf_tools conf::tool() + { + return {}; + } + + conf_transifex conf::transifex() + { + return {}; + } + + conf_prebuilt conf::prebuilt() + { + return {}; + } + + conf_versions conf::version() + { + return {}; + } + + conf_paths conf::path() + { + return {}; + } + + conf_global::conf_global() : conf_section("global") {} + + int conf_global::output_log_level() const + { + return details::g_output_log_level; + } + + int conf_global::file_log_level() const + { + return details::g_file_log_level; + } + + bool conf_global::dry() const + { + return details::g_dry; + } + + conf_task::conf_task(std::vector names) : names_(std::move(names)) {} + + std::string conf_task::get(std::string_view key) const + { + return details::get_string_for_task(names_, key); + } + + bool conf_task::get_bool(std::string_view key) const + { + return details::get_bool_for_task(names_, key); + } + + bool conf_cmake::cmake_constant::is_equivalent(std::string_view other) const + { + // _strcmpi does not have a n-overload, and since string_view is not + // necessarily null-terminated, _strmcpi cannot be safely used + return std::equal(std::begin(value_), std::end(value_), std::begin(other), + std::end(other), [](auto&& c1, auto&& c2) { + return ::tolower(c1) == ::tolower(c2); + }); + } + + conf_cmake::conf_cmake() : conf_section("cmake") {} + + const conf_cmake::cmake_constant conf_cmake::ALWAYS{"always"}; + const conf_cmake::cmake_constant conf_cmake::LAZY{"lazy"}; + const conf_cmake::cmake_constant conf_cmake::NEVER{"never"}; + + conf_cmake::cmake_constant conf_cmake::install_message() const + { + return read_cmake_constant("install_message", {ALWAYS, LAZY, NEVER}); + } + + std::string conf_cmake::host() const + { + return details::get_string(name(), "host"); + } + + conf_cmake::cmake_constant + conf_cmake::read_cmake_constant(std::string_view key, + std::vector const& allowed) const + { + const auto value = details::get_string(name(), key); + for (const auto& constant : allowed) { + if (constant.is_equivalent(value)) { + return constant; + } + } + + gcx().bail_out(context::conf, "bad value '{}' for {}/{} (expected one of {})", + value, name(), key, join(allowed, ", ", std::string{})); + } + + conf_tools::conf_tools() : conf_section("tools") {} + + conf_transifex::conf_transifex() : conf_section("transifex") {} + + conf_versions::conf_versions() : conf_section("versions") {} + + conf_prebuilt::conf_prebuilt() : conf_section("prebuilt") {} + + conf_paths::conf_paths() : conf_section("paths") {} + +} // namespace mob diff --git a/src/core/conf.h b/src/core/conf.h index ae61b64..891fc83 100644 --- a/src/core/conf.h +++ b/src/core/conf.h @@ -3,310 +3,292 @@ // these shouldn't be called directly, they're used by some of the template // below // -namespace mob::details -{ - -// returns an option named `key` from the given `section` -// -std::string get_string(std::string_view section, std::string_view key); - -// calls get_string(), converts to bool -// -bool get_bool(std::string_view section, std::string_view key); - -// calls get_string(), converts to in -// -int get_int(std::string_view section, std::string_view key); - -// sets the given option, bails out if the option doesn't exist -// -void set_string( - std::string_view section, std::string_view key, std::string_view value); - -} // namespace - - -namespace mob -{ - -// reads options from the given inis and option strings, resolves all the paths -// and necessary tools, also adds a couple of things to PATH -// -void init_options( - const std::vector& inis, const std::vector& opts); - -// checks some of the options once everything is loaded, returns false if -// something's wrong -// -bool verify_options(); - -// returns all options formatted as three columns -// -std::vector format_options(); - - -// base class for all conf structs -// -template -class conf_section -{ -public: - DefaultType get(std::string_view key) const - { - return details::get_string(name_, key); - } - - // undefined - template - T get(std::string_view key) const; - - template <> - bool get(std::string_view key) const - { - return details::get_bool(name_, key); - } - - template <> - int get(std::string_view key) const - { - return details::get_int(name_, key); - } - - void set(std::string_view key, std::string_view value) - { - details::set_string(name_, key, value); - } - -protected: - conf_section(std::string section_name) - : name_(std::move(section_name)) - { - } - - const auto& name() const { return name_; } - -private: - std::string name_; -}; - - -// options in [global] -// -class conf_global : public conf_section -{ -public: - conf_global(); - - // convenience, doesn't need string manipulation - int output_log_level() const; - int file_log_level() const; - bool dry() const; - - // convenience - bool redownload() const { return get("redownload"); } - bool reextract() const { return get("reextract"); } - bool reconfigure() const { return get("reconfigure"); } - bool rebuild() const { return get("rebuild"); } - bool clean() const { return get("clean_task"); } - bool fetch() const { return get("fetch_task"); } - bool build() const { return get("build_task"); } -}; - - -// options in [task] or [task_name:task] -// -class conf_task -{ -public: - conf_task(std::vector names); - - std::string get(std::string_view key) const; - - template - T get(std::string_view key) const; - - template <> - bool get(std::string_view key) const - { - return get_bool(key); - } - - std::string mo_org() const { return get("mo_org"); } - std::string mo_branch() const { return get("mo_branch"); } - std::string mo_fallback_branch() const { return get("mo_fallback"); } - bool no_pull() const { return get("no_pull"); } - bool revert_ts() const { return get("revert_ts"); } - bool ignore_ts()const { return get("ignore_ts"); } - std::string git_url_prefix() const { return get("git_url_prefix"); } - bool git_shallow() const { return get("git_shallow"); } - std::string git_user() const { return get("git_username"); } - std::string git_email() const { return get("git_email"); } - bool set_origin_remote() const { return get("set_origin_remote"); } - std::string remote_org() const { return get("remote_org"); } - std::string remote_key() const { return get("remote_key"); } - bool remote_no_push_upstream() const { return get("remote_no_push_upstream"); } - bool remote_push_default_origin() const { return get("remote_push_default_origin"); } - -private: - std::vector names_; - - bool get_bool(std::string_view name) const; -}; - - -// options in [task] or [task_name:task] -// -class conf_cmake : conf_section -{ -public: - class cmake_constant { - std::string value_; - - // check if the given string is equivalent to this constant - // - bool is_equivalent(std::string_view other) const; - - friend class conf_cmake; - - public: - constexpr cmake_constant(std::string_view value) : value_{value} { } - constexpr const auto& value() const { return value_; } - constexpr operator const std::string& () const { return value(); } - - - friend bool operator==(cmake_constant const& lhs, cmake_constant const& rhs) - { - return lhs.is_equivalent(rhs.value()); - } - friend bool operator!=(cmake_constant const& lhs, cmake_constant const& rhs) - { - return !(lhs == rhs); - } - - }; - - static const cmake_constant ALWAYS; - static const cmake_constant LAZY; - static const cmake_constant NEVER; - - -public: - conf_cmake(); - - // specify the value for CMAKE_INSTALL_MESSAGE - // - cmake_constant install_message() const; - - // specify the toolset host configuration, if any, this is equivalent - // to -T host=XXX on the command line - // - // an empty string means no host configured - // - std::string host() const; - -private: - cmake_constant read_cmake_constant( - std::string_view key, std::vector const& allowed) const; -}; - - -// options in [tools] -// -class conf_tools : public conf_section -{ -public: - conf_tools(); -}; - - -// options in [transifex] -// -class conf_transifex : public conf_section -{ -public: - conf_transifex(); -}; - - -// options in [versions] -// -class conf_versions : public conf_section -{ -public: - conf_versions(); -}; - - -// options in [prebuilt] -// -class conf_prebuilt : public conf_section -{ -public: - conf_prebuilt(); -}; - - -// options in [paths] -// -class conf_paths : public conf_section -{ -public: - conf_paths(); - -#define VALUE(NAME) fs::path NAME() const { return get(#NAME); } - - VALUE(third_party); - VALUE(prefix); - VALUE(cache); - VALUE(patches); - VALUE(licenses); - VALUE(build); - - VALUE(install); - VALUE(install_installer); - VALUE(install_bin); - VALUE(install_libs); - VALUE(install_pdbs); - - VALUE(install_dlls); - VALUE(install_loot); - VALUE(install_plugins); - VALUE(install_stylesheets); - VALUE(install_licenses); - VALUE(install_pythoncore); - VALUE(install_translations); - - VALUE(vs); - VALUE(qt_install); - VALUE(qt_bin); - VALUE(qt_translations); - - VALUE(pf_x86); - VALUE(pf_x64); - VALUE(temp_dir); +namespace mob::details { + + // returns an option named `key` from the given `section` + // + std::string get_string(std::string_view section, std::string_view key); + + // calls get_string(), converts to bool + // + bool get_bool(std::string_view section, std::string_view key); + + // calls get_string(), converts to in + // + int get_int(std::string_view section, std::string_view key); + + // sets the given option, bails out if the option doesn't exist + // + void set_string(std::string_view section, std::string_view key, + std::string_view value); + +} // namespace mob::details + +namespace mob { + + // reads options from the given inis and option strings, resolves all the paths + // and necessary tools, also adds a couple of things to PATH + // + void init_options(const std::vector& inis, + const std::vector& opts); + + // checks some of the options once everything is loaded, returns false if + // something's wrong + // + bool verify_options(); + + // returns all options formatted as three columns + // + std::vector format_options(); + + // base class for all conf structs + // + template + class conf_section { + public: + DefaultType get(std::string_view key) const + { + return details::get_string(name_, key); + } + + // undefined + template + T get(std::string_view key) const; + + template <> + bool get(std::string_view key) const + { + return details::get_bool(name_, key); + } + + template <> + int get(std::string_view key) const + { + return details::get_int(name_, key); + } + + void set(std::string_view key, std::string_view value) + { + details::set_string(name_, key, value); + } + + protected: + conf_section(std::string section_name) : name_(std::move(section_name)) {} + + const auto& name() const { return name_; } + + private: + std::string name_; + }; + + // options in [global] + // + class conf_global : public conf_section { + public: + conf_global(); + + // convenience, doesn't need string manipulation + int output_log_level() const; + int file_log_level() const; + bool dry() const; + + // convenience + bool redownload() const { return get("redownload"); } + bool reextract() const { return get("reextract"); } + bool reconfigure() const { return get("reconfigure"); } + bool rebuild() const { return get("rebuild"); } + bool clean() const { return get("clean_task"); } + bool fetch() const { return get("fetch_task"); } + bool build() const { return get("build_task"); } + }; + + // options in [task] or [task_name:task] + // + class conf_task { + public: + conf_task(std::vector names); + + std::string get(std::string_view key) const; + + template + T get(std::string_view key) const; + + template <> + bool get(std::string_view key) const + { + return get_bool(key); + } + + std::string mo_org() const { return get("mo_org"); } + std::string mo_branch() const { return get("mo_branch"); } + std::string mo_fallback_branch() const { return get("mo_fallback"); } + bool no_pull() const { return get("no_pull"); } + bool revert_ts() const { return get("revert_ts"); } + bool ignore_ts() const { return get("ignore_ts"); } + std::string git_url_prefix() const { return get("git_url_prefix"); } + bool git_shallow() const { return get("git_shallow"); } + std::string git_user() const { return get("git_username"); } + std::string git_email() const { return get("git_email"); } + bool set_origin_remote() const { return get("set_origin_remote"); } + std::string remote_org() const { return get("remote_org"); } + std::string remote_key() const { return get("remote_key"); } + bool remote_no_push_upstream() const + { + return get("remote_no_push_upstream"); + } + bool remote_push_default_origin() const + { + return get("remote_push_default_origin"); + } + + private: + std::vector names_; + + bool get_bool(std::string_view name) const; + }; + + // options in [task] or [task_name:task] + // + class conf_cmake : conf_section { + public: + class cmake_constant { + std::string value_; + + // check if the given string is equivalent to this constant + // + bool is_equivalent(std::string_view other) const; + + friend class conf_cmake; + + public: + constexpr cmake_constant(std::string_view value) : value_{value} {} + constexpr const auto& value() const { return value_; } + constexpr operator const std::string&() const { return value(); } + + friend bool operator==(cmake_constant const& lhs, cmake_constant const& rhs) + { + return lhs.is_equivalent(rhs.value()); + } + friend bool operator!=(cmake_constant const& lhs, cmake_constant const& rhs) + { + return !(lhs == rhs); + } + }; + + static const cmake_constant ALWAYS; + static const cmake_constant LAZY; + static const cmake_constant NEVER; + + public: + conf_cmake(); + + // specify the value for CMAKE_INSTALL_MESSAGE + // + cmake_constant install_message() const; + + // specify the toolset host configuration, if any, this is equivalent + // to -T host=XXX on the command line + // + // an empty string means no host configured + // + std::string host() const; + + private: + cmake_constant + read_cmake_constant(std::string_view key, + std::vector const& allowed) const; + }; + + // options in [tools] + // + class conf_tools : public conf_section { + public: + conf_tools(); + }; + + // options in [transifex] + // + class conf_transifex : public conf_section { + public: + conf_transifex(); + }; + + // options in [versions] + // + class conf_versions : public conf_section { + public: + conf_versions(); + }; + + // options in [prebuilt] + // + class conf_prebuilt : public conf_section { + public: + conf_prebuilt(); + }; + + // options in [paths] + // + class conf_paths : public conf_section { + public: + conf_paths(); + +#define VALUE(NAME) \ + fs::path NAME() const \ + { \ + return get(#NAME); \ + } + + VALUE(third_party); + VALUE(prefix); + VALUE(cache); + VALUE(patches); + VALUE(licenses); + VALUE(build); + + VALUE(install); + VALUE(install_installer); + VALUE(install_bin); + VALUE(install_libs); + VALUE(install_pdbs); + + VALUE(install_dlls); + VALUE(install_loot); + VALUE(install_plugins); + VALUE(install_stylesheets); + VALUE(install_licenses); + VALUE(install_pythoncore); + VALUE(install_translations); + + VALUE(vs); + VALUE(qt_install); + VALUE(qt_bin); + VALUE(qt_translations); + + VALUE(pf_x86); + VALUE(pf_x64); + VALUE(temp_dir); #undef VALUE -}; - - -// should be used as conf().global().whatever(), doesn't actually hold anything, -// but it's better than a bunch of static functions -// -class conf -{ -public: - conf_global global(); - conf_task task(const std::vector& names); - conf_cmake cmake(); - conf_tools tool(); - conf_transifex transifex(); - conf_prebuilt prebuilt(); - conf_versions version(); - conf_paths path(); - - // opens the log file, creates the directory if needed - // - void set_log_file(); -}; - -} // namespace + }; + + // should be used as conf().global().whatever(), doesn't actually hold anything, + // but it's better than a bunch of static functions + // + class conf { + public: + conf_global global(); + conf_task task(const std::vector& names); + conf_cmake cmake(); + conf_tools tool(); + conf_transifex transifex(); + conf_prebuilt prebuilt(); + conf_versions version(); + conf_paths path(); + + // opens the log file, creates the directory if needed + // + void set_log_file(); + }; + +} // namespace mob diff --git a/src/core/context.cpp b/src/core/context.cpp index a628820..538d627 100644 --- a/src/core/context.cpp +++ b/src/core/context.cpp @@ -1,486 +1,459 @@ #include "pch.h" #include "context.h" -#include "conf.h" -#include "../utility.h" #include "../tasks/task.h" #include "../tools/tools.h" +#include "../utility.h" +#include "conf.h" -namespace mob::details -{ - -std::string converter::convert(const std::wstring& s) -{ - return utf16_to_utf8(s); -} - -std::string converter::convert(const fs::path& s) -{ - return utf16_to_utf8(s.native()); -} - -std::string converter::convert(const url& u) -{ - return u.string(); -} - -} // namespace - - -namespace mob -{ - -// timestamps are relative to this -static hr_clock::time_point g_start_time = hr_clock::now(); - -// accumulated errors and warnings; only used if should_dump_logs() is true, -// dumped on the console just before mob exits -static std::vector g_errors, g_warnings; - -// handle to log file -static handle_ptr g_log_file; - -// global output mutex to avoid interleaving, but also mixing colors -static std::mutex g_mutex; - - -// returns the color associated with the given level -// -console_color level_color(context::level lv) -{ - switch (lv) - { - case context::level::dump: - case context::level::trace: - case context::level::debug: - return console_color::grey; - - case context::level::warning: - return console_color::yellow; - - case context::level::error: - return console_color::red; - - case context::level::info: - default: - return console_color::white; - } -} - -// converts a reason to string -// -const char* reason_string(context::reason r) -{ - switch (r) - { - case context::bypass: return "bypass"; - case context::redownload: return "re-dl"; - case context::rebuild: return "re-bd"; - case context::reextract: return "re-ex"; - case context::interruption: return "int"; - case context::cmd: return "cmd"; - case context::std_out: return "stdout"; - case context::std_err: return "stderr"; - case context::fs: return (conf().global().dry() ? "fs-dry" : "fs"); - case context::net: return "net"; - case context::generic: return ""; - case context::conf: return "conf"; - default: return "?"; - } -} - -// retrieves the error message from the system for the given id -// -std::string error_message(DWORD id) -{ - wchar_t* message = nullptr; - - const auto ret = FormatMessageW( - FORMAT_MESSAGE_ALLOCATE_BUFFER | - FORMAT_MESSAGE_FROM_SYSTEM | - FORMAT_MESSAGE_IGNORE_INSERTS, - NULL, - id, - MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), - reinterpret_cast(&message), - 0, NULL); - - std::wstring s; - - std::wostringstream oss; - - // hex error code - oss << L"0x" << std::hex << id; - - if (ret == 0 || !message) { - // error message not found, just use the hex error code - s = oss.str(); - } else { - // FormatMessage() includes a newline, trim it and put the hex code too - s = trim_copy(message) + L" (" + oss.str() + L")"; - } - - LocalFree(message); - - return utf16_to_utf8(s); -} - -std::chrono::nanoseconds timestamp() -{ - return (hr_clock::now() - g_start_time); -} - -std::string_view timestamp_string() -{ - // thread local buffer to avoid allocation - static thread_local char buffer[50]; - - using namespace std::chrono; - - // getting time in seconds as a float - const auto ms = duration_cast(timestamp()); - const auto frac = static_cast(ms.count()) / 1000.0; - - // to string with 2 digits precision - const auto r = std::to_chars( - std::begin(buffer), std::end(buffer), frac, - std::chars_format::fixed, 2); - - if (r.ec != std::errc()) - return "?"; - - return {buffer, r.ptr}; -} - -// returns if the given level should be enabled based on the given level from -// the ini -// -// unfortunately, log levels in the ini have dump as the highest number, but -// the level enum is the reverse -// -bool log_enabled(context::level lv, int conf_lv) -{ - switch (lv) - { - case context::level::dump: - return conf_lv > 5; - - case context::level::trace: - return conf_lv > 4; - - case context::level::debug: - return conf_lv > 3; - - case context::level::info: - return conf_lv > 2; - - case context::level::warning: - return conf_lv > 1; - - case context::level::error: - return conf_lv > 0; - - default: - return true; - } -} - -// whether errors and warnings should be dumped at the end, only returns true -// for debug level and higher, lower levels don't have enough stuff on the -// console to make it worth, it's just annoying to have duplicate logs -// -// in fact, this feature might not be very useful at all -// -bool should_dump_logs() -{ - return log_enabled( - context::level::debug, - conf().global().output_log_level()); -} - - -context::context(std::string task_name) - : task_(std::move(task_name)), tool_(nullptr) -{ -} - -void context::set_tool(tool* t) -{ - tool_ = t; -} - -const context* context::global() -{ - static thread_local context c(""); - return &c; -} - -bool context::enabled(level lv) -{ - // a log level is enabled if it's included in either the console or the log - // file, which have independent levels - const int minimum_log_level = std::max( - mob::conf().global().output_log_level(), - mob::conf().global().file_log_level()); - - return log_enabled(lv, minimum_log_level); -} - -void context::set_log_file(const fs::path& p) -{ - if (!mob::conf().global().dry() && !p.empty()) - { - // creating directory - if (!exists(p.parent_path())) - op::create_directories(gcx(), mob::conf().path().prefix()); - - HANDLE h = CreateFileW( - p.native().c_str(), GENERIC_WRITE, FILE_SHARE_READ, - nullptr, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, 0); - - if (h == INVALID_HANDLE_VALUE) - { - const auto e = GetLastError(); - gcx().bail_out(context::generic, - "failed to open log file {}, {}", p, error_message(e)); - } - - g_log_file.reset(h); - } -} - -void context::close_log_file() -{ - g_log_file.reset(); -} - -void context::log_string(reason r, level lv, std::string_view s) const -{ - if (!enabled(lv)) - return; - - do_log_impl(false, r, lv, s); -} - -void context::do_log_impl( - bool bail, reason r, level lv, std::string_view utf8) const -{ - std::string_view sv = make_log_string(r, lv, utf8); - - if (bail) - { - // log the string with "(bailing out)" at the end, but throw the - // original, it's prettier that way - const std::string s(sv); - emit_log(lv, s + " (bailing out)"); - throw bailed(s); - } - else - { - emit_log(lv, sv); - } -} - -void context::emit_log(level lv, std::string_view utf8) const -{ - std::scoped_lock lock(g_mutex); - - // console - if (log_enabled(lv, mob::conf().global().output_log_level())) - { - // will revert color in dtor - console_color c = level_color(lv); - u8cout.write_ln(utf8); - } - - // log file - if (g_log_file && log_enabled(lv, mob::conf().global().file_log_level())) - { - DWORD written = 0; - - ::WriteFile( - g_log_file.get(), utf8.data(), static_cast(utf8.size()), - &written, nullptr); - - ::WriteFile(g_log_file.get(), "\r\n", 2, &written, nullptr); - } - - // remember warnings and errors - if (should_dump_logs()) - { - if (lv == level::error) - g_errors.emplace_back(utf8); - else if (lv == level::warning) - g_warnings.emplace_back(utf8); - } -} - -// used by make_log_string(), appends `what` to `s`, with padding on the right -// up to `max_length` characters, including the length of `what` -// -void append(std::string& s, std::string_view what, std::size_t max_length) -{ - if (what.empty()) - { - s.append(max_length, ' '); - } - else - { - s.append(what); - - // padding - const std::size_t written = what.size(); - if (written < max_length) - s.append(max_length - written, ' '); - } -} - -// used by make_log_string(), same as append() above but puts `what` between -// [brackets] -// -// avoids unnecessary memory allocations by appending the brackets directly -// -void append_with_brackets( - std::string& s, std::string_view what, std::size_t max_length) -{ - if (what.empty()) - { - s.append(max_length, ' '); - } - else - { - s.append(1, '['); - s.append(what); - s.append(1, ']'); - - // padding - const std::size_t written = what.size() + 2; // "[what]" - if (written < max_length) - s.append(max_length - written, ' '); - } -} - -// used by make_log_string(), can append some stuff depending on the reason -// -void append_context(std::string& ls, context::reason r) -{ - switch (r) - { - case context::redownload: - { - ls.append(" (happened because of --redownload)"); - break; - } - - case context::rebuild: - { - ls.append(" (happened because of --rebuild)"); - break; - } - - case context::reextract: - { - ls.append(" (happened because of --reextract)"); - break; - } - - case context::cmd: - case context::bypass: - case context::std_out: - case context::std_err: - case context::fs: - case context::net: - case context::generic: - case context::conf: - case context::interruption: - default: - break; - } -} - -std::string_view context::make_log_string(reason r, level, std::string_view s) const -{ - // maximum lengths of the various components below, used for padding - - // mob shouldn't run for more than three hours, includes space - const std::size_t timestamp_max_length = 8; // '0000.00 ' - - // cut task name at 15, +3 because brackets + space at the end - const std::size_t longest_task_name = 15; - const std::size_t task_name_max_length = longest_task_name + 3; - - // cut tool name at 7, +3 because brackets + space at the end - const std::size_t longest_tool_name = 7; - const std::size_t tool_name_max_length = longest_tool_name + 3; - - // cut reason name at 7, +3 because brackets + space at the end - const std::size_t longest_reason = 7; - const std::size_t reason_max_length = longest_reason + 3; - - - // keep a thread local string to avoid memory allocations - static thread_local std::string ls; - - // clear previous log - ls.clear(); - - - // a full log line might look like: - // "2.77 [cmake_common] [git] [cmd] creating process" - - - // timestamp - append(ls, timestamp_string(), timestamp_max_length); - - // task name - append_with_brackets( - ls, task_.substr(0, longest_task_name), task_name_max_length); - - // tool - if (tool_) - { - append_with_brackets( - ls, tool_->name().substr(0, longest_tool_name), - tool_name_max_length); - } - else - { - ls.append(tool_name_max_length, ' '); - } - - // reason - append_with_brackets(ls, reason_string(r), reason_max_length); - - // log message - ls.append(s); - - // context - append_context(ls, r); - - - return ls; -} - -void dump_logs() -{ - if (!should_dump_logs()) - return; - - if (!g_warnings.empty() || !g_errors.empty()) - { - u8cout << "\n\nthere were problems:\n"; - - { - auto c = level_color(context::level::warning); - for (auto&& s : g_warnings) - u8cout << s << "\n"; - } - - { - auto c = level_color(context::level::error); - for (auto&& s : g_errors) - u8cout << s << "\n"; - } - } -} - -} // namespace +namespace mob::details { + + std::string converter::convert(const std::wstring& s) + { + return utf16_to_utf8(s); + } + + std::string converter::convert(const fs::path& s) + { + return utf16_to_utf8(s.native()); + } + + std::string converter::convert(const url& u) + { + return u.string(); + } + +} // namespace mob::details + +namespace mob { + + // timestamps are relative to this + static hr_clock::time_point g_start_time = hr_clock::now(); + + // accumulated errors and warnings; only used if should_dump_logs() is true, + // dumped on the console just before mob exits + static std::vector g_errors, g_warnings; + + // handle to log file + static handle_ptr g_log_file; + + // global output mutex to avoid interleaving, but also mixing colors + static std::mutex g_mutex; + + // returns the color associated with the given level + // + console_color level_color(context::level lv) + { + switch (lv) { + case context::level::dump: + case context::level::trace: + case context::level::debug: + return console_color::grey; + + case context::level::warning: + return console_color::yellow; + + case context::level::error: + return console_color::red; + + case context::level::info: + default: + return console_color::white; + } + } + + // converts a reason to string + // + const char* reason_string(context::reason r) + { + switch (r) { + case context::bypass: + return "bypass"; + case context::redownload: + return "re-dl"; + case context::rebuild: + return "re-bd"; + case context::reextract: + return "re-ex"; + case context::interruption: + return "int"; + case context::cmd: + return "cmd"; + case context::std_out: + return "stdout"; + case context::std_err: + return "stderr"; + case context::fs: + return (conf().global().dry() ? "fs-dry" : "fs"); + case context::net: + return "net"; + case context::generic: + return ""; + case context::conf: + return "conf"; + default: + return "?"; + } + } + + // retrieves the error message from the system for the given id + // + std::string error_message(DWORD id) + { + wchar_t* message = nullptr; + + const auto ret = + FormatMessageW(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | + FORMAT_MESSAGE_IGNORE_INSERTS, + NULL, id, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), + reinterpret_cast(&message), 0, NULL); + + std::wstring s; + + std::wostringstream oss; + + // hex error code + oss << L"0x" << std::hex << id; + + if (ret == 0 || !message) { + // error message not found, just use the hex error code + s = oss.str(); + } + else { + // FormatMessage() includes a newline, trim it and put the hex code too + s = trim_copy(message) + L" (" + oss.str() + L")"; + } + + LocalFree(message); + + return utf16_to_utf8(s); + } + + std::chrono::nanoseconds timestamp() + { + return (hr_clock::now() - g_start_time); + } + + std::string_view timestamp_string() + { + // thread local buffer to avoid allocation + static thread_local char buffer[50]; + + using namespace std::chrono; + + // getting time in seconds as a float + const auto ms = duration_cast(timestamp()); + const auto frac = static_cast(ms.count()) / 1000.0; + + // to string with 2 digits precision + const auto r = std::to_chars(std::begin(buffer), std::end(buffer), frac, + std::chars_format::fixed, 2); + + if (r.ec != std::errc()) + return "?"; + + return {buffer, r.ptr}; + } + + // returns if the given level should be enabled based on the given level from + // the ini + // + // unfortunately, log levels in the ini have dump as the highest number, but + // the level enum is the reverse + // + bool log_enabled(context::level lv, int conf_lv) + { + switch (lv) { + case context::level::dump: + return conf_lv > 5; + + case context::level::trace: + return conf_lv > 4; + + case context::level::debug: + return conf_lv > 3; + + case context::level::info: + return conf_lv > 2; + + case context::level::warning: + return conf_lv > 1; + + case context::level::error: + return conf_lv > 0; + + default: + return true; + } + } + + // whether errors and warnings should be dumped at the end, only returns true + // for debug level and higher, lower levels don't have enough stuff on the + // console to make it worth, it's just annoying to have duplicate logs + // + // in fact, this feature might not be very useful at all + // + bool should_dump_logs() + { + return log_enabled(context::level::debug, conf().global().output_log_level()); + } + + context::context(std::string task_name) + : task_(std::move(task_name)), tool_(nullptr) + { + } + + void context::set_tool(tool* t) + { + tool_ = t; + } + + const context* context::global() + { + static thread_local context c(""); + return &c; + } + + bool context::enabled(level lv) + { + // a log level is enabled if it's included in either the console or the log + // file, which have independent levels + const int minimum_log_level = std::max(mob::conf().global().output_log_level(), + mob::conf().global().file_log_level()); + + return log_enabled(lv, minimum_log_level); + } + + void context::set_log_file(const fs::path& p) + { + if (!mob::conf().global().dry() && !p.empty()) { + // creating directory + if (!exists(p.parent_path())) + op::create_directories(gcx(), mob::conf().path().prefix()); + + HANDLE h = CreateFileW(p.native().c_str(), GENERIC_WRITE, FILE_SHARE_READ, + nullptr, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, 0); + + if (h == INVALID_HANDLE_VALUE) { + const auto e = GetLastError(); + gcx().bail_out(context::generic, "failed to open log file {}, {}", p, + error_message(e)); + } + + g_log_file.reset(h); + } + } + + void context::close_log_file() + { + g_log_file.reset(); + } + + void context::log_string(reason r, level lv, std::string_view s) const + { + if (!enabled(lv)) + return; + + do_log_impl(false, r, lv, s); + } + + void context::do_log_impl(bool bail, reason r, level lv, + std::string_view utf8) const + { + std::string_view sv = make_log_string(r, lv, utf8); + + if (bail) { + // log the string with "(bailing out)" at the end, but throw the + // original, it's prettier that way + const std::string s(sv); + emit_log(lv, s + " (bailing out)"); + throw bailed(s); + } + else { + emit_log(lv, sv); + } + } + + void context::emit_log(level lv, std::string_view utf8) const + { + std::scoped_lock lock(g_mutex); + + // console + if (log_enabled(lv, mob::conf().global().output_log_level())) { + // will revert color in dtor + console_color c = level_color(lv); + u8cout.write_ln(utf8); + } + + // log file + if (g_log_file && log_enabled(lv, mob::conf().global().file_log_level())) { + DWORD written = 0; + + ::WriteFile(g_log_file.get(), utf8.data(), static_cast(utf8.size()), + &written, nullptr); + + ::WriteFile(g_log_file.get(), "\r\n", 2, &written, nullptr); + } + + // remember warnings and errors + if (should_dump_logs()) { + if (lv == level::error) + g_errors.emplace_back(utf8); + else if (lv == level::warning) + g_warnings.emplace_back(utf8); + } + } + + // used by make_log_string(), appends `what` to `s`, with padding on the right + // up to `max_length` characters, including the length of `what` + // + void append(std::string& s, std::string_view what, std::size_t max_length) + { + if (what.empty()) { + s.append(max_length, ' '); + } + else { + s.append(what); + + // padding + const std::size_t written = what.size(); + if (written < max_length) + s.append(max_length - written, ' '); + } + } + + // used by make_log_string(), same as append() above but puts `what` between + // [brackets] + // + // avoids unnecessary memory allocations by appending the brackets directly + // + void append_with_brackets(std::string& s, std::string_view what, + std::size_t max_length) + { + if (what.empty()) { + s.append(max_length, ' '); + } + else { + s.append(1, '['); + s.append(what); + s.append(1, ']'); + + // padding + const std::size_t written = what.size() + 2; // "[what]" + if (written < max_length) + s.append(max_length - written, ' '); + } + } + + // used by make_log_string(), can append some stuff depending on the reason + // + void append_context(std::string& ls, context::reason r) + { + switch (r) { + case context::redownload: { + ls.append(" (happened because of --redownload)"); + break; + } + + case context::rebuild: { + ls.append(" (happened because of --rebuild)"); + break; + } + + case context::reextract: { + ls.append(" (happened because of --reextract)"); + break; + } + + case context::cmd: + case context::bypass: + case context::std_out: + case context::std_err: + case context::fs: + case context::net: + case context::generic: + case context::conf: + case context::interruption: + default: + break; + } + } + + std::string_view context::make_log_string(reason r, level, std::string_view s) const + { + // maximum lengths of the various components below, used for padding + + // mob shouldn't run for more than three hours, includes space + const std::size_t timestamp_max_length = 8; // '0000.00 ' + + // cut task name at 15, +3 because brackets + space at the end + const std::size_t longest_task_name = 15; + const std::size_t task_name_max_length = longest_task_name + 3; + + // cut tool name at 7, +3 because brackets + space at the end + const std::size_t longest_tool_name = 7; + const std::size_t tool_name_max_length = longest_tool_name + 3; + + // cut reason name at 7, +3 because brackets + space at the end + const std::size_t longest_reason = 7; + const std::size_t reason_max_length = longest_reason + 3; + + // keep a thread local string to avoid memory allocations + static thread_local std::string ls; + + // clear previous log + ls.clear(); + + // a full log line might look like: + // "2.77 [cmake_common] [git] [cmd] creating process" + + // timestamp + append(ls, timestamp_string(), timestamp_max_length); + + // task name + append_with_brackets(ls, task_.substr(0, longest_task_name), + task_name_max_length); + + // tool + if (tool_) { + append_with_brackets(ls, tool_->name().substr(0, longest_tool_name), + tool_name_max_length); + } + else { + ls.append(tool_name_max_length, ' '); + } + + // reason + append_with_brackets(ls, reason_string(r), reason_max_length); + + // log message + ls.append(s); + + // context + append_context(ls, r); + + return ls; + } + + void dump_logs() + { + if (!should_dump_logs()) + return; + + if (!g_warnings.empty() || !g_errors.empty()) { + u8cout << "\n\nthere were problems:\n"; + + { + auto c = level_color(context::level::warning); + for (auto&& s : g_warnings) + u8cout << s << "\n"; + } + + { + auto c = level_color(context::level::error); + for (auto&& s : g_errors) + u8cout << s << "\n"; + } + } + } + +} // namespace mob diff --git a/src/core/context.h b/src/core/context.h index 6a38141..7630598 100644 --- a/src/core/context.h +++ b/src/core/context.h @@ -7,314 +7,292 @@ // those are kept in this namespace so they don't leak all over the place; // they're used directly by context::do_log() below -namespace mob::details -{ - -class mob::url; - -template -struct converter -{ - static const T& convert(const T& t) - { - return t; - } -}; - -template <> -struct converter -{ - static std::string convert(const std::wstring& s); -}; - -template <> -struct converter -{ - static std::string convert(const fs::path& s); -}; - -template <> -struct converter -{ - static std::string convert(const url& u); -}; - -template -struct converter>> -{ - static std::string convert(T e) - { - return std::to_string(static_cast>(e)); - } -}; - -} // namespace - - -namespace mob -{ - -class tool; - -// system error message -// -std::string error_message(DWORD e); - - -// a logger with some context, this is passed around everywhere and knows which -// task and tool is currently running to get better context when logging -// -// each log must have a reason, `generic` can be used if no reason makes sense -// -// in places where there is no context available, there's a global one can that -// be retrieved with gcx() for logging -// -// all log functions will use fmt::format() internally, so they can be used -// like: -// -// cx.log(context::generic, "eat more {}", "potatoes"); -// -class context -{ -public: - // reason for a log or bailing out - // - enum reason - { - // generic - generic, - - // a configuration action - conf, - - // something was bypassed because it was already done - bypass, - - // something was done because the --redownload option was set - redownload, - - // something was done because the --rebuild option was set - rebuild, - - // something was done because the --reextract option was set - reextract, - - // something was done in case of interruption or because something - // was interrupted - interruption, - - // command line of a process - cmd, - - // output of a process - std_out, - std_err, - - // a filesystem action - fs, - - // a network action - net, - }; - - // level of a log entry, `dump` should only be used for really verbose - // stuff that shouldn't be very useful, like curl's debugging logs - // - enum class level - { - dump = 1, - trace, - debug, - info, - warning, - error, - }; - - // returns the global context, used by gcx() below - // - static const context* global(); - - // whether logs of this level are enabled; this normally doesn't need to be - // called because the logging functions will discard entries for levels that - // are not enabled, but it can be used for log strings that expensive to - // create - // - // since there are two log levels (one for console, one for file), enabled() - // will return true if the given level is enabled for at least one of them - // - static bool enabled(level lv); - - // sets the output file for logs - // - static void set_log_file(const fs::path& p); - - // closes the output file for logs, see release_command::check_clean_prefix() - // - static void close_log_file(); - - - // creates a context for a task; the global context has no name - // - context(std::string task_name); - - // sets the tool that's currently running, may be null if there isn't one; - // log entries will have the name of the tool if one is set - // - void set_tool(tool* t); - - // logs a simple string with the given level - // - void log_string(reason r, level lv, std::string_view s) const; - - // logs a formatted string with the given level - // - template - void log(reason r, level lv, const char* f, Args&&... args) const - { - do_log(false, r, lv, f, std::forward(args)...); - } - - // logs a formatted string with the dump level - // - template - void dump(reason r, const char* f, Args&&... args) const - { - do_log(false, r, level::dump, f, std::forward(args)...); - } - - // logs a formatted string with the trace level - // - template - void trace(reason r, const char* f, Args&&... args) const - { - do_log(false, r, level::trace, f, std::forward(args)...); - } - - // logs a formatted string with the debug level - // - template - void debug(reason r, const char* f, Args&&... args) const - { - do_log(false, r, level::debug, f, std::forward(args)...); - } - - // logs a formatted string with the info level - // - template - void info(reason r, const char* f, Args&&... args) const - { - do_log(false, r, level::info, f, std::forward(args)...); - } - - // logs a formatted string with the warning level - // - template - void warning(reason r, const char* f, Args&&... args) const - { - do_log(false, r, level::warning, f, std::forward(args)...); - } - - // logs a formatted string with the error level - // - template - void error(reason r, const char* f, Args&&... args) const - { - do_log(false, r, level::error, f, std::forward(args)...); - } - - // logs a formatted string with the error level and throws a bailed - // exception, which will exit mob as quickly as it can, interrupting all - // tasks - // - template - [[noreturn]] void bail_out(reason r, const char* f, Args&&... args) const - { - do_log(true, r, level::error, f, std::forward(args)...); - } - -private: - // current task, may be empty - std::string task_; - - // current tool, may be null - const tool* tool_; - - - // all logs above end up in here; if `bail` is true, this will throw a - // bailed exception after logging - // - template - void do_log(bool bail, reason r, level lv, const char* f, Args&&... args) const - { - // discard log if it's not enabled and it's not bailing out - if (!bail && !enabled(lv)) - return; - - try - { - // formatting string - const std::string s = fmt::format( - f, details::converter>::convert( - std::forward(args))...); - - do_log_impl(bail, r, lv, s); - } - catch(std::exception&) - { - // this is typically a bad format string, but there's not a lot - // that can be done except logging to stderr and asserting - - // try to display the format string, but the console is in utf16 - // mode and the string is utf8, so it would have to be converted, - // which could also fail - // - // since pretty much all format strings are ascii anyway, just do - // a ghetto conversion and hope it gives enough info - std::wstring s; - - const char* p = f; - while (*p) - { - s += (wchar_t)*p; - ++p; - } - - std::wcerr << "bad format string '" << s << "'\n"; - - if (IsDebuggerPresent()) - DebugBreak(); - } - } - - // all the calls above end up here; calls make_log_string() to get the full - // log line calls emit_log() with it; throws after if `bail` is true - // - void do_log_impl(bool bail, reason r, level lv, std::string_view s) const; - - // formats the log line: adds the timestamp, task name and tool name, if - // any - // - std::string_view make_log_string( - reason r, level lv, std::string_view s) const; - - // writes the given string to the console and the log file, and keeps all - // errors and warnings in global lists so they can be dumped just before mob - // exits - // - void emit_log(level lv, std::string_view s) const; -}; - - -// global context, convenience -// -inline const context& gcx() -{ - return *context::global(); -} - -// called in main() just before mob exits, dumps all errors and warnings seen -// during the build if the console log level was high enough -// -void dump_logs(); - -} // namespace +namespace mob::details { + + class mob::url; + + template + struct converter { + static const T& convert(const T& t) { return t; } + }; + + template <> + struct converter { + static std::string convert(const std::wstring& s); + }; + + template <> + struct converter { + static std::string convert(const fs::path& s); + }; + + template <> + struct converter { + static std::string convert(const url& u); + }; + + template + struct converter>> { + static std::string convert(T e) + { + return std::to_string(static_cast>(e)); + } + }; + +} // namespace mob::details + +namespace mob { + + class tool; + + // system error message + // + std::string error_message(DWORD e); + + // a logger with some context, this is passed around everywhere and knows which + // task and tool is currently running to get better context when logging + // + // each log must have a reason, `generic` can be used if no reason makes sense + // + // in places where there is no context available, there's a global one can that + // be retrieved with gcx() for logging + // + // all log functions will use fmt::format() internally, so they can be used + // like: + // + // cx.log(context::generic, "eat more {}", "potatoes"); + // + class context { + public: + // reason for a log or bailing out + // + enum reason { + // generic + generic, + + // a configuration action + conf, + + // something was bypassed because it was already done + bypass, + + // something was done because the --redownload option was set + redownload, + + // something was done because the --rebuild option was set + rebuild, + + // something was done because the --reextract option was set + reextract, + + // something was done in case of interruption or because something + // was interrupted + interruption, + + // command line of a process + cmd, + + // output of a process + std_out, + std_err, + + // a filesystem action + fs, + + // a network action + net, + }; + + // level of a log entry, `dump` should only be used for really verbose + // stuff that shouldn't be very useful, like curl's debugging logs + // + enum class level { + dump = 1, + trace, + debug, + info, + warning, + error, + }; + + // returns the global context, used by gcx() below + // + static const context* global(); + + // whether logs of this level are enabled; this normally doesn't need to be + // called because the logging functions will discard entries for levels that + // are not enabled, but it can be used for log strings that expensive to + // create + // + // since there are two log levels (one for console, one for file), enabled() + // will return true if the given level is enabled for at least one of them + // + static bool enabled(level lv); + + // sets the output file for logs + // + static void set_log_file(const fs::path& p); + + // closes the output file for logs, see release_command::check_clean_prefix() + // + static void close_log_file(); + + // creates a context for a task; the global context has no name + // + context(std::string task_name); + + // sets the tool that's currently running, may be null if there isn't one; + // log entries will have the name of the tool if one is set + // + void set_tool(tool* t); + + // logs a simple string with the given level + // + void log_string(reason r, level lv, std::string_view s) const; + + // logs a formatted string with the given level + // + template + void log(reason r, level lv, const char* f, Args&&... args) const + { + do_log(false, r, lv, f, std::forward(args)...); + } + + // logs a formatted string with the dump level + // + template + void dump(reason r, const char* f, Args&&... args) const + { + do_log(false, r, level::dump, f, std::forward(args)...); + } + + // logs a formatted string with the trace level + // + template + void trace(reason r, const char* f, Args&&... args) const + { + do_log(false, r, level::trace, f, std::forward(args)...); + } + + // logs a formatted string with the debug level + // + template + void debug(reason r, const char* f, Args&&... args) const + { + do_log(false, r, level::debug, f, std::forward(args)...); + } + + // logs a formatted string with the info level + // + template + void info(reason r, const char* f, Args&&... args) const + { + do_log(false, r, level::info, f, std::forward(args)...); + } + + // logs a formatted string with the warning level + // + template + void warning(reason r, const char* f, Args&&... args) const + { + do_log(false, r, level::warning, f, std::forward(args)...); + } + + // logs a formatted string with the error level + // + template + void error(reason r, const char* f, Args&&... args) const + { + do_log(false, r, level::error, f, std::forward(args)...); + } + + // logs a formatted string with the error level and throws a bailed + // exception, which will exit mob as quickly as it can, interrupting all + // tasks + // + template + [[noreturn]] void bail_out(reason r, const char* f, Args&&... args) const + { + do_log(true, r, level::error, f, std::forward(args)...); + } + + private: + // current task, may be empty + std::string task_; + + // current tool, may be null + const tool* tool_; + + // all logs above end up in here; if `bail` is true, this will throw a + // bailed exception after logging + // + template + void do_log(bool bail, reason r, level lv, const char* f, Args&&... args) const + { + // discard log if it's not enabled and it's not bailing out + if (!bail && !enabled(lv)) + return; + + try { + // formatting string + const std::string s = + fmt::format(f, details::converter>::convert( + std::forward(args))...); + + do_log_impl(bail, r, lv, s); + } + catch (std::exception&) { + // this is typically a bad format string, but there's not a lot + // that can be done except logging to stderr and asserting + + // try to display the format string, but the console is in utf16 + // mode and the string is utf8, so it would have to be converted, + // which could also fail + // + // since pretty much all format strings are ascii anyway, just do + // a ghetto conversion and hope it gives enough info + std::wstring s; + + const char* p = f; + while (*p) { + s += (wchar_t)*p; + ++p; + } + + std::wcerr << "bad format string '" << s << "'\n"; + + if (IsDebuggerPresent()) + DebugBreak(); + } + } + + // all the calls above end up here; calls make_log_string() to get the full + // log line calls emit_log() with it; throws after if `bail` is true + // + void do_log_impl(bool bail, reason r, level lv, std::string_view s) const; + + // formats the log line: adds the timestamp, task name and tool name, if + // any + // + std::string_view make_log_string(reason r, level lv, std::string_view s) const; + + // writes the given string to the console and the log file, and keeps all + // errors and warnings in global lists so they can be dumped just before mob + // exits + // + void emit_log(level lv, std::string_view s) const; + }; + + // global context, convenience + // + inline const context& gcx() + { + return *context::global(); + } + + // called in main() just before mob exits, dumps all errors and warnings seen + // during the build if the console log level was high enough + // + void dump_logs(); + +} // namespace mob diff --git a/src/core/env.cpp b/src/core/env.cpp index 51cc203..4ccf7a1 100644 --- a/src/core/env.cpp +++ b/src/core/env.cpp @@ -1,545 +1,511 @@ #include "pch.h" #include "env.h" +#include "../tools/tools.h" +#include "../utility.h" #include "conf.h" -#include "process.h" -#include "op.h" #include "context.h" -#include "../utility.h" -#include "../tools/tools.h" - -namespace mob -{ - -// retrieves the Visual Studio environment variables for the given architecture; -// this is pretty expensive, so it's called on demand and only once, and is -// stored as a static variable in vs_x86() and vs_x64() below -// -env get_vcvars_env(arch a) -{ - // translate arch to the string needed by vcvars - std::string arch_s; +#include "op.h" +#include "process.h" - switch (a) - { - case arch::x86: - arch_s = "x86"; - break; +namespace mob { - case arch::x64: - arch_s = "amd64"; - break; + // retrieves the Visual Studio environment variables for the given architecture; + // this is pretty expensive, so it's called on demand and only once, and is + // stored as a static variable in vs_x86() and vs_x64() below + // + env get_vcvars_env(arch a) + { + // translate arch to the string needed by vcvars + std::string arch_s; - case arch::dont_care: - default: - gcx().bail_out(context::generic, "get_vcvars_env: bad arch"); - } + switch (a) { + case arch::x86: + arch_s = "x86"; + break; - gcx().trace(context::generic, "looking for vcvars for {}", arch_s); + case arch::x64: + arch_s = "amd64"; + break; + case arch::dont_care: + default: + gcx().bail_out(context::generic, "get_vcvars_env: bad arch"); + } - // the only way to get these variables is to - // 1) run vcvars in a cmd instance, - // 2) call `set`, which outputs all the variables to stdout, and - // 3) parse it - // - // the process class doesn't really have a good way of dealing with this - // and it's not worth adding all this crap to it just for vcvars, so most - // of this is done manually - - // stdout will be redirected to this - const fs::path tmp = make_temp_file(); - - // runs `"vcvarsall.bat" amd64 && set > temp_file` - const std::string cmd = - "\"" + path_to_utf8(vs::vcvars()) + "\" " + arch_s + - " && set > \"" + path_to_utf8(tmp) + "\""; - - // cmd_unicode() is necessary so `set` outputs in utf16 instead of codepage - process::raw(gcx(), cmd) - .cmd_unicode(true) - .run(); - - gcx().trace(context::generic, "reading from {}", tmp); - - // reads the file, converting utf16 to utf8 - std::stringstream ss(op::read_text_file(gcx(), encodings::utf16, tmp)); - op::delete_file(gcx(), tmp); - - // `ss` contains all the variables in utf8 - - env e; - - gcx().trace(context::generic, "parsing variables"); - - for (;;) - { - std::string line; - std::getline(ss, line); - if (!ss) - break; - - const auto sep = line.find('='); - - if (sep == std::string::npos) - continue; - - std::string name = line.substr(0, sep); - std::string value = line.substr(sep + 1); - - gcx().trace(context::generic, "{} = {}", name, value); - e.set(std::move(name), std::move(value)); - } - - return e; -} - - -env env::vs_x86() -{ - static env e = get_vcvars_env(arch::x86); - return e; -} - -env env::vs_x64() -{ - static env e = get_vcvars_env(arch::x64); - return e; -} - -env env::vs(arch a) -{ - switch (a) - { - case arch::x86: - return vs_x86(); - - case arch::x64: - return vs_x64(); - - case arch::dont_care: - return {}; - - default: - gcx().bail_out(context::generic, "bad arch for env"); - } -} - - -env::env() - : own_(false) -{ - // empty env, does not own -} - -env::env(const env& e) - : data_(e.data_), own_(false) -{ - // copy data, does not own -} - -env::env(env&& e) - : data_(std::move(e.data_)), own_(e.own_) -{ - // move data, owns if `e` did -} - -env& env::operator=(const env& e) -{ - // copy data, does not own - data_ = e.data_; - own_ = false; - return *this; -} - -env& env::operator=(env&& e) -{ - // copy data, owns if `e` did - data_ = std::move(e.data_); - own_ = e.own_; - return *this; -} - -env& env::append_path(const fs::path& p) -{ - append_path(std::vector{p}); - return *this; -} - -env& env::prepend_path(const fs::path& p) -{ - prepend_path(std::vector{p}); - return *this; -} - -env& env::prepend_path(const std::vector& v) -{ - change_path(v, prepend); - return *this; -} - -env& env::append_path(const std::vector& v) -{ - change_path(v, append); - return *this; -} - -env& env::change_path(const std::vector& v, flags f) -{ - copy_for_write(); - - std::wstring path; - - switch (f) - { - case replace: - { - // convert to utf16 strings, join with ; - const auto strings = - mob::map(v, [&](auto&& p){ return p.native(); }); - - path = join(strings, L";"); - - break; - } - - case append: - { - auto current = find(L"PATH"); - if (current) - path = *current; - - // append all paths as utf16 strings to the current value, if any - for (auto&& p : v) - { - if (!path.empty()) - path += L";"; - - path += p.native(); - } - - break; - } - - case prepend: - { - auto current = find(L"PATH"); - if (current) - path = *current; - - // prepend all paths as utf16 strings to the current value, if any - for (auto&& p : v) - { - if (!path.empty()) - path = L";" + path; - - path = p.native() + path; - } - - break; - } - } - - set(L"PATH", path, replace); - - return *this; -} - -env& env::set(std::string_view k, std::string_view v, flags f) -{ - copy_for_write(); - set_impl(utf8_to_utf16(k), utf8_to_utf16(v), f); - return *this; -} - -env& env::set(std::wstring k, std::wstring v, flags f) -{ - copy_for_write(); - set_impl(std::move(k), std::move(v), f); - return *this; -} - -void env::set_impl(std::wstring k, std::wstring v, flags f) -{ - auto current = find(k); - - if (!current) - { - data_->vars.emplace(std::move(k), std::move(v)); - return; - } - - switch (f) - { - case replace: - *current = std::move(v); - break; - - case append: - *current += v; - break; - - case prepend: - *current = v + *current; - break; - } -} - -std::string env::get(std::string_view k) const -{ - if (!data_) - return {}; - - auto current = find(utf8_to_utf16(k)); - if (!current) - return {}; - - return utf16_to_utf8(*current); -} - -env::map env::get_map() const -{ - if (!data_) - return {}; - - std::scoped_lock lock(data_->m); - return data_->vars; -} - -void env::create_sys() const -{ - // CreateProcess() wants a string where every key=value is separated by a - // null and also terminated by a null, so there are two null characters at - // the end - - data_->sys.clear(); - - for (auto&& v : data_->vars) - { - data_->sys += v.first + L"=" + v.second; - data_->sys.append(1, L'\0'); - } - - data_->sys.append(1, L'\0'); -} - -std::wstring* env::find(std::wstring_view name) -{ - return const_cast(std::as_const(*this).find(name)); -} - -const std::wstring* env::find(std::wstring_view name) const -{ - if (!data_) - return {}; - - for (auto itor=data_->vars.begin(); itor!=data_->vars.end(); ++itor) - { - if (_wcsicmp(itor->first.c_str(), name.data()) == 0) - return &itor->second; - } - - return {}; -} - -void* env::get_unicode_pointers() const -{ - if (!data_ || data_->vars.empty()) - return nullptr; - - // create string if it doesn't exist - { - std::scoped_lock lock(data_->m); - if (data_->sys.empty()) - create_sys(); - } - - return (void*)data_->sys.c_str(); -} - -void env::copy_for_write() -{ - if (own_) - { - // this is called every time something is about to change; if this - // instance already owns the data, the sys strings must still be cleared - // out so they're recreated if get_unicode_pointers() is every called - if (data_) - data_->sys.clear(); - - return; - } - - if (data_) - { - // remember the shared data - auto shared = data_; - - // create a new owned instance - data_.reset(new data); - - // copying - std::scoped_lock lock(shared->m); - data_->vars = shared->vars; - } - else - { - // creating own, empty data - data_.reset(new data); - } - - // this instance owns the data - own_ = true; -} - - -// mob's environment variables are only retrieved once and are kept in sync -// after that; this must also be thread-safe -static std::mutex g_sys_env_mutex; -static env g_sys_env; -static bool g_sys_env_inited; - - -env this_env::get() -{ - std::scoped_lock lock(g_sys_env_mutex); - - if (g_sys_env_inited) - { - // already done - return g_sys_env; - } - - - // first time, get the variables from the system - - auto free = [](wchar_t* p) { FreeEnvironmentStringsW(p); }; - - auto env_block = std::unique_ptr{ - GetEnvironmentStringsW(), free}; - - // GetEnvironmentStringsW() returns a string where each variable=value - // is separated by a null character - - for (const wchar_t* name = env_block.get(); *name != L'\0'; ) - { - // equal sign - const wchar_t* equal = std::wcschr(name, '='); - - // key - std::wstring key(name, static_cast(equal - name)); - - // value - const wchar_t* value_start = equal + 1; - std::wstring value(value_start); - - // the strings contain all sorts of weird stuff, like variables to - // keep track of the current directory, those start with an equal sign, - // so just ignore them - if (!key.empty()) - g_sys_env.set(key, value); - - // next string is one past end of value to account for null byte - name = value_start + value.length() + 1; - } - - g_sys_env_inited = true; - - return g_sys_env; -} - -void this_env::set(const std::string& k, const std::string& v, env::flags f) -{ - const std::wstring wk = utf8_to_utf16(k); - std::wstring wv = utf8_to_utf16(v); - - switch (f) - { - case env::replace: - { - ::SetEnvironmentVariableW(wk.c_str(), wv.c_str()); - break; - } - - case env::append: - { - const std::wstring current = get_impl(k).value_or(L""); - wv = current + wv; - ::SetEnvironmentVariableW(wk.c_str(), wv.c_str()); - break; - } - - case env::prepend: - { - const std::wstring current = get_impl(k).value_or(L""); - wv = wv + current; - ::SetEnvironmentVariableW(wk.c_str(), wv.c_str()); - break; - } - } - - // keep in sync - { - std::scoped_lock lock(g_sys_env_mutex); - if (g_sys_env_inited) - g_sys_env.set(utf8_to_utf16(k), wv); - } -} - -void this_env::prepend_to_path(const fs::path& p) -{ - gcx().trace(context::generic, "prepending to PATH: {}", p); - set("PATH", path_to_utf8(p) + ";", env::prepend); -} - -void this_env::append_to_path(const fs::path& p) -{ - gcx().trace(context::generic, "appending to PATH: {}", p); - set("PATH", ";" + path_to_utf8(p), env::append); -} - -std::string this_env::get(const std::string& name) -{ - auto v = get_impl(name); - if (!v) - { - gcx().bail_out(context::generic, - "environment variable {} doesn't exist", name); - } - - return utf16_to_utf8(*v); -} - -std::optional this_env::get_opt(const std::string& name) -{ - auto v = get_impl(name); - if (v) - return utf16_to_utf8(*v); - else - return {}; -} - -std::optional this_env::get_impl(const std::string& k) -{ - const std::wstring wk = utf8_to_utf16(k); - - const std::size_t buffer_size = GetEnvironmentVariableW( - wk.c_str(), nullptr, 0); - - if (buffer_size == 0) - return {}; - - auto buffer = std::make_unique(buffer_size + 1); - std::fill(buffer.get(), buffer.get() + buffer_size + 1, 0); - - const std::size_t written = GetEnvironmentVariableW( - wk.c_str(), buffer.get(), static_cast(buffer_size)); - - if (written == 0) - return {}; - - MOB_ASSERT((written + 1) == buffer_size); - - return std::wstring(buffer.get(), buffer.get() + written); -} - -} // namespace + gcx().trace(context::generic, "looking for vcvars for {}", arch_s); + + // the only way to get these variables is to + // 1) run vcvars in a cmd instance, + // 2) call `set`, which outputs all the variables to stdout, and + // 3) parse it + // + // the process class doesn't really have a good way of dealing with this + // and it's not worth adding all this crap to it just for vcvars, so most + // of this is done manually + + // stdout will be redirected to this + const fs::path tmp = make_temp_file(); + + // runs `"vcvarsall.bat" amd64 && set > temp_file` + const std::string cmd = "\"" + path_to_utf8(vs::vcvars()) + "\" " + arch_s + + " && set > \"" + path_to_utf8(tmp) + "\""; + + // cmd_unicode() is necessary so `set` outputs in utf16 instead of codepage + process::raw(gcx(), cmd).cmd_unicode(true).run(); + + gcx().trace(context::generic, "reading from {}", tmp); + + // reads the file, converting utf16 to utf8 + std::stringstream ss(op::read_text_file(gcx(), encodings::utf16, tmp)); + op::delete_file(gcx(), tmp); + + // `ss` contains all the variables in utf8 + + env e; + + gcx().trace(context::generic, "parsing variables"); + + for (;;) { + std::string line; + std::getline(ss, line); + if (!ss) + break; + + const auto sep = line.find('='); + + if (sep == std::string::npos) + continue; + + std::string name = line.substr(0, sep); + std::string value = line.substr(sep + 1); + + gcx().trace(context::generic, "{} = {}", name, value); + e.set(std::move(name), std::move(value)); + } + + return e; + } + + env env::vs_x86() + { + static env e = get_vcvars_env(arch::x86); + return e; + } + + env env::vs_x64() + { + static env e = get_vcvars_env(arch::x64); + return e; + } + + env env::vs(arch a) + { + switch (a) { + case arch::x86: + return vs_x86(); + + case arch::x64: + return vs_x64(); + + case arch::dont_care: + return {}; + + default: + gcx().bail_out(context::generic, "bad arch for env"); + } + } + + env::env() : own_(false) + { + // empty env, does not own + } + + env::env(const env& e) : data_(e.data_), own_(false) + { + // copy data, does not own + } + + env::env(env&& e) : data_(std::move(e.data_)), own_(e.own_) + { + // move data, owns if `e` did + } + + env& env::operator=(const env& e) + { + // copy data, does not own + data_ = e.data_; + own_ = false; + return *this; + } + + env& env::operator=(env&& e) + { + // copy data, owns if `e` did + data_ = std::move(e.data_); + own_ = e.own_; + return *this; + } + + env& env::append_path(const fs::path& p) + { + append_path(std::vector{p}); + return *this; + } + + env& env::prepend_path(const fs::path& p) + { + prepend_path(std::vector{p}); + return *this; + } + + env& env::prepend_path(const std::vector& v) + { + change_path(v, prepend); + return *this; + } + + env& env::append_path(const std::vector& v) + { + change_path(v, append); + return *this; + } + + env& env::change_path(const std::vector& v, flags f) + { + copy_for_write(); + + std::wstring path; + + switch (f) { + case replace: { + // convert to utf16 strings, join with ; + const auto strings = mob::map(v, [&](auto&& p) { + return p.native(); + }); + + path = join(strings, L";"); + + break; + } + + case append: { + auto current = find(L"PATH"); + if (current) + path = *current; + + // append all paths as utf16 strings to the current value, if any + for (auto&& p : v) { + if (!path.empty()) + path += L";"; + + path += p.native(); + } + + break; + } + + case prepend: { + auto current = find(L"PATH"); + if (current) + path = *current; + + // prepend all paths as utf16 strings to the current value, if any + for (auto&& p : v) { + if (!path.empty()) + path = L";" + path; + + path = p.native() + path; + } + + break; + } + } + + set(L"PATH", path, replace); + + return *this; + } + + env& env::set(std::string_view k, std::string_view v, flags f) + { + copy_for_write(); + set_impl(utf8_to_utf16(k), utf8_to_utf16(v), f); + return *this; + } + + env& env::set(std::wstring k, std::wstring v, flags f) + { + copy_for_write(); + set_impl(std::move(k), std::move(v), f); + return *this; + } + + void env::set_impl(std::wstring k, std::wstring v, flags f) + { + auto current = find(k); + + if (!current) { + data_->vars.emplace(std::move(k), std::move(v)); + return; + } + + switch (f) { + case replace: + *current = std::move(v); + break; + + case append: + *current += v; + break; + + case prepend: + *current = v + *current; + break; + } + } + + std::string env::get(std::string_view k) const + { + if (!data_) + return {}; + + auto current = find(utf8_to_utf16(k)); + if (!current) + return {}; + + return utf16_to_utf8(*current); + } + + env::map env::get_map() const + { + if (!data_) + return {}; + + std::scoped_lock lock(data_->m); + return data_->vars; + } + + void env::create_sys() const + { + // CreateProcess() wants a string where every key=value is separated by a + // null and also terminated by a null, so there are two null characters at + // the end + + data_->sys.clear(); + + for (auto&& v : data_->vars) { + data_->sys += v.first + L"=" + v.second; + data_->sys.append(1, L'\0'); + } + + data_->sys.append(1, L'\0'); + } + + std::wstring* env::find(std::wstring_view name) + { + return const_cast(std::as_const(*this).find(name)); + } + + const std::wstring* env::find(std::wstring_view name) const + { + if (!data_) + return {}; + + for (auto itor = data_->vars.begin(); itor != data_->vars.end(); ++itor) { + if (_wcsicmp(itor->first.c_str(), name.data()) == 0) + return &itor->second; + } + + return {}; + } + + void* env::get_unicode_pointers() const + { + if (!data_ || data_->vars.empty()) + return nullptr; + + // create string if it doesn't exist + { + std::scoped_lock lock(data_->m); + if (data_->sys.empty()) + create_sys(); + } + + return (void*)data_->sys.c_str(); + } + + void env::copy_for_write() + { + if (own_) { + // this is called every time something is about to change; if this + // instance already owns the data, the sys strings must still be cleared + // out so they're recreated if get_unicode_pointers() is every called + if (data_) + data_->sys.clear(); + + return; + } + + if (data_) { + // remember the shared data + auto shared = data_; + + // create a new owned instance + data_.reset(new data); + + // copying + std::scoped_lock lock(shared->m); + data_->vars = shared->vars; + } + else { + // creating own, empty data + data_.reset(new data); + } + + // this instance owns the data + own_ = true; + } + + // mob's environment variables are only retrieved once and are kept in sync + // after that; this must also be thread-safe + static std::mutex g_sys_env_mutex; + static env g_sys_env; + static bool g_sys_env_inited; + + env this_env::get() + { + std::scoped_lock lock(g_sys_env_mutex); + + if (g_sys_env_inited) { + // already done + return g_sys_env; + } + + // first time, get the variables from the system + + auto free = [](wchar_t* p) { + FreeEnvironmentStringsW(p); + }; + + auto env_block = + std::unique_ptr{GetEnvironmentStringsW(), free}; + + // GetEnvironmentStringsW() returns a string where each variable=value + // is separated by a null character + + for (const wchar_t* name = env_block.get(); *name != L'\0';) { + // equal sign + const wchar_t* equal = std::wcschr(name, '='); + + // key + std::wstring key(name, static_cast(equal - name)); + + // value + const wchar_t* value_start = equal + 1; + std::wstring value(value_start); + + // the strings contain all sorts of weird stuff, like variables to + // keep track of the current directory, those start with an equal sign, + // so just ignore them + if (!key.empty()) + g_sys_env.set(key, value); + + // next string is one past end of value to account for null byte + name = value_start + value.length() + 1; + } + + g_sys_env_inited = true; + + return g_sys_env; + } + + void this_env::set(const std::string& k, const std::string& v, env::flags f) + { + const std::wstring wk = utf8_to_utf16(k); + std::wstring wv = utf8_to_utf16(v); + + switch (f) { + case env::replace: { + ::SetEnvironmentVariableW(wk.c_str(), wv.c_str()); + break; + } + + case env::append: { + const std::wstring current = get_impl(k).value_or(L""); + wv = current + wv; + ::SetEnvironmentVariableW(wk.c_str(), wv.c_str()); + break; + } + + case env::prepend: { + const std::wstring current = get_impl(k).value_or(L""); + wv = wv + current; + ::SetEnvironmentVariableW(wk.c_str(), wv.c_str()); + break; + } + } + + // keep in sync + { + std::scoped_lock lock(g_sys_env_mutex); + if (g_sys_env_inited) + g_sys_env.set(utf8_to_utf16(k), wv); + } + } + + void this_env::prepend_to_path(const fs::path& p) + { + gcx().trace(context::generic, "prepending to PATH: {}", p); + set("PATH", path_to_utf8(p) + ";", env::prepend); + } + + void this_env::append_to_path(const fs::path& p) + { + gcx().trace(context::generic, "appending to PATH: {}", p); + set("PATH", ";" + path_to_utf8(p), env::append); + } + + std::string this_env::get(const std::string& name) + { + auto v = get_impl(name); + if (!v) { + gcx().bail_out(context::generic, "environment variable {} doesn't exist", + name); + } + + return utf16_to_utf8(*v); + } + + std::optional this_env::get_opt(const std::string& name) + { + auto v = get_impl(name); + if (v) + return utf16_to_utf8(*v); + else + return {}; + } + + std::optional this_env::get_impl(const std::string& k) + { + const std::wstring wk = utf8_to_utf16(k); + + const std::size_t buffer_size = GetEnvironmentVariableW(wk.c_str(), nullptr, 0); + + if (buffer_size == 0) + return {}; + + auto buffer = std::make_unique(buffer_size + 1); + std::fill(buffer.get(), buffer.get() + buffer_size + 1, 0); + + const std::size_t written = GetEnvironmentVariableW( + wk.c_str(), buffer.get(), static_cast(buffer_size)); + + if (written == 0) + return {}; + + MOB_ASSERT((written + 1) == buffer_size); + + return std::wstring(buffer.get(), buffer.get() + written); + } + +} // namespace mob diff --git a/src/core/env.h b/src/core/env.h index a832216..4ead684 100644 --- a/src/core/env.h +++ b/src/core/env.h @@ -2,154 +2,140 @@ #include "../utility.h" -namespace mob -{ - -// a set of environment variables; copy-on-write because this gets copied a lot -// -class env -{ -public: - using map = std::map; - - // used in set(); replaces, appends or prepends to a variable if it already - // exists - // - enum flags - { - replace = 1, - append, - prepend - }; - - // Visual Studio environment variables for 32-bit - // - static env vs_x86(); - - // Visual Studio environment variables for 64-bit - // - static env vs_x64(); - - // Visual Studio environment variables for the given architecture - // - static env vs(arch a); - - - // empty set - // - env(); - - // handle ref count - // - env(const env& e); - env(env&& e); - env& operator=(const env& e); - env& operator=(env&& e); - - // prepends to PATH - // - env& prepend_path(const fs::path& p); - env& prepend_path(const std::vector& v); - - // appends to PATH - // - env& append_path(const fs::path& p); - env& append_path(const std::vector& v); - - // sets k=v - // - env& set(std::string_view k, std::string_view v, flags f=replace); - env& set(std::wstring k, std::wstring v, flags f=replace); - - // returns the variable' value, empty if not found - // - std::string get(std::string_view k) const; - - // map of variables - // - map get_map() const; - - // passed to CreateProcess() in the process class; returns a pointer to a - // block of utf16 strings, owned by this, created on demand - // - void* get_unicode_pointers() const; - -private: - // shared between copies - // - struct data - { - std::mutex m; - map vars; - - // unicode strings, see get_unicode_pointers() - mutable std::wstring sys; - }; - - // shared data - std::shared_ptr data_; - - // whether this instance owns the data, set to true in copy_for_write() - // when the data must be modified - bool own_; - - - // creates the unicode strings - // - void create_sys() const; - - // returns a pointer inside the map, null if not found - // - std::wstring* find(std::wstring_view name); - const std::wstring* find(std::wstring_view name) const; - - // called by set(), sets the value in the map - // - void set_impl(std::wstring k, std::wstring v, flags f); - - // duplicates the data, sets own_=true - // - void copy_for_write(); - - // called by the various *_path() functions, actually changes the PATH - // value - // - env& change_path(const std::vector& v, flags f); -}; - - -// represents mob's environment variables -// -struct this_env -{ - // sets a variable - // - static void set( - const std::string& k, - const std::string& v, - env::flags f=env::replace); - - // changes PATH - // - static void prepend_to_path(const fs::path& p); - static void append_to_path(const fs::path& p); - - // returns mob's environment variables - // - static env get(); - - // returns a specific variable; bails out if it doesn't exist - // - static std::string get(const std::string& k); - - // returns a specific variable, or empty if it doesn't exist - // - static std::optional get_opt(const std::string& k); - -private: - // used by get() and get_opt(), does the actual work - // - static std::optional get_impl(const std::string& k); -}; - -} // namespace +namespace mob { + + // a set of environment variables; copy-on-write because this gets copied a lot + // + class env { + public: + using map = std::map; + + // used in set(); replaces, appends or prepends to a variable if it already + // exists + // + enum flags { replace = 1, append, prepend }; + + // Visual Studio environment variables for 32-bit + // + static env vs_x86(); + + // Visual Studio environment variables for 64-bit + // + static env vs_x64(); + + // Visual Studio environment variables for the given architecture + // + static env vs(arch a); + + // empty set + // + env(); + + // handle ref count + // + env(const env& e); + env(env&& e); + env& operator=(const env& e); + env& operator=(env&& e); + + // prepends to PATH + // + env& prepend_path(const fs::path& p); + env& prepend_path(const std::vector& v); + + // appends to PATH + // + env& append_path(const fs::path& p); + env& append_path(const std::vector& v); + + // sets k=v + // + env& set(std::string_view k, std::string_view v, flags f = replace); + env& set(std::wstring k, std::wstring v, flags f = replace); + + // returns the variable' value, empty if not found + // + std::string get(std::string_view k) const; + + // map of variables + // + map get_map() const; + + // passed to CreateProcess() in the process class; returns a pointer to a + // block of utf16 strings, owned by this, created on demand + // + void* get_unicode_pointers() const; + + private: + // shared between copies + // + struct data { + std::mutex m; + map vars; + + // unicode strings, see get_unicode_pointers() + mutable std::wstring sys; + }; + + // shared data + std::shared_ptr data_; + + // whether this instance owns the data, set to true in copy_for_write() + // when the data must be modified + bool own_; + + // creates the unicode strings + // + void create_sys() const; + + // returns a pointer inside the map, null if not found + // + std::wstring* find(std::wstring_view name); + const std::wstring* find(std::wstring_view name) const; + + // called by set(), sets the value in the map + // + void set_impl(std::wstring k, std::wstring v, flags f); + + // duplicates the data, sets own_=true + // + void copy_for_write(); + + // called by the various *_path() functions, actually changes the PATH + // value + // + env& change_path(const std::vector& v, flags f); + }; + + // represents mob's environment variables + // + struct this_env { + // sets a variable + // + static void set(const std::string& k, const std::string& v, + env::flags f = env::replace); + + // changes PATH + // + static void prepend_to_path(const fs::path& p); + static void append_to_path(const fs::path& p); + + // returns mob's environment variables + // + static env get(); + + // returns a specific variable; bails out if it doesn't exist + // + static std::string get(const std::string& k); + + // returns a specific variable, or empty if it doesn't exist + // + static std::optional get_opt(const std::string& k); + + private: + // used by get() and get_opt(), does the actual work + // + static std::optional get_impl(const std::string& k); + }; + +} // namespace mob diff --git a/src/core/ini.cpp b/src/core/ini.cpp index fe5ef81..3378168 100644 --- a/src/core/ini.cpp +++ b/src/core/ini.cpp @@ -1,330 +1,287 @@ #include "pch.h" #include "ini.h" -#include "paths.h" -#include "context.h" -#include "env.h" -#include "conf.h" #include "../tasks/task_manager.h" #include "../utility/string.h" +#include "conf.h" +#include "context.h" +#include "env.h" +#include "paths.h" -namespace mob -{ - -template -void ini_error( - const ini_data& ini, std::size_t line, std::string_view f, Args&&... args) -{ - gcx().bail_out(context::conf, - "{}:{}: {}", - path_to_utf8(ini.path), (line + 1), - fmt::format(f, std::forward(args)...)); -} - - -ini_data::kv_map& ini_data::get_section(std::string_view name) -{ - for (auto itor=sections.begin(); itor!=sections.end(); ++itor) - { - if (itor->first == name) - return itor->second; - } - - sections.push_back({std::string(name), kv_map()}); - return sections.back().second; -} - -void ini_data::set(std::string_view section, std::string key, std::string value) -{ - auto& s = get_section(section); - s.emplace(std::move(key), std::move(value)); -} - - - -std::string default_ini_filename() -{ - return "mob.ini"; -} - -std::vector find_inis( - bool auto_detect, const std::vector& from_cl, bool verbose) -{ - // the string is just for verbose - std::vector> v; - - // adds a path to the vector; if the path already exists, moves it to - // the last element - auto add_or_move_up = [&](std::string where, fs::path p) - { - for (auto itor=v.begin(); itor!=v.end(); ++itor) - { - if (fs::equivalent(p, itor->second)) - { - auto pair = std::move(*itor); - v.erase(itor); - v.push_back({where + ", was " + pair.first, p}); - return; - } - } - - v.push_back({where, p}); - }; - - // whether the ini is already in the list - auto ini_already_found = [&](auto&& p) - { - for (auto itor=v.begin(); itor!=v.end(); ++itor) - { - if (fs::equivalent(p, itor->second)) - return true; - } - - return false; - }; - - - fs::path master; - - // auto detect from exe directory - if (auto_detect) - { - if (verbose) - { - const auto r = find_root(verbose); - u8cout << "root is " << path_to_utf8(r) << "\n"; - } - - master = find_in_root(default_ini_filename()); - - if (verbose) - u8cout << "found master " << path_to_utf8(master) << "\n"; - - v.push_back({"master", master}); - } - - - // MOBINI environment variable - if (auto e=this_env::get_opt("MOBINI")) - { - if (verbose) - u8cout << "found env MOBINI: '" << *e << "'\n"; - - for (auto&& i : split(*e, ";")) - { - auto p = fs::path(i); - if (!fs::exists(p)) - { - u8cerr << "ini from env MOBINI " << i << " not found\n"; - throw bailed(); - } - - p = fs::canonical(p); - - if (verbose) - u8cout << "ini from env: " << path_to_utf8(p) << "\n"; - - add_or_move_up("env", p); - } - } - - - // auto detect from the current directory - if (auto_detect) - { - MOB_ASSERT(!master.empty()); - - auto cwd = fs::current_path(); - - while (!cwd.empty()) - { - const auto in_cwd = cwd / default_ini_filename(); - - if (fs::exists(in_cwd) && !ini_already_found(in_cwd)) - { - if (verbose) - u8cout << "also found in cwd " << path_to_utf8(in_cwd) << "\n"; - - v.push_back({ "cwd", fs::canonical(in_cwd) }); - break; - } - - const auto parent = cwd.parent_path(); - if (cwd == parent) - break; - - cwd = parent; - } - } - - - // command line - for (auto&& i : from_cl) - { - auto p = fs::path(i); - if (!fs::exists(p)) - { - u8cerr << "ini " << i << " not found\n"; - throw bailed(); - } - - p = fs::canonical(p); +namespace mob { + + template + void ini_error(const ini_data& ini, std::size_t line, std::string_view f, + Args&&... args) + { + gcx().bail_out(context::conf, "{}:{}: {}", path_to_utf8(ini.path), (line + 1), + fmt::format(f, std::forward(args)...)); + } + + ini_data::kv_map& ini_data::get_section(std::string_view name) + { + for (auto itor = sections.begin(); itor != sections.end(); ++itor) { + if (itor->first == name) + return itor->second; + } + + sections.push_back({std::string(name), kv_map()}); + return sections.back().second; + } + + void ini_data::set(std::string_view section, std::string key, std::string value) + { + auto& s = get_section(section); + s.emplace(std::move(key), std::move(value)); + } + + std::string default_ini_filename() + { + return "mob.ini"; + } + + std::vector + find_inis(bool auto_detect, const std::vector& from_cl, bool verbose) + { + // the string is just for verbose + std::vector> v; + + // adds a path to the vector; if the path already exists, moves it to + // the last element + auto add_or_move_up = [&](std::string where, fs::path p) { + for (auto itor = v.begin(); itor != v.end(); ++itor) { + if (fs::equivalent(p, itor->second)) { + auto pair = std::move(*itor); + v.erase(itor); + v.push_back({where + ", was " + pair.first, p}); + return; + } + } + + v.push_back({where, p}); + }; + + // whether the ini is already in the list + auto ini_already_found = [&](auto&& p) { + for (auto itor = v.begin(); itor != v.end(); ++itor) { + if (fs::equivalent(p, itor->second)) + return true; + } + + return false; + }; + + fs::path master; + + // auto detect from exe directory + if (auto_detect) { + if (verbose) { + const auto r = find_root(verbose); + u8cout << "root is " << path_to_utf8(r) << "\n"; + } + + master = find_in_root(default_ini_filename()); + + if (verbose) + u8cout << "found master " << path_to_utf8(master) << "\n"; + + v.push_back({"master", master}); + } + + // MOBINI environment variable + if (auto e = this_env::get_opt("MOBINI")) { + if (verbose) + u8cout << "found env MOBINI: '" << *e << "'\n"; + + for (auto&& i : split(*e, ";")) { + auto p = fs::path(i); + if (!fs::exists(p)) { + u8cerr << "ini from env MOBINI " << i << " not found\n"; + throw bailed(); + } + + p = fs::canonical(p); + + if (verbose) + u8cout << "ini from env: " << path_to_utf8(p) << "\n"; + + add_or_move_up("env", p); + } + } + + // auto detect from the current directory + if (auto_detect) { + MOB_ASSERT(!master.empty()); + + auto cwd = fs::current_path(); + + while (!cwd.empty()) { + const auto in_cwd = cwd / default_ini_filename(); + + if (fs::exists(in_cwd) && !ini_already_found(in_cwd)) { + if (verbose) + u8cout << "also found in cwd " << path_to_utf8(in_cwd) << "\n"; + + v.push_back({"cwd", fs::canonical(in_cwd)}); + break; + } + + const auto parent = cwd.parent_path(); + if (cwd == parent) + break; + + cwd = parent; + } + } + + // command line + for (auto&& i : from_cl) { + auto p = fs::path(i); + if (!fs::exists(p)) { + u8cerr << "ini " << i << " not found\n"; + throw bailed(); + } + + p = fs::canonical(p); + + if (verbose) + u8cout << "ini from command line: " << path_to_utf8(p) << "\n"; + + add_or_move_up("cl", p); + } + + if (verbose) { + u8cout << "\nhigher number overrides lower\n"; + + for (std::size_t i = 0; i < v.size(); ++i) { + u8cout << " " << (i + 1) << ") " << path_to_utf8(v[i].second) << " (" + << v[i].first << ")\n"; + } + } - if (verbose) - u8cout << "ini from command line: " << path_to_utf8(p) << "\n"; + return map(v, [&](auto&& p) { + return p.second; + }); + } - add_or_move_up("cl", p); - } + std::vector read_ini(const fs::path& ini) + { + std::ifstream in(ini); + std::vector lines; - if (verbose) - { - u8cout << "\nhigher number overrides lower\n"; - - for (std::size_t i=0; i read_ini(const fs::path& ini) -{ - std::ifstream in(ini); + const auto sep = line.find("="); + if (sep == std::string::npos) + ini_error(ini, i, "bad line '{}'", line); - std::vector lines; + const std::string k = trim_copy(line.substr(0, sep)); + const std::string v = trim_copy(line.substr(sep + 1)); - for (;;) - { - std::string line; - std::getline(in, line); - trim(line); - - if (!in) - break; + if (k.empty()) + ini_error(ini, i, "bad line '{}'", line); - lines.push_back(std::move(line)); - } - - if (in.bad()) - gcx().bail_out(context::conf, "failed to read ini {}", ini); + if (section == "aliases") { + tm.add_alias(k, split_quoted(v, " ")); + } + else if (task.empty()) { + ini.set(section, k, v); + } + else { + if (!tm.valid_task_name(task)) + ini_error(ini, i, "no task matching '{}' found", task); - return lines; -} + ini.set(task + ":" + section, k, v); + } + } + + void parse_section(ini_data& ini, std::size_t& i, + const std::vector& lines, + const std::string& section_string) + { + std::string task, section; + + const auto col = section_string.find(":"); + + if (col == std::string::npos) { + section = section_string; + } + else { + task = section_string.substr(0, col); + section = section_string.substr(col + 1); + } + + ++i; + + for (;;) { + if (i >= lines.size() || lines[i][0] == '[') + break; + + const auto& line = lines[i]; -void parse_line( - ini_data& ini, std::size_t i, const std::string& line, - const std::string& task, const std::string& section) -{ - auto& tm = task_manager::instance(); - - const auto sep = line.find("="); - if (sep == std::string::npos) - ini_error(ini, i, "bad line '{}'", line); - - const std::string k = trim_copy(line.substr(0, sep)); - const std::string v = trim_copy(line.substr(sep + 1)); - - if (k.empty()) - ini_error(ini, i, "bad line '{}'", line); - - if (section == "aliases") - { - tm.add_alias(k, split_quoted(v, " ")); - } - else if (task.empty()) - { - ini.set(section, k, v); - } - else - { - if (!tm.valid_task_name(task)) - ini_error(ini, i, "no task matching '{}' found", task); - - ini.set(task + ":" + section, k, v); - } -} - -void parse_section( - ini_data& ini, std::size_t& i, const std::vector& lines, - const std::string& section_string) -{ - std::string task, section; - - const auto col = section_string.find(":"); - - if (col == std::string::npos) - { - section = section_string; - } - else - { - task = section_string.substr(0, col); - section = section_string.substr(col + 1); - } - - - ++i; - - for (;;) - { - if (i >= lines.size() || lines[i][0] == '[') - break; - - const auto& line = lines[i]; - - // empty or comment - if (line.empty() || line[0] == '#' || line[0] == ';') - { - ++i; - continue; - } - - parse_line(ini, i, line, task, section); - ++i; - } -} - -ini_data parse_ini(const fs::path& path) -{ - gcx().debug(context::conf, "using ini at {}", path); - - ini_data ini; - ini.path = path; - - const auto lines = read_ini(path); - std::size_t i = 0; - - for (;;) - { - if (i >= lines.size()) - break; - - const auto& line = lines[i]; - - // empty or comment - if (line.empty() || line[0] == '#' || line[0] == ';') - { - ++i; - continue; - } - - if (line.starts_with("[") && line.ends_with("]")) - { - const std::string name = line.substr(1, line.size() - 2); - parse_section(ini, i, lines, name); - } - else - { - ini_error(ini, i, "bad line '{}'", line); - } - } - - return ini; -} - -} // namespace + // empty or comment + if (line.empty() || line[0] == '#' || line[0] == ';') { + ++i; + continue; + } + + parse_line(ini, i, line, task, section); + ++i; + } + } + + ini_data parse_ini(const fs::path& path) + { + gcx().debug(context::conf, "using ini at {}", path); + + ini_data ini; + ini.path = path; + + const auto lines = read_ini(path); + std::size_t i = 0; + + for (;;) { + if (i >= lines.size()) + break; + + const auto& line = lines[i]; + + // empty or comment + if (line.empty() || line[0] == '#' || line[0] == ';') { + ++i; + continue; + } + + if (line.starts_with("[") && line.ends_with("]")) { + const std::string name = line.substr(1, line.size() - 2); + parse_section(ini, i, lines, name); + } + else { + ini_error(ini, i, "bad line '{}'", line); + } + } + + return ini; + } + +} // namespace mob diff --git a/src/core/ini.h b/src/core/ini.h index 4957078..26920cd 100644 --- a/src/core/ini.h +++ b/src/core/ini.h @@ -1,32 +1,28 @@ #pragma once -namespace mob -{ +namespace mob { -std::string default_ini_filename(); + std::string default_ini_filename(); -std::vector find_inis( - bool auto_detect, const std::vector& from_cl, - bool verbose); + std::vector + find_inis(bool auto_detect, const std::vector& from_cl, bool verbose); + struct ini_data { + using alias_patterns = std::vector; + using aliases_map = std::map; -struct ini_data -{ - using alias_patterns = std::vector; - using aliases_map = std::map; + using kv_map = std::map; + using sections_vector = std::vector>; - using kv_map = std::map; - using sections_vector = std::vector>; + fs::path path; + aliases_map aliases; + sections_vector sections; - fs::path path; - aliases_map aliases; - sections_vector sections; + kv_map& get_section(std::string_view name); + void set(std::string_view section, std::string key, std::string value); + }; - kv_map& get_section(std::string_view name); - void set(std::string_view section, std::string key, std::string value); -}; + ini_data parse_ini(const fs::path& ini); + std::string default_ini_filename(); -ini_data parse_ini(const fs::path& ini); -std::string default_ini_filename(); - -} // namespace +} // namespace mob diff --git a/src/core/op.cpp b/src/core/op.cpp index 5de359d..bf0a051 100644 --- a/src/core/op.cpp +++ b/src/core/op.cpp @@ -1,698 +1,617 @@ #include "pch.h" #include "op.h" +#include "../tools/tools.h" +#include "../utility.h" #include "conf.h" #include "context.h" -#include "../utility.h" -#include "../tools/tools.h" -namespace mob::op -{ - -// most of the functions from the header will check the paths, return early -// for dry run, and then forward to these to do the actual work -void do_touch(const context& cx, const fs::path& p); -void do_create_directories(const context& cx, const fs::path& p); -void do_delete_directory(const context& cx, const fs::path& p); -void do_delete_file(const context& cx, const fs::path& p); -void do_copy_file_to_dir(const context& cx, const fs::path& f, const fs::path& d); -void do_copy_file_to_file(const context& cx, const fs::path& f, const fs::path& d); -void do_remove_readonly(const context& cx, const fs::path& p); -void do_rename(const context& cx, const fs::path& src, const fs::path& dest); - -// checks whether the path is valid, bails out if not -// -void check(const context& cx, const fs::path& p, flags f); - - -void touch(const context& cx, const fs::path& p, flags f) -{ - cx.trace(context::fs, "touching {}", p); - check(cx, p, f); - - if (!conf().global().dry()) - do_touch(cx, p); -} - -void create_directories(const context& cx, const fs::path& p, flags f) -{ - cx.trace(context::fs, "creating dir {}", p); - check(cx, p, f); - - if (!conf().global().dry()) - do_create_directories(cx, p); -} - -void delete_directory(const context& cx, const fs::path& p, flags f) -{ - cx.trace(context::fs, "deleting dir {}", p); - check(cx, p, f); - - if (!fs::exists(p)) - { - if (f & optional) - { - cx.trace(context::fs, - "not deleting dir {}, doesn't exist (optional)", p); - - return; - } - - cx.bail_out(context::fs, "can't delete dir {}, doesn't exist", p); - } - - if (fs::exists(p) && !fs::is_directory(p)) - cx.bail_out(context::fs, "{} is not a dir", p); - - if (!conf().global().dry()) - do_delete_directory(cx, p); -} - -void delete_file(const context& cx, const fs::path& p, flags f) -{ - cx.trace(context::fs, "deleting file {}", p); - check(cx, p, f); - - if (!fs::exists(p)) - { - if (f & optional) - { - cx.trace(context::fs, - "not deleting file {}, doesn't exist (optional)", p); - - return; - } - - cx.bail_out(context::fs, "can't delete file {}, doesn't exist", p); - } - - if (fs::exists(p) && !fs::is_regular_file(p)) - { - if (f & optional) - cx.warning(context::fs, "can't delete {}, not a file", p); - else - cx.bail_out(context::fs, "can't delete {}, not a file", p); - - return; - } - - if (!conf().global().dry()) - do_delete_file(cx, p); -} - -void delete_file_glob(const context& cx, const fs::path& glob, flags f) -{ - cx.trace(context::fs, "deleting glob {}", glob); - - const auto parent = glob.parent_path(); - const auto wildcard = glob.filename().native(); - - if (!fs::exists(parent)) - return; - - for (auto&& e : fs::directory_iterator(parent)) - { - const auto p = e.path(); - const auto name = p.filename().native(); - - if (!PathMatchSpecW(name.c_str(), wildcard.c_str())) - { - cx.trace(context::fs, - "{} did not match {}; skipping", name, wildcard); - - continue; - } - - delete_file(cx, p, f); - } -} - -void remove_readonly(const context& cx, const fs::path& dir, flags f) -{ - cx.trace(context::fs, "removing read-only from {}", dir); - check(cx, dir, f); - - if (!conf().global().dry()) - { - for (auto&& p : fs::recursive_directory_iterator(dir)) - { - if (fs::is_regular_file(p)) - do_remove_readonly(cx, p); - } - } -} - -bool is_source_better( - const context& cx, const fs::path& src, const fs::path& dest) -{ - if (!fs::exists(dest)) - { - cx.trace(context::fs, "target {} doesn't exist; copying", dest); - return true; - } - - std::error_code ec; - - const auto src_size = fs::file_size(src, ec); - if (ec) - { - cx.warning(context::fs, - "failed to get size of {}, {}; forcing copy", - src, ec.message()); - - return true; - } - - const auto dest_size = fs::file_size(dest, ec); - if (ec) - { - cx.warning(context::fs, - "failed to get size of {}, {}; forcing copy", - dest, ec.message()); - - return true; - } - - if (src_size != dest_size) - { - cx.trace(context::fs, - "src {} bytes, dest {} bytes; different, copying", - src, src_size, dest, dest_size); - - return true; - } - - - const auto src_time = fs::last_write_time(src, ec); - if (ec) - { - cx.warning(context::fs, - "failed to get time of {}, {}; forcing copy", - src, ec.message()); - - return true; - } - - const auto dest_time = fs::last_write_time(dest, ec); - if (ec) - { - cx.warning(context::fs, - "failed to get time of {}, {}; forcing copy", - dest, ec.message()); - - return true; - } - - if (src_time > dest_time) - { - cx.trace(context::fs, - "src {} is newer than dest {}; copying", - src, dest); - - return true; - } - - // same size, same date - return false; -} - -void rename(const context& cx, const fs::path& src, const fs::path& dest, flags f) -{ - check(cx, src, f); - check(cx, dest, f); - - if (fs::exists(dest)) - { - cx.bail_out(context::fs, - "can't rename {} to {}, already exists", src, dest); - } - - cx.trace(context::fs, "renaming {} to {}", src, dest); - - if (!conf().global().dry()) - do_rename(cx, src, dest); -} - -void move_to_directory( - const context& cx, const fs::path& src, const fs::path& dest_dir, flags f) -{ - check(cx, src, f); - check(cx, dest_dir, f); - - const auto target = dest_dir / src.filename(); - - if (fs::exists(target)) - { - cx.bail_out(context::fs, - "can't move {} to directory {}, {} already exists", - src, dest_dir, target); - } - - cx.trace(context::fs, "moving {} to {}", src, target); - - if (!conf().global().dry()) - do_rename(cx, src, target); -} - -void copy_file_to_dir_if_better( - const context& cx, const fs::path& file, const fs::path& dir, flags f) -{ - check(cx, file, f); - check(cx, dir, f); - - if (file.u8string().find(u8"*") != std::string::npos) - cx.bail_out(context::fs, "{} contains a glob", file); - - if (!conf().global().dry()) - { - if (!fs::exists(file) || !fs::is_regular_file(file)) - { - if (f & optional) - { - cx.trace(context::fs, - "not copying {}, doesn't exist (optional)", file); - - return; - } - - cx.bail_out(context::fs, "can't copy {}, not a file", file); - } - - if (fs::exists(dir) && !fs::is_directory(dir)) - cx.bail_out(context::fs, "can't copy to {}, not a dir", dir); - } - - const auto target = dir / file.filename(); - if (is_source_better(cx, file, target)) - { - cx.trace(context::fs, "{} -> {}", file, dir); - - if (!conf().global().dry()) - do_copy_file_to_dir(cx, file, dir); - } - else - { - cx.trace(context::bypass, "(skipped) {} -> {}", file, dir); - } -} - -void copy_file_to_file_if_better( - const context& cx, const fs::path& src, const fs::path& dest, flags f) -{ - check(cx, src, f); - check(cx, dest, f); - - if (src.u8string().find(u8"*") != std::string::npos) - cx.bail_out(context::fs, "{} contains a glob", src); - - if (!conf().global().dry()) - { - if (!fs::exists(src)) - { - if (f & optional) - { - cx.trace(context::fs, - "not copying {}, doesn't exist (optional)", src); - - return; - } - - cx.bail_out(context::fs, "can't copy {}, doesn't exist", src); - } - - if (fs::exists(dest) && fs::is_directory(dest)) - { - cx.bail_out(context::fs, - "can't copy to {}, already exists but is a directory", dest); - } - } - - if (is_source_better(cx, src, dest)) - { - cx.trace(context::fs, "{} -> {}", src, dest); - - if (!conf().global().dry()) - do_copy_file_to_file(cx, src, dest); - } - else - { - cx.trace(context::bypass, "(skipped) {} -> {}", src, dest); - } -} - -void copy_glob_to_dir_if_better( - const context& cx, - const fs::path& src_glob, const fs::path& dest_dir, flags f) -{ - check(cx, dest_dir, f); - - const auto file_parent = src_glob.parent_path(); - const auto wildcard = src_glob.filename().native(); - - if (!fs::exists(file_parent)) - { - cx.bail_out(context::fs, - "can't copy glob {} to {}, parent directory {} doesn't exist", - src_glob, dest_dir, file_parent); - } - - - for (auto&& e : fs::directory_iterator(file_parent)) - { - const auto name = e.path().filename().native(); - - if (!PathMatchSpecW(name.c_str(), wildcard.c_str())) - { - cx.trace(context::fs, - "{} did not match {}; skipping", name, wildcard); - - continue; - } - - if (e.is_regular_file()) - { - if (f & copy_files) - { - copy_file_to_dir_if_better(cx, e.path(), dest_dir); - } - else - { - cx.trace(context::fs, - "file {} matched {} but files are not copied", - name, wildcard); - } - } - else if (e.is_directory()) - { - if (f & copy_dirs) - { - const fs::path sub = dest_dir / e.path().filename(); - - create_directories(cx, sub); - copy_glob_to_dir_if_better(cx, e.path() / "*", sub, f); - } - else - { - cx.trace(context::fs, - "directory {} matched {} but directories are not copied", - name, wildcard); - } - } - } -} - -void replace_file( - const context& cx, const fs::path& src, const fs::path& dest, - const fs::path& backup, flags f) -{ - cx.trace(context::fs, "swapping {} and {}", src, dest); - - check(cx, src, f); - check(cx, dest, f); - - if (conf().global().dry()) - return; - - const wchar_t* backup_p = nullptr; - std::wstring backup_s; - - if (!backup.empty()) - { - backup_s = backup.native(); - backup_p = backup_s.c_str(); - } - - const auto r = ::ReplaceFileW( - src.native().c_str(), dest.native().c_str(), backup_p, - REPLACEFILE_IGNORE_MERGE_ERRORS | REPLACEFILE_IGNORE_ACL_ERRORS, - nullptr, nullptr); - - if (r) - return; - - const auto e = GetLastError(); - - cx.warning( - context::generic, - "failed to atomically rename {} to {}, {}; hoping for the best", - src, dest, error_message(e)); - - op::rename(cx, src, backup); - op::rename(cx, dest, src); -} - -std::string read_text_file_impl(const context& cx, const fs::path& p, flags f) -{ - cx.trace(context::fs, "reading {}", p); - - std::string s; - std::ifstream in(p, std::ios::binary); - - in.seekg(0, std::ios::end); - s.resize(static_cast(in.tellg())); - in.seekg(0, std::ios::beg); - in.read(&s[0], static_cast(s.size())); - - if (in.bad()) - { - if (f & optional) - cx.debug(context::fs, "can't read from {} (optional)", p); - else - cx.bail_out(context::fs, "can't read from {}", p); - } - else - { - cx.trace(context::fs, "finished reading {}, {} bytes", p, s.size()); - } - - return s; -} - -std::string read_text_file( - const context& cx, encodings e, const fs::path& p, flags f) -{ - std::string bytes = read_text_file_impl(cx, p, f); - if (bytes.empty()) - return bytes; - - std::string utf8 = bytes_to_utf8(e, bytes); - utf8 = replace_all(utf8, "\r\n", "\n"); - - return utf8; -} - -void write_text_file( - const context& cx, encodings e, const fs::path& p, - std::string_view utf8, flags f) -{ - const std::string bytes = utf8_to_bytes(e, utf8); - cx.trace(context::fs, "writing {} bytes to {}", bytes.size(), p); - - check(cx, p, f); - - if (conf().global().dry()) - return; - - { - std::ofstream out(p, std::ios::binary); - out.write(bytes.data(), static_cast(bytes.size())); - out.close(); - - if (out.bad()) - { - if (f & optional) - { - cx.debug(context::fs, "can't write to {} (optional)", p); - } - else - { - cx.bail_out(context::fs, "can't write to {}", p); - } - } - } - - cx.trace(context::fs, - "finished writing {} bytes to {}", bytes.size(), p); -} - -void archive_from_glob( - const context& cx, - const fs::path& src_glob, const fs::path& dest_file, - const std::vector& ignore, flags f) -{ - cx.trace(context::fs, "archiving {} into {}", src_glob, dest_file); - check(cx, dest_file, f); - - if (conf().global().dry()) - return; - - archiver::create_from_glob(cx, dest_file, src_glob, ignore); -} - -void archive_from_files( - const context& cx, - const std::vector& files, const fs::path& files_root, - const fs::path& dest_file, flags f) -{ - check(cx, dest_file, f); - - cx.trace(context::fs, - "archiving {} files rooted in {} into {}", - files.size(), files_root, dest_file); - - if (conf().global().dry()) - return; - - archiver::create_from_files(cx, dest_file, files, files_root); -} - - -void do_touch(const context& cx, const fs::path& p) -{ - op::create_directories(cx, p.parent_path()); - - std::ofstream out(p); - if (!out) - cx.bail_out(context::fs, "failed to touch {}", p); -} - -void do_create_directories(const context& cx, const fs::path& p) -{ - std::error_code ec; - fs::create_directories(p, ec); - - if (ec) - cx.bail_out(context::fs, "can't create {}, {}", p, ec.message()); -} - -void do_delete_directory(const context& cx, const fs::path& p) -{ - std::error_code ec; - fs::remove_all(p, ec); - - if (ec) - { - if (ec.value() == ERROR_ACCESS_DENIED) - { - cx.trace( - context::fs, - "got access denied trying to delete dir {}, " - "trying to remove read-only flag recursively", p); - - remove_readonly(cx, p); - fs::remove_all(p, ec); - - if (!ec) - return; - } - - cx.bail_out(context::fs, "failed to delete {}, {}", p, ec.message()); - } -} - -void do_delete_file(const context& cx, const fs::path& p) -{ - std::error_code ec; - fs::remove(p, ec); - - if (ec) - cx.bail_out(context::fs, "can't delete {}, {}", p, ec.message()); -} - -void do_copy_file_to_dir( - const context& cx, const fs::path& f, const fs::path& d) -{ - if (!fs::exists(d)) - op::create_directories(cx, d); - - std::error_code ec; - fs::copy_file( - f, d / f.filename(), - fs::copy_options::overwrite_existing, ec); - - if (ec) - { - cx.bail_out(context::fs, - "can't copy {} to {}, {}", f, d, ec.message()); - } -} - -void do_copy_file_to_file( - const context& cx, const fs::path& src, const fs::path& dest) -{ - op::create_directories(cx, dest.parent_path()); - - std::error_code ec; - fs::copy_file( - src, dest, - fs::copy_options::overwrite_existing, ec); - - if (ec) - { - cx.bail_out(context::fs, - "can't copy {} to {}, {}", src, dest, ec.message()); - } -} - -void do_remove_readonly(const context& cx, const fs::path& p) -{ - cx.trace(context::fs, "chmod +x {}", p); - - std::error_code ec; - fs::permissions(p, fs::perms::owner_write, fs::perm_options::add, ec); - - if (ec) - { - cx.bail_out(context::fs, - "can't remove read-only flag on {}, {}", p, ec.message()); - } -} - -void do_rename(const context& cx, const fs::path& src, const fs::path& dest) -{ - std::error_code ec; - fs::rename(src, dest, ec); - - if (ec) - { - cx.bail_out(context::fs, - "can't rename {} to {}, {}", src, dest, ec.message()); - } -} - -void check(const context& cx, const fs::path& p, flags f) -{ - if (p.empty()) - cx.bail_out(context::fs, "path is empty"); - - if (is_set(f, unsafe)) - return; - - auto is_inside = [](auto&& p, auto&& dir) - { - const std::string s = path_to_utf8(p); - const std::string prefix = path_to_utf8(dir); - - if (s.size() < prefix.size()) - return false; - - const std::string scut = s.substr(0, prefix.size()); - - if (_stricmp(scut.c_str(), prefix.c_str()) != 0) - return false; - - return true; - }; - - if (is_inside(p, conf().path().prefix())) - return; - - if (is_inside(p, conf().path().temp_dir())) - return; - - if (is_inside(p, conf().path().licenses())) - return; - - cx.bail_out(context::fs, "path {} is outside prefix", p); -} - -} // namespace +namespace mob::op { + + // most of the functions from the header will check the paths, return early + // for dry run, and then forward to these to do the actual work + void do_touch(const context& cx, const fs::path& p); + void do_create_directories(const context& cx, const fs::path& p); + void do_delete_directory(const context& cx, const fs::path& p); + void do_delete_file(const context& cx, const fs::path& p); + void do_copy_file_to_dir(const context& cx, const fs::path& f, const fs::path& d); + void do_copy_file_to_file(const context& cx, const fs::path& f, const fs::path& d); + void do_remove_readonly(const context& cx, const fs::path& p); + void do_rename(const context& cx, const fs::path& src, const fs::path& dest); + + // checks whether the path is valid, bails out if not + // + void check(const context& cx, const fs::path& p, flags f); + + void touch(const context& cx, const fs::path& p, flags f) + { + cx.trace(context::fs, "touching {}", p); + check(cx, p, f); + + if (!conf().global().dry()) + do_touch(cx, p); + } + + void create_directories(const context& cx, const fs::path& p, flags f) + { + cx.trace(context::fs, "creating dir {}", p); + check(cx, p, f); + + if (!conf().global().dry()) + do_create_directories(cx, p); + } + + void delete_directory(const context& cx, const fs::path& p, flags f) + { + cx.trace(context::fs, "deleting dir {}", p); + check(cx, p, f); + + if (!fs::exists(p)) { + if (f & optional) { + cx.trace(context::fs, "not deleting dir {}, doesn't exist (optional)", + p); + + return; + } + + cx.bail_out(context::fs, "can't delete dir {}, doesn't exist", p); + } + + if (fs::exists(p) && !fs::is_directory(p)) + cx.bail_out(context::fs, "{} is not a dir", p); + + if (!conf().global().dry()) + do_delete_directory(cx, p); + } + + void delete_file(const context& cx, const fs::path& p, flags f) + { + cx.trace(context::fs, "deleting file {}", p); + check(cx, p, f); + + if (!fs::exists(p)) { + if (f & optional) { + cx.trace(context::fs, "not deleting file {}, doesn't exist (optional)", + p); + + return; + } + + cx.bail_out(context::fs, "can't delete file {}, doesn't exist", p); + } + + if (fs::exists(p) && !fs::is_regular_file(p)) { + if (f & optional) + cx.warning(context::fs, "can't delete {}, not a file", p); + else + cx.bail_out(context::fs, "can't delete {}, not a file", p); + + return; + } + + if (!conf().global().dry()) + do_delete_file(cx, p); + } + + void delete_file_glob(const context& cx, const fs::path& glob, flags f) + { + cx.trace(context::fs, "deleting glob {}", glob); + + const auto parent = glob.parent_path(); + const auto wildcard = glob.filename().native(); + + if (!fs::exists(parent)) + return; + + for (auto&& e : fs::directory_iterator(parent)) { + const auto p = e.path(); + const auto name = p.filename().native(); + + if (!PathMatchSpecW(name.c_str(), wildcard.c_str())) { + cx.trace(context::fs, "{} did not match {}; skipping", name, wildcard); + + continue; + } + + delete_file(cx, p, f); + } + } + + void remove_readonly(const context& cx, const fs::path& dir, flags f) + { + cx.trace(context::fs, "removing read-only from {}", dir); + check(cx, dir, f); + + if (!conf().global().dry()) { + for (auto&& p : fs::recursive_directory_iterator(dir)) { + if (fs::is_regular_file(p)) + do_remove_readonly(cx, p); + } + } + } + + bool is_source_better(const context& cx, const fs::path& src, const fs::path& dest) + { + if (!fs::exists(dest)) { + cx.trace(context::fs, "target {} doesn't exist; copying", dest); + return true; + } + + std::error_code ec; + + const auto src_size = fs::file_size(src, ec); + if (ec) { + cx.warning(context::fs, "failed to get size of {}, {}; forcing copy", src, + ec.message()); + + return true; + } + + const auto dest_size = fs::file_size(dest, ec); + if (ec) { + cx.warning(context::fs, "failed to get size of {}, {}; forcing copy", dest, + ec.message()); + + return true; + } + + if (src_size != dest_size) { + cx.trace(context::fs, "src {} bytes, dest {} bytes; different, copying", + src, src_size, dest, dest_size); + + return true; + } + + const auto src_time = fs::last_write_time(src, ec); + if (ec) { + cx.warning(context::fs, "failed to get time of {}, {}; forcing copy", src, + ec.message()); + + return true; + } + + const auto dest_time = fs::last_write_time(dest, ec); + if (ec) { + cx.warning(context::fs, "failed to get time of {}, {}; forcing copy", dest, + ec.message()); + + return true; + } + + if (src_time > dest_time) { + cx.trace(context::fs, "src {} is newer than dest {}; copying", src, dest); + + return true; + } + + // same size, same date + return false; + } + + void rename(const context& cx, const fs::path& src, const fs::path& dest, flags f) + { + check(cx, src, f); + check(cx, dest, f); + + if (fs::exists(dest)) { + cx.bail_out(context::fs, "can't rename {} to {}, already exists", src, + dest); + } + + cx.trace(context::fs, "renaming {} to {}", src, dest); + + if (!conf().global().dry()) + do_rename(cx, src, dest); + } + + void move_to_directory(const context& cx, const fs::path& src, + const fs::path& dest_dir, flags f) + { + check(cx, src, f); + check(cx, dest_dir, f); + + const auto target = dest_dir / src.filename(); + + if (fs::exists(target)) { + cx.bail_out(context::fs, "can't move {} to directory {}, {} already exists", + src, dest_dir, target); + } + + cx.trace(context::fs, "moving {} to {}", src, target); + + if (!conf().global().dry()) + do_rename(cx, src, target); + } + + void copy_file_to_dir_if_better(const context& cx, const fs::path& file, + const fs::path& dir, flags f) + { + check(cx, file, f); + check(cx, dir, f); + + if (file.u8string().find(u8"*") != std::string::npos) + cx.bail_out(context::fs, "{} contains a glob", file); + + if (!conf().global().dry()) { + if (!fs::exists(file) || !fs::is_regular_file(file)) { + if (f & optional) { + cx.trace(context::fs, "not copying {}, doesn't exist (optional)", + file); + + return; + } + + cx.bail_out(context::fs, "can't copy {}, not a file", file); + } + + if (fs::exists(dir) && !fs::is_directory(dir)) + cx.bail_out(context::fs, "can't copy to {}, not a dir", dir); + } + + const auto target = dir / file.filename(); + if (is_source_better(cx, file, target)) { + cx.trace(context::fs, "{} -> {}", file, dir); + + if (!conf().global().dry()) + do_copy_file_to_dir(cx, file, dir); + } + else { + cx.trace(context::bypass, "(skipped) {} -> {}", file, dir); + } + } + + void copy_file_to_file_if_better(const context& cx, const fs::path& src, + const fs::path& dest, flags f) + { + check(cx, src, f); + check(cx, dest, f); + + if (src.u8string().find(u8"*") != std::string::npos) + cx.bail_out(context::fs, "{} contains a glob", src); + + if (!conf().global().dry()) { + if (!fs::exists(src)) { + if (f & optional) { + cx.trace(context::fs, "not copying {}, doesn't exist (optional)", + src); + + return; + } + + cx.bail_out(context::fs, "can't copy {}, doesn't exist", src); + } + + if (fs::exists(dest) && fs::is_directory(dest)) { + cx.bail_out(context::fs, + "can't copy to {}, already exists but is a directory", + dest); + } + } + + if (is_source_better(cx, src, dest)) { + cx.trace(context::fs, "{} -> {}", src, dest); + + if (!conf().global().dry()) + do_copy_file_to_file(cx, src, dest); + } + else { + cx.trace(context::bypass, "(skipped) {} -> {}", src, dest); + } + } + + void copy_glob_to_dir_if_better(const context& cx, const fs::path& src_glob, + const fs::path& dest_dir, flags f) + { + check(cx, dest_dir, f); + + const auto file_parent = src_glob.parent_path(); + const auto wildcard = src_glob.filename().native(); + + if (!fs::exists(file_parent)) { + cx.bail_out(context::fs, + "can't copy glob {} to {}, parent directory {} doesn't exist", + src_glob, dest_dir, file_parent); + } + + for (auto&& e : fs::directory_iterator(file_parent)) { + const auto name = e.path().filename().native(); + + if (!PathMatchSpecW(name.c_str(), wildcard.c_str())) { + cx.trace(context::fs, "{} did not match {}; skipping", name, wildcard); + + continue; + } + + if (e.is_regular_file()) { + if (f & copy_files) { + copy_file_to_dir_if_better(cx, e.path(), dest_dir); + } + else { + cx.trace(context::fs, "file {} matched {} but files are not copied", + name, wildcard); + } + } + else if (e.is_directory()) { + if (f & copy_dirs) { + const fs::path sub = dest_dir / e.path().filename(); + + create_directories(cx, sub); + copy_glob_to_dir_if_better(cx, e.path() / "*", sub, f); + } + else { + cx.trace(context::fs, + "directory {} matched {} but directories are not copied", + name, wildcard); + } + } + } + } + + void replace_file(const context& cx, const fs::path& src, const fs::path& dest, + const fs::path& backup, flags f) + { + cx.trace(context::fs, "swapping {} and {}", src, dest); + + check(cx, src, f); + check(cx, dest, f); + + if (conf().global().dry()) + return; + + const wchar_t* backup_p = nullptr; + std::wstring backup_s; + + if (!backup.empty()) { + backup_s = backup.native(); + backup_p = backup_s.c_str(); + } + + const auto r = ::ReplaceFileW( + src.native().c_str(), dest.native().c_str(), backup_p, + REPLACEFILE_IGNORE_MERGE_ERRORS | REPLACEFILE_IGNORE_ACL_ERRORS, nullptr, + nullptr); + + if (r) + return; + + const auto e = GetLastError(); + + cx.warning(context::generic, + "failed to atomically rename {} to {}, {}; hoping for the best", src, + dest, error_message(e)); + + op::rename(cx, src, backup); + op::rename(cx, dest, src); + } + + std::string read_text_file_impl(const context& cx, const fs::path& p, flags f) + { + cx.trace(context::fs, "reading {}", p); + + std::string s; + std::ifstream in(p, std::ios::binary); + + in.seekg(0, std::ios::end); + s.resize(static_cast(in.tellg())); + in.seekg(0, std::ios::beg); + in.read(&s[0], static_cast(s.size())); + + if (in.bad()) { + if (f & optional) + cx.debug(context::fs, "can't read from {} (optional)", p); + else + cx.bail_out(context::fs, "can't read from {}", p); + } + else { + cx.trace(context::fs, "finished reading {}, {} bytes", p, s.size()); + } + + return s; + } + + std::string read_text_file(const context& cx, encodings e, const fs::path& p, + flags f) + { + std::string bytes = read_text_file_impl(cx, p, f); + if (bytes.empty()) + return bytes; + + std::string utf8 = bytes_to_utf8(e, bytes); + utf8 = replace_all(utf8, "\r\n", "\n"); + + return utf8; + } + + void write_text_file(const context& cx, encodings e, const fs::path& p, + std::string_view utf8, flags f) + { + const std::string bytes = utf8_to_bytes(e, utf8); + cx.trace(context::fs, "writing {} bytes to {}", bytes.size(), p); + + check(cx, p, f); + + if (conf().global().dry()) + return; + + { + std::ofstream out(p, std::ios::binary); + out.write(bytes.data(), static_cast(bytes.size())); + out.close(); + + if (out.bad()) { + if (f & optional) { + cx.debug(context::fs, "can't write to {} (optional)", p); + } + else { + cx.bail_out(context::fs, "can't write to {}", p); + } + } + } + + cx.trace(context::fs, "finished writing {} bytes to {}", bytes.size(), p); + } + + void archive_from_glob(const context& cx, const fs::path& src_glob, + const fs::path& dest_file, + const std::vector& ignore, flags f) + { + cx.trace(context::fs, "archiving {} into {}", src_glob, dest_file); + check(cx, dest_file, f); + + if (conf().global().dry()) + return; + + archiver::create_from_glob(cx, dest_file, src_glob, ignore); + } + + void archive_from_files(const context& cx, const std::vector& files, + const fs::path& files_root, const fs::path& dest_file, + flags f) + { + check(cx, dest_file, f); + + cx.trace(context::fs, "archiving {} files rooted in {} into {}", files.size(), + files_root, dest_file); + + if (conf().global().dry()) + return; + + archiver::create_from_files(cx, dest_file, files, files_root); + } + + void do_touch(const context& cx, const fs::path& p) + { + op::create_directories(cx, p.parent_path()); + + std::ofstream out(p); + if (!out) + cx.bail_out(context::fs, "failed to touch {}", p); + } + + void do_create_directories(const context& cx, const fs::path& p) + { + std::error_code ec; + fs::create_directories(p, ec); + + if (ec) + cx.bail_out(context::fs, "can't create {}, {}", p, ec.message()); + } + + void do_delete_directory(const context& cx, const fs::path& p) + { + std::error_code ec; + fs::remove_all(p, ec); + + if (ec) { + if (ec.value() == ERROR_ACCESS_DENIED) { + cx.trace(context::fs, + "got access denied trying to delete dir {}, " + "trying to remove read-only flag recursively", + p); + + remove_readonly(cx, p); + fs::remove_all(p, ec); + + if (!ec) + return; + } + + cx.bail_out(context::fs, "failed to delete {}, {}", p, ec.message()); + } + } + + void do_delete_file(const context& cx, const fs::path& p) + { + std::error_code ec; + fs::remove(p, ec); + + if (ec) + cx.bail_out(context::fs, "can't delete {}, {}", p, ec.message()); + } + + void do_copy_file_to_dir(const context& cx, const fs::path& f, const fs::path& d) + { + if (!fs::exists(d)) + op::create_directories(cx, d); + + std::error_code ec; + fs::copy_file(f, d / f.filename(), fs::copy_options::overwrite_existing, ec); + + if (ec) { + cx.bail_out(context::fs, "can't copy {} to {}, {}", f, d, ec.message()); + } + } + + void do_copy_file_to_file(const context& cx, const fs::path& src, + const fs::path& dest) + { + op::create_directories(cx, dest.parent_path()); + + std::error_code ec; + fs::copy_file(src, dest, fs::copy_options::overwrite_existing, ec); + + if (ec) { + cx.bail_out(context::fs, "can't copy {} to {}, {}", src, dest, + ec.message()); + } + } + + void do_remove_readonly(const context& cx, const fs::path& p) + { + cx.trace(context::fs, "chmod +x {}", p); + + std::error_code ec; + fs::permissions(p, fs::perms::owner_write, fs::perm_options::add, ec); + + if (ec) { + cx.bail_out(context::fs, "can't remove read-only flag on {}, {}", p, + ec.message()); + } + } + + void do_rename(const context& cx, const fs::path& src, const fs::path& dest) + { + std::error_code ec; + fs::rename(src, dest, ec); + + if (ec) { + cx.bail_out(context::fs, "can't rename {} to {}, {}", src, dest, + ec.message()); + } + } + + void check(const context& cx, const fs::path& p, flags f) + { + if (p.empty()) + cx.bail_out(context::fs, "path is empty"); + + if (is_set(f, unsafe)) + return; + + auto is_inside = [](auto&& p, auto&& dir) { + const std::string s = path_to_utf8(p); + const std::string prefix = path_to_utf8(dir); + + if (s.size() < prefix.size()) + return false; + + const std::string scut = s.substr(0, prefix.size()); + + if (_stricmp(scut.c_str(), prefix.c_str()) != 0) + return false; + + return true; + }; + + if (is_inside(p, conf().path().prefix())) + return; + + if (is_inside(p, conf().path().temp_dir())) + return; + + if (is_inside(p, conf().path().licenses())) + return; + + cx.bail_out(context::fs, "path {} is outside prefix", p); + } + +} // namespace mob::op diff --git a/src/core/op.h b/src/core/op.h index f094ac4..eafd58a 100644 --- a/src/core/op.h +++ b/src/core/op.h @@ -2,151 +2,138 @@ #include "../utility.h" -namespace mob { class context; } - -namespace mob::op -{ - -// filesystem operations, also handle --dry -// -// for functions that end with _if_better(): the source is considered better -// than the destination if: -// 1) the destination doesn't exist, or -// 2) the size is different, or -// 3) the date is newer - - -// various flags for the operations below, only some of them are used by some -// functions -// -enum flags -{ - noflags = 0x00, - - // the operation is optional, don't bail out if it fails - optional = 0x01, - - // used by copy_glob_to_dir_if_better() to decide if files and/or - // directories are copied - copy_files = 0x02, - copy_dirs = 0x04, - - // operations will typically fail early if paths are empty or if they're not - // inside a list of approved locations, like the prefix, %TEMP%, etc. - // - // this is to prevent mob from going on a deletion spree in case of bugs - unsafe = 0x08 -}; - -MOB_ENUM_OPERATORS(flags); - - -// creates the given file if it doesn't exist -// -void touch(const context& cx, const fs::path& p, flags f=noflags); - -// creates all the directories in the given path -// -void create_directories(const context& cx, const fs::path& p, flags f=noflags); - -// deletes the given directory, recursive -// -// if deletion fails because of access denied, attempts to remove the readonly -// flag on all files and tries again; this happens with some archives like 7z -// -// if the directory is controlled by git, prefer git_wrap::delete_directory(), -// which checks for uncommitted changes before -// -void delete_directory(const context& cx, const fs::path& p, flags f=noflags); - -// deletes the given file -// -void delete_file(const context& cx, const fs::path& p, flags f=noflags); - -// deletes all files matching the glob in the glob's parent directory -// -void delete_file_glob(const context& cx, const fs::path& glob, flags f=noflags); - -// removes the readonly flag for all files in `dir`, recursive -// -void remove_readonly(const context& cx, const fs::path& dir, flags f=noflags); - -// renames `src` to `dest`, files or directories; fails if it already exists -// -void rename( - const context& cx, const fs::path& src, const fs::path& dest, - flags f=noflags); - -// moves a file or directory `src` into dir `dest_dir`, using the same name -// (renames src to dest_dir/src.filename()); fails if it already exists -// -void move_to_directory( - const context& cx, const fs::path& src, const fs::path& dest_dir, - flags f=noflags); - -// copies a single file `file` into `dest_dir`; if the file already exists, only -// copies it if it's considered better (see comment on top); doesn't support -// globs or directories -// -void copy_file_to_dir_if_better( - const context& cx, - const fs::path& file, const fs::path& dest_dir, flags f=noflags); - -// same as copy_file_to_dir_if_better(), but the `dest_file` contains the -// target filename instead of being constructed from dest_dir/src.filename() -// -void copy_file_to_file_if_better( - const context& cx, - const fs::path& src_file, const fs::path& dest_file, flags f=noflags); - -// basically calls copy_file_to_dir_if_better() for every file matching the -// glob; recursive -// -void copy_glob_to_dir_if_better( - const context& cx, - const fs::path& src_glob, const fs::path& dest_dir, flags f); - -// renames `dest` to `src`, deleting `src` if it exists; if `backup` is given, -// `src` is first renamed to it -// -// this attempts an atomic rename with ReplaceFile(), falls back to non-atomic -// renames if it fails -// -void replace_file( - const context& cx, const fs::path& src, const fs::path& dest, - const fs::path& backup={}, flags f=noflags); - -// reads the given file, converts it to utf8 from the given encoding, returns -// the utf8 string; if `e` is `dont_know`, returns the bytes as-is -// -std::string read_text_file( - const context& cx, encodings e, const fs::path& p, flags f=noflags); - -// creates file `p`, writes the given utf8 string into it, converting the string -// to the given encoding; if `e` is dont_know, the bytes are written as-is -// -void write_text_file( - const context& cx, encodings e, const fs::path& p, std::string_view utf8, - flags f=noflags); - -// creates an archive `dest_file` and puts all the files matching `src_glob` -// into it, ignoring any file in `ignore` by name -// -// uses tools::archiver -// -void archive_from_glob( - const context& cx, - const fs::path& src_glob, const fs::path& dest_file, - const std::vector& ignore, - flags f=noflags); - -// creates an archive `dest_file` and puts all the files from `files` in it, -// resolving relative paths against `files_root` -// -void archive_from_files( - const context& cx, - const std::vector& files, const fs::path& files_root, - const fs::path& dest_file, - flags f=noflags); - -} // namespace +namespace mob { + class context; +} + +namespace mob::op { + + // filesystem operations, also handle --dry + // + // for functions that end with _if_better(): the source is considered better + // than the destination if: + // 1) the destination doesn't exist, or + // 2) the size is different, or + // 3) the date is newer + + // various flags for the operations below, only some of them are used by some + // functions + // + enum flags { + noflags = 0x00, + + // the operation is optional, don't bail out if it fails + optional = 0x01, + + // used by copy_glob_to_dir_if_better() to decide if files and/or + // directories are copied + copy_files = 0x02, + copy_dirs = 0x04, + + // operations will typically fail early if paths are empty or if they're not + // inside a list of approved locations, like the prefix, %TEMP%, etc. + // + // this is to prevent mob from going on a deletion spree in case of bugs + unsafe = 0x08 + }; + + MOB_ENUM_OPERATORS(flags); + + // creates the given file if it doesn't exist + // + void touch(const context& cx, const fs::path& p, flags f = noflags); + + // creates all the directories in the given path + // + void create_directories(const context& cx, const fs::path& p, flags f = noflags); + + // deletes the given directory, recursive + // + // if deletion fails because of access denied, attempts to remove the readonly + // flag on all files and tries again; this happens with some archives like 7z + // + // if the directory is controlled by git, prefer git_wrap::delete_directory(), + // which checks for uncommitted changes before + // + void delete_directory(const context& cx, const fs::path& p, flags f = noflags); + + // deletes the given file + // + void delete_file(const context& cx, const fs::path& p, flags f = noflags); + + // deletes all files matching the glob in the glob's parent directory + // + void delete_file_glob(const context& cx, const fs::path& glob, flags f = noflags); + + // removes the readonly flag for all files in `dir`, recursive + // + void remove_readonly(const context& cx, const fs::path& dir, flags f = noflags); + + // renames `src` to `dest`, files or directories; fails if it already exists + // + void rename(const context& cx, const fs::path& src, const fs::path& dest, + flags f = noflags); + + // moves a file or directory `src` into dir `dest_dir`, using the same name + // (renames src to dest_dir/src.filename()); fails if it already exists + // + void move_to_directory(const context& cx, const fs::path& src, + const fs::path& dest_dir, flags f = noflags); + + // copies a single file `file` into `dest_dir`; if the file already exists, only + // copies it if it's considered better (see comment on top); doesn't support + // globs or directories + // + void copy_file_to_dir_if_better(const context& cx, const fs::path& file, + const fs::path& dest_dir, flags f = noflags); + + // same as copy_file_to_dir_if_better(), but the `dest_file` contains the + // target filename instead of being constructed from dest_dir/src.filename() + // + void copy_file_to_file_if_better(const context& cx, const fs::path& src_file, + const fs::path& dest_file, flags f = noflags); + + // basically calls copy_file_to_dir_if_better() for every file matching the + // glob; recursive + // + void copy_glob_to_dir_if_better(const context& cx, const fs::path& src_glob, + const fs::path& dest_dir, flags f); + + // renames `dest` to `src`, deleting `src` if it exists; if `backup` is given, + // `src` is first renamed to it + // + // this attempts an atomic rename with ReplaceFile(), falls back to non-atomic + // renames if it fails + // + void replace_file(const context& cx, const fs::path& src, const fs::path& dest, + const fs::path& backup = {}, flags f = noflags); + + // reads the given file, converts it to utf8 from the given encoding, returns + // the utf8 string; if `e` is `dont_know`, returns the bytes as-is + // + std::string read_text_file(const context& cx, encodings e, const fs::path& p, + flags f = noflags); + + // creates file `p`, writes the given utf8 string into it, converting the string + // to the given encoding; if `e` is dont_know, the bytes are written as-is + // + void write_text_file(const context& cx, encodings e, const fs::path& p, + std::string_view utf8, flags f = noflags); + + // creates an archive `dest_file` and puts all the files matching `src_glob` + // into it, ignoring any file in `ignore` by name + // + // uses tools::archiver + // + void archive_from_glob(const context& cx, const fs::path& src_glob, + const fs::path& dest_file, + const std::vector& ignore, flags f = noflags); + + // creates an archive `dest_file` and puts all the files from `files` in it, + // resolving relative paths against `files_root` + // + void archive_from_files(const context& cx, const std::vector& files, + const fs::path& files_root, const fs::path& dest_file, + flags f = noflags); + +} // namespace mob::op diff --git a/src/core/paths.cpp b/src/core/paths.cpp index 382e884..8595aa5 100644 --- a/src/core/paths.cpp +++ b/src/core/paths.cpp @@ -4,428 +4,385 @@ #include "../tasks/tasks.h" #include "../utility/string.h" -namespace mob -{ - -// returns a path to the given known folder, empty on error -// -fs::path get_known_folder(const GUID& id) -{ - wchar_t* buffer = nullptr; - const auto r = ::SHGetKnownFolderPath(id, 0, 0, &buffer); - - if (r != S_OK) - return {}; - - fs::path p = buffer; - ::CoTaskMemFree(buffer); - - return p; -} - -// searches PATH for the given executable, returns empty if not found -// -fs::path find_in_path(std::string_view exe) -{ - const std::wstring wexe = utf8_to_utf16(exe); - - const std::size_t size = MAX_PATH; - wchar_t buffer[size + 1] = {}; - - if (SearchPathW(nullptr, wexe.c_str(), nullptr, size, buffer, nullptr) == 0) - return {}; - - return buffer; -} - -// checks if a path exists that starts with `check` and ends with as many parts -// as possible -// -// for example: -// -// try_parts("c:/", {"1", "2", "3"}) -// -// will try in order: -// -// c:/1/2/3 -// c:/2/3 -// c:/3 -// -// if none of the paths exist, returns false; if one of the paths exists, -// `check` is set to it and returns true -// -bool try_parts(fs::path& check, const std::vector& parts) -{ - for (std::size_t i=0; i(buffer_size + 1); - DWORD n = GetModuleFileNameW(0, buffer.get(), buffer_size); - - if (n == 0) - { - const auto e = GetLastError(); - - gcx().bail_out(context::conf, - "can't get module filename, {}", error_message(e)); - } - else if (n >= buffer_size) - { - // buffer is too small, try again - buffer_size *= 2; - } - else - { - // if GetModuleFileName() works, `n` does not include the null - // terminator - const std::wstring s(buffer.get(), n); - return fs::canonical(s); - } - } - - gcx().bail_out(context::conf, "can't get module filename"); -} - -fs::path find_root(bool verbose) -{ - gcx().trace(context::conf, "looking for root directory"); - - fs::path mob_exe_dir = mob_exe_path().parent_path(); - - auto third_party = mob_exe_dir / "third-party"; - - if (!fs::exists(third_party)) - { - // doesn't exist, maybe this is the build directory - - auto p = mob_exe_dir; - - if (p.filename().u8string() == u8"x64") - { - p = p.parent_path(); - - if (p.filename().u8string() == u8"Debug" || - p.filename().u8string() == u8"Release") - { - if (verbose) - u8cout << "mob.exe is in its build directory, looking up\n"; - - // mob_exe_dir is in the build directory - third_party = mob_exe_dir / ".." / ".." / ".." / "third-party"; - } - } - } - - if (!fs::exists(third_party)) - gcx().bail_out(context::conf, "root directory not found"); - - const auto p = fs::canonical(third_party.parent_path()); - gcx().trace(context::conf, "found root directory at {}", p); - - return p; -} - -fs::path find_in_root(const fs::path& file) -{ - static fs::path root = find_root(); - - fs::path p = root / file; - if (!fs::exists(p)) - gcx().bail_out(context::conf, "{} not found", p); - - gcx().trace(context::conf, "found {}", p); - return p; -} - -fs::path find_third_party_directory() -{ - static fs::path path = find_in_root("third-party"); - return path; -} - -fs::path find_program_files_x86() -{ - fs::path p = get_known_folder(FOLDERID_ProgramFilesX86); - - if (p.empty()) - { - const auto e = GetLastError(); - - p = fs::path(R"(C:\Program Files (x86))"); - - gcx().warning(context::conf, - "failed to get x86 program files folder, defaulting to {}, {}", - p, error_message(e)); - } - else - { - gcx().trace(context::conf, "x86 program files is {}", p); - } - - return p; -} - -fs::path find_program_files_x64() -{ - fs::path p = get_known_folder(FOLDERID_ProgramFilesX64); - - if (p.empty()) - { - const auto e = GetLastError(); - - p = fs::path(R"(C:\Program Files)"); - - gcx().warning(context::conf, - "failed to get x64 program files folder, defaulting to {}, {}", - p, error_message(e)); - } - else - { - gcx().trace(context::conf, "x64 program files is {}", p); - } - - return p; -} - -fs::path find_vs() -{ - // asking vswhere - const auto output = vswhere::find_vs(); - if (output.empty()) - gcx().bail_out(context::conf, "vswhere failed"); - - const auto lines = split(output, "\r\n"); - - if (lines.empty()) - { - gcx().bail_out(context::conf, "vswhere didn't output anything"); - } - else if (lines.size() > 1) - { - gcx().error(context::conf, "vswhere returned multiple installations:"); - - for (auto&& line : lines) - gcx().error(context::conf, " - {}", line); - - gcx().bail_out(context::conf, - "specify the `vs` path in the `[paths]` section of the INI, or " - "pass -s paths/vs=PATH` to pick an installation"); - } - else - { - // only one line - const fs::path path(output); - - if (!fs::exists(path)) - { - gcx().bail_out(context::conf, - "the path given by vswhere doesn't exist: {}", path); - } - - return path; - } -} - -fs::path find_qt() -{ - // check from the ini first - fs::path p = conf().path().qt_install(); - - if (!p.empty()) - { - p = fs::absolute(p); - - // check if qmake exists in there - if (find_qmake(p)) - { - p = fs::absolute(p.parent_path() / ".."); - return p; - } - - // fail early, don't try to guess if the user had something in the ini - gcx().bail_out(context::conf, "no qt install in {}", p); - } - - - // a list of possible location - std::deque locations = - { - conf().path().pf_x64(), - "C:\\", - "D:\\" - }; - - // look for qmake in PATH, which is in %qt%/version/msvc.../bin - fs::path qmake = find_in_path("qmake.exe"); - if (!qmake.empty()) - locations.push_front(qmake.parent_path() / "../../"); - - // look for qtcreator.exe in PATH, which is in %qt%/Tools/QtCreator/bin - fs::path qtcreator = find_in_path("qtcreator.exe"); - if (!qtcreator.empty()) - locations.push_front(qtcreator.parent_path() / "../../../"); - - // check each location - for (fs::path loc : locations) - { - loc = fs::absolute(loc); - - // check for qmake in there - if (find_qmake(loc)) - { - loc = fs::absolute(loc.parent_path() / ".."); - return loc; - } - } - - gcx().bail_out(context::conf, "can't find qt install"); -} - -fs::path find_iscc() -{ - // don't bother if the installer isn't enabled, it might fail anyway - if (!task_manager::instance().find_one("installer")->enabled()) - return {}; - - // check from the ini first, supports both relative and absolute - const auto iscc = conf().tool().get("iscc"); - - if (iscc.is_absolute()) - { - if (!fs::exists(iscc)) - { - gcx().bail_out(context::conf, - "{} doesn't exist (from ini, absolute path)", iscc); - } - - return iscc; - } - - // path is relative - - // check in PATH - fs::path p = find_in_path(path_to_utf8(iscc)); - if (fs::exists(p)) - return fs::canonical(fs::absolute(p)); - - // check known installation paths for a bunch of versions - for (int v : {5, 6, 7, 8}) - { - const fs::path inno_dir = fmt::format("Inno Setup {}", v); - - // check for both architectures - for (fs::path pf : {conf().path().pf_x86(), conf().path().pf_x64()}) - { - p = pf / inno_dir / iscc; - if (fs::exists(p)) - return fs::canonical(fs::absolute(p)); - } - } - - gcx().bail_out(context::conf, "can't find {} anywhere", iscc); -} - -fs::path find_temp_dir() -{ - const std::size_t buffer_size = MAX_PATH + 2; - wchar_t buffer[buffer_size] = {}; - - if (GetTempPathW(static_cast(buffer_size), buffer) == 0) - { - const auto e = GetLastError(); - gcx().bail_out(context::conf, "can't get temp path", error_message(e)); - } - - fs::path p(buffer); - gcx().trace(context::conf, "temp dir is {}", p); - - return p; -} - -fs::path find_vcvars() -{ - // check from the ini first - fs::path bat = conf().tool().get("vcvars"); - - if (conf().global().dry()) - { - if (bat.empty()) - bat = "vcvars.bat"; - - return bat; - } - - if (bat.empty()) - { - // derive from vs installation - bat = vs::installation_path() - / "VC" / "Auxiliary" / "Build" / "vcvarsall.bat"; - } - - if (!fs::exists(bat)) - gcx().bail_out(context::conf, "vcvars not found at {}", bat); - - if (bat.is_relative()) - bat = fs::absolute(bat); - - bat = fs::canonical(bat); - gcx().trace(context::conf, "using vcvars at {}", bat); - - return bat; -} - -} // namespace +namespace mob { + + // returns a path to the given known folder, empty on error + // + fs::path get_known_folder(const GUID& id) + { + wchar_t* buffer = nullptr; + const auto r = ::SHGetKnownFolderPath(id, 0, 0, &buffer); + + if (r != S_OK) + return {}; + + fs::path p = buffer; + ::CoTaskMemFree(buffer); + + return p; + } + + // searches PATH for the given executable, returns empty if not found + // + fs::path find_in_path(std::string_view exe) + { + const std::wstring wexe = utf8_to_utf16(exe); + + const std::size_t size = MAX_PATH; + wchar_t buffer[size + 1] = {}; + + if (SearchPathW(nullptr, wexe.c_str(), nullptr, size, buffer, nullptr) == 0) + return {}; + + return buffer; + } + + // checks if a path exists that starts with `check` and ends with as many parts + // as possible + // + // for example: + // + // try_parts("c:/", {"1", "2", "3"}) + // + // will try in order: + // + // c:/1/2/3 + // c:/2/3 + // c:/3 + // + // if none of the paths exist, returns false; if one of the paths exists, + // `check` is set to it and returns true + // + bool try_parts(fs::path& check, const std::vector& parts) + { + for (std::size_t i = 0; i < parts.size(); ++i) { + fs::path p = check; + + for (std::size_t j = i; j < parts.size(); ++j) + p /= parts[j]; + + gcx().trace(context::conf, "trying parts {}", p); + + if (fs::exists(p)) { + check = p; + return true; + } + } + + return false; + } + + // looks for `qmake.exe` in the given path, tries a variety of subdirectories + // + bool find_qmake(fs::path& check) + { + // try Qt/Qt5.14.2/msvc*/bin/qmake.exe + if (try_parts(check, {"Qt", "Qt" + qt::version(), + "msvc" + qt::vs_version() + "_64", "bin", "qmake.exe"})) { + return true; + } + + // try Qt/5.14.2/msvc*/bin/qmake.exe + if (try_parts(check, {"Qt", qt::version(), "msvc" + qt::vs_version() + "_64", + "bin", "qmake.exe"})) { + return true; + } + + return false; + } + + fs::path mob_exe_path() + { + const int max_tries = 3; + + DWORD buffer_size = MAX_PATH; + + for (int tries = 0; tries < max_tries; ++tries) { + auto buffer = std::make_unique(buffer_size + 1); + DWORD n = GetModuleFileNameW(0, buffer.get(), buffer_size); + + if (n == 0) { + const auto e = GetLastError(); + + gcx().bail_out(context::conf, "can't get module filename, {}", + error_message(e)); + } + else if (n >= buffer_size) { + // buffer is too small, try again + buffer_size *= 2; + } + else { + // if GetModuleFileName() works, `n` does not include the null + // terminator + const std::wstring s(buffer.get(), n); + return fs::canonical(s); + } + } + + gcx().bail_out(context::conf, "can't get module filename"); + } + + fs::path find_root(bool verbose) + { + gcx().trace(context::conf, "looking for root directory"); + + fs::path mob_exe_dir = mob_exe_path().parent_path(); + + auto third_party = mob_exe_dir / "third-party"; + + if (!fs::exists(third_party)) { + // doesn't exist, maybe this is the build directory + + auto p = mob_exe_dir; + + if (p.filename().u8string() == u8"x64") { + p = p.parent_path(); + + if (p.filename().u8string() == u8"Debug" || + p.filename().u8string() == u8"Release") { + if (verbose) + u8cout << "mob.exe is in its build directory, looking up\n"; + + // mob_exe_dir is in the build directory + third_party = mob_exe_dir / ".." / ".." / ".." / "third-party"; + } + } + } + + if (!fs::exists(third_party)) + gcx().bail_out(context::conf, "root directory not found"); + + const auto p = fs::canonical(third_party.parent_path()); + gcx().trace(context::conf, "found root directory at {}", p); + + return p; + } + + fs::path find_in_root(const fs::path& file) + { + static fs::path root = find_root(); + + fs::path p = root / file; + if (!fs::exists(p)) + gcx().bail_out(context::conf, "{} not found", p); + + gcx().trace(context::conf, "found {}", p); + return p; + } + + fs::path find_third_party_directory() + { + static fs::path path = find_in_root("third-party"); + return path; + } + + fs::path find_program_files_x86() + { + fs::path p = get_known_folder(FOLDERID_ProgramFilesX86); + + if (p.empty()) { + const auto e = GetLastError(); + + p = fs::path(R"(C:\Program Files (x86))"); + + gcx().warning( + context::conf, + "failed to get x86 program files folder, defaulting to {}, {}", p, + error_message(e)); + } + else { + gcx().trace(context::conf, "x86 program files is {}", p); + } + + return p; + } + + fs::path find_program_files_x64() + { + fs::path p = get_known_folder(FOLDERID_ProgramFilesX64); + + if (p.empty()) { + const auto e = GetLastError(); + + p = fs::path(R"(C:\Program Files)"); + + gcx().warning( + context::conf, + "failed to get x64 program files folder, defaulting to {}, {}", p, + error_message(e)); + } + else { + gcx().trace(context::conf, "x64 program files is {}", p); + } + + return p; + } + + fs::path find_vs() + { + // asking vswhere + const auto output = vswhere::find_vs(); + if (output.empty()) + gcx().bail_out(context::conf, "vswhere failed"); + + const auto lines = split(output, "\r\n"); + + if (lines.empty()) { + gcx().bail_out(context::conf, "vswhere didn't output anything"); + } + else if (lines.size() > 1) { + gcx().error(context::conf, "vswhere returned multiple installations:"); + + for (auto&& line : lines) + gcx().error(context::conf, " - {}", line); + + gcx().bail_out( + context::conf, + "specify the `vs` path in the `[paths]` section of the INI, or " + "pass -s paths/vs=PATH` to pick an installation"); + } + else { + // only one line + const fs::path path(output); + + if (!fs::exists(path)) { + gcx().bail_out(context::conf, + "the path given by vswhere doesn't exist: {}", path); + } + + return path; + } + } + + fs::path find_qt() + { + // check from the ini first + fs::path p = conf().path().qt_install(); + + if (!p.empty()) { + p = fs::absolute(p); + + // check if qmake exists in there + if (find_qmake(p)) { + p = fs::absolute(p.parent_path() / ".."); + return p; + } + + // fail early, don't try to guess if the user had something in the ini + gcx().bail_out(context::conf, "no qt install in {}", p); + } + + // a list of possible location + std::deque locations = {conf().path().pf_x64(), "C:\\", "D:\\"}; + + // look for qmake in PATH, which is in %qt%/version/msvc.../bin + fs::path qmake = find_in_path("qmake.exe"); + if (!qmake.empty()) + locations.push_front(qmake.parent_path() / "../../"); + + // look for qtcreator.exe in PATH, which is in %qt%/Tools/QtCreator/bin + fs::path qtcreator = find_in_path("qtcreator.exe"); + if (!qtcreator.empty()) + locations.push_front(qtcreator.parent_path() / "../../../"); + + // check each location + for (fs::path loc : locations) { + loc = fs::absolute(loc); + + // check for qmake in there + if (find_qmake(loc)) { + loc = fs::absolute(loc.parent_path() / ".."); + return loc; + } + } + + gcx().bail_out(context::conf, "can't find qt install"); + } + + fs::path find_iscc() + { + // don't bother if the installer isn't enabled, it might fail anyway + if (!task_manager::instance().find_one("installer")->enabled()) + return {}; + + // check from the ini first, supports both relative and absolute + const auto iscc = conf().tool().get("iscc"); + + if (iscc.is_absolute()) { + if (!fs::exists(iscc)) { + gcx().bail_out(context::conf, + "{} doesn't exist (from ini, absolute path)", iscc); + } + + return iscc; + } + + // path is relative + + // check in PATH + fs::path p = find_in_path(path_to_utf8(iscc)); + if (fs::exists(p)) + return fs::canonical(fs::absolute(p)); + + // check known installation paths for a bunch of versions + for (int v : {5, 6, 7, 8}) { + const fs::path inno_dir = fmt::format("Inno Setup {}", v); + + // check for both architectures + for (fs::path pf : {conf().path().pf_x86(), conf().path().pf_x64()}) { + p = pf / inno_dir / iscc; + if (fs::exists(p)) + return fs::canonical(fs::absolute(p)); + } + } + + gcx().bail_out(context::conf, "can't find {} anywhere", iscc); + } + + fs::path find_temp_dir() + { + const std::size_t buffer_size = MAX_PATH + 2; + wchar_t buffer[buffer_size] = {}; + + if (GetTempPathW(static_cast(buffer_size), buffer) == 0) { + const auto e = GetLastError(); + gcx().bail_out(context::conf, "can't get temp path", error_message(e)); + } + + fs::path p(buffer); + gcx().trace(context::conf, "temp dir is {}", p); + + return p; + } + + fs::path find_vcvars() + { + // check from the ini first + fs::path bat = conf().tool().get("vcvars"); + + if (conf().global().dry()) { + if (bat.empty()) + bat = "vcvars.bat"; + + return bat; + } + + if (bat.empty()) { + // derive from vs installation + bat = vs::installation_path() / "VC" / "Auxiliary" / "Build" / + "vcvarsall.bat"; + } + + if (!fs::exists(bat)) + gcx().bail_out(context::conf, "vcvars not found at {}", bat); + + if (bat.is_relative()) + bat = fs::absolute(bat); + + bat = fs::canonical(bat); + gcx().trace(context::conf, "using vcvars at {}", bat); + + return bat; + } + +} // namespace mob diff --git a/src/core/paths.h b/src/core/paths.h index 972c6f5..597823a 100644 --- a/src/core/paths.h +++ b/src/core/paths.h @@ -1,55 +1,53 @@ #pragma once -namespace mob -{ - -// returns the path to mob.exe that's currently running, including filename; -// bails on failure -// -fs::path mob_exe_path(); - -// returns mob's root directory, contains third-party/, etc.; bails on failure -// -// this is not necessarily the parent of mob_exe_path(), it mob could be running -// from the build directory -// -fs::path find_root(bool verbose=false); - -// resolves the given relative path against find_root(); bails when not found -// -fs::path find_in_root(const fs::path& file); - - -// returns the absolute path of mob's third-party directory; bails when not -// found -// -fs::path find_third_party_directory(); - -// returns the absolute path to x86/x64 program files directory -// -fs::path find_program_files_x86(); -fs::path find_program_files_x64(); - -// returns the absolute path to visual studio's root directory, the one that -// contains Common7, VC, etc.; bails if not found -// -fs::path find_vs(); - -// returns the absolute path to Qt's root directory, the one that contains -// bin, include, etc.; bails if not found -// -fs::path find_qt(); - -// returns the absolute path to iscc.exe; bails if not found -// -fs::path find_iscc(); - -// returns the absolute path to the system's temp directory; bails on error -// -fs::path find_temp_dir(); - -// returns the absolute path to the vcvars batch file, bails if not found -// -fs::path find_vcvars(); - -} // namespace +namespace mob { + + // returns the path to mob.exe that's currently running, including filename; + // bails on failure + // + fs::path mob_exe_path(); + + // returns mob's root directory, contains third-party/, etc.; bails on failure + // + // this is not necessarily the parent of mob_exe_path(), it mob could be running + // from the build directory + // + fs::path find_root(bool verbose = false); + + // resolves the given relative path against find_root(); bails when not found + // + fs::path find_in_root(const fs::path& file); + + // returns the absolute path of mob's third-party directory; bails when not + // found + // + fs::path find_third_party_directory(); + + // returns the absolute path to x86/x64 program files directory + // + fs::path find_program_files_x86(); + fs::path find_program_files_x64(); + + // returns the absolute path to visual studio's root directory, the one that + // contains Common7, VC, etc.; bails if not found + // + fs::path find_vs(); + + // returns the absolute path to Qt's root directory, the one that contains + // bin, include, etc.; bails if not found + // + fs::path find_qt(); + + // returns the absolute path to iscc.exe; bails if not found + // + fs::path find_iscc(); + + // returns the absolute path to the system's temp directory; bails on error + // + fs::path find_temp_dir(); + + // returns the absolute path to the vcvars batch file, bails if not found + // + fs::path find_vcvars(); + +} // namespace mob diff --git a/src/core/pipe.cpp b/src/core/pipe.cpp index 8164766..e40ed43 100644 --- a/src/core/pipe.cpp +++ b/src/core/pipe.cpp @@ -3,355 +3,319 @@ #include "context.h" #include "process.h" -namespace mob -{ - -// many processes may be started simultaneously, this is incremented each time -// a pipe is created to make sure pipe names are unique -static std::atomic g_next_pipe_id(0); - - -async_pipe_stdout::async_pipe_stdout(const context& cx) - : cx_(cx), pending_(false), closed_(true) -{ - buffer_ = std::make_unique(buffer_size); - - std::memset(buffer_.get(), 0, buffer_size); - std::memset(&ov_, 0, sizeof(ov_)); -} - -bool async_pipe_stdout::closed() const -{ - return closed_; -} - -handle_ptr async_pipe_stdout::create() -{ - // creating pipe - handle_ptr out(create_named_pipe()); - if (out.get() == INVALID_HANDLE_VALUE) - return {}; - - // creating event - ov_.hEvent = ::CreateEvent(nullptr, TRUE, FALSE, nullptr); - - if (ov_.hEvent == NULL) - { - const auto e = GetLastError(); - cx_.bail_out(context::cmd, - "CreateEvent failed, {}", error_message(e)); - } - - event_.reset(ov_.hEvent); - closed_ = false; - - return out; -} - -std::string_view async_pipe_stdout::read(bool finish) -{ - std::string_view s; - - if (closed_) - { - // no-op - return s; - } - - if (pending_) - { - // the last read() call started an async operation, check if it's done - s = check_pending(); - } - else - { - // there's no async operation in progress, try to read - s = try_read(); - } - - if (finish && s.empty()) - { - // nothing was read from the pipe and `finish` is true, so the process - // has terminated; in this case, assume the pipe is empty and everything - // has been read - - // even if the pipe is empty and nothing more is available, try_read() - // may have started an async operation that would return no data in the - // future - // - // make sure that operation is cancelled, because the kernel would try - // to use the OVERLAPPED buffer, which will probably have been destroyed - // by that time - ::CancelIo(pipe_.get()); - - // a future call to read() will be a no-op and closed() will return true - closed_ = true; - } - - // the bytes that were read, if any - return s; -} - -HANDLE async_pipe_stdout::create_named_pipe() -{ - // unique name - const auto pipe_id = g_next_pipe_id.fetch_add(1) + 1; - - const std::wstring pipe_name = - LR"(\\.\pipe\mob_pipe)" + std::to_wstring(pipe_id); - - - // creating pipe - { - const DWORD open_flags = - PIPE_ACCESS_INBOUND | // the pipe will be read from - FILE_FLAG_OVERLAPPED | // non blocking - FILE_FLAG_FIRST_PIPE_INSTANCE; // the pipe must not exist - - // pipes support either bytes or messages, use bytes - const DWORD mode_flags = PIPE_TYPE_BYTE | PIPE_READMODE_BYTE; - - HANDLE pipe_handle = ::CreateNamedPipeW( - pipe_name.c_str(), open_flags, mode_flags, - 1, buffer_size, buffer_size, process::wait_timeout, nullptr); - - if (pipe_handle == INVALID_HANDLE_VALUE) - { - const auto e = GetLastError(); - cx_.bail_out(context::cmd, - "CreateNamedPipeW failed, {}", error_message(e)); - } - - pipe_.reset(pipe_handle); - } - - - // creating the write-only end of the pipe that will be passed to - // CreateProcess() - SECURITY_ATTRIBUTES sa = {}; - sa.nLength = sizeof(SECURITY_ATTRIBUTES); - - // mark this handle as being inherited by the process; if this is not TRUE, - // the connection between the two ends of the pipe will be broken - sa.bInheritHandle = TRUE; - - HANDLE output_write = ::CreateFileW( - pipe_name.c_str(), FILE_WRITE_DATA|SYNCHRONIZE, 0, - &sa, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, 0); - - if (output_write == INVALID_HANDLE_VALUE) - { - const auto e = GetLastError(); - cx_.bail_out(context::cmd, - "CreateFileW for pipe failed, {}", error_message(e)); - } - - return output_write; -} - -std::string_view async_pipe_stdout::try_read() -{ - DWORD bytes_read = 0; - - // read bytes from the pipe - const auto r = ::ReadFile( - pipe_.get(), buffer_.get(), buffer_size, &bytes_read, &ov_); - - if (r) - { - // some bytes were read, so ReadFile() turned out to be a synchronous - // operation; this happens when there's already stuff in the pipe - MOB_ASSERT(bytes_read <= buffer_size); - return {buffer_.get(), bytes_read}; - } - - - // ReadFile() failed, but it's not necessarily an error - - const auto e = GetLastError(); - - switch (e) - { - case ERROR_IO_PENDING: - { - // an async operation was started by the kernel, future calls to - // read() will end up in check_pending() to see if it completed - pending_ = true; - break; - } - - case ERROR_BROKEN_PIPE: - { - // broken pipe means the process is finished - closed_ = true; - break; - } - - default: - { - // some other hard error - cx_.bail_out(context::cmd, - "async_pipe_stdout read failed, {}", error_message(e)); - break; - } - } - - // nothing available - return {}; -} - -std::string_view async_pipe_stdout::check_pending() -{ - // check if the async operation finished, wait for a short amount of time - const auto wr = WaitForSingleObject(event_.get(), process::wait_timeout); - - if (wr == WAIT_TIMEOUT) - { - // nothing's available - return {}; - } - else if (wr == WAIT_FAILED) - { - // hard error - const auto e = GetLastError(); - cx_.bail_out(context::cmd, - "WaitForSingleObject in async_pipe_stdout failed, {}", - error_message(e)); - } - - - // the operation seems to be finished - - DWORD bytes_read = 0; - - // getting status of the async read operation - const auto r = ::GetOverlappedResult(pipe_.get(), &ov_, &bytes_read, FALSE); - - if (r) - { - // the operation has completed - - // reset for the next read() - ::ResetEvent(event_.get()); - pending_ = false; - - // return the data, if any - MOB_ASSERT(bytes_read <= buffer_size); - return {buffer_.get(), bytes_read}; - } - - // GetOverlappedResult() failed, but it's not necessarily an error - - const auto e = GetLastError(); - - switch (e) - { - case ERROR_IO_INCOMPLETE: - case WAIT_TIMEOUT: - { - // still not completed - break; - } - - case ERROR_BROKEN_PIPE: - { - // broken pipe means the process is finished - closed_ = true; - break; - } - - default: - { - // some other hard error - cx_.bail_out(context::cmd, - "GetOverlappedResult failed in async_pipe_stdout, {}", - error_message(e)); - - break; - } - } - - return {}; -} - - -async_pipe_stdin::async_pipe_stdin(const context& cx) - : cx_(cx) -{ -} - -handle_ptr async_pipe_stdin::create() -{ - // this pipe has two ends: - // - write_pipe is this end of the pipe, it will be written to - // - read_pipe is the process's end of the pipe, it will be read from - // - // read_pipe is given to the process in CreateProcess() and it needs to be - // inheritable or the connection between both ends of the pipe will be - // broken - // - // write_pipe does not need to be inheritable - - - SECURITY_ATTRIBUTES saAttr = {}; - saAttr.nLength = sizeof(SECURITY_ATTRIBUTES); - - // set both ends of the pipe as inheritable, write_pipe will be changed - // after - saAttr.bInheritHandle = TRUE; - - // creating pipe - HANDLE read_pipe, write_pipe; - if (!CreatePipe(&read_pipe, &write_pipe, &saAttr, 0)) - { - const auto e = GetLastError(); - cx_.bail_out(context::cmd, - "CreatePipe failed, {}", error_message(e)); - } - - // write_pipe does not need to be inheritable, it's not given to the process - if (!SetHandleInformation(write_pipe, HANDLE_FLAG_INHERIT, 0)) - { - const auto e = GetLastError(); - cx_.bail_out(context::cmd, - "SetHandleInformation failed, {}", error_message(e)); - } - - // keep the end that's written to - pipe_.reset(write_pipe); - - // give to other end to the new process - return handle_ptr(read_pipe); -} - -std::size_t async_pipe_stdin::write(std::string_view s) -{ - // bytes to write - const DWORD n = static_cast(s.size()); - - // bytes actually written - DWORD written = 0; - - // send to bytes down the pipe - const auto r = ::WriteFile(pipe_.get(), s.data(), n, &written, nullptr); - - if (!r) - { - // hard error - const auto e = GetLastError(); - - cx_.bail_out(context::cmd, - "WriteFile failed in async_pipe_stdin, {}", - error_message(e)); - } - - // some bytes were written correctly - return written; -} - -void async_pipe_stdin::close() -{ - pipe_ = {}; -} - -} // namespace +namespace mob { + + // many processes may be started simultaneously, this is incremented each time + // a pipe is created to make sure pipe names are unique + static std::atomic g_next_pipe_id(0); + + async_pipe_stdout::async_pipe_stdout(const context& cx) + : cx_(cx), pending_(false), closed_(true) + { + buffer_ = std::make_unique(buffer_size); + + std::memset(buffer_.get(), 0, buffer_size); + std::memset(&ov_, 0, sizeof(ov_)); + } + + bool async_pipe_stdout::closed() const + { + return closed_; + } + + handle_ptr async_pipe_stdout::create() + { + // creating pipe + handle_ptr out(create_named_pipe()); + if (out.get() == INVALID_HANDLE_VALUE) + return {}; + + // creating event + ov_.hEvent = ::CreateEvent(nullptr, TRUE, FALSE, nullptr); + + if (ov_.hEvent == NULL) { + const auto e = GetLastError(); + cx_.bail_out(context::cmd, "CreateEvent failed, {}", error_message(e)); + } + + event_.reset(ov_.hEvent); + closed_ = false; + + return out; + } + + std::string_view async_pipe_stdout::read(bool finish) + { + std::string_view s; + + if (closed_) { + // no-op + return s; + } + + if (pending_) { + // the last read() call started an async operation, check if it's done + s = check_pending(); + } + else { + // there's no async operation in progress, try to read + s = try_read(); + } + + if (finish && s.empty()) { + // nothing was read from the pipe and `finish` is true, so the process + // has terminated; in this case, assume the pipe is empty and everything + // has been read + + // even if the pipe is empty and nothing more is available, try_read() + // may have started an async operation that would return no data in the + // future + // + // make sure that operation is cancelled, because the kernel would try + // to use the OVERLAPPED buffer, which will probably have been destroyed + // by that time + ::CancelIo(pipe_.get()); + + // a future call to read() will be a no-op and closed() will return true + closed_ = true; + } + + // the bytes that were read, if any + return s; + } + + HANDLE async_pipe_stdout::create_named_pipe() + { + // unique name + const auto pipe_id = g_next_pipe_id.fetch_add(1) + 1; + + const std::wstring pipe_name = + LR"(\\.\pipe\mob_pipe)" + std::to_wstring(pipe_id); + + // creating pipe + { + const DWORD open_flags = + PIPE_ACCESS_INBOUND | // the pipe will be read from + FILE_FLAG_OVERLAPPED | // non blocking + FILE_FLAG_FIRST_PIPE_INSTANCE; // the pipe must not exist + + // pipes support either bytes or messages, use bytes + const DWORD mode_flags = PIPE_TYPE_BYTE | PIPE_READMODE_BYTE; + + HANDLE pipe_handle = ::CreateNamedPipeW( + pipe_name.c_str(), open_flags, mode_flags, 1, buffer_size, buffer_size, + process::wait_timeout, nullptr); + + if (pipe_handle == INVALID_HANDLE_VALUE) { + const auto e = GetLastError(); + cx_.bail_out(context::cmd, "CreateNamedPipeW failed, {}", + error_message(e)); + } + + pipe_.reset(pipe_handle); + } + + // creating the write-only end of the pipe that will be passed to + // CreateProcess() + SECURITY_ATTRIBUTES sa = {}; + sa.nLength = sizeof(SECURITY_ATTRIBUTES); + + // mark this handle as being inherited by the process; if this is not TRUE, + // the connection between the two ends of the pipe will be broken + sa.bInheritHandle = TRUE; + + HANDLE output_write = + ::CreateFileW(pipe_name.c_str(), FILE_WRITE_DATA | SYNCHRONIZE, 0, &sa, + OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, 0); + + if (output_write == INVALID_HANDLE_VALUE) { + const auto e = GetLastError(); + cx_.bail_out(context::cmd, "CreateFileW for pipe failed, {}", + error_message(e)); + } + + return output_write; + } + + std::string_view async_pipe_stdout::try_read() + { + DWORD bytes_read = 0; + + // read bytes from the pipe + const auto r = + ::ReadFile(pipe_.get(), buffer_.get(), buffer_size, &bytes_read, &ov_); + + if (r) { + // some bytes were read, so ReadFile() turned out to be a synchronous + // operation; this happens when there's already stuff in the pipe + MOB_ASSERT(bytes_read <= buffer_size); + return {buffer_.get(), bytes_read}; + } + + // ReadFile() failed, but it's not necessarily an error + + const auto e = GetLastError(); + + switch (e) { + case ERROR_IO_PENDING: { + // an async operation was started by the kernel, future calls to + // read() will end up in check_pending() to see if it completed + pending_ = true; + break; + } + + case ERROR_BROKEN_PIPE: { + // broken pipe means the process is finished + closed_ = true; + break; + } + + default: { + // some other hard error + cx_.bail_out(context::cmd, "async_pipe_stdout read failed, {}", + error_message(e)); + break; + } + } + + // nothing available + return {}; + } + + std::string_view async_pipe_stdout::check_pending() + { + // check if the async operation finished, wait for a short amount of time + const auto wr = WaitForSingleObject(event_.get(), process::wait_timeout); + + if (wr == WAIT_TIMEOUT) { + // nothing's available + return {}; + } + else if (wr == WAIT_FAILED) { + // hard error + const auto e = GetLastError(); + cx_.bail_out(context::cmd, + "WaitForSingleObject in async_pipe_stdout failed, {}", + error_message(e)); + } + + // the operation seems to be finished + + DWORD bytes_read = 0; + + // getting status of the async read operation + const auto r = ::GetOverlappedResult(pipe_.get(), &ov_, &bytes_read, FALSE); + + if (r) { + // the operation has completed + + // reset for the next read() + ::ResetEvent(event_.get()); + pending_ = false; + + // return the data, if any + MOB_ASSERT(bytes_read <= buffer_size); + return {buffer_.get(), bytes_read}; + } + + // GetOverlappedResult() failed, but it's not necessarily an error + + const auto e = GetLastError(); + + switch (e) { + case ERROR_IO_INCOMPLETE: + case WAIT_TIMEOUT: { + // still not completed + break; + } + + case ERROR_BROKEN_PIPE: { + // broken pipe means the process is finished + closed_ = true; + break; + } + + default: { + // some other hard error + cx_.bail_out(context::cmd, + "GetOverlappedResult failed in async_pipe_stdout, {}", + error_message(e)); + + break; + } + } + + return {}; + } + + async_pipe_stdin::async_pipe_stdin(const context& cx) : cx_(cx) {} + + handle_ptr async_pipe_stdin::create() + { + // this pipe has two ends: + // - write_pipe is this end of the pipe, it will be written to + // - read_pipe is the process's end of the pipe, it will be read from + // + // read_pipe is given to the process in CreateProcess() and it needs to be + // inheritable or the connection between both ends of the pipe will be + // broken + // + // write_pipe does not need to be inheritable + + SECURITY_ATTRIBUTES saAttr = {}; + saAttr.nLength = sizeof(SECURITY_ATTRIBUTES); + + // set both ends of the pipe as inheritable, write_pipe will be changed + // after + saAttr.bInheritHandle = TRUE; + + // creating pipe + HANDLE read_pipe, write_pipe; + if (!CreatePipe(&read_pipe, &write_pipe, &saAttr, 0)) { + const auto e = GetLastError(); + cx_.bail_out(context::cmd, "CreatePipe failed, {}", error_message(e)); + } + + // write_pipe does not need to be inheritable, it's not given to the process + if (!SetHandleInformation(write_pipe, HANDLE_FLAG_INHERIT, 0)) { + const auto e = GetLastError(); + cx_.bail_out(context::cmd, "SetHandleInformation failed, {}", + error_message(e)); + } + + // keep the end that's written to + pipe_.reset(write_pipe); + + // give to other end to the new process + return handle_ptr(read_pipe); + } + + std::size_t async_pipe_stdin::write(std::string_view s) + { + // bytes to write + const DWORD n = static_cast(s.size()); + + // bytes actually written + DWORD written = 0; + + // send to bytes down the pipe + const auto r = ::WriteFile(pipe_.get(), s.data(), n, &written, nullptr); + + if (!r) { + // hard error + const auto e = GetLastError(); + + cx_.bail_out(context::cmd, "WriteFile failed in async_pipe_stdin, {}", + error_message(e)); + } + + // some bytes were written correctly + return written; + } + + void async_pipe_stdin::close() + { + pipe_ = {}; + } + +} // namespace mob diff --git a/src/core/pipe.h b/src/core/pipe.h index 6befb48..1572ccc 100644 --- a/src/core/pipe.h +++ b/src/core/pipe.h @@ -2,113 +2,108 @@ #include "../utility.h" -namespace mob -{ - -// a pipe connected to a process's stdout or stderr, it is read from -// -class async_pipe_stdout -{ -public: - async_pipe_stdout(const context& cx); - - // a pipe has two ends: one that's given to the process so it can write to - // it, and another that's kept so it can be read from - // - // this creates both ends and returns the handle that should be given to - // the process - // - handle_ptr create(); - - // reads from the pipe and returns bytes, if any - // - // if `finish` is true (happens when the process has terminated) and nothing - // is available in the pipe, closed() will return true - // - // this may start an async read, so read() must be called repeatedly until - // the process terminates - // - std::string_view read(bool finish); - - // if this returns true, everything has been read from the pipe - // - bool closed() const; - -private: - // the maximum number of bytes that can be put in the pipe - static const std::size_t buffer_size = 50'000; - - // calling context, used for logging - const context& cx_; - - // end of the pipe that is read from - handle_ptr pipe_; - - // an event that's given to pipe for overlapped reads, signalled when data - // is available - handle_ptr event_; - - // internal buffer of `buffer_size` bytes, the data from the pipe is put - // in there - std::unique_ptr buffer_; - - // used for async reads - OVERLAPPED ov_; - - // whether the last read attempt said an async operation was started - bool pending_; - - // whether the last read attempt had `finished` true and nothing was - // available in the pipe; in this case, the pipe is considered closed - bool closed_; - - - // creates the actual pipe, sets stdout_ and returns the other end so it - // can be given to the process - // - HANDLE create_named_pipe(); - - // called when pending_ is false; tries to read from the pipe, which may - // start an async operation, in which case pending_ is set to true and an - // empty string is returned - // - // if the read operation was completed synchronously, returns the bytes - // that were read - // - std::string_view try_read(); - - // called when pending_ is true; if the async operation is finished, resets - // pending_ and returns the bytes that were read - // - std::string_view check_pending(); -}; - - -// a pipe connected to a process's stdin, it is written to; this pipe is -// synchronous and does not keep a copy of the given buffer, see write() -// -class async_pipe_stdin -{ -public: - async_pipe_stdin(const context& cx); - - handle_ptr create(); - - // tries to send all of `s` down the pipe, returns the number of bytes - // actually written - // - std::size_t write(std::string_view s); - - // closes the pipe, should be called as soon as everything has been written - // - void close(); - -private: - // calling context, used for logging - const context& cx_; - - // end of the pipe that is written to - handle_ptr pipe_; -}; - -} // namespace +namespace mob { + + // a pipe connected to a process's stdout or stderr, it is read from + // + class async_pipe_stdout { + public: + async_pipe_stdout(const context& cx); + + // a pipe has two ends: one that's given to the process so it can write to + // it, and another that's kept so it can be read from + // + // this creates both ends and returns the handle that should be given to + // the process + // + handle_ptr create(); + + // reads from the pipe and returns bytes, if any + // + // if `finish` is true (happens when the process has terminated) and nothing + // is available in the pipe, closed() will return true + // + // this may start an async read, so read() must be called repeatedly until + // the process terminates + // + std::string_view read(bool finish); + + // if this returns true, everything has been read from the pipe + // + bool closed() const; + + private: + // the maximum number of bytes that can be put in the pipe + static const std::size_t buffer_size = 50'000; + + // calling context, used for logging + const context& cx_; + + // end of the pipe that is read from + handle_ptr pipe_; + + // an event that's given to pipe for overlapped reads, signalled when data + // is available + handle_ptr event_; + + // internal buffer of `buffer_size` bytes, the data from the pipe is put + // in there + std::unique_ptr buffer_; + + // used for async reads + OVERLAPPED ov_; + + // whether the last read attempt said an async operation was started + bool pending_; + + // whether the last read attempt had `finished` true and nothing was + // available in the pipe; in this case, the pipe is considered closed + bool closed_; + + // creates the actual pipe, sets stdout_ and returns the other end so it + // can be given to the process + // + HANDLE create_named_pipe(); + + // called when pending_ is false; tries to read from the pipe, which may + // start an async operation, in which case pending_ is set to true and an + // empty string is returned + // + // if the read operation was completed synchronously, returns the bytes + // that were read + // + std::string_view try_read(); + + // called when pending_ is true; if the async operation is finished, resets + // pending_ and returns the bytes that were read + // + std::string_view check_pending(); + }; + + // a pipe connected to a process's stdin, it is written to; this pipe is + // synchronous and does not keep a copy of the given buffer, see write() + // + class async_pipe_stdin { + public: + async_pipe_stdin(const context& cx); + + handle_ptr create(); + + // tries to send all of `s` down the pipe, returns the number of bytes + // actually written + // + std::size_t write(std::string_view s); + + // closes the pipe, should be called as soon as everything has been written + // + void close(); + + private: + // calling context, used for logging + const context& cx_; + + // end of the pipe that is written to + handle_ptr pipe_; + }; + +} // namespace mob diff --git a/src/core/process.cpp b/src/core/process.cpp index a7305bb..b6f933d 100644 --- a/src/core/process.cpp +++ b/src/core/process.cpp @@ -1,953 +1,882 @@ #include "pch.h" #include "process.h" +#include "../net.h" #include "conf.h" #include "context.h" #include "op.h" #include "pipe.h" -#include "../net.h" -namespace mob -{ - -// handle to dev/null -// -HANDLE get_bit_bucket() -{ - SECURITY_ATTRIBUTES sa { .nLength = sizeof(sa), .bInheritHandle = TRUE }; - return ::CreateFileW(L"NUL", GENERIC_WRITE, 0, &sa, OPEN_EXISTING, 0, 0); -} - - -process::filter::filter(std::string_view line, context::reason r, context::level lv) - : line(line), r(r), lv(lv), discard(false) -{ -} - -process::impl::impl(const impl& i) - : interrupt(i.interrupt.load()) -{ -} - -process::impl& process::impl::operator=(const impl&) -{ - // none of these things should be copied when copying a process object, - // process should not normally be copied after they've started - - handle = {}; - job = {}; - interrupt = false; - stdout_pipe = {}; - stderr_pipe = {}; - stdin_pipe = {}; - - return *this; -} - - -process::io::io() : - unicode(false), chcp(-1), - out(context::level::trace), err(context::level::error), in_offset(0) -{ -} - -process::exec::exec() - : code(0) -{ - // default success exit code is just 0 - success.insert(0); -} - - -process::process() - : cx_(&gcx()), flags_(process::noflags) -{ -} - -// anchors -process::process(process&&) = default; -process::process(const process&) = default; -process& process::operator=(const process&) = default; -process& process::operator=(process&&) = default; - -process::~process() -{ - join(); -} - -process process::raw(const context& cx, const std::string& cmd) -{ - process p; - p.cx_ = &cx; - p.exec_.raw = cmd; - return p; -} - -process& process::set_context(const context* cx) -{ - cx_ = cx; - return *this; -} - -process& process::name(const std::string& name) -{ - name_ = name; - return *this; -} - -std::string process::name() const -{ - if (name_.empty()) - return path_to_utf8(exec_.bin.stem()); - else - return name_; -} - -process& process::binary(const fs::path& p) -{ - exec_.bin = p; - return *this; -} - -const fs::path& process::binary() const -{ - return exec_.bin; -} - -process& process::cwd(const fs::path& p) -{ - exec_.cwd = p; - return *this; -} - -const fs::path& process::cwd() const -{ - return exec_.cwd; -} - -process& process::stdout_flags(stream_flags s) -{ - io_.out.flags = s; - return *this; -} - -process& process::stdout_level(context::level lv) -{ - io_.out.level = lv; - return *this; -} - -process& process::stdout_filter(filter_fun f) -{ - io_.out.filter = f; - return *this; -} - -process& process::stdout_encoding(encodings e) -{ - io_.out.encoding = e; - return *this; -} - -process& process::stderr_flags(stream_flags s) -{ - io_.err.flags = s; - return *this; -} - -process& process::stderr_level(context::level lv) -{ - io_.err.level = lv; - return *this; -} - -process& process::stderr_filter(filter_fun f) -{ - io_.err.filter = f; - return *this; -} - -process& process::stderr_encoding(encodings e) -{ - io_.err.encoding = e; - return *this; -} - -process& process::stdin_string(std::string s) -{ - io_.in = s; - return *this; -} - -process& process::cmd_unicode(bool b) -{ - io_.unicode = b; - - if (b) - { - io_.out.encoding = encodings::utf16; - io_.err.encoding = encodings::utf16; - } - - return *this; -} - -process& process::chcp(int i) -{ - io_.chcp = i; - return *this; -} - -process& process::external_error_log(const fs::path& p) -{ - io_.error_log_file = p; - return *this; -} - -process& process::flags(process_flags f) -{ - flags_ = f; - return *this; -} - -process::process_flags process::flags() const -{ - return flags_; -} - -process& process::success_exit_codes(const std::set& v) -{ - exec_.success = v; - return *this; -} - -process& process::env(const mob::env& e) -{ - exec_.env = e; - return *this; -} - -std::string process::make_name() const -{ - if (!name().empty()) - return name(); - - return make_cmd(); -} - -std::string process::make_cmd() const -{ - if (!exec_.raw.empty()) - return exec_.raw; - - // "bin" args... - return "\"" + path_to_utf8(exec_.bin) + "\"" + exec_.cmd; -} - -void process::pipe_into(const process& p) -{ - exec_.raw = make_cmd() + " | " + p.make_cmd(); -} - -void process::run() -{ - // log cwd - if (!exec_.cwd.empty()) - cx_->debug(context::cmd, "> cd {}", exec_.cwd); - - const auto what = make_cmd(); - cx_->debug(context::cmd, "> {}", what); - - if (conf().global().dry()) - return; - - // shouldn't happen - if (exec_.raw.empty() && exec_.bin.empty()) - cx_->bail_out(context::cmd, "process: nothing to run"); - - do_run(what); -} - -void process::do_run(const std::string& what) -{ - delete_external_log_file(); - create_job(); - - io_.out.buffer = encoded_buffer(io_.out.encoding); - io_.err.buffer = encoded_buffer(io_.err.encoding); - - STARTUPINFOW si = {}; - si.cb = sizeof(si); - si.dwFlags = STARTF_USESTDHANDLES; - - // these handles are given to STARTUPINFOW and must stay alive until the - // process is created in create(), they can be closed after that - handle_ptr stdout_handle = redirect_stdout(si); - handle_ptr stderr_handle = redirect_stderr(si); - handle_ptr stdin_handle = redirect_stdin(si); - - const std::wstring cmd = utf8_to_utf16(this_env::get("COMSPEC")); - const std::wstring args = make_cmd_args(what); - const std::wstring cwd = exec_.cwd.native(); - - create(cmd, args, cwd, si); -} - -void process::delete_external_log_file() -{ - if (fs::exists(io_.error_log_file)) - { - cx_->trace(context::cmd, - "external error log file {} exists, deleting", io_.error_log_file); - - op::delete_file(*cx_, io_.error_log_file, op::optional); - } -} - -void process::create_job() -{ - SetLastError(0); - HANDLE job = CreateJobObjectW(nullptr, nullptr); - const auto e = GetLastError(); - - if (job == 0) - { - cx_->warning(context::cmd, - "failed to create job, {}", error_message(e)); - } - else - { - MOB_ASSERT(e != ERROR_ALREADY_EXISTS); - impl_.job.reset(job); - } -} - -handle_ptr process::redirect_stdout(STARTUPINFOW& si) -{ - handle_ptr h; - - switch (io_.out.flags) - { - case forward_to_log: - case keep_in_string: - { - impl_.stdout_pipe.reset(new async_pipe_stdout(*cx_)); - h = impl_.stdout_pipe->create(); - si.hStdOutput = h.get(); - break; - } - - case bit_bucket: - { - si.hStdOutput = get_bit_bucket(); - break; - } - - case inherit: - { - si.hStdOutput = ::GetStdHandle(STD_OUTPUT_HANDLE); - break; - } - } - - return h; -} - -handle_ptr process::redirect_stderr(STARTUPINFOW& si) -{ - handle_ptr h; - - switch (io_.err.flags) - { - case forward_to_log: - case keep_in_string: - { - impl_.stderr_pipe.reset(new async_pipe_stdout(*cx_)); - h = impl_.stderr_pipe->create(); - si.hStdError = h.get(); - break; - } - - case bit_bucket: - { - si.hStdError = get_bit_bucket(); - break; - } - - case inherit: - { - si.hStdError = ::GetStdHandle(STD_ERROR_HANDLE); - break; - } - } - - return h; -} - -handle_ptr process::redirect_stdin(STARTUPINFOW& si) -{ - handle_ptr h; - - if (io_.in) - { - impl_.stdin_pipe.reset(new async_pipe_stdin(*cx_)); - h = impl_.stdin_pipe->create(); - } - else - { - h.reset(get_bit_bucket()); - } - - si.hStdInput = h.get(); - - return h; -} - -void process::create( - std::wstring cmd, std::wstring args, std::wstring cwd, STARTUPINFOW si) -{ - cx_->trace(context::cmd, "creating process"); - - if (!cwd.empty()) - { - // the path might be relative, especially when it comes from the command - // line, in which case it would fail the safety check - op::create_directories(*cx_, fs::absolute(cwd)); - } - - // cwd - const wchar_t* cwd_p = (cwd.empty() ? nullptr : cwd.c_str()); - - // flags - const DWORD flags = - // will forward sigint to child processes - CREATE_NEW_PROCESS_GROUP | - - // the pointer given for environment variables is a utf16 string, not - // codepage - CREATE_UNICODE_ENVIRONMENT; - - - // creating process - PROCESS_INFORMATION pi = {}; - const auto r = ::CreateProcessW( - cmd.c_str(), args.data(), nullptr, nullptr, - TRUE, // inherit handles - flags, exec_.env.get_unicode_pointers(), cwd_p, &si, &pi); - - - if (!r) - { - const auto e = GetLastError(); - cx_->bail_out(context::cmd, - "failed to start '{}', {}", args, error_message(e)); - } - - if (impl_.job) - { - if (!::AssignProcessToJobObject(impl_.job.get(), pi.hProcess)) - { - // this shouldn't fail, but the only consequence is that ctrl-c - // won't be able to kill everything, so make it a warning - const auto e = GetLastError(); - cx_->warning(context::cmd, - "can't assign process to job, {}", error_message(e)); - } - } - - cx_->trace(context::cmd, "pid {}", pi.dwProcessId); - - // not needed - ::CloseHandle(pi.hThread); - - // process handle - impl_.handle.reset(pi.hProcess); -} - -std::wstring process::make_cmd_args(const std::string& what) const -{ - std::wstring s; - - // /U forces cmd builtins to output utf16, such as `set` or `env`, used by - // vcvars to get the environment variables - if (io_.unicode) - s += L"/U "; - - // /C runs the command and exits - s += L"/C "; - - - s += L"\""; - - // run chcp first if necessary - if (io_.chcp != -1) - s += L"chcp " + std::to_wstring(io_.chcp) + L" && "; - - // process command line - s += utf8_to_utf16(what); - - s += L"\""; - - return s; -} - -void process::interrupt() -{ - impl_.interrupt = true; - cx_->trace(context::cmd, "will interrupt"); -} - -void process::join() -{ - if (!impl_.handle) - return; - - // remembers if the process was already interrupted - bool interrupted = false; - - // close the handle quickly after termination - guard g([&] { impl_.handle = {}; }); - - cx_->trace(context::cmd, "joining"); - - for (;;) - { - // returns if the process is done or after the timeout - const auto r = WaitForSingleObject(impl_.handle.get(), wait_timeout); - - if (r == WAIT_OBJECT_0) - { - on_completed(); - break; - } - else if (r == WAIT_TIMEOUT) - { - on_timeout(interrupted); - } - else - { - const auto e = GetLastError(); - cx_->bail_out(context::cmd, - "failed to wait on process", error_message(e)); - } - } - - if (interrupted) - cx_->trace(context::cmd, "process interrupted and finished"); -} - -int process::run_and_join() -{ - run(); - join(); - return exit_code(); -} - -void process::on_timeout(bool& already_interrupted) -{ - read_pipes(false); - feed_stdin(); - - if (!already_interrupted) - already_interrupted = check_interrupted(); -} - -void process::read_pipes(bool finish) -{ - if (impl_.stdout_pipe) - read_pipe(finish, io_.out, *impl_.stdout_pipe, context::std_out); - - if (impl_.stderr_pipe) - read_pipe(finish, io_.err, *impl_.stderr_pipe, context::std_err); -} - -void process::read_pipe( - bool finish, stream& s, async_pipe_stdout& pipe, context::reason r) -{ - switch (s.flags) - { - case forward_to_log: - { - // read from the pipe, add the bytes to the buffer - s.buffer.add(pipe.read(finish)); - - // for each line in the buffer - s.buffer.next_utf8_lines(finish, [&](std::string&& line) - { - // filter it, if there's a callback - filter f(line, r, s.level); - - if (s.filter) - { - s.filter(f); - if (f.discard) - return; - } - - // don't log when ignore_output_on_success was specified, the - // process must finish before knowing whether to log or not - if (!is_set(flags_, ignore_output_on_success)) - cx_->log_string(f.r, f.lv, f.line); - - // remember this log line, can be dumped after the process - // terminates - io_.logs[f.lv].emplace_back(std::move(line)); - }); - - break; - } - - case keep_in_string: - { - // read from the pipe, add the bytes to the buffer - s.buffer.add(pipe.read(finish)); - break; - } - - case bit_bucket: - case inherit: - { - // no-op - break; - } - } -} - -void process::feed_stdin() -{ - if (io_.in && io_.in_offset < io_.in->size()) - { - io_.in_offset += impl_.stdin_pipe->write({ - io_.in->data() + io_.in_offset, io_.in->size() - io_.in_offset}); - - if (io_.in_offset >= io_.in->size()) - { - impl_.stdin_pipe->close(); - io_.in = {}; - } - } -} - -void process::on_completed() -{ - // none of this stuff is needed if the process was interrupted, mob will - // exit shortly - if (impl_.interrupt) - return; - - if (!GetExitCodeProcess(impl_.handle.get(), &exec_.code)) - { - const auto e = GetLastError(); - - cx_->error(context::cmd, - "failed to get exit code, ", error_message(e)); - - exec_.code = 0xffff; - } - - // pipes are finicky, or I just don't understand how they work - // - // I've seen empty pipes after processes finish even though there was still - // data left somewhere in between that hadn't reached this end of the pipe - // yet - // - // so pipes are read one last time with `finish` false, meaning that an - // empty pipe won't be closed and the last line of the buffer won't be - // processed yet - // - // then pipes are read again in a loop with `finish` true, and hopefully - // all the content will have been read by that time - - read_pipes(false); - - for (;;) - { - read_pipes(true); - - // loop until both pipes are closed - if (impl_.stdout_pipe && !impl_.stdout_pipe->closed()) - continue; - - if (impl_.stderr_pipe && !impl_.stderr_pipe->closed()) - continue; - - break; - } - - - // check if the exit code is considered success - if (exec_.success.contains(static_cast(exec_.code))) - on_process_successful(); - else - on_process_failed(); -} - -void process::on_process_successful() -{ - const bool ignore_output = is_set(flags_, ignore_output_on_success); - const auto& warnings = io_.logs[context::level::warning]; - const auto& errors = io_.logs[context::level::error]; - - if (ignore_output || (warnings.empty() && errors.empty())) - { - // the process was successful and there were no warnings or errors, - // or they should be ignored - cx_->trace(context::cmd, - "process exit code is {} (considered success)", exec_.code); - } - else - { - // the process was successful, but there were warnings or errors, log - // them - - cx_->warning( - context::cmd, - "process exit code is {} (considered success), " - "but stderr had something", exec_.code); - - // don't re-log the same stuff - if (io_.err.flags != forward_to_log) - { - cx_->warning(context::cmd, "process was: {}", make_cmd()); - cx_->warning(context::cmd, "stderr:"); - - for (auto&& line : warnings) - cx_->warning(context::std_err, " {}", line); - - for (auto&& line : errors) - cx_->warning(context::std_err, " {}", line); - } - } -} - -void process::on_process_failed() -{ - if (flags_ & allow_failure) - { - // ignore failure if the flag is set, it's used for optional things - cx_->trace(context::cmd, - "process failed but failure was allowed"); - } - else - { - dump_error_log_file(); - dump_stderr(); - cx_->bail_out(context::cmd, "{} returned {}", make_name(), exec_.code); - } -} - -bool process::check_interrupted() -{ - if (!impl_.interrupt) - return false; - - const auto pid = GetProcessId(impl_.handle.get()); - - // interruption is normally done by sending sigint, which requires a pid; - // without a pid, the process can be killed from the handle - - if (pid == 0) - { - cx_->trace(context::cmd, - "process id is 0, terminating instead"); - - terminate(); - } - else - { - cx_->trace(context::cmd, "sending sigint to {}", pid); - GenerateConsoleCtrlEvent(CTRL_BREAK_EVENT, pid); - - if (flags_ & terminate_on_interrupt) - { - // this process doesn't support sigint or doesn't handle it very - // well; sigint is also sent for good measure - - cx_->trace(context::cmd, - "terminating process (flag is set)"); - - terminate(); - } - } - - return true; -} - -void process::terminate() -{ - UINT exit_code = 0xff; - - if (impl_.job) - { - // kill all the child processes in the job - - JOBOBJECT_BASIC_ACCOUNTING_INFORMATION info = {}; - - const auto r = ::QueryInformationJobObject( - impl_.job.get(), JobObjectBasicAccountingInformation, - &info, sizeof(info), nullptr); - - if (r) - { - gcx().trace(context::cmd, - "terminating job, {} processes ({} spawned total)", - info.ActiveProcesses, info.TotalProcesses); - } - else - { - gcx().trace(context::cmd, "terminating job"); - } - - if (::TerminateJobObject(impl_.job.get(), exit_code)) - { - // done - return; - } - - const auto e = GetLastError(); - gcx().warning(context::cmd, - "failed to terminate job, {}", error_message(e)); - } - - // either job creation failed or job termination failed, last ditch attempt - ::TerminateProcess(impl_.handle.get(), exit_code); -} - -void process::dump_error_log_file() noexcept -{ - if (io_.error_log_file.empty()) - return; - - if (!fs::exists(io_.error_log_file)) - { - cx_->debug(context::cmd, - "external error log file {} doesn't exist", io_.error_log_file); - - return; - } - - - const std::string log = op::read_text_file( - *cx_, encodings::dont_know, io_.error_log_file, op::optional); - - if (log.empty()) - return; - - cx_->error(context::cmd, - "{} failed, content of {}:", make_name(), io_.error_log_file); - - for_each_line(log, [&](auto&& line) - { - cx_->error(context::cmd, " {}", line); - }); -} - -void process::dump_stderr() noexcept -{ - const std::string s = io_.err.buffer.utf8_string(); - - if (s.empty()) - { - cx_->error(context::cmd, - "{} failed, stderr was empty", make_name()); - } - else - { - cx_->error(context::cmd, - "{} failed, {}, content of stderr:", make_name(), make_cmd()); - - for_each_line(s, [&](auto&& line) - { - cx_->error(context::cmd, " {}", line); - }); - } -} - -int process::exit_code() const -{ - return static_cast(exec_.code); -} - -std::string process::stdout_string() -{ - return io_.out.buffer.utf8_string(); -} - -std::string process::stderr_string() -{ - return io_.err.buffer.utf8_string(); -} - -void process::add_arg(const std::string& k, const std::string& v, arg_flags f) -{ - // don't add the argument if it's for a log level that's not active - if ((f & log_debug) && !context::enabled(context::level::debug)) - return; - - if ((f & log_trace) && !context::enabled(context::level::trace)) - return; - - if ((f & log_dump) && !context::enabled(context::level::dump)) - return; - - if ((f & log_quiet) && context::enabled(context::level::trace)) - return; - - // empty? - if (k.empty() && v.empty()) - return; - - if (k.empty()) - { - exec_.cmd += " " + v; - } - else - { - // key and value, don't put space if the key ends with = or the flag is - // set - if ((f & nospace) || k.back() == '=') - exec_.cmd += " " + k + v; - else - exec_.cmd += " " + k + " " + v; - } -} - -std::string process::arg_to_string(const char* s, arg_flags f) -{ - if (f & quote) - return "\"" + std::string(s) + "\""; - else - return s; -} - -std::string process::arg_to_string(const std::string& s, arg_flags f) -{ - if (f & quote) - return "\"" + std::string(s) + "\""; - else - return s; -} - -std::string process::arg_to_string(const fs::path& p, arg_flags f) -{ - std::string s = path_to_utf8(p); - - if (f & forward_slashes) - s = replace_all(s, "\\", "/"); - - return "\"" + s + "\""; -} - -std::string process::arg_to_string(const url& u, arg_flags f) -{ - if (f & quote) - return "\"" + u.string() + "\""; - else - return u.string(); -} - -std::string process::arg_to_string(int i, arg_flags) -{ - return std::to_string(i); -} - -} // namespace +namespace mob { + + // handle to dev/null + // + HANDLE get_bit_bucket() + { + SECURITY_ATTRIBUTES sa{.nLength = sizeof(sa), .bInheritHandle = TRUE}; + return ::CreateFileW(L"NUL", GENERIC_WRITE, 0, &sa, OPEN_EXISTING, 0, 0); + } + + process::filter::filter(std::string_view line, context::reason r, context::level lv) + : line(line), r(r), lv(lv), discard(false) + { + } + + process::impl::impl(const impl& i) : interrupt(i.interrupt.load()) {} + + process::impl& process::impl::operator=(const impl&) + { + // none of these things should be copied when copying a process object, + // process should not normally be copied after they've started + + handle = {}; + job = {}; + interrupt = false; + stdout_pipe = {}; + stderr_pipe = {}; + stdin_pipe = {}; + + return *this; + } + + process::io::io() + : unicode(false), chcp(-1), out(context::level::trace), + err(context::level::error), in_offset(0) + { + } + + process::exec::exec() : code(0) + { + // default success exit code is just 0 + success.insert(0); + } + + process::process() : cx_(&gcx()), flags_(process::noflags) {} + + // anchors + process::process(process&&) = default; + process::process(const process&) = default; + process& process::operator=(const process&) = default; + process& process::operator=(process&&) = default; + + process::~process() + { + join(); + } + + process process::raw(const context& cx, const std::string& cmd) + { + process p; + p.cx_ = &cx; + p.exec_.raw = cmd; + return p; + } + + process& process::set_context(const context* cx) + { + cx_ = cx; + return *this; + } + + process& process::name(const std::string& name) + { + name_ = name; + return *this; + } + + std::string process::name() const + { + if (name_.empty()) + return path_to_utf8(exec_.bin.stem()); + else + return name_; + } + + process& process::binary(const fs::path& p) + { + exec_.bin = p; + return *this; + } + + const fs::path& process::binary() const + { + return exec_.bin; + } + + process& process::cwd(const fs::path& p) + { + exec_.cwd = p; + return *this; + } + + const fs::path& process::cwd() const + { + return exec_.cwd; + } + + process& process::stdout_flags(stream_flags s) + { + io_.out.flags = s; + return *this; + } + + process& process::stdout_level(context::level lv) + { + io_.out.level = lv; + return *this; + } + + process& process::stdout_filter(filter_fun f) + { + io_.out.filter = f; + return *this; + } + + process& process::stdout_encoding(encodings e) + { + io_.out.encoding = e; + return *this; + } + + process& process::stderr_flags(stream_flags s) + { + io_.err.flags = s; + return *this; + } + + process& process::stderr_level(context::level lv) + { + io_.err.level = lv; + return *this; + } + + process& process::stderr_filter(filter_fun f) + { + io_.err.filter = f; + return *this; + } + + process& process::stderr_encoding(encodings e) + { + io_.err.encoding = e; + return *this; + } + + process& process::stdin_string(std::string s) + { + io_.in = s; + return *this; + } + + process& process::cmd_unicode(bool b) + { + io_.unicode = b; + + if (b) { + io_.out.encoding = encodings::utf16; + io_.err.encoding = encodings::utf16; + } + + return *this; + } + + process& process::chcp(int i) + { + io_.chcp = i; + return *this; + } + + process& process::external_error_log(const fs::path& p) + { + io_.error_log_file = p; + return *this; + } + + process& process::flags(process_flags f) + { + flags_ = f; + return *this; + } + + process::process_flags process::flags() const + { + return flags_; + } + + process& process::success_exit_codes(const std::set& v) + { + exec_.success = v; + return *this; + } + + process& process::env(const mob::env& e) + { + exec_.env = e; + return *this; + } + + std::string process::make_name() const + { + if (!name().empty()) + return name(); + + return make_cmd(); + } + + std::string process::make_cmd() const + { + if (!exec_.raw.empty()) + return exec_.raw; + + // "bin" args... + return "\"" + path_to_utf8(exec_.bin) + "\"" + exec_.cmd; + } + + void process::pipe_into(const process& p) + { + exec_.raw = make_cmd() + " | " + p.make_cmd(); + } + + void process::run() + { + // log cwd + if (!exec_.cwd.empty()) + cx_->debug(context::cmd, "> cd {}", exec_.cwd); + + const auto what = make_cmd(); + cx_->debug(context::cmd, "> {}", what); + + if (conf().global().dry()) + return; + + // shouldn't happen + if (exec_.raw.empty() && exec_.bin.empty()) + cx_->bail_out(context::cmd, "process: nothing to run"); + + do_run(what); + } + + void process::do_run(const std::string& what) + { + delete_external_log_file(); + create_job(); + + io_.out.buffer = encoded_buffer(io_.out.encoding); + io_.err.buffer = encoded_buffer(io_.err.encoding); + + STARTUPINFOW si = {}; + si.cb = sizeof(si); + si.dwFlags = STARTF_USESTDHANDLES; + + // these handles are given to STARTUPINFOW and must stay alive until the + // process is created in create(), they can be closed after that + handle_ptr stdout_handle = redirect_stdout(si); + handle_ptr stderr_handle = redirect_stderr(si); + handle_ptr stdin_handle = redirect_stdin(si); + + const std::wstring cmd = utf8_to_utf16(this_env::get("COMSPEC")); + const std::wstring args = make_cmd_args(what); + const std::wstring cwd = exec_.cwd.native(); + + create(cmd, args, cwd, si); + } + + void process::delete_external_log_file() + { + if (fs::exists(io_.error_log_file)) { + cx_->trace(context::cmd, "external error log file {} exists, deleting", + io_.error_log_file); + + op::delete_file(*cx_, io_.error_log_file, op::optional); + } + } + + void process::create_job() + { + SetLastError(0); + HANDLE job = CreateJobObjectW(nullptr, nullptr); + const auto e = GetLastError(); + + if (job == 0) { + cx_->warning(context::cmd, "failed to create job, {}", error_message(e)); + } + else { + MOB_ASSERT(e != ERROR_ALREADY_EXISTS); + impl_.job.reset(job); + } + } + + handle_ptr process::redirect_stdout(STARTUPINFOW& si) + { + handle_ptr h; + + switch (io_.out.flags) { + case forward_to_log: + case keep_in_string: { + impl_.stdout_pipe.reset(new async_pipe_stdout(*cx_)); + h = impl_.stdout_pipe->create(); + si.hStdOutput = h.get(); + break; + } + + case bit_bucket: { + si.hStdOutput = get_bit_bucket(); + break; + } + + case inherit: { + si.hStdOutput = ::GetStdHandle(STD_OUTPUT_HANDLE); + break; + } + } + + return h; + } + + handle_ptr process::redirect_stderr(STARTUPINFOW& si) + { + handle_ptr h; + + switch (io_.err.flags) { + case forward_to_log: + case keep_in_string: { + impl_.stderr_pipe.reset(new async_pipe_stdout(*cx_)); + h = impl_.stderr_pipe->create(); + si.hStdError = h.get(); + break; + } + + case bit_bucket: { + si.hStdError = get_bit_bucket(); + break; + } + + case inherit: { + si.hStdError = ::GetStdHandle(STD_ERROR_HANDLE); + break; + } + } + + return h; + } + + handle_ptr process::redirect_stdin(STARTUPINFOW& si) + { + handle_ptr h; + + if (io_.in) { + impl_.stdin_pipe.reset(new async_pipe_stdin(*cx_)); + h = impl_.stdin_pipe->create(); + } + else { + h.reset(get_bit_bucket()); + } + + si.hStdInput = h.get(); + + return h; + } + + void process::create(std::wstring cmd, std::wstring args, std::wstring cwd, + STARTUPINFOW si) + { + cx_->trace(context::cmd, "creating process"); + + if (!cwd.empty()) { + // the path might be relative, especially when it comes from the command + // line, in which case it would fail the safety check + op::create_directories(*cx_, fs::absolute(cwd)); + } + + // cwd + const wchar_t* cwd_p = (cwd.empty() ? nullptr : cwd.c_str()); + + // flags + const DWORD flags = + // will forward sigint to child processes + CREATE_NEW_PROCESS_GROUP | + + // the pointer given for environment variables is a utf16 string, not + // codepage + CREATE_UNICODE_ENVIRONMENT; + + // creating process + PROCESS_INFORMATION pi = {}; + const auto r = + ::CreateProcessW(cmd.c_str(), args.data(), nullptr, nullptr, + TRUE, // inherit handles + flags, exec_.env.get_unicode_pointers(), cwd_p, &si, &pi); + + if (!r) { + const auto e = GetLastError(); + cx_->bail_out(context::cmd, "failed to start '{}', {}", args, + error_message(e)); + } + + if (impl_.job) { + if (!::AssignProcessToJobObject(impl_.job.get(), pi.hProcess)) { + // this shouldn't fail, but the only consequence is that ctrl-c + // won't be able to kill everything, so make it a warning + const auto e = GetLastError(); + cx_->warning(context::cmd, "can't assign process to job, {}", + error_message(e)); + } + } + + cx_->trace(context::cmd, "pid {}", pi.dwProcessId); + + // not needed + ::CloseHandle(pi.hThread); + + // process handle + impl_.handle.reset(pi.hProcess); + } + + std::wstring process::make_cmd_args(const std::string& what) const + { + std::wstring s; + + // /U forces cmd builtins to output utf16, such as `set` or `env`, used by + // vcvars to get the environment variables + if (io_.unicode) + s += L"/U "; + + // /C runs the command and exits + s += L"/C "; + + s += L"\""; + + // run chcp first if necessary + if (io_.chcp != -1) + s += L"chcp " + std::to_wstring(io_.chcp) + L" && "; + + // process command line + s += utf8_to_utf16(what); + + s += L"\""; + + return s; + } + + void process::interrupt() + { + impl_.interrupt = true; + cx_->trace(context::cmd, "will interrupt"); + } + + void process::join() + { + if (!impl_.handle) + return; + + // remembers if the process was already interrupted + bool interrupted = false; + + // close the handle quickly after termination + guard g([&] { + impl_.handle = {}; + }); + + cx_->trace(context::cmd, "joining"); + + for (;;) { + // returns if the process is done or after the timeout + const auto r = WaitForSingleObject(impl_.handle.get(), wait_timeout); + + if (r == WAIT_OBJECT_0) { + on_completed(); + break; + } + else if (r == WAIT_TIMEOUT) { + on_timeout(interrupted); + } + else { + const auto e = GetLastError(); + cx_->bail_out(context::cmd, "failed to wait on process", + error_message(e)); + } + } + + if (interrupted) + cx_->trace(context::cmd, "process interrupted and finished"); + } + + int process::run_and_join() + { + run(); + join(); + return exit_code(); + } + + void process::on_timeout(bool& already_interrupted) + { + read_pipes(false); + feed_stdin(); + + if (!already_interrupted) + already_interrupted = check_interrupted(); + } + + void process::read_pipes(bool finish) + { + if (impl_.stdout_pipe) + read_pipe(finish, io_.out, *impl_.stdout_pipe, context::std_out); + + if (impl_.stderr_pipe) + read_pipe(finish, io_.err, *impl_.stderr_pipe, context::std_err); + } + + void process::read_pipe(bool finish, stream& s, async_pipe_stdout& pipe, + context::reason r) + { + switch (s.flags) { + case forward_to_log: { + // read from the pipe, add the bytes to the buffer + s.buffer.add(pipe.read(finish)); + + // for each line in the buffer + s.buffer.next_utf8_lines(finish, [&](std::string&& line) { + // filter it, if there's a callback + filter f(line, r, s.level); + + if (s.filter) { + s.filter(f); + if (f.discard) + return; + } + + // don't log when ignore_output_on_success was specified, the + // process must finish before knowing whether to log or not + if (!is_set(flags_, ignore_output_on_success)) + cx_->log_string(f.r, f.lv, f.line); + + // remember this log line, can be dumped after the process + // terminates + io_.logs[f.lv].emplace_back(std::move(line)); + }); + + break; + } + + case keep_in_string: { + // read from the pipe, add the bytes to the buffer + s.buffer.add(pipe.read(finish)); + break; + } + + case bit_bucket: + case inherit: { + // no-op + break; + } + } + } + + void process::feed_stdin() + { + if (io_.in && io_.in_offset < io_.in->size()) { + io_.in_offset += impl_.stdin_pipe->write( + {io_.in->data() + io_.in_offset, io_.in->size() - io_.in_offset}); + + if (io_.in_offset >= io_.in->size()) { + impl_.stdin_pipe->close(); + io_.in = {}; + } + } + } + + void process::on_completed() + { + // none of this stuff is needed if the process was interrupted, mob will + // exit shortly + if (impl_.interrupt) + return; + + if (!GetExitCodeProcess(impl_.handle.get(), &exec_.code)) { + const auto e = GetLastError(); + + cx_->error(context::cmd, "failed to get exit code, ", error_message(e)); + + exec_.code = 0xffff; + } + + // pipes are finicky, or I just don't understand how they work + // + // I've seen empty pipes after processes finish even though there was still + // data left somewhere in between that hadn't reached this end of the pipe + // yet + // + // so pipes are read one last time with `finish` false, meaning that an + // empty pipe won't be closed and the last line of the buffer won't be + // processed yet + // + // then pipes are read again in a loop with `finish` true, and hopefully + // all the content will have been read by that time + + read_pipes(false); + + for (;;) { + read_pipes(true); + + // loop until both pipes are closed + if (impl_.stdout_pipe && !impl_.stdout_pipe->closed()) + continue; + + if (impl_.stderr_pipe && !impl_.stderr_pipe->closed()) + continue; + + break; + } + + // check if the exit code is considered success + if (exec_.success.contains(static_cast(exec_.code))) + on_process_successful(); + else + on_process_failed(); + } + + void process::on_process_successful() + { + const bool ignore_output = is_set(flags_, ignore_output_on_success); + const auto& warnings = io_.logs[context::level::warning]; + const auto& errors = io_.logs[context::level::error]; + + if (ignore_output || (warnings.empty() && errors.empty())) { + // the process was successful and there were no warnings or errors, + // or they should be ignored + cx_->trace(context::cmd, "process exit code is {} (considered success)", + exec_.code); + } + else { + // the process was successful, but there were warnings or errors, log + // them + + cx_->warning(context::cmd, + "process exit code is {} (considered success), " + "but stderr had something", + exec_.code); + + // don't re-log the same stuff + if (io_.err.flags != forward_to_log) { + cx_->warning(context::cmd, "process was: {}", make_cmd()); + cx_->warning(context::cmd, "stderr:"); + + for (auto&& line : warnings) + cx_->warning(context::std_err, " {}", line); + + for (auto&& line : errors) + cx_->warning(context::std_err, " {}", line); + } + } + } + + void process::on_process_failed() + { + if (flags_ & allow_failure) { + // ignore failure if the flag is set, it's used for optional things + cx_->trace(context::cmd, "process failed but failure was allowed"); + } + else { + dump_error_log_file(); + dump_stderr(); + cx_->bail_out(context::cmd, "{} returned {}", make_name(), exec_.code); + } + } + + bool process::check_interrupted() + { + if (!impl_.interrupt) + return false; + + const auto pid = GetProcessId(impl_.handle.get()); + + // interruption is normally done by sending sigint, which requires a pid; + // without a pid, the process can be killed from the handle + + if (pid == 0) { + cx_->trace(context::cmd, "process id is 0, terminating instead"); + + terminate(); + } + else { + cx_->trace(context::cmd, "sending sigint to {}", pid); + GenerateConsoleCtrlEvent(CTRL_BREAK_EVENT, pid); + + if (flags_ & terminate_on_interrupt) { + // this process doesn't support sigint or doesn't handle it very + // well; sigint is also sent for good measure + + cx_->trace(context::cmd, "terminating process (flag is set)"); + + terminate(); + } + } + + return true; + } + + void process::terminate() + { + UINT exit_code = 0xff; + + if (impl_.job) { + // kill all the child processes in the job + + JOBOBJECT_BASIC_ACCOUNTING_INFORMATION info = {}; + + const auto r = ::QueryInformationJobObject( + impl_.job.get(), JobObjectBasicAccountingInformation, &info, + sizeof(info), nullptr); + + if (r) { + gcx().trace(context::cmd, + "terminating job, {} processes ({} spawned total)", + info.ActiveProcesses, info.TotalProcesses); + } + else { + gcx().trace(context::cmd, "terminating job"); + } + + if (::TerminateJobObject(impl_.job.get(), exit_code)) { + // done + return; + } + + const auto e = GetLastError(); + gcx().warning(context::cmd, "failed to terminate job, {}", + error_message(e)); + } + + // either job creation failed or job termination failed, last ditch attempt + ::TerminateProcess(impl_.handle.get(), exit_code); + } + + void process::dump_error_log_file() noexcept + { + if (io_.error_log_file.empty()) + return; + + if (!fs::exists(io_.error_log_file)) { + cx_->debug(context::cmd, "external error log file {} doesn't exist", + io_.error_log_file); + + return; + } + + const std::string log = op::read_text_file(*cx_, encodings::dont_know, + io_.error_log_file, op::optional); + + if (log.empty()) + return; + + cx_->error(context::cmd, "{} failed, content of {}:", make_name(), + io_.error_log_file); + + for_each_line(log, [&](auto&& line) { + cx_->error(context::cmd, " {}", line); + }); + } + + void process::dump_stderr() noexcept + { + const std::string s = io_.err.buffer.utf8_string(); + + if (s.empty()) { + cx_->error(context::cmd, "{} failed, stderr was empty", make_name()); + } + else { + cx_->error(context::cmd, "{} failed, {}, content of stderr:", make_name(), + make_cmd()); + + for_each_line(s, [&](auto&& line) { + cx_->error(context::cmd, " {}", line); + }); + } + } + + int process::exit_code() const + { + return static_cast(exec_.code); + } + + std::string process::stdout_string() + { + return io_.out.buffer.utf8_string(); + } + + std::string process::stderr_string() + { + return io_.err.buffer.utf8_string(); + } + + void process::add_arg(const std::string& k, const std::string& v, arg_flags f) + { + // don't add the argument if it's for a log level that's not active + if ((f & log_debug) && !context::enabled(context::level::debug)) + return; + + if ((f & log_trace) && !context::enabled(context::level::trace)) + return; + + if ((f & log_dump) && !context::enabled(context::level::dump)) + return; + + if ((f & log_quiet) && context::enabled(context::level::trace)) + return; + + // empty? + if (k.empty() && v.empty()) + return; + + if (k.empty()) { + exec_.cmd += " " + v; + } + else { + // key and value, don't put space if the key ends with = or the flag is + // set + if ((f & nospace) || k.back() == '=') + exec_.cmd += " " + k + v; + else + exec_.cmd += " " + k + " " + v; + } + } + + std::string process::arg_to_string(const char* s, arg_flags f) + { + if (f & quote) + return "\"" + std::string(s) + "\""; + else + return s; + } + + std::string process::arg_to_string(const std::string& s, arg_flags f) + { + if (f & quote) + return "\"" + std::string(s) + "\""; + else + return s; + } + + std::string process::arg_to_string(const fs::path& p, arg_flags f) + { + std::string s = path_to_utf8(p); + + if (f & forward_slashes) + s = replace_all(s, "\\", "/"); + + return "\"" + s + "\""; + } + + std::string process::arg_to_string(const url& u, arg_flags f) + { + if (f & quote) + return "\"" + u.string() + "\""; + else + return u.string(); + } + + std::string process::arg_to_string(int i, arg_flags) + { + return std::to_string(i); + } + +} // namespace mob diff --git a/src/core/process.h b/src/core/process.h index 881d7fe..b7e6eab 100644 --- a/src/core/process.h +++ b/src/core/process.h @@ -1,572 +1,546 @@ #pragma once -#include "env.h" -#include "context.h" #include "../utility.h" +#include "context.h" +#include "env.h" -namespace mob -{ - -class url; -class async_pipe_stdout; -class async_pipe_stdin; - - -class process -{ -public: - static constexpr DWORD wait_timeout = 50; - - // given in flags(), control process creation and termination - // - enum process_flags - { - noflags = 0x00, - - // will not bail out on failure, used for optional processes - allow_failure = 0x01, - - // some processes just refuse to die when given sigint, like jom, so - // this just kills the process - terminate_on_interrupt = 0x02, - - // some processes output useless stuff even when they're successful, - // so to try to keep the amount of logs down, specifying this will - // discard the output of the process is successful - ignore_output_on_success = 0x04 - }; - - - // used in arg(), controls how args are converted to string and whether - // they're tied to a specific log level - // - enum arg_flags - { - noargflags = 0x00, - - // the argument should only be used when the given log level is active; - // a /q switch for quiet output would have the log_quiet flag, for - // example - log_debug = 0x01, - log_trace = 0x02, - log_dump = 0x04, - log_quiet = 0x08, - - // for arg(k, v), doesn't put a space between `k` and `v`; some programs - // are pretty strict with their arguments, like 7z working for `-opath` - // but not `-o path` - nospace = 0x10, - - // for arg(k, v) or arg(v), forces the value to be double-quoted (but - // not the key); note that fs::path and url objects are always quoted - // automatically - quote = 0x20, - - // converts backslashes to forward slashes for the given fs::path - // value; ignored for other types - forward_slashes = 0x40 - }; - - - // used in stdout_flags() and stderr_flags(), controls what to do with the - // process' output - // - enum stream_flags - { - // default, forwards the output directly to mob's logs, both file and - // console, depending on the configuration - forward_to_log = 1, - - // discards the output - bit_bucket, - - // does not log the output, keeps it in a string; can be retrieved in - // stdout_string() and stderr_string() - keep_in_string, - - // inherits stdout/stderr from this process; this is only used by - // processes started early in mob before things are set up, like when - // calling vswhere, so it just dumps stuff to the console - inherit - }; - - - // used to filter the output of a process, because some programs really suck - // at being quiet and will output crap that's not needed but cannot be - // inhibited - // - // a lambda is given to stdout_filter() and stderr_filter() that will - // receive a `filter` object, allowing to change the log level of a line - // or just discard it completely - // - struct filter - { - // a line from the process' output - std::string_view line; - - // the base reason for the stream, either context::std_out or - // context::std_err, can be modified by the filter - context::reason r; - - // the base level for the stream, defaults to stdout=trace and - // stderr=error, or whatever was given to stdout_level()/stderr_level() - // below; can be modified by the filter - context::level lv; - - // whether to discard this log line; can be set to true by the filter - bool discard; - - filter(std::string_view line, context::reason r, context::level lv); - - // this struct is only used by filter_fun callback, where copying a - // filter wouldn't make any sense - filter(const filter&) = delete; - filter& operator=(const filter&) = delete; - }; - - using filter_fun = std::function; - - - // empty process - // - process(); - - // joins - // - ~process(); - - // anchors, all defaults - process(process&&); - process(const process&); - process& operator=(const process&); - process& operator=(process&&); - - - // creates a process from the given command line instead of using the - // various binary(), arg(), etc. - // - static process raw(const context& cx, const std::string& cmd); - - // used by pipe(...) below to finish recursion - // - static process pipe(process p) - { - return p; - } - - // constructs a process object by concatenating the command line of the - // given processes with " | " in between; this can only be used with fully - // set up processes, their command line is extracted immediately - // - // it's basically only used by 7z to pipe tar into it and is not a very - // generic solution - // - template - static process pipe(const process& p1, const process& p2, Processes&&... ps) - { - auto r = p1; - r.pipe_into(p2); - pipe(r, std::forward(ps)...); - return r; - } - - // sets the context of this process, used for all logging, bailing out, - // filesystem operations, etc.; the process runners use this before spawning - // the process - // - process& set_context(const context* cx); - - // display name for the process, used in logging; if not set, returns the - // filename without extension of the binary, which may be an empty string - // - process& name(const std::string& name); - std::string name() const; - - // path to the executable - // - process& binary(const fs::path& p); - const fs::path& binary() const; - - // working directory - // - process& cwd(const fs::path& p); - const fs::path& cwd() const; - - // process flags - // - process& flags(process_flags f); - process_flags flags() const; - - // sets flags for stdout/stderr - // - process& stdout_flags(stream_flags s); - process& stderr_flags(stream_flags s); - - // sets the default log level for stdout/stderr; if not given, stdout is - // trace, stderr is error - // - process& stdout_level(context::level lv); - process& stderr_level(context::level lv); - - // sets a callback to filter the output - // - process& stdout_filter(filter_fun f); - process& stderr_filter(filter_fun f); - - // sets the encoding of stdout/stderr, defaults to dont_know, which doesn't - // do any conversion - // - process& stdout_encoding(encodings e); - process& stderr_encoding(encodings e); - - // if the string is not empty, the process' stdin will be redirected and - // given the string - // - process& stdin_string(std::string s); - - // if not -1, `chcp cp` will be executed before spawning the process; note - // that processes are started in their own cmd instance, so this won't leak - // - process& chcp(int cp); - - // passes /U to cmd when spawning the process - // - // this is basically only used when getting the environment variables after - // calling vcvars to force `set` to output in utf16, because it normally - // outputs in the current codepage - // - // also forces stdout_encoding() and stderr_encoding() to utf16 - // - process& cmd_unicode(bool b); - - // some processes output to an external log file instead of to - // stdout/stderr, such as boost's boostrap.bat - // - // if the process failed (returned an exit code considered failure), the - // content of the file will be dumped to the logs as errors; if the process - // succeeded, the file is ignored - // - // the file is always deleted before starting the process - // - process& external_error_log(const fs::path& p); - - // the default success exit code is 0, this can be used to override it; any - // exit code found in the set is considered success - // - // basically only used by transifex `init`, which exits with 2 when the - // directory already has a .tx folder - // - process& success_exit_codes(const std::set& v); - - - // adds an argument to the command line, see comment on top for conversions - // - template >> - process& arg(const T& value, arg_flags f=noargflags) - { - add_arg("", arg_to_string(value, f), f); - return *this; - } - - // adds a name=value argument to the command line, see comment on top for - // conversions - // - template >> - process& arg(const std::string& name, const T& value, arg_flags f=noargflags) - { - add_arg(name, arg_to_string(value, f), f); - return *this; - } - - // adds every name=value pair to the command line, see comment on top for - // conversions - // - template class Container, class K, class V, class Alloc> - process& args(const Container, Alloc>& v, arg_flags f=noargflags) - { - for (auto&& [name, value] : v) - arg(name, value, f); - - return *this; - } - - // adds every string from the container to the command line as-is - // - process& args(const std::vector& v, arg_flags f=noargflags) - { - for (auto&& e : v) - add_arg(e, "", f); - - return *this; - } - - // sets the environment variables for this process - // - // this can use something like this_env() to add variables, or env::vs() - // to get a VS environment - // - process& env(const mob::env& e); - - // spawns the process, returns immediately; bails out if it fails - // - // join() must be called shortly after, that's where the process is - // monitored and redirected streams handled - // - void run(); - - // forces the process to exit by sending sigint or killing it, depending - // on process flags; this just sets a flag, join() does the work - // - void interrupt(); - - // reads from streams, writes to stdin if needed, monitors for termination, - // handles interrupt(); bails out on failure - // - void join(); - - // calls run(), join() and returns exit_code() - // - int run_and_join(); - - // exit code of the process, only valid after join() returns - // - int exit_code() const; - - // content of stdout/stderr if keep_in_string is set - // - std::string stdout_string(); - std::string stderr_string(); - -private: - // stuff that must be handled when copying process objects - // - struct impl - { - // process handle - handle_ptr handle; - - // job handle; processes are added to a job so child processes can be - // monitored and terminated - handle_ptr job; - - // whether the process should be killed - std::atomic interrupt{false}; - - // pipes - std::unique_ptr stdout_pipe; - std::unique_ptr stderr_pipe; - std::unique_ptr stdin_pipe; - - impl() = default; - impl(const impl&); - impl& operator=(const impl&); - }; - - // info about stdout/stderr - // - struct stream - { - stream_flags flags; - context::level level; - filter_fun filter; - encodings encoding; - - // anything output to stdout/stderr ends up here - encoded_buffer buffer; - - stream(context::level lv) - : flags(forward_to_log), level(lv), encoding(encodings::dont_know) - { - } - }; - - // info about the process io - // - struct io - { - // whether /U is given to cmd, see cmd_unicode() - bool unicode; - - // if not -1, calls chcp before spawning, see chcp() - int chcp; - - // stdout/stderr - stream out; - stream err; - - // stdin string that's fed to the process - std::optional in; - - // index of last character written to stdin - std::size_t in_offset; - - // see external_error_log() - fs::path error_log_file; - - // each line from the process is saved in this map so it can be output - // after the process has completed successfully but had stuff in stderr - std::map> logs; - - io(); - }; - - // info about execution - // - struct exec - { - fs::path bin; - fs::path cwd; - mob::env env; - - // success exit codes, defaults to 0 - std::set success; - - // set in process::raw() or process::pipe(), used instead of cmd - std::string raw; - - // built by calling arg() or args() - std::string cmd; - - // exit code - DWORD code; - - exec(); - }; - - // log context - const context* cx_; - - // display name - std::string name_; - - // flags - process_flags flags_; - - // non copyable stuff - impl impl_; - - // to avoid having 30 member variables - io io_; - exec exec_; - - - // returns name() or the command line - // - std::string make_name() const; - - // the command line for the process itself - // - std::string make_cmd() const; - - // returns arguments given to cmd, `what` is the whole command line for - // the process itself; this includes flags to cmd like /U, but also stuff - // like chcp - // - std::wstring make_cmd_args(const std::string& what) const; - - // sets the raw command line to `make_cmd() | p.make_cmd()` - // - void pipe_into(const process& p); - - - // builds the command line, sets up redirections and and calls - // CreateProcess() - // - void do_run(const std::string& what); - - // deletes the external log file, if any - // - void delete_external_log_file(); - - // creates the job object - // - void create_job(); - - // sets up stdout/stderr/stdin redirection - // - handle_ptr redirect_stdout(STARTUPINFOW& si); - handle_ptr redirect_stderr(STARTUPINFOW& si); - handle_ptr redirect_stdin(STARTUPINFOW& si); - - // calls CreateProcess() with the given stuff - // - void create( - std::wstring cmd, std::wstring args, std::wstring cwd, STARTUPINFOW si); - - - // called regularly in join(), checks for termination or interruption, - // handles pipes - // - void on_timeout(bool& already_interrupted); - - // reads from stdin and stderr, `finish` must be true when the process has - // terminated - // - void read_pipes(bool finish); - - // reads from the given stream and pipe, `finish` must be true when the - // process has terminated - // - void read_pipe( - bool finish, stream& s, - async_pipe_stdout& pipe, context::reason r); - - // sends stuff to stdin, if any - // - void feed_stdin(); - - - // called when the process has terminated; checks exit code, logs stuff and - // bails out on errors - // - void on_completed(); - - // called from on_completed() when the process' exit code was successful - // - void on_process_successful(); - - // called from on_completed() when the process' exit code was a failure - // - void on_process_failed(); - - // interrupts the process if needed, returns true if interrupted - // - bool check_interrupted(); - - // forcefully kills the process and its children - // - void terminate(); - - - // if external_error_log() was called, dumps the contenf of the log file as - // errors - // - void dump_error_log_file() noexcept; - - // logs the content of stderr, used when the process failed before bailing - // out - // - void dump_stderr() noexcept; - - - // adds the given argument to the command line - // - // depending on the flags, the argument may be discarded; for example, if - // the argument is marked as arg_flags::log_quiet but the log level is - // trace, it is not included - // - void add_arg(const std::string& k, const std::string& v, arg_flags f); - - // argument conversions - // - static std::string arg_to_string(const char* s, arg_flags f); - static std::string arg_to_string(const std::string& s, arg_flags f); - static std::string arg_to_string(const fs::path& p, arg_flags f); - static std::string arg_to_string(const url& u, arg_flags f); - static std::string arg_to_string(int i, arg_flags f); -}; - - -MOB_ENUM_OPERATORS(process::process_flags); - -} // namespace +namespace mob { + + class url; + class async_pipe_stdout; + class async_pipe_stdin; + + class process { + public: + static constexpr DWORD wait_timeout = 50; + + // given in flags(), control process creation and termination + // + enum process_flags { + noflags = 0x00, + + // will not bail out on failure, used for optional processes + allow_failure = 0x01, + + // some processes just refuse to die when given sigint, like jom, so + // this just kills the process + terminate_on_interrupt = 0x02, + + // some processes output useless stuff even when they're successful, + // so to try to keep the amount of logs down, specifying this will + // discard the output of the process is successful + ignore_output_on_success = 0x04 + }; + + // used in arg(), controls how args are converted to string and whether + // they're tied to a specific log level + // + enum arg_flags { + noargflags = 0x00, + + // the argument should only be used when the given log level is active; + // a /q switch for quiet output would have the log_quiet flag, for + // example + log_debug = 0x01, + log_trace = 0x02, + log_dump = 0x04, + log_quiet = 0x08, + + // for arg(k, v), doesn't put a space between `k` and `v`; some programs + // are pretty strict with their arguments, like 7z working for `-opath` + // but not `-o path` + nospace = 0x10, + + // for arg(k, v) or arg(v), forces the value to be double-quoted (but + // not the key); note that fs::path and url objects are always quoted + // automatically + quote = 0x20, + + // converts backslashes to forward slashes for the given fs::path + // value; ignored for other types + forward_slashes = 0x40 + }; + + // used in stdout_flags() and stderr_flags(), controls what to do with the + // process' output + // + enum stream_flags { + // default, forwards the output directly to mob's logs, both file and + // console, depending on the configuration + forward_to_log = 1, + + // discards the output + bit_bucket, + + // does not log the output, keeps it in a string; can be retrieved in + // stdout_string() and stderr_string() + keep_in_string, + + // inherits stdout/stderr from this process; this is only used by + // processes started early in mob before things are set up, like when + // calling vswhere, so it just dumps stuff to the console + inherit + }; + + // used to filter the output of a process, because some programs really suck + // at being quiet and will output crap that's not needed but cannot be + // inhibited + // + // a lambda is given to stdout_filter() and stderr_filter() that will + // receive a `filter` object, allowing to change the log level of a line + // or just discard it completely + // + struct filter { + // a line from the process' output + std::string_view line; + + // the base reason for the stream, either context::std_out or + // context::std_err, can be modified by the filter + context::reason r; + + // the base level for the stream, defaults to stdout=trace and + // stderr=error, or whatever was given to stdout_level()/stderr_level() + // below; can be modified by the filter + context::level lv; + + // whether to discard this log line; can be set to true by the filter + bool discard; + + filter(std::string_view line, context::reason r, context::level lv); + + // this struct is only used by filter_fun callback, where copying a + // filter wouldn't make any sense + filter(const filter&) = delete; + filter& operator=(const filter&) = delete; + }; + + using filter_fun = std::function; + + // empty process + // + process(); + + // joins + // + ~process(); + + // anchors, all defaults + process(process&&); + process(const process&); + process& operator=(const process&); + process& operator=(process&&); + + // creates a process from the given command line instead of using the + // various binary(), arg(), etc. + // + static process raw(const context& cx, const std::string& cmd); + + // used by pipe(...) below to finish recursion + // + static process pipe(process p) { return p; } + + // constructs a process object by concatenating the command line of the + // given processes with " | " in between; this can only be used with fully + // set up processes, their command line is extracted immediately + // + // it's basically only used by 7z to pipe tar into it and is not a very + // generic solution + // + template + static process pipe(const process& p1, const process& p2, Processes&&... ps) + { + auto r = p1; + r.pipe_into(p2); + pipe(r, std::forward(ps)...); + return r; + } + + // sets the context of this process, used for all logging, bailing out, + // filesystem operations, etc.; the process runners use this before spawning + // the process + // + process& set_context(const context* cx); + + // display name for the process, used in logging; if not set, returns the + // filename without extension of the binary, which may be an empty string + // + process& name(const std::string& name); + std::string name() const; + + // path to the executable + // + process& binary(const fs::path& p); + const fs::path& binary() const; + + // working directory + // + process& cwd(const fs::path& p); + const fs::path& cwd() const; + + // process flags + // + process& flags(process_flags f); + process_flags flags() const; + + // sets flags for stdout/stderr + // + process& stdout_flags(stream_flags s); + process& stderr_flags(stream_flags s); + + // sets the default log level for stdout/stderr; if not given, stdout is + // trace, stderr is error + // + process& stdout_level(context::level lv); + process& stderr_level(context::level lv); + + // sets a callback to filter the output + // + process& stdout_filter(filter_fun f); + process& stderr_filter(filter_fun f); + + // sets the encoding of stdout/stderr, defaults to dont_know, which doesn't + // do any conversion + // + process& stdout_encoding(encodings e); + process& stderr_encoding(encodings e); + + // if the string is not empty, the process' stdin will be redirected and + // given the string + // + process& stdin_string(std::string s); + + // if not -1, `chcp cp` will be executed before spawning the process; note + // that processes are started in their own cmd instance, so this won't leak + // + process& chcp(int cp); + + // passes /U to cmd when spawning the process + // + // this is basically only used when getting the environment variables after + // calling vcvars to force `set` to output in utf16, because it normally + // outputs in the current codepage + // + // also forces stdout_encoding() and stderr_encoding() to utf16 + // + process& cmd_unicode(bool b); + + // some processes output to an external log file instead of to + // stdout/stderr, such as boost's boostrap.bat + // + // if the process failed (returned an exit code considered failure), the + // content of the file will be dumped to the logs as errors; if the process + // succeeded, the file is ignored + // + // the file is always deleted before starting the process + // + process& external_error_log(const fs::path& p); + + // the default success exit code is 0, this can be used to override it; any + // exit code found in the set is considered success + // + // basically only used by transifex `init`, which exits with 2 when the + // directory already has a .tx folder + // + process& success_exit_codes(const std::set& v); + + // adds an argument to the command line, see comment on top for conversions + // + template >> + process& arg(const T& value, arg_flags f = noargflags) + { + add_arg("", arg_to_string(value, f), f); + return *this; + } + + // adds a name=value argument to the command line, see comment on top for + // conversions + // + template >> + process& arg(const std::string& name, const T& value, arg_flags f = noargflags) + { + add_arg(name, arg_to_string(value, f), f); + return *this; + } + + // adds every name=value pair to the command line, see comment on top for + // conversions + // + template