Test features in ci

include/vcpkg/binarycaching.h

@@ -30,6 +30,7 @@ namespace vcpkg
         void mark_unavailable(const IBinaryProvider* sender);
         void mark_available(const IBinaryProvider* sender) noexcept;
         void mark_restored() noexcept;
+        void mark_unrestored() noexcept;
 
     private:
         CacheStatusState m_status = CacheStatusState::unknown;
@@ -142,6 +143,8 @@ namespace vcpkg
         /// Returns a vector where each index corresponds to the matching index in `actions`.
         std::vector<CacheAvailability> precheck(View<InstallPlanAction> actions);
 
+        void clear_cache();
+
     private:
         std::unordered_map<std::string, CacheStatus> m_status;
         std::vector<std::unique_ptr<IBinaryProvider>> m_providers;

include/vcpkg/commands.setinstalled.h

@@ -8,6 +8,14 @@
 namespace vcpkg::Commands::SetInstalled
 {
     extern const CommandStructure COMMAND_STRUCTURE;
+
+    /**
+     * @brief adjust_action_plan_to_status_db creates an action plan that installs only the requested ports
+     * @param action_plan An action plan that was created with an empty status db
+     * @param status_db The status db of the installed folder
+     */
+    void adjust_action_plan_to_status_db(ActionPlan& action_plan, const StatusParagraphs& status_db);
+
     void perform_and_exit_ex(const VcpkgCmdArguments& args,
                              const VcpkgPaths& paths,
                              const PathsPortFileProvider& provider,

include/vcpkg/install.h

@@ -9,6 +9,7 @@
 #include <vcpkg/binaryparagraph.h>
 #include <vcpkg/build.h>
 #include <vcpkg/dependencies.h>
+#include <vcpkg/remove.h>
 #include <vcpkg/vcpkgcmdarguments.h>
 #include <vcpkg/vcpkgpaths.h>
 
@@ -32,6 +33,10 @@ namespace vcpkg
 
         const BinaryParagraph* get_binary_paragraph() const;
         const PackageSpec& get_spec() const { return m_spec; }
+        Optional<const std::string&> get_abi() const
+        {
+            return m_install_action ? m_install_action->package_abi() : nullopt;
+        }
         bool is_user_requested_install() const;
         Optional<ExtendedBuildResult> build_result;
         vcpkg::ElapsedTime timing;
@@ -115,7 +120,8 @@ namespace vcpkg
                                StatusParagraphs& status_db,
                                BinaryCache& binary_cache,
                                const IBuildLogsRecorder& build_logs_recorder,
-                               const CMakeVars::CMakeVarProvider& var_provider);
+                               const CMakeVars::CMakeVarProvider& var_provider,
+                               Remove::Purge purge = Remove::Purge::YES);
 
         void perform_and_exit(const VcpkgCmdArguments& args,
                               const VcpkgPaths& paths,

include/vcpkg/packagespec.h

@@ -114,6 +114,9 @@ namespace vcpkg
         {
         }
 
+        std::string to_string() const;
+        void to_string(std::string& s) const;
+
         /// Splats into individual FeatureSpec's
         void expand_fspecs_to(std::vector<FeatureSpec>& oFut) const;
 

src/vcpkg/binarycaching.cpp

@@ -1458,6 +1458,14 @@ namespace vcpkg
         return results;
     }
 
+    void BinaryCache::clear_cache()
+    {
+        for (auto& entry : m_status)
+        {
+            entry.second.mark_unrestored();
+        }
+    }
+
     bool CacheStatus::should_attempt_precheck(const IBinaryProvider* sender) const noexcept
     {
         switch (m_status)
@@ -1534,6 +1542,8 @@ namespace vcpkg
         }
     }
 
+    void CacheStatus::mark_unrestored() noexcept { m_status = CacheStatusState::available; }
+
     const IBinaryProvider* CacheStatus::get_available_provider() const noexcept
     {
         switch (m_status)

src/vcpkg/commands.ci.cpp

@@ -15,6 +15,7 @@
 #include <vcpkg/ci-baseline.h>
 #include <vcpkg/cmakevars.h>
 #include <vcpkg/commands.ci.h>
+#include <vcpkg/commands.setinstalled.h>
 #include <vcpkg/dependencies.h>
 #include <vcpkg/globalstate.h>
 #include <vcpkg/help.h>
@@ -101,26 +102,38 @@ namespace vcpkg::Commands::CI
     static constexpr StringLiteral OPTION_OUTPUT_HASHES = "output-hashes";
     static constexpr StringLiteral OPTION_PARENT_HASHES = "parent-hashes";
     static constexpr StringLiteral OPTION_SKIPPED_CASCADE_COUNT = "x-skipped-cascade-count";
+    static constexpr StringLiteral OPTION_TEST_FEATURE_CORE = "test-feature-core";
+    static constexpr StringLiteral OPTION_TEST_FEATURES_COMBINED = "test-features-combined";
+    static constexpr StringLiteral OPTION_TEST_FEATURES_SEPERATLY = "test-features-seperatly";
+    static constexpr StringLiteral OPTION_RUN_FEATURE_TESTS_PORTS = "run-feature-tests-for-ports";
+    static constexpr StringLiteral OPTION_RUN_FEATURE_TESTS_ALL_PORTS = "run-feature-tests-for-all-ports";
+
+    static constexpr std::array<CommandSetting, 9> CI_SETTINGS = {{
+        {OPTION_EXCLUDE, "Comma separated list of ports to skip"},
+        {OPTION_HOST_EXCLUDE, "Comma separated list of ports to skip for the host triplet"},
+        {OPTION_XUNIT, "File to output results in XUnit format (internal)"},
+        {OPTION_CI_BASELINE, "Path to the ci.baseline.txt file. Used to skip ports and detect regressions."},
+        {OPTION_FAILURE_LOGS, "Directory to which failure logs will be copied"},
+        {OPTION_OUTPUT_HASHES, "File to output all determined package hashes"},
+        {OPTION_PARENT_HASHES,
+         "File to read package hashes for a parent CI state, to reduce the set of changed packages"},
+        {OPTION_SKIPPED_CASCADE_COUNT,
+         "Asserts that the number of --exclude and supports skips exactly equal this number"},
+        {OPTION_RUN_FEATURE_TESTS_PORTS,
+         "A comma seperated list of ports for which the specified feature tests should be run"},
+    }};
 
-    static constexpr std::array<CommandSetting, 8> CI_SETTINGS = {
-        {{OPTION_EXCLUDE, "Comma separated list of ports to skip"},
-         {OPTION_HOST_EXCLUDE, "Comma separated list of ports to skip for the host triplet"},
-         {OPTION_XUNIT, "File to output results in XUnit format (internal)"},
-         {OPTION_CI_BASELINE, "Path to the ci.baseline.txt file. Used to skip ports and detect regressions."},
-         {OPTION_FAILURE_LOGS, "Directory to which failure logs will be copied"},
-         {OPTION_OUTPUT_HASHES, "File to output all determined package hashes"},
-         {OPTION_PARENT_HASHES,
-          "File to read package hashes for a parent CI state, to reduce the set of changed packages"},
-         {OPTION_SKIPPED_CASCADE_COUNT,
-          "Asserts that the number of --exclude and supports skips exactly equal this number"}}};
-
-    static constexpr std::array<CommandSwitch, 5> CI_SWITCHES = {{
+    static constexpr std::array<CommandSwitch, 9> CI_SWITCHES = {{
         {OPTION_DRY_RUN, "Print out plan without execution"},
         {OPTION_RANDOMIZE, "Randomize the install order"},
         {OPTION_ALLOW_UNEXPECTED_PASSING,
          "Indicates that 'Passing, remove from fail list' results should not be emitted."},
         {OPTION_SKIP_FAILURES, "Indicates that ports marked `=fail` in ci.baseline.txt should be skipped."},
         {OPTION_XUNIT_ALL, "Report also unchanged ports to the XUnit output (internal)"},
+        {OPTION_RUN_FEATURE_TESTS_ALL_PORTS, "Runs the specified tests for all ports"},
+        {OPTION_TEST_FEATURE_CORE, "Tests the 'core' feature for every specified port"},
+        {OPTION_TEST_FEATURES_SEPERATLY, "Tests every feature of a port seperatly for every specified port"},
+        {OPTION_TEST_FEATURES_COMBINED, "Tests the combination of every feature of a port for every specified port"},
     }};
 
     const CommandStructure COMMAND_STRUCTURE = {
@@ -270,6 +283,33 @@ namespace vcpkg::Commands::CI
         });
     }
 
+    static auto get_ports_to_test_with_features(const ParsedArguments& args, ActionPlan& action_plan)
+    {
+        const auto all_ports = Util::Sets::contains(args.switches, OPTION_RUN_FEATURE_TESTS_ALL_PORTS);
+        std::vector<std::string> ports;
+        auto iter = args.settings.find(OPTION_RUN_FEATURE_TESTS_PORTS);
+        if (iter != args.settings.end())
+        {
+            ports = Strings::split(iter->second, ',');
+        }
+        std::vector<SourceControlFile*> ports_to_test;
+        if (all_ports || !ports.empty())
+        {
+            for (const auto& action : action_plan.install_actions)
+            {
+                const auto& source_control_file =
+                    action.source_control_file_and_location.value_or_exit(VCPKG_LINE_INFO).source_control_file;
+
+                if (all_ports || Util::Vectors::contains(ports, source_control_file->core_paragraph->name))
+                {
+                    ports_to_test.push_back(action.source_control_file_and_location.value_or_exit(VCPKG_LINE_INFO)
+                                                .source_control_file.get());
+                }
+            }
+        }
+        return ports_to_test;
+    }
+
     static void parse_exclusions(const std::map<std::string, std::string, std::less<>>& settings,
                                  StringLiteral opt,
                                  Triplet triplet,
@@ -336,6 +376,20 @@ namespace vcpkg::Commands::CI
         const ParsedArguments options = args.parse_arguments(COMMAND_STRUCTURE);
         const auto& settings = options.settings;
 
+        const auto test_feature_core = Util::Sets::contains(options.switches, OPTION_TEST_FEATURE_CORE);
+        const auto test_features_combined = Util::Sets::contains(options.switches, OPTION_TEST_FEATURES_COMBINED);
+        const auto test_features_seperatly = Util::Sets::contains(options.switches, OPTION_TEST_FEATURES_SEPERATLY);
+
+        const auto run_tests_all_ports = Util::Sets::contains(options.switches, OPTION_RUN_FEATURE_TESTS_ALL_PORTS);
+        const auto run_tests_ports_list = Util::Sets::contains(options.settings, OPTION_RUN_FEATURE_TESTS_PORTS);
+        {
+            const auto tests_selected = test_feature_core || test_features_combined || test_features_seperatly;
+            const auto ports_selected = run_tests_all_ports || run_tests_ports_list;
+            Checks::check_exit(VCPKG_LINE_INFO,
+                               (tests_selected && ports_selected) || (!tests_selected && !ports_selected),
+                               "You specify a flag to test features, but not which port should be checked");
+        }
+
         BinaryCache binary_cache{args, paths};
 
         ExclusionsMap exclusions_map;
@@ -473,7 +527,6 @@ namespace vcpkg::Commands::CI
                 return abi->string(VCPKG_LINE_INFO).to_string();
             });
         }
-
         reduce_action_plan(action_plan, split_specs->known, parent_hashes);
 
         msg::println(msgElapsedTimeForChecks, msg::elapsed = timer.elapsed());
@@ -494,6 +547,138 @@ namespace vcpkg::Commands::CI
         else
         {
             StatusParagraphs status_db = database_load_check(paths.get_filesystem(), paths.installed());
+
+            // test port features
+            std::unordered_set<std::string> known_failures;
+            for (const auto port : get_ports_to_test_with_features(options, action_plan))
+            {
+                PackageSpec package_spec(port->core_paragraph->name, target_triplet);
+                var_provider.load_dep_info_vars(Span<PackageSpec>(&package_spec, 1), host_triplet);
+                const auto dep_info_vars = var_provider.get_dep_info_vars(package_spec).value_or_exit(VCPKG_LINE_INFO);
+                std::vector<FullPackageSpec> specs_to_test;
+                if (test_feature_core)
+                {
+                    specs_to_test.emplace_back(package_spec, InternalFeatureSet{{"core"}});
+                }
+                InternalFeatureSet all_features{{"core"}};
+                for (const auto& feature : port->feature_paragraphs)
+                {
+                    if (feature->supports_expression.evaluate(dep_info_vars))
+                    {
+                        all_features.push_back(feature->name);
+                        if (test_features_seperatly)
+                        {
+                            specs_to_test.emplace_back(package_spec, InternalFeatureSet{{"core", feature->name}});
+                        }
+                    }
+                }
+                if (test_features_combined && all_features.size() > (test_features_seperatly ? size_t{2} : size_t{1}))
+                {
+                    specs_to_test.emplace_back(package_spec, all_features);
+                }
+
+                for (auto& spec : specs_to_test)
+                {
+                    auto install_plan = create_feature_install_plan(provider,
+                                                                    var_provider,
+                                                                    Span<FullPackageSpec>(&spec, 1),
+                                                                    {},
+                                                                    {host_triplet, UnsupportedPortAction::Warn});
+                    if (!install_plan.warnings.empty())
+                    {
+                        print2("Skipping testing of ",
+                               install_plan.install_actions.back().displayname(),
+                               " because of the following warnings: \n",
+                               Strings::join("\n", install_plan.warnings),
+                               '\n');
+                        continue;
+                    }
+
+                    compute_all_abis(paths, install_plan, var_provider, status_db);
+                    if (Util::any_of(install_plan.install_actions, [&known_failures](const auto& install_action) {
+                            return Util::Sets::contains(known_failures,
+                                                        install_action.package_abi().value_or_exit(VCPKG_LINE_INFO));
+                        }))
+                    {
+                        print2(spec, " will fail tested\n");
+                        continue;
+                    }
+
+                    // only install the absolute minimum
+                    SetInstalled::adjust_action_plan_to_status_db(install_plan, status_db);
+                    if (install_plan.install_actions.empty()) // already installed
+                    {
+                        continue;
+                    }
+
+                    if (binary_cache.precheck({&install_plan.install_actions.back(), 1}).front() ==
+                        CacheAvailability::available)
+                        continue;
+                    print2("Test feature ", spec, '\n');
+                    for (auto&& action : install_plan.install_actions)
+                    {
+                        action.build_options = backcompat_prohibiting_package_options;
+                    }
+                    const auto summary = Install::perform(args,
+                                                          install_plan,
+                                                          KeepGoing::YES,
+                                                          paths,
+                                                          status_db,
+                                                          binary_cache,
+                                                          null_build_logs_recorder(),
+                                                          var_provider);
+                    binary_cache.clear_cache();
+                    for (const auto& result : summary.results)
+                    {
+                        switch (result.build_result.value_or_exit(VCPKG_LINE_INFO).code)
+                        {
+                            case BuildResult::BUILD_FAILED:
+                            case BuildResult::POST_BUILD_CHECKS_FAILED:
+                            case BuildResult::FILE_CONFLICTS:
+                                known_failures.insert(result.get_abi().value_or_exit(VCPKG_LINE_INFO));
+                                break;
+                            default: break;
+                        }
+                    }
+                    switch (summary.results.back().build_result.value_or_exit(VCPKG_LINE_INFO).code)
+                    {
+                        case vcpkg::BuildResult::SUCCEEDED:
+                        case vcpkg::BuildResult::CASCADED_DUE_TO_MISSING_DEPENDENCIES:
+                            print2("Feature ", spec, " works \n");
+                            break;
+                        default: print2("Feature ", spec, " failed with \n");
+                    }
+                }
+            }
+
+            if (!known_failures.empty())
+            {
+                // remove known failures from the action_plan.install_actions
+                std::set<PackageSpec> known_failure_specs;
+                for (const auto& install_action : action_plan.install_actions)
+                {
+                    if (Util::Sets::contains(known_failures,
+                                             install_action.package_abi().value_or_exit(VCPKG_LINE_INFO)))
+                    {
+                        split_specs->known.emplace(install_action.spec, BuildResult::BUILD_FAILED);
+                        known_failure_specs.emplace(install_action.spec);
+                    }
+                    else if (Util::any_of(install_action.package_dependencies, [&](const PackageSpec& spec) {
+                                 return Util::Sets::contains(known_failure_specs, spec);
+                             }))
+                    {
+                        split_specs->known.emplace(install_action.spec,
+                                                   BuildResult::CASCADED_DUE_TO_MISSING_DEPENDENCIES);
+                        known_failure_specs.emplace(install_action.spec);
+                    }
+                }
+                Util::erase_remove_if(action_plan.install_actions,
+                                      [&known_failure_specs](const InstallPlanAction& action) {
+                                          return Util::Sets::contains(known_failure_specs, action.spec);
+                                      });
+            }
+            SetInstalled::adjust_action_plan_to_status_db(action_plan, status_db);
+
             auto summary = Install::perform(
                 args, action_plan, KeepGoing::YES, paths, status_db, binary_cache, build_logs_recorder, var_provider);