Add a metadata collection mode to the STF back end.

backends/p4tools/modules/testgen/cmake/TestUtils.cmake

@@ -2,9 +2,6 @@
 # from the testsuite patterns by calling p4c_find_test_names then pass the list
 # to p4tools_add_test_list
 # Arguments:
-#   - template_file The path to the extension/target file that implements
-#     the logic of the target test. "p4tools_add_tests" includes this file and then
-#     executes the logic defined "p4tools_add_test_with_args".
 #   - tag is a label for the set of test suite where this test belongs
 #     (for example, p4ctest)
 #   - driver is the script that is used to run the tests and compare the results
@@ -21,7 +18,7 @@
 #
 macro(p4tools_add_tests)
   set(options "")
-  set(oneValueArgs TEMPLATE_FILE TAG DRIVER)
+  set(oneValueArgs TAG DRIVER)
   set(multiValueArgs TESTSUITES)
   cmake_parse_arguments(
     TOOLS_TESTS "${options}" "${oneValueArgs}"
@@ -31,7 +28,6 @@ macro(p4tools_add_tests)
   set(__tests "")
   p4c_find_test_names("${TOOLS_TESTS_TESTSUITES}" __tests)
 
-  include(${TOOLS_TESTS_TEMPLATE_FILE})
   list(LENGTH __tests __nTests)
   foreach(t ${__tests})
     get_filename_component(p4name ${t} NAME)

backends/p4tools/modules/testgen/targets/bmv2/CMakeLists.txt

@@ -9,6 +9,7 @@ set(
   TESTGEN_SOURCES
   ${TESTGEN_SOURCES}
   ${CMAKE_CURRENT_SOURCE_DIR}/backend/protobuf/protobuf.cpp
+  ${CMAKE_CURRENT_SOURCE_DIR}/backend/metadata/metadata.cpp
   ${CMAKE_CURRENT_SOURCE_DIR}/backend/ptf/ptf.cpp
   ${CMAKE_CURRENT_SOURCE_DIR}/backend/stf/stf.cpp
   ${CMAKE_CURRENT_SOURCE_DIR}/bmv2.cpp

backends/p4tools/modules/testgen/targets/bmv2/backend/metadata/metadata.cpp

@@ -0,0 +1,166 @@
+#include "backends/p4tools/modules/testgen/targets/bmv2/backend/metadata/metadata.h"
+
+#include <iomanip>
+#include <map>
+#include <stdexcept>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include <boost/filesystem.hpp>
+#include <boost/format.hpp>
+#include <boost/none.hpp>
+#include <boost/variant/apply_visitor.hpp>
+#include <boost/variant/static_visitor.hpp>
+#include <inja/inja.hpp>
+
+#include "backends/p4tools/common/lib/format_int.h"
+#include "backends/p4tools/common/lib/trace_events.h"
+#include "backends/p4tools/common/lib/util.h"
+#include "gsl/gsl-lite.hpp"
+#include "ir/declaration.h"
+#include "ir/ir.h"
+#include "ir/vector.h"
+#include "lib/big_int_util.h"
+#include "lib/error.h"
+#include "lib/error_catalog.h"
+#include "lib/exceptions.h"
+#include "lib/log.h"
+#include "lib/null.h"
+#include "nlohmann/json.hpp"
+
+#include "backends/p4tools/modules/testgen/lib/tf.h"
+#include "backends/p4tools/modules/testgen/targets/bmv2/test_spec.h"
+
+namespace P4Tools::P4Testgen::Bmv2 {
+
+Metadata::Metadata(cstring testName, boost::optional<unsigned int> seed = boost::none)
+    : TF(testName, seed) {}
+
+std::vector<std::pair<size_t, size_t>> Metadata::getIgnoreMasks(const IR::Constant *mask) {
+    std::vector<std::pair<size_t, size_t>> ignoreMasks;
+    if (mask == nullptr) {
+        return ignoreMasks;
+    }
+    auto maskBinStr = formatBinExpr(mask, false, true, false);
+    int countZeroes = 0;
+    size_t offset = 0;
+    for (; offset < maskBinStr.size(); ++offset) {
+        if (maskBinStr.at(offset) == '0') {
+            countZeroes++;
+        } else {
+            if (countZeroes > 0) {
+                ignoreMasks.emplace_back(offset - countZeroes, countZeroes);
+                countZeroes = 0;
+            }
+        }
+    }
+    if (countZeroes > 0) {
+        ignoreMasks.emplace_back(offset - countZeroes, countZeroes);
+    }
+    return ignoreMasks;
+}
+
+inja::json Metadata::getSend(const TestSpec *testSpec) {
+    const auto *iPacket = testSpec->getIngressPacket();
+    const auto *payload = iPacket->getEvaluatedPayload();
+    inja::json sendJson;
+    sendJson["ig_port"] = iPacket->getPort();
+    auto dataStr = formatHexExpr(payload);
+    sendJson["pkt"] = dataStr;
+    sendJson["pkt_size"] = payload->type->width_bits();
+    return sendJson;
+}
+
+inja::json Metadata::getVerify(const TestSpec *testSpec) {
+    inja::json verifyData = inja::json::object();
+    if (testSpec->getEgressPacket() != boost::none) {
+        const auto &packet = **testSpec->getEgressPacket();
+        verifyData["eg_port"] = packet.getPort();
+        const auto *payload = packet.getEvaluatedPayload();
+        const auto *payloadMask = packet.getEvaluatedPayloadMask();
+        verifyData["ignore_mask"] = formatHexExpr(payloadMask);
+        verifyData["exp_pkt"] = formatHexExpr(payload);
+    }
+    return verifyData;
+}
+
+std::string Metadata::getTestCaseTemplate() {
+    static std::string TEST_CASE(
+        R"""(# A P4TestGen-generated test case for {{test_name}}.p4
+# p4testgen seed: {{ default(seed, "none") }}
+# Date generated: {{timestamp}}
+## if length(selected_branches) > 0
+# {{selected_branches}}
+## endif
+# Current statement coverage: {{coverage}}
+
+## for trace_item in trace
+# "{{trace_item}}"
+## endfor
+
+input_packet: {{send.pkt}}
+input_port: {{send.ig_port}}
+
+## if verify
+output_packet: "{{verify.exp_pkt}}"
+output_port: {{verify.eg_port}}
+output_packet_mask: "{{verify.ignore_mask}}"
+## endif
+
+
+## for metadata_field in metadata_fields
+Metadata: {{metadata_field.name}}:{{metadata_field.value}}
+## endfor
+
+)""");
+    return TEST_CASE;
+}
+
+void Metadata::emitTestcase(const TestSpec *testSpec, cstring selectedBranches, size_t testId,
+                            const std::string &testCase, float currentCoverage) {
+    inja::json dataJson;
+    if (selectedBranches != nullptr) {
+        dataJson["selected_branches"] = selectedBranches.c_str();
+    }
+    boost::filesystem::path testFile(testName + ".proto");
+    cstring testNameOnly(testFile.stem().c_str());
+    if (seed) {
+        dataJson["seed"] = *seed;
+    }
+    dataJson["test_name"] = testNameOnly.c_str();
+    dataJson["test_id"] = testId + 1;
+    dataJson["trace"] = getTrace(testSpec);
+    dataJson["send"] = getSend(testSpec);
+    dataJson["verify"] = getVerify(testSpec);
+    dataJson["timestamp"] = Utils::getTimeStamp();
+    std::stringstream coverageStr;
+    coverageStr << std::setprecision(2) << currentCoverage;
+    dataJson["coverage"] = coverageStr.str();
+    dataJson["metadata_fields"] = inja::json::array();
+    const auto *metadataCollection =
+        testSpec->getTestObject("metadata_collection", "metadata_collection", true)
+            ->checkedTo<MetadataCollection>();
+    for (auto const &metadataField : metadataCollection->getMetadataFields()) {
+        inja::json jsonField;
+        jsonField["name"] = metadataField.first;
+        jsonField["value"] = formatHexExpr(metadataField.second);
+        dataJson["metadata_fields"].push_back(jsonField);
+    }
+
+    LOG5("Metadata back end: emitting testcase:" << std::setw(4) << dataJson);
+
+    inja::render_to(metadataFile, testCase, dataJson);
+    metadataFile.flush();
+}
+
+void Metadata::outputTest(const TestSpec *testSpec, cstring selectedBranches, size_t testIdx,
+                          float currentCoverage) {
+    auto incrementedTestName = testName + "_" + std::to_string(testIdx);
+
+    metadataFile = std::ofstream(incrementedTestName + ".metadata");
+    std::string testCase = getTestCaseTemplate();
+    emitTestcase(testSpec, selectedBranches, testIdx, testCase, currentCoverage);
+}
+
+}  // namespace P4Tools::P4Testgen::Bmv2

backends/p4tools/modules/testgen/targets/bmv2/backend/metadata/metadata.h

@@ -0,0 +1,73 @@
+#ifndef BACKENDS_P4TOOLS_MODULES_TESTGEN_TARGETS_BMV2_BACKEND_METADATA_METADATA_H_
+#define BACKENDS_P4TOOLS_MODULES_TESTGEN_TARGETS_BMV2_BACKEND_METADATA_METADATA_H_
+
+#include <cstddef>
+#include <fstream>
+#include <map>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include <boost/optional/optional.hpp>
+#include <inja/inja.hpp>
+
+#include "ir/ir.h"
+#include "lib/cstring.h"
+
+#include "backends/p4tools/modules/testgen/lib/test_spec.h"
+#include "backends/p4tools/modules/testgen/lib/tf.h"
+
+namespace P4Tools::P4Testgen::Bmv2 {
+
+/// Extracts information from the @testSpec to emit a Metadata test case.
+class Metadata : public TF {
+    /// The output file.
+    std::ofstream metadataFile;
+
+ public:
+    virtual ~Metadata() = default;
+
+    Metadata(const Metadata &) = delete;
+
+    Metadata(Metadata &&) = delete;
+
+    Metadata &operator=(const Metadata &) = delete;
+
+    Metadata &operator=(Metadata &&) = delete;
+
+    Metadata(cstring testName, boost::optional<unsigned int> seed);
+
+    /// Produce a Metadata test.
+    void outputTest(const TestSpec *spec, cstring selectedBranches, size_t testIdx,
+                    float currentCoverage) override;
+
+ private:
+    /// Emits the test preamble. This is only done once for all generated tests.
+    /// For the Metadata back end this is the "p4testgen.proto" file.
+    void emitPreamble(const std::string &preamble);
+
+    /// Emits a test case.
+    /// @param testId specifies the test name.
+    /// @param selectedBranches enumerates the choices the interpreter made for this path.
+    /// @param currentCoverage contains statistics  about the current coverage of this test and its
+    /// preceding tests.
+    void emitTestcase(const TestSpec *testSpec, cstring selectedBranches, size_t testId,
+                      const std::string &testCase, float currentCoverage);
+
+    /// @returns the inja test case template as a string.
+    static std::string getTestCaseTemplate();
+
+    /// Converts the input packet and port into Inja format.
+    static inja::json getSend(const TestSpec *testSpec);
+
+    /// Converts the output packet, port, and mask into Inja format.
+    static inja::json getVerify(const TestSpec *testSpec);
+
+    /// Helper function for @getVerify. Matches the mask value against the input packet value and
+    /// generates the appropriate ignore ranges.
+    static std::vector<std::pair<size_t, size_t>> getIgnoreMasks(const IR::Constant *mask);
+};
+
+}  // namespace P4Tools::P4Testgen::Bmv2
+
+#endif /* BACKENDS_P4TOOLS_MODULES_TESTGEN_TARGETS_BMV2_BACKEND_METADATA_METADATA_H_ */

backends/p4tools/modules/testgen/targets/bmv2/program_info.h

@@ -13,11 +13,7 @@
 #include "backends/p4tools/modules/testgen/core/program_info.h"
 #include "backends/p4tools/modules/testgen/lib/continuation.h"
 
-namespace P4Tools {
-
-namespace P4Testgen {
-
-namespace Bmv2 {
+namespace P4Tools::P4Testgen::Bmv2 {
 
 class BMv2_V1ModelProgramInfo : public ProgramInfo {
  private:
@@ -69,10 +65,6 @@ class BMv2_V1ModelProgramInfo : public ProgramInfo {
                                         size_t paramIndex, cstring paramLabel) const override;
 };
 
-}  // namespace Bmv2
-
-}  // namespace P4Testgen
-
-}  // namespace P4Tools
+}  // namespace P4Tools::P4Testgen::Bmv2
 
 #endif /* BACKENDS_P4TOOLS_MODULES_TESTGEN_TARGETS_BMV2_PROGRAM_INFO_H_ */

backends/p4tools/modules/testgen/targets/bmv2/table_stepper.cpp

@@ -399,13 +399,15 @@ void BMv2_V1ModelTableStepper::evalTargetTable(
         auto testBackend = TestgenOptions::get().testBackend;
         if (testBackend == "STF" && !properties.defaultIsImmutable) {
             setTableDefaultEntries(tableActionList);
-        } else {
+            return;
+        }
+        if (!properties.defaultIsImmutable) {
             ::warning(
-                "Overriding default actions not supported for test back end %1%. Choosing default "
-                "action",
-                testBackend);
-            addDefaultAction(boost::none);
+                "Table %1%: Overriding default actions not supported for test back end %2%. "
+                "Choosing default action",
+                properties.tableName, testBackend);
         }
+        addDefaultAction(boost::none);
         return;
     }
 

backends/p4tools/modules/testgen/targets/bmv2/test/AssumeAssertTests.cmake

@@ -1,35 +1,35 @@
 p4tools_add_test_with_args(
   P4TEST "backends/p4tools/modules/testgen/targets/bmv2/test/p4-programs/assert_assume_tests/bmv2_testgen_assert_1.p4"
-  TAG "testgen-p4c-bmv2-assert" ALIAS "bmv2_testgen_assert_1.p4" DRIVER ${P4TESTGEN_DRIVER} TEMPLATE_FILE ${TEMPLATE_FILE}
+  TAG "testgen-p4c-bmv2-assert" ALIAS "bmv2_testgen_assert_1.p4" DRIVER ${P4TESTGEN_DRIVER}
   TARGET "bmv2" ARCH "v1model" USE_ASSERT_MODE DISABLE_ASSUME_MODE TEST_ARGS "-I${P4C_BINARY_DIR}/p4include --test-backend STF --print-traces --max-tests 10 "
 )
 
 p4tools_add_test_with_args(
   P4TEST "backends/p4tools/modules/testgen/targets/bmv2/test/p4-programs/assert_assume_tests/bmv2_testgen_assert_2.p4"
-  TAG "testgen-p4c-bmv2-assert" ALIAS "bmv2_testgen_assert_2.p4" DRIVER ${P4TESTGEN_DRIVER} TEMPLATE_FILE ${TEMPLATE_FILE}
+  TAG "testgen-p4c-bmv2-assert" ALIAS "bmv2_testgen_assert_2.p4" DRIVER ${P4TESTGEN_DRIVER}
   TARGET "bmv2" ARCH "v1model" USE_ASSERT_MODE DISABLE_ASSUME_MODE CHECK_EMPTY TEST_ARGS "-I${P4C_BINARY_DIR}/p4include --test-backend STF --print-traces --max-tests 10 "
 )
 
 p4tools_add_test_with_args(
   P4TEST "backends/p4tools/modules/testgen/targets/bmv2/test/p4-programs/assert_assume_tests/bmv2_testgen_assert_assume_1.p4"
-  TAG "testgen-p4c-bmv2-assert" ALIAS "bmv2_testgen_assert_assume_1.p4" DRIVER ${P4TESTGEN_DRIVER} TEMPLATE_FILE ${TEMPLATE_FILE}
+  TAG "testgen-p4c-bmv2-assert" ALIAS "bmv2_testgen_assert_assume_1.p4" DRIVER ${P4TESTGEN_DRIVER}
   TARGET "bmv2" ARCH "v1model" USE_ASSERT_MODE USE_ASSUME_MODE CHECK_EMPTY TEST_ARGS "-I${P4C_BINARY_DIR}/p4include --test-backend STF --print-traces --max-tests 10 "
 )
 
 p4tools_add_test_with_args(
   P4TEST "backends/p4tools/modules/testgen/targets/bmv2/test/p4-programs/assert_assume_tests/bmv2_testgen_assert_assume_1.p4"
-  TAG "testgen-p4c-bmv2-assert" ALIAS "bmv2_testgen_assert_assume_1_neg.p4" DRIVER ${P4TESTGEN_DRIVER} TEMPLATE_FILE ${TEMPLATE_FILE}
+  TAG "testgen-p4c-bmv2-assert" ALIAS "bmv2_testgen_assert_assume_1_neg.p4" DRIVER ${P4TESTGEN_DRIVER}
   TARGET "bmv2" ARCH "v1model" USE_ASSERT_MODE DISABLE_ASSUME_MODE TEST_ARGS "-I${P4C_BINARY_DIR}/p4include --test-backend STF --print-traces --max-tests 10 "
 )
 
 p4tools_add_test_with_args(
   P4TEST "backends/p4tools/modules/testgen/targets/bmv2/test/p4-programs/assert_assume_tests/bmv2_testgen_assert_assume_2.p4"
-  TAG "testgen-p4c-bmv2-assert" ALIAS "bmv2_testgen_assert_assume_2.p4" DRIVER ${P4TESTGEN_DRIVER} TEMPLATE_FILE ${TEMPLATE_FILE}
+  TAG "testgen-p4c-bmv2-assert" ALIAS "bmv2_testgen_assert_assume_2.p4" DRIVER ${P4TESTGEN_DRIVER}
   TARGET "bmv2" ARCH "v1model" USE_ASSERT_MODE CHECK_EMPTY TEST_ARGS "-I${P4C_BINARY_DIR}/p4include --test-backend STF --print-traces --max-tests 10 "
 )
 
 p4tools_add_test_with_args(
   P4TEST "backends/p4tools/modules/testgen/targets/bmv2/test/p4-programs/assert_assume_tests/bmv2_testgen_assert_assume_3.p4"
-  TAG "testgen-p4c-bmv2-assert" ALIAS "bmv2_testgen_assert_assume_3.p4" DRIVER ${P4TESTGEN_DRIVER} TEMPLATE_FILE ${TEMPLATE_FILE}
+  TAG "testgen-p4c-bmv2-assert" ALIAS "bmv2_testgen_assert_assume_3.p4" DRIVER ${P4TESTGEN_DRIVER}
   TARGET "bmv2" ARCH "v1model" USE_ASSERT_MODE TEST_ARGS "-I${P4C_BINARY_DIR}/p4include --test-backend STF --print-traces --max-tests 10 "
 )