From c5b81b66cd06e49b14e29606247ec2fd5ad4d441 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Mon, 21 Aug 2023 10:26:01 +0200 Subject: [PATCH 1/5] Define Error::severity() --- liblangutil/Exceptions.h | 1 + 1 file changed, 1 insertion(+) diff --git a/liblangutil/Exceptions.h b/liblangutil/Exceptions.h index fc7286ed1c73..df3128fe2abb 100644 --- a/liblangutil/Exceptions.h +++ b/liblangutil/Exceptions.h @@ -206,6 +206,7 @@ class Error: virtual public util::Exception ErrorId errorId() const { return m_errorId; } Type type() const { return m_type; } + Severity severity() const { return errorSeverity(m_type); } SourceLocation const* sourceLocation() const noexcept; SecondarySourceLocation const* secondarySourceLocation() const noexcept; From b1ead4af941f337c8103abcb6868777fef7aca02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Mon, 21 Aug 2023 10:22:23 +0200 Subject: [PATCH 2/5] Order Error::Severity enum from most to least severe - Also reorder Error::Type to match initial values for some consistency --- liblangutil/Exceptions.h | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/liblangutil/Exceptions.h b/liblangutil/Exceptions.h index df3128fe2abb..65362df31d40 100644 --- a/liblangutil/Exceptions.h +++ b/liblangutil/Exceptions.h @@ -170,6 +170,8 @@ class Error: virtual public util::Exception public: enum class Type { + Info, + Warning, CodeGenerationError, DeclarationError, DocstringParsingError, @@ -185,15 +187,14 @@ class Error: virtual public util::Exception UnimplementedFeatureError, YulException, SMTLogicException, - Warning, - Info }; enum class Severity { - Error, + // NOTE: We rely on these being ordered from least to most severe. + Info, Warning, - Info + Error, }; Error( From 73b9077ab08860ccbe57f2fcf474446b723d636f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Mon, 21 Aug 2023 10:13:43 +0200 Subject: [PATCH 3/5] SyntaxTest: Default-initialize boolean fields without showing the value - These get re-initialized in constructor anyway. The only purpose if initializing here is our convention to always initialize primitive types at declaration time. We don't want to have to repeat the defaults though. --- test/libsolidity/SyntaxTest.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/libsolidity/SyntaxTest.h b/test/libsolidity/SyntaxTest.h index f4ca6d9bc955..f9fd65319b46 100644 --- a/test/libsolidity/SyntaxTest.h +++ b/test/libsolidity/SyntaxTest.h @@ -48,7 +48,7 @@ class SyntaxTest: public AnalysisFramework, public solidity::test::CommonSyntaxT void parseAndAnalyze() override; virtual void filterObtainedErrors(); - bool m_optimiseYul = true; + bool m_optimiseYul{}; }; } From c965d6332c88e626249c7c3086b1f1c5df6129da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Mon, 21 Aug 2023 10:29:16 +0200 Subject: [PATCH 4/5] SyntaxTest: Allow derived test cases to filter out warnings and infos --- test/libsolidity/SyntaxTest.cpp | 11 ++++++++++- test/libsolidity/SyntaxTest.h | 7 ++++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/test/libsolidity/SyntaxTest.cpp b/test/libsolidity/SyntaxTest.cpp index e0d7a17e925a..ad53c2b265ec 100644 --- a/test/libsolidity/SyntaxTest.cpp +++ b/test/libsolidity/SyntaxTest.cpp @@ -38,7 +38,13 @@ using namespace solidity::frontend::test; using namespace boost::unit_test; namespace fs = boost::filesystem; -SyntaxTest::SyntaxTest(string const& _filename, langutil::EVMVersion _evmVersion): CommonSyntaxTest(_filename, _evmVersion) +SyntaxTest::SyntaxTest( + string const& _filename, + langutil::EVMVersion _evmVersion, + Error::Severity _minSeverity +): + CommonSyntaxTest(_filename, _evmVersion), + m_minSeverity(_minSeverity) { m_optimiseYul = m_reader.boolSetting("optimize-yul", true); } @@ -98,6 +104,9 @@ void SyntaxTest::filterObtainedErrors() { for (auto const& currentError: filteredErrors()) { + if (currentError->severity() < m_minSeverity) + continue; + int locationStart = -1; int locationEnd = -1; string sourceName; diff --git a/test/libsolidity/SyntaxTest.h b/test/libsolidity/SyntaxTest.h index f9fd65319b46..a85e1f53febe 100644 --- a/test/libsolidity/SyntaxTest.h +++ b/test/libsolidity/SyntaxTest.h @@ -41,7 +41,11 @@ class SyntaxTest: public AnalysisFramework, public solidity::test::CommonSyntaxT { return std::make_unique(_config.filename, _config.evmVersion); } - SyntaxTest(std::string const& _filename, langutil::EVMVersion _evmVersion); + SyntaxTest( + std::string const& _filename, + langutil::EVMVersion _evmVersion, + langutil::Error::Severity _minSeverity = langutil::Error::Severity::Info + ); protected: virtual void setupCompiler(); @@ -49,6 +53,7 @@ class SyntaxTest: public AnalysisFramework, public solidity::test::CommonSyntaxT virtual void filterObtainedErrors(); bool m_optimiseYul{}; + langutil::Error::Severity m_minSeverity{}; }; } From e847596e3950862a23e88cb21df4062ad0f3a398 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Fri, 18 Aug 2023 15:44:13 +0200 Subject: [PATCH 5/5] CommonSyntaxTest: Add support for syntax tests with custom expectations in addition to expected errors --- test/CommonSyntaxTest.cpp | 46 +++++++++++++++++++++++++++++++++++---- test/CommonSyntaxTest.h | 17 +++++++++++++-- test/TestCaseReader.cpp | 8 ++++--- 3 files changed, 62 insertions(+), 9 deletions(-) diff --git a/test/CommonSyntaxTest.cpp b/test/CommonSyntaxTest.cpp index f794439de86a..79b59780411b 100644 --- a/test/CommonSyntaxTest.cpp +++ b/test/CommonSyntaxTest.cpp @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -29,6 +30,7 @@ using namespace std; using namespace solidity; +using namespace solidity::util; using namespace solidity::util::formatting; using namespace solidity::langutil; using namespace solidity::frontend; @@ -66,6 +68,7 @@ CommonSyntaxTest::CommonSyntaxTest(string const& _filename, langutil::EVMVersion TestCase::TestResult CommonSyntaxTest::run(ostream& _stream, string const& _linePrefix, bool _formatted) { + parseCustomExpectations(m_reader.stream()); parseAndAnalyze(); return conclude(_stream, _linePrefix, _formatted); @@ -73,7 +76,7 @@ TestCase::TestResult CommonSyntaxTest::run(ostream& _stream, string const& _line TestCase::TestResult CommonSyntaxTest::conclude(ostream& _stream, string const& _linePrefix, bool _formatted) { - if (m_expectations == m_errorList) + if (expectationsMatch()) return TestResult::Success; printExpectationAndError(_stream, _linePrefix, _formatted); @@ -84,9 +87,9 @@ void CommonSyntaxTest::printExpectationAndError(ostream& _stream, string const& { string nextIndentLevel = _linePrefix + " "; util::AnsiColorized(_stream, _formatted, {BOLD, CYAN}) << _linePrefix << "Expected result:" << endl; - printErrorList(_stream, m_expectations, nextIndentLevel, _formatted); + printExpectedResult(_stream, nextIndentLevel, _formatted); util::AnsiColorized(_stream, _formatted, {BOLD, CYAN}) << _linePrefix << "Obtained result:" << endl; - printErrorList(_stream, m_errorList, nextIndentLevel, _formatted); + printObtainedResult(_stream, nextIndentLevel, _formatted); } void CommonSyntaxTest::printSource(ostream& _stream, string const& _linePrefix, bool _formatted) const @@ -149,6 +152,30 @@ void CommonSyntaxTest::printSource(ostream& _stream, string const& _linePrefix, } } +void CommonSyntaxTest::parseCustomExpectations(istream& _stream) +{ + string remainingExpectations = boost::trim_copy(readUntilEnd(_stream)); + soltestAssert( + remainingExpectations.empty(), + "Found custom expectations not supported by the test case:\n" + remainingExpectations + ); +} + +bool CommonSyntaxTest::expectationsMatch() +{ + return m_expectations == m_errorList; +} + +void CommonSyntaxTest::printExpectedResult(ostream& _stream, string const& _linePrefix, bool _formatted) const +{ + printErrorList(_stream, m_expectations, _linePrefix, _formatted); +} + +void CommonSyntaxTest::printObtainedResult(ostream& _stream, string const& _linePrefix, bool _formatted) const +{ + printErrorList(_stream, m_errorList, _linePrefix, _formatted); +} + void CommonSyntaxTest::printErrorList( ostream& _stream, vector const& _errorList, @@ -157,7 +184,10 @@ void CommonSyntaxTest::printErrorList( ) { if (_errorList.empty()) - util::AnsiColorized(_stream, _formatted, {BOLD, GREEN}) << _linePrefix << "Success" << endl; + { + if (_formatted) + util::AnsiColorized(_stream, _formatted, {BOLD, GREEN}) << _linePrefix << "Success" << endl; + } else for (auto const& error: _errorList) { @@ -194,12 +224,20 @@ string CommonSyntaxTest::errorMessage(util::Exception const& _e) vector CommonSyntaxTest::parseExpectations(istream& _stream) { + static string const customExpectationsDelimiter("// ----"); + vector expectations; string line; while (getline(_stream, line)) { auto it = line.begin(); + // Anything below the delimiter is left up to the derived class to process in a custom way. + // The delimiter is optional and identical to the one that starts error expectations in + // TestCaseReader::parseSourcesAndSettingsWithLineNumber(). + if (boost::algorithm::starts_with(line, customExpectationsDelimiter)) + break; + skipSlashes(it, line.end()); skipWhitespace(it, line.end()); diff --git a/test/CommonSyntaxTest.h b/test/CommonSyntaxTest.h index 169824252d93..606f61a88d0c 100644 --- a/test/CommonSyntaxTest.h +++ b/test/CommonSyntaxTest.h @@ -62,14 +62,27 @@ class CommonSyntaxTest: public frontend::test::EVMVersionRestrictedTestCase void printSource(std::ostream& _stream, std::string const &_linePrefix = "", bool _formatted = false) const override; void printUpdatedExpectations(std::ostream& _stream, std::string const& _linePrefix) const override { - if (!m_errorList.empty()) - printErrorList(_stream, m_errorList, _linePrefix, false); + printObtainedResult(_stream, _linePrefix, false); } static std::string errorMessage(util::Exception const& _e); protected: + /// Should be implemented by those derived test cases that want to allow extra expectations + /// after the error/warning expectations. The default implementation does not allow them and + /// fails instead. + /// @param _stream Input stream positioned at the beginning of the extra expectations. + virtual void parseCustomExpectations(std::istream& _stream); + virtual void parseAndAnalyze() = 0; + /// Should return true if obtained values match expectations. + /// The default implementation only compares the error list. Derived classes that support + /// custom expectations should override this to include them in the comparison. + virtual bool expectationsMatch(); + + virtual void printExpectedResult(std::ostream& _stream, std::string const& _linePrefix, bool _formatted) const; + virtual void printObtainedResult(std::ostream& _stream, std::string const& _linePrefix, bool _formatted) const; + static void printErrorList( std::ostream& _stream, std::vector const& _errors, diff --git a/test/TestCaseReader.cpp b/test/TestCaseReader.cpp index 43e4a6fc36b6..7ad752ef816e 100644 --- a/test/TestCaseReader.cpp +++ b/test/TestCaseReader.cpp @@ -114,15 +114,17 @@ pair TestCaseReader::parseSourcesAndSettingsWithLineNumber(is static string const sourceDelimiterEnd("===="); static string const comment("// "); static string const settingsDelimiter("// ===="); - static string const delimiter("// ----"); + static string const expectationsDelimiter("// ----"); bool sourcePart = true; while (getline(_stream, line)) { lineNumber++; - if (boost::algorithm::starts_with(line, delimiter)) + // Anything below the delimiter is left up to the test case to process in a custom way. + if (boost::algorithm::starts_with(line, expectationsDelimiter)) break; - else if (boost::algorithm::starts_with(line, settingsDelimiter)) + + if (boost::algorithm::starts_with(line, settingsDelimiter)) sourcePart = false; else if (sourcePart) {