Bison 3.8 Parser

cmake/ADIOSBisonFlexSub.cmake

@@ -0,0 +1,15 @@
+FUNCTION (SETUP_ADIOS_BISON_FLEX_SUB)
+
+set (BISON_FLEX_PRECOMPILE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/toolkit/derived/parser/pregen-source")
+
+ADD_CUSTOM_COMMAND(OUTPUT parser.cpp
+        COMMAND ${CMAKE_COMMAND} -E copy ${BISON_FLEX_PRECOMPILE_DIR}/parser.cpp ${CMAKE_CURRENT_BINARY_DIR}
+        COMMAND ${CMAKE_COMMAND} -E copy ${BISON_FLEX_PRECOMPILE_DIR}/parser.h ${CMAKE_CURRENT_BINARY_DIR}
+        COMMAND ${CMAKE_COMMAND} -E copy ${BISON_FLEX_PRECOMPILE_DIR}/location.hh ${CMAKE_CURRENT_BINARY_DIR}
+	COMMENT "Using pre-generated Bison Output from ${BISON_FLEX_PRECOMPILE_DIR}")
+ADD_CUSTOM_COMMAND(OUTPUT lexer.cpp
+	COMMAND ${CMAKE_COMMAND} -E copy ${BISON_FLEX_PRECOMPILE_DIR}/lexer.cpp ${CMAKE_CURRENT_BINARY_DIR}
+	COMMENT "Using pre-generated Flex Output from ${BISON_FLEX_PRECOMPILE_DIR}")
+
+set (BISON_Parser_OUTPUT_SOURCE parser.cpp PARENT_SCOPE)
+ENDFUNCTION()

scripts/ci/scripts/run-clang-format.sh

@@ -12,7 +12,7 @@ then
 fi
 
 # Check C and C++ code with clang-format
-find source plugins testing examples bindings -regextype posix-extended -iregex '.*\.(h|c|cpp|tcc|cu)'  | xargs clang-format -i
+find source plugins testing examples bindings  -regextype posix-extended -iregex '.*\.(h|c|cpp|tcc|cu)' ! -path "source/adios2/toolkit/derived/parser/pregen-source/*" | xargs clang-format -i
 DIFF="$(git diff)"
 if [ -n "${DIFF}" ]
 then

source/adios2/CMakeLists.txt

@@ -135,12 +135,51 @@ if (ADIOS2_HAVE_Derived_Variable)
       toolkit/derived/Expression.cpp
       toolkit/derived/Function.cpp toolkit/derived/Function.tcc
       toolkit/derived/ExprHelper.h)
+  set_target_properties(adios2_core PROPERTIES
+  				   INCLUDE_DIRECTORIES "$<BUILD_INTERFACE:${ADIOS2_SOURCE_DIR}/source/adios2/toolkit/derived/parser>;$<BUILD_INTERFACE:${ADIOS2_BINARY_DIR}/source/adios2>")
+  find_package(BISON "3.8.2")
+  find_package(FLEX)
+
+  if(NOT BISON_FOUND OR NOT FLEX_FOUND)
+    include(ADIOSBisonFlexSub)
+    SETUP_ADIOS_BISON_FLEX_SUB()
+  else()
+    BISON_TARGET(MyParser
+      toolkit/derived/parser/parser.y
+      ${CMAKE_CURRENT_BINARY_DIR}/parser.cpp
+      COMPILE_FLAGS "-o parser.cpp --header=parser.h"
+      DEFINES_FILE ${CMAKE_CURRENT_BINARY_DIR}/parser.h)
+    FLEX_TARGET(MyScanner
+      toolkit/derived/parser/lexer.l
+      COMPILE_FLAGS "-o lexer.cpp --header-file=lexer.h" 
+      ${CMAKE_CURRENT_BINARY_DIR}/lexer.cpp
+      DEFINES_FILE ${CMAKE_CURRENT_BINARY_DIR}/lexer.h)
+    ADD_FLEX_BISON_DEPENDENCY(MyScanner MyParser)
+  endif()
+  if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
+    SET_SOURCE_FILES_PROPERTIES(${CMAKE_CURRENT_BINARY_DIR}/lexer.cpp PROPERTIES COMPILE_FLAGS -Wno-sign-compare)
+  elseif(CMAKE_CXX_COMPILER_ID STREQUAL "Intel")
+    SET_SOURCE_FILES_PROPERTIES(${CMAKE_CURRENT_BINARY_DIR}/parser.cpp PROPERTIES COMPILE_FLAGS -Wno-unused-but-set-variable)
+  elseif(CMAKE_CXX_COMPILER_ID STREQUAL "IntelLLVM")
+    SET_SOURCE_FILES_PROPERTIES(${CMAKE_CURRENT_BINARY_DIR}/parser.cpp PROPERTIES COMPILE_FLAGS -Wno-unused-but-set-variable)
+  elseif(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC")
+    SET_SOURCE_FILES_PROPERTIES(toolkit/derived/Expression.cpp toolkit/derived/Function.cpp PROPERTIES COMPILE_FLAGS "/wd4005 /wd4065 /wd4267 -DYY_NO_UNISTD_H")
+  endif()
   add_library(adios2_core_derived
-      toolkit/derived/parser/lexer.cpp
-      toolkit/derived/parser/parser.cpp
+      ${CMAKE_CURRENT_BINARY_DIR}/lexer.cpp
+      ${CMAKE_CURRENT_BINARY_DIR}/parser.cpp
+      toolkit/derived/parser/ASTDriver.cpp
       toolkit/derived/parser/ASTNode.cpp)
+  set_target_properties(adios2_core_derived PROPERTIES
+  					   VISIBILITY_INLINES_HIDDEN ON
+					   INCLUDE_DIRECTORIES "$<BUILD_INTERFACE:${ADIOS2_SOURCE_DIR}/source/adios2/toolkit/derived/parser>;$<BUILD_INTERFACE:${ADIOS2_BINARY_DIR}/source/adios2>"
+					   EXPORT_NAME core_derived
+					   OUTPUT_NAME adios2${ADIOS2_LIBRARY_SUFFIX}_core_derived)
   target_link_libraries(adios2_core PRIVATE adios2_core_derived)
   set(maybe_adios2_core_derived adios2_core_derived)
+  if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC")
+    set_target_properties(adios2_core_derived PROPERTIES COMPILE_FLAGS "/wd4005 /wd4267 /wd4065 -DYY_NO_UNISTD_H")
+  endif()
 endif()
 
 set(maybe_adios2_core_cuda)

source/adios2/core/VariableDerived.cpp

@@ -11,7 +11,7 @@ VariableDerived::VariableDerived(const std::string &name, adios2::derived::Expre
                                  const DerivedVarType varType)
 : VariableBase(name, exprType, helper::GetDataTypeSize(exprType), expr.GetShape(), expr.GetStart(),
                expr.GetCount(), isConstant),
-  m_Expr(expr), m_DerivedType(varType)
+  m_DerivedType(varType), m_Expr(expr)
 {
 }
 
@@ -27,15 +27,15 @@ void VariableDerived::UpdateExprDim(std::map<std::string, std::tuple<Dims, Dims,
 }
 
 std::vector<std::tuple<void *, Dims, Dims>>
-VariableDerived::ApplyExpression(std::map<std::string, MinVarInfo> NameToMVI)
+VariableDerived::ApplyExpression(std::map<std::string, std::unique_ptr<MinVarInfo>> &NameToMVI)
 {
     size_t numBlocks = 0;
     // check that all variables have the same number of blocks
-    for (auto variable : NameToMVI)
+    for (const auto &variable : NameToMVI)
     {
         if (numBlocks == 0)
-            numBlocks = variable.second.BlocksInfo.size();
-        if (numBlocks != variable.second.BlocksInfo.size())
+            numBlocks = variable.second->BlocksInfo.size();
+        if (numBlocks != variable.second->BlocksInfo.size())
             helper::Throw<std::invalid_argument>("Core", "VariableDerived", "ApplyExpression",
                                                  " variables do not have the same number of blocks "
                                                  " in computing the derived variable " +
@@ -44,21 +44,21 @@ VariableDerived::ApplyExpression(std::map<std::string, MinVarInfo> NameToMVI)
 
     std::map<std::string, std::vector<adios2::derived::DerivedData>> inputData;
     // create the map between variable name and DerivedData object
-    for (auto variable : NameToMVI)
+    for (const auto &variable : NameToMVI)
     {
         // add the dimensions of all blocks into a vector
         std::vector<adios2::derived::DerivedData> varData;
         for (size_t i = 0; i < numBlocks; i++)
         {
             Dims start;
             Dims count;
-            for (size_t d = 0; d < variable.second.Dims; d++)
+            for (int d = 0; d < variable.second->Dims; d++)
             {
-                start.push_back(variable.second.BlocksInfo[i].Start[d]);
-                count.push_back(variable.second.BlocksInfo[i].Count[d]);
+                start.push_back(variable.second->BlocksInfo[i].Start[d]);
+                count.push_back(variable.second->BlocksInfo[i].Count[d]);
             }
             varData.push_back(adios2::derived::DerivedData(
-                {variable.second.BlocksInfo[i].BufferP, start, count}));
+                {variable.second->BlocksInfo[i].BufferP, start, count}));
         }
         inputData.insert({variable.first, varData});
     }

source/adios2/core/VariableDerived.h

@@ -32,7 +32,7 @@ class VariableDerived : public VariableBase
     ApplyExpression(std::map<std::string, std::vector<void *>> NameToData,
                     std::map<std::string, std::tuple<Dims, Dims, Dims>> NameToDims);
     std::vector<std::tuple<void *, Dims, Dims>>
-    ApplyExpression(std::map<std::string, MinVarInfo> mvi);
+    ApplyExpression(std::map<std::string, std::unique_ptr<MinVarInfo>> &mvi);
 };
 
 } // end namespace core

source/adios2/engine/bp5/BP5Writer.cpp

@@ -516,7 +516,7 @@ void BP5Writer::ComputeDerivedVariables()
         auto derivedVar = dynamic_cast<core::VariableDerived *>((*it).second.get());
         std::vector<std::string> varList = derivedVar->VariableNameList();
         // to create a mapping between variable name and the varInfo (dim and data pointer)
-        std::map<std::string, MinVarInfo> nameToVarInfo;
+        std::map<std::string, std::unique_ptr<MinVarInfo>> nameToVarInfo;
         bool computeDerived = true;
         for (auto varName : varList)
         {
@@ -536,7 +536,7 @@ void BP5Writer::ComputeDerivedVariables()
                 std::cout << " .. skip derived variable " << (*it).second->m_Name << std::endl;
                 break;
             }
-            nameToVarInfo.insert({varName, *mvi});
+            nameToVarInfo.insert({varName, std::unique_ptr<MinVarInfo>(mvi)});
         }
         // skip computing derived variables if it contains variables that are not written this step
         if (!computeDerived)

source/adios2/toolkit/derived/ExprHelper.h

@@ -57,4 +57,4 @@ inline ExpressionOperator get_op(std::string op) { return string_to_op.at(op); }
 
 }
 }
-#endif
+#endif

source/adios2/toolkit/derived/Expression.cpp

@@ -2,19 +2,35 @@
 #define ADIOS2_DERIVED_Expression_CPP_
 
 #include "Expression.h"
-#include "parser/ASTNode.h"
-#include "parser/parser.h"
+#include "parser/ASTDriver.h"
 
 namespace adios2
 {
 namespace detail
 {
+// helper function
+adios2::detail::ExpressionOperator convert_op(std::string opname)
+{
+    adios2::detail::ExpressionOperator op;
+    try
+    {
+        op = adios2::detail::get_op(opname);
+    }
+    catch (std::out_of_range &e)
+    {
+        (void)e; // use e
+        helper::Throw<std::invalid_argument>("Derived", "ExprHelper", "get_op",
+                                             "Parser cannot recognize operator '" + opname + "'.");
+    }
+    return op;
+};
+
 adios2::derived::ExpressionTree ASTNode_to_ExpressionTree(adios2::detail::ASTNode *node)
 {
-    adios2::derived::ExpressionTree exprTree_node(node->operation);
-    for (adios2::detail::ASTNode *e : node->sub_expr)
+    adios2::derived::ExpressionTree exprTree_node(convert_op(node->get_opname()));
+    for (adios2::detail::ASTNode *e : node->get_subexprs())
     {
-        switch (e->operation)
+        switch (convert_op(e->get_opname()))
         {
         case adios2::detail::ExpressionOperator::OP_ALIAS: // add variable given by alias
             // add an index operation in the chain if the variable contains indeces
@@ -25,19 +41,19 @@ adios2::derived::ExpressionTree ASTNode_to_ExpressionTree(adios2::detail::ASTNod
                 index_expr.add_child(e->lookup_var_path(e->alias));
                 expTree_node->add_child(expr);
             }*/
-            exprTree_node.add_child(e->lookup_var_path(e->alias));
+            exprTree_node.add_child(e->get_varname());
             break;
         case adios2::detail::ExpressionOperator::OP_PATH: // add variable name
-            exprTree_node.add_child(e->alias);
+            exprTree_node.add_child(e->get_varname());
             break;
         case adios2::detail::ExpressionOperator::OP_NUM: // set the base value for the operation
-            exprTree_node.set_base(e->value);
+            exprTree_node.set_base(e->get_value());
             break;
         default: // if the children nodes are other expressions, convert them to expressions
             auto temp_node = ASTNode_to_ExpressionTree(e);
             // move from a binary to a multinary tree if the child has the same operation
-            if (e->operation == node->operation &&
-                adios2::detail::op_property.at(e->operation).is_associative)
+            if (convert_op(e->get_opname()) == convert_op(node->get_opname()) &&
+                adios2::detail::op_property.at(convert_op(e->get_opname())).is_associative)
             {
                 // concatenate exprTree with temp_node
                 for (std::tuple<adios2::derived::ExpressionTree, std::string, bool> childTree :
@@ -63,10 +79,10 @@ namespace derived
 {
 
 Expression::Expression(std::string string_exp)
-: ExprString(string_exp), m_Shape({0}), m_Start({0}), m_Count({0})
+: m_Shape({0}), m_Start({0}), m_Count({0}), ExprString(string_exp)
 {
-    adios2::detail::ASTNode *root_node = adios2::detail::parse_expression(string_exp);
-    m_Expr = adios2::detail::ASTNode_to_ExpressionTree(root_node);
+    adios2::detail::ASTDriver drv(string_exp);
+    m_Expr = adios2::detail::ASTNode_to_ExpressionTree(drv.getAST());
 }
 
 std::vector<std::string> Expression::VariableNameList() { return m_Expr.VariableNameList(); }

source/adios2/toolkit/derived/Expression.h

@@ -34,7 +34,7 @@ class ExpressionTree
     std::vector<std::tuple<ExpressionTree, std::string, bool>> sub_exprs;
     OpInfo detail;
 
-    ExpressionTree(){};
+    ExpressionTree() : detail({adios2::detail::ExpressionOperator::OP_NULL, {}, 0}) {}
     ExpressionTree(adios2::detail::ExpressionOperator o) : detail({o, {}, 0}) {}
     ExpressionTree(adios2::detail::ExpressionOperator o, double c) : detail({o, {}, 0}) {}
     ExpressionTree(std::vector<std::tuple<size_t, size_t, size_t>> indices)

source/adios2/toolkit/derived/Function.cpp

@@ -39,7 +39,7 @@ DerivedData MagnitudeFunc(std::vector<DerivedData> inputData, DataType type)
         T *magValues = ApplyOneToOne<T>(inputData, dataSize, [](T a, T b) { return a + b * b; });  \
         for (size_t i = 0; i < dataSize; i++)                                                      \
         {                                                                                          \
-            magValues[i] = std::sqrt(magValues[i]);                                                \
+            magValues[i] = (T)std::sqrt(magValues[i]);                                             \
         }                                                                                          \
         return DerivedData({(void *)magValues, inputData[0].Start, inputData[0].Count});           \
     }

source/adios2/toolkit/derived/Function.tcc

@@ -22,7 +22,7 @@ T *ApplyOneToOne(std::vector<DerivedData> inputData, size_t dataSize,
         std::cout << "Allocation failed for the derived data" << std::endl;
         // TODO - throw an exception
     }
-    memset(outValues, 0, dataSize * sizeof(T));
+    memset((void *)outValues, 0, dataSize * sizeof(T));
     for (auto &variable : inputData)
     {
         for (size_t i = 0; i < dataSize; i++)

source/adios2/toolkit/derived/parser/ASTDriver.cpp

@@ -0,0 +1,101 @@
+#include "ASTDriver.h"
+
+namespace adios2
+{
+namespace detail
+{
+
+using indx_type = std::vector<std::tuple<int, int, int>>;
+
+ASTDriver::ASTDriver() {}
+
+ASTDriver::ASTDriver(const std::string input) { ASTDriver::parse(input); }
+
+ASTDriver::~ASTDriver()
+{
+    while (holding.size() > 0)
+    {
+        delete holding.top();
+        holding.pop();
+    }
+}
+
+ASTNode *ASTDriver::getAST()
+{
+    // TODO: check only one ASTNode remains in holding
+    // else throw error that parsing failed
+    resolve(holding.top());
+    return holding.top();
+}
+
+void ASTDriver::resolve(ASTNode *node)
+{
+    if (!node->get_alias().empty())
+    {
+        std::tuple<std::string, indx_type> var_info;
+        var_info = lookup_var(node->get_alias());
+        node->set_varname(std::get<0>(var_info));
+        node->set_indices(std::get<1>(var_info));
+    }
+    for (ASTNode *subexpr : node->get_subexprs())
+    {
+        resolve(subexpr);
+    }
+}
+
+std::tuple<std::string, indx_type> ASTDriver::lookup_var(const std::string alias)
+{
+    return aliases[alias];
+}
+
+std::string ASTDriver::lookup_var_name(const std::string alias)
+{
+    std::tuple<std::string, indx_type> var = aliases[alias];
+    return std::get<0>(var);
+}
+
+indx_type ASTDriver::lookup_var_indices(const std::string alias)
+{
+    std::tuple<std::string, indx_type> var = aliases[alias];
+    return std::get<1>(var);
+}
+
+void ASTDriver::add_lookup_entry(std::string alias, std::string var_name, indx_type indices)
+{
+    aliases.insert({alias, {var_name, indices}});
+}
+
+void ASTDriver::add_lookup_entry(std::string alias, std::string var_name)
+{
+    aliases.insert({alias, {var_name, {}}});
+}
+
+void ASTDriver::createNode(std::string op_name, size_t numsubexprs)
+{
+    ASTNode *node = new ASTNode(op_name, numsubexprs);
+    for (size_t i = 1; i <= numsubexprs; ++i)
+    {
+        // TODO: check that holding contains ASTNode(s)
+        // else throw error that parsing failed
+        ASTNode *subexpr = holding.top();
+        node->insert_subexpr_n(subexpr, numsubexprs - i);
+        holding.pop();
+    }
+    holding.push(node);
+}
+
+void ASTDriver::createNode(std::string alias)
+{
+    ASTNode *node = new ASTNode("ALIAS", alias);
+    holding.push(node);
+}
+
+void ASTDriver::createNode(std::string alias, indx_type indices)
+{
+    ASTNode *node = new ASTNode("INDEX", indices);
+    node->pushback_subexpr(new ASTNode("ALIAS", alias));
+    holding.push(node);
+}
+
+}
+}