Skip to content

Commit

Permalink
Bison 3.8 Parser (#4062)
Browse files Browse the repository at this point in the history
* Derived Expressions Curl function with correctness test
  • Loading branch information
lizdulac authored Mar 19, 2024
1 parent 34093ee commit ab1cf30
Show file tree
Hide file tree
Showing 30 changed files with 6,174 additions and 4,506 deletions.
15 changes: 15 additions & 0 deletions cmake/ADIOSBisonFlexSub.cmake
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
FUNCTION (SETUP_ADIOS_BISON_FLEX_SUB)

set (BISON_FLEX_PRECOMPILE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/toolkit/derived/parser/pregen-source")

ADD_CUSTOM_COMMAND(OUTPUT parser.cpp
COMMAND ${CMAKE_COMMAND} -E copy ${BISON_FLEX_PRECOMPILE_DIR}/parser.cpp ${CMAKE_CURRENT_BINARY_DIR}
COMMAND ${CMAKE_COMMAND} -E copy ${BISON_FLEX_PRECOMPILE_DIR}/parser.h ${CMAKE_CURRENT_BINARY_DIR}
COMMAND ${CMAKE_COMMAND} -E copy ${BISON_FLEX_PRECOMPILE_DIR}/location.hh ${CMAKE_CURRENT_BINARY_DIR}
COMMENT "Using pre-generated Bison Output from ${BISON_FLEX_PRECOMPILE_DIR}")
ADD_CUSTOM_COMMAND(OUTPUT lexer.cpp
COMMAND ${CMAKE_COMMAND} -E copy ${BISON_FLEX_PRECOMPILE_DIR}/lexer.cpp ${CMAKE_CURRENT_BINARY_DIR}
COMMENT "Using pre-generated Flex Output from ${BISON_FLEX_PRECOMPILE_DIR}")

set (BISON_Parser_OUTPUT_SOURCE parser.cpp PARENT_SCOPE)
ENDFUNCTION()
2 changes: 1 addition & 1 deletion scripts/ci/scripts/run-clang-format.sh
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ then
fi

# Check C and C++ code with clang-format
find source plugins testing examples bindings -regextype posix-extended -iregex '.*\.(h|c|cpp|tcc|cu)' | xargs clang-format -i
find source plugins testing examples bindings -regextype posix-extended -iregex '.*\.(h|c|cpp|tcc|cu)' ! -path "source/adios2/toolkit/derived/parser/pregen-source/*" | xargs clang-format -i
DIFF="$(git diff)"
if [ -n "${DIFF}" ]
then
Expand Down
43 changes: 41 additions & 2 deletions source/adios2/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -135,12 +135,51 @@ if (ADIOS2_HAVE_Derived_Variable)
toolkit/derived/Expression.cpp
toolkit/derived/Function.cpp toolkit/derived/Function.tcc
toolkit/derived/ExprHelper.h)
set_target_properties(adios2_core PROPERTIES
INCLUDE_DIRECTORIES "$<BUILD_INTERFACE:${ADIOS2_SOURCE_DIR}/source/adios2/toolkit/derived/parser>;$<BUILD_INTERFACE:${ADIOS2_BINARY_DIR}/source/adios2>")
find_package(BISON "3.8.2")
find_package(FLEX)

if(NOT BISON_FOUND OR NOT FLEX_FOUND)
include(ADIOSBisonFlexSub)
SETUP_ADIOS_BISON_FLEX_SUB()
else()
BISON_TARGET(MyParser
toolkit/derived/parser/parser.y
${CMAKE_CURRENT_BINARY_DIR}/parser.cpp
COMPILE_FLAGS "-o parser.cpp --header=parser.h"
DEFINES_FILE ${CMAKE_CURRENT_BINARY_DIR}/parser.h)
FLEX_TARGET(MyScanner
toolkit/derived/parser/lexer.l
COMPILE_FLAGS "-o lexer.cpp --header-file=lexer.h"
${CMAKE_CURRENT_BINARY_DIR}/lexer.cpp
DEFINES_FILE ${CMAKE_CURRENT_BINARY_DIR}/lexer.h)
ADD_FLEX_BISON_DEPENDENCY(MyScanner MyParser)
endif()
if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
SET_SOURCE_FILES_PROPERTIES(${CMAKE_CURRENT_BINARY_DIR}/lexer.cpp PROPERTIES COMPILE_FLAGS -Wno-sign-compare)
elseif(CMAKE_CXX_COMPILER_ID STREQUAL "Intel")
SET_SOURCE_FILES_PROPERTIES(${CMAKE_CURRENT_BINARY_DIR}/parser.cpp PROPERTIES COMPILE_FLAGS -Wno-unused-but-set-variable)
elseif(CMAKE_CXX_COMPILER_ID STREQUAL "IntelLLVM")
SET_SOURCE_FILES_PROPERTIES(${CMAKE_CURRENT_BINARY_DIR}/parser.cpp PROPERTIES COMPILE_FLAGS -Wno-unused-but-set-variable)
elseif(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC")
SET_SOURCE_FILES_PROPERTIES(toolkit/derived/Expression.cpp toolkit/derived/Function.cpp PROPERTIES COMPILE_FLAGS "/wd4005 /wd4065 /wd4267 -DYY_NO_UNISTD_H")
endif()
add_library(adios2_core_derived
toolkit/derived/parser/lexer.cpp
toolkit/derived/parser/parser.cpp
${CMAKE_CURRENT_BINARY_DIR}/lexer.cpp
${CMAKE_CURRENT_BINARY_DIR}/parser.cpp
toolkit/derived/parser/ASTDriver.cpp
toolkit/derived/parser/ASTNode.cpp)
set_target_properties(adios2_core_derived PROPERTIES
VISIBILITY_INLINES_HIDDEN ON
INCLUDE_DIRECTORIES "$<BUILD_INTERFACE:${ADIOS2_SOURCE_DIR}/source/adios2/toolkit/derived/parser>;$<BUILD_INTERFACE:${ADIOS2_BINARY_DIR}/source/adios2>"
EXPORT_NAME core_derived
OUTPUT_NAME adios2${ADIOS2_LIBRARY_SUFFIX}_core_derived)
target_link_libraries(adios2_core PRIVATE adios2_core_derived)
set(maybe_adios2_core_derived adios2_core_derived)
if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC")
set_target_properties(adios2_core_derived PROPERTIES COMPILE_FLAGS "/wd4005 /wd4267 /wd4065 -DYY_NO_UNISTD_H")
endif()
endif()

set(maybe_adios2_core_cuda)
Expand Down
20 changes: 10 additions & 10 deletions source/adios2/core/VariableDerived.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ VariableDerived::VariableDerived(const std::string &name, adios2::derived::Expre
const DerivedVarType varType)
: VariableBase(name, exprType, helper::GetDataTypeSize(exprType), expr.GetShape(), expr.GetStart(),
expr.GetCount(), isConstant),
m_Expr(expr), m_DerivedType(varType)
m_DerivedType(varType), m_Expr(expr)
{
}

Expand All @@ -27,15 +27,15 @@ void VariableDerived::UpdateExprDim(std::map<std::string, std::tuple<Dims, Dims,
}

std::vector<std::tuple<void *, Dims, Dims>>
VariableDerived::ApplyExpression(std::map<std::string, MinVarInfo> NameToMVI)
VariableDerived::ApplyExpression(std::map<std::string, std::unique_ptr<MinVarInfo>> &NameToMVI)
{
size_t numBlocks = 0;
// check that all variables have the same number of blocks
for (auto variable : NameToMVI)
for (const auto &variable : NameToMVI)
{
if (numBlocks == 0)
numBlocks = variable.second.BlocksInfo.size();
if (numBlocks != variable.second.BlocksInfo.size())
numBlocks = variable.second->BlocksInfo.size();
if (numBlocks != variable.second->BlocksInfo.size())
helper::Throw<std::invalid_argument>("Core", "VariableDerived", "ApplyExpression",
" variables do not have the same number of blocks "
" in computing the derived variable " +
Expand All @@ -44,21 +44,21 @@ VariableDerived::ApplyExpression(std::map<std::string, MinVarInfo> NameToMVI)

std::map<std::string, std::vector<adios2::derived::DerivedData>> inputData;
// create the map between variable name and DerivedData object
for (auto variable : NameToMVI)
for (const auto &variable : NameToMVI)
{
// add the dimensions of all blocks into a vector
std::vector<adios2::derived::DerivedData> varData;
for (size_t i = 0; i < numBlocks; i++)
{
Dims start;
Dims count;
for (size_t d = 0; d < variable.second.Dims; d++)
for (int d = 0; d < variable.second->Dims; d++)
{
start.push_back(variable.second.BlocksInfo[i].Start[d]);
count.push_back(variable.second.BlocksInfo[i].Count[d]);
start.push_back(variable.second->BlocksInfo[i].Start[d]);
count.push_back(variable.second->BlocksInfo[i].Count[d]);
}
varData.push_back(adios2::derived::DerivedData(
{variable.second.BlocksInfo[i].BufferP, start, count}));
{variable.second->BlocksInfo[i].BufferP, start, count}));
}
inputData.insert({variable.first, varData});
}
Expand Down
2 changes: 1 addition & 1 deletion source/adios2/core/VariableDerived.h
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ class VariableDerived : public VariableBase
ApplyExpression(std::map<std::string, std::vector<void *>> NameToData,
std::map<std::string, std::tuple<Dims, Dims, Dims>> NameToDims);
std::vector<std::tuple<void *, Dims, Dims>>
ApplyExpression(std::map<std::string, MinVarInfo> mvi);
ApplyExpression(std::map<std::string, std::unique_ptr<MinVarInfo>> &mvi);
};

} // end namespace core
Expand Down
4 changes: 2 additions & 2 deletions source/adios2/engine/bp5/BP5Writer.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -516,7 +516,7 @@ void BP5Writer::ComputeDerivedVariables()
auto derivedVar = dynamic_cast<core::VariableDerived *>((*it).second.get());
std::vector<std::string> varList = derivedVar->VariableNameList();
// to create a mapping between variable name and the varInfo (dim and data pointer)
std::map<std::string, MinVarInfo> nameToVarInfo;
std::map<std::string, std::unique_ptr<MinVarInfo>> nameToVarInfo;
bool computeDerived = true;
for (auto varName : varList)
{
Expand All @@ -536,7 +536,7 @@ void BP5Writer::ComputeDerivedVariables()
std::cout << " .. skip derived variable " << (*it).second->m_Name << std::endl;
break;
}
nameToVarInfo.insert({varName, *mvi});
nameToVarInfo.insert({varName, std::unique_ptr<MinVarInfo>(mvi)});
}
// skip computing derived variables if it contains variables that are not written this step
if (!computeDerived)
Expand Down
2 changes: 1 addition & 1 deletion source/adios2/toolkit/derived/ExprHelper.h
Original file line number Diff line number Diff line change
Expand Up @@ -57,4 +57,4 @@ inline ExpressionOperator get_op(std::string op) { return string_to_op.at(op); }

}
}
#endif
#endif
42 changes: 29 additions & 13 deletions source/adios2/toolkit/derived/Expression.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -2,19 +2,35 @@
#define ADIOS2_DERIVED_Expression_CPP_

#include "Expression.h"
#include "parser/ASTNode.h"
#include "parser/parser.h"
#include "parser/ASTDriver.h"

namespace adios2
{
namespace detail
{
// helper function
adios2::detail::ExpressionOperator convert_op(std::string opname)
{
adios2::detail::ExpressionOperator op;
try
{
op = adios2::detail::get_op(opname);
}
catch (std::out_of_range &e)
{
(void)e; // use e
helper::Throw<std::invalid_argument>("Derived", "ExprHelper", "get_op",
"Parser cannot recognize operator '" + opname + "'.");
}
return op;
};

adios2::derived::ExpressionTree ASTNode_to_ExpressionTree(adios2::detail::ASTNode *node)
{
adios2::derived::ExpressionTree exprTree_node(node->operation);
for (adios2::detail::ASTNode *e : node->sub_expr)
adios2::derived::ExpressionTree exprTree_node(convert_op(node->get_opname()));
for (adios2::detail::ASTNode *e : node->get_subexprs())
{
switch (e->operation)
switch (convert_op(e->get_opname()))
{
case adios2::detail::ExpressionOperator::OP_ALIAS: // add variable given by alias
// add an index operation in the chain if the variable contains indeces
Expand All @@ -25,19 +41,19 @@ adios2::derived::ExpressionTree ASTNode_to_ExpressionTree(adios2::detail::ASTNod
index_expr.add_child(e->lookup_var_path(e->alias));
expTree_node->add_child(expr);
}*/
exprTree_node.add_child(e->lookup_var_path(e->alias));
exprTree_node.add_child(e->get_varname());
break;
case adios2::detail::ExpressionOperator::OP_PATH: // add variable name
exprTree_node.add_child(e->alias);
exprTree_node.add_child(e->get_varname());
break;
case adios2::detail::ExpressionOperator::OP_NUM: // set the base value for the operation
exprTree_node.set_base(e->value);
exprTree_node.set_base(e->get_value());
break;
default: // if the children nodes are other expressions, convert them to expressions
auto temp_node = ASTNode_to_ExpressionTree(e);
// move from a binary to a multinary tree if the child has the same operation
if (e->operation == node->operation &&
adios2::detail::op_property.at(e->operation).is_associative)
if (convert_op(e->get_opname()) == convert_op(node->get_opname()) &&
adios2::detail::op_property.at(convert_op(e->get_opname())).is_associative)
{
// concatenate exprTree with temp_node
for (std::tuple<adios2::derived::ExpressionTree, std::string, bool> childTree :
Expand All @@ -63,10 +79,10 @@ namespace derived
{

Expression::Expression(std::string string_exp)
: ExprString(string_exp), m_Shape({0}), m_Start({0}), m_Count({0})
: m_Shape({0}), m_Start({0}), m_Count({0}), ExprString(string_exp)
{
adios2::detail::ASTNode *root_node = adios2::detail::parse_expression(string_exp);
m_Expr = adios2::detail::ASTNode_to_ExpressionTree(root_node);
adios2::detail::ASTDriver drv(string_exp);
m_Expr = adios2::detail::ASTNode_to_ExpressionTree(drv.getAST());
}

std::vector<std::string> Expression::VariableNameList() { return m_Expr.VariableNameList(); }
Expand Down
2 changes: 1 addition & 1 deletion source/adios2/toolkit/derived/Expression.h
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ class ExpressionTree
std::vector<std::tuple<ExpressionTree, std::string, bool>> sub_exprs;
OpInfo detail;

ExpressionTree(){};
ExpressionTree() : detail({adios2::detail::ExpressionOperator::OP_NULL, {}, 0}) {}
ExpressionTree(adios2::detail::ExpressionOperator o) : detail({o, {}, 0}) {}
ExpressionTree(adios2::detail::ExpressionOperator o, double c) : detail({o, {}, 0}) {}
ExpressionTree(std::vector<std::tuple<size_t, size_t, size_t>> indices)
Expand Down
2 changes: 1 addition & 1 deletion source/adios2/toolkit/derived/Function.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ DerivedData MagnitudeFunc(std::vector<DerivedData> inputData, DataType type)
T *magValues = ApplyOneToOne<T>(inputData, dataSize, [](T a, T b) { return a + b * b; }); \
for (size_t i = 0; i < dataSize; i++) \
{ \
magValues[i] = std::sqrt(magValues[i]); \
magValues[i] = (T)std::sqrt(magValues[i]); \
} \
return DerivedData({(void *)magValues, inputData[0].Start, inputData[0].Count}); \
}
Expand Down
2 changes: 1 addition & 1 deletion source/adios2/toolkit/derived/Function.tcc
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ T *ApplyOneToOne(std::vector<DerivedData> inputData, size_t dataSize,
std::cout << "Allocation failed for the derived data" << std::endl;
// TODO - throw an exception
}
memset(outValues, 0, dataSize * sizeof(T));
memset((void *)outValues, 0, dataSize * sizeof(T));
for (auto &variable : inputData)
{
for (size_t i = 0; i < dataSize; i++)
Expand Down
101 changes: 101 additions & 0 deletions source/adios2/toolkit/derived/parser/ASTDriver.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,101 @@
#include "ASTDriver.h"

namespace adios2
{
namespace detail
{

using indx_type = std::vector<std::tuple<int, int, int>>;

ASTDriver::ASTDriver() {}

ASTDriver::ASTDriver(const std::string input) { ASTDriver::parse(input); }

ASTDriver::~ASTDriver()
{
while (holding.size() > 0)
{
delete holding.top();
holding.pop();
}
}

ASTNode *ASTDriver::getAST()
{
// TODO: check only one ASTNode remains in holding
// else throw error that parsing failed
resolve(holding.top());
return holding.top();
}

void ASTDriver::resolve(ASTNode *node)
{
if (!node->get_alias().empty())
{
std::tuple<std::string, indx_type> var_info;
var_info = lookup_var(node->get_alias());
node->set_varname(std::get<0>(var_info));
node->set_indices(std::get<1>(var_info));
}
for (ASTNode *subexpr : node->get_subexprs())
{
resolve(subexpr);
}
}

std::tuple<std::string, indx_type> ASTDriver::lookup_var(const std::string alias)
{
return aliases[alias];
}

std::string ASTDriver::lookup_var_name(const std::string alias)
{
std::tuple<std::string, indx_type> var = aliases[alias];
return std::get<0>(var);
}

indx_type ASTDriver::lookup_var_indices(const std::string alias)
{
std::tuple<std::string, indx_type> var = aliases[alias];
return std::get<1>(var);
}

void ASTDriver::add_lookup_entry(std::string alias, std::string var_name, indx_type indices)
{
aliases.insert({alias, {var_name, indices}});
}

void ASTDriver::add_lookup_entry(std::string alias, std::string var_name)
{
aliases.insert({alias, {var_name, {}}});
}

void ASTDriver::createNode(std::string op_name, size_t numsubexprs)
{
ASTNode *node = new ASTNode(op_name, numsubexprs);
for (size_t i = 1; i <= numsubexprs; ++i)
{
// TODO: check that holding contains ASTNode(s)
// else throw error that parsing failed
ASTNode *subexpr = holding.top();
node->insert_subexpr_n(subexpr, numsubexprs - i);
holding.pop();
}
holding.push(node);
}

void ASTDriver::createNode(std::string alias)
{
ASTNode *node = new ASTNode("ALIAS", alias);
holding.push(node);
}

void ASTDriver::createNode(std::string alias, indx_type indices)
{
ASTNode *node = new ASTNode("INDEX", indices);
node->pushback_subexpr(new ASTNode("ALIAS", alias));
holding.push(node);
}

}
}
Loading

0 comments on commit ab1cf30

Please sign in to comment.