From d55d0c80a018cb19c442ba102d2394b9756bb895 Mon Sep 17 00:00:00 2001 From: abachma2 Date: Thu, 23 Mar 2023 11:12:21 -0500 Subject: [PATCH 001/102] remove inner loop for test The inner loop attempted to iterate over the variable 'x', which was NoneType and can't be iterated over. This may be a change in going from python 3.6 to 3.10. --- tests/test_null_sink.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/test_null_sink.py b/tests/test_null_sink.py index b1799156f5..98c2006acd 100644 --- a/tests/test_null_sink.py +++ b/tests/test_null_sink.py @@ -71,7 +71,6 @@ def check_null_sink(fname, given_spec): # No resource exchange is expected assert not tables_exist(outfile, illegal_paths) - assert 1 == 2 # FIX ME clean_outs() @@ -79,6 +78,5 @@ def test_null_sink(): cases = [("null_sink.xml", ":agents:Sink"), ("null_sink.py", ":cyclus.pyagents:Sink")] for case in cases: - for x in check_null_sink(*case): - pass + check_null_sink(*case) From 84b8b399202c5908fcd764cef5994090615aca08 Mon Sep 17 00:00:00 2001 From: abachma2 Date: Thu, 23 Mar 2023 11:32:29 -0500 Subject: [PATCH 002/102] remove inner loop, same as test_null_sink.py --- tests/test_source_to_sink.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_source_to_sink.py b/tests/test_source_to_sink.py index 7750691ec0..1ab5045ef7 100644 --- a/tests/test_source_to_sink.py +++ b/tests/test_source_to_sink.py @@ -107,5 +107,4 @@ def test_source_to_sink(): ("source_to_sink.py", ":cyclus.pyagents:Source", ":cyclus.pyagents:Sink"), ] for case in cases: - for x in check_source_to_sink(*case): - pass + check_source_to_sink(*case) From 7ea673b31387b5413c4839f0ad384e93c9baed28 Mon Sep 17 00:00:00 2001 From: abachma2 Date: Thu, 23 Mar 2023 14:31:11 -0500 Subject: [PATCH 003/102] change deprecated tests to pytest syntax --- tests/test_smbchk.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_smbchk.py b/tests/test_smbchk.py index 5229434acc..1bc97e011f 100644 --- a/tests/test_smbchk.py +++ b/tests/test_smbchk.py @@ -2,6 +2,7 @@ import os import platform import sys +import pytest from argparse import Namespace from tools import skip_then_continue @@ -16,8 +17,8 @@ except ImportError: smbchk = False +@pytest.mark.deprecated("symbol test has been deprecated") def test_load(): - raise DeprecatedTest("symbol test has been deprecated") if not smbchk: return ns = Namespace(filename=os.path.join(reldir, 'symbols.json')) From 266eb4d5655f30c654eba697cd8051c6c380032a Mon Sep 17 00:00:00 2001 From: abachma2 Date: Tue, 28 Mar 2023 16:22:08 -0500 Subject: [PATCH 004/102] remove file and tests All four tests in this file were previously deprecated, so with the new updates it felt reasonable to remove them entirely --- tests/test_smbchk.py | 70 -------------------------------------------- 1 file changed, 70 deletions(-) delete mode 100644 tests/test_smbchk.py diff --git a/tests/test_smbchk.py b/tests/test_smbchk.py deleted file mode 100644 index 1bc97e011f..0000000000 --- a/tests/test_smbchk.py +++ /dev/null @@ -1,70 +0,0 @@ -from __future__ import print_function, unicode_literals -import os -import platform -import sys -import pytest -from argparse import Namespace - -from tools import skip_then_continue - -cycdir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -reldir = os.path.join(cycdir, 'release') -blddir = os.path.join(cycdir, 'build') -sys.path.insert(0, reldir) - -try: - import smbchk -except ImportError: - smbchk = False - -@pytest.mark.deprecated("symbol test has been deprecated") -def test_load(): - if not smbchk: - return - ns = Namespace(filename=os.path.join(reldir, 'symbols.json')) - db = smbchk.load(ns) - assert(isinstance(db, list)) - -def test_nm(): - raise DeprecatedTest("symbol test has been deprecated") - if platform.system() == 'Darwin': - skip_then_continue("Skipping for Mac") - if not smbchk: - return - if os.name != 'posix' or not os.path.isdir(blddir): - return - ns = Namespace(prefix=blddir) - syms = smbchk.nm(ns) - assert ("cyclus::Agent::Agent(cyclus::Context*)" in syms) - -def test_diff(): - raise DeprecatedTest("symbol test has been deprecated") - if not smbchk: - return - db = [{'symbols': ["cyclus::Agent::Agent(cyclus::Context*)"], - 'version': 'X', 'date': 'x.x.x'}, - {'symbols': ["cyclus::Agent::Agent(cyclus::Context*)", - "cyclus::Agent::~Agent()"], - 'version': 'Y', 'date': 'y.y.y'},] - obs = smbchk.diff(db, 0, 1) - assert(len(obs) > 0) - -def test_check(): - raise DeprecatedTest("symbol test has been deprecated") - if not smbchk: - return - # adds to API - db = [{'symbols': ["cyclus::Agent::Agent(cyclus::Context*)"], - 'version': 'X', 'date': 'x.x.x'}, - {'symbols': ["cyclus::Agent::Agent(cyclus::Context*)", - "cyclus::Agent::~Agent()"], - 'version': 'Y', 'date': 'y.y.y'},] - obs = smbchk.check(db) - assert(obs) - - # removes from API - db.append({'symbols': ["cyclus::Agent::~Agent()"], - 'version': 'Z', 'date': 'z.z.z'}) - obs = smbchk.check(db) - assert not(obs) - From 10231a05f61468a821148d8ee830eba19a2644d4 Mon Sep 17 00:00:00 2001 From: abachma2 Date: Tue, 28 Mar 2023 16:50:31 -0500 Subject: [PATCH 005/102] rename to pick up by pytest Rename the file to *_test.py because pytest did not pick up the *_tests.py name. All tests are passing --- tests/cycpp_tests.py | 1194 ------------------------------------------ 1 file changed, 1194 deletions(-) delete mode 100644 tests/cycpp_tests.py diff --git a/tests/cycpp_tests.py b/tests/cycpp_tests.py deleted file mode 100644 index 65083f86d3..0000000000 --- a/tests/cycpp_tests.py +++ /dev/null @@ -1,1194 +0,0 @@ -import os -import sys -import uuid -import pprint -import tempfile -import pytest - -from collections import OrderedDict -from subprocess import Popen, PIPE, STDOUT - - -cycdir = os.path.dirname(os.path.dirname(__file__)) -sys.path.insert(0, os.path.join(cycdir, 'cli')) - -# pass 1 Filters -from cycpp import NamespaceFilter, TypedefFilter, UsingFilter,\ - UsingNamespaceFilter, NamespaceAliasFilter, ClassFilter, \ - AccessFilter, PragmaCyclusErrorFilter - -# pass 2 Filters -from cycpp import VarDecorationFilter, VarDeclarationFilter, ExecFilter, \ - NoteDecorationFilter, StateAccumulator - -# pass 3 Filters -from cycpp import CloneFilter, InitFromCopyFilter, \ - InitFromDbFilter, InfileToDbFilter, SchemaFilter, SnapshotFilter, \ - SnapshotInvFilter, InitInvFilter, DefaultPragmaFilter, AnnotationsFilter - -import cycpp - -# assert_equal.__self__.maxDiff = None !! this may not be necessary for pytest??? - -class MockMachine(object): - def __init__(self): - self.depth = 0 - self.execns = {} - self.context = {} - self.statements = [] - self.classes = [] - self.superclasses = {} - self.access = {} - self.namespaces = [] - self.using_namespaces = set() - self.aliases = set() - self.var_annotations = None - self.filters = [] - - def includeloc(self, statement=None): - return "" - - def classname(self): - """Implemented just for testing""" - return "" - - def ensure_class_context(self, classname): - if classname not in self.context: - self.context[classname] = OrderedDict() - - -# -# pass 1 Filters -# - -def test_tffilt(): - """Test TypedefFilter""" - m = MockMachine() - f = TypedefFilter(m) - assert not f.isvalid("mis typedef kind") - assert not f.isvalid("typedef kind") - - statement, sep = "typedef double db", ";" - assert f.isvalid(statement) - f.transform(statement, sep) - assert len(m.aliases) == 1 - assert (0, "double", "db") == m.aliases.pop() - - statement, sep = "typedef struct {int a; int b;} S, *pS", ";" - assert f.isvalid(statement) - -def test_uffilt(): - """Test UsingFilter""" - m = MockMachine() - f = UsingFilter(m) - assert not f.isvalid("not using namespace") - - statement, sep = "using std::cout", "" - assert f.isvalid(statement) - f.transform(statement, sep) - assert len(m.aliases) == 1 - assert (0, "std::cout", "cout") == m.aliases.pop() - -def test_nsfilt(): - """Test NamespaceFilter""" - m = MockMachine() - f = NamespaceFilter(m) - assert not f.isvalid("olzhas is not a namespace") - - # anonymous namespaces - statement, sep = " namespace ", "{" - assert f.isvalid(statement) - f.transform(statement, sep) - assert len(m.namespaces) == 1 - assert m.namespaces[0] == (0, '') - f.revert(statement, sep) - assert len(m.namespaces) == 0 - - # nymous namespace - statement, sep = "namespace gorgus ", "{" - assert f.isvalid(statement) - f.transform(statement, sep) - assert len(m.namespaces) == 1 - assert m.namespaces[0] == (0, "gorgus") - f.revert(statement, sep) - assert len(m.namespaces) == 0 - -def test_unfilt(): - """Test UsingNamespaseFilter""" - m = MockMachine() - f = UsingNamespaceFilter(m) - assert not f.isvalid("using cycamore") - - statement, sep = "using namespace std", "" - assert f.isvalid(statement) - f.transform(statement, sep) - assert len(m.using_namespaces) == 1 - assert (0, "std") == m.using_namespaces.pop() - f.revert(statement, sep) - assert len(m.using_namespaces) == 0 - -def test_nafilter(): - """Test NamespaceAliasFilter""" - m = MockMachine() - f = NamespaceAliasFilter(m) - assert not f.isvalid("namespace cycamore") - - statement, sep = "namespace cycamore = cm", "" - assert f.isvalid(statement) - f.transform(statement, sep) - assert len(m.aliases) == 1 - assert (0, "cm", "cycamore") == m.aliases.pop() - -def test_cfilter(): - """Test ClassFilter""" - m = MockMachine() - f = ClassFilter(m) - assert not f.isvalid("class ") - - statement, sep = "class Cyclus", "" - assert f.isvalid(statement) - f.transform(statement, sep) - assert len(m.classes) == 1 - assert m.classes[0] == (0, "Cyclus") - assert m.access[tuple(m.classes)] == "private" - f.revert(statement, sep) - assert len(m.classes) == 0 - -def test_afilter(): - """Test AccessFilter""" - m = MockMachine() - f = AccessFilter(m) - assert not f.isvalid("new private") - - statement, sep = "private:", "" - assert f.isvalid(statement) - f.transform(statement, sep) - assert m.access[tuple(m.classes)] == "private" - -def test_synerror(): - """Test PragmaCyclusErrorFilter""" - m = MockMachine() - f = PragmaCyclusErrorFilter(m) - assert not f.isvalid("#pragma cyclus var {}") - assert not f.isvalid("#pragma cyclus") - - assert f.isvalid('#pragma cyclus nooooo') - statement, sep = "#pragma cyclus var{}", "\n" - assert f.isvalid(statement) - with pytest.raises(SyntaxError): - f.transform(statement, sep) - -# -# pass 2 Filters -# -def test_vdecorfilter(): - """Test VarDecorationFilter""" - m = MockMachine() - f = VarDecorationFilter(m) - assert not f.isvalid("#pragma cyclus") - - statement, sep = "#pragma cyclus var {'name': 'James Bond'} ", "\n" - assert f.isvalid(statement) - f.transform(statement, sep) - assert m.var_annotations == {'name': 'James Bond'} - -def test_vdeclarfilter(): - """Test VarDeclarationFilter""" - m = MockMachine() - f = VarDeclarationFilter(m) - assert not f.isvalid("one ") - - statement, sep = "one two", "\n" - assert f.isvalid(statement) - m.classes = [(0, "trader")] - m.access = {"trader": "public"} - # m.var_annotations = {'name': 'James Bond'} - f.transform(statement, sep) - assert m.var_annotations == None - -def test_vdeclarfilter_canonize_alias(): - m = MockMachine() - f = VarDeclarationFilter(m) - cases = [ - # exp type name alias - ('x', 'float', 'x', None), - (['x', 'val'], ['std::set', 'float'], 'x', None), - (['x', 'first', 'second'], ['std::pair', 'int', 'int'], 'x', None), - ([['x', 'item'], 'key', 'val'], ['std::map', 'int', 'int'], 'x', None), - ([['x', 'item'], ['key', 'first', 'second'], ['val', 'val']], - ['std::map', ['std::pair', 'int', 'int'], ['std::set', 'int']], - 'x', None), - ('y', 'float', 'x', 'y'), - (['x', 'y'], ['std::set', 'float'], 'x', [None, 'y']), - (['y', 'first', 'second'], ['std::pair', 'int', 'int'], - 'x', ['y']), - ([['x', 'item'], 'y', 'val'], ['std::map', 'int', 'int'], 'x', - [None, 'y']), - ([['x', 'ytem'], 'y', 'val'], ['std::map', 'int', 'int'], 'x', - [['x', 'ytem'], 'y']), - ([['x', 'ytem'], ['key', 'f', 's'], ['data', 'val']], - ['std::map', ['std::pair', 'int', 'int'], ['std::set', 'int']], - 'x', [[None, 'ytem'], [None, 'f', 's'], 'data']), - ] - for exp, t, name, alias in cases: - obs = f.canonize_alias(t, name, alias=alias) - assert exp == obs - -def test_vdeclarfilter_canonize_ui(): - m = MockMachine() - f = VarDeclarationFilter(m) - cases = [ - # exp type name uilabel/tooltip - ('foo', 'float', 'x', 'foo'), - (['foo', ''], ['std::set', 'float'], 'x', 'foo'), - (['foo', 'bar'], ['std::set', 'float'], 'x', ['foo', 'bar']), - ([['foo', ''], ['', '', ''], ['', '']], - ['std::map', ['std::pair', 'int', 'int'], ['std::set', 'int']], - 'x', - 'foo'), - ([['foo', ''], ['bar', '', ''], ['', '']], - ['std::map', ['std::pair', 'int', 'int'], ['std::set', 'int']], - 'x', - ['foo', 'bar']), - ([['foo', ''], ['bar', '', ''], ['baz', '']], - ['std::map', ['std::pair', 'int', 'int'], ['std::set', 'int']], - 'x', - ['foo', 'bar', 'baz']), - ] - for exp, t, name, x in cases: - obs = f.canonize_uilabel(t, name, uilabel=x) - assert exp == obs - obs = f.canonize_tooltip(t, name, tooltip=x) - assert exp == obs - - -def test_execfilter(): - """Test ExecFilter""" - m = MockMachine() - f = ExecFilter(m) - assert not f.isvalid("#pragma cyclus") - - statement, sep = "#pragma cyclus exec x = 42", "\n" - assert f.isvalid(statement) - f.transform(statement, sep) - # What are the other possible tests - assert m.execns["x"] == 42 - -def test_notefilter(): - """Test NoteDecorationFilter""" - m = MockMachine() - f = NoteDecorationFilter(m) - assert not f.isvalid("#pragma cyclus") - - statement, sep = "#pragma cyclus note {'doc': 'string'} ", "\n" - assert f.isvalid(statement) - f.transform(statement, sep) - assert m.context['']['doc'] == 'string' - -class MockAliasCodeGenMachine(object): - """Mock machine for testing aliasing on pass 3 filters""" - def __init__(self): - self.depth = 0 - self.execns = {} - self.context = {"MyFactory": OrderedDict([('vars', OrderedDict([ - ('bar_var', { - 'type': 'std::string', - 'alias': 'foo_alias', - }),('foo_alias', 'bar_var'), - ('bar_map_var', { - 'type': ('std::map', 'std::string', 'int'), - 'alias': 'foo_map_alias', - }),('foo_map_alias', 'bar_map_var'), - ])) - ])} - self.statements = [] - self.classes = [] - self.superclasses = {'MyFactory': ()} - self.access = {} - self.namespaces = [] - self.using_namespaces = set() - self.aliases = set() - self.var_annotations = None - self.filters = [] - self.local_classname = "MyFactory" - - def classname(self): - return self.local_classname - - -def test_canon_type(): - sa = StateAccumulator() - cases = [ - ('double', 'double'), - ('std::string', 'std::string'), - ('std::vector', ('std::vector', 'int')), - ('std::map', ('std::map', 'int', 'cyclus::Blob')), - ('std::pair', ('std::pair', 'int', 'std::string')), - ('std::map, double>', - ('std::map', ('std::pair', 'int', 'std::string'), 'double')), - ] - for t, exp in cases: - obs = sa.canonize_type(t) - assert exp == obs - -# -# pass 3 Filters -# -class MockCodeGenMachine(object): - """Mock machine for testing pass 3 filters""" - def __init__(self): - self.depth = 0 - self.execns = {} - self.context = {"MyFactory": OrderedDict([('vars', OrderedDict([ - ('x', {'type': 'int', 'schematype': 'positiveInteger'}), - ('y', {'type': 'std::string', - 'shape': [42], - 'initfromcopy': 'y=m -> y;\n', - 'initfromdb': 'WAKKA JAWAKA', - 'infiletodb': {'read': 'THINGFISH\n', - 'write': 'ABSOLUTELY FREE\n'}, - 'schema': "FREAK OUT\n", - 'snapshot': "JUST ANOTHER BAND FROM LA\n" - }), - ])) - ])} - self.statements = [] - self.classes = [] - self.superclasses = {'MyFactory': ()} - self.access = {} - self.namespaces = [] - self.using_namespaces = set() - self.aliases = set() - self.var_annotations = None - self.filters = [] - self.local_classname = "MyFactory" - - def classname(self): - return self.local_classname - -def test_clonefilter(): - """Test CloneFilter""" - m = MockCodeGenMachine() - f = CloneFilter(m) - f.given_classname = 'MyFactory' - impl = f.impl() - exp_impl = " MyFactory* m = new MyFactory(context());\n" + \ - " m->InitFrom(this);\n return m;\n" - assert exp_impl == impl - -def test_ifcfilter(): - """Test InitFromCopyFilter""" - m = MockCodeGenMachine() - f = InitFromCopyFilter(m) - f.given_classname = 'MyFactory' - - args = f.methodargs() - exp_args = "MyFactory* m" - assert exp_args == args - - impl = f.impl() - exp_impl = (' int rawcycpp_shape_y[1] = {42};\n' - ' cycpp_shape_y = std::vector(rawcycpp_shape_y, ' - 'rawcycpp_shape_y + 1);\n' - " x = m->x;\n" - "y=m -> y;\n") - assert exp_impl == impl - -def test_ifdbfilter(): - """Test InitFromDbFilter""" - m = MockCodeGenMachine() - f = InitFromDbFilter(m) - f.given_classname = 'MyFactory' - - args = f.methodargs() - exp_args = "cyclus::QueryableBackend* b" - assert exp_args == args - - impl = f.impl() - exp_impl = (' int rawcycpp_shape_y[1] = {42};\n' - ' cycpp_shape_y = std::vector(rawcycpp_shape_y, ' - 'rawcycpp_shape_y + 1);\n' - ' cyclus::QueryResult qr = b->Query("Info", NULL);\n' - ' x = qr.GetVal("x");\n' - "WAKKA JAWAKA") - assert exp_impl == impl - -def test_aliasing_schemafilter(): - impl = setup_alias(SchemaFilter) - - assert('foo_alias' in impl) - assert not('bar_var' in impl) - assert('foo_map_alias' in impl) - assert not('bar_map_var' in impl) - -def test_aliasing_snapshotfilter(): - impl = setup_alias(SnapshotFilter) - - assert not('foo_alias' in impl) - assert('bar_var' in impl) - assert not('foo_map_alias' in impl) - assert('bar_map_var' in impl) - -def test_aliasing_infiletodbfilter(): - impl = setup_alias(InfileToDbFilter) - - assert('foo_alias' in impl) - assert('bar_var' in impl) - assert('foo_map_alias' in impl) - assert('bar_map_var' in impl) - -def setup_alias(filt): - m = MockAliasCodeGenMachine() - f = filt(m) - f.given_classname = 'MyFactory' - return f.impl() - -def test_itdbfilter(): - """Test InfileToDbFilter""" - m = MockCodeGenMachine() - f = InfileToDbFilter(m) - f.given_classname = 'MyFactory' - - args = f.methodargs() - exp_args = "cyclus::InfileTree* tree, cyclus::DbInit di" - assert exp_args == args - - impl = f.impl() - exp_impl = ( - ' int rawcycpp_shape_y[1] = {42};\n cycpp_shape_y = std::vector(rawcycpp_shape_y, rawcycpp_shape_y + 1);\n cyclus::InfileTree* sub = tree->SubTree("config/*");\n int i;\n int n;\n {\n int x_val = cyclus::Query(sub, "x");\n x = x_val;\n }\nTHINGFISH\n di.NewDatum("Info")\n ->AddVal("x", x)\nABSOLUTELY FREE\n ->Record();\n' - ) - - assert exp_impl == impl - -def test_itdbfilter_val(): - """Test InfileToDbFilter._val() Defaults""" - m = MockCodeGenMachine() - f = InfileToDbFilter(m) - - cases = [ - ('bool', True, 'foo', None, 'bool foo = true;\n'), - ('bool', False, 'foo', None, 'bool foo = false;\n'), - ('int', 42, 'foo', None, 'int foo = 42;\n'), - ('int', 92235, 'foo', 'nuclide', 'int foo = pyne::nucname::id(92235);\n'), - ('int', 'U-235', 'foo', 'nuclide', 'int foo = pyne::nucname::id("U-235");\n'), - ('float', 42.0, 'foo', None, 'double foo = 42.0;\n'), - ('double', 42.0, 'foo', None, 'double foo = 42.0;\n'), - ('std::string', 'wakka', 'foo', None, 'std::string foo("wakka");\n'), - ('cyclus::Blob', 'wakka', 'foo', None, 'cyclus::Blob foo("wakka");\n'), - ('boost::uuids::uuid', - '/#\xfb\xaf\x90\xc9N\xe9\x98:S\xea\xd6\xd6\x0fb', 'foo', None, - 'boost::uuids::uuid foo = "/#\xfb\xaf\x90\xc9N\xe9\x98:S\xea\xd6\xd6\x0fb";\n'), - ('boost::uuids::uuid', - uuid.UUID('2f23fbaf-90c9-4ee9-983a-53ead6d60f62'), 'foo', None, - 'boost::uuids::uuid foo = {0x2f, 0xf3, 0x2b, 0x3f, 0xf0, 0xb9, 0xae, 0xf9, 0x98, 0x0a, 0xc3, 0x9a, 0x46, 0xe6, 0xef, 0x92};\n'), - (('std::vector', 'int'), [42], 'foo', None, - ('std::vector< int > foo;\n' - 'foo.resize(1);\n' - '{\n' - ' {\n' - ' int elem = 42;\n' - ' foo[0] = elem;\n' - ' }\n' - '}\n'), - ), - (('std::vector', 'int'), [92235], 'foo', [None, 'nuclide'], - ('std::vector< int > foo;\n' - 'foo.resize(1);\n' - '{\n' - ' {\n' - ' int elem = pyne::nucname::id(92235);\n' - ' foo[0] = elem;\n' - ' }\n' - '}\n'), - ), - (('std::set', 'int'), [42, 65], 'foo', None, - ('std::set< int > foo;\n' - '{\n' - ' {\n' - ' int elem = 42;\n' - ' foo.insert(elem);\n' - ' }\n' - ' {\n' - ' int elem = 65;\n' - ' foo.insert(elem);\n' - ' }\n' - '}\n'), - ), - (('std::list', 'int'), [42, 65], 'foo', None, - ('std::list< int > foo;\n' - '{\n' - ' {\n' - ' int elem = 42;\n' - ' foo.push_back(elem);\n' - ' }\n' - ' {\n' - ' int elem = 65;\n' - ' foo.push_back(elem);\n' - ' }\n' - '}\n'), - ), - (('std::pair', 'int', 'double'), [42, 65.0], 'foo', None, - ('std::pair< int, double > foo;\n' - '{\n' - ' int first = 42;\n' - ' double second = 65.0;\n' - ' foo.first = first;\n' - ' foo.second = second;\n' - '}\n'), - ), - (('std::map', 'int', 'double'), {42: 65.0}, 'foo', None, - ('std::map< int, double > foo;\n' - '{\n' - ' {\n' - ' int key = 42;\n' - ' double val = 65.0;\n' - ' foo[key] = val;\n' - ' }\n' - '}\n'), - ), - (('std::map', 'std::string', ('std::pair', 'bool', ('std::vector', 'double'))), - OrderedDict([('hello', [True, [1.1, 2.2, 3.3]]), ('goodbye', [False, [3.3, 2.2, 1.1]])]), - 'foo', None, - ('std::map< std::string, std::pair< bool, std::vector< double > > > foo;\n' - '{\n' - ' {\n' - ' std::string key("hello");\n' - ' std::pair< bool, std::vector< double > > val;\n' - ' {\n' - ' bool first = true;\n' - ' std::vector< double > second;\n' - ' second.resize(3);\n' - ' {\n' - ' {\n' - ' double elem = 1.1;\n' - ' second[0] = elem;\n' - ' }\n' - ' {\n' - ' double elem = 2.2;\n' - ' second[1] = elem;\n' - ' }\n' - ' {\n' - ' double elem = 3.3;\n' - ' second[2] = elem;\n' - ' }\n' - ' }\n' - ' val.first = first;\n' - ' val.second = second;\n' - ' }\n' - ' foo[key] = val;\n' - ' }\n' - ' {\n' - ' std::string key("goodbye");\n' - ' std::pair< bool, std::vector< double > > val;\n' - ' {\n' - ' bool first = false;\n' - ' std::vector< double > second;\n' - ' second.resize(3);\n' - ' {\n' - ' {\n' - ' double elem = 3.3;\n' - ' second[0] = elem;\n' - ' }\n' - ' {\n' - ' double elem = 2.2;\n' - ' second[1] = elem;\n' - ' }\n' - ' {\n' - ' double elem = 1.1;\n' - ' second[2] = elem;\n' - ' }\n' - ' }\n' - ' val.first = first;\n' - ' val.second = second;\n' - ' }\n' - ' foo[key] = val;\n' - ' }\n' - '}\n'), - ), - ] - - for t, v, name, uitype, exp in cases: - obs = f._val(t, val=v, name=name, uitype=uitype) - assert exp == obs - -def test_schemafilter(): - """Test SchemaFilter""" - m = MockCodeGenMachine() - f = SchemaFilter(m) - f.given_classname = 'MyFactory' - - args = f.methodargs() - exp_args = "" - assert exp_args == args - - impl = f.impl() - exp_impl = (' return ""\n' - ' "\\n"\n' - ' " \\n"\n' - ' " \\n"\n' - ' " \\n"\n' - ' " FREAK OUT\\n"\n' - ' "\\n";\n') - assert exp_impl == impl - - # schema type tests - assert 'string' == f._type('std::string') - assert 'boolean' == f._type('bool') - assert 'token' == f._type('std::string', 'token') - - m.context = {"MyFactory": OrderedDict([('vars', OrderedDict([ - ('x', {'type': ('std::map', 'int', 'double')}), - ])) - ])} - impl = f.impl() - exp_impl = ( - ' return ""\n' - ' "\\n"\n' - ' " \\n"\n' - ' " \\n"\n' - ' " \\n"\n' - ' " \\n"\n' - ' " \\n"\n' - ' " \\n"\n' - ' " \\n"\n' - ' " \\n"\n' - ' " \\n"\n' - ' " \\n"\n' - ' " \\n"\n' - ' " \\n"\n' - ' " \\n"\n' - ' " \\n"\n' - ' "\\n";\n') - assert exp_impl == impl - -def test_annotationsfilter(): - """Test SchemaFilter""" - m = MockCodeGenMachine() - f = AnnotationsFilter(m) - f.given_classname = 'MyFactory' - - args = f.methodargs() - exp_args = "" - assert exp_args == args - - impl = f.impl() - assert isinstance(impl, str) - -def test_snapshotfilter(): - """Test SnapshotFilter""" - m = MockCodeGenMachine() - f = SnapshotFilter(m) - f.given_classname = 'MyFactory' - - args = f.methodargs() - exp_args = 'cyclus::DbInit di' - assert exp_args == args - - impl = f.impl() - exp_impl = (' di.NewDatum("Info")\n' - ' ->AddVal("x", x)\n' - 'JUST ANOTHER BAND FROM LA\n' - ' ->Record();\n') - assert exp_impl == impl - -def test_sshinvfilter(): - """Test SnapshotInvFilter""" - m = MockCodeGenMachine() - f = SnapshotInvFilter(m) - f.given_classname = 'MyFactory' - - args = f.methodargs() - exp_args = '' - assert exp_args == args - - impl = f.impl() - exp_impl = (" cyclus::Inventories invs;\n" - " return invs;\n") - assert exp_impl == impl - - f = SnapshotInvFilter(m) - f.given_classname = 'MyFactory' - f.mode = 'impl' - impl = f.impl() - exp_impl = (" cyclus::Inventories invs;\n") - assert exp_impl == impl - -def test_intinvfilter(): - """Test InitInvFilter""" - m = MockCodeGenMachine() - f = InitInvFilter(m) - f.given_classname = 'MyFactory' - - args = f.methodargs() - exp_args = "cyclus::Inventories& inv" - assert exp_args == args - - impl = f.impl() - exp_impl = '' - assert exp_impl == impl - -def test_defpragmafilter(): - """Test DefaultPragmaFilter""" - m = MockCodeGenMachine() - f = DefaultPragmaFilter(m) - -def test_schemafilter_buildschema(): - m = MockCodeGenMachine() - m.context = {"MyFactory": OrderedDict([('vars', OrderedDict([ - ('x', {'type': 'int', 'uitype': 'nuclide'}), - ])) - ])} - f = SchemaFilter(m) - - schematype = None - uitype = None - names = None - - cpptype = ['std::map', 'std::string', ['std::vector', 'double']] - names = ['streams'] - exp = ('' - '' - '' - '' - '') - obs = f._buildschema(cpptype, schematype, uitype, names) - assert exp == obs - - cpptype = ['std::map', 'std::string', ['std::vector', 'double']] - names = ['streams', 'name', ['efficiencies', 'val']] - exp = ('' - '' - '' - '' - '') - obs = f._buildschema(cpptype, schematype, uitype, names) - assert exp == obs - - # test item aliasing - cpptype = ['std::map', 'std::string', ['std::vector', 'double']] - names = [['streams', 'entry'], 'name', ['efficiencies', 'val']] - exp = ('' - '' - '' - '' - '') - obs = f._buildschema(cpptype, schematype, uitype, names) - assert exp == obs - -def test_escape_xml(): - """Test escape_xml""" - xml = '' \ - '' \ - '' - got = cycpp.escape_xml(xml) - - s = ' "\\n"\n' \ - ' " \\n"\n' \ - ' " \\n"\n' \ - ' " \\n"\n' \ - ' " \\n"\n' \ - ' " \\n"\n' \ - ' " \\n"\n' \ - ' "\\n"' - - assert s == got - -def test_infiletodb_read_member1(): - m = MockCodeGenMachine() - m.context = {"MyFactory": OrderedDict([('vars', OrderedDict([ - ('x', {'type': 'int', 'uitype': 'nuclide'}), - ])) - ])} - f = InfileToDbFilter(m) - - cpptype = ('std::map', 'std::string', ('std::vector', - ('std::vector', ('std::pair', 'double', - ('std::pair', 'int', ('std::list', ('std::set', 'bool'))))))) - alias = ['streams', 'name', ['efficiencies', 'val']] - gen = f.read_member('mymap', alias, cpptype, uitype=None) - exp_gen = ( - ' std::map< std::string, std::vector< std::vector< std::pair< double, std::pair< int, std::list< std::set< bool > > > > > > > mymap;\n' - ' {\n' - ' cyclus::InfileTree* bub = sub->SubTree("streams", 0);\n' - ' cyclus::InfileTree* sub = bub;\n' - ' int n1 = sub->NMatches("item");\n' - ' std::map< std::string, std::vector< std::vector< std::pair< double, std::pair< int, std::list< std::set< bool > > > > > > > mymap_in;\n' - ' for (int i1 = 0; i1 < n1; ++i1) {\n' - ' std::string key;\n' - ' {\n' - ' std::string key_in = cyclus::Query(sub, "item/name", i1);\n' - ' key = key_in;\n' - ' }\n' - ' std::vector< std::vector< std::pair< double, std::pair< int, std::list< std::set< bool > > > > > > val;\n' - ' {\n' - ' cyclus::InfileTree* bub = sub->SubTree("item/efficiencies", i1);\n' - ' cyclus::InfileTree* sub = bub;\n' - ' int n2 = sub->NMatches("val");\n' - ' std::vector< std::vector< std::pair< double, std::pair< int, std::list< std::set< bool > > > > > > val_in;\n' - ' val_in.resize(n2);\n' - ' for (int i2 = 0; i2 < n2; ++i2) {\n' - ' std::vector< std::pair< double, std::pair< int, std::list< std::set< bool > > > > > elem;\n' - ' {\n' - ' cyclus::InfileTree* bub = sub->SubTree("val", i2);\n' - ' cyclus::InfileTree* sub = bub;\n' - ' int n3 = sub->NMatches("val");\n' - ' std::vector< std::pair< double, std::pair< int, std::list< std::set< bool > > > > > elem_in;\n' - ' elem_in.resize(n3);\n' - ' for (int i3 = 0; i3 < n3; ++i3) {\n' - ' std::pair< double, std::pair< int, std::list< std::set< bool > > > > elem;\n' - ' {\n' - ' cyclus::InfileTree* bub = sub->SubTree("val", i3);\n' - ' cyclus::InfileTree* sub = bub;\n' - ' double firsti3;\n' - ' {\n' - ' double firsti3_in = cyclus::Query(sub, "first", 0);\n' - ' firsti3 = firsti3_in;\n' - ' }\n' - ' std::pair< int, std::list< std::set< bool > > > secondi3;\n' - ' {\n' - ' cyclus::InfileTree* bub = sub->SubTree("second", 0);\n' - ' cyclus::InfileTree* sub = bub;\n' - ' int first0;\n' - ' {\n' - ' int first0_in = cyclus::Query(sub, "first", 0);\n' - ' first0 = first0_in;\n' - ' }\n' - ' std::list< std::set< bool > > second0;\n' - ' {\n' - ' cyclus::InfileTree* bub = sub->SubTree("second", 0);\n' - ' cyclus::InfileTree* sub = bub;\n' - ' int n4 = sub->NMatches("val");\n' - ' std::list< std::set< bool > > second0_in;\n' - ' for (int i4 = 0; i4 < n4; ++i4) {\n' - ' std::set< bool > elem;\n' - ' {\n' - ' cyclus::InfileTree* bub = sub->SubTree("val", i4);\n' - ' cyclus::InfileTree* sub = bub;\n' - ' int n5 = sub->NMatches("val");\n' - ' std::set< bool > elem_in;\n' - ' for (int i5 = 0; i5 < n5; ++i5) {\n' - ' bool elem;\n' - ' {\n' - ' bool elem_in = cyclus::Query(sub, "val", i5);\n' - ' elem = elem_in;\n' - ' }\n' - ' elem_in.insert(elem);\n' - ' }\n' - ' elem = elem_in;\n' - ' }\n' - ' second0_in.push_back(elem);\n' - ' }\n' - ' second0 = second0_in;\n' - ' }\n' - ' std::pair< int, std::list< std::set< bool > > > secondi3_in(first0, second0);\n' - ' secondi3 = secondi3_in;\n' - ' }\n' - ' std::pair< double, std::pair< int, std::list< std::set< bool > > > > elem_in(firsti3, secondi3);\n' - ' elem = elem_in;\n' - ' }\n' - ' elem_in[i3] = elem;\n' - ' }\n' - ' elem = elem_in;\n' - ' }\n' - ' val_in[i2] = elem;\n' - ' }\n' - ' val = val_in;\n' - ' }\n' - ' mymap_in[key] = val;\n' - ' }\n' - ' mymap = mymap_in;\n' - ' }\n') - - ## useful for debugging test failures - #print(gen) - #print(exp_gen) - - assert exp_gen == gen - -def test_infiletodb_read_member2(): - m = MockCodeGenMachine() - m.context = {"MyFactory": OrderedDict([('vars', OrderedDict([ - ('x', {'type': 'int', 'uitype': 'nuclide'}), - ])) - ])} - f = InfileToDbFilter(m) - - alias = ['map', 'int', ['vector', ['pair', 'str1', 'str2']]] - cpptype = ('std::map', 'int', ('std::vector', ('std::pair', 'std::string', 'std::string'))) - gen = f.read_member('mymap', alias, cpptype, uitype=None) - exp_gen = ( - ' std::map< int, std::vector< std::pair< std::string, std::string > > > mymap;\n' - ' {\n' - ' cyclus::InfileTree* bub = sub->SubTree("map", 0);\n' - ' cyclus::InfileTree* sub = bub;\n' - ' int n1 = sub->NMatches("item");\n' - ' std::map< int, std::vector< std::pair< std::string, std::string > > > mymap_in;\n' - ' for (int i1 = 0; i1 < n1; ++i1) {\n' - ' int key;\n' - ' {\n' - ' int key_in = cyclus::Query(sub, "item/int", i1);\n' - ' key = key_in;\n' - ' }\n' - ' std::vector< std::pair< std::string, std::string > > val;\n' - ' {\n' - ' cyclus::InfileTree* bub = sub->SubTree("item/vector", i1);\n' - ' cyclus::InfileTree* sub = bub;\n' - ' int n2 = sub->NMatches("pair");\n' - ' std::vector< std::pair< std::string, std::string > > val_in;\n' - ' val_in.resize(n2);\n' - ' for (int i2 = 0; i2 < n2; ++i2) {\n' - ' std::pair< std::string, std::string > elem;\n' - ' {\n' - ' cyclus::InfileTree* bub = sub->SubTree("pair", i2);\n' - ' cyclus::InfileTree* sub = bub;\n' - ' std::string firsti2;\n' - ' {\n' - ' std::string firsti2_in = cyclus::Query(sub, "str1", 0);\n' - ' firsti2 = firsti2_in;\n' - ' }\n' - ' std::string secondi2;\n' - ' {\n' - ' std::string secondi2_in = cyclus::Query(sub, "str2", 0);\n' - ' secondi2 = secondi2_in;\n' - ' }\n' - ' std::pair< std::string, std::string > elem_in(firsti2, secondi2);\n' - ' elem = elem_in;\n' - ' }\n' - ' val_in[i2] = elem;\n' - ' }\n' - ' val = val_in;\n' - ' }\n' - ' mymap_in[key] = val;\n' - ' }\n' - ' mymap = mymap_in;\n' - ' }\n' - ) - # # useful for debugging test failures - # print() - # print(gen) - # print(exp_gen) - assert exp_gen == gen - -def test_infiletodb_read_map(): - m = MockCodeGenMachine() - m.context = {"MyFactory": OrderedDict([('vars', OrderedDict([ - ('x', {'type': 'int', 'uitype': 'nuclide'}), - ])) - ])} - f = InfileToDbFilter(m) - - cpptype = ('std::map', 'int', 'double') - alias = [['streams', 'entry'], 'id', 'mass'] - obs = f.read_member('mymap', alias, cpptype, uitype=None) - - exp = ( - ' std::map< int, double > mymap;\n' - ' {\n' - ' cyclus::InfileTree* bub = sub->SubTree("streams", 0);\n' - ' cyclus::InfileTree* sub = bub;\n' - ' int n1 = sub->NMatches("entry");\n' - ' std::map< int, double > mymap_in;\n' - ' for (int i1 = 0; i1 < n1; ++i1) {\n' - ' int key;\n' - ' {\n' - ' int key_in = cyclus::Query(sub, "entry/id", i1);\n' - ' key = key_in;\n' - ' }\n' - ' double val;\n' - ' {\n' - ' double val_in = cyclus::Query(sub, "entry/mass", i1);\n' - ' val = val_in;\n' - ' }\n' - ' mymap_in[key] = val;\n' - ' }\n' - ' mymap = mymap_in;\n' - ' }\n') - - assert exp == obs - -def test_internal_schema(): - cases = [ - {'params': {'internal': True, 'default': 7}, 'want': - ' return ""\n "\\n";\n' - }, - {'params': {'derived_init': 'x *= 42;'}, 'want': - (' return ""\n' - ' "\\n"\n' - ' " \\n"\n' - ' " \\n"\n' - ' " \\n"\n' - ' "\\n";\n') - }, - {'params': {'derived_init': 'x *= 42;', 'default': 7}, 'want': - (' return ""\n' - ' "\\n"\n' - ' " \\n"\n' - ' " \\n"\n' - ' " \\n"\n' - ' " \\n"\n' - ' " \\n"\n' - ' "\\n";\n') - }, - {'params': {'internal': True, 'derived_init': 'x *= 42;', 'default': 7}, 'want': - ' return ""\n "\\n";\n' - }, - ] - - for i, case in enumerate(cases): - keys = case['params'].copy() - m = MockCodeGenMachine() - params = {'type': 'int'} - params.update(keys) - m.context = {"MyFactory": OrderedDict([('vars', OrderedDict([ - ('x', params), - ]))])} - f = SchemaFilter(m) - f.given_classname = 'MyFactory' - - impl = f.impl() - - want = case['want'] - msg = 'case {0} failed\n ---- got ----\n {1}\n ---- want ----\n {2}'.format(i + 1, impl.replace('\n', '\n '), want.replace('\n', '\n ')) - if want != impl: - pprint.pprint(impl) - assert(False, msg) - -def test_internal_infiletodb(): - # the expected output (i.e. 'want':...) is set as 'throw' if the - # configuration should cause an exception. - cases = [ - {'params': {'internal': True}, 'want': 'throw'}, - {'params': {'internal': True, 'default': 7}, 'want': - (' cyclus::InfileTree* sub = tree->SubTree("config/*");\n' - ' int i;\n' - ' int n;\n' - ' int x_tmp = 7;\n' - ' x = x_tmp;\n' - ' di.NewDatum("Info")\n' - ' ->AddVal("x", x)\n' - ' ->Record();\n') - }, - {'params': {'derived_init': 'x *= 42;'}, 'want': - (' cyclus::InfileTree* sub = tree->SubTree("config/*");\n' - ' int i;\n' - ' int n;\n' - ' {\n' - ' int x_val = cyclus::Query(sub, "x");\n' - ' x = x_val;\n' - ' }\n' - ' x *= 42;\n' - ' di.NewDatum("Info")\n' - ' ->AddVal("x", x)\n' - ' ->Record();\n') - }, - {'params': {'derived_init': 'x *= 42;', 'default': 7}, 'want': - (' cyclus::InfileTree* sub = tree->SubTree("config/*");\n' - ' int i;\n' - ' int n;\n' - ' if (sub->NMatches("x") > 0) {\n' - ' {\n' - ' int x_val = cyclus::Query(sub, "x");\n' - ' x = x_val;\n' - ' }\n' - ' } else {\n' - ' int x_tmp = 7;\n' - ' x = x_tmp;\n' - ' }\n' - ' x *= 42;\n' - ' di.NewDatum("Info")\n' - ' ->AddVal("x", x)\n' - ' ->Record();\n') - }, - {'params': {'internal': True, 'derived_init': 'x *= 42;', 'default': 7}, 'want': - (' cyclus::InfileTree* sub = tree->SubTree("config/*");\n' - ' int i;\n' - ' int n;\n' - ' int x_tmp = 7;\n' - ' x = x_tmp;\n' - ' x *= 42;\n' - ' di.NewDatum("Info")\n' - ' ->AddVal("x", x)\n' - ' ->Record();\n') - }, - ] - - for i, case in enumerate(cases): - keys = case['params'].copy() - m = MockCodeGenMachine() - params = {'type': 'int'} - params.update(keys) - m.context = {"MyFactory": OrderedDict([('vars', OrderedDict([ - ('x', params), - ]))])} - f = InfileToDbFilter(m) - f.given_classname = 'MyFactory' - - want = case['want'] - - impl = '' - if want == 'throw': - haderr = False - try: - impl = f.impl() - except: - haderr = True - msg = 'case {0} failed: expected raised exception, got none.' - assert(haderr, msg) - continue - else: - impl = f.impl() - - msg = 'case {0} failed\n ---- got ----\n {1}\n ---- want ----\n {2}'.format(i + 1, impl.replace('\n', '\n '), want.replace('\n', '\n ')) - if want != impl: - pprint.pprint(impl) - assert(False, msg) - -def test_nuclide_uitype(): - m = MockCodeGenMachine() - m.context = {"MyFactory": OrderedDict([('vars', OrderedDict([ - ('x', {'type': 'int', 'uitype': 'nuclide'}), - ])) - ])} - - # test schema is set to string - f = SchemaFilter(m) - f.given_classname = 'MyFactory' - impl = f.impl() - exp_impl = (' return ""\n' - ' "\\n"\n' - ' " \\n"\n' - ' " \\n"\n' - ' " \\n"\n' - ' "\\n";\n') - assert exp_impl == impl - - # test infiletodb updates - f = InfileToDbFilter(m) - f.given_classname = 'MyFactory' - impl = f.impl() - exp_impl = ' cyclus::InfileTree* sub = tree->SubTree("config/*");\n int i;\n int n;\n {\n int x_val = pyne::nucname::id(cyclus::Query(sub, "x"));\n x = x_val;\n }\n di.NewDatum("Info")\n ->AddVal("x", x)\n ->Record();\n' - - assert exp_impl == impl - - # test bad uitypes values fail - m.context = {"MyFactory": OrderedDict([('vars', OrderedDict([ - ('x', {'type': 'int', 'uitype': 'WRONG'}), - ])) - ])} - f = SchemaFilter(m) - f.given_classname = 'MyFactory' - with pytest.raises(TypeError): - f.impl() - -def test_integration(): - inf = os.path.join(os.path.dirname(__file__), 'cycpp_tests.h') - outf = tempfile.NamedTemporaryFile() - cmd = "" - # if CXX is set use it, fallback on g++ otherwise - if os.getenv("CXX") is not None: - cmd = 'cycpp.py {} -o {} --cpp-path `which $CXX`'.format(inf, outf.name) - else: - cmd = 'cycpp.py {} -o {} --cpp-path `which g++`'.format(inf, outf.name) - p = Popen(cmd, shell=True, stdin=PIPE, stdout=PIPE, stderr=STDOUT, close_fds=True) - assert '' == p.stdout.read().decode() - From 1f889fe57345f13450b797ed1517da27ecd2989a Mon Sep 17 00:00:00 2001 From: abachma2 Date: Tue, 28 Mar 2023 16:52:25 -0500 Subject: [PATCH 006/102] rename to be picked up by pytest Similar to the cycpp_tests.py, this test file wasn't picked up by pytest because it didn't recognize the *_tests.py file name, it had to be *_test.py --- tests/hdf5_back_gen_tests.py | 276 ----------------------------------- 1 file changed, 276 deletions(-) delete mode 100644 tests/hdf5_back_gen_tests.py diff --git a/tests/hdf5_back_gen_tests.py b/tests/hdf5_back_gen_tests.py deleted file mode 100644 index 6b2850ed21..0000000000 --- a/tests/hdf5_back_gen_tests.py +++ /dev/null @@ -1,276 +0,0 @@ -import os -import sys -import json -import subprocess -from random import randint -import uuid -import pandas as pd -from pandas.util.testing import assert_frame_equal -import pytest - -from cyclus.lib import Hdf5Back, Recorder -import cyclus.typesystem as ts - -cycdir = os.path.dirname(os.path.dirname(__file__)) -sys.path.insert(0, os.path.join(cycdir, 'src')) -import hdf5_back_gen -#from hdf5_back_gen import resolve_unicode, convert_canonical, setup, ORIGIN_DICT - -#Call to hdf5_back_gen function -hdf5_back_gen.setup() - -is_primitive = lambda canon: isinstance(canon, str) - -CHARS = r'abcdefghijklmnopqrstuv1234567890`/\_.,-+@$#!% ' - -CANON_TYPES = [] -CANON_TO_DB = {} -CANON_TO_VL = {} -def setup(): - with open(os.path.join(os.path.dirname(__file__), '..', 'share', - 'dbtypes.json')) as f: - RAW_TABLE = hdf5_back_gen.resolve_unicode(json.load(f)) - - VERSION = "" - TABLE_START = 0 - TABLE_END = 0 - for row in range(len(RAW_TABLE)): - current = tuple(RAW_TABLE[row]) - if current[4] == "HDF5": - if current[5] > VERSION: - VERSION = current[5] - TABLE_START = row - if current[5] == VERSION: - TABLE_END = row - - TYPES_TABLE = list(tuple(row) for row in RAW_TABLE[TABLE_START:TABLE_END+1]) - for row in TYPES_TABLE: - if row[6] == 1 and row[4] == "HDF5" and row[5] == VERSION: - db = row[1] - is_vl = row[8] - canon = hdf5_back_gen.convert_canonical(row[7]) - if canon not in CANON_TYPES: - CANON_TYPES.append(canon) - CANON_TO_DB[canon] = db - CANON_TO_VL[canon] = is_vl - -def make_bytes(string): - return string.encode() - -VARIABLE_LENGTH_TYPES = ['LIST', 'VECTOR', 'MAP', 'SET', 'STRING'] - -CONTAINER_TYPES = {'LIST': 'LIST', 'VL_LIST': 'LIST', - 'VECTOR': 'VECTOR', 'VL_VECTOR': 'VECTOR', - 'MAP': 'MAP', 'VL_MAP': 'MAP', - 'SET': 'SET', 'VL_SET': 'SET', - 'PAIR': 'PAIR'} - -CREATE_FUNCTIONS = {'MAP': dict, 'LIST': list, 'SET': set, 'PAIR': list, - 'VECTOR': list, 'INT': int, 'DOUBLE': float, 'FLOAT': float, - 'BOOL': bool, 'STRING': str, 'UUID': list, 'BLOB': str} - -UNIQUE_TYPES = ['MAP', 'SET'] - -TYPE_SHAPE = 0 -TYPE_FUNCTION = 1 -TYPE_CANON = 2 -CONTAINER_MIN = 4 -CONTAINER_MAX = 8 -def generate_meta(canon, depth=0): - """Produces metadata about a type to be created. - - This includes a shape, constructor function, and canonical form. This - 3-tuple can be passed to the populate function to create data in the form - specified by the metadata. - - Parameters - ---------- - canon : tuple or str - Unique identifier of current data type - depth : int, optional - Recursive depth value - - Returns - ------- - meta_shape : tuple or list - Meta data specification for type - """ - meta_shape = [] - my_shape = None - my_type = None - origin = hdf5_back_gen.ORIGIN_DICT[canon] - if is_primitive(canon): - if CANON_TO_VL[canon]: - my_shape = -1 - elif origin in VARIABLE_LENGTH_TYPES: - my_shape = randint(CONTAINER_MIN, CONTAINER_MAX) - else: - my_shape = -1 - my_type = CREATE_FUNCTIONS[origin] - return (my_shape, my_type, canon) - elif CANON_TO_VL[canon]: - my_shape = -1 - my_type = CREATE_FUNCTIONS[origin[0]] - elif origin[0] in VARIABLE_LENGTH_TYPES: - my_shape = randint(CONTAINER_MIN, CONTAINER_MAX) - my_type = CREATE_FUNCTIONS[origin[0]] - else: - my_shape = -1 - my_type = CREATE_FUNCTIONS[origin[0]] - meta_shape.append((my_shape, my_type, canon)) - for i in canon[1:]: - meta_shape.append(generate_meta(i, depth=depth+1)) - return meta_shape - -def add_item(container, value): - """Attempts to add a value to a container of unknown type. - - Parameters - ---------- - container : iterable - The type into which the value is inserted - value : type - Data point to insert - - Returns - ------- - container : iterable - The container with added value - """ - if len(value) == 1: - try: - container.append(value[0]) - except AttributeError: - try: - container.add(value[0]) - except AttributeError: - print("Unsure how to add " + value[0] + " of type " - + str(type(value[0])) + " to " + str(type(container))) - sys.exit(1) - else: - try: - container.update({value[0]: value[1]}) - except AttributeError: - try: - container = (value[0], value[1]) - except AttributeError: - print("Unsure how to add " + str(value[0]) + " of type " - + str(type(value[0])) + " and " + str(value[1]) - + " of type " + str(type(value[1])) + " to " - + str(type(container))) - sys.exit(1) - return container - -def populate(meta): - """Generate data based upon the metadata for a type. - - Uses the shape values within meta to create data with the correct dimensions - for adding to the table. - - Parameters - ---------- - meta : tuple or list - Metadata for type - - Returns - ------- - data : type specified in meta - Populated container or primitive - """ - if isinstance(meta, tuple): - current_type = meta - else: - current_type = meta[0] - my_shape = current_type[TYPE_SHAPE] - my_type = current_type[TYPE_FUNCTION] - canon = current_type[TYPE_CANON] - data = my_type() - origin = hdf5_back_gen.ORIGIN_DICT[canon] - if is_primitive(canon): - if origin == 'STRING' or origin == 'BLOB': - if my_shape > 0: - for i in range(my_shape): - data += CHARS[randint(0, len(CHARS)-1)] - else: - for i in range(randint(CONTAINER_MIN, CONTAINER_MAX)): - data += CHARS[randint(0, len(CHARS)-1)] - if origin == 'BLOB': - data = make_bytes(data) - elif canon == 'BOOL': - data = bool(randint(0,1)) - elif canon == 'UUID': - data = uuid.uuid5(uuid.NAMESPACE_DNS, CHARS[randint(0,len(CHARS)-1)]) - else: - data = my_type(randint(0, 9) * (data + 1)) - elif origin[0] in VARIABLE_LENGTH_TYPES: - if my_shape > 0: - item_count = my_shape - else: - item_count = randint(CONTAINER_MIN, CONTAINER_MAX) - add_count = 0 - while add_count < item_count: - #reset our current child - child_data = [] - #fill with child values - for j in meta[1:]: - child_data.append(populate(j)) - if origin[0] in UNIQUE_TYPES and child_data[0] in data: - continue - else: - #add to parent - data = add_item(data, child_data) - add_count += 1 - else: - #pairs, etc. - child_data = [] - for i in meta[1:]: - child_data.append(populate(i)) - data = add_item(data, child_data) - return data - -def get_shape(meta): - """Make shape into a flat int list for the Hdf5 backend.""" - shape = [] - if isinstance(meta, tuple): - return [meta[TYPE_SHAPE]] - else: - current_type = meta[0] - shape.append(current_type[TYPE_SHAPE]) - for i in meta[1:]: - shape.extend(get_shape(i)) - return shape - -ROW_NUM = 3 -PATH = 'gen_db.h5' -def generate_and_test(): - """Generate and run tests for supported Hdf5 datatypes.""" - if sys.version_info[0] == 2: - msg = 'Hdf5 backend gen tests do not support Python 2.x' - pytest.skip(msg) - if os.path.isfile(PATH): - os.remove(PATH) - for i in CANON_TYPES: - print(CANON_TO_DB[i],'\n') - rec = Recorder(inject_sim_id=False) - back = Hdf5Back(PATH) - rec.register_backend(back) - data_meta = generate_meta(i) - shape = get_shape(data_meta) - print("shape: ", shape) - data = [] - for j in range(ROW_NUM): - data.append(populate(data_meta)) - exp = pd.DataFrame({'col0': data}, columns=['col0']) - print("expected: \n", exp) - for j in data: - d = rec.new_datum("test0") - d.add_val("col0", j, shape=shape, type=ts.IDS[CANON_TO_DB[i]]) - d.record() - rec.flush() - obs = back.query("test0") - print("observed: \n", obs) - assert_frame_equal, exp, obs - rec.close() - os.remove(PATH) - - From 5468bd2c432dc4b2e080831b125557dd68d0916f Mon Sep 17 00:00:00 2001 From: abachma2 Date: Tue, 28 Mar 2023 16:56:11 -0500 Subject: [PATCH 007/102] add files for renaming Add the files that the previous commits were trying to change the file names to, so that the tests would be picked up by pytest --- tests/cycpp_test.py | 1194 +++++++++++++++++++++++++++++++++++ tests/hdf5_back_gen_test.py | 276 ++++++++ 2 files changed, 1470 insertions(+) create mode 100644 tests/cycpp_test.py create mode 100644 tests/hdf5_back_gen_test.py diff --git a/tests/cycpp_test.py b/tests/cycpp_test.py new file mode 100644 index 0000000000..65083f86d3 --- /dev/null +++ b/tests/cycpp_test.py @@ -0,0 +1,1194 @@ +import os +import sys +import uuid +import pprint +import tempfile +import pytest + +from collections import OrderedDict +from subprocess import Popen, PIPE, STDOUT + + +cycdir = os.path.dirname(os.path.dirname(__file__)) +sys.path.insert(0, os.path.join(cycdir, 'cli')) + +# pass 1 Filters +from cycpp import NamespaceFilter, TypedefFilter, UsingFilter,\ + UsingNamespaceFilter, NamespaceAliasFilter, ClassFilter, \ + AccessFilter, PragmaCyclusErrorFilter + +# pass 2 Filters +from cycpp import VarDecorationFilter, VarDeclarationFilter, ExecFilter, \ + NoteDecorationFilter, StateAccumulator + +# pass 3 Filters +from cycpp import CloneFilter, InitFromCopyFilter, \ + InitFromDbFilter, InfileToDbFilter, SchemaFilter, SnapshotFilter, \ + SnapshotInvFilter, InitInvFilter, DefaultPragmaFilter, AnnotationsFilter + +import cycpp + +# assert_equal.__self__.maxDiff = None !! this may not be necessary for pytest??? + +class MockMachine(object): + def __init__(self): + self.depth = 0 + self.execns = {} + self.context = {} + self.statements = [] + self.classes = [] + self.superclasses = {} + self.access = {} + self.namespaces = [] + self.using_namespaces = set() + self.aliases = set() + self.var_annotations = None + self.filters = [] + + def includeloc(self, statement=None): + return "" + + def classname(self): + """Implemented just for testing""" + return "" + + def ensure_class_context(self, classname): + if classname not in self.context: + self.context[classname] = OrderedDict() + + +# +# pass 1 Filters +# + +def test_tffilt(): + """Test TypedefFilter""" + m = MockMachine() + f = TypedefFilter(m) + assert not f.isvalid("mis typedef kind") + assert not f.isvalid("typedef kind") + + statement, sep = "typedef double db", ";" + assert f.isvalid(statement) + f.transform(statement, sep) + assert len(m.aliases) == 1 + assert (0, "double", "db") == m.aliases.pop() + + statement, sep = "typedef struct {int a; int b;} S, *pS", ";" + assert f.isvalid(statement) + +def test_uffilt(): + """Test UsingFilter""" + m = MockMachine() + f = UsingFilter(m) + assert not f.isvalid("not using namespace") + + statement, sep = "using std::cout", "" + assert f.isvalid(statement) + f.transform(statement, sep) + assert len(m.aliases) == 1 + assert (0, "std::cout", "cout") == m.aliases.pop() + +def test_nsfilt(): + """Test NamespaceFilter""" + m = MockMachine() + f = NamespaceFilter(m) + assert not f.isvalid("olzhas is not a namespace") + + # anonymous namespaces + statement, sep = " namespace ", "{" + assert f.isvalid(statement) + f.transform(statement, sep) + assert len(m.namespaces) == 1 + assert m.namespaces[0] == (0, '') + f.revert(statement, sep) + assert len(m.namespaces) == 0 + + # nymous namespace + statement, sep = "namespace gorgus ", "{" + assert f.isvalid(statement) + f.transform(statement, sep) + assert len(m.namespaces) == 1 + assert m.namespaces[0] == (0, "gorgus") + f.revert(statement, sep) + assert len(m.namespaces) == 0 + +def test_unfilt(): + """Test UsingNamespaseFilter""" + m = MockMachine() + f = UsingNamespaceFilter(m) + assert not f.isvalid("using cycamore") + + statement, sep = "using namespace std", "" + assert f.isvalid(statement) + f.transform(statement, sep) + assert len(m.using_namespaces) == 1 + assert (0, "std") == m.using_namespaces.pop() + f.revert(statement, sep) + assert len(m.using_namespaces) == 0 + +def test_nafilter(): + """Test NamespaceAliasFilter""" + m = MockMachine() + f = NamespaceAliasFilter(m) + assert not f.isvalid("namespace cycamore") + + statement, sep = "namespace cycamore = cm", "" + assert f.isvalid(statement) + f.transform(statement, sep) + assert len(m.aliases) == 1 + assert (0, "cm", "cycamore") == m.aliases.pop() + +def test_cfilter(): + """Test ClassFilter""" + m = MockMachine() + f = ClassFilter(m) + assert not f.isvalid("class ") + + statement, sep = "class Cyclus", "" + assert f.isvalid(statement) + f.transform(statement, sep) + assert len(m.classes) == 1 + assert m.classes[0] == (0, "Cyclus") + assert m.access[tuple(m.classes)] == "private" + f.revert(statement, sep) + assert len(m.classes) == 0 + +def test_afilter(): + """Test AccessFilter""" + m = MockMachine() + f = AccessFilter(m) + assert not f.isvalid("new private") + + statement, sep = "private:", "" + assert f.isvalid(statement) + f.transform(statement, sep) + assert m.access[tuple(m.classes)] == "private" + +def test_synerror(): + """Test PragmaCyclusErrorFilter""" + m = MockMachine() + f = PragmaCyclusErrorFilter(m) + assert not f.isvalid("#pragma cyclus var {}") + assert not f.isvalid("#pragma cyclus") + + assert f.isvalid('#pragma cyclus nooooo') + statement, sep = "#pragma cyclus var{}", "\n" + assert f.isvalid(statement) + with pytest.raises(SyntaxError): + f.transform(statement, sep) + +# +# pass 2 Filters +# +def test_vdecorfilter(): + """Test VarDecorationFilter""" + m = MockMachine() + f = VarDecorationFilter(m) + assert not f.isvalid("#pragma cyclus") + + statement, sep = "#pragma cyclus var {'name': 'James Bond'} ", "\n" + assert f.isvalid(statement) + f.transform(statement, sep) + assert m.var_annotations == {'name': 'James Bond'} + +def test_vdeclarfilter(): + """Test VarDeclarationFilter""" + m = MockMachine() + f = VarDeclarationFilter(m) + assert not f.isvalid("one ") + + statement, sep = "one two", "\n" + assert f.isvalid(statement) + m.classes = [(0, "trader")] + m.access = {"trader": "public"} + # m.var_annotations = {'name': 'James Bond'} + f.transform(statement, sep) + assert m.var_annotations == None + +def test_vdeclarfilter_canonize_alias(): + m = MockMachine() + f = VarDeclarationFilter(m) + cases = [ + # exp type name alias + ('x', 'float', 'x', None), + (['x', 'val'], ['std::set', 'float'], 'x', None), + (['x', 'first', 'second'], ['std::pair', 'int', 'int'], 'x', None), + ([['x', 'item'], 'key', 'val'], ['std::map', 'int', 'int'], 'x', None), + ([['x', 'item'], ['key', 'first', 'second'], ['val', 'val']], + ['std::map', ['std::pair', 'int', 'int'], ['std::set', 'int']], + 'x', None), + ('y', 'float', 'x', 'y'), + (['x', 'y'], ['std::set', 'float'], 'x', [None, 'y']), + (['y', 'first', 'second'], ['std::pair', 'int', 'int'], + 'x', ['y']), + ([['x', 'item'], 'y', 'val'], ['std::map', 'int', 'int'], 'x', + [None, 'y']), + ([['x', 'ytem'], 'y', 'val'], ['std::map', 'int', 'int'], 'x', + [['x', 'ytem'], 'y']), + ([['x', 'ytem'], ['key', 'f', 's'], ['data', 'val']], + ['std::map', ['std::pair', 'int', 'int'], ['std::set', 'int']], + 'x', [[None, 'ytem'], [None, 'f', 's'], 'data']), + ] + for exp, t, name, alias in cases: + obs = f.canonize_alias(t, name, alias=alias) + assert exp == obs + +def test_vdeclarfilter_canonize_ui(): + m = MockMachine() + f = VarDeclarationFilter(m) + cases = [ + # exp type name uilabel/tooltip + ('foo', 'float', 'x', 'foo'), + (['foo', ''], ['std::set', 'float'], 'x', 'foo'), + (['foo', 'bar'], ['std::set', 'float'], 'x', ['foo', 'bar']), + ([['foo', ''], ['', '', ''], ['', '']], + ['std::map', ['std::pair', 'int', 'int'], ['std::set', 'int']], + 'x', + 'foo'), + ([['foo', ''], ['bar', '', ''], ['', '']], + ['std::map', ['std::pair', 'int', 'int'], ['std::set', 'int']], + 'x', + ['foo', 'bar']), + ([['foo', ''], ['bar', '', ''], ['baz', '']], + ['std::map', ['std::pair', 'int', 'int'], ['std::set', 'int']], + 'x', + ['foo', 'bar', 'baz']), + ] + for exp, t, name, x in cases: + obs = f.canonize_uilabel(t, name, uilabel=x) + assert exp == obs + obs = f.canonize_tooltip(t, name, tooltip=x) + assert exp == obs + + +def test_execfilter(): + """Test ExecFilter""" + m = MockMachine() + f = ExecFilter(m) + assert not f.isvalid("#pragma cyclus") + + statement, sep = "#pragma cyclus exec x = 42", "\n" + assert f.isvalid(statement) + f.transform(statement, sep) + # What are the other possible tests + assert m.execns["x"] == 42 + +def test_notefilter(): + """Test NoteDecorationFilter""" + m = MockMachine() + f = NoteDecorationFilter(m) + assert not f.isvalid("#pragma cyclus") + + statement, sep = "#pragma cyclus note {'doc': 'string'} ", "\n" + assert f.isvalid(statement) + f.transform(statement, sep) + assert m.context['']['doc'] == 'string' + +class MockAliasCodeGenMachine(object): + """Mock machine for testing aliasing on pass 3 filters""" + def __init__(self): + self.depth = 0 + self.execns = {} + self.context = {"MyFactory": OrderedDict([('vars', OrderedDict([ + ('bar_var', { + 'type': 'std::string', + 'alias': 'foo_alias', + }),('foo_alias', 'bar_var'), + ('bar_map_var', { + 'type': ('std::map', 'std::string', 'int'), + 'alias': 'foo_map_alias', + }),('foo_map_alias', 'bar_map_var'), + ])) + ])} + self.statements = [] + self.classes = [] + self.superclasses = {'MyFactory': ()} + self.access = {} + self.namespaces = [] + self.using_namespaces = set() + self.aliases = set() + self.var_annotations = None + self.filters = [] + self.local_classname = "MyFactory" + + def classname(self): + return self.local_classname + + +def test_canon_type(): + sa = StateAccumulator() + cases = [ + ('double', 'double'), + ('std::string', 'std::string'), + ('std::vector', ('std::vector', 'int')), + ('std::map', ('std::map', 'int', 'cyclus::Blob')), + ('std::pair', ('std::pair', 'int', 'std::string')), + ('std::map, double>', + ('std::map', ('std::pair', 'int', 'std::string'), 'double')), + ] + for t, exp in cases: + obs = sa.canonize_type(t) + assert exp == obs + +# +# pass 3 Filters +# +class MockCodeGenMachine(object): + """Mock machine for testing pass 3 filters""" + def __init__(self): + self.depth = 0 + self.execns = {} + self.context = {"MyFactory": OrderedDict([('vars', OrderedDict([ + ('x', {'type': 'int', 'schematype': 'positiveInteger'}), + ('y', {'type': 'std::string', + 'shape': [42], + 'initfromcopy': 'y=m -> y;\n', + 'initfromdb': 'WAKKA JAWAKA', + 'infiletodb': {'read': 'THINGFISH\n', + 'write': 'ABSOLUTELY FREE\n'}, + 'schema': "FREAK OUT\n", + 'snapshot': "JUST ANOTHER BAND FROM LA\n" + }), + ])) + ])} + self.statements = [] + self.classes = [] + self.superclasses = {'MyFactory': ()} + self.access = {} + self.namespaces = [] + self.using_namespaces = set() + self.aliases = set() + self.var_annotations = None + self.filters = [] + self.local_classname = "MyFactory" + + def classname(self): + return self.local_classname + +def test_clonefilter(): + """Test CloneFilter""" + m = MockCodeGenMachine() + f = CloneFilter(m) + f.given_classname = 'MyFactory' + impl = f.impl() + exp_impl = " MyFactory* m = new MyFactory(context());\n" + \ + " m->InitFrom(this);\n return m;\n" + assert exp_impl == impl + +def test_ifcfilter(): + """Test InitFromCopyFilter""" + m = MockCodeGenMachine() + f = InitFromCopyFilter(m) + f.given_classname = 'MyFactory' + + args = f.methodargs() + exp_args = "MyFactory* m" + assert exp_args == args + + impl = f.impl() + exp_impl = (' int rawcycpp_shape_y[1] = {42};\n' + ' cycpp_shape_y = std::vector(rawcycpp_shape_y, ' + 'rawcycpp_shape_y + 1);\n' + " x = m->x;\n" + "y=m -> y;\n") + assert exp_impl == impl + +def test_ifdbfilter(): + """Test InitFromDbFilter""" + m = MockCodeGenMachine() + f = InitFromDbFilter(m) + f.given_classname = 'MyFactory' + + args = f.methodargs() + exp_args = "cyclus::QueryableBackend* b" + assert exp_args == args + + impl = f.impl() + exp_impl = (' int rawcycpp_shape_y[1] = {42};\n' + ' cycpp_shape_y = std::vector(rawcycpp_shape_y, ' + 'rawcycpp_shape_y + 1);\n' + ' cyclus::QueryResult qr = b->Query("Info", NULL);\n' + ' x = qr.GetVal("x");\n' + "WAKKA JAWAKA") + assert exp_impl == impl + +def test_aliasing_schemafilter(): + impl = setup_alias(SchemaFilter) + + assert('foo_alias' in impl) + assert not('bar_var' in impl) + assert('foo_map_alias' in impl) + assert not('bar_map_var' in impl) + +def test_aliasing_snapshotfilter(): + impl = setup_alias(SnapshotFilter) + + assert not('foo_alias' in impl) + assert('bar_var' in impl) + assert not('foo_map_alias' in impl) + assert('bar_map_var' in impl) + +def test_aliasing_infiletodbfilter(): + impl = setup_alias(InfileToDbFilter) + + assert('foo_alias' in impl) + assert('bar_var' in impl) + assert('foo_map_alias' in impl) + assert('bar_map_var' in impl) + +def setup_alias(filt): + m = MockAliasCodeGenMachine() + f = filt(m) + f.given_classname = 'MyFactory' + return f.impl() + +def test_itdbfilter(): + """Test InfileToDbFilter""" + m = MockCodeGenMachine() + f = InfileToDbFilter(m) + f.given_classname = 'MyFactory' + + args = f.methodargs() + exp_args = "cyclus::InfileTree* tree, cyclus::DbInit di" + assert exp_args == args + + impl = f.impl() + exp_impl = ( + ' int rawcycpp_shape_y[1] = {42};\n cycpp_shape_y = std::vector(rawcycpp_shape_y, rawcycpp_shape_y + 1);\n cyclus::InfileTree* sub = tree->SubTree("config/*");\n int i;\n int n;\n {\n int x_val = cyclus::Query(sub, "x");\n x = x_val;\n }\nTHINGFISH\n di.NewDatum("Info")\n ->AddVal("x", x)\nABSOLUTELY FREE\n ->Record();\n' + ) + + assert exp_impl == impl + +def test_itdbfilter_val(): + """Test InfileToDbFilter._val() Defaults""" + m = MockCodeGenMachine() + f = InfileToDbFilter(m) + + cases = [ + ('bool', True, 'foo', None, 'bool foo = true;\n'), + ('bool', False, 'foo', None, 'bool foo = false;\n'), + ('int', 42, 'foo', None, 'int foo = 42;\n'), + ('int', 92235, 'foo', 'nuclide', 'int foo = pyne::nucname::id(92235);\n'), + ('int', 'U-235', 'foo', 'nuclide', 'int foo = pyne::nucname::id("U-235");\n'), + ('float', 42.0, 'foo', None, 'double foo = 42.0;\n'), + ('double', 42.0, 'foo', None, 'double foo = 42.0;\n'), + ('std::string', 'wakka', 'foo', None, 'std::string foo("wakka");\n'), + ('cyclus::Blob', 'wakka', 'foo', None, 'cyclus::Blob foo("wakka");\n'), + ('boost::uuids::uuid', + '/#\xfb\xaf\x90\xc9N\xe9\x98:S\xea\xd6\xd6\x0fb', 'foo', None, + 'boost::uuids::uuid foo = "/#\xfb\xaf\x90\xc9N\xe9\x98:S\xea\xd6\xd6\x0fb";\n'), + ('boost::uuids::uuid', + uuid.UUID('2f23fbaf-90c9-4ee9-983a-53ead6d60f62'), 'foo', None, + 'boost::uuids::uuid foo = {0x2f, 0xf3, 0x2b, 0x3f, 0xf0, 0xb9, 0xae, 0xf9, 0x98, 0x0a, 0xc3, 0x9a, 0x46, 0xe6, 0xef, 0x92};\n'), + (('std::vector', 'int'), [42], 'foo', None, + ('std::vector< int > foo;\n' + 'foo.resize(1);\n' + '{\n' + ' {\n' + ' int elem = 42;\n' + ' foo[0] = elem;\n' + ' }\n' + '}\n'), + ), + (('std::vector', 'int'), [92235], 'foo', [None, 'nuclide'], + ('std::vector< int > foo;\n' + 'foo.resize(1);\n' + '{\n' + ' {\n' + ' int elem = pyne::nucname::id(92235);\n' + ' foo[0] = elem;\n' + ' }\n' + '}\n'), + ), + (('std::set', 'int'), [42, 65], 'foo', None, + ('std::set< int > foo;\n' + '{\n' + ' {\n' + ' int elem = 42;\n' + ' foo.insert(elem);\n' + ' }\n' + ' {\n' + ' int elem = 65;\n' + ' foo.insert(elem);\n' + ' }\n' + '}\n'), + ), + (('std::list', 'int'), [42, 65], 'foo', None, + ('std::list< int > foo;\n' + '{\n' + ' {\n' + ' int elem = 42;\n' + ' foo.push_back(elem);\n' + ' }\n' + ' {\n' + ' int elem = 65;\n' + ' foo.push_back(elem);\n' + ' }\n' + '}\n'), + ), + (('std::pair', 'int', 'double'), [42, 65.0], 'foo', None, + ('std::pair< int, double > foo;\n' + '{\n' + ' int first = 42;\n' + ' double second = 65.0;\n' + ' foo.first = first;\n' + ' foo.second = second;\n' + '}\n'), + ), + (('std::map', 'int', 'double'), {42: 65.0}, 'foo', None, + ('std::map< int, double > foo;\n' + '{\n' + ' {\n' + ' int key = 42;\n' + ' double val = 65.0;\n' + ' foo[key] = val;\n' + ' }\n' + '}\n'), + ), + (('std::map', 'std::string', ('std::pair', 'bool', ('std::vector', 'double'))), + OrderedDict([('hello', [True, [1.1, 2.2, 3.3]]), ('goodbye', [False, [3.3, 2.2, 1.1]])]), + 'foo', None, + ('std::map< std::string, std::pair< bool, std::vector< double > > > foo;\n' + '{\n' + ' {\n' + ' std::string key("hello");\n' + ' std::pair< bool, std::vector< double > > val;\n' + ' {\n' + ' bool first = true;\n' + ' std::vector< double > second;\n' + ' second.resize(3);\n' + ' {\n' + ' {\n' + ' double elem = 1.1;\n' + ' second[0] = elem;\n' + ' }\n' + ' {\n' + ' double elem = 2.2;\n' + ' second[1] = elem;\n' + ' }\n' + ' {\n' + ' double elem = 3.3;\n' + ' second[2] = elem;\n' + ' }\n' + ' }\n' + ' val.first = first;\n' + ' val.second = second;\n' + ' }\n' + ' foo[key] = val;\n' + ' }\n' + ' {\n' + ' std::string key("goodbye");\n' + ' std::pair< bool, std::vector< double > > val;\n' + ' {\n' + ' bool first = false;\n' + ' std::vector< double > second;\n' + ' second.resize(3);\n' + ' {\n' + ' {\n' + ' double elem = 3.3;\n' + ' second[0] = elem;\n' + ' }\n' + ' {\n' + ' double elem = 2.2;\n' + ' second[1] = elem;\n' + ' }\n' + ' {\n' + ' double elem = 1.1;\n' + ' second[2] = elem;\n' + ' }\n' + ' }\n' + ' val.first = first;\n' + ' val.second = second;\n' + ' }\n' + ' foo[key] = val;\n' + ' }\n' + '}\n'), + ), + ] + + for t, v, name, uitype, exp in cases: + obs = f._val(t, val=v, name=name, uitype=uitype) + assert exp == obs + +def test_schemafilter(): + """Test SchemaFilter""" + m = MockCodeGenMachine() + f = SchemaFilter(m) + f.given_classname = 'MyFactory' + + args = f.methodargs() + exp_args = "" + assert exp_args == args + + impl = f.impl() + exp_impl = (' return ""\n' + ' "\\n"\n' + ' " \\n"\n' + ' " \\n"\n' + ' " \\n"\n' + ' " FREAK OUT\\n"\n' + ' "\\n";\n') + assert exp_impl == impl + + # schema type tests + assert 'string' == f._type('std::string') + assert 'boolean' == f._type('bool') + assert 'token' == f._type('std::string', 'token') + + m.context = {"MyFactory": OrderedDict([('vars', OrderedDict([ + ('x', {'type': ('std::map', 'int', 'double')}), + ])) + ])} + impl = f.impl() + exp_impl = ( + ' return ""\n' + ' "\\n"\n' + ' " \\n"\n' + ' " \\n"\n' + ' " \\n"\n' + ' " \\n"\n' + ' " \\n"\n' + ' " \\n"\n' + ' " \\n"\n' + ' " \\n"\n' + ' " \\n"\n' + ' " \\n"\n' + ' " \\n"\n' + ' " \\n"\n' + ' " \\n"\n' + ' " \\n"\n' + ' "\\n";\n') + assert exp_impl == impl + +def test_annotationsfilter(): + """Test SchemaFilter""" + m = MockCodeGenMachine() + f = AnnotationsFilter(m) + f.given_classname = 'MyFactory' + + args = f.methodargs() + exp_args = "" + assert exp_args == args + + impl = f.impl() + assert isinstance(impl, str) + +def test_snapshotfilter(): + """Test SnapshotFilter""" + m = MockCodeGenMachine() + f = SnapshotFilter(m) + f.given_classname = 'MyFactory' + + args = f.methodargs() + exp_args = 'cyclus::DbInit di' + assert exp_args == args + + impl = f.impl() + exp_impl = (' di.NewDatum("Info")\n' + ' ->AddVal("x", x)\n' + 'JUST ANOTHER BAND FROM LA\n' + ' ->Record();\n') + assert exp_impl == impl + +def test_sshinvfilter(): + """Test SnapshotInvFilter""" + m = MockCodeGenMachine() + f = SnapshotInvFilter(m) + f.given_classname = 'MyFactory' + + args = f.methodargs() + exp_args = '' + assert exp_args == args + + impl = f.impl() + exp_impl = (" cyclus::Inventories invs;\n" + " return invs;\n") + assert exp_impl == impl + + f = SnapshotInvFilter(m) + f.given_classname = 'MyFactory' + f.mode = 'impl' + impl = f.impl() + exp_impl = (" cyclus::Inventories invs;\n") + assert exp_impl == impl + +def test_intinvfilter(): + """Test InitInvFilter""" + m = MockCodeGenMachine() + f = InitInvFilter(m) + f.given_classname = 'MyFactory' + + args = f.methodargs() + exp_args = "cyclus::Inventories& inv" + assert exp_args == args + + impl = f.impl() + exp_impl = '' + assert exp_impl == impl + +def test_defpragmafilter(): + """Test DefaultPragmaFilter""" + m = MockCodeGenMachine() + f = DefaultPragmaFilter(m) + +def test_schemafilter_buildschema(): + m = MockCodeGenMachine() + m.context = {"MyFactory": OrderedDict([('vars', OrderedDict([ + ('x', {'type': 'int', 'uitype': 'nuclide'}), + ])) + ])} + f = SchemaFilter(m) + + schematype = None + uitype = None + names = None + + cpptype = ['std::map', 'std::string', ['std::vector', 'double']] + names = ['streams'] + exp = ('' + '' + '' + '' + '') + obs = f._buildschema(cpptype, schematype, uitype, names) + assert exp == obs + + cpptype = ['std::map', 'std::string', ['std::vector', 'double']] + names = ['streams', 'name', ['efficiencies', 'val']] + exp = ('' + '' + '' + '' + '') + obs = f._buildschema(cpptype, schematype, uitype, names) + assert exp == obs + + # test item aliasing + cpptype = ['std::map', 'std::string', ['std::vector', 'double']] + names = [['streams', 'entry'], 'name', ['efficiencies', 'val']] + exp = ('' + '' + '' + '' + '') + obs = f._buildschema(cpptype, schematype, uitype, names) + assert exp == obs + +def test_escape_xml(): + """Test escape_xml""" + xml = '' \ + '' \ + '' + got = cycpp.escape_xml(xml) + + s = ' "\\n"\n' \ + ' " \\n"\n' \ + ' " \\n"\n' \ + ' " \\n"\n' \ + ' " \\n"\n' \ + ' " \\n"\n' \ + ' " \\n"\n' \ + ' "\\n"' + + assert s == got + +def test_infiletodb_read_member1(): + m = MockCodeGenMachine() + m.context = {"MyFactory": OrderedDict([('vars', OrderedDict([ + ('x', {'type': 'int', 'uitype': 'nuclide'}), + ])) + ])} + f = InfileToDbFilter(m) + + cpptype = ('std::map', 'std::string', ('std::vector', + ('std::vector', ('std::pair', 'double', + ('std::pair', 'int', ('std::list', ('std::set', 'bool'))))))) + alias = ['streams', 'name', ['efficiencies', 'val']] + gen = f.read_member('mymap', alias, cpptype, uitype=None) + exp_gen = ( + ' std::map< std::string, std::vector< std::vector< std::pair< double, std::pair< int, std::list< std::set< bool > > > > > > > mymap;\n' + ' {\n' + ' cyclus::InfileTree* bub = sub->SubTree("streams", 0);\n' + ' cyclus::InfileTree* sub = bub;\n' + ' int n1 = sub->NMatches("item");\n' + ' std::map< std::string, std::vector< std::vector< std::pair< double, std::pair< int, std::list< std::set< bool > > > > > > > mymap_in;\n' + ' for (int i1 = 0; i1 < n1; ++i1) {\n' + ' std::string key;\n' + ' {\n' + ' std::string key_in = cyclus::Query(sub, "item/name", i1);\n' + ' key = key_in;\n' + ' }\n' + ' std::vector< std::vector< std::pair< double, std::pair< int, std::list< std::set< bool > > > > > > val;\n' + ' {\n' + ' cyclus::InfileTree* bub = sub->SubTree("item/efficiencies", i1);\n' + ' cyclus::InfileTree* sub = bub;\n' + ' int n2 = sub->NMatches("val");\n' + ' std::vector< std::vector< std::pair< double, std::pair< int, std::list< std::set< bool > > > > > > val_in;\n' + ' val_in.resize(n2);\n' + ' for (int i2 = 0; i2 < n2; ++i2) {\n' + ' std::vector< std::pair< double, std::pair< int, std::list< std::set< bool > > > > > elem;\n' + ' {\n' + ' cyclus::InfileTree* bub = sub->SubTree("val", i2);\n' + ' cyclus::InfileTree* sub = bub;\n' + ' int n3 = sub->NMatches("val");\n' + ' std::vector< std::pair< double, std::pair< int, std::list< std::set< bool > > > > > elem_in;\n' + ' elem_in.resize(n3);\n' + ' for (int i3 = 0; i3 < n3; ++i3) {\n' + ' std::pair< double, std::pair< int, std::list< std::set< bool > > > > elem;\n' + ' {\n' + ' cyclus::InfileTree* bub = sub->SubTree("val", i3);\n' + ' cyclus::InfileTree* sub = bub;\n' + ' double firsti3;\n' + ' {\n' + ' double firsti3_in = cyclus::Query(sub, "first", 0);\n' + ' firsti3 = firsti3_in;\n' + ' }\n' + ' std::pair< int, std::list< std::set< bool > > > secondi3;\n' + ' {\n' + ' cyclus::InfileTree* bub = sub->SubTree("second", 0);\n' + ' cyclus::InfileTree* sub = bub;\n' + ' int first0;\n' + ' {\n' + ' int first0_in = cyclus::Query(sub, "first", 0);\n' + ' first0 = first0_in;\n' + ' }\n' + ' std::list< std::set< bool > > second0;\n' + ' {\n' + ' cyclus::InfileTree* bub = sub->SubTree("second", 0);\n' + ' cyclus::InfileTree* sub = bub;\n' + ' int n4 = sub->NMatches("val");\n' + ' std::list< std::set< bool > > second0_in;\n' + ' for (int i4 = 0; i4 < n4; ++i4) {\n' + ' std::set< bool > elem;\n' + ' {\n' + ' cyclus::InfileTree* bub = sub->SubTree("val", i4);\n' + ' cyclus::InfileTree* sub = bub;\n' + ' int n5 = sub->NMatches("val");\n' + ' std::set< bool > elem_in;\n' + ' for (int i5 = 0; i5 < n5; ++i5) {\n' + ' bool elem;\n' + ' {\n' + ' bool elem_in = cyclus::Query(sub, "val", i5);\n' + ' elem = elem_in;\n' + ' }\n' + ' elem_in.insert(elem);\n' + ' }\n' + ' elem = elem_in;\n' + ' }\n' + ' second0_in.push_back(elem);\n' + ' }\n' + ' second0 = second0_in;\n' + ' }\n' + ' std::pair< int, std::list< std::set< bool > > > secondi3_in(first0, second0);\n' + ' secondi3 = secondi3_in;\n' + ' }\n' + ' std::pair< double, std::pair< int, std::list< std::set< bool > > > > elem_in(firsti3, secondi3);\n' + ' elem = elem_in;\n' + ' }\n' + ' elem_in[i3] = elem;\n' + ' }\n' + ' elem = elem_in;\n' + ' }\n' + ' val_in[i2] = elem;\n' + ' }\n' + ' val = val_in;\n' + ' }\n' + ' mymap_in[key] = val;\n' + ' }\n' + ' mymap = mymap_in;\n' + ' }\n') + + ## useful for debugging test failures + #print(gen) + #print(exp_gen) + + assert exp_gen == gen + +def test_infiletodb_read_member2(): + m = MockCodeGenMachine() + m.context = {"MyFactory": OrderedDict([('vars', OrderedDict([ + ('x', {'type': 'int', 'uitype': 'nuclide'}), + ])) + ])} + f = InfileToDbFilter(m) + + alias = ['map', 'int', ['vector', ['pair', 'str1', 'str2']]] + cpptype = ('std::map', 'int', ('std::vector', ('std::pair', 'std::string', 'std::string'))) + gen = f.read_member('mymap', alias, cpptype, uitype=None) + exp_gen = ( + ' std::map< int, std::vector< std::pair< std::string, std::string > > > mymap;\n' + ' {\n' + ' cyclus::InfileTree* bub = sub->SubTree("map", 0);\n' + ' cyclus::InfileTree* sub = bub;\n' + ' int n1 = sub->NMatches("item");\n' + ' std::map< int, std::vector< std::pair< std::string, std::string > > > mymap_in;\n' + ' for (int i1 = 0; i1 < n1; ++i1) {\n' + ' int key;\n' + ' {\n' + ' int key_in = cyclus::Query(sub, "item/int", i1);\n' + ' key = key_in;\n' + ' }\n' + ' std::vector< std::pair< std::string, std::string > > val;\n' + ' {\n' + ' cyclus::InfileTree* bub = sub->SubTree("item/vector", i1);\n' + ' cyclus::InfileTree* sub = bub;\n' + ' int n2 = sub->NMatches("pair");\n' + ' std::vector< std::pair< std::string, std::string > > val_in;\n' + ' val_in.resize(n2);\n' + ' for (int i2 = 0; i2 < n2; ++i2) {\n' + ' std::pair< std::string, std::string > elem;\n' + ' {\n' + ' cyclus::InfileTree* bub = sub->SubTree("pair", i2);\n' + ' cyclus::InfileTree* sub = bub;\n' + ' std::string firsti2;\n' + ' {\n' + ' std::string firsti2_in = cyclus::Query(sub, "str1", 0);\n' + ' firsti2 = firsti2_in;\n' + ' }\n' + ' std::string secondi2;\n' + ' {\n' + ' std::string secondi2_in = cyclus::Query(sub, "str2", 0);\n' + ' secondi2 = secondi2_in;\n' + ' }\n' + ' std::pair< std::string, std::string > elem_in(firsti2, secondi2);\n' + ' elem = elem_in;\n' + ' }\n' + ' val_in[i2] = elem;\n' + ' }\n' + ' val = val_in;\n' + ' }\n' + ' mymap_in[key] = val;\n' + ' }\n' + ' mymap = mymap_in;\n' + ' }\n' + ) + # # useful for debugging test failures + # print() + # print(gen) + # print(exp_gen) + assert exp_gen == gen + +def test_infiletodb_read_map(): + m = MockCodeGenMachine() + m.context = {"MyFactory": OrderedDict([('vars', OrderedDict([ + ('x', {'type': 'int', 'uitype': 'nuclide'}), + ])) + ])} + f = InfileToDbFilter(m) + + cpptype = ('std::map', 'int', 'double') + alias = [['streams', 'entry'], 'id', 'mass'] + obs = f.read_member('mymap', alias, cpptype, uitype=None) + + exp = ( + ' std::map< int, double > mymap;\n' + ' {\n' + ' cyclus::InfileTree* bub = sub->SubTree("streams", 0);\n' + ' cyclus::InfileTree* sub = bub;\n' + ' int n1 = sub->NMatches("entry");\n' + ' std::map< int, double > mymap_in;\n' + ' for (int i1 = 0; i1 < n1; ++i1) {\n' + ' int key;\n' + ' {\n' + ' int key_in = cyclus::Query(sub, "entry/id", i1);\n' + ' key = key_in;\n' + ' }\n' + ' double val;\n' + ' {\n' + ' double val_in = cyclus::Query(sub, "entry/mass", i1);\n' + ' val = val_in;\n' + ' }\n' + ' mymap_in[key] = val;\n' + ' }\n' + ' mymap = mymap_in;\n' + ' }\n') + + assert exp == obs + +def test_internal_schema(): + cases = [ + {'params': {'internal': True, 'default': 7}, 'want': + ' return ""\n "\\n";\n' + }, + {'params': {'derived_init': 'x *= 42;'}, 'want': + (' return ""\n' + ' "\\n"\n' + ' " \\n"\n' + ' " \\n"\n' + ' " \\n"\n' + ' "\\n";\n') + }, + {'params': {'derived_init': 'x *= 42;', 'default': 7}, 'want': + (' return ""\n' + ' "\\n"\n' + ' " \\n"\n' + ' " \\n"\n' + ' " \\n"\n' + ' " \\n"\n' + ' " \\n"\n' + ' "\\n";\n') + }, + {'params': {'internal': True, 'derived_init': 'x *= 42;', 'default': 7}, 'want': + ' return ""\n "\\n";\n' + }, + ] + + for i, case in enumerate(cases): + keys = case['params'].copy() + m = MockCodeGenMachine() + params = {'type': 'int'} + params.update(keys) + m.context = {"MyFactory": OrderedDict([('vars', OrderedDict([ + ('x', params), + ]))])} + f = SchemaFilter(m) + f.given_classname = 'MyFactory' + + impl = f.impl() + + want = case['want'] + msg = 'case {0} failed\n ---- got ----\n {1}\n ---- want ----\n {2}'.format(i + 1, impl.replace('\n', '\n '), want.replace('\n', '\n ')) + if want != impl: + pprint.pprint(impl) + assert(False, msg) + +def test_internal_infiletodb(): + # the expected output (i.e. 'want':...) is set as 'throw' if the + # configuration should cause an exception. + cases = [ + {'params': {'internal': True}, 'want': 'throw'}, + {'params': {'internal': True, 'default': 7}, 'want': + (' cyclus::InfileTree* sub = tree->SubTree("config/*");\n' + ' int i;\n' + ' int n;\n' + ' int x_tmp = 7;\n' + ' x = x_tmp;\n' + ' di.NewDatum("Info")\n' + ' ->AddVal("x", x)\n' + ' ->Record();\n') + }, + {'params': {'derived_init': 'x *= 42;'}, 'want': + (' cyclus::InfileTree* sub = tree->SubTree("config/*");\n' + ' int i;\n' + ' int n;\n' + ' {\n' + ' int x_val = cyclus::Query(sub, "x");\n' + ' x = x_val;\n' + ' }\n' + ' x *= 42;\n' + ' di.NewDatum("Info")\n' + ' ->AddVal("x", x)\n' + ' ->Record();\n') + }, + {'params': {'derived_init': 'x *= 42;', 'default': 7}, 'want': + (' cyclus::InfileTree* sub = tree->SubTree("config/*");\n' + ' int i;\n' + ' int n;\n' + ' if (sub->NMatches("x") > 0) {\n' + ' {\n' + ' int x_val = cyclus::Query(sub, "x");\n' + ' x = x_val;\n' + ' }\n' + ' } else {\n' + ' int x_tmp = 7;\n' + ' x = x_tmp;\n' + ' }\n' + ' x *= 42;\n' + ' di.NewDatum("Info")\n' + ' ->AddVal("x", x)\n' + ' ->Record();\n') + }, + {'params': {'internal': True, 'derived_init': 'x *= 42;', 'default': 7}, 'want': + (' cyclus::InfileTree* sub = tree->SubTree("config/*");\n' + ' int i;\n' + ' int n;\n' + ' int x_tmp = 7;\n' + ' x = x_tmp;\n' + ' x *= 42;\n' + ' di.NewDatum("Info")\n' + ' ->AddVal("x", x)\n' + ' ->Record();\n') + }, + ] + + for i, case in enumerate(cases): + keys = case['params'].copy() + m = MockCodeGenMachine() + params = {'type': 'int'} + params.update(keys) + m.context = {"MyFactory": OrderedDict([('vars', OrderedDict([ + ('x', params), + ]))])} + f = InfileToDbFilter(m) + f.given_classname = 'MyFactory' + + want = case['want'] + + impl = '' + if want == 'throw': + haderr = False + try: + impl = f.impl() + except: + haderr = True + msg = 'case {0} failed: expected raised exception, got none.' + assert(haderr, msg) + continue + else: + impl = f.impl() + + msg = 'case {0} failed\n ---- got ----\n {1}\n ---- want ----\n {2}'.format(i + 1, impl.replace('\n', '\n '), want.replace('\n', '\n ')) + if want != impl: + pprint.pprint(impl) + assert(False, msg) + +def test_nuclide_uitype(): + m = MockCodeGenMachine() + m.context = {"MyFactory": OrderedDict([('vars', OrderedDict([ + ('x', {'type': 'int', 'uitype': 'nuclide'}), + ])) + ])} + + # test schema is set to string + f = SchemaFilter(m) + f.given_classname = 'MyFactory' + impl = f.impl() + exp_impl = (' return ""\n' + ' "\\n"\n' + ' " \\n"\n' + ' " \\n"\n' + ' " \\n"\n' + ' "\\n";\n') + assert exp_impl == impl + + # test infiletodb updates + f = InfileToDbFilter(m) + f.given_classname = 'MyFactory' + impl = f.impl() + exp_impl = ' cyclus::InfileTree* sub = tree->SubTree("config/*");\n int i;\n int n;\n {\n int x_val = pyne::nucname::id(cyclus::Query(sub, "x"));\n x = x_val;\n }\n di.NewDatum("Info")\n ->AddVal("x", x)\n ->Record();\n' + + assert exp_impl == impl + + # test bad uitypes values fail + m.context = {"MyFactory": OrderedDict([('vars', OrderedDict([ + ('x', {'type': 'int', 'uitype': 'WRONG'}), + ])) + ])} + f = SchemaFilter(m) + f.given_classname = 'MyFactory' + with pytest.raises(TypeError): + f.impl() + +def test_integration(): + inf = os.path.join(os.path.dirname(__file__), 'cycpp_tests.h') + outf = tempfile.NamedTemporaryFile() + cmd = "" + # if CXX is set use it, fallback on g++ otherwise + if os.getenv("CXX") is not None: + cmd = 'cycpp.py {} -o {} --cpp-path `which $CXX`'.format(inf, outf.name) + else: + cmd = 'cycpp.py {} -o {} --cpp-path `which g++`'.format(inf, outf.name) + p = Popen(cmd, shell=True, stdin=PIPE, stdout=PIPE, stderr=STDOUT, close_fds=True) + assert '' == p.stdout.read().decode() + diff --git a/tests/hdf5_back_gen_test.py b/tests/hdf5_back_gen_test.py new file mode 100644 index 0000000000..6b2850ed21 --- /dev/null +++ b/tests/hdf5_back_gen_test.py @@ -0,0 +1,276 @@ +import os +import sys +import json +import subprocess +from random import randint +import uuid +import pandas as pd +from pandas.util.testing import assert_frame_equal +import pytest + +from cyclus.lib import Hdf5Back, Recorder +import cyclus.typesystem as ts + +cycdir = os.path.dirname(os.path.dirname(__file__)) +sys.path.insert(0, os.path.join(cycdir, 'src')) +import hdf5_back_gen +#from hdf5_back_gen import resolve_unicode, convert_canonical, setup, ORIGIN_DICT + +#Call to hdf5_back_gen function +hdf5_back_gen.setup() + +is_primitive = lambda canon: isinstance(canon, str) + +CHARS = r'abcdefghijklmnopqrstuv1234567890`/\_.,-+@$#!% ' + +CANON_TYPES = [] +CANON_TO_DB = {} +CANON_TO_VL = {} +def setup(): + with open(os.path.join(os.path.dirname(__file__), '..', 'share', + 'dbtypes.json')) as f: + RAW_TABLE = hdf5_back_gen.resolve_unicode(json.load(f)) + + VERSION = "" + TABLE_START = 0 + TABLE_END = 0 + for row in range(len(RAW_TABLE)): + current = tuple(RAW_TABLE[row]) + if current[4] == "HDF5": + if current[5] > VERSION: + VERSION = current[5] + TABLE_START = row + if current[5] == VERSION: + TABLE_END = row + + TYPES_TABLE = list(tuple(row) for row in RAW_TABLE[TABLE_START:TABLE_END+1]) + for row in TYPES_TABLE: + if row[6] == 1 and row[4] == "HDF5" and row[5] == VERSION: + db = row[1] + is_vl = row[8] + canon = hdf5_back_gen.convert_canonical(row[7]) + if canon not in CANON_TYPES: + CANON_TYPES.append(canon) + CANON_TO_DB[canon] = db + CANON_TO_VL[canon] = is_vl + +def make_bytes(string): + return string.encode() + +VARIABLE_LENGTH_TYPES = ['LIST', 'VECTOR', 'MAP', 'SET', 'STRING'] + +CONTAINER_TYPES = {'LIST': 'LIST', 'VL_LIST': 'LIST', + 'VECTOR': 'VECTOR', 'VL_VECTOR': 'VECTOR', + 'MAP': 'MAP', 'VL_MAP': 'MAP', + 'SET': 'SET', 'VL_SET': 'SET', + 'PAIR': 'PAIR'} + +CREATE_FUNCTIONS = {'MAP': dict, 'LIST': list, 'SET': set, 'PAIR': list, + 'VECTOR': list, 'INT': int, 'DOUBLE': float, 'FLOAT': float, + 'BOOL': bool, 'STRING': str, 'UUID': list, 'BLOB': str} + +UNIQUE_TYPES = ['MAP', 'SET'] + +TYPE_SHAPE = 0 +TYPE_FUNCTION = 1 +TYPE_CANON = 2 +CONTAINER_MIN = 4 +CONTAINER_MAX = 8 +def generate_meta(canon, depth=0): + """Produces metadata about a type to be created. + + This includes a shape, constructor function, and canonical form. This + 3-tuple can be passed to the populate function to create data in the form + specified by the metadata. + + Parameters + ---------- + canon : tuple or str + Unique identifier of current data type + depth : int, optional + Recursive depth value + + Returns + ------- + meta_shape : tuple or list + Meta data specification for type + """ + meta_shape = [] + my_shape = None + my_type = None + origin = hdf5_back_gen.ORIGIN_DICT[canon] + if is_primitive(canon): + if CANON_TO_VL[canon]: + my_shape = -1 + elif origin in VARIABLE_LENGTH_TYPES: + my_shape = randint(CONTAINER_MIN, CONTAINER_MAX) + else: + my_shape = -1 + my_type = CREATE_FUNCTIONS[origin] + return (my_shape, my_type, canon) + elif CANON_TO_VL[canon]: + my_shape = -1 + my_type = CREATE_FUNCTIONS[origin[0]] + elif origin[0] in VARIABLE_LENGTH_TYPES: + my_shape = randint(CONTAINER_MIN, CONTAINER_MAX) + my_type = CREATE_FUNCTIONS[origin[0]] + else: + my_shape = -1 + my_type = CREATE_FUNCTIONS[origin[0]] + meta_shape.append((my_shape, my_type, canon)) + for i in canon[1:]: + meta_shape.append(generate_meta(i, depth=depth+1)) + return meta_shape + +def add_item(container, value): + """Attempts to add a value to a container of unknown type. + + Parameters + ---------- + container : iterable + The type into which the value is inserted + value : type + Data point to insert + + Returns + ------- + container : iterable + The container with added value + """ + if len(value) == 1: + try: + container.append(value[0]) + except AttributeError: + try: + container.add(value[0]) + except AttributeError: + print("Unsure how to add " + value[0] + " of type " + + str(type(value[0])) + " to " + str(type(container))) + sys.exit(1) + else: + try: + container.update({value[0]: value[1]}) + except AttributeError: + try: + container = (value[0], value[1]) + except AttributeError: + print("Unsure how to add " + str(value[0]) + " of type " + + str(type(value[0])) + " and " + str(value[1]) + + " of type " + str(type(value[1])) + " to " + + str(type(container))) + sys.exit(1) + return container + +def populate(meta): + """Generate data based upon the metadata for a type. + + Uses the shape values within meta to create data with the correct dimensions + for adding to the table. + + Parameters + ---------- + meta : tuple or list + Metadata for type + + Returns + ------- + data : type specified in meta + Populated container or primitive + """ + if isinstance(meta, tuple): + current_type = meta + else: + current_type = meta[0] + my_shape = current_type[TYPE_SHAPE] + my_type = current_type[TYPE_FUNCTION] + canon = current_type[TYPE_CANON] + data = my_type() + origin = hdf5_back_gen.ORIGIN_DICT[canon] + if is_primitive(canon): + if origin == 'STRING' or origin == 'BLOB': + if my_shape > 0: + for i in range(my_shape): + data += CHARS[randint(0, len(CHARS)-1)] + else: + for i in range(randint(CONTAINER_MIN, CONTAINER_MAX)): + data += CHARS[randint(0, len(CHARS)-1)] + if origin == 'BLOB': + data = make_bytes(data) + elif canon == 'BOOL': + data = bool(randint(0,1)) + elif canon == 'UUID': + data = uuid.uuid5(uuid.NAMESPACE_DNS, CHARS[randint(0,len(CHARS)-1)]) + else: + data = my_type(randint(0, 9) * (data + 1)) + elif origin[0] in VARIABLE_LENGTH_TYPES: + if my_shape > 0: + item_count = my_shape + else: + item_count = randint(CONTAINER_MIN, CONTAINER_MAX) + add_count = 0 + while add_count < item_count: + #reset our current child + child_data = [] + #fill with child values + for j in meta[1:]: + child_data.append(populate(j)) + if origin[0] in UNIQUE_TYPES and child_data[0] in data: + continue + else: + #add to parent + data = add_item(data, child_data) + add_count += 1 + else: + #pairs, etc. + child_data = [] + for i in meta[1:]: + child_data.append(populate(i)) + data = add_item(data, child_data) + return data + +def get_shape(meta): + """Make shape into a flat int list for the Hdf5 backend.""" + shape = [] + if isinstance(meta, tuple): + return [meta[TYPE_SHAPE]] + else: + current_type = meta[0] + shape.append(current_type[TYPE_SHAPE]) + for i in meta[1:]: + shape.extend(get_shape(i)) + return shape + +ROW_NUM = 3 +PATH = 'gen_db.h5' +def generate_and_test(): + """Generate and run tests for supported Hdf5 datatypes.""" + if sys.version_info[0] == 2: + msg = 'Hdf5 backend gen tests do not support Python 2.x' + pytest.skip(msg) + if os.path.isfile(PATH): + os.remove(PATH) + for i in CANON_TYPES: + print(CANON_TO_DB[i],'\n') + rec = Recorder(inject_sim_id=False) + back = Hdf5Back(PATH) + rec.register_backend(back) + data_meta = generate_meta(i) + shape = get_shape(data_meta) + print("shape: ", shape) + data = [] + for j in range(ROW_NUM): + data.append(populate(data_meta)) + exp = pd.DataFrame({'col0': data}, columns=['col0']) + print("expected: \n", exp) + for j in data: + d = rec.new_datum("test0") + d.add_val("col0", j, shape=shape, type=ts.IDS[CANON_TO_DB[i]]) + d.record() + rec.flush() + obs = back.query("test0") + print("observed: \n", obs) + assert_frame_equal, exp, obs + rec.close() + os.remove(PATH) + + From c9bb8f635968a1dc81afb1909caf5f92baca417f Mon Sep 17 00:00:00 2001 From: abachma2 Date: Wed, 29 Mar 2023 09:59:08 -0500 Subject: [PATCH 008/102] change Mapping call to collections.abc --- cyclus/gentypesystem.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cyclus/gentypesystem.py b/cyclus/gentypesystem.py index 506dca2a9d..f20e13b24b 100644 --- a/cyclus/gentypesystem.py +++ b/cyclus/gentypesystem.py @@ -700,7 +700,7 @@ def convert_to_cpp(self, x, t): '{valdecl}\n' 'cdef {type} cpp{var}\n', 'cpp{var} = {type}()\n' - 'if not isinstance({var}, collections.Mapping):\n' + 'if not isinstance({var}, collections.abc.Mapping):\n' ' {var} = dict({var})\n' 'for {keyname}, {valname} in {var}.items():\n' ' {keybody.indent4}\n' From 70f50bc8cf5e7da040bd23a60ae3e2b804692189 Mon Sep 17 00:00:00 2001 From: abachma2 Date: Wed, 29 Mar 2023 10:01:02 -0500 Subject: [PATCH 009/102] add yield statements back in The yield statements seem to be needed to pass information out of the two functions. Return statements gave an error. The one test is being skipped, but that seems to be because of something else later in the test --- tests/test_stubs.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_stubs.py b/tests/test_stubs.py index c2d8008481..152c53d69f 100644 --- a/tests/test_stubs.py +++ b/tests/test_stubs.py @@ -11,11 +11,12 @@ @contextmanager def tmpdir(): d = tempfile.mkdtemp() + yield d shutil.rmtree(d) @contextmanager def tmplog(fname): - io.open(fname, mode='w') + yield io.open(fname, mode='w') os.remove(fname) def test_stubs(): From fab462188198bfe817956bcb8c5005e2ca284e1a Mon Sep 17 00:00:00 2001 From: abachma2 Date: Thu, 30 Mar 2023 10:57:58 -0500 Subject: [PATCH 010/102] rename to be picked up by pytest Pytest won't find tests in files named tests_*.py, it must be test_*.py or *_test.py --- tests/{tests_hdf5_back_gen.py => test_hdf5_back_gen.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/{tests_hdf5_back_gen.py => test_hdf5_back_gen.py} (100%) diff --git a/tests/tests_hdf5_back_gen.py b/tests/test_hdf5_back_gen.py similarity index 100% rename from tests/tests_hdf5_back_gen.py rename to tests/test_hdf5_back_gen.py From 182faaa83d62a96806510c8f4db8618165496ca3 Mon Sep 17 00:00:00 2001 From: abachma2 Date: Tue, 4 Apr 2023 13:12:00 -0500 Subject: [PATCH 011/102] update pandas testing module --- tests/hdf5_back_gen_test.py | 2 +- tests/test_memback.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/hdf5_back_gen_test.py b/tests/hdf5_back_gen_test.py index 6b2850ed21..e51dfe416c 100644 --- a/tests/hdf5_back_gen_test.py +++ b/tests/hdf5_back_gen_test.py @@ -5,7 +5,7 @@ from random import randint import uuid import pandas as pd -from pandas.util.testing import assert_frame_equal +from pandas.testing import assert_frame_equal import pytest from cyclus.lib import Hdf5Back, Recorder diff --git a/tests/test_memback.py b/tests/test_memback.py index d50fe28655..aac617e3e6 100644 --- a/tests/test_memback.py +++ b/tests/test_memback.py @@ -7,7 +7,7 @@ import numpy as np import pandas as pd -from pandas.util.testing import assert_frame_equal +from pandas.testing import assert_frame_equal def make_rec_back(inject_sim_id=False): From f87c0aca5fdd111a06eb42444a90e3b74e6ae29c Mon Sep 17 00:00:00 2001 From: abachma2 Date: Wed, 5 Apr 2023 15:44:18 -0500 Subject: [PATCH 012/102] update conda install instructions --- README.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.rst b/README.rst index e33233ae7d..a865ae27a3 100644 --- a/README.rst +++ b/README.rst @@ -79,7 +79,7 @@ Package Minimum Version ``boost`` 1.46.1 ``libxml2`` 2 ``libxml++`` 2.36 -``python`` 2.7 or 3.3+ +``python`` 2.7 or 3.10 ``sqlite3`` 3.7.10 ``HDF5`` 1.8.4 ``Coin-Cbc`` 2.5 @@ -108,10 +108,10 @@ To install Cyclus and its dependencies onto a clean Ubuntu machine (tested on 18 - ``conda config --add channels conda-forge`` -- ``conda install -y openssh gxx_linux-64 gcc_linux-64 cmake make docker-pycreds git xo - python-json-logger python=3.6 glibmm glib=2.56 libxml2 libxmlpp libblas libcblas - liblapack pkg-config coincbc=2.9 boost-cpp hdf5 sqlite pcre gettext bzip2 xz - setuptools nose pytables pandas jinja2 cython==0.26 websockets pprintpp`` +- ``conda install -y openssh gxx_linux-64=12.2 gcc_linux-64=12.2 cmake make + docker-pycreds git xo python-json-logger python=3.10 glibmm glib libxml2 libxmlpp + libblas libcblas liblapack pkg-config coincbc boost-cpp hdf5 sqlite pcre gettext + bzip2 xz setuptools nose pytables pandas jinja2 cython websockets pprintpp`` - Use ``sudo apt install`` to install and configure git From 85efafb3602edb424596e902d536cd3529b2726c Mon Sep 17 00:00:00 2001 From: abachma2 Date: Tue, 9 May 2023 14:05:30 -0500 Subject: [PATCH 013/102] change to import importlib to remove deprecation warning --- cyclus/gentypesystem.py | 2 +- tests/tools.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cyclus/gentypesystem.py b/cyclus/gentypesystem.py index f20e13b24b..80897b1ff1 100644 --- a/cyclus/gentypesystem.py +++ b/cyclus/gentypesystem.py @@ -10,7 +10,7 @@ import io import os import sys -import imp +import importlib import json import argparse import platform diff --git a/tests/tools.py b/tests/tools.py index 772a3e5d5d..e13a5eed00 100644 --- a/tests/tools.py +++ b/tests/tools.py @@ -3,7 +3,7 @@ import os import re import sys -import imp +import importlib import shutil import unittest import subprocess From 459e8867508079adc800d5340498b245b7c400ee Mon Sep 17 00:00:00 2001 From: abachma2 Date: Mon, 15 May 2023 13:50:26 -0500 Subject: [PATCH 014/102] fix spelling errors --- src/request_portfolio.h | 6 +++--- src/xml_file_loader.cc | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/request_portfolio.h b/src/request_portfolio.h index 003f1f0e7a..8a497769d1 100644 --- a/src/request_portfolio.h +++ b/src/request_portfolio.h @@ -50,7 +50,7 @@ template struct QtyCoeffConverter : public Converter { /// @class RequestPortfolio /// -/// @brief A RequestPortfolio is a group of (possibly constrainted) requests for +/// @brief A RequestPortfolio is a group of (possibly constrained) requests for /// resources /// /// The portfolio contains a grouping of resource requests that may be mutually @@ -69,7 +69,7 @@ template struct QtyCoeffConverter : public Converter { /// RequestPortfolio()); /// // add some requests /// rp->AddRequest(/* args */); -/// // declare some of them as multicommodity requsts (i.e., any one will +/// // declare some of them as multi-commodity requests (i.e., any one will /// // satisfy this demand). /// rp->AddMutualReqs(/* args */); /// @@ -229,7 +229,7 @@ class RequestPortfolio /// constraints_ is a set because constraints are assumed to be unique std::set> constraints_; - /// the total quantity of resources assocaited with the portfolio + /// the total quantity of resources associated with the portfolio double qty_; Trader* requester_; }; diff --git a/src/xml_file_loader.cc b/src/xml_file_loader.cc index d47b082df9..184f4a527c 100644 --- a/src/xml_file_loader.cc +++ b/src/xml_file_loader.cc @@ -255,7 +255,7 @@ void XMLFileLoader::LoadSolver() { if (!exclusive) { std::stringstream ss; ss << "You have set allow_exclusive_orders to False." - << " Many archetypes (e.g., :cycamore:Reactor will not work" + << " Many archetypes (e.g., :cycamore:Reactor) will not work" << " as intended with this feature turned off."; Warn(ss.str()); } From f7235b0eb51ef97111373f765b35bb2317b6578b Mon Sep 17 00:00:00 2001 From: abachma2 Date: Mon, 22 May 2023 08:37:48 -0500 Subject: [PATCH 015/102] comment out raising error from invalid schema --- src/relax_ng_validator.cc | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/relax_ng_validator.cc b/src/relax_ng_validator.cc index 2ddf8a8630..2825315420 100644 --- a/src/relax_ng_validator.cc +++ b/src/relax_ng_validator.cc @@ -15,6 +15,7 @@ #include #include "error.h" +#include namespace cyclus { @@ -81,10 +82,13 @@ bool RelaxNGValidator::Validate(const xmlpp::Document* doc) { throw ValidationError(e.message); } - if (res != 0) { - throw ValidationError("Document failed schema validation"); - } + std::cout << (xmlDocPtr)doc->cobj(); + + // if (res != 0) { + // throw ValidationError("Document failed schema validation"); + //} + return res; } From 2512d5911b850728e2681f70e8a373a0ee309470 Mon Sep 17 00:00:00 2001 From: abachma2 Date: Mon, 22 May 2023 13:29:00 -0500 Subject: [PATCH 016/102] update request dictionary to include a preference key This is an attempt to get the AddMutualReqs function added into the Python API. --- cyclus/gentypesystem.py | 6 +++++- cyclus/lib.pyx | 7 ++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/cyclus/gentypesystem.py b/cyclus/gentypesystem.py index f20e13b24b..54c84c1b83 100644 --- a/cyclus/gentypesystem.py +++ b/cyclus/gentypesystem.py @@ -2280,7 +2280,11 @@ class {{rclsname}}Request(_{{rclsname}}Request): if req['cost'] is not None: raise ValueError('setting cost functions from Python is not yet ' 'supported.') - port.get().AddRequest(targ_ptr, requester, commod, req['preference'], + if pyport['preference'] is not None: + port.get().AddRequest(targ_ptr, requester, commod, pyport['preference'], + req['exclusive']) + else: + port.get().AddRequest(targ_ptr, requester, commod, req['preference'], req['exclusive']) # add constraints for constr in pyport['constraints']: diff --git a/cyclus/lib.pyx b/cyclus/lib.pyx index 258533e2f0..17f92ede3c 100644 --- a/cyclus/lib.pyx +++ b/cyclus/lib.pyx @@ -1810,6 +1810,7 @@ cpdef dict normalize_request_portfolio(object inp): if 'commodities' in inp: commods = inp['commodities'] constrs = inp.get('constraints', []) + prefs = inp.get('preference', []) else: commods = inp constrs = [] @@ -1825,7 +1826,9 @@ cpdef dict normalize_request_portfolio(object inp): if isinstance(val, ts.Resource): req = default_req.copy() req['target'] = val + req['preference'] = prefs commods[key] = [req] + elif isinstance(val, Mapping): req = default_req.copy() req.update(val) @@ -1846,7 +1849,9 @@ cpdef dict normalize_request_portfolio(object inp): else: raise TypeError('Did not recognize type of commodity while ' 'converting to portfolio: ' + repr(inp)) - cdef dict rtn = {'commodities': commods, 'constraints': constrs} + + cdef dict rtn = {'commodities': commods, 'constraints': constrs, + 'preference': prefs} return rtn From e4da2973f848717d4a6873fe5ec75c25c0cf388d Mon Sep 17 00:00:00 2001 From: abachma2 Date: Tue, 23 May 2023 11:59:57 -0500 Subject: [PATCH 017/102] convert normalized dictionary to new format for multiple requests --- cyclus/lib.pyx | 83 ++++++++++++++++++++++++++++++-------------------- 1 file changed, 50 insertions(+), 33 deletions(-) diff --git a/cyclus/lib.pyx b/cyclus/lib.pyx index 17f92ede3c..d0d9f372b7 100644 --- a/cyclus/lib.pyx +++ b/cyclus/lib.pyx @@ -1804,54 +1804,71 @@ cpdef dict normalize_request_portfolio(object inp): """Normalizes a request portfolio into a standard Python form, ready to be traded. Note that this does not include the requester object. """ + print("normalizing input") # get initial values if not isinstance(inp, Mapping): inp = dict(inp) if 'commodities' in inp: - commods = inp['commodities'] + commods = [] + prefs = [] + for commodity in inp['commodities']: + for name, reqs in commodity.items(): + if name == 'preference': + continue + commods.append({name:reqs}) constrs = inp.get('constraints', []) - prefs = inp.get('preference', []) else: - commods = inp + commods = [] + for name, reqs in inp: + if name == 'preference': + continue + commods.append({name:reqs}) constrs = [] # canonize constraints if not isinstance(constrs, Iterable): constrs = [constrs] # canonize commods - if not isinstance(commods, Mapping): - commods = dict(commods) + if not isinstance(commods, Iterable): + commods = list(commods) cdef dict default_req = {'target': None, 'preference': 1.0, 'exclusive': False, 'cost': None} - for key, val in commods.items(): - if isinstance(val, ts.Resource): - req = default_req.copy() - req['target'] = val - req['preference'] = prefs - commods[key] = [req] - - elif isinstance(val, Mapping): - req = default_req.copy() - req.update(val) - commods[key] = [req] - elif isinstance(val, Sequence): - newval = [] - for x in val: + + print(default_req) + print(commods) + for index, commodity in enumerate(commods): + print(index, commodity) + for key, val in commodity.items(): + print("looping", key, val) + if isinstance(val, ts.Resource): req = default_req.copy() - if isinstance(x, ts.Resource): - req['target'] = x - elif isinstance(x, Mapping): - req.update(x) - else: - raise TypeError('Did not recognize type of request while ' - 'converting to portfolio: ' + repr(inp)) - newval.append(req) - commods[key] = newval - else: - raise TypeError('Did not recognize type of commodity while ' - 'converting to portfolio: ' + repr(inp)) + req['target'] = val + req['preference'] = inp['commodities'][index]['preference'] + commods[index][key] = [req] + print(commods) + + elif isinstance(val, Mapping): + req = default_req.copy() + req.update(val) + commods[key] = [req] + elif isinstance(val, Sequence): + newval = [] + for x in val: + req = default_req.copy() + if isinstance(x, ts.Resource): + req['target'] = x + elif isinstance(x, Mapping): + req.update(x) + else: + raise TypeError('Did not recognize type of request while ' + 'converting to portfolio: ' + repr(inp)) + newval.append(req) + commods[key] = newval + else: + raise TypeError('Did not recognize type of commodity while ' + 'converting to portfolio: ' + repr(inp)) - cdef dict rtn = {'commodities': commods, 'constraints': constrs, - 'preference': prefs} + cdef dict rtn = {'commodities': commods, 'constraints': constrs} + print("returned dict:", rtn) return rtn From 06abaa93f5016704b4549c6284e51dcbc0a7c240 Mon Sep 17 00:00:00 2001 From: abachma2 Date: Tue, 23 May 2023 12:12:22 -0500 Subject: [PATCH 018/102] modify format of _request_portfolio to match updated portfolio format --- cyclus/gentypesystem.py | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/cyclus/gentypesystem.py b/cyclus/gentypesystem.py index ba1399b2f6..e1b5627b79 100644 --- a/cyclus/gentypesystem.py +++ b/cyclus/gentypesystem.py @@ -2271,21 +2271,25 @@ class {{rclsname}}Request(_{{rclsname}}Request): cdef _{{rclsname}} targ cdef shared_ptr[{{cyr}}] targ_ptr # add requests - for name, reqs in pyport['commodities'].items(): - commod = str_py_to_cpp(name) - for req in reqs: - targ = <_{{rclsname}}> req['target'] - targ_ptr = reinterpret_pointer_cast[{{ts.cython_type(r)}}, - cpp_cyclus.Resource](targ.ptx) - if req['cost'] is not None: - raise ValueError('setting cost functions from Python is not yet ' - 'supported.') - if pyport['preference'] is not None: - port.get().AddRequest(targ_ptr, requester, commod, pyport['preference'], - req['exclusive']) - else: + print("adding requests") + for commodity in pyport['commodities']: + for name, reqs in commodity.items(): + commod = str_py_to_cpp(name) + print(commod, reqs) + for req in reqs: + targ = <_{{rclsname}}> req['target'] + targ_ptr = reinterpret_pointer_cast[{{ts.cython_type(r)}}, + cpp_cyclus.Resource](targ.ptx) + if req['cost'] is not None: + raise ValueError('setting cost functions from Python is not yet ' + 'supported.') + #if pyport['preference'] is not None: + # port.get().AddRequest(targ_ptr, requester, commod, pyport['preference'], + # req['exclusive']) + #else: port.get().AddRequest(targ_ptr, requester, commod, req['preference'], - req['exclusive']) + req['exclusive']) + print("added request for", req) # add constraints for constr in pyport['constraints']: port.get().AddConstraint( From f235582242b1768a7e54e54515b74e876b2356d5 Mon Sep 17 00:00:00 2001 From: abachma2 Date: Wed, 24 May 2023 14:30:30 -0500 Subject: [PATCH 019/102] remove print statements --- cyclus/lib.pyx | 7 ------- 1 file changed, 7 deletions(-) diff --git a/cyclus/lib.pyx b/cyclus/lib.pyx index d0d9f372b7..59c9a92ae9 100644 --- a/cyclus/lib.pyx +++ b/cyclus/lib.pyx @@ -1804,7 +1804,6 @@ cpdef dict normalize_request_portfolio(object inp): """Normalizes a request portfolio into a standard Python form, ready to be traded. Note that this does not include the requester object. """ - print("normalizing input") # get initial values if not isinstance(inp, Mapping): inp = dict(inp) @@ -1833,18 +1832,13 @@ cpdef dict normalize_request_portfolio(object inp): cdef dict default_req = {'target': None, 'preference': 1.0, 'exclusive': False, 'cost': None} - print(default_req) - print(commods) for index, commodity in enumerate(commods): - print(index, commodity) for key, val in commodity.items(): - print("looping", key, val) if isinstance(val, ts.Resource): req = default_req.copy() req['target'] = val req['preference'] = inp['commodities'][index]['preference'] commods[index][key] = [req] - print(commods) elif isinstance(val, Mapping): req = default_req.copy() @@ -1868,7 +1862,6 @@ cpdef dict normalize_request_portfolio(object inp): 'converting to portfolio: ' + repr(inp)) cdef dict rtn = {'commodities': commods, 'constraints': constrs} - print("returned dict:", rtn) return rtn From 10845979e546a66184455bf8c43c7f286e494eeb Mon Sep 17 00:00:00 2001 From: abachma2 Date: Wed, 24 May 2023 14:30:42 -0500 Subject: [PATCH 020/102] attempt to add AddMutualReqs, but struggling with data types --- cyclus/gentypesystem.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/cyclus/gentypesystem.py b/cyclus/gentypesystem.py index e1b5627b79..e485fcad5b 100644 --- a/cyclus/gentypesystem.py +++ b/cyclus/gentypesystem.py @@ -2267,6 +2267,9 @@ class {{rclsname}}Request(_{{rclsname}}Request): shared_ptr[cpp_cyclus.RequestPortfolio[{{cyr}}]]( new cpp_cyclus.RequestPortfolio[{{cyr}}]() ) + + cdef const std_vector[cpp_cyclus.RequestPortfolio[{{cyr}}].request_ptr] mreqs + cdef cpp_cyclus.RequestPortfolio[{{cyr}}]request cdef std_string commod cdef _{{rclsname}} targ cdef shared_ptr[{{cyr}}] targ_ptr @@ -2275,7 +2278,6 @@ class {{rclsname}}Request(_{{rclsname}}Request): for commodity in pyport['commodities']: for name, reqs in commodity.items(): commod = str_py_to_cpp(name) - print(commod, reqs) for req in reqs: targ = <_{{rclsname}}> req['target'] targ_ptr = reinterpret_pointer_cast[{{ts.cython_type(r)}}, @@ -2283,13 +2285,12 @@ class {{rclsname}}Request(_{{rclsname}}Request): if req['cost'] is not None: raise ValueError('setting cost functions from Python is not yet ' 'supported.') - #if pyport['preference'] is not None: - # port.get().AddRequest(targ_ptr, requester, commod, pyport['preference'], - # req['exclusive']) - #else: - port.get().AddRequest(targ_ptr, requester, commod, req['preference'], + request.AddRequest(targ_ptr, requester, commod, req['preference'], req['exclusive']) print("added request for", req) + mreqs.push_back(request) + #print(mreqs) + port.get().AddMutualReqs(mreqs) # add constraints for constr in pyport['constraints']: port.get().AddConstraint( From 4f05e413de3618b038013c8e46766ab8bac166aa Mon Sep 17 00:00:00 2001 From: abachma2 Date: Wed, 24 May 2023 15:24:30 -0500 Subject: [PATCH 021/102] mimic setup and datatypes used in cycamore reactors more --- cyclus/gentypesystem.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/cyclus/gentypesystem.py b/cyclus/gentypesystem.py index e485fcad5b..325560640b 100644 --- a/cyclus/gentypesystem.py +++ b/cyclus/gentypesystem.py @@ -2267,9 +2267,12 @@ class {{rclsname}}Request(_{{rclsname}}Request): shared_ptr[cpp_cyclus.RequestPortfolio[{{cyr}}]]( new cpp_cyclus.RequestPortfolio[{{cyr}}]() ) + # print(port) + + cdef std_vector[cpp_cyclus.RequestPortfolio[{{cyr}}].request_ptr] mreqs + cdef cpp_cyclus.RequestPortfolio[{{cyr}}].request_ptr request + cdef cpp_cyclus.Request[{{cyr}}]* single - cdef const std_vector[cpp_cyclus.RequestPortfolio[{{cyr}}].request_ptr] mreqs - cdef cpp_cyclus.RequestPortfolio[{{cyr}}]request cdef std_string commod cdef _{{rclsname}} targ cdef shared_ptr[{{cyr}}] targ_ptr @@ -2285,12 +2288,12 @@ class {{rclsname}}Request(_{{rclsname}}Request): if req['cost'] is not None: raise ValueError('setting cost functions from Python is not yet ' 'supported.') - request.AddRequest(targ_ptr, requester, commod, req['preference'], + single = port.get().AddRequest(targ_ptr, requester, commod, req['preference'], req['exclusive']) print("added request for", req) - mreqs.push_back(request) + #mreqs.push_back(single) #print(mreqs) - port.get().AddMutualReqs(mreqs) + #port.get().AddMutualReqs(mreqs) # add constraints for constr in pyport['constraints']: port.get().AddConstraint( From 916dbab3b91f4099cb15a0bed109087229e06288 Mon Sep 17 00:00:00 2001 From: abachma2 Date: Wed, 24 May 2023 16:07:26 -0500 Subject: [PATCH 022/102] adjusted datatypes some Based on the messages about expected datatypes, this should work. But based on the datatypes used in Cycamore and other c++ based archetypesmreqs should be a vector of Requests. My guess is there is a mismatch elsewhere with the data types. --- cyclus/gentypesystem.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/cyclus/gentypesystem.py b/cyclus/gentypesystem.py index 325560640b..eb73b99546 100644 --- a/cyclus/gentypesystem.py +++ b/cyclus/gentypesystem.py @@ -2267,11 +2267,9 @@ class {{rclsname}}Request(_{{rclsname}}Request): shared_ptr[cpp_cyclus.RequestPortfolio[{{cyr}}]]( new cpp_cyclus.RequestPortfolio[{{cyr}}]() ) - # print(port) - cdef std_vector[cpp_cyclus.RequestPortfolio[{{cyr}}].request_ptr] mreqs - cdef cpp_cyclus.RequestPortfolio[{{cyr}}].request_ptr request - cdef cpp_cyclus.Request[{{cyr}}]* single + cdef const std_vector[cpp_cyclus.RequestPortfolio[{{cyr}}].request_ptr] mreqs + cdef cpp_cyclus.Request[{{cyr}}]* single_req cdef std_string commod cdef _{{rclsname}} targ @@ -2288,10 +2286,10 @@ class {{rclsname}}Request(_{{rclsname}}Request): if req['cost'] is not None: raise ValueError('setting cost functions from Python is not yet ' 'supported.') - single = port.get().AddRequest(targ_ptr, requester, commod, req['preference'], + single_req = port.get().AddRequest(targ_ptr, requester, commod, req['preference'], req['exclusive']) print("added request for", req) - #mreqs.push_back(single) + #mreqs.push_back(single_req) #print(mreqs) #port.get().AddMutualReqs(mreqs) # add constraints From b5caaa85de95f3e984ca9f983899f80571abf6a1 Mon Sep 17 00:00:00 2001 From: Amanda Bachmann Date: Fri, 26 May 2023 11:14:16 -0500 Subject: [PATCH 023/102] Add notes about datatypes for AddMutualReqs --- cyclus/gentypesystem.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/cyclus/gentypesystem.py b/cyclus/gentypesystem.py index e485fcad5b..d5b99e7802 100644 --- a/cyclus/gentypesystem.py +++ b/cyclus/gentypesystem.py @@ -2269,7 +2269,14 @@ class {{rclsname}}Request(_{{rclsname}}Request): ) cdef const std_vector[cpp_cyclus.RequestPortfolio[{{cyr}}].request_ptr] mreqs - cdef cpp_cyclus.RequestPortfolio[{{cyr}}]request + # const vector[cyclus::RequestPortfolio::request_ptr] according to compiler + # cdef std_vector[cpp_cyclus.Request[{{cyr}}]*] mreqs + # std::vector*> mreqs defined in cycamore + + + cdef cpp_cyclus.Request[{{cyr}}]* single_request + # Request* r defined in cycamore + cdef std_string commod cdef _{{rclsname}} targ cdef shared_ptr[{{cyr}}] targ_ptr @@ -2285,10 +2292,10 @@ class {{rclsname}}Request(_{{rclsname}}Request): if req['cost'] is not None: raise ValueError('setting cost functions from Python is not yet ' 'supported.') - request.AddRequest(targ_ptr, requester, commod, req['preference'], + single_request = port.get().AddRequest(targ_ptr, requester, commod, req['preference'], req['exclusive']) print("added request for", req) - mreqs.push_back(request) + mreqs.push_back(single_request) #print(mreqs) port.get().AddMutualReqs(mreqs) # add constraints From dec137dcb2c9e578f59658cf56df6b448b5a70d2 Mon Sep 17 00:00:00 2001 From: abachma2 Date: Fri, 26 May 2023 11:26:12 -0500 Subject: [PATCH 024/102] remove cython compiling errors, but getting errors later in the build --- cyclus/gentypesystem.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cyclus/gentypesystem.py b/cyclus/gentypesystem.py index b2db239768..d5b99e7802 100644 --- a/cyclus/gentypesystem.py +++ b/cyclus/gentypesystem.py @@ -2297,7 +2297,7 @@ class {{rclsname}}Request(_{{rclsname}}Request): print("added request for", req) mreqs.push_back(single_request) #print(mreqs) - #port.get().AddMutualReqs(mreqs) + port.get().AddMutualReqs(mreqs) # add constraints for constr in pyport['constraints']: port.get().AddConstraint( From 5b93eb1b323a8e2636c8a1bbdabf17fd0f434589 Mon Sep 17 00:00:00 2001 From: abachma2 Date: Tue, 30 May 2023 10:30:00 -0500 Subject: [PATCH 025/102] add request_ptr object --- src/request_portfolio.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/request_portfolio.h b/src/request_portfolio.h index 8a497769d1..fbfc42a6ff 100644 --- a/src/request_portfolio.h +++ b/src/request_portfolio.h @@ -88,6 +88,7 @@ class RequestPortfolio public: typedef boost::shared_ptr> Ptr; typedef std::function)> cost_function_t; + typedef Request* request_ptr; RequestPortfolio() : requester_(NULL), qty_(0) {} From aa3c64c222e169692df0977fb87c3d53f4abc6d8 Mon Sep 17 00:00:00 2001 From: abachma2 Date: Tue, 30 May 2023 11:26:19 -0500 Subject: [PATCH 026/102] remove const for mreqs, remove unneeded comments and prints --- cyclus/gentypesystem.py | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/cyclus/gentypesystem.py b/cyclus/gentypesystem.py index d5b99e7802..d3a7fcef11 100644 --- a/cyclus/gentypesystem.py +++ b/cyclus/gentypesystem.py @@ -2268,18 +2268,12 @@ class {{rclsname}}Request(_{{rclsname}}Request): new cpp_cyclus.RequestPortfolio[{{cyr}}]() ) - cdef const std_vector[cpp_cyclus.RequestPortfolio[{{cyr}}].request_ptr] mreqs - # const vector[cyclus::RequestPortfolio::request_ptr] according to compiler - # cdef std_vector[cpp_cyclus.Request[{{cyr}}]*] mreqs - # std::vector*> mreqs defined in cycamore - - + cdef std_vector[cpp_cyclus.RequestPortfolio[{{cyr}}].request_ptr] mreqs cdef cpp_cyclus.Request[{{cyr}}]* single_request - # Request* r defined in cycamore - cdef std_string commod cdef _{{rclsname}} targ cdef shared_ptr[{{cyr}}] targ_ptr + # add requests print("adding requests") for commodity in pyport['commodities']: @@ -2294,10 +2288,9 @@ class {{rclsname}}Request(_{{rclsname}}Request): 'supported.') single_request = port.get().AddRequest(targ_ptr, requester, commod, req['preference'], req['exclusive']) - print("added request for", req) mreqs.push_back(single_request) - #print(mreqs) port.get().AddMutualReqs(mreqs) + # add constraints for constr in pyport['constraints']: port.get().AddConstraint( From 2fa03617a69db4b168801f82a4b10da8f03515ee Mon Sep 17 00:00:00 2001 From: abachma2 Date: Tue, 30 May 2023 15:18:04 -0500 Subject: [PATCH 027/102] remove extra print statement --- cyclus/gentypesystem.py | 1 - 1 file changed, 1 deletion(-) diff --git a/cyclus/gentypesystem.py b/cyclus/gentypesystem.py index d3a7fcef11..5f7ed449d9 100644 --- a/cyclus/gentypesystem.py +++ b/cyclus/gentypesystem.py @@ -2275,7 +2275,6 @@ class {{rclsname}}Request(_{{rclsname}}Request): cdef shared_ptr[{{cyr}}] targ_ptr # add requests - print("adding requests") for commodity in pyport['commodities']: for name, reqs in commodity.items(): commod = str_py_to_cpp(name) From 332d32ba1fa80b8a191b7041f6dda1cde2e34c62 Mon Sep 17 00:00:00 2001 From: Amanda Date: Wed, 28 Jun 2023 11:20:56 -0500 Subject: [PATCH 028/102] remove print statement --- cyclus/agents.pyx | 1 - 1 file changed, 1 deletion(-) diff --git a/cyclus/agents.pyx b/cyclus/agents.pyx index 9154d35691..61e800b03f 100644 --- a/cyclus/agents.pyx +++ b/cyclus/agents.pyx @@ -1508,7 +1508,6 @@ cdef class _Facility(_Agent): function, they must call their superclass' decommission function at the END of their decommission() function. """ - print('decom_fac') ( (<_Agent> self).shim).Decommission() @property From 9ffceb98e0d4cded87075fe8c13952b5ebabce4c Mon Sep 17 00:00:00 2001 From: Amanda Date: Wed, 28 Jun 2023 14:16:36 -0500 Subject: [PATCH 029/102] add way to specify if a request is exclusive --- cyclus/lib.pyx | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cyclus/lib.pyx b/cyclus/lib.pyx index 59c9a92ae9..bb6178c086 100644 --- a/cyclus/lib.pyx +++ b/cyclus/lib.pyx @@ -1812,14 +1812,15 @@ cpdef dict normalize_request_portfolio(object inp): prefs = [] for commodity in inp['commodities']: for name, reqs in commodity.items(): - if name == 'preference': + if name == 'preference' or name == 'exclusive': continue + commods.append({name:reqs}) constrs = inp.get('constraints', []) else: commods = [] for name, reqs in inp: - if name == 'preference': + if name == 'preference' or name == 'exclusive': continue commods.append({name:reqs}) constrs = [] @@ -1838,6 +1839,7 @@ cpdef dict normalize_request_portfolio(object inp): req = default_req.copy() req['target'] = val req['preference'] = inp['commodities'][index]['preference'] + req['exclusive'] = inp['commodities'][index]['exclusive'] commods[index][key] = [req] elif isinstance(val, Mapping): From ae4afcef343a2e457063de658f5c01d22ab30b16 Mon Sep 17 00:00:00 2001 From: Amanda Date: Sat, 1 Jul 2023 12:04:46 -0500 Subject: [PATCH 030/102] start adding context()->AddRecipe to python API --- cyclus/cpp_cyclus.pxd | 1 + cyclus/lib.pyx | 13 +++++++++++++ 2 files changed, 14 insertions(+) diff --git a/cyclus/cpp_cyclus.pxd b/cyclus/cpp_cyclus.pxd index 2472b7fa02..e1d11b346c 100644 --- a/cyclus/cpp_cyclus.pxd +++ b/cyclus/cpp_cyclus.pxd @@ -795,6 +795,7 @@ cdef extern from "context.h" namespace "cyclus": void SchedBuild(Agent*, std_string, int) void SchedDecom(Agent*) void SchedDecom(Agent*, int) + void AddRecipe(std_string, shared_ptr[Composition]) Datum* NewDatum(std_string) #void RegisterAgent(Agent*) # private void RegisterTrader(Trader*) diff --git a/cyclus/lib.pyx b/cyclus/lib.pyx index bb6178c086..d21156bdd5 100644 --- a/cyclus/lib.pyx +++ b/cyclus/lib.pyx @@ -1621,6 +1621,19 @@ cdef class _Context: return del self.ptx + def add_recipe(self, comp, basis): + """Adds a new recipe to a simulation + + Parameters: + ---------- + comp: dict + dictionary mapping nuclides to their compostion fraction + basis: str + 'atom' or 'mass' to specify the type of composition fraction + """ + comp = ts.composition_ptr_from_py(comp, basis) + + def del_agent(self, agent): """Destructs and cleans up an agent (and it's children recursively).""" self.ptx.DelAgent(dynamic_agent_ptr(agent)) From a2dcc25da543859ec3553a23f68894a9a53dc7df Mon Sep 17 00:00:00 2001 From: Amanda Date: Wed, 5 Jul 2023 14:25:58 -0500 Subject: [PATCH 031/102] finish adding AddRecipe method to Python API --- cyclus/lib.pyx | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/cyclus/lib.pyx b/cyclus/lib.pyx index d21156bdd5..9c08ff961d 100644 --- a/cyclus/lib.pyx +++ b/cyclus/lib.pyx @@ -1621,17 +1621,22 @@ cdef class _Context: return del self.ptx - def add_recipe(self, comp, basis): - """Adds a new recipe to a simulation + def add_recipe(self, name, comp, basis): + """ + Adds a new recipe to a simulation Parameters: ---------- + name: str + name for recipe comp: dict dictionary mapping nuclides to their compostion fraction basis: str 'atom' or 'mass' to specify the type of composition fraction """ - comp = ts.composition_ptr_from_py(comp, basis) + cdef std_string cpp_name = str_py_to_cpp(name) + cpp_comp = ts.composition_ptr_from_py(comp, basis) + self.ptx.AddRecipe(cpp_name, cpp_comp) def del_agent(self, agent): @@ -1851,7 +1856,9 @@ cpdef dict normalize_request_portfolio(object inp): if isinstance(val, ts.Resource): req = default_req.copy() req['target'] = val + #if 'preference' in inp['commodities'][index]: req['preference'] = inp['commodities'][index]['preference'] + #if 'exclusive' in inp['commoditites'][index]: req['exclusive'] = inp['commodities'][index]['exclusive'] commods[index][key] = [req] From b2b2d3a32a5b0b9c0819f47683d450c89c034e04 Mon Sep 17 00:00:00 2001 From: Amanda Date: Tue, 3 Oct 2023 09:51:02 -0500 Subject: [PATCH 032/102] update CHANGELOG with added functionality --- CHANGELOG.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 880d1831db..7b2b8cd08c 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -17,6 +17,7 @@ Since last release will fail unless update-alternatives has been used to point python at the correct python3 version (#1558) * build and test are now fown on githubAction in place or CircleCI (#1569) +* AddMutalReqs functionality and exclusive bids in python API of DRE (#1584) **Changed:** From 8497e44a88ea0626d143aa6958719f966a615c36 Mon Sep 17 00:00:00 2001 From: Amanda Date: Tue, 3 Oct 2023 09:58:03 -0500 Subject: [PATCH 033/102] remove changes --- src/relax_ng_validator.cc | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/relax_ng_validator.cc b/src/relax_ng_validator.cc index 2825315420..df52b43ba1 100644 --- a/src/relax_ng_validator.cc +++ b/src/relax_ng_validator.cc @@ -15,7 +15,6 @@ #include #include "error.h" -#include namespace cyclus { @@ -83,11 +82,9 @@ bool RelaxNGValidator::Validate(const xmlpp::Document* doc) { } - std::cout << (xmlDocPtr)doc->cobj(); - - // if (res != 0) { - // throw ValidationError("Document failed schema validation"); - //} + if (res != 0) { + throw ValidationError("Document failed schema validation"); + } return res; } From 6baa9fa3493ed4194724666321b3eca23edf4a93 Mon Sep 17 00:00:00 2001 From: Amanda Date: Tue, 3 Oct 2023 10:03:24 -0500 Subject: [PATCH 034/102] add version specification for cython --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index d9b2839802..ec5510f505 100644 --- a/README.rst +++ b/README.rst @@ -111,7 +111,7 @@ To install Cyclus and its dependencies onto a clean Ubuntu machine (tested on 18 - ``conda install -y openssh gxx_linux-64=12.2 gcc_linux-64=12.2 cmake make docker-pycreds git xo python-json-logger python=3.10 glibmm glib libxml2 libxmlpp libblas libcblas liblapack pkg-config coincbc boost-cpp hdf5 sqlite pcre gettext - bzip2 xz setuptools nose pytables pandas jinja2 cython websockets pprintpp`` + bzip2 xz setuptools nose pytables pandas jinja2 cython=0.29 websockets pprintpp`` - Use ``sudo apt install`` to install and configure git From 087486718040ceb6e1c426f454d71ea1368ab571 Mon Sep 17 00:00:00 2001 From: Amanda Date: Tue, 3 Oct 2023 10:25:40 -0500 Subject: [PATCH 035/102] remove changes to files --- src/relax_ng_validator.cc | 1 - tests/test_source_to_sink.py | 4 +--- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/src/relax_ng_validator.cc b/src/relax_ng_validator.cc index df52b43ba1..d08df04266 100644 --- a/src/relax_ng_validator.cc +++ b/src/relax_ng_validator.cc @@ -81,7 +81,6 @@ bool RelaxNGValidator::Validate(const xmlpp::Document* doc) { throw ValidationError(e.message); } - if (res != 0) { throw ValidationError("Document failed schema validation"); } diff --git a/tests/test_source_to_sink.py b/tests/test_source_to_sink.py index ad1597a20b..edc9151bc7 100644 --- a/tests/test_source_to_sink.py +++ b/tests/test_source_to_sink.py @@ -108,6 +108,4 @@ def test_source_to_sink(source_to_sink_case): assert_array_equal, quantities, expected_quantities - clean_outs() - - + clean_outs() \ No newline at end of file From 50f23cacdb84758fd598cf337ef68d2f7751b26e Mon Sep 17 00:00:00 2001 From: Amanda Date: Tue, 3 Oct 2023 10:25:58 -0500 Subject: [PATCH 036/102] temporarily remove apt build test --- .github/workflows/build_test_publish.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/build_test_publish.yml b/.github/workflows/build_test_publish.yml index 5311b3fcd3..9b49a89dc7 100644 --- a/.github/workflows/build_test_publish.yml +++ b/.github/workflows/build_test_publish.yml @@ -18,7 +18,6 @@ jobs: 22.04, ] pkg_mgr : [ - apt, conda, ] From 9bb80768945fc9a4e10ce553507b890f1d551400 Mon Sep 17 00:00:00 2001 From: Amanda Date: Tue, 3 Oct 2023 10:28:47 -0500 Subject: [PATCH 037/102] add back in apt build test --- .github/workflows/build_test_publish.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build_test_publish.yml b/.github/workflows/build_test_publish.yml index 9b49a89dc7..5311b3fcd3 100644 --- a/.github/workflows/build_test_publish.yml +++ b/.github/workflows/build_test_publish.yml @@ -18,6 +18,7 @@ jobs: 22.04, ] pkg_mgr : [ + apt, conda, ] From 4af5448ec42e367b6dfab4e54de384efa1889a24 Mon Sep 17 00:00:00 2001 From: Amanda Date: Tue, 3 Oct 2023 10:29:33 -0500 Subject: [PATCH 038/102] checkout from main branch No changes are meant for these files. Checking them out from the main branch ensures that no changes are recorded --- src/relax_ng_validator.cc | 2 +- tests/test_source_to_sink.py | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/relax_ng_validator.cc b/src/relax_ng_validator.cc index d08df04266..2ddf8a8630 100644 --- a/src/relax_ng_validator.cc +++ b/src/relax_ng_validator.cc @@ -84,7 +84,7 @@ bool RelaxNGValidator::Validate(const xmlpp::Document* doc) { if (res != 0) { throw ValidationError("Document failed schema validation"); } - + return res; } diff --git a/tests/test_source_to_sink.py b/tests/test_source_to_sink.py index edc9151bc7..2e83ea4459 100644 --- a/tests/test_source_to_sink.py +++ b/tests/test_source_to_sink.py @@ -108,4 +108,7 @@ def test_source_to_sink(source_to_sink_case): assert_array_equal, quantities, expected_quantities - clean_outs() \ No newline at end of file + clean_outs() + + + From 684f91bfb5e8dfaaf48ca9bcb0bcb6c447a64c38 Mon Sep 17 00:00:00 2001 From: Amanda Date: Tue, 3 Oct 2023 10:30:31 -0500 Subject: [PATCH 039/102] add note about AddRecipes added to python API --- CHANGELOG.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 7b2b8cd08c..827dd25a3e 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -17,7 +17,7 @@ Since last release will fail unless update-alternatives has been used to point python at the correct python3 version (#1558) * build and test are now fown on githubAction in place or CircleCI (#1569) -* AddMutalReqs functionality and exclusive bids in python API of DRE (#1584) +* AddMutalReqs and AddReciepe functions and exclusive bids in python API of DRE (#1584) **Changed:** From ec0714c88cad8d86ce0934b526c4140dbbf47fd1 Mon Sep 17 00:00:00 2001 From: Amanda Date: Tue, 3 Oct 2023 11:38:40 -0500 Subject: [PATCH 040/102] uncomment if loops for normalize_request_portfolio These if loops provide flexibility for using the default preferences of 1 and non-exclusive when archetype developers do not need to provide this information. --- cyclus/lib.pyx | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cyclus/lib.pyx b/cyclus/lib.pyx index 3df04c89ce..c3f75c66ef 100644 --- a/cyclus/lib.pyx +++ b/cyclus/lib.pyx @@ -1856,10 +1856,10 @@ cpdef dict normalize_request_portfolio(object inp): if isinstance(val, ts.Resource): req = default_req.copy() req['target'] = val - #if 'preference' in inp['commodities'][index]: - req['preference'] = inp['commodities'][index]['preference'] - #if 'exclusive' in inp['commoditites'][index]: - req['exclusive'] = inp['commodities'][index]['exclusive'] + if 'preference' in inp['commodities'][index]: + req['preference'] = inp['commodities'][index]['preference'] + if 'exclusive' in inp['commoditites'][index]: + req['exclusive'] = inp['commodities'][index]['exclusive'] commods[index][key] = [req] elif isinstance(val, Mapping): From 43606a59f0cd9562a235be4a4da4b763273be256 Mon Sep 17 00:00:00 2001 From: bennibbelink Date: Thu, 12 Oct 2023 20:29:54 -0500 Subject: [PATCH 041/102] first pass at a testing workflow --- .github/workflows/build_test.yml | 51 ++++++++++++++++++++++++ .github/workflows/build_test_publish.yml | 8 +++- 2 files changed, 57 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/build_test.yml diff --git a/.github/workflows/build_test.yml b/.github/workflows/build_test.yml new file mode 100644 index 0000000000..ba22f1176a --- /dev/null +++ b/.github/workflows/build_test.yml @@ -0,0 +1,51 @@ +name: Linux Build/Test for PR and collaborator push + +on: + # allows us to run workflows manually + workflow_dispatch: + pull_request: + branches: + - main + paths-ignore: + - '.github/workflows/build_test_publish.yml' + - '.github/workflows/changelog_test.yml' + - 'docker/**' + - 'doc/**' + push: + branches: + - main + paths-ignore: + - '.github/workflows/build_test_publish.yml' + - '.github/workflows/changelog_test.yml' + - 'docker/**' + - 'doc/**' + +jobs: + BuildTest: + runs-on: ubuntu-latest + + strategy: + matrix: + ubuntu_versions : [ + 22.04, + ] + pkg_mgr : [ + apt, + conda + ] + + container: + image: ghcr.io/${{ github.repository_owner }}/cyclus_${{ matrix.ubuntu_versions }}_${{ matrix.pkg_mgr }} + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Building Cyclus + run: python install.py -j 2 --build-type=Release --core-version 999999.999999 + + - name: Cyclus Unit Tests + run: cyclus_unit + + - name: Cyclus Python Tests + run: pytest tests \ No newline at end of file diff --git a/.github/workflows/build_test_publish.yml b/.github/workflows/build_test_publish.yml index 5311b3fcd3..3dfad73747 100644 --- a/.github/workflows/build_test_publish.yml +++ b/.github/workflows/build_test_publish.yml @@ -5,8 +5,12 @@ on: workflow_dispatch: pull_request: push: - branches: - - main + paths: + - '.github/workflows/build_test_publish.yml' + - '.github/workflows/changelog_test.yml' + - 'docker/**' + paths-ignore: + - 'doc/**' jobs: build-dependency-and-test-img: From f493d99b9e36a5d1bdb5391ed7749e2169af8475 Mon Sep 17 00:00:00 2001 From: bennibbelink Date: Thu, 12 Oct 2023 20:30:56 -0500 Subject: [PATCH 042/102] can only define paths or paths-ignore, oops --- .github/workflows/build_test_publish.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/build_test_publish.yml b/.github/workflows/build_test_publish.yml index 3dfad73747..6a150f5184 100644 --- a/.github/workflows/build_test_publish.yml +++ b/.github/workflows/build_test_publish.yml @@ -9,8 +9,6 @@ on: - '.github/workflows/build_test_publish.yml' - '.github/workflows/changelog_test.yml' - 'docker/**' - paths-ignore: - - 'doc/**' jobs: build-dependency-and-test-img: From 902ae6812c50b0a955fd8c2c83974cac422b4c76 Mon Sep 17 00:00:00 2001 From: bennibbelink Date: Thu, 12 Oct 2023 20:35:59 -0500 Subject: [PATCH 043/102] changed titles and changed registry path --- .github/workflows/build_test.yml | 4 ++-- .github/workflows/build_test_publish.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build_test.yml b/.github/workflows/build_test.yml index ba22f1176a..b6c997fd78 100644 --- a/.github/workflows/build_test.yml +++ b/.github/workflows/build_test.yml @@ -1,4 +1,4 @@ -name: Linux Build/Test for PR and collaborator push +name: Build/Test for PR and collaborator push on: # allows us to run workflows manually @@ -35,7 +35,7 @@ jobs: ] container: - image: ghcr.io/${{ github.repository_owner }}/cyclus_${{ matrix.ubuntu_versions }}_${{ matrix.pkg_mgr }} + image: ghcr.io/cyclus/cyclus_${{ matrix.ubuntu_versions }}_${{ matrix.pkg_mgr }}/${{ matrix.pkg_mgr }}-deps steps: - name: Checkout repository diff --git a/.github/workflows/build_test_publish.yml b/.github/workflows/build_test_publish.yml index 6a150f5184..328d506239 100644 --- a/.github/workflows/build_test_publish.yml +++ b/.github/workflows/build_test_publish.yml @@ -1,4 +1,4 @@ -name: Build, Test & Publish docker images for future CI and other users +name: Build and Test Dependency Images on: # allows us to run workflows manually From 0cf64a2274e49e1ba00d3616d88a6f6f8e1d06a9 Mon Sep 17 00:00:00 2001 From: bennibbelink Date: Thu, 12 Oct 2023 20:48:11 -0500 Subject: [PATCH 044/102] fixed typo in cyclus_unit_tests --- .github/workflows/build_test.yml | 5 +++-- .github/workflows/build_test_publish.yml | 1 - 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build_test.yml b/.github/workflows/build_test.yml index b6c997fd78..3ad867af0f 100644 --- a/.github/workflows/build_test.yml +++ b/.github/workflows/build_test.yml @@ -8,6 +8,7 @@ on: - main paths-ignore: - '.github/workflows/build_test_publish.yml' + - '.github/workflows/build_test.yml' - '.github/workflows/changelog_test.yml' - 'docker/**' - 'doc/**' @@ -21,7 +22,7 @@ on: - 'doc/**' jobs: - BuildTest: + build-and-test: runs-on: ubuntu-latest strategy: @@ -45,7 +46,7 @@ jobs: run: python install.py -j 2 --build-type=Release --core-version 999999.999999 - name: Cyclus Unit Tests - run: cyclus_unit + run: cyclus_unit_test - name: Cyclus Python Tests run: pytest tests \ No newline at end of file diff --git a/.github/workflows/build_test_publish.yml b/.github/workflows/build_test_publish.yml index 328d506239..b04314dfe6 100644 --- a/.github/workflows/build_test_publish.yml +++ b/.github/workflows/build_test_publish.yml @@ -7,7 +7,6 @@ on: push: paths: - '.github/workflows/build_test_publish.yml' - - '.github/workflows/changelog_test.yml' - 'docker/**' jobs: From de49e40791f5f1edf845efc99926517ed132f0cd Mon Sep 17 00:00:00 2001 From: bennibbelink Date: Thu, 12 Oct 2023 20:49:13 -0500 Subject: [PATCH 045/102] fail-fast = false --- .github/workflows/build_test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build_test.yml b/.github/workflows/build_test.yml index 3ad867af0f..d913fa2e7c 100644 --- a/.github/workflows/build_test.yml +++ b/.github/workflows/build_test.yml @@ -26,6 +26,7 @@ jobs: runs-on: ubuntu-latest strategy: + fail-fast: false matrix: ubuntu_versions : [ 22.04, From f9697365fa6564f154b77a5a56312d53fe3c736a Mon Sep 17 00:00:00 2001 From: bennibbelink Date: Thu, 12 Oct 2023 21:06:01 -0500 Subject: [PATCH 046/102] another stupid typo --- .github/workflows/build_test.yml | 3 +-- .github/workflows/build_test_publish.yml | 2 ++ 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_test.yml b/.github/workflows/build_test.yml index d913fa2e7c..2a7a96de1f 100644 --- a/.github/workflows/build_test.yml +++ b/.github/workflows/build_test.yml @@ -8,7 +8,6 @@ on: - main paths-ignore: - '.github/workflows/build_test_publish.yml' - - '.github/workflows/build_test.yml' - '.github/workflows/changelog_test.yml' - 'docker/**' - 'doc/**' @@ -47,7 +46,7 @@ jobs: run: python install.py -j 2 --build-type=Release --core-version 999999.999999 - name: Cyclus Unit Tests - run: cyclus_unit_test + run: cyclus_unit_tests - name: Cyclus Python Tests run: pytest tests \ No newline at end of file diff --git a/.github/workflows/build_test_publish.yml b/.github/workflows/build_test_publish.yml index b04314dfe6..ffcb7f2081 100644 --- a/.github/workflows/build_test_publish.yml +++ b/.github/workflows/build_test_publish.yml @@ -5,6 +5,8 @@ on: workflow_dispatch: pull_request: push: + branches: + - main paths: - '.github/workflows/build_test_publish.yml' - 'docker/**' From 6df6ae8816e1892ff503f11a098c49e543974f4a Mon Sep 17 00:00:00 2001 From: bennibbelink Date: Thu, 12 Oct 2023 21:28:39 -0500 Subject: [PATCH 047/102] added binaries to path hopefully, added release workflow --- .github/workflows/build_test.yml | 3 ++ .github/workflows/publish_release.yml | 55 +++++++++++++++++++++++++++ 2 files changed, 58 insertions(+) create mode 100644 .github/workflows/publish_release.yml diff --git a/.github/workflows/build_test.yml b/.github/workflows/build_test.yml index 2a7a96de1f..edec292de2 100644 --- a/.github/workflows/build_test.yml +++ b/.github/workflows/build_test.yml @@ -45,6 +45,9 @@ jobs: - name: Building Cyclus run: python install.py -j 2 --build-type=Release --core-version 999999.999999 + - name: Add Cyclus binaries to PATH + run: export PATH=/root/.local/bin:$PATH + - name: Cyclus Unit Tests run: cyclus_unit_tests diff --git a/.github/workflows/publish_release.yml b/.github/workflows/publish_release.yml new file mode 100644 index 0000000000..6996da6926 --- /dev/null +++ b/.github/workflows/publish_release.yml @@ -0,0 +1,55 @@ +name: Publish Cyclus Release + +on: + # allows us to run workflows manually + workflow_dispatch: + release: + types: [published, created, edited] + +jobs: + build-and-test: + runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: + ubuntu_versions : [ + 22.04, + ] + pkg_mgr : [ + apt, + conda + ] + + name: Installing Dependencies, Building cyclus and running tests + steps: + - name: default environment + run: | + echo "tag-latest-on-default=false" >> "$GITHUB_ENV" + + - name: condition on trigger parameters + if: ${{ github.repository_owner == 'cyclus' }} + run: | + echo "tag-latest-on-default=true" >> "$GITHUB_ENV" + + - name: Log in to the Container registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Installing Dependencies in Docker image + uses: firehed/multistage-docker-build-action@v1 + with: + repository: ghcr.io/${{ github.repository_owner }}/cyclus_${{ matrix.ubuntu_versions }}_${{ matrix.pkg_mgr }}_${{ github.ref }} + stages: ${{ matrix.pkg_mgr }}-deps, cyclus # nice to have the deps image pushed for a release + server-stage: cyclus-test + quiet: false + parallel: true + tag-latest-on-default: ${{ env.tag-latest-on-default }} + dockerfile: docker/Dockerfile + build-args: pkg_mgr=${{ matrix.pkg_mgr }} \ No newline at end of file From a0fdfe63ecee738545aa96daa609928fc8ffb8c9 Mon Sep 17 00:00:00 2001 From: bennibbelink Date: Thu, 12 Oct 2023 21:49:03 -0500 Subject: [PATCH 048/102] correctly(?) adding bin to path --- .github/workflows/build_test.yml | 2 +- .github/workflows/publish_release.yml | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build_test.yml b/.github/workflows/build_test.yml index edec292de2..e322f53d06 100644 --- a/.github/workflows/build_test.yml +++ b/.github/workflows/build_test.yml @@ -46,7 +46,7 @@ jobs: run: python install.py -j 2 --build-type=Release --core-version 999999.999999 - name: Add Cyclus binaries to PATH - run: export PATH=/root/.local/bin:$PATH + run: echo "PATH=/root/.local/bin:$PATH" >> "$GITHUB_ENV" - name: Cyclus Unit Tests run: cyclus_unit_tests diff --git a/.github/workflows/publish_release.yml b/.github/workflows/publish_release.yml index 6996da6926..19deba0a2a 100644 --- a/.github/workflows/publish_release.yml +++ b/.github/workflows/publish_release.yml @@ -1,8 +1,6 @@ name: Publish Cyclus Release on: - # allows us to run workflows manually - workflow_dispatch: release: types: [published, created, edited] @@ -46,7 +44,7 @@ jobs: uses: firehed/multistage-docker-build-action@v1 with: repository: ghcr.io/${{ github.repository_owner }}/cyclus_${{ matrix.ubuntu_versions }}_${{ matrix.pkg_mgr }}_${{ github.ref }} - stages: ${{ matrix.pkg_mgr }}-deps, cyclus # nice to have the deps image pushed for a release + stages: ${{ matrix.pkg_mgr }}-deps, cyclus server-stage: cyclus-test quiet: false parallel: true From 714730b870b37280b02e7e0054fad63a0b199fe9 Mon Sep 17 00:00:00 2001 From: bennibbelink Date: Thu, 12 Oct 2023 22:06:19 -0500 Subject: [PATCH 049/102] just calling tests by absolute path --- .github/workflows/build_test.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build_test.yml b/.github/workflows/build_test.yml index e322f53d06..8563484dda 100644 --- a/.github/workflows/build_test.yml +++ b/.github/workflows/build_test.yml @@ -45,11 +45,8 @@ jobs: - name: Building Cyclus run: python install.py -j 2 --build-type=Release --core-version 999999.999999 - - name: Add Cyclus binaries to PATH - run: echo "PATH=/root/.local/bin:$PATH" >> "$GITHUB_ENV" - - name: Cyclus Unit Tests - run: cyclus_unit_tests + run: /root/.local/bin/cyclus_unit_tests - name: Cyclus Python Tests - run: pytest tests \ No newline at end of file + run: pytest-3 tests \ No newline at end of file From b837f40c8bc497c6b653beab068f847d4631ff89 Mon Sep 17 00:00:00 2001 From: bennibbelink Date: Thu, 12 Oct 2023 22:18:00 -0500 Subject: [PATCH 050/102] call pytest differently so it works in both pkg_mgr versions --- .github/workflows/build_test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_test.yml b/.github/workflows/build_test.yml index 8563484dda..d43a5256b5 100644 --- a/.github/workflows/build_test.yml +++ b/.github/workflows/build_test.yml @@ -46,7 +46,7 @@ jobs: run: python install.py -j 2 --build-type=Release --core-version 999999.999999 - name: Cyclus Unit Tests - run: /root/.local/bin/cyclus_unit_tests + run: ./build/bin/cyclus_unit_tests - name: Cyclus Python Tests - run: pytest-3 tests \ No newline at end of file + run: cd tests && python -m pytest \ No newline at end of file From 4ded756621c30eb7f5276eeee044200dbe74039e Mon Sep 17 00:00:00 2001 From: bennibbelink Date: Fri, 13 Oct 2023 16:01:41 -0500 Subject: [PATCH 051/102] ignore test_main.py --- .github/workflows/build_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_test.yml b/.github/workflows/build_test.yml index d43a5256b5..aefb3cb473 100644 --- a/.github/workflows/build_test.yml +++ b/.github/workflows/build_test.yml @@ -49,4 +49,4 @@ jobs: run: ./build/bin/cyclus_unit_tests - name: Cyclus Python Tests - run: cd tests && python -m pytest \ No newline at end of file + run: cd tests && python -m pytest --ignore test_main.py \ No newline at end of file From 9f6237121a52e30db2d9948c4c87c9dbc8a9d607 Mon Sep 17 00:00:00 2001 From: bennibbelink Date: Fri, 13 Oct 2023 16:06:08 -0500 Subject: [PATCH 052/102] update changelog --- CHANGELOG.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 827dd25a3e..aa81caca72 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -17,7 +17,7 @@ Since last release will fail unless update-alternatives has been used to point python at the correct python3 version (#1558) * build and test are now fown on githubAction in place or CircleCI (#1569) -* AddMutalReqs and AddReciepe functions and exclusive bids in python API of DRE (#1584) +* Have separate workflows for testing, publishing dependency images, and publishing release images (#1597) **Changed:** From 48995562692f746c4d1141cf50460e3ecec82ce3 Mon Sep 17 00:00:00 2001 From: bennibbelink Date: Sat, 14 Oct 2023 20:23:55 -0500 Subject: [PATCH 053/102] modify env vars after building --- .github/workflows/build_test.yml | 12 +++++++++--- .github/workflows/publish_release.yml | 2 +- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build_test.yml b/.github/workflows/build_test.yml index aefb3cb473..426c8c18f1 100644 --- a/.github/workflows/build_test.yml +++ b/.github/workflows/build_test.yml @@ -34,7 +34,7 @@ jobs: apt, conda ] - + container: image: ghcr.io/cyclus/cyclus_${{ matrix.ubuntu_versions }}_${{ matrix.pkg_mgr }}/${{ matrix.pkg_mgr }}-deps @@ -45,8 +45,14 @@ jobs: - name: Building Cyclus run: python install.py -j 2 --build-type=Release --core-version 999999.999999 + - name: Add bins to PATH + run: echo "/root/.local/bin:$PATH" >> $GITHUB_PATH + + - name: Add libs to LD_LIBRARY_PATH + run: echo "/root/.local/lib:/root/.local/lib/cyclus:$LD_LIBRARYPATH" >> $GITHUB_PATH + - name: Cyclus Unit Tests - run: ./build/bin/cyclus_unit_tests + run: cyclus_unit_tests - name: Cyclus Python Tests - run: cd tests && python -m pytest --ignore test_main.py \ No newline at end of file + run: cd tests && python -m pytest --ignore test_main.py diff --git a/.github/workflows/publish_release.yml b/.github/workflows/publish_release.yml index 19deba0a2a..dc697bb72f 100644 --- a/.github/workflows/publish_release.yml +++ b/.github/workflows/publish_release.yml @@ -5,7 +5,7 @@ on: types: [published, created, edited] jobs: - build-and-test: + build-and-test-for-release: runs-on: ubuntu-latest strategy: From 2135a808eb06d6adc83502f44edf7984379cc0a8 Mon Sep 17 00:00:00 2001 From: bennibbelink Date: Sat, 14 Oct 2023 20:58:56 -0500 Subject: [PATCH 054/102] export env vars in same shell? --- .github/workflows/build_test.yml | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build_test.yml b/.github/workflows/build_test.yml index 426c8c18f1..6bcc2f8230 100644 --- a/.github/workflows/build_test.yml +++ b/.github/workflows/build_test.yml @@ -45,14 +45,21 @@ jobs: - name: Building Cyclus run: python install.py -j 2 --build-type=Release --core-version 999999.999999 - - name: Add bins to PATH - run: echo "/root/.local/bin:$PATH" >> $GITHUB_PATH + # - name: Add bins to PATH + # run: echo "/root/.local/bin:$PATH" >> $GITHUB_PATH - - name: Add libs to LD_LIBRARY_PATH - run: echo "/root/.local/lib:/root/.local/lib/cyclus:$LD_LIBRARYPATH" >> $GITHUB_PATH + # - name: Add libs to LD_LIBRARY_PATH + # run: echo "/root/.local/lib:/root/.local/lib/cyclus:$LD_LIBRARY_PATH" >> $GITHUB_PATH - name: Cyclus Unit Tests - run: cyclus_unit_tests + run: | + export PATH=$PATH:/root/.local/bin + export LD_LIBRARY_PATH=/root/.local/lib:/root/.local/lib/cyclus:$LD_LIBRARY_PATH + cyclus_unit_tests - name: Cyclus Python Tests - run: cd tests && python -m pytest --ignore test_main.py + run: | + export PATH=$PATH:/root/.local/bin + export LD_LIBRARY_PATH=/root/.local/lib:/root/.local/lib/cyclus:$LD_LIBRARY_PATH + cd tests + python -m pytest --ignore test_main.py From 1f109ad5085aec49f757a83869b9712877401b56 Mon Sep 17 00:00:00 2001 From: bennibbelink Date: Sat, 14 Oct 2023 21:03:53 -0500 Subject: [PATCH 055/102] GITHUB_ENV typo, now env vars might work --- .github/workflows/build_test.yml | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/.github/workflows/build_test.yml b/.github/workflows/build_test.yml index 6bcc2f8230..e4861040bb 100644 --- a/.github/workflows/build_test.yml +++ b/.github/workflows/build_test.yml @@ -45,21 +45,14 @@ jobs: - name: Building Cyclus run: python install.py -j 2 --build-type=Release --core-version 999999.999999 - # - name: Add bins to PATH - # run: echo "/root/.local/bin:$PATH" >> $GITHUB_PATH + - name: Add bins to PATH + run: echo "/root/.local/bin:$PATH" >> $GITHUB_ENV - # - name: Add libs to LD_LIBRARY_PATH - # run: echo "/root/.local/lib:/root/.local/lib/cyclus:$LD_LIBRARY_PATH" >> $GITHUB_PATH + - name: Add libs to LD_LIBRARY_PATH + run: echo "/root/.local/lib:/root/.local/lib/cyclus:$LD_LIBRARY_PATH" >> $GITHUB_ENV - name: Cyclus Unit Tests - run: | - export PATH=$PATH:/root/.local/bin - export LD_LIBRARY_PATH=/root/.local/lib:/root/.local/lib/cyclus:$LD_LIBRARY_PATH - cyclus_unit_tests + run: cyclus_unit_tests - name: Cyclus Python Tests - run: | - export PATH=$PATH:/root/.local/bin - export LD_LIBRARY_PATH=/root/.local/lib:/root/.local/lib/cyclus:$LD_LIBRARY_PATH - cd tests - python -m pytest --ignore test_main.py + run: cd tests python -m pytest --ignore test_main.py From bde36e68804477931e6748a721aa75242c454488 Mon Sep 17 00:00:00 2001 From: bennibbelink Date: Sat, 14 Oct 2023 21:13:49 -0500 Subject: [PATCH 056/102] fix another mistake setting the path --- .github/workflows/build_test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_test.yml b/.github/workflows/build_test.yml index e4861040bb..c226da07b6 100644 --- a/.github/workflows/build_test.yml +++ b/.github/workflows/build_test.yml @@ -46,10 +46,10 @@ jobs: run: python install.py -j 2 --build-type=Release --core-version 999999.999999 - name: Add bins to PATH - run: echo "/root/.local/bin:$PATH" >> $GITHUB_ENV + run: echo "PATH=/root/.local/bin:$PATH" >> $GITHUB_ENV - name: Add libs to LD_LIBRARY_PATH - run: echo "/root/.local/lib:/root/.local/lib/cyclus:$LD_LIBRARY_PATH" >> $GITHUB_ENV + run: echo "LD_LIBRARY_PATH=/root/.local/lib:/root/.local/lib/cyclus:$LD_LIBRARY_PATH" >> $GITHUB_ENV - name: Cyclus Unit Tests run: cyclus_unit_tests From 7cb2a4179a199305745ca6be0e6d59b3518d39d6 Mon Sep 17 00:00:00 2001 From: bennibbelink Date: Sat, 14 Oct 2023 21:53:46 -0500 Subject: [PATCH 057/102] echo PATH --- .github/workflows/build_test.yml | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build_test.yml b/.github/workflows/build_test.yml index c226da07b6..a35e3db642 100644 --- a/.github/workflows/build_test.yml +++ b/.github/workflows/build_test.yml @@ -46,13 +46,25 @@ jobs: run: python install.py -j 2 --build-type=Release --core-version 999999.999999 - name: Add bins to PATH - run: echo "PATH=/root/.local/bin:$PATH" >> $GITHUB_ENV + run: | + echo "PATH=/root/.local/bin:$PATH" >> $GITHUB_ENV - name: Add libs to LD_LIBRARY_PATH - run: echo "LD_LIBRARY_PATH=/root/.local/lib:/root/.local/lib/cyclus:$LD_LIBRARY_PATH" >> $GITHUB_ENV + run: | + echo "LD_LIBRARY_PATH=/root/.local/lib:/root/.local/lib/cyclus:$LD_LIBRARY_PATH" >> $GITHUB_ENV + + - name: Confirm PATH + run: | + echo $PATH + + - name: Confirm LD_LIBRARY_PATH + run: | + echo $LD_LIBRARY_PATH - name: Cyclus Unit Tests - run: cyclus_unit_tests + run: | + cyclus_unit_tests - name: Cyclus Python Tests - run: cd tests python -m pytest --ignore test_main.py + run: | + cd tests && python -m pytest --ignore test_main.py From d3a20c9c8cbef7fc9fe1b5be87fadfd730047e17 Mon Sep 17 00:00:00 2001 From: bennibbelink Date: Sat, 14 Oct 2023 21:54:33 -0500 Subject: [PATCH 058/102] added missing quotation marks --- .github/workflows/build_test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_test.yml b/.github/workflows/build_test.yml index a35e3db642..6f2fcb0734 100644 --- a/.github/workflows/build_test.yml +++ b/.github/workflows/build_test.yml @@ -47,11 +47,11 @@ jobs: - name: Add bins to PATH run: | - echo "PATH=/root/.local/bin:$PATH" >> $GITHUB_ENV + echo "PATH=/root/.local/bin:$PATH" >> "$GITHUB_ENV" - name: Add libs to LD_LIBRARY_PATH run: | - echo "LD_LIBRARY_PATH=/root/.local/lib:/root/.local/lib/cyclus:$LD_LIBRARY_PATH" >> $GITHUB_ENV + echo "LD_LIBRARY_PATH=/root/.local/lib:/root/.local/lib/cyclus:$LD_LIBRARY_PATH" >> "$GITHUB_ENV" - name: Confirm PATH run: | From 3f258bf5d5b5bee247178d3c0b291708024abc75 Mon Sep 17 00:00:00 2001 From: bennibbelink Date: Sat, 14 Oct 2023 22:17:06 -0500 Subject: [PATCH 059/102] not sure why it won't find cyclus_unit_tests --- .github/workflows/build_test.yml | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/.github/workflows/build_test.yml b/.github/workflows/build_test.yml index 6f2fcb0734..4f5efedbde 100644 --- a/.github/workflows/build_test.yml +++ b/.github/workflows/build_test.yml @@ -53,17 +53,9 @@ jobs: run: | echo "LD_LIBRARY_PATH=/root/.local/lib:/root/.local/lib/cyclus:$LD_LIBRARY_PATH" >> "$GITHUB_ENV" - - name: Confirm PATH - run: | - echo $PATH - - - name: Confirm LD_LIBRARY_PATH - run: | - echo $LD_LIBRARY_PATH - - name: Cyclus Unit Tests run: | - cyclus_unit_tests + /root/.local/bin/cyclus_unit_tests - name: Cyclus Python Tests run: | From df12cec6655b55c02f0877045360b671860a5224 Mon Sep 17 00:00:00 2001 From: bennibbelink Date: Sat, 14 Oct 2023 22:39:30 -0500 Subject: [PATCH 060/102] use /home/ben instead of hardcoded path --- .github/workflows/build_test.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build_test.yml b/.github/workflows/build_test.yml index 4f5efedbde..84967db05f 100644 --- a/.github/workflows/build_test.yml +++ b/.github/workflows/build_test.yml @@ -47,15 +47,15 @@ jobs: - name: Add bins to PATH run: | - echo "PATH=/root/.local/bin:$PATH" >> "$GITHUB_ENV" + echo "PATH=${HOME}/.local/bin:$PATH" >> "$GITHUB_ENV" - name: Add libs to LD_LIBRARY_PATH run: | - echo "LD_LIBRARY_PATH=/root/.local/lib:/root/.local/lib/cyclus:$LD_LIBRARY_PATH" >> "$GITHUB_ENV" + echo "LD_LIBRARY_PATH=${HOME}/.local/lib:${HOME}/.local/lib/cyclus:$LD_LIBRARY_PATH" >> "$GITHUB_ENV" - name: Cyclus Unit Tests run: | - /root/.local/bin/cyclus_unit_tests + cyclus_unit_tests - name: Cyclus Python Tests run: | From 633e57318d47cd3818063758c043eb7fb57687bb Mon Sep 17 00:00:00 2001 From: bennibbelink Date: Mon, 16 Oct 2023 13:00:32 -0500 Subject: [PATCH 061/102] using libmamba solver to install mamba --- docker/Dockerfile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docker/Dockerfile b/docker/Dockerfile index baa59ade8b..d944986004 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -59,8 +59,13 @@ RUN echo 'export PATH=/opt/conda/bin:$PATH' > /etc/profile.d/conda.sh && \ ENV PATH /opt/conda/bin:$PATH RUN conda config --add channels conda-forge +# RUN conda config --set channel_priority flexible RUN conda update -n base -c defaults conda +RUN conda install -y conda-libmamba-solver +RUN conda config --set solver libmamba RUN conda install -y mamba +RUN conda uninstall conda-libmamba-solver +RUN conda config --set solver classic RUN conda update -y --all && \ mamba install -y \ openssh \ From a6a0b7f1538952c9947681a7713439b29b7aa154 Mon Sep 17 00:00:00 2001 From: bennibbelink Date: Mon, 16 Oct 2023 15:59:57 -0500 Subject: [PATCH 062/102] using libmamba and mamba --- docker/Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index d944986004..16f8531364 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -59,12 +59,11 @@ RUN echo 'export PATH=/opt/conda/bin:$PATH' > /etc/profile.d/conda.sh && \ ENV PATH /opt/conda/bin:$PATH RUN conda config --add channels conda-forge -# RUN conda config --set channel_priority flexible RUN conda update -n base -c defaults conda RUN conda install -y conda-libmamba-solver RUN conda config --set solver libmamba RUN conda install -y mamba -RUN conda uninstall conda-libmamba-solver +RUN conda uninstall -y conda-libmamba-solver RUN conda config --set solver classic RUN conda update -y --all && \ mamba install -y \ From 6562dc6bbbd89215543fee59478386ea0abf7eee Mon Sep 17 00:00:00 2001 From: bennibbelink Date: Mon, 16 Oct 2023 16:01:01 -0500 Subject: [PATCH 063/102] trigger on push to any branch --- .github/workflows/build_test.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/build_test.yml b/.github/workflows/build_test.yml index 84967db05f..b2faed4bf2 100644 --- a/.github/workflows/build_test.yml +++ b/.github/workflows/build_test.yml @@ -12,8 +12,6 @@ on: - 'docker/**' - 'doc/**' push: - branches: - - main paths-ignore: - '.github/workflows/build_test_publish.yml' - '.github/workflows/changelog_test.yml' From 94b2f97abf2e4279a209766f630c0b76764b775f Mon Sep 17 00:00:00 2001 From: bennibbelink Date: Mon, 16 Oct 2023 16:07:45 -0500 Subject: [PATCH 064/102] combine workflow steps --- .github/workflows/build_test.yml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/.github/workflows/build_test.yml b/.github/workflows/build_test.yml index b2faed4bf2..33de9b83ae 100644 --- a/.github/workflows/build_test.yml +++ b/.github/workflows/build_test.yml @@ -41,14 +41,9 @@ jobs: uses: actions/checkout@v3 - name: Building Cyclus - run: python install.py -j 2 --build-type=Release --core-version 999999.999999 - - - name: Add bins to PATH run: | + python install.py -j 2 --build-type=Release --core-version 999999.999999 echo "PATH=${HOME}/.local/bin:$PATH" >> "$GITHUB_ENV" - - - name: Add libs to LD_LIBRARY_PATH - run: | echo "LD_LIBRARY_PATH=${HOME}/.local/lib:${HOME}/.local/lib/cyclus:$LD_LIBRARY_PATH" >> "$GITHUB_ENV" - name: Cyclus Unit Tests From c7e52b8ce23345656373f3ee49255899a6b47292 Mon Sep 17 00:00:00 2001 From: bennibbelink Date: Mon, 16 Oct 2023 16:20:02 -0500 Subject: [PATCH 065/102] pull from ghcr of repository owner instead of hardcoded cyclus --- .github/workflows/build_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_test.yml b/.github/workflows/build_test.yml index 33de9b83ae..de2eb13bd4 100644 --- a/.github/workflows/build_test.yml +++ b/.github/workflows/build_test.yml @@ -34,7 +34,7 @@ jobs: ] container: - image: ghcr.io/cyclus/cyclus_${{ matrix.ubuntu_versions }}_${{ matrix.pkg_mgr }}/${{ matrix.pkg_mgr }}-deps + image: ghcr.io/${{ github.repository_owner }}/cyclus_${{ matrix.ubuntu_versions }}_${{ matrix.pkg_mgr }}/${{ matrix.pkg_mgr }}-deps steps: - name: Checkout repository From 61c9ff1bb8d5cfb0caa000836c5881454dbddfef Mon Sep 17 00:00:00 2001 From: bennibbelink Date: Mon, 16 Oct 2023 16:55:01 -0500 Subject: [PATCH 066/102] removed branch requirement in build_test.yml --- .github/workflows/build_test.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/build_test.yml b/.github/workflows/build_test.yml index de2eb13bd4..3dfcf942fc 100644 --- a/.github/workflows/build_test.yml +++ b/.github/workflows/build_test.yml @@ -4,8 +4,6 @@ on: # allows us to run workflows manually workflow_dispatch: pull_request: - branches: - - main paths-ignore: - '.github/workflows/build_test_publish.yml' - '.github/workflows/changelog_test.yml' @@ -37,7 +35,7 @@ jobs: image: ghcr.io/${{ github.repository_owner }}/cyclus_${{ matrix.ubuntu_versions }}_${{ matrix.pkg_mgr }}/${{ matrix.pkg_mgr }}-deps steps: - - name: Checkout repository + - name: Checkout and Build Cyclus uses: actions/checkout@v3 - name: Building Cyclus From 80852a3771084035f5f45aecca746f4e016ea7fe Mon Sep 17 00:00:00 2001 From: bennibbelink Date: Thu, 19 Oct 2023 12:17:23 -0500 Subject: [PATCH 067/102] add 20.04 support --- .github/workflows/build_test.yml | 1 + .github/workflows/build_test_publish.yml | 3 ++- .github/workflows/publish_release.yml | 3 ++- docker/Dockerfile | 12 +++++++----- 4 files changed, 12 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build_test.yml b/.github/workflows/build_test.yml index 3dfcf942fc..118f5fa643 100644 --- a/.github/workflows/build_test.yml +++ b/.github/workflows/build_test.yml @@ -24,6 +24,7 @@ jobs: fail-fast: false matrix: ubuntu_versions : [ + 20.04, 22.04, ] pkg_mgr : [ diff --git a/.github/workflows/build_test_publish.yml b/.github/workflows/build_test_publish.yml index ffcb7f2081..3cc6962548 100644 --- a/.github/workflows/build_test_publish.yml +++ b/.github/workflows/build_test_publish.yml @@ -18,6 +18,7 @@ jobs: strategy: matrix: ubuntu_versions : [ + 20.04, 22.04, ] pkg_mgr : [ @@ -56,4 +57,4 @@ jobs: parallel: true tag-latest-on-default: ${{ env.tag-latest-on-default }} dockerfile: docker/Dockerfile - build-args: pkg_mgr=${{ matrix.pkg_mgr }} \ No newline at end of file + build-args: pkg_mgr=${{ matrix.pkg_mgr }}, ubuntu_version=${{ matrix.ubuntu_versions }} \ No newline at end of file diff --git a/.github/workflows/publish_release.yml b/.github/workflows/publish_release.yml index dc697bb72f..b52b924fd0 100644 --- a/.github/workflows/publish_release.yml +++ b/.github/workflows/publish_release.yml @@ -12,6 +12,7 @@ jobs: fail-fast: false matrix: ubuntu_versions : [ + 20.04, 22.04, ] pkg_mgr : [ @@ -50,4 +51,4 @@ jobs: parallel: true tag-latest-on-default: ${{ env.tag-latest-on-default }} dockerfile: docker/Dockerfile - build-args: pkg_mgr=${{ matrix.pkg_mgr }} \ No newline at end of file + build-args: pkg_mgr=${{ matrix.pkg_mgr }}, ubuntu_version=${{ matrix.ubuntu_versions }} \ No newline at end of file diff --git a/docker/Dockerfile b/docker/Dockerfile index 16f8531364..963966c5c4 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,7 +1,8 @@ ARG pkg_mgr=apt ARG make_cores=2 +ARG ubuntu_version=22.04 -FROM ubuntu:22.04 as common-base +FROM ubuntu:${ubuntu_version} as common-base ENV TZ=America/Chicago RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone @@ -37,9 +38,9 @@ RUN apt install -y \ python3-pandas \ python3-jinja2 \ cython3 \ - libwebsockets-dev \ - python3-pprintpp \ - && apt clean -y all + libwebsockets-dev +RUN apt install -y python3-pprintpp; exit 0 +RUN apt clean -y all RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 10 @@ -119,6 +120,7 @@ WORKDIR /cyclus # You may add the option "--cmake-debug" to the following command # for further CMake debugging. +RUN mkdir -p `python -m site --user-site` RUN python install.py -j ${make_cores} --build-type=Release --core-version 999999.999999 ENV PATH /root/.local/bin:$PATH ENV LD_LIBRARY_PATH /root/.local/lib:/root/.local/lib/cyclus @@ -129,5 +131,5 @@ RUN cyclus_unit_tests FROM cyclus-test as cyclus-pytest -RUN cd tests && pytest +RUN cd tests && python -m pytest --ignore test_main.py From de8fe02be981524f457e558b4622cb1f7664dbb0 Mon Sep 17 00:00:00 2001 From: bennibbelink Date: Thu, 19 Oct 2023 12:24:29 -0500 Subject: [PATCH 068/102] update changelog --- CHANGELOG.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index aa81caca72..8600c6a9f4 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -18,6 +18,7 @@ Since last release correct python3 version (#1558) * build and test are now fown on githubAction in place or CircleCI (#1569) * Have separate workflows for testing, publishing dependency images, and publishing release images (#1597) +* Add support for Ubuntu 20.04 (#1605) **Changed:** From 8ff7aa14d5220a537ed36c3111346ec422ce2778 Mon Sep 17 00:00:00 2001 From: "Paul P.H. Wilson" Date: Wed, 18 Oct 2023 11:08:52 -0500 Subject: [PATCH 069/102] limit PR conditions for running this workflow --- .github/workflows/build_test_publish.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/build_test_publish.yml b/.github/workflows/build_test_publish.yml index 3cc6962548..176264cf6b 100644 --- a/.github/workflows/build_test_publish.yml +++ b/.github/workflows/build_test_publish.yml @@ -4,6 +4,9 @@ on: # allows us to run workflows manually workflow_dispatch: pull_request: + paths: + - '.github/workflows/build_test_publish.yml' + - 'docker/**' push: branches: - main From 6717e252a9fe00d48d562de27a7e208a78d1198c Mon Sep 17 00:00:00 2001 From: "Paul P.H. Wilson" Date: Wed, 18 Oct 2023 11:10:07 -0500 Subject: [PATCH 070/102] update changelog --- CHANGELOG.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 8600c6a9f4..a9e0561aca 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -17,8 +17,7 @@ Since last release will fail unless update-alternatives has been used to point python at the correct python3 version (#1558) * build and test are now fown on githubAction in place or CircleCI (#1569) -* Have separate workflows for testing, publishing dependency images, and publishing release images (#1597) -* Add support for Ubuntu 20.04 (#1605) +* Have separate workflows for testing, publishing dependency images, and publishing release images (#1597, #1601) **Changed:** From 7c5b9a4af0542c1be408ac670c3a7cf555c3ff13 Mon Sep 17 00:00:00 2001 From: "Paul P.H. Wilson" Date: Wed, 18 Oct 2023 12:26:51 -0500 Subject: [PATCH 071/102] update PR number --- CHANGELOG.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index a9e0561aca..1481d27826 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -17,7 +17,7 @@ Since last release will fail unless update-alternatives has been used to point python at the correct python3 version (#1558) * build and test are now fown on githubAction in place or CircleCI (#1569) -* Have separate workflows for testing, publishing dependency images, and publishing release images (#1597, #1601) +* Have separate workflows for testing, publishing dependency images, and publishing release images (#1597, #1602) **Changed:** From f4c4eef260b055e9c90aac40a84560da72f0f905 Mon Sep 17 00:00:00 2001 From: bennibbelink Date: Thu, 19 Oct 2023 12:37:52 -0500 Subject: [PATCH 072/102] hardcoded cyclus --- .github/workflows/build_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_test.yml b/.github/workflows/build_test.yml index 118f5fa643..738f5dc857 100644 --- a/.github/workflows/build_test.yml +++ b/.github/workflows/build_test.yml @@ -33,7 +33,7 @@ jobs: ] container: - image: ghcr.io/${{ github.repository_owner }}/cyclus_${{ matrix.ubuntu_versions }}_${{ matrix.pkg_mgr }}/${{ matrix.pkg_mgr }}-deps + image: ghcr.io/cyclus/cyclus_${{ matrix.ubuntu_versions }}_${{ matrix.pkg_mgr }}/${{ matrix.pkg_mgr }}-deps steps: - name: Checkout and Build Cyclus From ddb3034cdf74f3012f9e8462b118ecb3f0bc9399 Mon Sep 17 00:00:00 2001 From: bennibbelink Date: Thu, 19 Oct 2023 12:39:01 -0500 Subject: [PATCH 073/102] update changelog --- CHANGELOG.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 1481d27826..e4a9df2277 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -17,7 +17,7 @@ Since last release will fail unless update-alternatives has been used to point python at the correct python3 version (#1558) * build and test are now fown on githubAction in place or CircleCI (#1569) -* Have separate workflows for testing, publishing dependency images, and publishing release images (#1597, #1602) +* Have separate workflows for testing, publishing dependency images, and publishing release images (#1597, #1606) **Changed:** From c8d85813e06510bc67f1d1a5c20367ddc40d70bf Mon Sep 17 00:00:00 2001 From: bennibbelink Date: Fri, 20 Oct 2023 12:39:09 -0500 Subject: [PATCH 074/102] only changing dep image --- docker/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 963966c5c4..b356f3be82 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -38,7 +38,8 @@ RUN apt install -y \ python3-pandas \ python3-jinja2 \ cython3 \ - libwebsockets-dev + libwebsockets-dev + RUN apt install -y python3-pprintpp; exit 0 RUN apt clean -y all From 24a0ccc28781e0dfb689fce6ab15c8c144b3604a Mon Sep 17 00:00:00 2001 From: bennibbelink Date: Fri, 20 Oct 2023 12:45:22 -0500 Subject: [PATCH 075/102] reset other changes --- .github/workflows/build_test.yml | 3 +-- .github/workflows/publish_release.yml | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build_test.yml b/.github/workflows/build_test.yml index 738f5dc857..c378260a75 100644 --- a/.github/workflows/build_test.yml +++ b/.github/workflows/build_test.yml @@ -24,7 +24,6 @@ jobs: fail-fast: false matrix: ubuntu_versions : [ - 20.04, 22.04, ] pkg_mgr : [ @@ -51,4 +50,4 @@ jobs: - name: Cyclus Python Tests run: | - cd tests && python -m pytest --ignore test_main.py + cd tests && python -m pytest --ignore test_main.py \ No newline at end of file diff --git a/.github/workflows/publish_release.yml b/.github/workflows/publish_release.yml index b52b924fd0..dc697bb72f 100644 --- a/.github/workflows/publish_release.yml +++ b/.github/workflows/publish_release.yml @@ -12,7 +12,6 @@ jobs: fail-fast: false matrix: ubuntu_versions : [ - 20.04, 22.04, ] pkg_mgr : [ @@ -51,4 +50,4 @@ jobs: parallel: true tag-latest-on-default: ${{ env.tag-latest-on-default }} dockerfile: docker/Dockerfile - build-args: pkg_mgr=${{ matrix.pkg_mgr }}, ubuntu_version=${{ matrix.ubuntu_versions }} \ No newline at end of file + build-args: pkg_mgr=${{ matrix.pkg_mgr }} \ No newline at end of file From b00d1915713854775772178d328bb8bb75d7aac7 Mon Sep 17 00:00:00 2001 From: bennibbelink Date: Fri, 20 Oct 2023 12:46:18 -0500 Subject: [PATCH 076/102] new line at end of file --- .github/workflows/build_test.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_test.yml b/.github/workflows/build_test.yml index c378260a75..bca3cc4b30 100644 --- a/.github/workflows/build_test.yml +++ b/.github/workflows/build_test.yml @@ -50,4 +50,5 @@ jobs: - name: Cyclus Python Tests run: | - cd tests && python -m pytest --ignore test_main.py \ No newline at end of file + cd tests && python -m pytest --ignore test_main.py + \ No newline at end of file From c00675b3041ee7006c6a1779e297cbcb8c7b3ab4 Mon Sep 17 00:00:00 2001 From: bennibbelink Date: Fri, 20 Oct 2023 13:02:22 -0500 Subject: [PATCH 077/102] reset files to main --- .github/workflows/build_test.yml | 1 - CHANGELOG.rst | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/build_test.yml b/.github/workflows/build_test.yml index bca3cc4b30..95aa501ffb 100644 --- a/.github/workflows/build_test.yml +++ b/.github/workflows/build_test.yml @@ -51,4 +51,3 @@ jobs: - name: Cyclus Python Tests run: | cd tests && python -m pytest --ignore test_main.py - \ No newline at end of file diff --git a/CHANGELOG.rst b/CHANGELOG.rst index e4a9df2277..7ac3e4e828 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -17,7 +17,7 @@ Since last release will fail unless update-alternatives has been used to point python at the correct python3 version (#1558) * build and test are now fown on githubAction in place or CircleCI (#1569) -* Have separate workflows for testing, publishing dependency images, and publishing release images (#1597, #1606) +* Have separate workflows for testing, publishing dependency images, and publishing release images (#1597, #1602, #1606) **Changed:** From a0045dda0d240724ce39a54164722521a4221a0d Mon Sep 17 00:00:00 2001 From: bennibbelink Date: Fri, 20 Oct 2023 13:09:30 -0500 Subject: [PATCH 078/102] update changelog --- CHANGELOG.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 7ac3e4e828..8b0821d040 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -18,7 +18,7 @@ Since last release correct python3 version (#1558) * build and test are now fown on githubAction in place or CircleCI (#1569) * Have separate workflows for testing, publishing dependency images, and publishing release images (#1597, #1602, #1606) - +* Add Ubuntu 20.04 to the list of supported platforms (#1605) **Changed:** From e2dcf53aeb797d6e8de005d4efb3491ad96222bc Mon Sep 17 00:00:00 2001 From: bennibbelink Date: Fri, 20 Oct 2023 22:15:41 -0500 Subject: [PATCH 079/102] added 20.04 to PR and release workflows --- .github/workflows/build_test.yml | 1 + .github/workflows/publish_release.yml | 3 ++- CHANGELOG.rst | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_test.yml b/.github/workflows/build_test.yml index 95aa501ffb..738f5dc857 100644 --- a/.github/workflows/build_test.yml +++ b/.github/workflows/build_test.yml @@ -24,6 +24,7 @@ jobs: fail-fast: false matrix: ubuntu_versions : [ + 20.04, 22.04, ] pkg_mgr : [ diff --git a/.github/workflows/publish_release.yml b/.github/workflows/publish_release.yml index dc697bb72f..b52b924fd0 100644 --- a/.github/workflows/publish_release.yml +++ b/.github/workflows/publish_release.yml @@ -12,6 +12,7 @@ jobs: fail-fast: false matrix: ubuntu_versions : [ + 20.04, 22.04, ] pkg_mgr : [ @@ -50,4 +51,4 @@ jobs: parallel: true tag-latest-on-default: ${{ env.tag-latest-on-default }} dockerfile: docker/Dockerfile - build-args: pkg_mgr=${{ matrix.pkg_mgr }} \ No newline at end of file + build-args: pkg_mgr=${{ matrix.pkg_mgr }}, ubuntu_version=${{ matrix.ubuntu_versions }} \ No newline at end of file diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 8b0821d040..8c74fe8d90 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -18,7 +18,7 @@ Since last release correct python3 version (#1558) * build and test are now fown on githubAction in place or CircleCI (#1569) * Have separate workflows for testing, publishing dependency images, and publishing release images (#1597, #1602, #1606) -* Add Ubuntu 20.04 to the list of supported platforms (#1605) +* Add Ubuntu 20.04 to the list of supported platforms (#1605, #1608) **Changed:** From 8a7ae5c3946a66886fdb84261539ad95d4ed932b Mon Sep 17 00:00:00 2001 From: bennibbelink Date: Fri, 20 Oct 2023 22:58:37 -0500 Subject: [PATCH 080/102] need to create site packages dir --- .github/workflows/build_test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build_test.yml b/.github/workflows/build_test.yml index 738f5dc857..a8cc6e649e 100644 --- a/.github/workflows/build_test.yml +++ b/.github/workflows/build_test.yml @@ -41,6 +41,7 @@ jobs: - name: Building Cyclus run: | + mkdir -p `python -m site --user-site` python install.py -j 2 --build-type=Release --core-version 999999.999999 echo "PATH=${HOME}/.local/bin:$PATH" >> "$GITHUB_ENV" echo "LD_LIBRARY_PATH=${HOME}/.local/lib:${HOME}/.local/lib/cyclus:$LD_LIBRARY_PATH" >> "$GITHUB_ENV" From 8f9aa1f64f89214893966c5b98df3718be06f93a Mon Sep 17 00:00:00 2001 From: bennibbelink Date: Sat, 21 Oct 2023 13:17:46 -0500 Subject: [PATCH 081/102] removed the mkdir command from the build_test workflow --- .github/workflows/build_test.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/build_test.yml b/.github/workflows/build_test.yml index a8cc6e649e..738f5dc857 100644 --- a/.github/workflows/build_test.yml +++ b/.github/workflows/build_test.yml @@ -41,7 +41,6 @@ jobs: - name: Building Cyclus run: | - mkdir -p `python -m site --user-site` python install.py -j 2 --build-type=Release --core-version 999999.999999 echo "PATH=${HOME}/.local/bin:$PATH" >> "$GITHUB_ENV" echo "LD_LIBRARY_PATH=${HOME}/.local/lib:${HOME}/.local/lib/cyclus:$LD_LIBRARY_PATH" >> "$GITHUB_ENV" From 6d6a1cb8ffce8e6981958ca6644f272f8dd1a7fd Mon Sep 17 00:00:00 2001 From: bennibbelink Date: Sat, 21 Oct 2023 18:38:47 -0500 Subject: [PATCH 082/102] debugging workflow --- .github/workflows/build_test.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/build_test.yml b/.github/workflows/build_test.yml index 738f5dc857..db65990814 100644 --- a/.github/workflows/build_test.yml +++ b/.github/workflows/build_test.yml @@ -39,6 +39,11 @@ jobs: - name: Checkout and Build Cyclus uses: actions/checkout@v3 + - name: echo some stuff + run: | + python -m site --user-site + ls /github/home/.local/lib/python3.8/site-packages + - name: Building Cyclus run: | python install.py -j 2 --build-type=Release --core-version 999999.999999 From 7a78f4b8aca49ad7e974f0586ac4bbf1e2ef0fef Mon Sep 17 00:00:00 2001 From: bennibbelink Date: Sat, 21 Oct 2023 18:50:07 -0500 Subject: [PATCH 083/102] add mkdir command to workflow --- .github/workflows/build_test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build_test.yml b/.github/workflows/build_test.yml index db65990814..c99dfc1313 100644 --- a/.github/workflows/build_test.yml +++ b/.github/workflows/build_test.yml @@ -46,6 +46,7 @@ jobs: - name: Building Cyclus run: | + mkdir `python -m site --user-site` python install.py -j 2 --build-type=Release --core-version 999999.999999 echo "PATH=${HOME}/.local/bin:$PATH" >> "$GITHUB_ENV" echo "LD_LIBRARY_PATH=${HOME}/.local/lib:${HOME}/.local/lib/cyclus:$LD_LIBRARY_PATH" >> "$GITHUB_ENV" From b70055cbf424aa5f45825bfde7a82cb63f0b1a50 Mon Sep 17 00:00:00 2001 From: bennibbelink Date: Sat, 21 Oct 2023 18:50:55 -0500 Subject: [PATCH 084/102] remove debugging step --- .github/workflows/build_test.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/build_test.yml b/.github/workflows/build_test.yml index c99dfc1313..8f1f99215e 100644 --- a/.github/workflows/build_test.yml +++ b/.github/workflows/build_test.yml @@ -39,11 +39,6 @@ jobs: - name: Checkout and Build Cyclus uses: actions/checkout@v3 - - name: echo some stuff - run: | - python -m site --user-site - ls /github/home/.local/lib/python3.8/site-packages - - name: Building Cyclus run: | mkdir `python -m site --user-site` From c6d40383d12820d1e520519eb9a9e68816c1bc67 Mon Sep 17 00:00:00 2001 From: bennibbelink <79653949+bennibbelink@users.noreply.github.com> Date: Sat, 21 Oct 2023 20:07:25 -0400 Subject: [PATCH 085/102] Forgot the -p flag --- .github/workflows/build_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_test.yml b/.github/workflows/build_test.yml index 8f1f99215e..a8cc6e649e 100644 --- a/.github/workflows/build_test.yml +++ b/.github/workflows/build_test.yml @@ -41,7 +41,7 @@ jobs: - name: Building Cyclus run: | - mkdir `python -m site --user-site` + mkdir -p `python -m site --user-site` python install.py -j 2 --build-type=Release --core-version 999999.999999 echo "PATH=${HOME}/.local/bin:$PATH" >> "$GITHUB_ENV" echo "LD_LIBRARY_PATH=${HOME}/.local/lib:${HOME}/.local/lib/cyclus:$LD_LIBRARY_PATH" >> "$GITHUB_ENV" From efb61b4db969cc2f2ddeba69e7c3c26aafb9df31 Mon Sep 17 00:00:00 2001 From: Paul Wilson Date: Sat, 21 Oct 2023 09:19:24 -0500 Subject: [PATCH 086/102] Fix pythonpath --- docker/Dockerfile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index b356f3be82..9b1b0f9750 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -44,6 +44,8 @@ RUN apt install -y python3-pprintpp; exit 0 RUN apt clean -y all RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 10 +RUN mkdir -p `python -m site --user-site` +ENV PYTHONPATH `python -m site --user-site` FROM common-base as conda-deps @@ -104,10 +106,11 @@ RUN conda update -y --all && \ && \ mamba install -y --force-reinstall libsqlite && \ conda clean -y --all +RUN mkdir -p `python -m site --user-site` +ENV PYTHONPATH `python -m site --user-site` ENV CC /opt/conda/bin/x86_64-conda_cos6-linux-gnu-gcc ENV CXX /opt/conda/bin/x86_64-conda_cos6-linux-gnu-g++ ENV CPP /opt/conda/bin/x86_64-conda_cos6-linux-gnu-cpp -ENV PYTHONPATH "/root/.local/lib/python3.10/site-packages/" FROM ${pkg_mgr}-deps as cyclus ARG make_cores=2 @@ -121,7 +124,6 @@ WORKDIR /cyclus # You may add the option "--cmake-debug" to the following command # for further CMake debugging. -RUN mkdir -p `python -m site --user-site` RUN python install.py -j ${make_cores} --build-type=Release --core-version 999999.999999 ENV PATH /root/.local/bin:$PATH ENV LD_LIBRARY_PATH /root/.local/lib:/root/.local/lib/cyclus From f15fd6cb5de10d47fe50b3b924364187e79c1343 Mon Sep 17 00:00:00 2001 From: Paul Wilson Date: Sat, 21 Oct 2023 09:21:40 -0500 Subject: [PATCH 087/102] Update CHANGELOG.rst --- CHANGELOG.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 8c74fe8d90..e2b265c4ae 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -17,8 +17,8 @@ Since last release will fail unless update-alternatives has been used to point python at the correct python3 version (#1558) * build and test are now fown on githubAction in place or CircleCI (#1569) -* Have separate workflows for testing, publishing dependency images, and publishing release images (#1597, #1602, #1606) -* Add Ubuntu 20.04 to the list of supported platforms (#1605, #1608) +* Have separate workflows for testing, publishing dependency images, and publishing release images (#1597, #1602, #1606, #1609) +* Add Ubuntu 20.04 to the list of supported platforms (#1605) **Changed:** From bef69a7d4bb928cfe90f9864cd8adda81cbac7ca Mon Sep 17 00:00:00 2001 From: Paul Wilson Date: Sat, 21 Oct 2023 15:26:15 -0500 Subject: [PATCH 088/102] =?UTF-8?q?Don=E2=80=99t=20set=20PYTHONPATH?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docker/Dockerfile | 2 -- 1 file changed, 2 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 9b1b0f9750..0bedf9d6a9 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -45,7 +45,6 @@ RUN apt clean -y all RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 10 RUN mkdir -p `python -m site --user-site` -ENV PYTHONPATH `python -m site --user-site` FROM common-base as conda-deps @@ -107,7 +106,6 @@ RUN conda update -y --all && \ mamba install -y --force-reinstall libsqlite && \ conda clean -y --all RUN mkdir -p `python -m site --user-site` -ENV PYTHONPATH `python -m site --user-site` ENV CC /opt/conda/bin/x86_64-conda_cos6-linux-gnu-gcc ENV CXX /opt/conda/bin/x86_64-conda_cos6-linux-gnu-g++ ENV CPP /opt/conda/bin/x86_64-conda_cos6-linux-gnu-cpp From cc323a2baff8359aa70b96014ca0ec97156516bd Mon Sep 17 00:00:00 2001 From: bennibbelink Date: Sat, 21 Oct 2023 15:31:24 -0500 Subject: [PATCH 089/102] got rid of boost deprecation warnings --- src/agent.h | 2 +- src/any.hpp | 24 ++++++++++++------------ src/bid.h | 2 +- src/bid_portfolio.h | 1 + src/capacity_constraint.h | 1 + src/composition.h | 1 + src/exchange_graph.h | 3 ++- src/material.h | 1 + src/product.h | 1 + src/request.h | 2 +- src/request_portfolio.h | 1 + src/res_tracker.h | 2 +- src/resource.h | 1 + src/sqlite_db.h | 1 + src/toolkit/infile_converters.cc | 1 + src/toolkit/symbolic_functions.h | 1 + src/xml_file_loader.h | 1 + src/xml_parser.h | 2 +- 18 files changed, 30 insertions(+), 18 deletions(-) diff --git a/src/agent.h b/src/agent.h index a3c53e24a9..ece9598f5c 100644 --- a/src/agent.h +++ b/src/agent.h @@ -6,7 +6,7 @@ #include #include -#include +#include #include "db_init.h" #include "dynamic_module.h" diff --git a/src/any.hpp b/src/any.hpp index 36b84343d1..573e12f1a1 100644 --- a/src/any.hpp +++ b/src/any.hpp @@ -53,7 +53,7 @@ original Boost 1.54.0 #include #include #include -#include +#include #include #include @@ -74,7 +74,7 @@ namespace boost { namespace spirit struct bad_any_cast : std::bad_cast { - bad_any_cast(boost::detail::sp_typeinfo const& src, boost::detail::sp_typeinfo const& dest) + bad_any_cast(boost::core::typeinfo const& src, boost::core::typeinfo const& dest) : from(src.name()), to(dest.name()) {} @@ -90,7 +90,7 @@ namespace boost { namespace spirit template struct fxn_ptr_table { - boost::detail::sp_typeinfo const& (*get_type)(); + boost::core::typeinfo const& (*get_type)(); void (*static_delete)(void**); void (*destruct)(void**); void (*clone)(void* const*, void**); @@ -107,9 +107,9 @@ namespace boost { namespace spirit template struct type { - static boost::detail::sp_typeinfo const& get_type() + static boost::core::typeinfo const& get_type() { - return BOOST_SP_TYPEID(T); + return BOOST_CORE_TYPEID(T); } static void static_delete(void** x) { @@ -138,9 +138,9 @@ namespace boost { namespace spirit template struct type { - static boost::detail::sp_typeinfo const& get_type() + static boost::core::typeinfo const& get_type() { - return BOOST_SP_TYPEID(T); + return BOOST_CORE_TYPEID(T); } static void static_delete(void** x) { @@ -300,7 +300,7 @@ namespace boost { namespace spirit return *this; } - boost::detail::sp_typeinfo const& type() const + boost::core::typeinfo const& type() const { return table->get_type(); } @@ -308,8 +308,8 @@ namespace boost { namespace spirit template T const& cast() const { - if (type() != BOOST_SP_TYPEID(T)) - throw bad_any_cast(type(), BOOST_SP_TYPEID(T)); + if (type() != BOOST_CORE_TYPEID(T)) + throw bad_any_cast(type(), BOOST_CORE_TYPEID(T)); return spirit::detail::get_table::is_small::value ? *reinterpret_cast(&object) : @@ -359,7 +359,7 @@ namespace boost { namespace spirit template inline T* any_cast (basic_hold_any* operand) { - if (operand && operand->type() == BOOST_SP_TYPEID(T)) { + if (operand && operand->type() == BOOST_CORE_TYPEID(T)) { return spirit::detail::get_table::is_small::value ? reinterpret_cast(&operand->object) : reinterpret_cast(operand->object); @@ -390,7 +390,7 @@ namespace boost { namespace spirit nonref* result = any_cast(&operand); if(!result) - boost::throw_exception(bad_any_cast(operand.type(), BOOST_SP_TYPEID(T))); + boost::throw_exception(bad_any_cast(operand.type(), BOOST_CORE_TYPEID(T))); return *result; } diff --git a/src/bid.h b/src/bid.h index 1403e1849d..a1bd3f1156 100644 --- a/src/bid.h +++ b/src/bid.h @@ -1,8 +1,8 @@ #ifndef CYCLUS_SRC_BID_H_ #define CYCLUS_SRC_BID_H_ +#include #include -#include #include #include "request.h" diff --git a/src/bid_portfolio.h b/src/bid_portfolio.h index 855c74b7e5..f402adfc11 100644 --- a/src/bid_portfolio.h +++ b/src/bid_portfolio.h @@ -5,6 +5,7 @@ #include #include +#include #include #include "bid.h" diff --git a/src/capacity_constraint.h b/src/capacity_constraint.h index fa5bcd32de..c6fe8ff1fe 100644 --- a/src/capacity_constraint.h +++ b/src/capacity_constraint.h @@ -1,6 +1,7 @@ #ifndef CYCLUS_SRC_CAPACITY_CONSTRAINT_H_ #define CYCLUS_SRC_CAPACITY_CONSTRAINT_H_ +#include #include #include "error.h" diff --git a/src/composition.h b/src/composition.h index cf6aca5dfe..994c740a37 100644 --- a/src/composition.h +++ b/src/composition.h @@ -3,6 +3,7 @@ #include #include +#include #include class SimInitTest; diff --git a/src/exchange_graph.h b/src/exchange_graph.h index 7eaa6a6f01..fd0401f45c 100644 --- a/src/exchange_graph.h +++ b/src/exchange_graph.h @@ -7,8 +7,9 @@ #include #include -#include +#include #include +#include namespace cyclus { diff --git a/src/material.h b/src/material.h index 958cac7498..a406ff27a9 100644 --- a/src/material.h +++ b/src/material.h @@ -2,6 +2,7 @@ #define CYCLUS_SRC_MATERIAL_H_ #include +#include #include #include "composition.h" diff --git a/src/product.h b/src/product.h index 6f0e9ee8c7..6fd626a596 100644 --- a/src/product.h +++ b/src/product.h @@ -1,6 +1,7 @@ #ifndef CYCLUS_SRC_PRODUCT_H_ #define CYCLUS_SRC_PRODUCT_H_ +#include #include #include "context.h" diff --git a/src/request.h b/src/request.h index 44a61510a3..70a9c52832 100644 --- a/src/request.h +++ b/src/request.h @@ -5,8 +5,8 @@ #include #include +#include #include -#include namespace cyclus { diff --git a/src/request_portfolio.h b/src/request_portfolio.h index fbfc42a6ff..3c357fdf5b 100644 --- a/src/request_portfolio.h +++ b/src/request_portfolio.h @@ -8,6 +8,7 @@ #include #include +#include #include #include "capacity_constraint.h" diff --git a/src/res_tracker.h b/src/res_tracker.h index 8c1772ca31..250b44a31e 100644 --- a/src/res_tracker.h +++ b/src/res_tracker.h @@ -3,7 +3,7 @@ #include #include -#include +#include #include "context.h" #include "resource.h" diff --git a/src/resource.h b/src/resource.h index b13b52431e..c9bfb9a5c5 100644 --- a/src/resource.h +++ b/src/resource.h @@ -3,6 +3,7 @@ #include #include +#include #include class SimInitTest; diff --git a/src/sqlite_db.h b/src/sqlite_db.h index 9e1bf66bbc..135862a085 100644 --- a/src/sqlite_db.h +++ b/src/sqlite_db.h @@ -3,6 +3,7 @@ #include #include +#include #include #include "error.h" diff --git a/src/toolkit/infile_converters.cc b/src/toolkit/infile_converters.cc index ac49b780cd..e1d94c4359 100644 --- a/src/toolkit/infile_converters.cc +++ b/src/toolkit/infile_converters.cc @@ -1,5 +1,6 @@ #include +#include #include #include diff --git a/src/toolkit/symbolic_functions.h b/src/toolkit/symbolic_functions.h index 1412ab062d..ccfe363589 100644 --- a/src/toolkit/symbolic_functions.h +++ b/src/toolkit/symbolic_functions.h @@ -4,6 +4,7 @@ #include #include +#include #include namespace cyclus { diff --git a/src/xml_file_loader.h b/src/xml_file_loader.h index aabd136312..c0d0985673 100644 --- a/src/xml_file_loader.h +++ b/src/xml_file_loader.h @@ -4,6 +4,7 @@ #include #include #include +#include #include #include "composition.h" diff --git a/src/xml_parser.h b/src/xml_parser.h index 1eed2d5b35..eaed81fa32 100644 --- a/src/xml_parser.h +++ b/src/xml_parser.h @@ -2,7 +2,7 @@ #define CYCLUS_SRC_XML_PARSER_H_ #include -#include +#include namespace xmlpp { class DomParser; From bd06eba8413bfd5f34be4c795be525262845a98a Mon Sep 17 00:00:00 2001 From: bennibbelink Date: Sat, 21 Oct 2023 16:33:00 -0500 Subject: [PATCH 090/102] got rid of unnecessary typeinfo headers --- src/agent.h | 2 -- src/bid.h | 1 - src/bid_portfolio.h | 1 - src/capacity_constraint.h | 1 - src/composition.h | 1 - src/exchange_graph.h | 1 - src/material.h | 1 - src/product.h | 1 - src/request.h | 1 - src/request_portfolio.h | 1 - src/res_tracker.h | 1 - src/resource.h | 1 - src/toolkit/infile_converters.cc | 1 - src/toolkit/symbolic_functions.h | 1 - src/xml_file_loader.h | 1 - src/xml_parser.h | 1 - 16 files changed, 17 deletions(-) diff --git a/src/agent.h b/src/agent.h index ece9598f5c..e6f10d721b 100644 --- a/src/agent.h +++ b/src/agent.h @@ -6,8 +6,6 @@ #include #include -#include - #include "db_init.h" #include "dynamic_module.h" #include "infile_tree.h" diff --git a/src/bid.h b/src/bid.h index a1bd3f1156..e4f3122334 100644 --- a/src/bid.h +++ b/src/bid.h @@ -1,7 +1,6 @@ #ifndef CYCLUS_SRC_BID_H_ #define CYCLUS_SRC_BID_H_ -#include #include #include diff --git a/src/bid_portfolio.h b/src/bid_portfolio.h index f402adfc11..855c74b7e5 100644 --- a/src/bid_portfolio.h +++ b/src/bid_portfolio.h @@ -5,7 +5,6 @@ #include #include -#include #include #include "bid.h" diff --git a/src/capacity_constraint.h b/src/capacity_constraint.h index c6fe8ff1fe..fa5bcd32de 100644 --- a/src/capacity_constraint.h +++ b/src/capacity_constraint.h @@ -1,7 +1,6 @@ #ifndef CYCLUS_SRC_CAPACITY_CONSTRAINT_H_ #define CYCLUS_SRC_CAPACITY_CONSTRAINT_H_ -#include #include #include "error.h" diff --git a/src/composition.h b/src/composition.h index 994c740a37..cf6aca5dfe 100644 --- a/src/composition.h +++ b/src/composition.h @@ -3,7 +3,6 @@ #include #include -#include #include class SimInitTest; diff --git a/src/exchange_graph.h b/src/exchange_graph.h index fd0401f45c..7a21a1be4a 100644 --- a/src/exchange_graph.h +++ b/src/exchange_graph.h @@ -7,7 +7,6 @@ #include #include -#include #include #include diff --git a/src/material.h b/src/material.h index a406ff27a9..958cac7498 100644 --- a/src/material.h +++ b/src/material.h @@ -2,7 +2,6 @@ #define CYCLUS_SRC_MATERIAL_H_ #include -#include #include #include "composition.h" diff --git a/src/product.h b/src/product.h index 6fd626a596..6f0e9ee8c7 100644 --- a/src/product.h +++ b/src/product.h @@ -1,7 +1,6 @@ #ifndef CYCLUS_SRC_PRODUCT_H_ #define CYCLUS_SRC_PRODUCT_H_ -#include #include #include "context.h" diff --git a/src/request.h b/src/request.h index 70a9c52832..6b7707f236 100644 --- a/src/request.h +++ b/src/request.h @@ -5,7 +5,6 @@ #include #include -#include #include namespace cyclus { diff --git a/src/request_portfolio.h b/src/request_portfolio.h index 3c357fdf5b..fbfc42a6ff 100644 --- a/src/request_portfolio.h +++ b/src/request_portfolio.h @@ -8,7 +8,6 @@ #include #include -#include #include #include "capacity_constraint.h" diff --git a/src/res_tracker.h b/src/res_tracker.h index 250b44a31e..59e73b540a 100644 --- a/src/res_tracker.h +++ b/src/res_tracker.h @@ -3,7 +3,6 @@ #include #include -#include #include "context.h" #include "resource.h" diff --git a/src/resource.h b/src/resource.h index c9bfb9a5c5..b13b52431e 100644 --- a/src/resource.h +++ b/src/resource.h @@ -3,7 +3,6 @@ #include #include -#include #include class SimInitTest; diff --git a/src/toolkit/infile_converters.cc b/src/toolkit/infile_converters.cc index e1d94c4359..ac49b780cd 100644 --- a/src/toolkit/infile_converters.cc +++ b/src/toolkit/infile_converters.cc @@ -1,6 +1,5 @@ #include -#include #include #include diff --git a/src/toolkit/symbolic_functions.h b/src/toolkit/symbolic_functions.h index ccfe363589..1412ab062d 100644 --- a/src/toolkit/symbolic_functions.h +++ b/src/toolkit/symbolic_functions.h @@ -4,7 +4,6 @@ #include #include -#include #include namespace cyclus { diff --git a/src/xml_file_loader.h b/src/xml_file_loader.h index c0d0985673..aabd136312 100644 --- a/src/xml_file_loader.h +++ b/src/xml_file_loader.h @@ -4,7 +4,6 @@ #include #include #include -#include #include #include "composition.h" diff --git a/src/xml_parser.h b/src/xml_parser.h index eaed81fa32..1b1aa5d7bd 100644 --- a/src/xml_parser.h +++ b/src/xml_parser.h @@ -2,7 +2,6 @@ #define CYCLUS_SRC_XML_PARSER_H_ #include -#include namespace xmlpp { class DomParser; From ce87c142860430df5ff30956d33d4d87faf5e078 Mon Sep 17 00:00:00 2001 From: bennibbelink Date: Sat, 21 Oct 2023 16:33:38 -0500 Subject: [PATCH 091/102] missed one --- src/sqlite_db.h | 1 - 1 file changed, 1 deletion(-) diff --git a/src/sqlite_db.h b/src/sqlite_db.h index 135862a085..9e1bf66bbc 100644 --- a/src/sqlite_db.h +++ b/src/sqlite_db.h @@ -3,7 +3,6 @@ #include #include -#include #include #include "error.h" From 91defd1ad8cd540b176702e1aa6b5e02a7a160cb Mon Sep 17 00:00:00 2001 From: bennibbelink Date: Sat, 21 Oct 2023 16:42:14 -0500 Subject: [PATCH 092/102] added explicit #include where they should be --- src/bid.h | 1 + src/greedy_solver.h | 1 + src/request.h | 1 + src/toolkit/symbolic_function_factories.h | 1 + tests/integ_tests.cc | 1 + 5 files changed, 5 insertions(+) diff --git a/src/bid.h b/src/bid.h index e4f3122334..1403e1849d 100644 --- a/src/bid.h +++ b/src/bid.h @@ -2,6 +2,7 @@ #define CYCLUS_SRC_BID_H_ #include +#include #include #include "request.h" diff --git a/src/greedy_solver.h b/src/greedy_solver.h index 610846a6bc..767f7315ad 100644 --- a/src/greedy_solver.h +++ b/src/greedy_solver.h @@ -1,6 +1,7 @@ #ifndef CYCLUS_SRC_GREEDY_SOLVER_H_ #define CYCLUS_SRC_GREEDY_SOLVER_H_ +#include #include "exchange_graph.h" #include "exchange_solver.h" #include "greedy_preconditioner.h" diff --git a/src/request.h b/src/request.h index 6b7707f236..44a61510a3 100644 --- a/src/request.h +++ b/src/request.h @@ -6,6 +6,7 @@ #include #include +#include namespace cyclus { diff --git a/src/toolkit/symbolic_function_factories.h b/src/toolkit/symbolic_function_factories.h index 2184eb3fda..0dfefd95bb 100644 --- a/src/toolkit/symbolic_function_factories.h +++ b/src/toolkit/symbolic_function_factories.h @@ -4,6 +4,7 @@ #include #include #include +#include #include "symbolic_functions.h" diff --git a/tests/integ_tests.cc b/tests/integ_tests.cc index 785d073c27..694c04f0af 100644 --- a/tests/integ_tests.cc +++ b/tests/integ_tests.cc @@ -1,6 +1,7 @@ #include #include +#include #include "cyclus.h" #include "platform.h" From b5235d89392e47060814f3fab8bf2acc7455f652 Mon Sep 17 00:00:00 2001 From: bennibbelink Date: Sat, 21 Oct 2023 16:58:59 -0500 Subject: [PATCH 093/102] update changelog --- CHANGELOG.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index e2b265c4ae..a8412f8b3b 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -36,6 +36,7 @@ Since last release * Issue #1312. (#1560) * fix sell_policy that was offering bids when capacity was inbetween 0 and the quantize, bids that one was not able to fullfill and caused cyclus to crash. (#1552) +* Deprecation warnings involving (#1611) From 5f816b5f0343f59a9d35043db4063822456583a1 Mon Sep 17 00:00:00 2001 From: bennibbelink Date: Sun, 22 Oct 2023 14:20:01 -0500 Subject: [PATCH 094/102] revert last commit --- .github/workflows/build_test.yml | 2 -- .github/workflows/build_test_publish.yml | 3 +-- .github/workflows/publish_release.yml | 3 +-- 3 files changed, 2 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build_test.yml b/.github/workflows/build_test.yml index a8cc6e649e..95aa501ffb 100644 --- a/.github/workflows/build_test.yml +++ b/.github/workflows/build_test.yml @@ -24,7 +24,6 @@ jobs: fail-fast: false matrix: ubuntu_versions : [ - 20.04, 22.04, ] pkg_mgr : [ @@ -41,7 +40,6 @@ jobs: - name: Building Cyclus run: | - mkdir -p `python -m site --user-site` python install.py -j 2 --build-type=Release --core-version 999999.999999 echo "PATH=${HOME}/.local/bin:$PATH" >> "$GITHUB_ENV" echo "LD_LIBRARY_PATH=${HOME}/.local/lib:${HOME}/.local/lib/cyclus:$LD_LIBRARY_PATH" >> "$GITHUB_ENV" diff --git a/.github/workflows/build_test_publish.yml b/.github/workflows/build_test_publish.yml index 176264cf6b..9d9c326be1 100644 --- a/.github/workflows/build_test_publish.yml +++ b/.github/workflows/build_test_publish.yml @@ -21,7 +21,6 @@ jobs: strategy: matrix: ubuntu_versions : [ - 20.04, 22.04, ] pkg_mgr : [ @@ -60,4 +59,4 @@ jobs: parallel: true tag-latest-on-default: ${{ env.tag-latest-on-default }} dockerfile: docker/Dockerfile - build-args: pkg_mgr=${{ matrix.pkg_mgr }}, ubuntu_version=${{ matrix.ubuntu_versions }} \ No newline at end of file + build-args: pkg_mgr=${{ matrix.pkg_mgr }} \ No newline at end of file diff --git a/.github/workflows/publish_release.yml b/.github/workflows/publish_release.yml index b52b924fd0..dc697bb72f 100644 --- a/.github/workflows/publish_release.yml +++ b/.github/workflows/publish_release.yml @@ -12,7 +12,6 @@ jobs: fail-fast: false matrix: ubuntu_versions : [ - 20.04, 22.04, ] pkg_mgr : [ @@ -51,4 +50,4 @@ jobs: parallel: true tag-latest-on-default: ${{ env.tag-latest-on-default }} dockerfile: docker/Dockerfile - build-args: pkg_mgr=${{ matrix.pkg_mgr }}, ubuntu_version=${{ matrix.ubuntu_versions }} \ No newline at end of file + build-args: pkg_mgr=${{ matrix.pkg_mgr }} \ No newline at end of file From a68632283006ff9caa6bfc7aa46b0654feeefc16 Mon Sep 17 00:00:00 2001 From: Amanda Date: Tue, 24 Oct 2023 11:10:29 -0500 Subject: [PATCH 095/102] update changelog after merging main --- CHANGELOG.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index d428a5c734..a16fdcff82 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -19,6 +19,7 @@ Since last release * build and test are now fown on githubAction in place or CircleCI (#1569) * Have separate workflows for testing, publishing dependency images, and publishing release images (#1597, #1602, #1606, #1609) * Add Ubuntu 20.04 to the list of supported platforms (#1605, #1608) +* AddMutalReqs and AddReciepe functions and exclusive bids in python API of DRE (#1584) **Changed:** From 7eaa706f4412637fa32c5972ee757221156a7c79 Mon Sep 17 00:00:00 2001 From: abachmann Date: Thu, 18 Jan 2024 10:45:01 -0600 Subject: [PATCH 096/102] fix typo in updating exculsive key of dictionary --- cyclus/lib.pyx | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/cyclus/lib.pyx b/cyclus/lib.pyx index c3f75c66ef..055d866590 100644 --- a/cyclus/lib.pyx +++ b/cyclus/lib.pyx @@ -1832,7 +1832,7 @@ cpdef dict normalize_request_portfolio(object inp): for name, reqs in commodity.items(): if name == 'preference' or name == 'exclusive': continue - + commods.append({name:reqs}) constrs = inp.get('constraints', []) else: @@ -1850,7 +1850,6 @@ cpdef dict normalize_request_portfolio(object inp): commods = list(commods) cdef dict default_req = {'target': None, 'preference': 1.0, 'exclusive': False, 'cost': None} - for index, commodity in enumerate(commods): for key, val in commodity.items(): if isinstance(val, ts.Resource): @@ -1858,7 +1857,7 @@ cpdef dict normalize_request_portfolio(object inp): req['target'] = val if 'preference' in inp['commodities'][index]: req['preference'] = inp['commodities'][index]['preference'] - if 'exclusive' in inp['commoditites'][index]: + if 'exclusive' in inp['commodities'][index]: req['exclusive'] = inp['commodities'][index]['exclusive'] commods[index][key] = [req] From 711e2d3eabb1305b4408b8a44c765f8549147198 Mon Sep 17 00:00:00 2001 From: abachmann Date: Thu, 18 Jan 2024 10:45:44 -0600 Subject: [PATCH 097/102] update request structurefor Sink agent The material and product request dictionaries had to be updated to have the inner dictionary be a list of dictionaries for each commodity to request --- cyclus/pyagents.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cyclus/pyagents.py b/cyclus/pyagents.py index 8f2ac2a7d9..8d5d4258ea 100644 --- a/cyclus/pyagents.py +++ b/cyclus/pyagents.py @@ -50,13 +50,13 @@ def get_material_requests(self): else: comp = self.context.get_recipe(self.recipe) mat = ts.Material.create_untracked(self.capacity, comp) - port = {"commodities": {c: mat for c in self.in_commods}, + port = {"commodities": [{c: mat} for c in self.in_commods], "constraints": self.capacity} return port def get_product_requests(self): prod = ts.Product.create_untracked(self.capacity, "") - port = {"commodities": {c: prod for c in self.in_commods}, + port = {"commodities": [{c: prod} for c in self.in_commods], "constraints": self.capacity} return port From afee0f8d7eeaa6e630935a6b4f440729e2cd761a Mon Sep 17 00:00:00 2001 From: abachmann Date: Tue, 23 Jan 2024 08:41:03 -0600 Subject: [PATCH 098/102] correct dictionary format for normalizing These changes allow an archetype to pass a dictionary as simple as {'commodity_name1':Resource1} as the request portfolio, and it can be normalized as needed to interact with the DRE properly --- cyclus/lib.pyx | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/cyclus/lib.pyx b/cyclus/lib.pyx index 055d866590..1e9ddd30c5 100644 --- a/cyclus/lib.pyx +++ b/cyclus/lib.pyx @@ -1837,7 +1837,7 @@ cpdef dict normalize_request_portfolio(object inp): constrs = inp.get('constraints', []) else: commods = [] - for name, reqs in inp: + for name, reqs in inp.items(): if name == 'preference' or name == 'exclusive': continue commods.append({name:reqs}) @@ -1855,10 +1855,11 @@ cpdef dict normalize_request_portfolio(object inp): if isinstance(val, ts.Resource): req = default_req.copy() req['target'] = val - if 'preference' in inp['commodities'][index]: - req['preference'] = inp['commodities'][index]['preference'] - if 'exclusive' in inp['commodities'][index]: - req['exclusive'] = inp['commodities'][index]['exclusive'] + if 'commodities' in inp: + if 'preference' in inp['commodities'][index]: + req['preference'] = inp['commodities'][index]['preference'] + if 'exclusive' in inp['commodities'][index]: + req['exclusive'] = inp['commodities'][index]['exclusive'] commods[index][key] = [req] elif isinstance(val, Mapping): From 5f9d9db8aa16104bf7eac620b07205473253621a Mon Sep 17 00:00:00 2001 From: abachmann Date: Mon, 5 Feb 2024 13:38:40 -0600 Subject: [PATCH 099/102] checking update for website --- INSTALL.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/INSTALL.rst b/INSTALL.rst index 2e82957d0f..bb9d9cfe9d 100644 --- a/INSTALL.rst +++ b/INSTALL.rst @@ -34,7 +34,7 @@ Installation Before going further with the installation procedure be sure you have installed all the required dependencies listed above. We have provided detailed -instructions for `installing dependencies `_ for the major supported +instructions for :doc:`installing dependencies `_ for the major supported systems. From c7413e087babb17b31e178b88648331d6e6691ca Mon Sep 17 00:00:00 2001 From: abachmann Date: Mon, 5 Feb 2024 13:57:26 -0600 Subject: [PATCH 100/102] updating depend link --- INSTALL.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/INSTALL.rst b/INSTALL.rst index bb9d9cfe9d..e067cde14d 100644 --- a/INSTALL.rst +++ b/INSTALL.rst @@ -34,7 +34,7 @@ Installation Before going further with the installation procedure be sure you have installed all the required dependencies listed above. We have provided detailed -instructions for :doc:`installing dependencies `_ for the major supported +instructions for :doc:`here `.for the major supported systems. From 2d01308ce8d98a3e648de1d359f1edd4b870d87b Mon Sep 17 00:00:00 2001 From: abachmann Date: Mon, 5 Feb 2024 13:58:10 -0600 Subject: [PATCH 101/102] update dependency text --- INSTALL.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/INSTALL.rst b/INSTALL.rst index e067cde14d..99a51e3f0a 100644 --- a/INSTALL.rst +++ b/INSTALL.rst @@ -34,7 +34,7 @@ Installation Before going further with the installation procedure be sure you have installed all the required dependencies listed above. We have provided detailed -instructions for :doc:`here `.for the major supported +instructions for :doc:`installing dependencies `.for the major supported systems. From b136b0c07de6655822d602e1425aee6cdf0b3986 Mon Sep 17 00:00:00 2001 From: abachmann Date: Thu, 8 Feb 2024 10:20:38 -0600 Subject: [PATCH 102/102] remove unused prefs list, per @gonuke request --- cyclus/lib.pyx | 1 - 1 file changed, 1 deletion(-) diff --git a/cyclus/lib.pyx b/cyclus/lib.pyx index 1e9ddd30c5..217c8d0f18 100644 --- a/cyclus/lib.pyx +++ b/cyclus/lib.pyx @@ -1827,7 +1827,6 @@ cpdef dict normalize_request_portfolio(object inp): inp = dict(inp) if 'commodities' in inp: commods = [] - prefs = [] for commodity in inp['commodities']: for name, reqs in commodity.items(): if name == 'preference' or name == 'exclusive':