From 2080ab15f9a3aafeca3d7efe87e4bd5c30b63f23 Mon Sep 17 00:00:00 2001 From: Baptiste Mouginot Date: Fri, 31 Jan 2020 16:00:53 -0600 Subject: [PATCH 01/11] setting up the archetechture for units --- cymetric/evaluator.py | 37 +++++++++++++++++++++++++++++++++++-- cymetric/metrics.py | 6 ++++-- cymetric/root_metrics.py | 8 +++++++- 3 files changed, 46 insertions(+), 5 deletions(-) diff --git a/cymetric/evaluator.py b/cymetric/evaluator.py index a198460b..94efe217 100644 --- a/cymetric/evaluator.py +++ b/cymetric/evaluator.py @@ -9,12 +9,14 @@ METRIC_REGISTRY = {} - +UNITS_REGISTRY = {} def register_metric(cls): """Adds a metric to the registry.""" METRIC_REGISTRY[cls.__name__] = cls - + if cls.__name__ not in UNITS_REGISTRY and cls.registry is not None: + UNITS_REGISTRY[cls.__name__] = cls.registry + build_normalisd_metric(cls) class Evaluator(object): """An evaluation context for metrics.""" @@ -40,6 +42,8 @@ def __init__(self, db, write=True): self.recorder = rec = lib.Recorder(inject_sim_id=False) rec.register_backend(db) self.known_tables = db.tables + #try: + # self.registry = self.db.query("table_registry") def get_metric(self, metric): """Checks if metric is already in the registry; adds it if not.""" @@ -81,3 +85,32 @@ def eval(metric, db, conds=None, write=True): """Evalutes a metric with the given conditions in a database.""" e = Evaluator(db, write=write) return e.eval(str(metric), conds=conds) + + +def build_normalisd_metric(cls): + _dsdeps = [cls.__name__] + + _dsschema = build_normalized_schema(cls.schema) + norm_name = cls.__name__ + "_norm" + + @metric(name=norm_name, depends=_dsdeps, schema=_dsschema) + def normalization(raw_metric): + return rtn + + del _dsdeps, _dsschema + +def build_normalized_schema(raw_cls): + # if not in the unit registery: nothing to do + if (raw_cls.__name__ not in UNITS_REGISTRY): + return raw_cls.schema + + # initialize the normed metric schema + norm_schema = raw_cls.schema + + # removing units columns form the new schema + unit_registry = UNITS_REGISTRY[raw_cls.__name__] + for key in unit_registry: + norm_schema.pop(unit_registry[key][0], None) + + return norm_schema + diff --git a/cymetric/metrics.py b/cymetric/metrics.py index aa62dcab..17b89a25 100644 --- a/cymetric/metrics.py +++ b/cymetric/metrics.py @@ -31,6 +31,7 @@ class Metric(object): """Metric class""" dependencies = NotImplemented schema = NotImplemented + registry = NotImplemented def __init__(self, db): self.db = db @@ -40,7 +41,7 @@ def name(self): return self.__class__.__name__ -def _genmetricclass(f, name, depends, scheme): +def _genmetricclass(f, name, depends, scheme, register): """Creates a new metric class with a given name, dependencies, and schema. Parameters @@ -58,6 +59,7 @@ def _genmetricclass(f, name, depends, scheme): class Cls(Metric): dependencies = depends schema = scheme + registry = register func = staticmethod(f) __doc__ = inspect.getdoc(f) @@ -400,7 +402,7 @@ def annual_electricity_generated_by_agent(elec): 'AgentId': elec.AgentId, 'Year': elec.Time.apply(lambda x: x//12), 'Energy': elec.Value.apply(lambda x: x/12)}, - columns=['SimId', 'AgentId', 'Year', 'Energy']) + columns=['SimId', 'AgentId', 'Year', 'Energy']) el_index = ['SimId', 'AgentId', 'Year'] elec = elec.groupby(el_index).sum() rtn = elec.reset_index() diff --git a/cymetric/root_metrics.py b/cymetric/root_metrics.py index d3e78be2..98ed3c2a 100644 --- a/cymetric/root_metrics.py +++ b/cymetric/root_metrics.py @@ -9,6 +9,12 @@ def _genrootclass(name): """Creates a new root metric class.""" class Cls(object): dependencies = () + + @property + def registry(self): + """Defines registry for root metric if provided.""" + if self._registry is not None: + return self._registry @property def schema(self): @@ -38,7 +44,7 @@ def __call__(self, conds=None, *args, **kwargs): return Cls -def root_metric(obj=None, name=None, schema=None, *args, **kwargs): +def root_metric(obj=None, name=None, schema=None, registry=None, *args, **kwargs): """Decorator that creates a root metric from a function or class.""" if obj is not None: raise RuntimeError From 78466185e9f3070a00afa5bee40ead39214c60d4 Mon Sep 17 00:00:00 2001 From: Baptiste Mouginot Date: Wed, 12 Feb 2020 11:59:08 -0600 Subject: [PATCH 02/11] start unit conversion --- cymetric/evaluator.py | 2 +- cymetric/units.py | 49 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 1 deletion(-) create mode 100644 cymetric/units.py diff --git a/cymetric/evaluator.py b/cymetric/evaluator.py index 94efe217..4d43b081 100644 --- a/cymetric/evaluator.py +++ b/cymetric/evaluator.py @@ -16,7 +16,7 @@ def register_metric(cls): METRIC_REGISTRY[cls.__name__] = cls if cls.__name__ not in UNITS_REGISTRY and cls.registry is not None: UNITS_REGISTRY[cls.__name__] = cls.registry - build_normalisd_metric(cls) + build_normalised_metric(cls) class Evaluator(object): """An evaluation context for metrics.""" diff --git a/cymetric/units.py b/cymetric/units.py new file mode 100644 index 00000000..a2c2d2c2 --- /dev/null +++ b/cymetric/units.py @@ -0,0 +1,49 @@ +""" Convert able to the default unit system. +""" + +try: + from cymetric import schemas + from cymetric import tools + from cymetric import evaluator +except ImportError: + # some wacky CI paths prevent absolute importing, try relative + from . import schemas + from . import tools + from . import evaluator + +import pint + +ureg = pint.UnitRegistry() + +def build_normalized_metric(raw_metric): + + _norm_deps = [raw_metric.__name__] + + _norm_schema = evaluator.build_normalized_metric(raw_metric) + _norm_name = "norm_" + raw_metric.__name__ + + @metric(name=_norm_name, depends=_norm_deps, schema=_norm_schema) + def norm_metric(raw): + if (raw_cls.__name__ not in UNITS_REGISTRY): + return raw + + unit_registry = evaluator.UNITS_REGISTRY[raw.__name__] + + for key in unit_registry: + for unit in unit_registry[key]: + u_col_name = unit[0] + conv = [ ureg.parse_expression(x).to_root_units().magnitude + for x in raw[u_col_name]] + + return z + + + del _matdeps, _matschema + + +def build_conversion_col(unit_col): + conversion_col = ureg.parse_expression(unit_col) + return conversion_col + + + From d7b34ef03df4687de3eeaecd628c20467e0c6192 Mon Sep 17 00:00:00 2001 From: Baptiste Mouginot Date: Wed, 12 Feb 2020 11:59:50 -0600 Subject: [PATCH 03/11] formating --- cymetric/root_metrics.py | 2 +- cymetric/units.py | 25 +++++++++++-------------- 2 files changed, 12 insertions(+), 15 deletions(-) diff --git a/cymetric/root_metrics.py b/cymetric/root_metrics.py index 98ed3c2a..5a131c93 100644 --- a/cymetric/root_metrics.py +++ b/cymetric/root_metrics.py @@ -9,7 +9,7 @@ def _genrootclass(name): """Creates a new root metric class.""" class Cls(object): dependencies = () - + @property def registry(self): """Defines registry for root metric if provided.""" diff --git a/cymetric/units.py b/cymetric/units.py index a2c2d2c2..e3cce4a0 100644 --- a/cymetric/units.py +++ b/cymetric/units.py @@ -6,15 +6,16 @@ from cymetric import tools from cymetric import evaluator except ImportError: - # some wacky CI paths prevent absolute importing, try relative - from . import schemas - from . import tools - from . import evaluator + # some wacky CI paths prevent absolute importing, try relative + from . import schemas + from . import tools + from . import evaluator import pint ureg = pint.UnitRegistry() + def build_normalized_metric(raw_metric): _norm_deps = [raw_metric.__name__] @@ -23,27 +24,23 @@ def build_normalized_metric(raw_metric): _norm_name = "norm_" + raw_metric.__name__ @metric(name=_norm_name, depends=_norm_deps, schema=_norm_schema) - def norm_metric(raw): - if (raw_cls.__name__ not in UNITS_REGISTRY): + def norm_metric(raw): + if (raw_cls.__name__ not in UNITS_REGISTRY): return raw - unit_registry = evaluator.UNITS_REGISTRY[raw.__name__] - - for key in unit_registry: + unit_registry = evaluator.UNITS_REGISTRY[raw.__name__] + + for key in unit_registry: for unit in unit_registry[key]: u_col_name = unit[0] - conv = [ ureg.parse_expression(x).to_root_units().magnitude + conv = [ureg.parse_expression(x).to_root_units().magnitude for x in raw[u_col_name]] return z - del _matdeps, _matschema def build_conversion_col(unit_col): conversion_col = ureg.parse_expression(unit_col) return conversion_col - - - From b560e45b0b712f3fe23c1a819a1364f67f218f4e Mon Sep 17 00:00:00 2001 From: Baptiste Mouginot Date: Fri, 14 Feb 2020 10:19:11 -0600 Subject: [PATCH 04/11] first working implementation --- cymetric/evaluator.py | 37 ++--------- cymetric/metrics.py | 21 ++++--- cymetric/root_metrics.py | 19 +++--- cymetric/units.py | 131 ++++++++++++++++++++++++++++++++++----- 4 files changed, 143 insertions(+), 65 deletions(-) diff --git a/cymetric/evaluator.py b/cymetric/evaluator.py index 4d43b081..651b308a 100644 --- a/cymetric/evaluator.py +++ b/cymetric/evaluator.py @@ -6,17 +6,18 @@ from cyclus import lib from cymetric.tools import raw_to_series - +from cymetric import units METRIC_REGISTRY = {} UNITS_REGISTRY = {} -def register_metric(cls): +def register_metric(cls, registry = None): """Adds a metric to the registry.""" METRIC_REGISTRY[cls.__name__] = cls - if cls.__name__ not in UNITS_REGISTRY and cls.registry is not None: - UNITS_REGISTRY[cls.__name__] = cls.registry - build_normalised_metric(cls) + if registry and registry != "NotImplemented": + print(registry) + UNITS_REGISTRY[cls.__name__] = registry + units.build_normalized_metric(cls) class Evaluator(object): """An evaluation context for metrics.""" @@ -87,30 +88,4 @@ def eval(metric, db, conds=None, write=True): return e.eval(str(metric), conds=conds) -def build_normalisd_metric(cls): - _dsdeps = [cls.__name__] - - _dsschema = build_normalized_schema(cls.schema) - norm_name = cls.__name__ + "_norm" - - @metric(name=norm_name, depends=_dsdeps, schema=_dsschema) - def normalization(raw_metric): - return rtn - - del _dsdeps, _dsschema - -def build_normalized_schema(raw_cls): - # if not in the unit registery: nothing to do - if (raw_cls.__name__ not in UNITS_REGISTRY): - return raw_cls.schema - - # initialize the normed metric schema - norm_schema = raw_cls.schema - - # removing units columns form the new schema - unit_registry = UNITS_REGISTRY[raw_cls.__name__] - for key in unit_registry: - norm_schema.pop(unit_registry[key][0], None) - - return norm_schema diff --git a/cymetric/metrics.py b/cymetric/metrics.py index 17b89a25..06d8994a 100644 --- a/cymetric/metrics.py +++ b/cymetric/metrics.py @@ -31,7 +31,7 @@ class Metric(object): """Metric class""" dependencies = NotImplemented schema = NotImplemented - registry = NotImplemented + registry = None def __init__(self, db): self.db = db @@ -41,7 +41,7 @@ def name(self): return self.__class__.__name__ -def _genmetricclass(f, name, depends, scheme, register): +def _genmetricclass(f, name, depends, scheme, registry): """Creates a new metric class with a given name, dependencies, and schema. Parameters @@ -59,10 +59,12 @@ def _genmetricclass(f, name, depends, scheme, register): class Cls(Metric): dependencies = depends schema = scheme - registry = register func = staticmethod(f) __doc__ = inspect.getdoc(f) + + def shema(self): + return schema def __init__(self, db): """Constructor for metric object in database.""" @@ -78,15 +80,19 @@ def __call__(self, frames, conds=None, known_tables=None, *args, **kwargs): return f(*frames) Cls.__name__ = str(name) - register_metric(Cls) + if registry != NotImplemented: + register_metric(Cls, registry) + else: + register_metric(Cls) + return Cls -def metric(name=None, depends=NotImplemented, schema=NotImplemented): +def metric(name=None, depends=NotImplemented, schema=NotImplemented, registery=NotImplemented): """Decorator that creates metric class from a function or class.""" def dec(f): clsname = name or f.__name__ - return _genmetricclass(f=f, name=clsname, scheme=schema, depends=depends) + return _genmetricclass(f=f, name=clsname, scheme=schema, depends=depends, registry=registery) return dec @@ -107,8 +113,9 @@ def dec(f): ('Units', ts.STRING), ('Mass', ts.DOUBLE) ] +_matregistery = { "Mass": ["Units", "kg", ""]} -@metric(name='Materials', depends=_matdeps, schema=_matschema) +@metric(name='Materials', depends=_matdeps, schema=_matschema, registery= _matregistery) def materials(rsrcs, comps): """Materials metric returns the material mass (quantity of material in Resources times the massfrac in Compositions) indexed by the SimId, QualId, diff --git a/cymetric/root_metrics.py b/cymetric/root_metrics.py index 5a131c93..9619c6d3 100644 --- a/cymetric/root_metrics.py +++ b/cymetric/root_metrics.py @@ -3,19 +3,18 @@ """ from __future__ import print_function, unicode_literals -from cymetric.evaluator import register_metric +try: + from cymetric.evaluator import register_metric +except ImportError: + # some wacky CI paths prevent absolute importing, try relative + from .evaluator import register_metric -def _genrootclass(name): +def _genrootclass(name, registry): """Creates a new root metric class.""" class Cls(object): dependencies = () - @property - def registry(self): - """Defines registry for root metric if provided.""" - if self._registry is not None: - return self._registry - + registery = registry @property def schema(self): """Defines schema for root metric if provided.""" @@ -44,13 +43,13 @@ def __call__(self, conds=None, *args, **kwargs): return Cls -def root_metric(obj=None, name=None, schema=None, registry=None, *args, **kwargs): +def root_metric(obj=None, name=None, schema=None, registery=None, *args, **kwargs): """Decorator that creates a root metric from a function or class.""" if obj is not None: raise RuntimeError if name is None: raise RuntimeError - return _genrootclass(name=name) + return _genrootclass(name=name, registry=registery) #core tables diff --git a/cymetric/units.py b/cymetric/units.py index e3cce4a0..4dbce432 100644 --- a/cymetric/units.py +++ b/cymetric/units.py @@ -1,46 +1,143 @@ """ Convert able to the default unit system. """ +import inspect try: from cymetric import schemas from cymetric import tools from cymetric import evaluator +# from cymetric.metrics import Metric, metric +# from cymetric.root_metrics import root_metric + except ImportError: # some wacky CI paths prevent absolute importing, try relative from . import schemas from . import tools from . import evaluator +# from .root_metrics import root_metric + from .metrics import Metric, metric import pint ureg = pint.UnitRegistry() +class NormMetric(object): + """Metric class""" + dependencies = NotImplemented + schema = NotImplemented + registry = NotImplemented + + def __init__(self, db): + self.db = db + + @property + def name(self): + return self.__class__.__name__ + +def _gen_norm_metricclass(f, name, r_name, depends, scheme): + """Creates a new metric class with a given name, dependencies, and schema. + + Parameters + ---------- + name : str + Metric name + depends : list of lists (table name, tuple of indices, column name) + Dependencies on other database tables (metrics or root metrics) + scheme : list of tuples (column name, data type) + Schema for metric + """ + if not isinstance(scheme, schemas.schema): + scheme = schemas.schema(scheme) + + class Cls(NormMetric): + dependencies = depends + schema = scheme + func = staticmethod(f) + raw_name = r_name + __doc__ = inspect.getdoc(f) + + def __init__(self, db): + """Constructor for metric object in database.""" + super(Cls, self).__init__(db) + + def __call__(self, frames, conds=None, known_tables=None, *args, **kwargs): + """Computes metric for given input data and conditions.""" + # FIXME test if I already exist in the db, read in if I do + if known_tables is None: + known_tables = self.db.tables() + if self.name in known_tables: + return self.db.query(self.name, conds=conds) + return f(self.raw_name, *frames) + + Cls.__name__ = str(name) + evaluator.register_metric(Cls) + return Cls + + + +def norm_metric(name=None, raw_name=NotImplemented, depends=NotImplemented, schema=NotImplemented): + """Decorator that creates metric class from a function or class.""" + def dec(f): + clsname = name or f.__name__ + return _gen_norm_metricclass(f=f, name=clsname, r_name=raw_name, scheme=schema, depends=depends) + return dec + + +def build_conversion_col(col): + conversion_col = [ureg.parse_expression( + x).to_root_units().magnitude for x in col] + return conversion_col + + +def build_normalized_schema(raw_cls): + # if not in the unit registery: nothing to do + if (raw_cls.__name__ not in evaluator.UNITS_REGISTRY): + return raw_cls.schema + + # initialize the normed metric schema + norm_schema = raw_cls.schema + + # removing units columns form the new schema + unit_registry = evaluator.UNITS_REGISTRY[raw_cls.__name__] + print(norm_schema) + for key in unit_registry: + idx = norm_schema.index( (unit_registry[key][0], 4, None)) + norm_schema.pop(idx) + print(norm_schema) + return norm_schema + def build_normalized_metric(raw_metric): _norm_deps = [raw_metric.__name__] - _norm_schema = evaluator.build_normalized_metric(raw_metric) + _norm_schema = build_normalized_schema(raw_metric) _norm_name = "norm_" + raw_metric.__name__ + _raw_name = raw_metric.__name__ - @metric(name=_norm_name, depends=_norm_deps, schema=_norm_schema) - def norm_metric(raw): - if (raw_cls.__name__ not in UNITS_REGISTRY): - return raw + @norm_metric(name=_norm_name, raw_name=_raw_name , depends=_norm_deps, schema=_norm_schema) + def new_norm_metric(raw_name, raw): + if (raw_name not in evaluator.UNITS_REGISTRY): + return raw - unit_registry = evaluator.UNITS_REGISTRY[raw.__name__] + unit_registry = evaluator.UNITS_REGISTRY[raw_name] + norm_pdf = raw.copy(deep=True) + for unit in unit_registry: + u_col_name = unit_registry[unit][0] + u_def_unit = unit_registry[unit][1] + + # if a column for unit exist parse the colunm convert the value + # drop the column + if ( u_col_name != ""): + conv = build_conversion_col(raw[u_col_name]) + norm_pdf[unit] *= conv + norm_pdf.drop([u_col_name], axis=1, inplace=True) + else: # else use the default unit to convert it + conv = ureg.parse_expression(u_def_unit).to_root_units().magnitude + norm_pdf[unit] *= conv - for key in unit_registry: - for unit in unit_registry[key]: - u_col_name = unit[0] - conv = [ureg.parse_expression(x).to_root_units().magnitude - for x in raw[u_col_name]] + return norm_pdf - return z + del _norm_deps, _norm_schema, _norm_name - del _matdeps, _matschema - -def build_conversion_col(unit_col): - conversion_col = ureg.parse_expression(unit_col) - return conversion_col From 590912971a105ef358841a5034da34b97f321f2f Mon Sep 17 00:00:00 2001 From: Baptiste Mouginot Date: Fri, 14 Feb 2020 17:34:49 -0600 Subject: [PATCH 05/11] fix some englsih issues --- cymetric/metrics.py | 12 ++++++------ cymetric/root_metrics.py | 13 ++++++++----- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/cymetric/metrics.py b/cymetric/metrics.py index 06d8994a..dd059c57 100644 --- a/cymetric/metrics.py +++ b/cymetric/metrics.py @@ -41,7 +41,7 @@ def name(self): return self.__class__.__name__ -def _genmetricclass(f, name, depends, scheme, registry): +def _genmetricclass(f, name, depends, scheme, registier): """Creates a new metric class with a given name, dependencies, and schema. Parameters @@ -60,7 +60,7 @@ class Cls(Metric): dependencies = depends schema = scheme func = staticmethod(f) - + regisry = register __doc__ = inspect.getdoc(f) def shema(self): @@ -88,11 +88,11 @@ def __call__(self, frames, conds=None, known_tables=None, *args, **kwargs): return Cls -def metric(name=None, depends=NotImplemented, schema=NotImplemented, registery=NotImplemented): +def metric(name=None, depends=NotImplemented, schema=NotImplemented,registry=NotImplemented): """Decorator that creates metric class from a function or class.""" def dec(f): clsname = name or f.__name__ - return _genmetricclass(f=f, name=clsname, scheme=schema, depends=depends, registry=registery) + return _genmetricclass(f=f, name=clsname, scheme=schema, depends=depends, register=registry) return dec @@ -113,9 +113,9 @@ def dec(f): ('Units', ts.STRING), ('Mass', ts.DOUBLE) ] -_matregistery = { "Mass": ["Units", "kg", ""]} +_matregistry = { "Mass": ["Units", "kg", ""]} -@metric(name='Materials', depends=_matdeps, schema=_matschema, registery= _matregistery) +@metric(name='Materials', depends=_matdeps, schema=_matschema, registry=_matregistry) def materials(rsrcs, comps): """Materials metric returns the material mass (quantity of material in Resources times the massfrac in Compositions) indexed by the SimId, QualId, diff --git a/cymetric/root_metrics.py b/cymetric/root_metrics.py index 9619c6d3..12703592 100644 --- a/cymetric/root_metrics.py +++ b/cymetric/root_metrics.py @@ -9,12 +9,12 @@ # some wacky CI paths prevent absolute importing, try relative from .evaluator import register_metric -def _genrootclass(name, registry): +def _genrootclass(name, register): """Creates a new root metric class.""" class Cls(object): dependencies = () - registery = registry + registry = register @property def schema(self): """Defines schema for root metric if provided.""" @@ -39,17 +39,20 @@ def __call__(self, conds=None, *args, **kwargs): return self.db.query(self.name, conds=conds) Cls.__name__ = str(name) - register_metric(Cls) + if registry is not None: + register_metric(Cls, registry) + else: + register_metric(Cls) return Cls -def root_metric(obj=None, name=None, schema=None, registery=None, *args, **kwargs): +def root_metric(obj=None, name=None, schema=None, registry=None, *args, **kwargs): """Decorator that creates a root metric from a function or class.""" if obj is not None: raise RuntimeError if name is None: raise RuntimeError - return _genrootclass(name=name, registry=registery) + return _genrootclass(name=name, register=registry) #core tables From 26977e267b840ae82a5e75d2ec245a1ef2464662 Mon Sep 17 00:00:00 2001 From: Baptiste Mouginot Date: Fri, 14 Feb 2020 17:51:10 -0600 Subject: [PATCH 06/11] now don't need a unit registry --- cymetric/evaluator.py | 7 +++---- cymetric/metrics.py | 11 ++++------- cymetric/root_metrics.py | 5 +---- cymetric/units.py | 21 ++++++++------------- 4 files changed, 16 insertions(+), 28 deletions(-) diff --git a/cymetric/evaluator.py b/cymetric/evaluator.py index 651b308a..a7d072f5 100644 --- a/cymetric/evaluator.py +++ b/cymetric/evaluator.py @@ -11,12 +11,11 @@ METRIC_REGISTRY = {} UNITS_REGISTRY = {} -def register_metric(cls, registry = None): +def register_metric(cls): """Adds a metric to the registry.""" METRIC_REGISTRY[cls.__name__] = cls - if registry and registry != "NotImplemented": - print(registry) - UNITS_REGISTRY[cls.__name__] = registry + if cls.registry and cls.registry is not NotImplemented: + print(cls.registry) units.build_normalized_metric(cls) class Evaluator(object): diff --git a/cymetric/metrics.py b/cymetric/metrics.py index dd059c57..ba99d750 100644 --- a/cymetric/metrics.py +++ b/cymetric/metrics.py @@ -31,7 +31,7 @@ class Metric(object): """Metric class""" dependencies = NotImplemented schema = NotImplemented - registry = None + registry = NotImplemented def __init__(self, db): self.db = db @@ -41,7 +41,7 @@ def name(self): return self.__class__.__name__ -def _genmetricclass(f, name, depends, scheme, registier): +def _genmetricclass(f, name, depends, scheme, register): """Creates a new metric class with a given name, dependencies, and schema. Parameters @@ -60,7 +60,7 @@ class Cls(Metric): dependencies = depends schema = scheme func = staticmethod(f) - regisry = register + registry = register __doc__ = inspect.getdoc(f) def shema(self): @@ -80,10 +80,7 @@ def __call__(self, frames, conds=None, known_tables=None, *args, **kwargs): return f(*frames) Cls.__name__ = str(name) - if registry != NotImplemented: - register_metric(Cls, registry) - else: - register_metric(Cls) + register_metric(Cls) return Cls diff --git a/cymetric/root_metrics.py b/cymetric/root_metrics.py index 12703592..f1181404 100644 --- a/cymetric/root_metrics.py +++ b/cymetric/root_metrics.py @@ -39,10 +39,7 @@ def __call__(self, conds=None, *args, **kwargs): return self.db.query(self.name, conds=conds) Cls.__name__ = str(name) - if registry is not None: - register_metric(Cls, registry) - else: - register_metric(Cls) + register_metric(Cls) return Cls diff --git a/cymetric/units.py b/cymetric/units.py index 4dbce432..94992acb 100644 --- a/cymetric/units.py +++ b/cymetric/units.py @@ -6,16 +6,12 @@ from cymetric import schemas from cymetric import tools from cymetric import evaluator -# from cymetric.metrics import Metric, metric -# from cymetric.root_metrics import root_metric except ImportError: # some wacky CI paths prevent absolute importing, try relative from . import schemas from . import tools from . import evaluator -# from .root_metrics import root_metric - from .metrics import Metric, metric import pint @@ -34,7 +30,7 @@ def __init__(self, db): def name(self): return self.__class__.__name__ -def _gen_norm_metricclass(f, name, r_name, depends, scheme): +def _gen_norm_metricclass(f, name, r_name, r_regitry, depends, scheme): """Creates a new metric class with a given name, dependencies, and schema. Parameters @@ -54,6 +50,7 @@ class Cls(NormMetric): schema = scheme func = staticmethod(f) raw_name = r_name + raw_unit_registry = r_regitry __doc__ = inspect.getdoc(f) def __init__(self, db): @@ -67,7 +64,7 @@ def __call__(self, frames, conds=None, known_tables=None, *args, **kwargs): known_tables = self.db.tables() if self.name in known_tables: return self.db.query(self.name, conds=conds) - return f(self.raw_name, *frames) + return f(self.raw_name, self.raw_unit_registry, *frames) Cls.__name__ = str(name) evaluator.register_metric(Cls) @@ -75,11 +72,11 @@ def __call__(self, frames, conds=None, known_tables=None, *args, **kwargs): -def norm_metric(name=None, raw_name=NotImplemented, depends=NotImplemented, schema=NotImplemented): +def norm_metric(name=None, raw_name=NotImplemented, raw_unit_registry=NotImplemented, depends=NotImplemented, schema=NotImplemented): """Decorator that creates metric class from a function or class.""" def dec(f): clsname = name or f.__name__ - return _gen_norm_metricclass(f=f, name=clsname, r_name=raw_name, scheme=schema, depends=depends) + return _gen_norm_metricclass(f=f, name=clsname, r_name=raw_name, r_regitry=raw_unit_registry, scheme=schema, depends=depends) return dec @@ -114,13 +111,11 @@ def build_normalized_metric(raw_metric): _norm_schema = build_normalized_schema(raw_metric) _norm_name = "norm_" + raw_metric.__name__ _raw_name = raw_metric.__name__ + _raw_units_registry = raw_metric.registry - @norm_metric(name=_norm_name, raw_name=_raw_name , depends=_norm_deps, schema=_norm_schema) - def new_norm_metric(raw_name, raw): - if (raw_name not in evaluator.UNITS_REGISTRY): - return raw + @norm_metric(name=_norm_name, raw_name=_raw_name, raw_unit_registry=_raw_units_registry, depends=_norm_deps, schema=_norm_schema) + def new_norm_metric(unit_registry, raw_name, raw): - unit_registry = evaluator.UNITS_REGISTRY[raw_name] norm_pdf = raw.copy(deep=True) for unit in unit_registry: u_col_name = unit_registry[unit][0] From d2f5d2b8869baafef8597f32fe208614b98d1598 Mon Sep 17 00:00:00 2001 From: Baptiste Mouginot Date: Fri, 14 Feb 2020 20:08:48 -0600 Subject: [PATCH 07/11] now adds the unit name in the columns of the pdf --- cymetric/units.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/cymetric/units.py b/cymetric/units.py index 94992acb..2f16c7aa 100644 --- a/cymetric/units.py +++ b/cymetric/units.py @@ -83,19 +83,16 @@ def dec(f): def build_conversion_col(col): conversion_col = [ureg.parse_expression( x).to_root_units().magnitude for x in col] - return conversion_col + default_unit = ureg.parse_expression(col[0]).to_root_units().units + return conversion_col, default_unit -def build_normalized_schema(raw_cls): - # if not in the unit registery: nothing to do - if (raw_cls.__name__ not in evaluator.UNITS_REGISTRY): - return raw_cls.schema +def build_normalized_schema(raw_cls, unit_registry): # initialize the normed metric schema norm_schema = raw_cls.schema # removing units columns form the new schema - unit_registry = evaluator.UNITS_REGISTRY[raw_cls.__name__] print(norm_schema) for key in unit_registry: idx = norm_schema.index( (unit_registry[key][0], 4, None)) @@ -108,28 +105,31 @@ def build_normalized_metric(raw_metric): _norm_deps = [raw_metric.__name__] - _norm_schema = build_normalized_schema(raw_metric) + _norm_schema = build_normalized_schema(raw_metric, raw_metric.registry) _norm_name = "norm_" + raw_metric.__name__ _raw_name = raw_metric.__name__ _raw_units_registry = raw_metric.registry @norm_metric(name=_norm_name, raw_name=_raw_name, raw_unit_registry=_raw_units_registry, depends=_norm_deps, schema=_norm_schema) - def new_norm_metric(unit_registry, raw_name, raw): + def new_norm_metric(raw_name, unit_registry, raw): norm_pdf = raw.copy(deep=True) for unit in unit_registry: u_col_name = unit_registry[unit][0] u_def_unit = unit_registry[unit][1] - + def_unit = "" # if a column for unit exist parse the colunm convert the value # drop the column if ( u_col_name != ""): - conv = build_conversion_col(raw[u_col_name]) + conv, def_unit = build_conversion_col(raw[u_col_name]) norm_pdf[unit] *= conv norm_pdf.drop([u_col_name], axis=1, inplace=True) else: # else use the default unit to convert it conv = ureg.parse_expression(u_def_unit).to_root_units().magnitude + def_unit = ureg.parse_expression(u_def_unit).to_root_units().units norm_pdf[unit] *= conv + print("defunit ", def_unit) + norm_pdf.rename(inplace=True, columns={unit : '{0} [{1:~P}]'.format(unit, def_unit)}) return norm_pdf From 446b53682fe2701bb2acdb18b2e1a040ca20449c Mon Sep 17 00:00:00 2001 From: Baptiste Mouginot Date: Fri, 14 Feb 2020 20:10:13 -0600 Subject: [PATCH 08/11] don't need to store the converted value --- cymetric/metrics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cymetric/metrics.py b/cymetric/metrics.py index ba99d750..804d6813 100644 --- a/cymetric/metrics.py +++ b/cymetric/metrics.py @@ -110,7 +110,7 @@ def dec(f): ('Units', ts.STRING), ('Mass', ts.DOUBLE) ] -_matregistry = { "Mass": ["Units", "kg", ""]} +_matregistry = { "Mass": ["Units", "kg"]} @metric(name='Materials', depends=_matdeps, schema=_matschema, registry=_matregistry) def materials(rsrcs, comps): From 1985dd08f00b4f9ffa95aad0838f48f4e55874b9 Mon Sep 17 00:00:00 2001 From: Baptiste Mouginot Date: Wed, 26 Feb 2020 09:50:10 -0600 Subject: [PATCH 09/11] now root metrics can have unit registery --- cymetric/evaluator.py | 22 +++++++++++--------- cymetric/metrics.py | 3 ++- cymetric/root_metrics.py | 43 ++++++++++++++++++++++++++++++---------- cymetric/units.py | 8 +++----- 4 files changed, 51 insertions(+), 25 deletions(-) diff --git a/cymetric/evaluator.py b/cymetric/evaluator.py index a7d072f5..18214590 100644 --- a/cymetric/evaluator.py +++ b/cymetric/evaluator.py @@ -9,19 +9,17 @@ from cymetric import units METRIC_REGISTRY = {} -UNITS_REGISTRY = {} def register_metric(cls): """Adds a metric to the registry.""" METRIC_REGISTRY[cls.__name__] = cls if cls.registry and cls.registry is not NotImplemented: - print(cls.registry) units.build_normalized_metric(cls) class Evaluator(object): """An evaluation context for metrics.""" - def __init__(self, db, write=True): + def __init__(self, db, write=True, normed=True): """Parameters ---------- db : database @@ -42,24 +40,30 @@ def __init__(self, db, write=True): self.recorder = rec = lib.Recorder(inject_sim_id=False) rec.register_backend(db) self.known_tables = db.tables - #try: - # self.registry = self.db.query("table_registry") + self.set_norm = normed - def get_metric(self, metric): + def get_metric(self, metric, normed=False): """Checks if metric is already in the registry; adds it if not.""" + normed_name = "norm_" + metric + if normed and normed_name in METRIC_REGISTRY: + metric = normed_name if metric not in self.metrics: self.metrics[metric] = METRIC_REGISTRY[metric](self.db) return self.metrics[metric] - def eval(self, metric, conds=None): + def eval(self, metric, conds=None, normed=None): """Evalutes a metric with the given conditions.""" + normed_name = "norm_" + metric + if (normed == True or (normed is None and self.set_norm == True)) and normed_name in METRIC_REGISTRY: + metric = normed_name + rawkey = (metric, conds if conds is None else frozenset(conds)) if rawkey in self.rawcache: return self.rawcache[rawkey] - m = self.get_metric(metric) + m = self.get_metric(metric, normed) frames = [] for dep in m.dependencies: - frame = self.eval(dep, conds=conds) + frame = self.eval(dep, conds=conds, normed=False) frames.append(frame) raw = m(frames=frames, conds=conds, known_tables=self.known_tables) if raw is None: diff --git a/cymetric/metrics.py b/cymetric/metrics.py index 804d6813..3673cae9 100644 --- a/cymetric/metrics.py +++ b/cymetric/metrics.py @@ -310,8 +310,9 @@ def agents(entry, exit, decom, info): ('Units', ts.STRING), ('Quantity', ts.DOUBLE) ] +_transregistry = { "Quantity": ["Units", "kg"]} -@metric(name='TransactionQuantity', depends=_transdeps, schema=_transschema) +@metric(name='TransactionQuantity', depends=_transdeps, schema=_transschema, registry=_transregistry) def transaction_quantity(mats, tranacts): """Transaction Quantity metric returns the quantity of each transaction throughout the simulation. diff --git a/cymetric/root_metrics.py b/cymetric/root_metrics.py index f1181404..a8506326 100644 --- a/cymetric/root_metrics.py +++ b/cymetric/root_metrics.py @@ -2,25 +2,30 @@ generated by Cyclus itself. """ from __future__ import print_function, unicode_literals +from cyclus import typesystem as ts try: from cymetric.evaluator import register_metric + from cymetric import schemas except ImportError: # some wacky CI paths prevent absolute importing, try relative + from . import schemas from .evaluator import register_metric -def _genrootclass(name, register): +def _genrootclass(name, schema, register): """Creates a new root metric class.""" + if schema != None and not isinstance(schema, schemas.schema): + schema = schemas.schema(schema) class Cls(object): dependencies = () registry = register - @property - def schema(self): - """Defines schema for root metric if provided.""" - if self._schema is not None: - return self._schema - # fill in schema code + #@property + #def schema(self): + # """Defines schema for root metric if provided.""" + # if self._schema is not None: + # return self._schema + # # fill in schema code @property def name(self): @@ -38,22 +43,40 @@ def __call__(self, conds=None, *args, **kwargs): return None return self.db.query(self.name, conds=conds) + Cls.schema = schema Cls.__name__ = str(name) register_metric(Cls) return Cls -def root_metric(obj=None, name=None, schema=None, registry=None, *args, **kwargs): +def root_metric(obj=None, name=None, schema=None, registry=NotImplemented, *args, **kwargs): """Decorator that creates a root metric from a function or class.""" if obj is not None: raise RuntimeError if name is None: raise RuntimeError - return _genrootclass(name=name, register=registry) + return _genrootclass(name=name, schema=schema, register=registry) #core tables -resources = root_metric(name='Resources') +_resour_registry = { "Quantity": ["Units", "kg"]} +_resource_shema = [ + ('SimId', ts.UUID), + ('ResourceId', ts.INT), + ('ObjId', ts.INT), + ('Type', ts.STRING), + ('TimeCreated', ts.INT), + ('Quantity', ts.DOUBLE), + ('Units', ts.STRING), + ('QualId', ts.INT), + ('Parent1', ts.INT), + ('Parent2', ts.INT) + ] +resources = root_metric(name='Resources', schema=_resource_shema, registry=_resour_registry) + +#del _resour_registry, _resource_shema +#resources = root_metric(name='Resources') + compositions = root_metric(name='Compositions') recipes = root_metric(name='Recipes') products = root_metric(name='Products') diff --git a/cymetric/units.py b/cymetric/units.py index 2f16c7aa..c97ed4d9 100644 --- a/cymetric/units.py +++ b/cymetric/units.py @@ -88,16 +88,15 @@ def build_conversion_col(col): def build_normalized_schema(raw_cls, unit_registry): - + if raw_cls.schema is None: + return None # initialize the normed metric schema norm_schema = raw_cls.schema - - # removing units columns form the new schema print(norm_schema) + # removing units columns form the new schema for key in unit_registry: idx = norm_schema.index( (unit_registry[key][0], 4, None)) norm_schema.pop(idx) - print(norm_schema) return norm_schema @@ -128,7 +127,6 @@ def new_norm_metric(raw_name, unit_registry, raw): conv = ureg.parse_expression(u_def_unit).to_root_units().magnitude def_unit = ureg.parse_expression(u_def_unit).to_root_units().units norm_pdf[unit] *= conv - print("defunit ", def_unit) norm_pdf.rename(inplace=True, columns={unit : '{0} [{1:~P}]'.format(unit, def_unit)}) return norm_pdf From c1a7ac0c23d698b98b01ec6c6107822d1d4749b2 Mon Sep 17 00:00:00 2001 From: Baptiste Mouginot Date: Fri, 28 Feb 2020 09:34:46 -0600 Subject: [PATCH 10/11] cleaning --- cymetric/units.py | 1 - 1 file changed, 1 deletion(-) diff --git a/cymetric/units.py b/cymetric/units.py index c97ed4d9..0d4a2877 100644 --- a/cymetric/units.py +++ b/cymetric/units.py @@ -92,7 +92,6 @@ def build_normalized_schema(raw_cls, unit_registry): return None # initialize the normed metric schema norm_schema = raw_cls.schema - print(norm_schema) # removing units columns form the new schema for key in unit_registry: idx = norm_schema.index( (unit_registry[key][0], 4, None)) From abc41ef8efdda8193eab51c102dce4a35be70e84 Mon Sep 17 00:00:00 2001 From: Baptiste Mouginot Date: Mon, 2 Mar 2020 12:27:58 -0600 Subject: [PATCH 11/11] add inception comment --- cymetric/evaluator.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cymetric/evaluator.py b/cymetric/evaluator.py index 18214590..ebaa4419 100644 --- a/cymetric/evaluator.py +++ b/cymetric/evaluator.py @@ -56,13 +56,14 @@ def eval(self, metric, conds=None, normed=None): normed_name = "norm_" + metric if (normed == True or (normed is None and self.set_norm == True)) and normed_name in METRIC_REGISTRY: metric = normed_name - + rawkey = (metric, conds if conds is None else frozenset(conds)) if rawkey in self.rawcache: return self.rawcache[rawkey] m = self.get_metric(metric, normed) frames = [] for dep in m.dependencies: + # norm=False to avoid inception frame = self.eval(dep, conds=conds, normed=False) frames.append(frame) raw = m(frames=frames, conds=conds, known_tables=self.known_tables)