diff --git a/cymetric/evaluator.py b/cymetric/evaluator.py index a198460b..d5453d5e 100644 --- a/cymetric/evaluator.py +++ b/cymetric/evaluator.py @@ -6,20 +6,20 @@ from cyclus import lib from cymetric.tools import raw_to_series - +from cymetric import units METRIC_REGISTRY = {} - def register_metric(cls): """Adds a metric to the registry.""" METRIC_REGISTRY[cls.__name__] = cls - + if cls.registry and cls.registry is not NotImplemented: + units.build_normalized_raw_metric(cls) class Evaluator(object): """An evaluation context for metrics.""" - def __init__(self, db, write=True): + def __init__(self, db, write=True, normed=True): """Parameters ---------- db : database @@ -40,22 +40,32 @@ def __init__(self, db, write=True): self.recorder = rec = lib.Recorder(inject_sim_id=False) rec.register_backend(db) self.known_tables = db.tables + self.set_norm = normed - def get_metric(self, metric): + def get_metric(self, metric, normed=False): """Checks if metric is already in the registry; adds it if not.""" + normed_name = "norm_" + metric + if normed and normed_name in METRIC_REGISTRY: + metric = normed_name if metric not in self.metrics: self.metrics[metric] = METRIC_REGISTRY[metric](self.db) return self.metrics[metric] - def eval(self, metric, conds=None): + def eval(self, metric, conds=None, normed=None): """Evalutes a metric with the given conditions.""" + requested_metric = metric + normed_name = "norm_" + metric + if (normed == True or (normed is None and self.set_norm == True)) and normed_name in METRIC_REGISTRY: + metric = normed_name + rawkey = (metric, conds if conds is None else frozenset(conds)) if rawkey in self.rawcache: return self.rawcache[rawkey] - m = self.get_metric(metric) + m = self.get_metric(metric, normed) frames = [] for dep in m.dependencies: - frame = self.eval(dep, conds=conds) + #normed condition avoid inceptions + frame = self.eval(dep, conds=conds, normed=(dep!=requested_metric) ) frames.append(frame) raw = m(frames=frames, conds=conds, known_tables=self.known_tables) if raw is None: @@ -81,3 +91,6 @@ def eval(metric, db, conds=None, write=True): """Evalutes a metric with the given conditions in a database.""" e = Evaluator(db, write=write) return e.eval(str(metric), conds=conds) + + + diff --git a/cymetric/metrics.py b/cymetric/metrics.py index aa62dcab..26e35641 100644 --- a/cymetric/metrics.py +++ b/cymetric/metrics.py @@ -20,17 +20,20 @@ from cymetric import schemas from cymetric import tools from cymetric.evaluator import register_metric + from cymetric.units import ureg except ImportError: # some wacky CI paths prevent absolute importing, try relative from . import schemas from . import tools from .evaluator import register_metric + from .units import ureg class Metric(object): """Metric class""" dependencies = NotImplemented schema = NotImplemented + registry = NotImplemented def __init__(self, db): self.db = db @@ -39,8 +42,7 @@ def __init__(self, db): def name(self): return self.__class__.__name__ - -def _genmetricclass(f, name, depends, scheme): +def _genmetricclass(f, name, depends, scheme, register): """Creates a new metric class with a given name, dependencies, and schema. Parameters @@ -59,8 +61,10 @@ class Cls(Metric): dependencies = depends schema = scheme func = staticmethod(f) - __doc__ = inspect.getdoc(f) + + def shema(self): + return schema def __init__(self, db): """Constructor for metric object in database.""" @@ -74,19 +78,39 @@ def __call__(self, frames, conds=None, known_tables=None, *args, **kwargs): if self.name in known_tables: return self.db.query(self.name, conds=conds) return f(*frames) - + + if register is not NotImplemented: + build_norm_metric(f, name, depends, scheme, register) + Cls.__name__ = str(name) register_metric(Cls) + return Cls -def metric(name=None, depends=NotImplemented, schema=NotImplemented): +def metric(name=None, depends=NotImplemented, schema=NotImplemented,registry=NotImplemented): """Decorator that creates metric class from a function or class.""" def dec(f): clsname = name or f.__name__ - return _genmetricclass(f=f, name=clsname, scheme=schema, depends=depends) + return _genmetricclass(f=f, name=clsname, scheme=schema, depends=depends, register=registry) return dec +def build_norm_metric(f, name, depends, scheme, register): + + _norm_name = "norm_" + name + _norm_schema = scheme + _norm_deps = depends + for unit in register: + unit_col, deps = register[unit] + + @metric(name=_norm_name, depends=_norm_deps, schema=_norm_schema) + def norm_metric(*frame): + return f(norm=True, *frame) + + del _norm_deps, _norm_schema, _norm_name + + + ##################### ## General Metrics ## @@ -105,18 +129,34 @@ def dec(f): ('Units', ts.STRING), ('Mass', ts.DOUBLE) ] +_matregistry = { "Mass": ["Units", "Resources"]} -@metric(name='Materials', depends=_matdeps, schema=_matschema) -def materials(rsrcs, comps): +@metric(name='Materials', depends=_matdeps, schema=_matschema, registry=_matregistry) +def materials(rsrcs, comps, norm=False): """Materials metric returns the material mass (quantity of material in Resources times the massfrac in Compositions) indexed by the SimId, QualId, ResourceId, ObjId, TimeCreated, and NucId. """ + index = ['SimId', 'QualId', 'ResourceId', 'ObjId', 'TimeCreated', 'NucId', 'Units'] + mass_col_name = "mass" + quantity_col_name = "Quantity" + # some change in case of normalisation + if norm: + index = ['SimId', 'QualId', 'ResourceId', 'ObjId', 'TimeCreated', 'NucId'] + for col in rsrcs.columns: + col_orign = col[:-1].split('[') + #detect col with units Resources + if col_orign[0] == 'Quantity ': + # get col name from + quantity_col_name = col + # form new col name for Material metric + def_unit = ureg.parse_expression(col_orign[1]).to_root_units().units + mass_col_name = '{0} [{1:~P}]'.format(col_orign[0], def_unit) + x = pd.merge(rsrcs, comps, on=['SimId', 'QualId'], how='inner') - x = x.set_index(['SimId', 'QualId', 'ResourceId', 'ObjId','TimeCreated', - 'NucId', 'Units']) - y = x['Quantity'] * x['MassFrac'] - y.name = 'Mass' + x = x.set_index(index) + y = x[quantity_col_name] * x['MassFrac'] + y.name = mass_col_name z = y.reset_index() return z @@ -304,8 +344,9 @@ def agents(entry, exit, decom, info): ('Units', ts.STRING), ('Quantity', ts.DOUBLE) ] +_transregistry = { "Quantity": ["Units", "kg"]} -@metric(name='TransactionQuantity', depends=_transdeps, schema=_transschema) +@metric(name='TransactionQuantity', depends=_transdeps, schema=_transschema, registry=_transregistry) def transaction_quantity(mats, tranacts): """Transaction Quantity metric returns the quantity of each transaction throughout the simulation. @@ -400,7 +441,7 @@ def annual_electricity_generated_by_agent(elec): 'AgentId': elec.AgentId, 'Year': elec.Time.apply(lambda x: x//12), 'Energy': elec.Value.apply(lambda x: x/12)}, - columns=['SimId', 'AgentId', 'Year', 'Energy']) + columns=['SimId', 'AgentId', 'Year', 'Energy']) el_index = ['SimId', 'AgentId', 'Year'] elec = elec.groupby(el_index).sum() rtn = elec.reset_index() diff --git a/cymetric/root_metrics.py b/cymetric/root_metrics.py index d3e78be2..37ed3981 100644 --- a/cymetric/root_metrics.py +++ b/cymetric/root_metrics.py @@ -2,20 +2,30 @@ generated by Cyclus itself. """ from __future__ import print_function, unicode_literals +from cyclus import typesystem as ts -from cymetric.evaluator import register_metric +try: + from cymetric.evaluator import register_metric + from cymetric import schemas +except ImportError: + # some wacky CI paths prevent absolute importing, try relative + from . import schemas + from .evaluator import register_metric -def _genrootclass(name): +def _genrootclass(name, schema, register): """Creates a new root metric class.""" + if schema != None and not isinstance(schema, schemas.schema): + schema = schemas.schema(schema) class Cls(object): dependencies = () - @property - def schema(self): - """Defines schema for root metric if provided.""" - if self._schema is not None: - return self._schema - # fill in schema code + registry = register + #@property + #def schema(self): + # """Defines schema for root metric if provided.""" + # if self._schema is not None: + # return self._schema + # # fill in schema code @property def name(self): @@ -33,22 +43,39 @@ def __call__(self, conds=None, *args, **kwargs): return None return self.db.query(self.name, conds=conds) + Cls.schema = schema Cls.__name__ = str(name) register_metric(Cls) return Cls -def root_metric(obj=None, name=None, schema=None, *args, **kwargs): +def root_metric(obj=None, name=None, schema=None, registry=NotImplemented, *args, **kwargs): """Decorator that creates a root metric from a function or class.""" if obj is not None: raise RuntimeError if name is None: raise RuntimeError - return _genrootclass(name=name) + return _genrootclass(name=name, schema=schema, register=registry) #core tables -resources = root_metric(name='Resources') +_resour_registry = { "Quantity": ["Units", "kg"]} +_resource_shema = [ + ('SimId', ts.UUID), + ('ResourceId', ts.INT), + ('ObjId', ts.INT), + ('Type', ts.STRING), + ('TimeCreated', ts.INT), + ('Quantity', ts.DOUBLE), + ('Units', ts.STRING), + ('QualId', ts.INT), + ('Parent1', ts.INT), + ('Parent2', ts.INT) + ] +resources = root_metric(name='Resources', schema=_resource_shema, registry=_resour_registry) +del _resour_registry, _resource_shema +#resources = root_metric(name='Resources') + compositions = root_metric(name='Compositions') recipes = root_metric(name='Recipes') products = root_metric(name='Products') diff --git a/cymetric/units.py b/cymetric/units.py new file mode 100644 index 00000000..62e05d2e --- /dev/null +++ b/cymetric/units.py @@ -0,0 +1,140 @@ +""" Convert able to the default unit system. +""" +import inspect + +try: + from cymetric import schemas + from cymetric import tools + from cymetric import evaluator +except ImportError: + # some wacky CI paths prevent absolute importing, try relative + from . import schemas + from . import tools + from . import evaluator + +import pint + +ureg = pint.UnitRegistry() + +class NormMetric(object): + """Metric class""" + dependencies = NotImplemented + schema = NotImplemented + registry = NotImplemented + + def __init__(self, db): + self.db = db + + @property + def name(self): + return self.__class__.__name__ + +def _gen_norm_metricclass(f, name, r_name, r_regitry, depends, scheme): + """Creates a new metric class with a given name, dependencies, and schema. + + Parameters + ---------- + name : str + Metric name + depends : list of lists (table name, tuple of indices, column name) + Dependencies on other database tables (metrics or root metrics) + scheme : list of tuples (column name, data type) + Schema for metric + """ + if not isinstance(scheme, schemas.schema): + scheme = schemas.schema(scheme) + + class Cls(NormMetric): + dependencies = depends + schema = scheme + func = staticmethod(f) + raw_name = r_name + raw_unit_registry = r_regitry + __doc__ = inspect.getdoc(f) + + def __init__(self, db): + """Constructor for metric object in database.""" + super(Cls, self).__init__(db) + + def __call__(self, frames, conds=None, known_tables=None, *args, **kwargs): + """Computes metric for given input data and conditions.""" + # FIXME test if I already exist in the db, read in if I do + if known_tables is None: + known_tables = self.db.tables() + if self.name in known_tables: + return self.db.query(self.name, conds=conds) + return f(self.raw_name, self.raw_unit_registry, *frames) + + Cls.__name__ = str(name) + evaluator.register_metric(Cls) + return Cls + + + +def norm_raw_metric(name=None, raw_name=NotImplemented, raw_unit_registry=NotImplemented, depends=NotImplemented, schema=NotImplemented): + """Decorator that creates metric class from a function or class.""" + def dec(f): + clsname = name or f.__name__ + return _gen_norm_metricclass(f=f, name=clsname, r_name=raw_name, r_regitry=raw_unit_registry, scheme=schema, depends=depends) + return dec + + +def build_conversion_col(col): + conversion_col = [ureg.parse_expression( + x).to_root_units().magnitude for x in col] + default_unit = ureg.parse_expression(col[0]).to_root_units().units + return conversion_col, default_unit + + +def build_normalized_schema(raw_cls, unit_registry): + if raw_cls.schema is None: + return None + # initialize the normed metric schema + norm_schema = raw_cls.schema + # removing units columns form the new schema + for key in unit_registry: + idx = norm_schema.index( (unit_registry[key][0], 4, None)) + norm_schema.pop(idx) + return norm_schema + + +def build_normalized_raw_metric(raw_metric): + + # Build raw normed metric + # name + _norm_name = "norm_" + raw_metric.__name__ + # deps + _norm_deps = [raw_metric.__name__] + # schema + _norm_schema = build_normalized_schema(raw_metric, raw_metric.registry) + + + _raw_name = raw_metric.__name__ + _raw_units_registry = raw_metric.registry + + @norm_raw_metric(name=_norm_name, raw_name=_raw_name, raw_unit_registry=_raw_units_registry, depends=_norm_deps, schema=_norm_schema) + def raw_norm_metric(raw_name, unit_registry, raw): + + norm_pdf = raw.copy(deep=True) + for unit in unit_registry: + u_col_name = unit_registry[unit][0] + u_def_unit = unit_registry[unit][1] + def_unit = "" + # if a column for unit exist parse the colunm convert the value + # drop the column + if ( u_col_name != ""): + conv, def_unit = build_conversion_col(raw[u_col_name]) + norm_pdf[unit] *= conv + norm_pdf.drop([u_col_name], axis=1, inplace=True) + else: # else use the default unit to convert it + conv = ureg.parse_expression(u_def_unit).to_root_units().magnitude + def_unit = ureg.parse_expression(u_def_unit).to_root_units().units + norm_pdf[unit] *= conv + norm_pdf.rename(inplace=True, columns={unit : '{0} [{1:~P}]'.format(unit, def_unit)}) + + return norm_pdf + + # clean your mess + del _norm_deps, _norm_schema, _norm_name, _raw_name, _raw_units_registry + +