Merge pull request #174 from KlimaDAO/dash-api

Dash api improvements for carbon dashboard

app-spec.yml

@@ -55,17 +55,17 @@ services:
       tag: ${GITHUB_SHA}
     health_check:
       http_path: /api/v1
-      initial_delay_seconds: 60
-      timeout_seconds: 1200
-      period_seconds: 20
+      initial_delay_seconds: 0
+      timeout_seconds: 1
+      period_seconds: 10
       success_threshold: 1
-      failure_threshold: 20
+      failure_threshold: 6
     http_port: 8050
     routes:
       - path: /api/v1/
         preserve_path_prefix: true
     instance_count: 1
-    instance_size_slug: basic-xs
+    instance_size_slug: basic-s
     name: carbon-api
-    run_command: gunicorn --worker-tmp-dir /dev/shm --timeout ${GUNICORN_TIMEOUT} src.apps.api.app:app
+    run_command: gunicorn --workers=1 --threads=2 --worker-tmp-dir /dev/shm --timeout ${GUNICORN_TIMEOUT} src.apps.api.app:app
     source_dir: /

src/apps/api/app.py

@@ -25,14 +25,27 @@ def output_json(data, code, headers=None):
 api.add_resource(endpoints.CreditsRaw, '/credits/raw')
 api.add_resource(endpoints.CreditsGlobalAggregation, '/credits/agg')
 api.add_resource(endpoints.CreditsDatesAggregation, '/credits/agg/<string:freq>')
-api.add_resource(endpoints.CreditsCountriesAggregation, '/credits/agg/countries')
-api.add_resource(endpoints.CreditsProjectsAggregation, '/credits/agg/projects')
-api.add_resource(endpoints.CreditsMethodologiesAggregation, '/credits/agg/methodologies')
+api.add_resource(endpoints.CreditsCountriesAggregation, '/credits/agg/country')
+api.add_resource(endpoints.CreditsProjectsAggregation, '/credits/agg/project')
+api.add_resource(endpoints.CreditsMethodologiesAggregation, '/credits/agg/methodology')
 api.add_resource(endpoints.CreditsVintageAggregation, '/credits/agg/vintage')
+api.add_resource(endpoints.CreditsPoolAggregation, '/credits/agg/pool')
+api.add_resource(endpoints.CreditsPoolVintageAggregation, '/credits/agg/pool/vintage')
+api.add_resource(endpoints.CreditsPoolMethodologyAggregation, '/credits/agg/pool/methodology')
+api.add_resource(endpoints.CreditsPoolProjectsAggregation, '/credits/agg/pool/project')
+api.add_resource(endpoints.CreditsPoolDatesAggregation, '/credits/agg/pool/<string:freq>')
+api.add_resource(endpoints.CreditsBridgeVintageAggregation, '/credits/agg/bridge/vintage')
+api.add_resource(endpoints.CreditsBridgeCountriesAggregation, '/credits/agg/bridge/country')
+api.add_resource(endpoints.CreditsBridgeProjectsAggregation, '/credits/agg/bridge/project')
+api.add_resource(endpoints.CreditsBridgeDateAggregation, '/credits/agg/bridge/<string:freq>')
+api.add_resource(endpoints.CreditsBridgeAggregation, '/credits/agg/bridge')
+
 
 api.add_resource(endpoints.PoolsRaw, '/pools/raw')
 api.add_resource(endpoints.PoolsGlobalAggregation, '/pools/agg')
 api.add_resource(endpoints.PoolsDatesAggregation, '/pools/agg/<string:freq>')
+api.add_resource(endpoints.PoolsTokensAndDatesAggregation, '/pools/agg/tokens/<string:freq>')
+
 
 api.add_resource(endpoints.Holders, '/holders')
 
@@ -48,7 +61,9 @@ def output_json(data, code, headers=None):
 api.add_resource(endpoints.RetirementsDatesAggregation, '/retirements/<string:filter>/agg/<string:freq>')
 api.add_resource(endpoints.RetirementsTokensAggregation, '/retirements/klima/agg/tokens')
 api.add_resource(endpoints.RetirementsTokensAndDatesAggregation, '/retirements/klima/agg/tokens/<string:freq>')
-api.add_resource(endpoints.RetirementsBeneficiariesAggregation, '/retirements/<string:filter>/agg/beneficiaries')
+api.add_resource(endpoints.RetirementsBeneficiariesAggregation, '/retirements/<string:filter>/agg/beneficiary')
+api.add_resource(endpoints.RetirementsOriginAndDatesAggregation, '/retirements/all/agg/origin/<string:freq>')
+
 
 api.add_resource(endpoints.Info, '', '/')
 

src/apps/api/endpoints/__init__.py

@@ -7,12 +7,23 @@
     CreditsProjectsAggregation,
     CreditsMethodologiesAggregation,
     CreditsVintageAggregation,
+    CreditsPoolAggregation,
+    CreditsPoolVintageAggregation,
+    CreditsPoolMethodologyAggregation,
+    CreditsPoolProjectsAggregation,
+    CreditsPoolDatesAggregation,
+    CreditsBridgeVintageAggregation,
+    CreditsBridgeCountriesAggregation,
+    CreditsBridgeProjectsAggregation,
+    CreditsBridgeDateAggregation,
+    CreditsBridgeAggregation,
     CreditsGlobalAggregation
 )
 from .pools import (  # noqa
     PoolsRaw,
     PoolsGlobalAggregation,
-    PoolsDatesAggregation
+    PoolsDatesAggregation,
+    PoolsTokensAndDatesAggregation
 )
 from .holders import Holders  # noqa
 from .prices import Prices  # noqa
@@ -23,6 +34,7 @@
     RetirementsBeneficiariesAggregation,
     RetirementsGlobalAggregation,
     RetirementsTokensAndDatesAggregation,
-    RetirementsTokensAggregation
+    RetirementsTokensAggregation,
+    RetirementsOriginAndDatesAggregation,
 )
 from .tokens import Tokens  # noqa

src/apps/api/endpoints/carbon_metrics.py

@@ -1,7 +1,10 @@
-from flask_restful import Resource
+from flask_restful import Resource, reqparse
 from src.apps.services import Metrics as Service, layout_cache, DashArgumentException
 from . import helpers
 
+carbon_metrics_parser = reqparse.RequestParser()
+carbon_metrics_parser.add_argument('sample', type=helpers.validate_list(["daily", "monthly"]), default="all")
+
 
 class CarbonMetrics(Resource):
     @layout_cache.cached(query_string=True)
@@ -18,5 +21,11 @@ def get(self, chain):
         service = Service()
         if (chain not in ["eth", "polygon", "celo", "all"]):
             raise DashArgumentException(f"Unknown chain '{chain}'")
+        args = carbon_metrics_parser.parse_args()
+        sample = args["sample"]
         metrics = getattr(service, chain)()
+
+        if sample == "monthly":
+            metrics = metrics.monthly_sample("date")
+
         return metrics

src/apps/api/endpoints/credits.py

@@ -4,8 +4,9 @@
 
 
 BRIDGES = ["all", "offchain", "toucan", "c3", "moss", "polygon", "eth"]
-POOLS = ["ubo", "nbo", "nct", "bct"]
-STATUSES = ["issued", "bridged", "deposited", "redeemed", "retired"]
+POOLS = ["ubo", "nbo", "nct", "bct", "all"]
+STATUSES = ["issued", "bridged", "deposited", "redeemed", "retired", "all_retired", "all"]
+OFFCHAIN_FILTER = ["tokenized", "toucan", "c3", "moss"]
 DATE_FIELDS = [
     "bridged_date",
     "issuance_date",
@@ -21,6 +22,11 @@
 credits_filter_parser.add_argument('bridge', type=helpers.validate_list(BRIDGES), default="all")
 credits_filter_parser.add_argument('pool', type=helpers.validate_list(POOLS + [None]), default=None)
 credits_filter_parser.add_argument('status', type=helpers.validate_list(STATUSES + [None]), default=None)
+credits_filter_parser.add_argument(
+    'offchain_filter',
+    type=helpers.validate_list(OFFCHAIN_FILTER + [None]),
+    default=None
+)
 
 
 class AbstractCredits(Resource):
@@ -30,23 +36,56 @@ def get_default_date_field(self):
         args = credits_filter_parser.parse_args()
         bridge = args["bridge"]
         status = args["status"]
+
         if status is None:
             return "issuance_date" if bridge == "offchain" else "bridged_date"
         else:
             return helpers.status_date_column(status)
 
-    def get_credits(self):
+    @helpers.with_daterange_filter("bridged_date")
+    @helpers.with_daterange_filter("issuance_date")
+    @helpers.with_daterange_filter("retirement_date")
+    @helpers.with_daterange_filter("deposited_date")
+    @helpers.with_daterange_filter("redeemed_date")
+    def get_credits(self, bridge=None):
         args = credits_filter_parser.parse_args()
-        bridge = args["bridge"]
+        if not bridge:
+            bridge = args["bridge"]
         pool = args["pool"]
         status = args["status"]
+        offchain_filter = args["offchain_filter"]
+
+        # Accept a 'all' value for pools
+        if pool == "all":
+            pool = None
+
+        # Accept a 'all' value for status
+        if status == "all":
+            status = None
 
         # auto select status
         if status is None:
             status = "issued" if bridge == "offchain" else "bridged"
 
-        # Return credits
-        return Service().filter(bridge, pool, status)
+        # Select credits
+        df = Service().filter(bridge, pool, status)
+
+        # Filter offchain credits
+        if offchain_filter:
+            df = df.offchain_filter(offchain_filter)
+
+        return df
+
+    def get_pooled_credits(self, bridge=None):
+        """
+        Hack: Filter the polygon datasets for pool analysis
+        because the pool columns quantities are not properly made
+        """
+        args = credits_filter_parser.parse_args()
+        credits = self.get_credits(bridge)
+        if args["bridge"] in ["toucan", "c3", "polygon"]:
+            credits = credits.pool_analysis()
+        return credits
 
 
 class CreditsRaw(AbstractCredits):
@@ -59,9 +98,6 @@ class CreditsRaw(AbstractCredits):
         """
     )
     @helpers.with_output_formatter
-    @helpers.with_daterange_filter("bridged_date")
-    @helpers.with_daterange_filter("issuance_date")
-    @helpers.with_daterange_filter("retirement_date")
     def get(self):
         return self.get_credits()
 
@@ -77,11 +113,6 @@ class CreditsDatesAggregation(AbstractCredits):
         """
     )
     @helpers.with_output_formatter
-    @helpers.with_daterange_filter("bridged_date")
-    @helpers.with_daterange_filter("issuance_date")
-    @helpers.with_daterange_filter("retirement_date")
-    @helpers.with_daterange_filter("deposited_date")
-    @helpers.with_daterange_filter("redeemed_date")
     def get(self, freq):
         return helpers.apply_date_aggregation(
             DATE_FIELDS,
@@ -147,6 +178,159 @@ def get(self):
         return credits
 
 
+class CreditsPoolAggregation(AbstractCredits):
+    @layout_cache.cached(query_string=True)
+    @helpers.with_errors_handler
+    @helpers.with_help(
+        f"""{BASE_HELP}
+        """
+    )
+    def get(self):
+        credits = self.get_credits().pool_analysis().pool_summary().resolve().to_dict(orient='records')[0]
+        return credits
+
+
+class CreditsPoolVintageAggregation(AbstractCredits):
+    @layout_cache.cached(query_string=True)
+    @helpers.with_errors_handler
+    @helpers.with_help(
+        f"""{BASE_HELP}
+        {helpers.OUTPUT_FORMATTER_HELP}
+        """
+    )
+    @helpers.with_output_formatter
+    def get(self):
+        credits = self.get_pooled_credits().vintage_agg().pool_summary("vintage")
+        return credits
+
+
+class CreditsPoolMethodologyAggregation(AbstractCredits):
+    @layout_cache.cached(query_string=True)
+    @helpers.with_errors_handler
+    @helpers.with_help(
+        f"""{BASE_HELP}
+        {helpers.OUTPUT_FORMATTER_HELP}
+        """
+    )
+    @helpers.with_output_formatter
+    def get(self):
+        credits = self.get_pooled_credits().methodologies_agg().pool_summary("methodology")
+        return credits
+
+
+class CreditsPoolCountriesAggregation(AbstractCredits):
+    @layout_cache.cached(query_string=True)
+    @helpers.with_errors_handler
+    @helpers.with_help(
+        f"""{BASE_HELP}
+        {helpers.OUTPUT_FORMATTER_HELP}
+        """
+    )
+    @helpers.with_output_formatter
+    def get(self):
+        credits = self.get_pooled_credits().countries_agg().pool_summary(["country_code", "country"])
+        return credits
+
+
+class CreditsPoolProjectsAggregation(AbstractCredits):
+    @layout_cache.cached(query_string=True)
+    @helpers.with_errors_handler
+    @helpers.with_help(
+        f"""{BASE_HELP}
+        {helpers.OUTPUT_FORMATTER_HELP}
+        """
+    )
+    @helpers.with_output_formatter
+    def get(self):
+        credits = self.get_pooled_credits().projects_agg().pool_summary("project_type")
+        return credits
+
+
+class CreditsPoolDatesAggregation(AbstractCredits):
+    @layout_cache.cached(query_string=True)
+    @helpers.with_errors_handler
+    @helpers.with_help(
+        f"""{BASE_HELP}
+        {helpers.OUTPUT_FORMATTER_HELP}
+        """
+    )
+    @helpers.with_output_formatter
+    def get(self, freq):
+        date_column = self.get_default_date_field()
+        credits = self.get_credits().date_agg(date_column, freq).pool_summary(date_column)
+        return credits
+
+
+class CreditsBridgeVintageAggregation(AbstractCredits):
+    @layout_cache.cached(query_string=True)
+    @helpers.with_errors_handler
+    @helpers.with_help(
+        f"""{BASE_HELP}
+        {helpers.OUTPUT_FORMATTER_HELP}
+        """
+    )
+    @helpers.with_output_formatter
+    def get(self):
+        credits = self.get_credits(bridge="offchain").vintage_agg().bridge_summary("vintage")
+        return credits
+
+
+class CreditsBridgeCountriesAggregation(AbstractCredits):
+    @layout_cache.cached(query_string=True)
+    @helpers.with_errors_handler
+    @helpers.with_help(
+        f"""{BASE_HELP}
+        {helpers.OUTPUT_FORMATTER_HELP}
+        """
+    )
+    @helpers.with_output_formatter
+    def get(self):
+        credits = self.get_credits(bridge="offchain").countries_agg().bridge_summary(["country_code", "country"])
+        return credits
+
+
+class CreditsBridgeProjectsAggregation(AbstractCredits):
+    @layout_cache.cached(query_string=True)
+    @helpers.with_errors_handler
+    @helpers.with_help(
+        f"""{BASE_HELP}
+        {helpers.OUTPUT_FORMATTER_HELP}
+        """
+    )
+    @helpers.with_output_formatter
+    def get(self):
+        credits = self.get_credits(bridge="offchain").projects_agg().bridge_summary("project_type")
+        return credits
+
+
+class CreditsBridgeDateAggregation(AbstractCredits):
+    @layout_cache.cached(query_string=True)
+    @helpers.with_errors_handler
+    @helpers.with_help(
+        f"""{BASE_HELP}
+        {helpers.dates_aggregation_help(DATE_FIELDS)}
+        {helpers.OUTPUT_FORMATTER_HELP}
+        {helpers.DATES_FILTER_HELP}
+        """
+    )
+    @helpers.with_output_formatter
+    def get(self, freq):
+        return self.get_credits(bridge="offchain").date_agg("issuance_date", freq).bridge_summary("issuance_date")
+
+
+class CreditsBridgeAggregation(AbstractCredits):
+    @layout_cache.cached(query_string=True)
+    @helpers.with_errors_handler
+    @helpers.with_help(
+        f"""{BASE_HELP}
+        {helpers.dates_aggregation_help(DATE_FIELDS)}
+        {helpers.DATES_FILTER_HELP}
+        """
+    )
+    def get(self):
+        return self.get_credits(bridge="offchain").bridge_summary("quantity").resolve().to_dict(orient='records')[0]
+
+
 class CreditsGlobalAggregation(AbstractCredits):
     @layout_cache.cached(query_string=True)
     @helpers.with_errors_handler