Working first version

.github/workflows/lint-and-test.yml

@@ -17,13 +17,19 @@ jobs:
   test:
     needs: check
     runs-on: ubuntu-latest
-    #strategy:  # TODO: make work with pre-commit
-    #  matrix:
-    #    python-version: [ '3.8', '3.6' ]
-    name: "Test (on Python3.8)"
+    strategy:
+      fail-fast: false
+      matrix:
+        py_version: [ '3.8', '3.9' ]
+    name: "Test (on Python ${{ matrix.py_version }})"
     steps:
       - uses: actions/setup-python@v2
         with:
-          python-version: 3.8
-      - uses: actions/checkout@v2
+          python-version: ${{ matrix.py_version }}
+      - name: Check out src from Git
+        uses: actions/checkout@v2
+      - name: Get history and tags for SCM versioning to work
+        run: |
+          git fetch --prune --unshallow
+          git fetch --depth=1 origin +refs/tags/*:refs/tags/*
       - run: make test

.gitignore

@@ -79,3 +79,4 @@ dmypy.json
 .ipynb_checkpoints/
 notebooks/.ipynb_checkpoints/
 
+flexmeasures.log

Makefile

@@ -33,8 +33,8 @@ install-pip-tools:
 freeze-deps:
 	make install-pip-tools
 	pip-compile -o requirements/app.txt requirements/app.in
-	pip-compile -o requirements/dev.txt requirements/dev.in
 	pip-compile -o requirements/test.txt requirements/test.in
+	pip-compile -o requirements/dev.txt requirements/dev.in
 
 upgrade-deps:
 	make install-pip-tools

README.md

@@ -1,16 +1,47 @@
-# flexmeasures-openweathermap - a plugin for FlexMeasures
-
+# flexmeasures-openweathermap - a plugin for FlexMeasures to integrate OpenWeatherMap data
 
 ## Usage
 
+To register a new weather sensor:
+
+`flexmeasures owm register-weather-sensor --name "wind speed" --latitude 30 --longitude 40`
+
+Currently supported: wind speed, temperature & irradiance.
+
+Notes about weather sensor setup: 
+
+- Weather sensors are public. They are accessible by all accounts on a FlexMeasures server. TODO: maybe limit this to a list of account roles.
+- The resolution is one hour. OWM also supports minutely data within the upcoming hour(s), but that is not supported here.
+
+To collect weather forecasts:
+
+`flexmeasures owm get-weather-forecasts --location 30,40`
+
+This saves forecasts for your registered sensors in the database.
+
+Use the `--help`` option for more options, e.g. for specifying two locations and requesting that a number of weather stations cover the bounding box between them (where the locations represent top left and bottom right).
+
+An alternative usage is to save raw results in JSON files (for later processing), like this:
+
+`flexmeasures owm get-weather-forecasts --location 30,40 --store-as-json-files --region somewhere`
+
+This saves the complete response from OpenWeatherMap in a local folder (i.e. no sensor registration needed, this is a direct way to use OWM, without FlexMeasures integration). `region` will become a subfolder.
+
+Finally, note that currently 1000 free calls per day can be made to the OpenWeatherMap API,
+so you can make a call every 15 minutes for up to 10 locations or every hour for up to 40 locations (or get a paid account).
+
 
 ## Installation
 
-1. Add "/path/to/flexmeasures-openweathermap/flexmeasures_openweathermap" to your FlexMeasures (>v0.7.0dev8) config file,
-   using the FLEXMEASURES_PLUGINS setting (a list).
-   Alternatively, if you installed this plugin as a package, then "flexmeasures_openweathermap" suffices.
+To install locally, run
+
+    make install
+
+To add as plugin to an existing FlexMeasures system, add "/path/to/flexmeasures-openweathermap/flexmeasures_openweathermap" to your FlexMeasures (>v0.7.0dev8) config file,
+using the FLEXMEASURES_PLUGINS setting (a list).
+
+Alternatively, if you installed this plugin as a package (e.g. via `python setup.py install`, `pip install -e` or `pip install flexmeasures_openweathermap` after this project is on Pypi), then "flexmeasures_openweathermap" suffices.
 
-2.  
 
 
 ## Development

flexmeasures_openweathermap/__init__.py

@@ -12,7 +12,7 @@
 
 from flask import Blueprint
 
-from .utils import ensure_bp_routes_are_loaded_fresh
+from .utils.blueprinting import ensure_bp_routes_are_loaded_fresh
 
 # Overwriting version (if possible) from the package metadata
 # ― if this plugin has been installed as a package.
@@ -24,13 +24,36 @@
     # package is not installed
     pass
 
+
+DEFAULT_FILE_PATH_LOCATION = "weather-forecasts"
+DEFAULT_DATA_SOURCE_NAME = "OpenWeatherMap"
+DEFAULT_WEATHER_STATION_NAME = "weather station (created by FM-OWM)"
+WEATHER_STATION_TYPE_NAME = "weather station"
+
+__version__ = "0.1"
+__settings__ = {
+    "OPENWEATHERMAP_API_KEY": dict(
+        description="You can generate this token after you made an account at OpenWeatherMap.",
+        level="error",
+    ),
+    "OPENWEATHERMAP_FILE_PATH_LOCATION": dict(
+        description="Location of JSON files (if you store weather data in this form). Absolute path.",
+        level="debug",
+    ),
+    "OPENWEATHERMAP_DATA_SOURCE_NAME": dict(
+        description=f"Name of the data source for OWM data, defaults to '{DEFAULT_DATA_SOURCE_NAME}'",
+        level="debug",
+    ),
+    "WEATHER_STATION_NAME": dict(
+        description=f"Name of the weather station asset, defaults to '{DEFAULT_WEATHER_STATION_NAME}'",
+        level="debug",
+    ),
+}
+
 # CLI
-flexmeasures_openweathermap_cli_bp: Blueprint = Blueprint(
-    "flexmeasures-openweathermap CLI",
-    __name__,
-    cli_group="flexmeasures-openweathermap"
+flexmeasures_openweathermap_bp: Blueprint = Blueprint(
+    "flexmeasures-openweathermap CLI", __name__, cli_group="owm"
 )
-flexmeasures_openweathermap_cli_bp.cli.help = "flexmeasures-openweathermap CLI commands"
+flexmeasures_openweathermap_bp.cli.help = "flexmeasures-openweathermap CLI commands"
 ensure_bp_routes_are_loaded_fresh("cli.commands")
 from flexmeasures_openweathermap.cli import commands  # noqa: E402,F401
-

flexmeasures_openweathermap/cli/commands.py

@@ -1,17 +1,146 @@
 from flask import current_app
+
 from flask.cli import with_appcontext
 import click
+from flexmeasures.data.models.time_series import Sensor
+
 from flexmeasures.data.transactional import task_with_status_report
+from flexmeasures.data.config import db
+
+from .. import flexmeasures_openweathermap_bp
+from .schemas.weather_sensor import WeatherSensorSchema
+from ..utils.modeling import (
+    get_or_create_weather_station,
+)
+from ..utils.locating import get_locations
+from ..utils.filing import make_file_path
+from ..utils.owm import (
+    save_forecasts_in_db,
+    save_forecasts_as_json,
+    get_supported_sensor_spec,
+)
+from ..sensor_specs import owm_to_sensor_map
 
-from .. import flexmeasures_openweathermap_cli_bp
 
+"""
+TODO: allow to also pass an asset ID or name for the weather station (instead of location) to both commands?
+      See https://github.com/SeitaBV/flexmeasures-openweathermap/issues/2
+"""
 
-@flexmeasures_openweathermap_cli_bp.cli.command("hello-world")
+supported_sensors_list = ", ".join([str(o["name"]) for o in owm_to_sensor_map.values()])
+
+
+@flexmeasures_openweathermap_bp.cli.command("register-weather-sensor")
+@with_appcontext
 @click.option(
     "--name",
+    required=True,
+    help=f"Name of the sensor. Has to be from the supported list ({supported_sensors_list})",
 )
+# @click.option("--generic-asset-id", required=False, help="The asset id of the weather station (you can also give its location).")
+@click.option(
+    "--latitude",
+    required=True,
+    type=float,
+    help="Latitude of where you want to measure.",
+)
+@click.option(
+    "--longitude",
+    required=True,
+    type=float,
+    help="Longitude of where you want to measure.",
+)
+@click.option(
+    "--timezone",
+    default="UTC",
+    help="The timezone of the sensor data as string, e.g. 'UTC' (default) or 'Europe/Amsterdam'",
+)
+def add_weather_sensor(**args):
+    """
+    Add a weather sensor.
+    This will first create a weather station asset if none exists at the location yet.
+
+    """
+    errors = WeatherSensorSchema().validate(args)
+    if errors:
+        click.echo(
+            f"[FLEXMEASURES-OWM] Please correct the following errors:\n{errors}.\n Use the --help flag to learn more."
+        )
+        raise click.Abort
+
+    weather_station = get_or_create_weather_station(args["latitude"], args["longitude"])
+
+    fm_sensor_specs = get_supported_sensor_spec(args["name"])
+    fm_sensor_specs["generic_asset"] = weather_station
+    fm_sensor_specs["timezone"] = args["timezone"]
+    sensor = Sensor(**fm_sensor_specs)
+    sensor.attributes = fm_sensor_specs["attributes"]
+
+    db.session.add(sensor)
+    db.session.commit()
+    click.echo(
+        f"[FLEXMEASURES-OWM] Successfully created weather sensor with ID {sensor.id}, at weather station with ID {weather_station.id}"
+    )
+    click.echo(
+        f"[FLEXMEASURES-OWM] You can access this sensor at its entity address {sensor.entity_address}"
+    )
+
+
+@flexmeasures_openweathermap_bp.cli.command("get-weather-forecasts")
 @with_appcontext
-@task_with_status_report
-def hello_world(name: str):
-    print(f"Hello, {name}!")
-    current_app.logger.info(f"'Hello, {name}!' printed")
+@click.option(
+    "--location",
+    type=str,
+    required=True,
+    help='Measurement location(s). "latitude,longitude" or "top-left-latitude,top-left-longitude:'
+    'bottom-right-latitude,bottom-right-longitude." The first format defines one location to measure.'
+    " The second format defines a region of interest with several (>=4) locations"
+    ' (see also the "method" and "num_cells" parameters for details on how to use this feature).',
+)
+@click.option(
+    "--store-in-db/--store-as-json-files",
+    default=True,
+    help="Store forecasts in the database, or simply save as json files (defaults to database).",
+)
+@click.option(
+    "--num_cells",
+    type=int,
+    default=1,
+    help="Number of cells on the grid. Only used if a region of interest has been mapped in the location parameter. Defaults to 1.",
+)
+@click.option(
+    "--method",
+    default="hex",
+    type=click.Choice(["hex", "square"]),
+    help="Grid creation method. Only used if a region of interest has been mapped in the location parameter.",
+)
+@click.option(
+    "--region",
+    type=str,
+    default="",
+    help="Name of the region (will create sub-folder if you store json files).",
+)
+@task_with_status_report("get-openweathermap-forecasts")
+def collect_weather_data(location, store_in_db, num_cells, method, region):
+    """
+    Collect weather forecasts from the OpenWeatherMap API.
+    This will be done for one or more locations, for which we first identify relevant weather stations.
+
+    This function can get weather data for one location or for several locations within
+    a geometrical grid (See the --location parameter).
+    """
+
+    api_key = str(current_app.config.get("OPENWEATHERMAP_API_KEY", ""))
+    if api_key == "":
+        raise Exception(
+            "[FLEXMEASURES-OWM] Setting OPENWEATHERMAP_API_KEY not available."
+        )
+    locations = get_locations(location, num_cells, method)
+
+    # Save the results
+    if store_in_db:
+        save_forecasts_in_db(api_key, locations)
+    else:
+        save_forecasts_as_json(
+            api_key, locations, data_path=make_file_path(current_app, region)
+        )

flexmeasures_openweathermap/cli/schemas/weather_sensor.py

@@ -0,0 +1,57 @@
+from marshmallow import (
+    Schema,
+    validates,
+    validates_schema,
+    ValidationError,
+    fields,
+    validate,
+)
+
+import pytz
+from flexmeasures.data.models.time_series import Sensor
+
+from ...utils.modeling import get_or_create_weather_station
+from ...utils.owm import get_supported_sensor_spec, get_supported_sensors_str
+
+
+class WeatherSensorSchema(Schema):
+    """
+    Schema for the weather sensor registration.
+    Based on flexmeasures.Sensor, plus some attributes for the weather station asset.
+    """
+
+    name = fields.Str(required=True)
+    timezone = fields.Str()
+    latitude = fields.Float(required=True, validate=validate.Range(min=-90, max=90))
+    longitude = fields.Float(required=True, validate=validate.Range(min=-180, max=180))
+
+    @validates("name")
+    def validate_name_is_supported(self, name: str):
+        if get_supported_sensor_spec(name):
+            return
+        raise ValidationError(
+            f"Weather sensors with name '{name}' are not supported by flexmeasures-openweathermap. For now, the following is supported: [{get_supported_sensors_str()}]"
+        )
+
+    @validates_schema(skip_on_field_errors=False)
+    def validate_name_is_unique_in_weather_station(self, data, **kwargs):
+        if "name" not in data or "latitude" not in data or "longitude" not in data:
+            return  # That's a different validation problem
+        weather_station = get_or_create_weather_station(
+            data["latitude"], data["longitude"]
+        )
+        sensor = Sensor.query.filter(
+            Sensor.name == data["name"].lower(),
+            Sensor.generic_asset == weather_station,
+        ).one_or_none()
+        if sensor:
+            raise ValidationError(
+                f"A '{data['name']}' - weather sensor already exists at this weather station (the station's ID is {weather_station.id}))."
+            )
+
+    @validates("timezone")
+    def validate_timezone(self, timezone: str):
+        try:
+            pytz.timezone(timezone)
+        except pytz.UnknownTimeZoneError:
+            raise ValidationError(f"Timezone {timezone} is unknown!")