Working first version

.gitignore

@@ -79,3 +79,4 @@ dmypy.json
 .ipynb_checkpoints/
 notebooks/.ipynb_checkpoints/
 
+flexmeasures.log

README.md

@@ -1,16 +1,42 @@
-# flexmeasures-openweathermap - a plugin for FlexMeasures
-
+# flexmeasures-openweathermap - a plugin for FlexMeasures to integrate OpenWeatherMap data
 
 ## Usage
 
+To register a new weather sensor:
+
+`flexmeasures owm register-weather-sensor --name wind-speed --latitude 30 --longitude 40`
+
+Currently supported: wind_speed, temperature & radiation.
+
+Notes about weather sensor setup: 
+
+- Weather sensors are public. They are accessible by all accounts on a FlexMeasures server. TODO: maybe limit this to a list of account roles.
+- The resolution is one hour. OWM also supports minutely data within the upcoming hour(s), but that is not supported here.
+
+To collect weather forecasts:
+
+`flexmeasures owm get-weather-forecasts --location 30,40`
+
+This saves forecasts for your registered sensors in the database.
+
+Use the `--help`` option for more options, e.g. for specifying two locations and ask for a few weather stations to cover the region between them.
+
+An alternative usage is to save raw results in JSON files (for later processing), like this:
+
+`flexmeasures owm get-weather-forecasts --location 30,40 --store-as-json-files --region somewhere`
+
+This saves the complete response from OpenWeatherMap in a local folder (i.e. no sensor registration needed, this is a direct way to use OWM, without FlexMeasures integration). `region` will become a subfolder.
+
+Finally, note that currently 1000 free calls per day can be made to the OpenWeatherMap API,
+so you can make a call every 15 minutes for up to 10 assets or every hour for up to 40 assets (or get a paid account).
+
 
 ## Installation
 
-1. Add "/path/to/flexmeasures-openweathermap/flexmeasures_openweathermap" to your FlexMeasures (>v0.7.0dev8) config file,
-   using the FLEXMEASURES_PLUGINS setting (a list).
-   Alternatively, if you installed this plugin as a package, then "flexmeasures_openweathermap" suffices.
+Add "/path/to/flexmeasures-openweathermap/flexmeasures_openweathermap" to your FlexMeasures (>v0.7.0dev8) config file,
+using the FLEXMEASURES_PLUGINS setting (a list).
 
-2.  
+Alternatively, if you installed this plugin as a package (e.g. via `python setup.py install`, `pip install -e` or `pip install flexmeasures_openweathermap` after this project is on Pypi), then "flexmeasures_openweathermap" suffices.
 
 
 ## Development

flexmeasures_openweathermap/__init__.py

@@ -12,7 +12,7 @@
 
 from flask import Blueprint
 
-from .utils import ensure_bp_routes_are_loaded_fresh
+from .utils.blueprinting import ensure_bp_routes_are_loaded_fresh
 
 # Overwriting version (if possible) from the package metadata
 # ― if this plugin has been installed as a package.
@@ -24,13 +24,41 @@
     # package is not installed
     pass
 
+
+DEFAULT_FILE_PATH_LOCATION = "weather-forecasts"
+DEFAULT_DATA_SOURCE_NAME = "OpenWeatherMap"
+WEATHER_STATION_TYPE_NAME = "Weather station"
+DEFAULT_WEATHER_STATION_NAME = "Weather station (created by FM-OWM)"
+
+__version__ = "0.1"
+__settings__ = {
+    "OPENWEATHERMAP_API_KEY": dict(
+        description="You can generate this token after you made an account at OpenWeatherMap.",
+        level="error",
+    ),
+    "OPENWEATHERMAP_DATA_SOURCE_NAME": dict(
+        description="Name of the data source for OWM data.",
+        default="OpenWeatherMap",
+        level="debug",
+    ),
+    "FILE_PATH_LOCATION": dict(
+        description="Location of JSON files (if you store weather data in this form). Absolute path.",
+        level="debug",
+    ),
+    "DATA_SOURCE_NAME": dict(
+        description=f"Name of the data source, defaults to '{DEFAULT_DATA_SOURCE_NAME}'",
+        level="debug",
+    ),
+    "WEATHER_STATION_NAME": dict(
+        description=f"Name of the weather station asset, defaults to '{DEFAULT_WEATHER_STATION_NAME}'",
+        level="debug",
+    ),
+}
+
 # CLI
-flexmeasures_openweathermap_cli_bp: Blueprint = Blueprint(
-    "flexmeasures-openweathermap CLI",
-    __name__,
-    cli_group="flexmeasures-openweathermap"
+flexmeasures_openweathermap_bp: Blueprint = Blueprint(
+    "flexmeasures-openweathermap CLI", __name__, cli_group="owm"
 )
-flexmeasures_openweathermap_cli_bp.cli.help = "flexmeasures-openweathermap CLI commands"
+flexmeasures_openweathermap_bp.cli.help = "flexmeasures-openweathermap CLI commands"
 ensure_bp_routes_are_loaded_fresh("cli.commands")
 from flexmeasures_openweathermap.cli import commands  # noqa: E402,F401
-

flexmeasures_openweathermap/cli/commands.py

@@ -1,17 +1,146 @@
+from datetime import timedelta
 from flask import current_app
+
 from flask.cli import with_appcontext
 import click
+from flexmeasures.data.models.time_series import Sensor
+
 from flexmeasures.data.transactional import task_with_status_report
+from flexmeasures.data.config import db
+
+from .. import flexmeasures_openweathermap_bp
+from .schemas.weather_sensor import WeatherSensorSchema
+from ..utils.modeling import get_weather_station, get_data_source
+from ..utils.locating import get_locations
+from ..utils.filing import make_file_path
+from ..utils.owm import (
+    save_forecasts_in_db,
+    save_forecasts_as_json,
+    owm_to_sensor_map,
+    get_supported_sensor_spec,
+)
+
 
-from .. import flexmeasures_openweathermap_cli_bp
+"""
+TODO: allow to also pass an asset ID for the weather station (instead of location) to both commands?
+"""
 
+supported_sensors_list = ", ".join([str(o["name"]) for o in owm_to_sensor_map.values()])
 
-@flexmeasures_openweathermap_cli_bp.cli.command("hello-world")
+
+@flexmeasures_openweathermap_bp.cli.command("register-weather-sensor")
+@with_appcontext
 @click.option(
     "--name",
+    required=True,
+    help=f"Name of the sensor. Has to be from the supported list ({supported_sensors_list})",
+)
+# @click.option("--generic-asset-id", required=False, help="The asset id of the weather station (you can also give its location).")
+@click.option(
+    "--latitude",
+    required=True,
+    type=float,
+    help="Latitude of where you want to measure.",
 )
+@click.option(
+    "--longitude",
+    required=True,
+    type=float,
+    help="Longitude of where you want to measure.",
+)
+@click.option(
+    "--timezone",
+    default="UTC",
+    help="The timezone of the sensor data as string, e.g. 'UTC' (default) or 'Europe/Amsterdam'",
+)
+def add_weather_sensor(**args):
+    """
+    Add a weather sensor.
+    This will first create a weather station asset if none exists at the location yet.
+
+    """
+    errors = WeatherSensorSchema().validate(args)
+    if errors:
+        click.echo(
+            f"Please correct the following errors:\n{errors}.\n Use the --help flag to learn more."
+        )
+        raise click.Abort
+
+    weather_station = get_weather_station(args["latitude"], args["longitude"])
+    args["generic_asset"] = weather_station
+    del args["latitude"]
+    del args["longitude"]
+
+    args["event_resolution"] = timedelta(minutes=60)  # OWM delivers hourly data
+
+    fm_sensor_specs = get_supported_sensor_spec(args["name"])
+    args["unit"] = fm_sensor_specs["unit"]
+    sensor = Sensor(**args)
+    sensor.attributes = fm_sensor_specs["seasonality"]
+
+    db.session.add(sensor)
+    db.session.commit()
+    click.echo(
+        f"Successfully created weather sensor with ID {sensor.id}, at weather station with ID {weather_station.id}"
+    )
+    click.echo(
+        f"You can access this sensor at its entity address {sensor.entity_address}"
+    )
+
+
+@flexmeasures_openweathermap_bp.cli.command("get-weather-forecasts")
 @with_appcontext
-@task_with_status_report
-def hello_world(name: str):
-    print(f"Hello, {name}!")
-    current_app.logger.info(f"'Hello, {name}!' printed")
+@click.option(
+    "--location",
+    type=str,
+    required=True,
+    help='Measurement location(s). "latitude,longitude" or "top-left-latitude,top-left-longitude:'
+    'bottom-right-latitude,bottom-right-longitude." The first format defines one location to measure.'
+    " The second format defines a region of interest with several (>=4) locations"
+    ' (see also the "method" and "num_cells" parameters for details on how to use this feature).',
+)
+@click.option(
+    "--store-in-db/--store-as-json-files",
+    default=True,
+    help="Store forecasts in the database, or simply save as json files (defaults to database).",
+)
+@click.option(
+    "--num_cells",
+    type=int,
+    default=1,
+    help="Number of cells on the grid. Only used if a region of interest has been mapped in the location parameter. Defaults to 1.",
+)
+@click.option(
+    "--method",
+    default="hex",
+    type=click.Choice(["hex", "square"]),
+    help="Grid creation method. Only used if a region of interest has been mapped in the location parameter.",
+)
+@click.option(
+    "--region",
+    type=str,
+    default="",
+    help="Name of the region (will create sub-folder if you store json files).",
+)
+@task_with_status_report("get-openweathermap-forecasts")
+def collect_weather_data(location, store_in_db, num_cells, method, region):
+    """
+    Collect weather forecasts from the OpenWeatherMap API.
+    This will be done for one or more locations, for which we first identify relevant weather stations.
+
+    This function can get weather data for one location or for several locations within
+    a geometrical grid (See the --location parameter).
+    """
+
+    api_key = str(current_app.config.get("OPENWEATHERMAP_API_KEY"))
+    if api_key is None:
+        raise Exception("Setting OPENWEATHERMAP_API_KEY not available.")
+    locations = get_locations(location, num_cells, method)
+
+    # Save the results
+    if store_in_db:
+        save_forecasts_in_db(api_key, locations, data_source=get_data_source())
+    else:
+        save_forecasts_as_json(
+            api_key, locations, data_path=make_file_path(current_app, region)
+        )

flexmeasures_openweathermap/cli/schemas/weather_sensor.py

@@ -0,0 +1,58 @@
+from marshmallow import (
+    Schema,
+    validates,
+    validates_schema,
+    ValidationError,
+    fields,
+    validate,
+)
+
+import pytz
+from flexmeasures.data.models.time_series import Sensor
+from flexmeasures.data import db
+
+from ...utils.modeling import get_weather_station
+from ...utils.owm import get_supported_sensor_spec, get_supported_sensors_str
+
+
+class WeatherSensorSchema(Schema):
+    """
+    Schema for the weather sensor registration.
+    Based on flexmeasures.Sensor, plus some attributes for the weather station asset.
+    """
+
+    name = fields.Str(required=True)
+    timezone = fields.Str()
+    latitude = fields.Float(required=True, validate=validate.Range(min=-90, max=90))
+    longitude = fields.Float(required=True, validate=validate.Range(min=-180, max=180))
+
+    @validates("name")
+    def validate_name_is_supported(self, name: str):
+        if get_supported_sensor_spec(name):
+            return
+        raise ValidationError(
+            f"Weather sensors with name '{name}' are not supported by flexmeasures-openweathermap. For now, the following is supported: [{get_supported_sensors_str()}]"
+        )
+
+    @validates_schema(skip_on_field_errors=False)
+    def validate_name_is_unique_in_weather_station(self, data, **kwargs):
+        if "name" not in data or "latitude" not in data or "longitude" not in data:
+            return  # That's a different validation problem
+        weather_station = get_weather_station(data["latitude"], data["longitude"])
+        if weather_station.id is None:
+            db.session.flush()
+        sensor = Sensor.query.filter(
+            Sensor.name == data["name"].lower(),
+            Sensor.generic_asset_id == weather_station.id,
+        ).one_or_none()
+        if sensor:
+            raise ValidationError(
+                f"A '{data['name']}' - weather sensor already exists at this weather station (with ID {weather_station.id}))."
+            )
+
+    @validates("timezone")
+    def validate_timezone(self, timezone: str):
+        try:
+            pytz.timezone(timezone)
+        except pytz.UnknownTimeZoneError:
+            raise ValidationError(f"Timezone {timezone} is unknown!")

flexmeasures_openweathermap/cli/tests/conftest.py

@@ -0,0 +1,4 @@
+from flexmeasures.conftest import (  # noqa: F401
+    add_weather_sensors_fresh_db,
+    setup_generic_asset_types_fresh_db,
+)

flexmeasures_openweathermap/cli/tests/test_get_forecasts.py

@@ -0,0 +1,53 @@
+from datetime import datetime, timedelta
+
+from flexmeasures.data.models.time_series import TimedBelief
+from flexmeasures.utils.time_utils import as_server_time, get_timezone
+
+from ..commands import collect_weather_data
+from ...utils import owm
+
+
+"""
+Useful resource: https://flask.palletsprojects.com/en/2.0.x/testing/#testing-cli-commands
+"""
+
+sensor_params = {"name": "wind_speed", "latitude": 30, "longitude": 40}
+
+
+def test_get_weather_forecasts_to_db(
+    app, fresh_db, monkeypatch, add_weather_sensors_fresh_db
+):
+    """ Test if we can process forecast and save them to the database."""
+    fresh_db.session.flush()
+    wind_sensor_id = add_weather_sensors_fresh_db["wind"].id
+
+    def mock_owm_response(api_key, location):
+        mock_date = datetime.now()
+        mock_date_tz_aware = as_server_time(
+            datetime.fromtimestamp(mock_date.timestamp(), tz=get_timezone())
+        ).replace(second=0, microsecond=0)
+        return mock_date_tz_aware, [
+            {"dt": mock_date.timestamp(), "temp": 40, "wind_speed": 100},
+            {
+                "dt": (mock_date + timedelta(hours=1)).timestamp(),
+                "temp": 42,
+                "wind_speed": 90,
+            },
+        ]
+
+    monkeypatch.setenv("OPENWEATHERMAP_API_KEY", "dummy")
+    monkeypatch.setattr(owm, "call_openweatherapi", mock_owm_response)
+
+    runner = app.test_cli_runner()
+    result = runner.invoke(collect_weather_data, ["--location", "33.484,126"])
+    print(result.output)
+    assert "Reported task get-openweathermap-forecasts status as True" in result.output
+
+    beliefs = (
+        fresh_db.session.query(TimedBelief)
+        .filter(TimedBelief.sensor_id == wind_sensor_id)
+        .all()
+    )
+    assert len(beliefs) == 2
+    for wind_speed in (100, 90):
+        assert wind_speed in [belief.event_value for belief in beliefs]