diff --git a/.github/workflows/lint-and-test.yml b/.github/workflows/lint-and-test.yml index 98eed3d..d150312 100644 --- a/.github/workflows/lint-and-test.yml +++ b/.github/workflows/lint-and-test.yml @@ -17,13 +17,19 @@ jobs: test: needs: check runs-on: ubuntu-latest - #strategy: # TODO: make work with pre-commit - # matrix: - # python-version: [ '3.8', '3.6' ] - name: "Test (on Python3.8)" + strategy: + fail-fast: false + matrix: + py_version: [ '3.8', '3.9' ] + name: "Test (on Python ${{ matrix.py_version }})" steps: - uses: actions/setup-python@v2 with: - python-version: 3.8 - - uses: actions/checkout@v2 + python-version: ${{ matrix.py_version }} + - name: Check out src from Git + uses: actions/checkout@v2 + - name: Get history and tags for SCM versioning to work + run: | + git fetch --prune --unshallow + git fetch --depth=1 origin +refs/tags/*:refs/tags/* - run: make test diff --git a/.gitignore b/.gitignore index 4642d71..3a74248 100644 --- a/.gitignore +++ b/.gitignore @@ -79,3 +79,4 @@ dmypy.json .ipynb_checkpoints/ notebooks/.ipynb_checkpoints/ +flexmeasures.log diff --git a/Makefile b/Makefile index 7c3d377..297ccd8 100644 --- a/Makefile +++ b/Makefile @@ -33,8 +33,8 @@ install-pip-tools: freeze-deps: make install-pip-tools pip-compile -o requirements/app.txt requirements/app.in - pip-compile -o requirements/dev.txt requirements/dev.in pip-compile -o requirements/test.txt requirements/test.in + pip-compile -o requirements/dev.txt requirements/dev.in upgrade-deps: make install-pip-tools diff --git a/README.md b/README.md index ef82bbd..b714be5 100644 --- a/README.md +++ b/README.md @@ -1,16 +1,47 @@ -# flexmeasures-openweathermap - a plugin for FlexMeasures - +# flexmeasures-openweathermap - a plugin for FlexMeasures to integrate OpenWeatherMap data ## Usage +To register a new weather sensor: + +`flexmeasures owm register-weather-sensor --name "wind speed" --latitude 30 --longitude 40` + +Currently supported: wind speed, temperature & irradiance. + +Notes about weather sensor setup: + +- Weather sensors are public. They are accessible by all accounts on a FlexMeasures server. TODO: maybe limit this to a list of account roles. +- The resolution is one hour. OWM also supports minutely data within the upcoming hour(s), but that is not supported here. + +To collect weather forecasts: + +`flexmeasures owm get-weather-forecasts --location 30,40` + +This saves forecasts for your registered sensors in the database. + +Use the `--help`` option for more options, e.g. for specifying two locations and requesting that a number of weather stations cover the bounding box between them (where the locations represent top left and bottom right). + +An alternative usage is to save raw results in JSON files (for later processing), like this: + +`flexmeasures owm get-weather-forecasts --location 30,40 --store-as-json-files --region somewhere` + +This saves the complete response from OpenWeatherMap in a local folder (i.e. no sensor registration needed, this is a direct way to use OWM, without FlexMeasures integration). `region` will become a subfolder. + +Finally, note that currently 1000 free calls per day can be made to the OpenWeatherMap API, +so you can make a call every 15 minutes for up to 10 locations or every hour for up to 40 locations (or get a paid account). + ## Installation -1. Add "/path/to/flexmeasures-openweathermap/flexmeasures_openweathermap" to your FlexMeasures (>v0.7.0dev8) config file, - using the FLEXMEASURES_PLUGINS setting (a list). - Alternatively, if you installed this plugin as a package, then "flexmeasures_openweathermap" suffices. +To install locally, run + + make install + +To add as plugin to an existing FlexMeasures system, add "/path/to/flexmeasures-openweathermap/flexmeasures_openweathermap" to your FlexMeasures (>v0.7.0dev8) config file, +using the FLEXMEASURES_PLUGINS setting (a list). + +Alternatively, if you installed this plugin as a package (e.g. via `python setup.py install`, `pip install -e` or `pip install flexmeasures_openweathermap` after this project is on Pypi), then "flexmeasures_openweathermap" suffices. -2. ## Development diff --git a/flexmeasures_openweathermap/__init__.py b/flexmeasures_openweathermap/__init__.py index b2c07f4..e62a566 100644 --- a/flexmeasures_openweathermap/__init__.py +++ b/flexmeasures_openweathermap/__init__.py @@ -12,7 +12,7 @@ from flask import Blueprint -from .utils import ensure_bp_routes_are_loaded_fresh +from .utils.blueprinting import ensure_bp_routes_are_loaded_fresh # Overwriting version (if possible) from the package metadata # ― if this plugin has been installed as a package. @@ -24,13 +24,36 @@ # package is not installed pass + +DEFAULT_FILE_PATH_LOCATION = "weather-forecasts" +DEFAULT_DATA_SOURCE_NAME = "OpenWeatherMap" +DEFAULT_WEATHER_STATION_NAME = "weather station (created by FM-OWM)" +WEATHER_STATION_TYPE_NAME = "weather station" + +__version__ = "0.1" +__settings__ = { + "OPENWEATHERMAP_API_KEY": dict( + description="You can generate this token after you made an account at OpenWeatherMap.", + level="error", + ), + "OPENWEATHERMAP_FILE_PATH_LOCATION": dict( + description="Location of JSON files (if you store weather data in this form). Absolute path.", + level="debug", + ), + "OPENWEATHERMAP_DATA_SOURCE_NAME": dict( + description=f"Name of the data source for OWM data, defaults to '{DEFAULT_DATA_SOURCE_NAME}'", + level="debug", + ), + "WEATHER_STATION_NAME": dict( + description=f"Name of the weather station asset, defaults to '{DEFAULT_WEATHER_STATION_NAME}'", + level="debug", + ), +} + # CLI -flexmeasures_openweathermap_cli_bp: Blueprint = Blueprint( - "flexmeasures-openweathermap CLI", - __name__, - cli_group="flexmeasures-openweathermap" +flexmeasures_openweathermap_bp: Blueprint = Blueprint( + "flexmeasures-openweathermap CLI", __name__, cli_group="owm" ) -flexmeasures_openweathermap_cli_bp.cli.help = "flexmeasures-openweathermap CLI commands" +flexmeasures_openweathermap_bp.cli.help = "flexmeasures-openweathermap CLI commands" ensure_bp_routes_are_loaded_fresh("cli.commands") from flexmeasures_openweathermap.cli import commands # noqa: E402,F401 - diff --git a/flexmeasures_openweathermap/cli/commands.py b/flexmeasures_openweathermap/cli/commands.py index 0e14863..9b824eb 100644 --- a/flexmeasures_openweathermap/cli/commands.py +++ b/flexmeasures_openweathermap/cli/commands.py @@ -1,17 +1,146 @@ from flask import current_app + from flask.cli import with_appcontext import click +from flexmeasures.data.models.time_series import Sensor + from flexmeasures.data.transactional import task_with_status_report +from flexmeasures.data.config import db + +from .. import flexmeasures_openweathermap_bp +from .schemas.weather_sensor import WeatherSensorSchema +from ..utils.modeling import ( + get_or_create_weather_station, +) +from ..utils.locating import get_locations +from ..utils.filing import make_file_path +from ..utils.owm import ( + save_forecasts_in_db, + save_forecasts_as_json, + get_supported_sensor_spec, +) +from ..sensor_specs import owm_to_sensor_map -from .. import flexmeasures_openweathermap_cli_bp +""" +TODO: allow to also pass an asset ID or name for the weather station (instead of location) to both commands? + See https://github.com/SeitaBV/flexmeasures-openweathermap/issues/2 +""" -@flexmeasures_openweathermap_cli_bp.cli.command("hello-world") +supported_sensors_list = ", ".join([str(o["name"]) for o in owm_to_sensor_map.values()]) + + +@flexmeasures_openweathermap_bp.cli.command("register-weather-sensor") +@with_appcontext @click.option( "--name", + required=True, + help=f"Name of the sensor. Has to be from the supported list ({supported_sensors_list})", ) +# @click.option("--generic-asset-id", required=False, help="The asset id of the weather station (you can also give its location).") +@click.option( + "--latitude", + required=True, + type=float, + help="Latitude of where you want to measure.", +) +@click.option( + "--longitude", + required=True, + type=float, + help="Longitude of where you want to measure.", +) +@click.option( + "--timezone", + default="UTC", + help="The timezone of the sensor data as string, e.g. 'UTC' (default) or 'Europe/Amsterdam'", +) +def add_weather_sensor(**args): + """ + Add a weather sensor. + This will first create a weather station asset if none exists at the location yet. + + """ + errors = WeatherSensorSchema().validate(args) + if errors: + click.echo( + f"[FLEXMEASURES-OWM] Please correct the following errors:\n{errors}.\n Use the --help flag to learn more." + ) + raise click.Abort + + weather_station = get_or_create_weather_station(args["latitude"], args["longitude"]) + + fm_sensor_specs = get_supported_sensor_spec(args["name"]) + fm_sensor_specs["generic_asset"] = weather_station + fm_sensor_specs["timezone"] = args["timezone"] + sensor = Sensor(**fm_sensor_specs) + sensor.attributes = fm_sensor_specs["attributes"] + + db.session.add(sensor) + db.session.commit() + click.echo( + f"[FLEXMEASURES-OWM] Successfully created weather sensor with ID {sensor.id}, at weather station with ID {weather_station.id}" + ) + click.echo( + f"[FLEXMEASURES-OWM] You can access this sensor at its entity address {sensor.entity_address}" + ) + + +@flexmeasures_openweathermap_bp.cli.command("get-weather-forecasts") @with_appcontext -@task_with_status_report -def hello_world(name: str): - print(f"Hello, {name}!") - current_app.logger.info(f"'Hello, {name}!' printed") +@click.option( + "--location", + type=str, + required=True, + help='Measurement location(s). "latitude,longitude" or "top-left-latitude,top-left-longitude:' + 'bottom-right-latitude,bottom-right-longitude." The first format defines one location to measure.' + " The second format defines a region of interest with several (>=4) locations" + ' (see also the "method" and "num_cells" parameters for details on how to use this feature).', +) +@click.option( + "--store-in-db/--store-as-json-files", + default=True, + help="Store forecasts in the database, or simply save as json files (defaults to database).", +) +@click.option( + "--num_cells", + type=int, + default=1, + help="Number of cells on the grid. Only used if a region of interest has been mapped in the location parameter. Defaults to 1.", +) +@click.option( + "--method", + default="hex", + type=click.Choice(["hex", "square"]), + help="Grid creation method. Only used if a region of interest has been mapped in the location parameter.", +) +@click.option( + "--region", + type=str, + default="", + help="Name of the region (will create sub-folder if you store json files).", +) +@task_with_status_report("get-openweathermap-forecasts") +def collect_weather_data(location, store_in_db, num_cells, method, region): + """ + Collect weather forecasts from the OpenWeatherMap API. + This will be done for one or more locations, for which we first identify relevant weather stations. + + This function can get weather data for one location or for several locations within + a geometrical grid (See the --location parameter). + """ + + api_key = str(current_app.config.get("OPENWEATHERMAP_API_KEY", "")) + if api_key == "": + raise Exception( + "[FLEXMEASURES-OWM] Setting OPENWEATHERMAP_API_KEY not available." + ) + locations = get_locations(location, num_cells, method) + + # Save the results + if store_in_db: + save_forecasts_in_db(api_key, locations) + else: + save_forecasts_as_json( + api_key, locations, data_path=make_file_path(current_app, region) + ) diff --git a/flexmeasures_openweathermap/cli/schemas/__init__.py b/flexmeasures_openweathermap/cli/schemas/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/flexmeasures_openweathermap/cli/schemas/weather_sensor.py b/flexmeasures_openweathermap/cli/schemas/weather_sensor.py new file mode 100644 index 0000000..de2b7e8 --- /dev/null +++ b/flexmeasures_openweathermap/cli/schemas/weather_sensor.py @@ -0,0 +1,57 @@ +from marshmallow import ( + Schema, + validates, + validates_schema, + ValidationError, + fields, + validate, +) + +import pytz +from flexmeasures.data.models.time_series import Sensor + +from ...utils.modeling import get_or_create_weather_station +from ...utils.owm import get_supported_sensor_spec, get_supported_sensors_str + + +class WeatherSensorSchema(Schema): + """ + Schema for the weather sensor registration. + Based on flexmeasures.Sensor, plus some attributes for the weather station asset. + """ + + name = fields.Str(required=True) + timezone = fields.Str() + latitude = fields.Float(required=True, validate=validate.Range(min=-90, max=90)) + longitude = fields.Float(required=True, validate=validate.Range(min=-180, max=180)) + + @validates("name") + def validate_name_is_supported(self, name: str): + if get_supported_sensor_spec(name): + return + raise ValidationError( + f"Weather sensors with name '{name}' are not supported by flexmeasures-openweathermap. For now, the following is supported: [{get_supported_sensors_str()}]" + ) + + @validates_schema(skip_on_field_errors=False) + def validate_name_is_unique_in_weather_station(self, data, **kwargs): + if "name" not in data or "latitude" not in data or "longitude" not in data: + return # That's a different validation problem + weather_station = get_or_create_weather_station( + data["latitude"], data["longitude"] + ) + sensor = Sensor.query.filter( + Sensor.name == data["name"].lower(), + Sensor.generic_asset == weather_station, + ).one_or_none() + if sensor: + raise ValidationError( + f"A '{data['name']}' - weather sensor already exists at this weather station (the station's ID is {weather_station.id}))." + ) + + @validates("timezone") + def validate_timezone(self, timezone: str): + try: + pytz.timezone(timezone) + except pytz.UnknownTimeZoneError: + raise ValidationError(f"Timezone {timezone} is unknown!") diff --git a/flexmeasures_openweathermap/cli/tests/conftest.py b/flexmeasures_openweathermap/cli/tests/conftest.py new file mode 100644 index 0000000..e69de29 diff --git a/flexmeasures_openweathermap/cli/tests/test_cli.py b/flexmeasures_openweathermap/cli/tests/test_cli.py deleted file mode 100644 index 1de1fa0..0000000 --- a/flexmeasures_openweathermap/cli/tests/test_cli.py +++ /dev/null @@ -1,11 +0,0 @@ -from ..commands import hello_world - -""" -Useful resource: https://flask.palletsprojects.com/en/2.0.x/testing/#testing-cli-commands -""" - - -def test_hello_world(app): - runner = app.test_cli_runner() - result = runner.invoke(hello_world, ["--name", "George"]) - assert "Hello, George!" in result.output diff --git a/flexmeasures_openweathermap/cli/tests/test_get_forecasts.py b/flexmeasures_openweathermap/cli/tests/test_get_forecasts.py new file mode 100644 index 0000000..337db9a --- /dev/null +++ b/flexmeasures_openweathermap/cli/tests/test_get_forecasts.py @@ -0,0 +1,63 @@ +from flexmeasures.data.models.time_series import TimedBelief + +from ..commands import collect_weather_data +from ...utils import owm +from .utils import mock_owm_response + + +""" +Useful resource: https://flask.palletsprojects.com/en/2.0.x/testing/#testing-cli-commands +""" + + +def test_get_weather_forecasts_to_db( + app, fresh_db, monkeypatch, add_weather_sensors_fresh_db +): + """ + Test if we can process forecast and save them to the database. + """ + wind_sensor = add_weather_sensors_fresh_db["wind"] + fresh_db.session.flush() + wind_sensor_id = wind_sensor.id + weather_station = wind_sensor.generic_asset + + monkeypatch.setitem(app.config, "OPENWEATHERMAP_API_KEY", "dummy") + monkeypatch.setattr(owm, "call_openweatherapi", mock_owm_response) + + runner = app.test_cli_runner() + result = runner.invoke( + collect_weather_data, + ["--location", f"{weather_station.latitude},{weather_station.longitude}"], + ) + print(result.output) + assert "Reported task get-openweathermap-forecasts status as True" in result.output + + beliefs = ( + fresh_db.session.query(TimedBelief) + .filter(TimedBelief.sensor_id == wind_sensor_id) + .all() + ) + assert len(beliefs) == 2 + for wind_speed in (100, 90): + assert wind_speed in [belief.event_value for belief in beliefs] + + +def test_get_weather_forecasts_no_close_sensors( + app, db, monkeypatch, add_weather_sensors_fresh_db +): + """ + Looking for a location too far away from existing weather stations means we fail. + """ + weather_station = add_weather_sensors_fresh_db["wind"].generic_asset + + monkeypatch.setitem(app.config, "OPENWEATHERMAP_API_KEY", "dummy") + monkeypatch.setattr(owm, "call_openweatherapi", mock_owm_response) + + runner = app.test_cli_runner() + result = runner.invoke( + collect_weather_data, + ["--location", f"{weather_station.latitude-5},{weather_station.longitude}"], + ) + print(result.output) + assert "Reported task get-openweathermap-forecasts status as False" in result.output + assert "No sufficiently close weather sensor found" in result.output diff --git a/flexmeasures_openweathermap/cli/tests/test_register.py b/flexmeasures_openweathermap/cli/tests/test_register.py new file mode 100644 index 0000000..cebcf7e --- /dev/null +++ b/flexmeasures_openweathermap/cli/tests/test_register.py @@ -0,0 +1,47 @@ +import pytest +from flexmeasures.data.models.time_series import Sensor + +from ..commands import add_weather_sensor +from .utils import cli_params_from_dict + + +""" +Useful resource: https://flask.palletsprojects.com/en/2.0.x/testing/#testing-cli-commands +""" + +sensor_params = {"name": "wind speed", "latitude": 30, "longitude": 40} + + +@pytest.mark.parametrize( + "invalid_param, invalid_value, expected_msg", + [ + ("name", "windd-speed", "not supported by flexmeasures-openweathermap"), + ("latitude", 93, "less than or equal to 90"), + ("timezone", "Erope/Amsterdam", "is unknown"), + ], +) +def test_register_weather_sensor_invalid_data( + app, db, invalid_param, invalid_value, expected_msg +): + test_sensor_params = sensor_params.copy() + test_sensor_params[invalid_param] = invalid_value + runner = app.test_cli_runner() + result = runner.invoke(add_weather_sensor, cli_params_from_dict(test_sensor_params)) + assert "Aborted" in result.output + assert expected_msg in result.output + + +def test_register_weather_sensor(app, fresh_db): + runner = app.test_cli_runner() + result = runner.invoke(add_weather_sensor, cli_params_from_dict(sensor_params)) + assert "Successfully created weather sensor with ID" in result.output + sensor = Sensor.query.filter(Sensor.name == sensor_params["name"]).one_or_none() + assert sensor is not None + + +def test_register_weather_sensor_twice(app, fresh_db): + runner = app.test_cli_runner() + result = runner.invoke(add_weather_sensor, cli_params_from_dict(sensor_params)) + assert "Successfully created weather sensor with ID" in result.output + result = runner.invoke(add_weather_sensor, cli_params_from_dict(sensor_params)) + assert "already exists" in result.output diff --git a/flexmeasures_openweathermap/cli/tests/utils.py b/flexmeasures_openweathermap/cli/tests/utils.py new file mode 100644 index 0000000..ea95339 --- /dev/null +++ b/flexmeasures_openweathermap/cli/tests/utils.py @@ -0,0 +1,27 @@ +from typing import List +from datetime import datetime, timedelta + +from flexmeasures.utils.time_utils import as_server_time, get_timezone + + +def cli_params_from_dict(d) -> List[str]: + cli_params = [] + for k, v in d.items(): + cli_params.append(f"--{k}") + cli_params.append(v) + return cli_params + + +def mock_owm_response(api_key, location): + mock_date = datetime.now() + mock_date_tz_aware = as_server_time( + datetime.fromtimestamp(mock_date.timestamp(), tz=get_timezone()) + ).replace(second=0, microsecond=0) + return mock_date_tz_aware, [ + {"dt": mock_date.timestamp(), "temp": 40, "wind_speed": 100}, + { + "dt": (mock_date + timedelta(hours=1)).timestamp(), + "temp": 42, + "wind_speed": 90, + }, + ] diff --git a/flexmeasures_openweathermap/conftest.py b/flexmeasures_openweathermap/conftest.py index e507c0d..cb323da 100644 --- a/flexmeasures_openweathermap/conftest.py +++ b/flexmeasures_openweathermap/conftest.py @@ -1,6 +1,14 @@ -import pytest +from typing import Dict +from datetime import timedelta +import pytest +from flask_sqlalchemy import SQLAlchemy from flexmeasures.app import create as create_flexmeasures_app +from flexmeasures.conftest import db, fresh_db # noqa: F401 +from flexmeasures.data.models.time_series import Sensor +from flexmeasures.data.models.generic_assets import GenericAsset, GenericAssetType + +from flexmeasures_openweathermap import WEATHER_STATION_TYPE_NAME @pytest.fixture(scope="session") @@ -8,7 +16,9 @@ def app(): print("APP FIXTURE") # Adding this plugin, making sure the name is known (as last part of plugin path) - test_app = create_flexmeasures_app(env="testing", plugins=["../flexmeasures_openweathermap"]) + test_app = create_flexmeasures_app( + env="testing", plugins=["../flexmeasures_openweathermap"] + ) # Establish an application context before running the tests. ctx = test_app.app_context() @@ -19,3 +29,44 @@ def app(): ctx.pop() print("DONE WITH APP FIXTURE") + + +@pytest.fixture(scope="module") +def add_weather_sensors(db) -> Dict[str, Sensor]: # noqa: F811 + return create_weather_sensors(db) + + +@pytest.fixture(scope="function") +def add_weather_sensors_fresh_db(fresh_db) -> Dict[str, Sensor]: # noqa: F811 + return create_weather_sensors(fresh_db) + + +def create_weather_sensors(db: SQLAlchemy): # noqa: F811 + """Add a weather station asset with two weather sensors.""" + weather_station_type = GenericAssetType(name=WEATHER_STATION_TYPE_NAME) + db.session.add(weather_station_type) + + weather_station = GenericAsset( + name="Test weather station", + generic_asset_type=weather_station_type, + latitude=33.4843866, + longitude=126, + ) + db.session.add(weather_station) + + wind_sensor = Sensor( + name="wind speed", + generic_asset=weather_station, + event_resolution=timedelta(minutes=60), + unit="m/s", + ) + db.session.add(wind_sensor) + + temp_sensor = Sensor( + name="temperature", + generic_asset=weather_station, + event_resolution=timedelta(minutes=60), + unit="°C", + ) + db.session.add(temp_sensor) + return {"wind": wind_sensor, "temperature": temp_sensor} diff --git a/flexmeasures_openweathermap/sensor_specs.py b/flexmeasures_openweathermap/sensor_specs.py new file mode 100644 index 0000000..cb9b018 --- /dev/null +++ b/flexmeasures_openweathermap/sensor_specs.py @@ -0,0 +1,37 @@ +from datetime import timedelta + + +""" +This maps sensor specs which we can use in FlexMeasures to OWM labels. +Note: Sensor names we use in FM need to be unique per weather station. +At the moment, we only extract from OWM hourly data. +""" + + +weather_attributes = { + "daily_seasonality": True, + "weekly_seasonality": False, + "yearly_seasonality": True, +} + + +owm_to_sensor_map = dict( + temp={ + "name": "temperature", + "unit": "°C", + "event_resolution": timedelta(minutes=60), + "attributes": weather_attributes, + }, + wind_speed={ + "name": "wind speed", + "unit": "m/s", + "event_resolution": timedelta(minutes=60), + "attributes": weather_attributes, + }, + clouds={ + "name": "irradiance", + "unit": "kW/m²", + "event_resolution": timedelta(minutes=60), + "attributes": weather_attributes, + }, +) diff --git a/flexmeasures_openweathermap/utils/__init__.py b/flexmeasures_openweathermap/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/flexmeasures_openweathermap/utils.py b/flexmeasures_openweathermap/utils/blueprinting.py similarity index 92% rename from flexmeasures_openweathermap/utils.py rename to flexmeasures_openweathermap/utils/blueprinting.py index d332709..2b672c3 100644 --- a/flexmeasures_openweathermap/utils.py +++ b/flexmeasures_openweathermap/utils/blueprinting.py @@ -8,10 +8,10 @@ def ensure_bp_routes_are_loaded_fresh(module_name): It's useful for situations in which some other process has read the module before, but you need some action to happen which only happens during module import ― decorators are a good example. - + One use case is pytest, which reads all python code when it collects tests. In our case, that happens before FlexMeasures' import mechanism - has had a chance to know which blueprints a plugin has. And + has had a chance to know which blueprints a plugin has. Seemingly, the importing code (plugin's __init__) can be imported later than the imported module (containing @route decorators). Re-importing helps to get this order right when FlexMeasures reads the diff --git a/flexmeasures_openweathermap/utils/filing.py b/flexmeasures_openweathermap/utils/filing.py new file mode 100644 index 0000000..e308872 --- /dev/null +++ b/flexmeasures_openweathermap/utils/filing.py @@ -0,0 +1,25 @@ +import os + +import click +from flask import Flask, current_app + +from flexmeasures_openweathermap import DEFAULT_FILE_PATH_LOCATION + + +def make_file_path(app: Flask, region: str) -> str: + """Ensure and return path for weather data""" + file_path = current_app.config.get( + "OPENWEATHERMAP_FILE_PATH_LOCATION", DEFAULT_FILE_PATH_LOCATION + ) + data_path = os.path.join(app.root_path, file_path) + if not os.path.exists(data_path): + click.echo("[FLEXMEASURES-OWM] Creating %s ..." % data_path) + os.mkdir(data_path) + # optional: extend with subpath for region + if region is not None and region != "": + region_data_path = "%s/%s" % (data_path, region) + if not os.path.exists(region_data_path): + click.echo("[FLEXMEASURES-OWM] Creating %s ..." % region_data_path) + os.mkdir(region_data_path) + data_path = region_data_path + return data_path diff --git a/flexmeasures_openweathermap/utils/locating.py b/flexmeasures_openweathermap/utils/locating.py new file mode 100644 index 0000000..6b21533 --- /dev/null +++ b/flexmeasures_openweathermap/utils/locating.py @@ -0,0 +1,108 @@ +from typing import Tuple, List, Optional + +import click + +from flexmeasures.utils.grid_cells import LatLngGrid, get_cell_nums +from flexmeasures.data.models.time_series import Sensor +from flexmeasures.data.models.generic_assets import GenericAsset +from flexmeasures.utils import flexmeasures_inflection + +from .. import WEATHER_STATION_TYPE_NAME + + +def get_locations( + location: str, + num_cells: int, + method: str, +) -> List[Tuple[float, float]]: + """ + Get locations for getting forecasts for, by parsing the location string, which possibly opens a latitude/longitude grid with several neatly ordered locations. + """ + if ( + location.count(",") == 0 + or location.count(",") != location.count(":") + 1 + or location.count(":") == 1 + and ( + location.find(",") > location.find(":") + or location.find(",", location.find(",") + 1) < location.find(":") + ) + ): + raise Exception( + '[FLEXMEASURES-OWM] location parameter "%s" seems malformed. Please use "latitude,longitude" or ' + ' "top-left-latitude,top-left-longitude:bottom-right-latitude,bottom-right-longitude"' + % location + ) + + location_identifiers = tuple(location.split(":")) + + if len(location_identifiers) == 1: + ll = location_identifiers[0].split(",") + locations = [(float(ll[0]), float(ll[1]))] + click.echo("[FLEXMEASURES-OWM] Only one location: %s,%s." % locations[0]) + elif len(location_identifiers) == 2: + click.echo( + "[FLEXMEASURES-OWM] Making a grid of locations between top/left %s and bottom/right %s ..." + % location_identifiers + ) + top_left = tuple(float(s) for s in location_identifiers[0].split(",")) + if len(top_left) != 2: + raise Exception( + "[FLEXMEASURES-OWM] top-left parameter '%s' is invalid." + % location_identifiers[0] + ) + bottom_right = tuple(float(s) for s in location_identifiers[1].split(",")) + if len(bottom_right) != 2: + raise Exception( + "[FLEXMEASURES-OWM] bottom-right parameter '%s' is invalid." + % location_identifiers[1] + ) + + num_lat, num_lng = get_cell_nums(top_left, bottom_right, num_cells) + + locations = LatLngGrid( + top_left=top_left, + bottom_right=bottom_right, + num_cells_lat=num_lat, + num_cells_lng=num_lng, + ).get_locations(method) + else: + raise Exception( + "[FLEXMEASURES-OWM] location parameter '%s' has too many locations." + % location + ) + return locations + + +def find_weather_sensor_by_location_or_fail( + location: Tuple[float, float], + max_degree_difference_for_nearest_weather_sensor: int, + sensor_name: str, +) -> Sensor: + """ + Try to find a weather sensor of fitting type close by. + Complain if the nearest weather sensor is further away than some minimum degrees. + """ + weather_sensor: Optional[Sensor] = Sensor.find_closest( + generic_asset_type_name=WEATHER_STATION_TYPE_NAME, + sensor_name=sensor_name, + lat=location[0], + lng=location[1], + n=1, + ) + if weather_sensor is not None: + weather_station: GenericAsset = weather_sensor.generic_asset + if abs( + location[0] - weather_station.location[0] + ) > max_degree_difference_for_nearest_weather_sensor or abs( + location[1] - weather_station.location[1] + > max_degree_difference_for_nearest_weather_sensor + ): + raise Exception( + f"[FLEXMEASURES-OWM] No sufficiently close weather sensor found (within {max_degree_difference_for_nearest_weather_sensor} {flexmeasures_inflection.pluralize('degree', max_degree_difference_for_nearest_weather_sensor)} distance) for measuring {sensor_name}! We're looking for: {location}, closest available: ({weather_station.location})" + ) + else: + raise Exception( + "[FLEXMEASURES-OWM] No weather sensor set up yet for measuring %s. Try the register-weather-sensor CLI task." + % sensor_name + ) + return weather_sensor diff --git a/flexmeasures_openweathermap/utils/modeling.py b/flexmeasures_openweathermap/utils/modeling.py new file mode 100644 index 0000000..56a96c8 --- /dev/null +++ b/flexmeasures_openweathermap/utils/modeling.py @@ -0,0 +1,65 @@ +from flask import current_app +from flexmeasures.data.config import db +from flexmeasures.data.models.generic_assets import GenericAssetType, GenericAsset +from flexmeasures.data.models.data_sources import DataSource +from flexmeasures.data.queries.data_sources import get_or_create_source + +from flexmeasures_openweathermap import DEFAULT_DATA_SOURCE_NAME +from flexmeasures_openweathermap import WEATHER_STATION_TYPE_NAME +from flexmeasures_openweathermap import DEFAULT_WEATHER_STATION_NAME + + +def get_or_create_owm_data_source() -> DataSource: + """Make sure we have am OWM data source""" + return get_or_create_source( + source=current_app.config.get( + "OPENWEATHERMAP_DATA_SOURCE_NAME", DEFAULT_DATA_SOURCE_NAME + ), + source_type="forecasting script", + flush=False, + ) + + +def get_or_create_owm_data_source_for_derived_data() -> DataSource: + owm_source_name = current_app.config.get( + "OPENWEATHERMAP_DATA_SOURCE_NAME", DEFAULT_DATA_SOURCE_NAME + ) + return get_or_create_source( + source=f"FlexMeasures {owm_source_name}", + source_type="forecasting script", + flush=False, + ) + + +def get_or_create_weather_station_type() -> GenericAssetType: + """Make sure a weather station type exists""" + weather_station_type = GenericAssetType.query.filter( + GenericAssetType.name == WEATHER_STATION_TYPE_NAME, + ).one_or_none() + if weather_station_type is None: + weather_station_type = GenericAssetType( + name=WEATHER_STATION_TYPE_NAME, + description="A weather station with various sensors.", + ) + db.session.add(weather_station_type) + return weather_station_type + + +def get_or_create_weather_station(latitude: float, longitude: float) -> GenericAsset: + """Make sure a weather station exists at this location.""" + station_name = current_app.config.get( + "WEATHER_STATION_NAME", DEFAULT_WEATHER_STATION_NAME + ) + weather_station = GenericAsset.query.filter( + GenericAsset.latitude == latitude, GenericAsset.longitude == longitude + ).one_or_none() + if weather_station is None: + weather_station_type = get_or_create_weather_station_type() + weather_station = GenericAsset( + name=station_name, + generic_asset_type=weather_station_type, + latitude=latitude, + longitude=longitude, + ) + db.session.add(weather_station) + return weather_station diff --git a/flexmeasures_openweathermap/utils/owm.py b/flexmeasures_openweathermap/utils/owm.py new file mode 100644 index 0000000..66b49a4 --- /dev/null +++ b/flexmeasures_openweathermap/utils/owm.py @@ -0,0 +1,209 @@ +from typing import Tuple, List, Dict, Optional +import os +from datetime import datetime, timedelta +import json + +import click +from flask import current_app +import requests +from humanize import naturaldelta +from timely_beliefs import BeliefsDataFrame +from flexmeasures.utils.time_utils import as_server_time, get_timezone, server_now +from flexmeasures.data.models.time_series import Sensor, TimedBelief +from flexmeasures.data.utils import save_to_db + +from .locating import find_weather_sensor_by_location_or_fail +from ..sensor_specs import owm_to_sensor_map +from .modeling import ( + get_or_create_owm_data_source, + get_or_create_owm_data_source_for_derived_data, +) +from .radiating import compute_irradiance + + +def get_supported_sensor_spec(name: str) -> Optional[dict]: + """ + Find the specs from a sensor by name. + """ + for supported_sensor_spec in owm_to_sensor_map.values(): + if supported_sensor_spec["name"] == name: + return supported_sensor_spec + return None + + +def get_supported_sensors_str() -> str: + """ A string - list of supported sensors, also revealing their unit""" + return ", ".join([f"{o['name']} ({o['unit']})" for o in owm_to_sensor_map.values()]) + + +def call_openweatherapi( + api_key: str, location: Tuple[float, float] +) -> Tuple[datetime, List[Dict]]: + """ + Make a single "one-call" to the Open Weather API and return the API timestamp as well as the 48 hourly forecasts. + See https://openweathermap.org/api/one-call-api for docs. + Note that the first forecast is about the current hour. + """ + query_str = f"lat={location[0]}&lon={location[1]}&units=metric&exclude=minutely,daily,alerts&appid={api_key}" + res = requests.get(f"http://api.openweathermap.org/data/2.5/onecall?{query_str}") + assert ( + res.status_code == 200 + ), f"OpenWeatherMap returned status code {res.status_code}: {res.text}" + data = res.json() + time_of_api_call = as_server_time( + datetime.fromtimestamp(data["current"]["dt"], tz=get_timezone()) + ).replace(second=0, microsecond=0) + return time_of_api_call, data["hourly"] + + +def save_forecasts_in_db( + api_key: str, + locations: List[Tuple[float, float]], + max_degree_difference_for_nearest_weather_sensor: int = 2, +): + """Process the response from OpenWeatherMap API into timed beliefs. + Collects all forecasts for all locations and all sensors at all locations, then bulk-saves them. + """ + click.echo("[FLEXMEASURES-OWM] Getting weather forecasts:") + click.echo("[FLEXMEASURES-OWM] Latitude, Longitude") + click.echo("[FLEXMEASURES-OWM] -----------------------") + + for location in locations: + click.echo("[FLEXMEASURES] %s, %s" % location) + weather_sensors: Dict[ + str, Sensor + ] = {} # keep track of the sensors to save lookups + db_forecasts: Dict[Sensor, List[TimedBelief]] = {} # collect beliefs per sensor + + now = server_now() + owm_time_of_api_call, forecasts = call_openweatherapi(api_key, location) + diff_fm_owm = now - owm_time_of_api_call + if abs(diff_fm_owm) > timedelta(minutes=10): + click.echo( + f"[FLEXMEASURES-OWM] Warning: difference between this server and OWM is {naturaldelta(diff_fm_owm)}" + ) + click.echo( + f"[FLEXMEASURES-OWM] Called OpenWeatherMap API successfully at {now}." + ) + + # loop through forecasts, including the one of current hour (horizon 0) + for fc in forecasts: + fc_datetime = as_server_time( + datetime.fromtimestamp(fc["dt"], get_timezone()) + ) + click.echo(f"[FLEXMEASURES-OWM] Processing forecast for {fc_datetime} ...") + for owm_response_label in owm_to_sensor_map: + data_source = get_or_create_owm_data_source() + sensor_specs = owm_to_sensor_map[owm_response_label] + sensor_name = str(sensor_specs["name"]) + if owm_response_label in fc: + weather_sensor = get_weather_sensor( + owm_response_label, + location, + weather_sensors, + max_degree_difference_for_nearest_weather_sensor, + ) + if weather_sensor not in db_forecasts.keys(): + db_forecasts[weather_sensor] = [] + + fc_value = fc[owm_response_label] + + # the radiation is not available in OWM -> we compute it ourselves + if sensor_name == "irradiance": + fc_value = compute_irradiance( + location[0], + location[1], + fc_datetime, + # OWM sends cloud coverage in percent, we need a ratio + fc_value / 100.0, + ) + data_source = get_or_create_owm_data_source_for_derived_data() + + db_forecasts[weather_sensor].append( + TimedBelief( + event_start=fc_datetime, + belief_time=now, + event_value=fc_value, + sensor=weather_sensor, + source=data_source, + ) + ) + else: + # we will not fail here, but issue a warning + msg = "No label '%s' in response data for time %s" % ( + owm_response_label, + fc_datetime, + ) + click.echo("[FLEXMEASURES-OWM] %s" % msg) + current_app.logger.warning(msg) + for sensor in db_forecasts.keys(): + click.echo(f"[FLEXMEASURES-OWM] Saving {sensor.name} forecasts ...") + if len(db_forecasts[sensor]) == 0: + # This is probably a serious problem + raise Exception( + "Nothing to put in the database was produced. That does not seem right..." + ) + status = save_to_db(BeliefsDataFrame(db_forecasts[sensor])) + if status == "success_but_nothing_new": + current_app.logger.info( + "[FLEXMEASURES-OWM] Done. These beliefs had already been saved before." + ) + elif status == "success_with_unchanged_beliefs_skipped": + current_app.logger.info( + "[FLEXMEASURES-OWM] Done. Some beliefs had already been saved before." + ) + + +def get_weather_sensor( + owm_response_label: str, + location: Tuple[float, float], + weather_sensors: Dict[str, Sensor], + max_degree_difference_for_nearest_weather_sensor: int, +) -> Sensor: + """Get the weather sensor for this own response label and location, if we haven't retrieved it already.""" + sensor_specs = owm_to_sensor_map[owm_response_label] + sensor_name = str(sensor_specs["name"]) + if sensor_name in weather_sensors: + weather_sensor = weather_sensors[sensor_name] + else: + weather_sensor = find_weather_sensor_by_location_or_fail( + location, + max_degree_difference_for_nearest_weather_sensor, + sensor_name=sensor_name, + ) + weather_sensors[sensor_name] = weather_sensor + if weather_sensor.event_resolution != sensor_specs["event_resolution"]: + raise Exception( + f"[FLEXMEASURES-OWM] The weather sensor found for {sensor_name} has an unfitting event resolution (should be {sensor_specs['event_resolution']}, but is {weather_sensor.event_resolution}." + ) + return weather_sensor + + +def save_forecasts_as_json( + api_key: str, locations: List[Tuple[float, float]], data_path: str +): + """Get forecasts, then store each as a raw JSON file, for later processing.""" + click.echo("[FLEXMEASURES-OWM] Getting weather forecasts:") + click.echo("[FLEXMEASURES-OWM] Latitude, Longitude") + click.echo("[FLEXMEASURES-OWM] ----------------------") + for location in locations: + click.echo("[FLEXMEASURES-OWM] %s, %s" % location) + now = server_now() + owm_time_of_api_call, forecasts = call_openweatherapi(api_key, location) + diff_fm_owm = now - owm_time_of_api_call + if abs(diff_fm_owm) > timedelta(minutes=10): + click.echo( + f"[FLEXMEASURES-OWM] Warning: difference between this server and OWM is {naturaldelta(diff_fm_owm)}" + ) + now_str = now.strftime("%Y-%m-%dT%H-%M-%S") + path_to_files = os.path.join(data_path, now_str) + if not os.path.exists(path_to_files): + click.echo(f"[FLEXMEASURES-OWM] Making directory: {path_to_files} ...") + os.mkdir(path_to_files) + forecasts_file = "%s/forecast_lat_%s_lng_%s.json" % ( + path_to_files, + str(location[0]), + str(location[1]), + ) + with open(forecasts_file, "w") as outfile: + json.dump(forecasts, outfile) diff --git a/flexmeasures_openweathermap/utils/radiating.py b/flexmeasures_openweathermap/utils/radiating.py new file mode 100644 index 0000000..90d6893 --- /dev/null +++ b/flexmeasures_openweathermap/utils/radiating.py @@ -0,0 +1,37 @@ +from datetime import datetime + +import pandas as pd +from pvlib.location import Location + + +def compute_irradiance( + latitude: float, longitude: float, dt: datetime, cloud_coverage: float +) -> float: + """Compute the irradiance received on a location at a specific time. + This uses pvlib to + 1) compute clear-sky irradiance as Global Horizontal Irradiance (GHI), + which includes both Direct Normal Irradiance (DNI) + and Diffuse Horizontal Irradiance (DHI). + 2) adjust the GHI for cloud coverage + """ + site = Location(latitude, longitude, tz=dt.tzinfo) + solpos = site.get_solarposition(pd.DatetimeIndex([dt])) + ghi_clear = site.get_clearsky(pd.DatetimeIndex([dt]), solar_position=solpos).loc[ + dt + ]["ghi"] + return ghi_clear_to_ghi(ghi_clear, cloud_coverage) + + +def ghi_clear_to_ghi(ghi_clear: float, cloud_coverage: float) -> float: + """Compute global horizontal irradiance (GHI) from clear-sky GHI, given a cloud coverage between 0 and 1. + + References + ---------- + Perez, R., Moore, K., Wilcox, S., Renne, D., Zelenka, A., 2007. + Forecasting solar radiation – preliminary evaluation of an + approach based upon the national forecast database. Solar Energy + 81, 809–812. + """ + if cloud_coverage < 0 or cloud_coverage > 1: + raise ValueError("cloud_coverage should lie in the interval [0, 1]") + return (1 - 0.87 * cloud_coverage ** 1.9) * ghi_clear diff --git a/flexmeasures_openweathermap/utils/tests/test_modeling.py b/flexmeasures_openweathermap/utils/tests/test_modeling.py new file mode 100644 index 0000000..19f9ce3 --- /dev/null +++ b/flexmeasures_openweathermap/utils/tests/test_modeling.py @@ -0,0 +1,15 @@ +from flexmeasures.data.models.generic_assets import GenericAsset + +from flexmeasures_openweathermap import DEFAULT_WEATHER_STATION_NAME +from flexmeasures_openweathermap.utils.modeling import get_or_create_weather_station + + +def test_creating_two_weather_stations(fresh_db): + get_or_create_weather_station(50, 40) + get_or_create_weather_station(40, 50) + assert ( + GenericAsset.query.filter( + GenericAsset.name == DEFAULT_WEATHER_STATION_NAME + ).count() + == 2 + ) diff --git a/requirements/app.in b/requirements/app.in index 8c65eb4..8fb01d9 100644 --- a/requirements/app.in +++ b/requirements/app.in @@ -1 +1,7 @@ -flexmeasures>=0.8.0 +flexmeasures>=0.9.0.dev37 +requests +pvlib +# the following three are optional in pvlib, but we use them +netCDF4 +siphon +tables \ No newline at end of file diff --git a/requirements/app.txt b/requirements/app.txt index deb8cca..77dfd34 100644 --- a/requirements/app.txt +++ b/requirements/app.txt @@ -1 +1,567 @@ -flexmeasures==0.8.0 +# +# This file is autogenerated by pip-compile with python 3.8 +# To update, run: +# +# pip-compile --output-file=requirements/app.txt requirements/app.in +# +alembic==1.7.6 + # via + # flask-migrate + # flexmeasures +altair==4.2.0 + # via + # flexmeasures + # timely-beliefs +arrow==1.2.2 + # via + # flexmeasures + # rq-dashboard +async-generator==1.10 + # via + # flexmeasures + # trio + # trio-websocket +attrs==21.4.0 + # via + # flexmeasures + # jsonschema + # outcome + # trio +babel==2.9.1 + # via + # flexmeasures + # py-moneyed +backports.zoneinfo==0.2.1 + # via + # flexmeasures + # pytz-deprecation-shim + # tzlocal + # workalendar +bcrypt==3.2.0 + # via flexmeasures +beautifulsoup4==4.10.0 + # via siphon +blinker==1.4 + # via + # flask-mail + # flask-principal + # flask-security-too + # flexmeasures + # sentry-sdk +bokeh==1.0.4 + # via + # flexmeasures + # pandas-bokeh +certifi==2021.10.8 + # via + # flexmeasures + # requests + # sentry-sdk + # urllib3 +cffi==1.15.0 + # via + # bcrypt + # cryptography + # flexmeasures +cftime==1.5.2 + # via netcdf4 +charset-normalizer==2.0.12 + # via + # flexmeasures + # requests +click==8.0.3 + # via + # flask + # flexmeasures + # rq +colour==0.1.5 + # via flexmeasures +convertdate==2.4.0 + # via + # flexmeasures + # workalendar +cryptography==36.0.1 + # via + # flexmeasures + # pyopenssl + # urllib3 +cycler==0.11.0 + # via + # flexmeasures + # matplotlib +deprecated==1.2.13 + # via + # flexmeasures + # redis +dill==0.3.4 + # via + # flexmeasures + # openturns +dnspython==2.2.0 + # via + # email-validator + # flexmeasures +email-validator==1.1.3 + # via + # flask-security-too + # flexmeasures +entrypoints==0.4 + # via + # altair + # flexmeasures +filelock==3.4.2 + # via + # flexmeasures + # tldextract +flask==2.0.2 + # via + # flask-classful + # flask-cors + # flask-json + # flask-login + # flask-mail + # flask-marshmallow + # flask-migrate + # flask-principal + # flask-security-too + # flask-sqlalchemy + # flask-sslify + # flask-wtf + # flexmeasures + # rq-dashboard + # sentry-sdk +flask-classful==0.14.2 + # via flexmeasures +flask-cors==3.0.10 + # via flexmeasures +flask-json==0.3.4 + # via flexmeasures +flask-login==0.5.0 + # via + # flask-security-too + # flexmeasures +flask-mail==0.9.1 + # via flexmeasures +flask-marshmallow==0.14.0 + # via flexmeasures +flask-migrate==3.1.0 + # via flexmeasures +flask-principal==0.4.0 + # via + # flask-security-too + # flexmeasures +flask-security-too==4.1.2 + # via flexmeasures +flask-sqlalchemy==2.5.1 + # via + # flask-migrate + # flexmeasures +flask-sslify==0.1.5 + # via flexmeasures +flask-wtf==1.0.0 + # via + # flask-security-too + # flexmeasures +flexmeasures==0.9.0.dev37 + # via -r requirements/app.in +fonttools==4.29.1 + # via + # flexmeasures + # matplotlib +greenlet==1.1.2 + # via + # flexmeasures + # sqlalchemy +h11==0.13.0 + # via + # flexmeasures + # wsproto +h5py==3.6.0 + # via pvlib +humanize==4.0.0 + # via flexmeasures +idna==3.3 + # via + # email-validator + # flexmeasures + # requests + # tldextract + # trio + # urllib3 +importlib-metadata==4.11.0 + # via + # alembic + # flexmeasures + # timely-beliefs +importlib-resources==5.4.0 + # via + # alembic + # flexmeasures + # jsonschema +inflect==5.4.0 + # via flexmeasures +inflection==0.5.1 + # via flexmeasures +iso8601==1.0.2 + # via flexmeasures +isodate==0.6.1 + # via + # flexmeasures + # timely-beliefs +itsdangerous==2.0.1 + # via + # flask + # flask-security-too + # flask-wtf + # flexmeasures +jinja2==3.0.3 + # via + # altair + # bokeh + # flask + # flexmeasures +joblib==1.1.0 + # via + # flexmeasures + # scikit-learn +jsonschema==4.4.0 + # via + # altair + # flexmeasures +kiwisolver==1.3.2 + # via + # flexmeasures + # matplotlib +lunardate==0.2.0 + # via + # flexmeasures + # workalendar +mako==1.1.6 + # via + # alembic + # flexmeasures +markupsafe==2.0.1 + # via + # flexmeasures + # jinja2 + # mako + # wtforms +marshmallow==3.14.1 + # via + # flask-marshmallow + # flexmeasures + # marshmallow-polyfield + # marshmallow-sqlalchemy + # webargs +marshmallow-polyfield==5.10 + # via flexmeasures +marshmallow-sqlalchemy==0.27.0 + # via flexmeasures +matplotlib==3.5.1 + # via + # flexmeasures + # timetomodel +netcdf4==1.5.8 + # via -r requirements/app.in +numexpr==2.8.1 + # via tables +numpy==1.22.2 + # via + # altair + # bokeh + # cftime + # flexmeasures + # h5py + # matplotlib + # netcdf4 + # numexpr + # pandas + # patsy + # properscoring + # pvlib + # scikit-learn + # scipy + # siphon + # statsmodels + # tables + # timely-beliefs + # timetomodel +openturns==1.18 + # via + # flexmeasures + # timely-beliefs +outcome==1.1.0 + # via + # flexmeasures + # trio +packaging==21.3 + # via + # bokeh + # flexmeasures + # matplotlib + # numexpr + # pint + # redis + # statsmodels + # tables + # webargs +pandas==1.2.5 + # via + # altair + # flexmeasures + # pandas-bokeh + # pvlib + # siphon + # statsmodels + # timely-beliefs + # timetomodel +pandas-bokeh==0.4.3 + # via flexmeasures +passlib==1.7.4 + # via + # flask-security-too + # flexmeasures +patsy==0.5.2 + # via + # flexmeasures + # statsmodels +pillow==9.0.1 + # via + # bokeh + # flexmeasures + # matplotlib +pint==0.18 + # via flexmeasures +ply==3.11 + # via + # flexmeasures + # pyomo +properscoring==0.1 + # via + # flexmeasures + # timely-beliefs +protobuf==3.19.4 + # via siphon +pscript==0.7.7 + # via flexmeasures +psutil==5.9.0 + # via + # flexmeasures + # openturns +psycopg2-binary==2.9.3 + # via + # flexmeasures + # timely-beliefs +pvlib==0.9.0 + # via -r requirements/app.in +py-moneyed==2.0 + # via flexmeasures +pycparser==2.21 + # via + # cffi + # flexmeasures +pyluach==1.3.0 + # via + # flexmeasures + # workalendar +pymeeus==0.5.11 + # via + # convertdate + # flexmeasures +pyomo==6.2 + # via flexmeasures +pyopenssl==22.0.0 + # via + # flexmeasures + # urllib3 +pyparsing==3.0.7 + # via + # flexmeasures + # matplotlib + # packaging +pyrsistent==0.18.1 + # via + # flexmeasures + # jsonschema +python-dateutil==2.8.2 + # via + # arrow + # bokeh + # flexmeasures + # matplotlib + # pandas + # timetomodel + # workalendar +python-dotenv==0.19.2 + # via flexmeasures +pytz==2021.3 + # via + # babel + # flexmeasures + # pandas + # pvlib + # timely-beliefs + # timetomodel +pytz-deprecation-shim==0.1.0.post0 + # via + # flexmeasures + # tzlocal +pyyaml==6.0 + # via + # bokeh + # flexmeasures +redis==4.1.3 + # via + # flexmeasures + # rq + # rq-dashboard +requests==2.27.1 + # via + # -r requirements/app.in + # flexmeasures + # pvlib + # requests-file + # siphon + # tldextract +requests-file==1.5.1 + # via + # flexmeasures + # tldextract +rq==1.10.1 + # via + # flexmeasures + # rq-dashboard +rq-dashboard==0.6.1 + # via flexmeasures +scikit-learn==1.0.2 + # via + # flexmeasures + # sklearn +scipy==1.8.0 + # via + # flexmeasures + # properscoring + # pvlib + # scikit-learn + # statsmodels + # timely-beliefs + # timetomodel +selenium==4.1.0 + # via + # flexmeasures + # timely-beliefs +sentry-sdk[flask]==1.5.5 + # via flexmeasures +siphon==0.9 + # via -r requirements/app.in +six==1.16.0 + # via + # bcrypt + # bokeh + # flask-cors + # flask-marshmallow + # flexmeasures + # isodate + # patsy + # python-dateutil + # requests-file +sklearn==0.0 + # via + # flexmeasures + # timetomodel +sniffio==1.2.0 + # via + # flexmeasures + # trio +sortedcontainers==2.4.0 + # via + # flexmeasures + # trio +soupsieve==2.3.1 + # via beautifulsoup4 +sqlalchemy==1.4.31 + # via + # alembic + # flask-sqlalchemy + # flexmeasures + # marshmallow-sqlalchemy + # timely-beliefs + # timetomodel +statsmodels==0.13.2 + # via + # flexmeasures + # timetomodel +tables==3.7.0 + # via -r requirements/app.in +tabulate==0.8.9 + # via flexmeasures +threadpoolctl==3.1.0 + # via + # flexmeasures + # scikit-learn +timely-beliefs==1.11.2 + # via flexmeasures +timetomodel==0.7.1 + # via flexmeasures +tldextract==3.1.2 + # via flexmeasures +toolz==0.11.2 + # via + # altair + # flexmeasures +tornado==6.1 + # via + # bokeh + # flexmeasures +trio==0.19.0 + # via + # flexmeasures + # selenium + # trio-websocket +trio-websocket==0.9.2 + # via + # flexmeasures + # selenium +typing-extensions==4.1.1 + # via + # flexmeasures + # py-moneyed +tzdata==2021.5 + # via + # flexmeasures + # pytz-deprecation-shim +tzlocal==4.1 + # via flexmeasures +urllib3[secure]==1.26.8 + # via + # flexmeasures + # requests + # selenium + # sentry-sdk +webargs==8.1.0 + # via flexmeasures +werkzeug==2.0.3 + # via + # flask + # flexmeasures +workalendar==16.2.0 + # via flexmeasures +wrapt==1.13.3 + # via + # deprecated + # flexmeasures +wsproto==1.0.0 + # via + # flexmeasures + # trio-websocket +wtforms==3.0.1 + # via + # flask-wtf + # flexmeasures +xlrd==2.0.1 + # via flexmeasures +zipp==3.7.0 + # via + # flexmeasures + # importlib-metadata + # importlib-resources diff --git a/requirements/dev.txt b/requirements/dev.txt index e69de29..2e32c8e 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -0,0 +1,92 @@ +# +# This file is autogenerated by pip-compile with python 3.8 +# To update, run: +# +# pip-compile --output-file=requirements/dev.txt requirements/dev.in +# +black==22.1.0 + # via -r requirements/dev.in +cfgv==3.3.1 + # via pre-commit +click==8.0.3 + # via + # -c requirements/app.txt + # -c requirements/test.txt + # black +distlib==0.3.4 + # via virtualenv +filelock==3.4.2 + # via + # -c requirements/app.txt + # virtualenv +flake8==4.0.1 + # via -r requirements/dev.in +flake8-blind-except==0.2.0 + # via -r requirements/dev.in +identify==2.4.10 + # via pre-commit +mccabe==0.6.1 + # via flake8 +mypy==0.931 + # via -r requirements/dev.in +mypy-extensions==0.4.3 + # via + # black + # mypy +nodeenv==1.6.0 + # via pre-commit +packaging==21.3 + # via + # -c requirements/app.txt + # -c requirements/test.txt + # setuptools-scm +pathspec==0.9.0 + # via black +platformdirs==2.5.0 + # via + # black + # virtualenv +pre-commit==2.17.0 + # via -r requirements/dev.in +pycodestyle==2.8.0 + # via flake8 +pyflakes==2.4.0 + # via flake8 +pyparsing==3.0.7 + # via + # -c requirements/app.txt + # -c requirements/test.txt + # packaging +pytest-runner==5.3.1 + # via -r requirements/dev.in +pyyaml==6.0 + # via + # -c requirements/app.txt + # pre-commit +setuptools-scm==6.4.2 + # via -r requirements/dev.in +six==1.16.0 + # via + # -c requirements/app.txt + # -c requirements/test.txt + # virtualenv +toml==0.10.2 + # via pre-commit +tomli==2.0.1 + # via + # -c requirements/test.txt + # black + # mypy + # setuptools-scm +typing-extensions==4.1.1 + # via + # -c requirements/app.txt + # black + # mypy +virtualenv==20.13.1 + # via pre-commit +watchdog==2.1.6 + # via -r requirements/dev.in + +# The following packages are considered to be unsafe in a requirements file: +# setuptools diff --git a/requirements/test.in b/requirements/test.in index af81a3d..8c1f66e 100644 --- a/requirements/test.in +++ b/requirements/test.in @@ -4,3 +4,7 @@ pytest pytest-flask pytest-sugar pytest-cov +# lets tests run successfully in containers +fakeredis +# required with fakeredis, maybe because we use rq +lupa diff --git a/requirements/test.txt b/requirements/test.txt index e69de29..54f47c4 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -0,0 +1,96 @@ +# +# This file is autogenerated by pip-compile with python 3.8 +# To update, run: +# +# pip-compile --output-file=requirements/test.txt requirements/test.in +# +attrs==21.4.0 + # via + # -c requirements/app.txt + # pytest +click==8.0.3 + # via + # -c requirements/app.txt + # flask +coverage[toml]==6.3.1 + # via pytest-cov +deprecated==1.2.13 + # via + # -c requirements/app.txt + # redis +fakeredis==1.7.1 + # via -r requirements/test.in +flask==2.0.2 + # via + # -c requirements/app.txt + # pytest-flask +iniconfig==1.1.1 + # via pytest +itsdangerous==2.0.1 + # via + # -c requirements/app.txt + # flask +jinja2==3.0.3 + # via + # -c requirements/app.txt + # flask +lupa==1.10 + # via -r requirements/test.in +markupsafe==2.0.1 + # via + # -c requirements/app.txt + # jinja2 +packaging==21.3 + # via + # -c requirements/app.txt + # fakeredis + # pytest + # pytest-sugar + # redis +pluggy==1.0.0 + # via pytest +py==1.11.0 + # via pytest +pyparsing==3.0.7 + # via + # -c requirements/app.txt + # packaging +pytest==7.0.1 + # via + # -r requirements/test.in + # pytest-cov + # pytest-flask + # pytest-sugar +pytest-cov==3.0.0 + # via -r requirements/test.in +pytest-flask==1.2.0 + # via -r requirements/test.in +pytest-sugar==0.9.4 + # via -r requirements/test.in +redis==4.1.3 + # via + # -c requirements/app.txt + # fakeredis +six==1.16.0 + # via + # -c requirements/app.txt + # fakeredis +sortedcontainers==2.4.0 + # via + # -c requirements/app.txt + # fakeredis +termcolor==1.1.0 + # via pytest-sugar +tomli==2.0.1 + # via + # coverage + # pytest +werkzeug==2.0.3 + # via + # -c requirements/app.txt + # flask + # pytest-flask +wrapt==1.13.3 + # via + # -c requirements/app.txt + # deprecated diff --git a/run_mypy.sh b/run_mypy.sh index 9519c89..3a25475 100755 --- a/run_mypy.sh +++ b/run_mypy.sh @@ -1,6 +1,6 @@ #!/bin/bash set -e pip install mypy -# We are checking python files which have type hints -files=$(find . -name \*.py -not -path "./venv/*") +pip install types-pytz types-requests types-Flask types-click types-redis types-tzlocal types-python-dateutil types-setuptools +files=$(find . -name \*.py -not -path "./venv/*" -not -path ".eggs/*" -not -path "./build/*" -not -path "./dist/*" -not -path "./scripts/*") mypy --follow-imports skip --ignore-missing-imports $files diff --git a/scripts/solartest.py b/scripts/solartest.py new file mode 100755 index 0000000..8ef0d70 --- /dev/null +++ b/scripts/solartest.py @@ -0,0 +1,177 @@ +#!/usr/bin/env python3 + +""" +Quick script to compare clear-sky irradiance computations +from three different libraries. +Among other considerations, this helped us to settle on pvlib. +""" +from typing import List, Dict +from datetime import datetime, timedelta + +import solarpy +import pvlib +import pysolar +import matplotlib.dates as mpl_dates +import matplotlib.pyplot as plt +import pytz +from pandas import DatetimeIndex +from tzwhere import tzwhere +from astral import LocationInfo +from astral.sun import sun + + +DAY = datetime(2021, 2, 10, tzinfo=pytz.utc) +tzwhere = tzwhere.tzwhere() + +locations = { + "Amsterdam": (52.370216, 4.895168), + "Tokyo": (35.6684415, 139.6007844), + "Dallas": (32.779167, -96.808891), + "Cape-Town": (-33.943707, 18.588740), # check southern hemisphere, too +} +datetimes = [DAY + timedelta(minutes=i * 20) for i in range(24 * 3)] +timezones = {k: tzwhere.tzNameAt(*v) for k, v in locations.items()} + + +def irradiance_by_solarpy( + latitude: float, longitude: float, dt: datetime, z: str, metric: str = "dni" +) -> float: + """Supports direct horizontal irradiance and direct normal irradiance.""" + h = 0 # sea-level + dt = dt.astimezone(pytz.timezone(z)).replace(tzinfo=None) # local time + dt = solarpy.standard2solar_time(dt, longitude) # solar time + if metric == "dhi": # direct horizontal irradiance + vnorm = [0, 0, -1] # plane pointing up + elif metric == "dni": # direct normal irradiance + vnorm = solarpy.solar_vector_ned( + dt, latitude + ) # plane pointing directly to the sun + vnorm[-1] = vnorm[-1] * 0.99999 # avoid floating point error + else: + return NotImplemented + return solarpy.irradiance_on_plane(vnorm, h, dt, latitude) + + +def irradiance_by_pysolar( + latitude: float, longitude: float, dt: datetime, method: str = "dni" +) -> float: + """Supports direct normal irradiance.""" + altitude_deg = pysolar.solar.get_altitude(latitude, longitude, dt) + if method == "dni": + return pysolar.radiation.get_radiation_direct(dt, altitude_deg) + else: + return NotImplemented + + +def irradiance_by_pvlib( + latitude: float, longitude: float, dt: datetime, method: str = "dni" +) -> float: + """ + Supports direct horizontal irradiance, direct normal irradiance and global horizontal irradiance. + https://firstgreenconsulting.wordpress.com/2012/04/26/differentiate-between-the-dni-dhi-and-ghi/ + """ + site = pvlib.location.Location(latitude, longitude, tz=pytz.utc) + solpos = site.get_solarposition(DatetimeIndex([dt])) + irradiance = site.get_clearsky(DatetimeIndex([dt]), solar_position=solpos).loc[dt] + if method in ("ghi", "dni", "dhi"): + return irradiance[method] + else: + return NotImplemented + + +def plot_irradiance( + city: str, + datetimes: List[datetime], + values: Dict[str, List[float]], + sun_times: Dict[str, datetime], +): + + fig, ax = plt.subplots() + + ax.set( + xlabel="Time (20m)", + ylabel="Direct Normal Irradiance (W/m²)", + title=f"Irradiance for {city} on {DAY.date()}", + ) + + # draw values + date_ticks = mpl_dates.date2num(datetimes) + for lib in ("pysolar", "solarpy", "pvlib"): + plt.plot_date(date_ticks, values[lib], "-", label=lib) + + # make date ticks look okay + plt.gca().xaxis.set_major_locator(mpl_dates.HourLocator()) + plt.setp(plt.gca().xaxis.get_majorticklabels(), "rotation", 40) + + # draw day phases boxes + dawn_tick, sunrise_tick, noon_tick, sunset_tick, dusk_tick = mpl_dates.date2num( + ( + sun_times["dawn"], + sun_times["sunrise"], + sun_times["noon"], + sun_times["sunset"], + sun_times["dusk"], + ) + ) + dawn_to_sunrise = plt.Rectangle( + (dawn_tick, -100), + sunrise_tick - dawn_tick, + 1100, + fc="floralwhite", + ec="lemonchiffon", + label="Dawn to Sunrise", + ) + plt.gca().add_patch(dawn_to_sunrise) + + sunrise_to_sunset = plt.Rectangle( + (sunrise_tick, -100), + sunset_tick - sunrise_tick, + 1100, + fc="lightyellow", + ec="lemonchiffon", + label="Sunrise to sunset", + ) + plt.gca().add_patch(sunrise_to_sunset) + + sunset_to_dusk = plt.Rectangle( + (sunset_tick, -100), + dusk_tick - sunset_tick, + 1100, + fc="oldlace", + ec="lemonchiffon", + label="Sunset to dusk", + ) + plt.gca().add_patch(sunset_to_dusk) + + # draw noon + plt.axvline(x=noon_tick, color="gold", label="Noon") + + plt.legend() + + fig.savefig(f"test-irradiance-{city}.png") + plt.show() + + +if __name__ == "__main__": + for city in locations: + values = dict(pysolar=[], solarpy=[], pvlib=[]) + lat, lon = locations[city] + timezone = timezones[city] + loc_info = LocationInfo(timezone=timezone, latitude=lat, longitude=lon) + # this gives 'dawn', 'sunrise', 'noon', 'sunset' and 'dusk' + sun_times = sun(loc_info.observer, date=DAY.date(), tzinfo=loc_info.timezone) + local_datetimes = [ + dt.replace(tzinfo=pytz.timezone(timezones[city])) for dt in datetimes + ] + + for dt in local_datetimes: + irrad_pysolar = irradiance_by_pysolar(lat, lon, dt) + values["pysolar"].append(irrad_pysolar) + irrad_solarpy = irradiance_by_solarpy(lat, lon, dt, timezone) + values["solarpy"].append(irrad_solarpy) + irrad_pvlib = irradiance_by_pvlib(lat, lon, dt) + values["pvlib"].append(irrad_pvlib) + print( + f"For {city} at {dt} {timezones[city]} ― pysolar: {irrad_pysolar:.2f}, solarpy: {irrad_solarpy:.2f}, pvlib: {irrad_pvlib:.2f}" + ) + plot_irradiance(city, local_datetimes, values, sun_times) diff --git a/setup.py b/setup.py index bb2c25c..d506e35 100644 --- a/setup.py +++ b/setup.py @@ -16,7 +16,7 @@ def load_requirements(use_case): setup( name="flexmeasures-openweathermap", - description="Integratig FlexMeasures with OpenWeatherMap", + description="Integrating FlexMeasures with OpenWeatherMap", author="Seita Energy Flexibility BV", author_email="nicolas@seita.nl", url="https://github.com/SeitaBV/flexmeasures-openweathermap", @@ -29,7 +29,7 @@ def load_requirements(use_case): include_package_data=True, # setuptools_scm takes care of adding the files in SCM classifiers=[ "Programming Language :: Python", - "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", "Development Status :: 3 - Alpha", "License :: OSI Approved :: Apache Software License", "Operating System :: OS Independent",