-
Notifications
You must be signed in to change notification settings - Fork 2
/
harvester_whale.cfg
66 lines (59 loc) · 2.49 KB
/
harvester_whale.cfg
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
# Smart Emission Data Harvester - Stetl config
#
# Just van den Broecke - 2016
#
# This config reads raw measurements from the CityGIS/Intemo (Whale) Sensor REST API.
# It inserts this data in Postgres. The progres of harvesting (date/time) is tracked, so this allows to harvest all data
# in multiple steps.
# The measurement data in Postgres is then available for further refinement (validation, aggregation, etc).
# The main Stetl ETL chain
[etl]
chains = input_raw_sensor_timeseries_2|output_postgres_insert
# [etl]
# chains = input_raw_sensor_timeseries_2|output_postgres_insert
# Read all history data from CityGIS Sensor REST Whale API
[input_raw_sensor_timeseries_1]
class = smartem.harvester.rawsensortimeseriesinput.RawSensorTimeseriesInput
url = {raw_device_url_1}
host = {pg_host}
database = {pg_database}
user = {pg_user}
password = {pg_password}
schema = {pg_schema_raw}
max_proc_time_secs = {max_proc_time_secs}
progress_table = harvester_progress
api_interval_secs = {api_interval_secs}
# migrated CityGIS to Intemo DC: 9,22,29,36,37,49,53,67,82,83,85,89,92,102,126,133 10.08.2018: 20,60,63 20.08.18: 47
# 22.08.18: 45 27.08:18: 13,15,26 28.08.18: 17 21.09.18: 84 05.11.2018: 66, 44, 23, 27, 32, 46, 16 -
# 07.11.2018: 33, 55, 30, 25, 62 - 13.11.18: 52, 54, 57 - 19.11.18: 31,35,42,50,59,61,87 and 111 03.12.18: 88,91
# 06.12.18: 19
skip_devices = 0,2,6,7,8,9,13,15,16,17,19,20,22,23,25,26,27,29,30,31,32,33,35,36,37,42,44,45,46,47,49,50,52,53,54,55,57,59,60,61,62,63,66,67,82,83,84,85,87,88,89,91,92,102,111,126,133,414,441,1298,2087,2191,6642,38444,55555,75662,604986,661785,828231,991007,995374,1908250,2005002,3092533,3487232,16777215
# Read all history data from Intemo Sensor REST Whale API
[input_raw_sensor_timeseries_2]
class = smartem.harvester.rawsensortimeseriesinput.RawSensorTimeseriesInput
url = {raw_device_url_2}
host = {pg_host}
database = {pg_database}
user = {pg_user}
password = {pg_password}
schema = {pg_schema_raw}
max_proc_time_secs = {max_proc_time_secs}
progress_table = harvester_progress
api_interval_secs = {api_interval_secs}
skip_devices = 2005002
auth = {{ 'type': 'token', 'keyword': 'Bearer', 'token': '{intemo_token}' }}
# for testing/debugging
[output_std]
class = outputs.standardoutput.StandardOutput
# Insert file records
[output_postgres_insert]
class = outputs.dboutput.PostgresInsertOutput
input_format = record_array
host = {pg_host}
database = {pg_database}
user = {pg_user}
password = {pg_password}
schema = {pg_schema_raw}
table = timeseries
key=unique_id
replace=True