This repository has been archived by the owner on Nov 27, 2023. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathimport_data.py
104 lines (94 loc) · 3.84 KB
/
import_data.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
import csv
import datetime
import re
import click
import sqlite_utils
def import_kurz_data(filename):
db = sqlite_utils.Database("meinsack.db")
# this regex requires only one date in the former year!
with open(filename, "r") as fp:
reader = csv.DictReader(fp)
for row in reader:
for _ in row["dates"].split("|"):
area_id = row["district"]
dt = datetime.datetime.strptime(_, "%Y-%m-%d").date()
q = list(
db["pickupdate"].rows_where(
"date = ? and area_id = ?", [dt, area_id]
)
)
if not q:
db["pickupdate"].insert(
{
"created": datetime.datetime.utcnow(),
"modified": datetime.datetime.utcnow(),
"date": dt,
"area_id": area_id,
}
)
def parse_schaal_und_mueller_csv_data(filename, year):
db = sqlite_utils.Database("meinsack.db")
# this regex requires only one date in the former year!
regex = r"(.*)\ ([0-9\.]*)\ (Mo.|Di.|Mi.|Do.|Fr.)\ ([0-9\.\ ]*)"
with open(filename, "r") as fp:
for line in fp.readlines():
if line.startswith("#"):
continue
# find weekday and split on it
area = re.findall(regex, line)[0][0]
a = list(db["area"].rows_where("description = ?", [area]))
if not a:
for in_db, in_csv in (
("Birkach, Botnang, Plieningen", "Birkach, Plieningen, Botnang"),
(
"Frauenkopf, Hedelfingen (ohne Hafen), Sillenbuch, Riedenberg",
"Frauenkopf, Hedelfingen (ohne Hafen) Sillenbuch (mit Riedenberg)",
),
(
"Stuttgart-West (ohne Kräherwald, Solitude, Wildpark)",
"Stuttgart-West (ohne Kräherwald, Solitude,Wildpark)",
),
(
"Bad Cannstatt I (ohneSteinhaldenfeld), Mühlhausen",
"Bad Cannstatt I (ohneSteinhaldenfeld) Mühlhausen",
),
(
"Büsnau, Degerloch, Dürrlewang, Kräherwald,Solitude, Wildpark",
"Büsnau, Degerloch, Dürrlewang,Kräherwald,Solitude, Wildpark",
),
):
if area == in_csv:
a = list(db["area"].rows_where("description = ?", [in_db]))
if not a:
assert f"Area not found, {area}"
area_id = a[0].get("id")
dates = re.findall(regex, line)[0][3].split()
# add years
dates_wyears = [i + str(year) for i in dates[:-1]]
dates_wyears.append(dates[-1] + str(year + 1))
for _ in dates_wyears:
dt = datetime.datetime.strptime(_, "%d.%m.%Y").date()
q = list(
db["pickupdate"].rows_where(
"date = ? and area_id = ?", [dt, area_id]
)
)
if not q:
db["pickupdate"].insert(
{
"created": datetime.datetime.utcnow(),
"modified": datetime.datetime.utcnow(),
"date": dt,
"area_id": area_id,
}
)
@click.command()
@click.option("--filename")
@click.option("--year", type=int)
def main(filename, year):
if year <= 2022:
parse_schaal_und_mueller_csv_data(filename, year)
else:
import_kurz_data(filename)
if __name__ == "__main__":
main()