forked from mrzmyr/leipzig-maps
-
Notifications
You must be signed in to change notification settings - Fork 2
/
leipzig-swimming-pools.js
82 lines (74 loc) · 2.49 KB
/
leipzig-swimming-pools.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
import scrapeIt from "scrape-it";
import fs from 'fs';
import { search } from "../../lib/nominatim.mjs";
const domain = `https://www.leipzig.de`;
const url_indoor_pool = `${domain}/freizeit-kultur-und-tourismus/sport/sportstaetten/schwimmhallen`;
const url_outdoor_pool = `${domain}/freizeit-kultur-und-tourismus/sport/sportstaetten/freibaeder`;
const __dirname = new URL('.', import.meta.url).pathname;
const scrapeDetailsUrl = {
list: {
listItem: '.project-attributes',
data: {
title: {
selector: 'h3 a'
},
detailsUrl: {
selector: 'h3 a',
attr: "href"
}
}
}
}
const scrapeDetailsData = {
address: {
selector: '.t3booking-t3booking-main-content p',
eq: 0,
convert: value => value.replace(' 04', ', 04').replace('Im Stadtplan anzeigen', '')
}
}
const enrichWithCoords = async (element) => {
console.log('search for ' + element.address)
try {
const resp = await search(element.address)
const search_results = resp.filter(r => ['water_park', 'sports_centre'].includes(r.type));
if(search_results.length > 0) {
return {
...element,
address: resp[0].address,
lat: resp[0].lat,
lon: resp[0].lon
}
} else {
console.log(element.title, element.address, 'could not be found')
console.log(resp)
return element;
}
} catch(e) {
console.log(element.title, element.address, 'could not be resolved')
console.log(resp)
return element;
}
}
const scrapeDetailsUrlIndoor = scrapeIt(url_indoor_pool, scrapeDetailsUrl);
const scrapeDetailsUrlOutdoor = scrapeIt(url_outdoor_pool, scrapeDetailsUrl);
const handleScrapeResponse = async ([indoor_pool, outdoor_pool]) => {
const data = [
...indoor_pool.data.list.map(sh => ({ ...sh, type: 'indoor_pool' })),
...outdoor_pool.data.list.map(sh => ({ ...sh, type: 'outdoor_pool' })),
]
const promises = [];
for (let i = 0; i < data.length; i++) {
const element = data[i];
const detailsUrl = `${domain}${element.detailsUrl}`;
const details = await scrapeIt(detailsUrl, scrapeDetailsData);
const result = await enrichWithCoords(details.data);
promises.push({
title: element.title,
...result,
link: detailsUrl
});
}
const details = await Promise.all(promises);
fs.writeFileSync(`${__dirname}../../public/data/leipzig-swimming-pools.json`, JSON.stringify(details), 'utf8')
};
Promise.all([scrapeDetailsUrlIndoor, scrapeDetailsUrlOutdoor]).then(handleScrapeResponse)