-
Notifications
You must be signed in to change notification settings - Fork 0
/
ScraperDataAccess.js
57 lines (48 loc) · 2.28 KB
/
ScraperDataAccess.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
const fs = require('fs');
const mysql = require('mysql');
const config = require('./config/scrapingConfig.json');
const axios = require('axios');
const get = require('lodash').get;
module.exports = class ScraperDataAccess {
constructor() {
this.urlBase = config.urlBase;
}
async saveExecutionLog(executionLogRecord) {
const url = `${this.urlBase}/api/workers/execution_log`;
return axios.put(url, executionLogRecord).then(response => response.data);
}
async saveScrapingPiecesIndex(scapingPiecesIndexRecord) {
const url = `${this.urlBase}/api/workers/scraping_piece_index`;
return axios.put(url, scapingPiecesIndexRecord).then(response => response.data);
}
async saveScrapingResults(scapingResultsRecord) {
const url = `${this.urlBase}/api/workers/scraping_results`;
return axios.put(url, scapingResultsRecord).then(response => response.data);
}
async dropIndex(device_id) {
const url = `${this.urlBase}/api/workers/drop_index?device_id=${device_id}`;
return axios.delete(url).then(response => response.data);
}
async regenerateScrapingIndex(device_id, method = "boundingBox", cities, appId="airbnb") {
const device = {device_id, method, cities, app_id:appId}
console.log(device);
const url = `${this.urlBase}/api/workers/regenerate_scraping_index`;
return axios.post(url, device).then(response => response.data);
}
async getNextPieceToScrap(device_id) {
const url = `${this.urlBase}/api/workers/next_piece_to_scrap?device_id=${device_id}`;
return axios.get(url).then(response => response.data);
}
async setIndexAsNotScraped(device_id) {
const url = `${this.urlBase}/api/workers/set_index_as_not_scraped?device_id=${device_id}`;
return axios.delete(url).then(response => response.data);
}
async setIndexPieceAsScraped(piece_id) {
const url = `${this.urlBase}/api/workers/set_index_piece_as_scraped?piece_id=${piece_id}`;
return axios.delete(url).then(response => response.data);
}
async countIndexEntries(device_id) {
const url = `${this.urlBase}/api/workers/count_index_pieces?device_id=${device_id}`;
return axios.get(url).then(response => get(response.data, 'count'));
}
}