-
Notifications
You must be signed in to change notification settings - Fork 0
/
scheduler_cronfile
33 lines (33 loc) · 3.17 KB
/
scheduler_cronfile
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
HOME=/opt/app/dfg-gepris/
PATH=/usr/local/bin:/usr/bin:/bin
### Data Monitor ###
# every day 00.00 - data monitor
### Search Results and newest Details ###
0 0 * * * scrapyd-client schedule -p gepris_crawler data_monitor > /proc/1/fd/1 2>/proc/1/fd/2
# every day 00.05 - search_results - institution
5 0 * * * scrapyd-client schedule -p gepris_crawler --arg context=institution search_results > /proc/1/fd/1 2>/proc/1/fd/2
# every day 00.50 - details - institution - only new (max 200)
50 0 * * * scrapyd-client schedule -p gepris_crawler --arg context=institution --arg ids=db:needed:200 details > /proc/1/fd/1 2>/proc/1/fd/2
# every day 01.00 - search_results - person
0 1 * * * scrapyd-client schedule -p gepris_crawler --arg context=person search_results > /proc/1/fd/1 2>/proc/1/fd/2
# every day 01.50 - details - person - only new (max 200)
50 1 * * * scrapyd-client schedule -p gepris_crawler --arg context=person --arg ids=db:needed:200 details > /proc/1/fd/1 2>/proc/1/fd/2
# every day 02.00 - search_results - projekt
0 2 * * * scrapyd-client schedule -p gepris_crawler --arg context=projekt search_results > /proc/1/fd/1 2>/proc/1/fd/2
# every day 03.15 - details - projekt - only new (max 500)
15 3 * * * scrapyd-client schedule -p gepris_crawler --arg context=projekt --arg ids=db:needed:500 details > /proc/1/fd/1 2>/proc/1/fd/2
### 50000 Details requests a day for Re-Indexing to detect changes ###
# 1. of month at 03.30 - details - institution - all (max 50000), sorted by last scrap descending
# 2.-4., 6.-9. of month at 03.30 - details - projekt - all (max 20000), sorted by last scrap descending
# 5., 10. of month at 03.30 - details - person - all (max 50000), sorted by last scrap descending
# every command is repeated every 10th day, except for 31. of month, where nothing is done
30 3 1,11,21 * * scrapyd-client schedule -p gepris_crawler --arg context=institution --arg ids=db:all:50000 details > /proc/1/fd/1 2>/proc/1/fd/2
30 3 2,12,22 * * scrapyd-client schedule -p gepris_crawler --arg context=projekt --arg ids=db:all:20000 details > /proc/1/fd/1 2>/proc/1/fd/2
30 3 3,13,23 * * scrapyd-client schedule -p gepris_crawler --arg context=projekt --arg ids=db:all:20000 details > /proc/1/fd/1 2>/proc/1/fd/2
30 3 4,14,24 * * scrapyd-client schedule -p gepris_crawler --arg context=projekt --arg ids=db:all:20000 details > /proc/1/fd/1 2>/proc/1/fd/2
30 3 5,15,25 * * scrapyd-client schedule -p gepris_crawler --arg context=person --arg ids=db:all:50000 details > /proc/1/fd/1 2>/proc/1/fd/2
30 3 6,16,26 * * scrapyd-client schedule -p gepris_crawler --arg context=projekt --arg ids=db:all:20000 details > /proc/1/fd/1 2>/proc/1/fd/2
30 3 7,17,27 * * scrapyd-client schedule -p gepris_crawler --arg context=projekt --arg ids=db:all:20000 details > /proc/1/fd/1 2>/proc/1/fd/2
30 3 8,18,28 * * scrapyd-client schedule -p gepris_crawler --arg context=projekt --arg ids=db:all:20000 details > /proc/1/fd/1 2>/proc/1/fd/2
30 3 9,19,29 * * scrapyd-client schedule -p gepris_crawler --arg context=projekt --arg ids=db:all:20000 details > /proc/1/fd/1 2>/proc/1/fd/2
30 3 10,20,30 * * scrapyd-client schedule -p gepris_crawler --arg context=person --arg ids=db:all:50000 details > /proc/1/fd/1 2>/proc/1/fd/2