-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathscript.py
206 lines (154 loc) · 6.9 KB
/
script.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
#!/usr/bin/env python
import csv
import os
import requests
#import unidecode
#import time
import json
import numpy
# Define region
re = 'KE'
re = 'ES'
# download list of hotspot for region
response = requests.request("GET", "https://ebird.org/ws2.0/ref/hotspot/"+re, headers={'X-eBirdApiToken': 'vcs68p4j67pt'}, params={"fmt":"csv"})
file = open(re+"/hotspots.csv", "w")
file.write(response.text)
file.close()
####################################################
# PART 1: READ/GET DATA
# Dowanload Hotspot as Json
response = requests.request("GET", "https://ebird.org/ws2.0/ref/hotspot/"+re, headers={'X-eBirdApiToken': 'vcs68p4j67pt'}, params={"fmt":"json"})
hotspots = json.loads(response.text)
# Download specie list
response = requests.request("GET", "https://ebird.org/barchartData?r="+re, params={"fmt":"json"})
tmp = json.loads(response.text)
species = [i['taxon'] for i in tmp['dataRows']]
# read KE of rht Book '50 Top Bird Sites in Kenya'
if re=="KE":
with open('50TopBirdingSites.csv') as f:
top50 = [{k: v for k, v in row.items()} for row in csv.DictReader(f, skipinitialspace=True)]
# Dowload Barchart KE for all hotspots
# Filter for hotspot having at leat 80 species recorded.
for hot in hotspots:
if 'numSpeciesAllTime' in hot:
if hot['numSpeciesAllTime']>80:
url = 'http://ebird.org/ebird/barchartData?r={code_loc}&bmo={bmonth}&emo={emonth}&byr={byear}&eyr={eyear}&fmt=json'.format(
code_loc=hot['locId'], byear=1900, eyear=2020, bmonth=1, emonth=12)
r = requests.get(url)
f = open(re+'/barchart/barchart_' + hot['locId']+'.json', 'wb')
f.write(r.content)
f.close()
####################################################
# PART 2: Build the structure
# hotspots -> species -> data
hotspots_enhanced = []
for bc in os.listdir(path=re+'/barchart/'):
with open(re+'/barchart/'+bc) as json_file:
BC = json.load(json_file)
hot = [item for item in hotspots if item["locId"] == bc.replace('.json','').replace('barchart_','')][0]
# hot.pop("countryCode", None)
# hot.pop("subnational1Code", None)
hot.pop("locID", None)
if re=="KE":
tmp = [item for item in top50 if item['code_loc'] == hot['locId']]
if len(tmp)>0:
hot['top50'] = tmp[0]['#']
else:
hot['top50'] = ''
hot['numSpecies'] = BC['numSpecies']
if len(BC['dataRows'])>0:
hot['numChecklists'] = sum(BC['dataRows'][0]['values_N'])
hot['values_N'] = [ sum(BC['dataRows'][0]['values_N'][i*4:(i*4+1*4)]) for i in range(0,12)]
else:
hot['numChecklists'] = 0
hot['values_N'] = []
hot['values'] = {}
for sp in BC['dataRows']:
sp['values'] = numpy.multiply(sp['values'], BC['dataRows'][0]['values_N'])
hot['values'].update({sp['speciesCode']: [ round(sum(sp['values'][i*4:(i*4+1*4)]),2) for i in range(0,12)] })
hotspots_enhanced.append(hot)
# Filter minimum checklist
hotspots_enhanced = list(filter(lambda x: x['numChecklists'] > 4, hotspots_enhanced))
# Writing the file
tmp = json.dumps(hotspots_enhanced)
tmp = tmp.replace(', ',',').replace(',0.0,',',0,').replace('.0,',',').replace('.0]',']')
text_file = open(re+"/hotspots_enhanced.json", "w")
text_file.write(tmp)
text_file.close()
## Export as xls
ct = [8];
hotspots_table=hotspots_enhanced
for h in hotspots_table:
h['values_N'] = sum([h['values_N'][i] for i in ct])
for v in h['values']:
h[v] = sum([h['values'][v][i] for i in ct])
h.pop("values", None)
keys = ['locId', 'locName', 'countryCode', 'subnational1Code', 'subnational2Code', 'lat', 'lng', 'latestObsDt', 'numSpeciesAllTime', 'numSpecies', 'numChecklists'];
keys.extend([sp['speciesCode'] for sp in species])
with open(re+'/hotspots_table.csv', 'w', newline='') as output_file:
dict_writer = csv.DictWriter(output_file,keys,extrasaction='ignore')
dict_writer.writeheader()
dict_writer.writerows(hotspots_table)
####################################################
# PART 3: Export species list
# Keep only species
species = list(filter(lambda x: x['category'] in 'species', species))
# Remove unsued field
for sp in species:
sp.pop('commonName', None)
sp.pop('reportAsSpeciesCode', None)
sp.pop('reportAsCategory', None)
sp.pop('showSpeciesPageLink', None)
sp.pop('category', None)
# Add Endemic or near-endemic species
EN = ["Jackson's Francolin","Williams's Lark","Taita Apalis","Tana River Cisticola","Aberdare Cisticola","Kikuyu White-eye","Taita White-eye","Hinde's Pied-Babbler","Taita Thrush","Clarke's Weaver","Sharpe's Longclaw"]
EN_ssp = ["White-headed Barbet","Heuglin's White-eye","Violet Woodhoopoe","Turner's Eremomela"]
NE = ["Friedmann's Lark","Golden-winged Sunbird","Donaldson-Smith's Sparrow-Weaver","Parrot-billed Sparrow","Fire-fronted Bishop","Parrot-billed Sparrow","Sokoke Pipit"]
INT = ["House Crow","House Sparrow","Yellow-collared Lovebird"]
END = ["Turner's Eremomela","Grey Crowned-Crane","Madagascar Pond Heron","Egyptian Vulture","White-headed Vulture","Lappet-faced Vulture","Hooded Vulture","White-backed Vulture","Rüppell's Griffon","Steppe Eagle","Sokoke Scops Owl","Saker Falcon","Turner's Eremomela","Taita Apalis","Basra Reed Warbler","Taita White-eye","Spotted Ground-Thrush","Taita Thrush","Amani Sunbird","Clarke's Weaver","Sokoke Pipit","Sharpe's Longclaw"]
for sp in species:
for i in EN:
if sp['displayName'] == i:
sp['class'] = ['EN']
print(sp)
for i in EN_ssp:
if sp['displayName'] == i:
try:
sp['class'].append('EN_ssp')
except :
sp['class'] = ['EN_ssp']
print(sp)
for i in NE:
if sp['displayName'] == i:
try:
sp['class'].append('NE')
except :
sp['class'] = ['NE']
print(sp)
for i in INT:
if sp['displayName'] == i:
try:
sp['class'].append('INT')
except :
sp['class'] = ['INT']
print(sp)
for i in END:
if sp['displayName'] == i:
try:
sp['class'].append('END')
except :
sp['class'] = ['END']
print(sp)
# Keep only field with specie in the data
tmp = []
for hot in hotspots_enhanced:
tmp += list(hot['values'].keys())
myset = set(tmp)
species_export = list(filter(lambda x: x['speciesCode'] in myset, species))
# Export
with open(re+'/species.json', 'w') as f:
json.dump(species_export, f)
with open(re+'/species_table.csv', 'w', newline='') as output_file:
dict_writer = csv.DictWriter(output_file,[*species_export[0]],extrasaction='ignore')
dict_writer.writeheader()
dict_writer.writerows(species_export)