-
Notifications
You must be signed in to change notification settings - Fork 3
/
Copy pathMapFeaturesFunctions.py
212 lines (172 loc) · 8.72 KB
/
MapFeaturesFunctions.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
"""
Created on Tue Oct 27 11:50:33 2020
@author: Santiago
"""
def GetFeatures(bbox, values, client_id, token, layers, filename, path):
import json, requests, os
# create our empty geojson
output = {"type":"FeatureCollection","features":[]}
# define the header that will be passed to the API request--it needs to include the token to authorize access to subscription data
header = {"Authorization" : "Bearer {}".format(token)}
# build the API call
url = ('https://a.mapillary.com/v3/map_features?layers={}&sort_by=key&client_id={}&per_page=500&values={}&bbox={}').format(layers,client_id,values,bbox)
# print the URL so we can preview it
print(url)
# send the API request with the header and no timeout
r = requests.get(url,timeout=None,headers=header)
# if call fails, keeping trying until it succeeds with a 200 status code
while r.status_code != 200:
r = requests.get(url,timeout=None,headers=header)
# get data response as a JSON and count how many features were found
data = r.json()
data_length = len(data['features'])
# print number of features
print(data_length)
# add each feature to the empty GeoJSON we created
for f in data['features']:
output['features'].append(f)
# loop through each new page and continue adding the results to the empty GeoJSON
while data_length == 500:
link = r.links['next']['url']
r = requests.get(link,timeout=None,headers=header)
while r.status_code != 200:
r = requests.get(url,timeout=None,headers=header)
data = r.json()
for f in data['features']:
output['features'].append(f)
# print total number of features found so far
print("Total features: {}".format(len(output['features'])))
# update length of data in last call to see if it still remains at 500 (maximum) indicating a next page
data_length = len(data['features'])
finalfile = "%s.geojson" % filename
with open(os.path.join(path, finalfile), 'w') as outfile:
print('DONE')
json.dump(output, outfile)
"""
Created on Fri Jan 08 2021
@author: gregoriiv
"""
# function returns features from Mapillary API for given shapefile
# --variables-- #
# path - path to shapefile
# fact - factor for disaggregation boundingbox of shapefile in degrees (ex. 0.02)
# values - reference values you'd like here, separate multiple by comma, leave empty quotes to get all values from Mapillary docs
# (ex. 'regulatory--priority-road--g1') https://www.mapillary.com/developer/api-documentation/#traffic-signs
# token - token from Mapillary API for authorized request
# layers - choose 'trafficsigns' or 'points' or 'lines'
# dir_feat - main path to place where catalog with returned files will be saved()
def MapillaryFeaturesFromStudyArea(path, fact, values, client_id, token, layers, dir_feat):
import json
import os
import time
import StudyArea2Bboxes
# create filename for result geojson file from 'values' variable
filename_fin = values
# create result geojson
output_result = {"type": "FeatureCollection", "features": []}
with open(os.path.join(dir_feat, (filename_fin + '.geojson')), 'w') as outfile:
json.dump(output_result, outfile)
# disaggregate bbox of shapefile to grid of bboxes
bboxes = StudyArea2Bboxes.Shp2Bbox(path, fact)
# filename for Mapillary request
filename = 'request'
# iterate through each bbox
for bb in bboxes:
# write geojson file with object for given bbox as request.geojson
GetFeatures(bb, values, client_id, token, layers, filename, dir_feat)
# open request.geojson
with open(os.path.join(dir_feat,(filename + '.geojson'))) as r:
request = json.load(r)
# open result.geojson
with open(os.path.join(dir_feat, (filename_fin +'.geojson'))) as res:
result = json.load(res)
# append features form request.geojson to result.geojson
for f in request['features']:
result['features'].append(f)
# write down new result.geojson
with open(os.path.join(dir_feat, (filename_fin +'.geojson')), 'w') as outfile:
json.dump(result, outfile)
print(filename_fin +'.UPDATED')
# remove request.geojson
os.remove(os.path.join(dir_feat,(filename + '.geojson')))
# sleep for 1 minute!!!
#time.sleep(60) #60sec
return print('DONE')
"""
Created on Sun Jan 24 2021
@author: gregoriiv
"""
# fuction returns geojson with features, set of value been requested for given shapefile
def MapillaryMultiFeaturesRequest(client_id, token):
import json, os
import yaml
import geopandas
import time
#import data from config.yaml
with open('mapil_request_config.yaml') as m:
config = yaml.safe_load(m)
var = config['VARIABLES_SET']
#create directory 'data' if not exists
for area in var:
dir_main = os.path.join('data', area)
if not os.path.exists(dir_main):
os.mkdir(dir_main)
dir_feat = os.path.join(dir_main, 'features')
if not os.path.exists(dir_feat):
os.mkdir(dir_feat)
#set variables from config yaml
fact = var[area]['fact']
path = var[area]['path']
#create metadata file
metadata = ["Study area: " + area +"\n", "Datetime of request: " + time.strftime(r"%Y-%m-%d %H:%M:%S", time.localtime()) +"\n",
"Path to study area shape file: " + path +"\n", "Factor of studyarea bounding box deaggregation: " + str(fact) +"\n", "Feature set: " + "\n"]
#create result files for each group of features with name of group
for v_cat in var[area]['custom_feature_set']:
metadata.append("- " + str(v_cat) + ": " + str(var[area]['custom_feature_set'][v_cat]) +"\n")
if len(var[area]['custom_feature_set'][v_cat]) > 0:
output_val_cat = {"type": "FeatureCollection", "features": []}
with open(os.path.join(dir_feat, (v_cat +'.geojson')), 'w') as outfile:
json.dump(output_val_cat, outfile)
# make request for each feature in each group
for feature in var[area]['custom_feature_set'][v_cat]:
MapillaryFeaturesFromStudyArea(path, fact, feature, client_id, token, v_cat, dir_feat)
# open request.geojson
with open(os.path.join(dir_feat, (feature + '.geojson'))) as r:
request = json.load(r)
# open result.geojson
with open(os.path.join(dir_feat, (v_cat +'.geojson'))) as res:
result = json.load(res)
# append features form request.geojson to result.geojson
for f in request['features']:
result['features'].append(f)
# write down new result.geojson
with open(os.path.join(dir_feat, (v_cat +'.geojson')), 'w') as outfile:
json.dump(result, outfile)
print(v_cat +'.UPDATED')
# remove request.geojson
os.remove(os.path.join(dir_feat, (feature + '.geojson')))
#remove features located outside the study area with geopandas
#open files with features and shapefile
categ_gjson = geopandas.read_file(os.path.join(dir_feat, (v_cat +'.geojson')))
st_area = geopandas.read_file(path + '.shp')
#save SRID for metadata
if ("Features SRID: " + str(categ_gjson.crs) + "\n") not in metadata:
metadata.insert(4, "Features SRID: " + str(categ_gjson.crs) + "\n")
if ("Study area SRID: " + str(st_area.crs) + "\n") not in metadata:
metadata.insert(1, "Study area SRID: " + str(st_area.crs) + "\n")
#create mask for intersected features and filter them
categ_mask = categ_gjson.within(st_area.loc[0,'geometry'])
categ_cut = categ_gjson[categ_mask]
#remove initial geojson file and save cut one
if categ_cut.empty:
metadata.append(" Number of features: 0" + "\n")
pass
else:
os.remove(os.path.join(dir_feat, (v_cat +'.geojson')))
metadata.append(" Number of features: " + str(len(categ_cut)) + "\n")
categ_cut.to_file(os.path.join(dir_feat, (v_cat +'.geojson')), driver='GeoJSON')
else:
metadata.append(" Number of features: 0" + "\n")
#write down metadata file
with open((os.path.join(dir_feat, ('metadata_features.txt'))), 'w') as md:
md.writelines(metadata)