Enhancement #2 address details option

openpoiservice/server/__init__.py

@@ -6,6 +6,7 @@
 from flask_cors import CORS
 from openpoiservice.server.categories.categories import CategoryTools
 from openpoiservice.server.api import api_exceptions
+from openpoiservice.server.db_import.objects import GeocoderSetup
 import yaml
 import os
 import time
@@ -19,6 +20,13 @@
 basedir = os.path.abspath(os.path.dirname(__file__))
 ops_settings = yaml.safe_load(open(os.path.join(basedir, 'ops_settings.yml')))
 
+geocoder = None
+geocode_categories = None
+if ops_settings['geocoder'] is not None:
+    geocoder = GeocoderSetup(list(ops_settings['geocoder'].items())[0]).define_geocoder()
+    geocode_categories = CategoryTools('categories.yml').generate_geocode_categories()
+
+
 if "TESTING" in os.environ:
     ops_settings['provider_parameters']['table_name'] = ops_settings['provider_parameters']['table_name'] + '_test'
 

openpoiservice/server/api/pois_post.yaml

@@ -449,8 +449,6 @@ definitions:
     properties:
       name:
         type: "string"
-      address:
-        type: "string"
       website:
         type: "string"
       opening_hours:

openpoiservice/server/api/query_builder.py

@@ -1,7 +1,7 @@
 # openpoiservice/server/query_builder.py
 
 from openpoiservice.server import db
-from openpoiservice.server import categories_tools, ops_settings
+from openpoiservice.server import categories_tools, ops_settings, geocoder
 import geoalchemy2.functions as geo_func
 from geoalchemy2.types import Geography, Geometry
 from geoalchemy2.elements import WKBElement, WKTElement
@@ -13,6 +13,7 @@
 from sqlalchemy import dialects
 import geojson as geojson
 import logging
+import json
 from timeit import default_timer as timer
 
 logger = logging.getLogger(__name__)
@@ -105,7 +106,8 @@ def request_pois(self):
                        bbox_query.c.geom,
                        keys_agg,
                        values_agg,
-                       categories_agg) \
+                       categories_agg,
+                       bbox_query.c.address) \
                 .order_by(*sortby_group) \
                 .filter(*category_filters) \
                 .filter(*custom_filters) \
@@ -115,14 +117,7 @@ def request_pois(self):
                 .group_by(bbox_query.c.osm_id) \
                 .group_by(bbox_query.c.osm_type) \
                 .group_by(bbox_query.c.geom) \
-                # .all()
-
-            # end = timer()
-            # print(end - start)
-
-            # print(str(pois_query))
-            # for dude in pois_query:
-            # print(dude)
+                .group_by(bbox_query.c.address)
 
             # response as geojson feature collection
             features = self.generate_geojson_features(pois_query, params['limit'])
@@ -147,12 +142,9 @@ def generate_geom_filters(geometry, Pois):
                                              type_coerce(geom_bbox, Geography)), Pois.geom, 0))
 
         elif 'bbox' not in geometry and 'geom' in geometry:
-
             geom = geometry['geom'].wkt
-
             filters.append(  # buffer around geom
-                geo_func.ST_DWithin(geo_func.ST_Buffer(type_coerce(geom, Geography), geometry['buffer']), Pois.geom, 0)
-            )
+                geo_func.ST_DWithin(geo_func.ST_Buffer(type_coerce(geom, Geography), geometry['buffer']), Pois.geom, 0))
 
         return filters, geom
 
@@ -264,12 +256,24 @@ def generate_geojson_features(cls, query, limit):
                 }
             properties["category_ids"] = category_ids_obj
 
+            # Checks if Tags are available
             if q[5][0] is not None:
                 key_values = {}
                 for idx, key in enumerate(q[4]):
                     key_values[key] = q[5][idx]
                 properties["osm_tags"] = key_values
 
+            # Checks if addresses are available
+            try:
+                if q[7] is not None:
+                    address_dict = {}
+                    address_data = json.loads(q[7])
+                    for key, value in address_data.items():
+                        address_dict[key] = value
+                    properties['address'] = address_dict
+            except IndexError:
+                pass
+
             geojson_feature = geojson.Feature(geometry=trimmed_point,
                                               properties=properties)
             geojson_features.append(geojson_feature)

openpoiservice/server/categories/categories.py

@@ -94,3 +94,16 @@ def get_category(self, tags):
                                 categories.append(category_id)
 
         return categories
+
+    def generate_geocode_categories(self):
+
+        geocode_categories = {}
+
+        for category in self.categories_object.items():
+            if 'geocoder' not in category[1]:
+                for child in category[1]['children'].values():
+                    for id in child.values():
+                        geocode_categories[id] = {}
+
+        return geocode_categories
+

openpoiservice/server/categories/categories.yml

@@ -49,6 +49,7 @@ education:
 
 facilities:
     id: 160
+    geocoder: False
     children:
         amenity:
             compressed_air: 161

openpoiservice/server/db_import/models.py

@@ -1,6 +1,6 @@
 # openpoiservice/server/models.py
 
-from openpoiservice.server import db, ops_settings
+from openpoiservice.server import db, ops_settings, geocoder
 from geoalchemy2 import Geography
 import logging
 
@@ -14,7 +14,6 @@ class Pois(db.Model):
     uuid = db.Column(db.LargeBinary, primary_key=True)
     osm_id = db.Column(db.BigInteger, nullable=False, index=True)
     osm_type = db.Column(db.Integer, nullable=False)
-    # address = db.Column(db.Text, nullable=True)
     geom = db.Column(Geography(geometry_type="POINT", srid=4326, spatial_index=True), nullable=False)
 
     tags = db.relationship("Tags", backref='{}'.format(ops_settings['provider_parameters']['table_name']),
@@ -23,6 +22,8 @@ class Pois(db.Model):
     categories = db.relationship("Categories", backref='{}'.format(ops_settings['provider_parameters']['table_name']),
                                  lazy='dynamic')
 
+    address = db.Column(db.String, nullable=True)
+
     def __repr__(self):
         return '<osm id %r>' % self.osm_id
 

openpoiservice/server/db_import/objects.py

@@ -1,9 +1,15 @@
 # openpoiservice/server/poi_entity.py
 
+import json
+import logging
+from geopy.geocoders import get_geocoder_for_service
+# from geopy.extra.rate_limiter import RateLimiter
+
+logger = logging.getLogger(__name__)
 
 class PoiObject(object):
 
-    def __init__(self, uuid, categories, osmid, lat_lng, osm_type):
+    def __init__(self, uuid, categories, osmid, lat_lng, osm_type, address=None):
         self.uuid = uuid
         self.osmid = int(osmid)
         self.type = int(osm_type)
@@ -14,9 +20,7 @@ def __init__(self, uuid, categories, osmid, lat_lng, osm_type):
 
         self.geom = 'SRID={};POINT({} {})'.format(4326, float(lat_lng[0]),
                                                   float(lat_lng[1]))
-
-        # add geocoder connector here...
-        self.address = None
+        self.address = address
 
 
 class TagsObject(object):
@@ -26,3 +30,56 @@ def __init__(self, uuid, osmid, key, value):
         self.osmid = int(osmid)
         self.key = key
         self.value = value
+
+
+class GeocoderSetup(object):
+    """Initialises geocoder"""
+
+    def __init__(self, geocoder_name):
+        self.geocoder_name = geocoder_name
+        self.geocoder = None
+
+    def define_geocoder(self):
+
+        # returns warning if no valid geocoder is provided
+        try:
+            self.geocoder_settings = get_geocoder_for_service(self.geocoder_name[0])
+
+        except Exception as err_geocoder:
+            logger.warning(err_geocoder)
+            return err_geocoder
+
+        # returns warning if no valid geocoder settings are provided
+        try:
+            if self.geocoder_name[1] is not None:
+                self.geocoder = self.geocoder_settings(**self.geocoder_name[1])
+
+            else:
+                self.geocoder = self.geocoder_settings()
+            return self.geocoder
+
+        except Exception as err_parameter:
+            logger.warning(err_parameter)
+            return err_parameter
+
+
+class AddressObject(object):
+
+    def __init__(self, lat_lng, geocoder):
+        self.lat_lng = lat_lng[::-1]
+        self.geocoder = geocoder
+
+    def address_request(self):
+
+        # address_delaied = RateLimiter(self.geocoder.reverse, min_delay_seconds=1)
+        response = self.geocoder.reverse(query=self.lat_lng)
+
+        # Checks if address for location is available
+        if response is not None:
+            return json.dumps(response.raw)
+            # try:
+            #     return json.dumps(response.raw)
+            # except AttributeError:
+            #     return json.dumps(response)
+
+        return None

openpoiservice/server/db_import/parse_osm.py

@@ -1,9 +1,9 @@
 # openpoiservice/server/parse_osm.py
 
 from openpoiservice.server import db
-from openpoiservice.server import categories_tools, ops_settings
+from openpoiservice.server import categories_tools, ops_settings, geocoder, geocode_categories
 from openpoiservice.server.db_import.models import Pois, Tags, Categories
-from openpoiservice.server.db_import.objects import PoiObject, TagsObject
+from openpoiservice.server.db_import.objects import PoiObject, TagsObject, AddressObject
 from openpoiservice.server.utils.decorators import get_size
 import shapely as shapely
 from shapely.geometry import Point, Polygon, LineString, MultiPoint
@@ -179,12 +179,21 @@ def store_poi(self, poi_object):
         """
 
         self.pois_cnt += 1
-        self.poi_objects.append(Pois(
-            uuid=poi_object.uuid,
-            osm_id=poi_object.osmid,
-            osm_type=poi_object.type,
-            geom=poi_object.geom
-        ))
+        if geocoder is not None:
+            self.poi_objects.append(Pois(
+                uuid=poi_object.uuid,
+                osm_id=poi_object.osmid,
+                osm_type=poi_object.type,
+                geom=poi_object.geom,
+                address=poi_object.address
+            ))
+        else:
+            self.poi_objects.append(Pois(
+                uuid=poi_object.uuid,
+                osm_id=poi_object.osmid,
+                osm_type=poi_object.type,
+                geom=poi_object.geom
+            ))
 
         if self.pois_cnt % 1000 == 0:
             logger.info('Pois: {}, tags: {}, categories: {}'.format(self.pois_cnt, self.tags_cnt, self.categories_cnt))
@@ -261,7 +270,13 @@ def create_poi(self, tags, osmid, lat_lng, osm_type, categories=[]):
                     self.tags_object = TagsObject(my_uuid, osmid, tag, value)
                     self.store_tags(self.tags_object)
 
-            self.poi_object = PoiObject(my_uuid, categories, osmid, lat_lng, osm_type)
+            address = None
+            if geocoder is not None and categories[0] in geocode_categories:
+                address = AddressObject(lat_lng, geocoder).address_request()
+                time.sleep(1)
+
+            self.poi_object = PoiObject(my_uuid, categories, osmid, lat_lng, osm_type, address)
+
             self.store_poi(self.poi_object)
 
             for category in categories:

openpoiservice/server/db_import/parser.py

@@ -2,10 +2,9 @@
 
 from openpoiservice.server.db_import.parse_osm import OsmImporter
 from openpoiservice.server.utils.decorators import timeit, processify
-from openpoiservice.server import ops_settings
+from openpoiservice.server import ops_settings, geocoder
 from imposm.parser import OSMParser
 import logging
-import time
 
 # from guppy import hpy
 from collections import deque
@@ -50,6 +49,10 @@ def parse_import(osm_file):
     coords = OSMParser(concurrency=1, coords_callback=osm_importer.parse_coords_for_ways)
     coords.parse(osm_file)
 
+    # Checks if geocoder is provided
+    if geocoder is not None:
+        logger.info('Importing addresses...')
+
     logger.info('Storing remaining pois')
     osm_importer.save_remainder()
 

openpoiservice/server/ops_settings.yml

@@ -16,13 +16,12 @@ concurrent_workers: 4
 # Database parameters
 provider_parameters:
   table_name: ops_planet_pois
-  db_name: gis
-  user_name: gis_admin
+  db_name: opsdb
+  user_name: postgres
   table_schema: public
-  #user_name: admin
-  password: admin
+  password: ops
   host: localhost
-  port: 5434
+  port: 5432
   port_tests: 5432
 # add any additional tags you want to save to the database
 column_mappings:
@@ -40,3 +39,24 @@ column_mappings:
   phone:
   # https://wiki.openstreetmap.org/wiki/Key:website
   website:
+# choose one geopy geocoder and provide all needed parameters - https://geopy.readthedocs.io/
+geocoder:
+#  ArcGIS:
+#    domain: 'geocode.arcgis.com'
+#  pelias:
+#    domain: 'api.geocode.earth'
+#    api_key: ''
+#    timeout: 60
+#  nominatim:
+#    timeout: 60
+#  GeocodeFarm:
+#    timeout: 60
+#  Azure:
+#    subscription_key: ''
+#    domain: 'atlas.microsoft.com'
+#  Baidu:
+#    api_key: ''
+#  BANFrance:
+#    domain: 'api-adresse.data.gouv.fr'
+#  Bing:
+#    api_key: ''