Utilize spatial queries to retrieve gyms within SCAN_AREA on raidscan init

database.py

@@ -4,6 +4,7 @@
 import math
 import datetime
 import time
+import decimal
 from sqlalchemy import create_engine, Column, Boolean, Integer, String, Float, SmallInteger, \
         BigInteger, ForeignKey, Index, UniqueConstraint, \
         create_engine, cast, func, desc, asc, desc, and_, exists
@@ -13,14 +14,19 @@
 from sqlalchemy.orm.exc import NoResultFound
 from logging import basicConfig, getLogger, FileHandler, StreamHandler, DEBUG, INFO, ERROR, Formatter
 from geopy.distance import vincenty
+from shapely.geometry import mapping
 from sys import argv
 import importlib
+
 LOG = getLogger('')
 
 if len(argv) >= 2:
     config = importlib.import_module(str(argv[1]))
 else:
     config = importlib.import_module('config')
+
+ctx = decimal.Context()
+ctx.prec = 20
 
 class DBCacheFortIdsWithinRange:
 
@@ -452,6 +458,27 @@ def get_device_location_history(session, near, uuid):
     device_location = session.query(DeviceLocationHistory).filter(DeviceLocationHistory.device_uuid == uuid).filter(DeviceLocationHistory.timestamp <= near).order_by(DeviceLocationHistory.timestamp.desc()).first()
     session.commit()
     return device_location
+
+def get_forts_inside_scan_area(session):
+    forts = []
+    poly_coords = ''
+
+    # Convert scan area to spatial coords string
+    mapped_polygon = mapping(config.SCAN_AREA)
+    for poly_coord in mapped_polygon['coordinates'][0]:
+        poly_coords = poly_coords + float_to_str(poly_coord[1]) + ' ' + float_to_str(poly_coord[0]) + ','
+
+    if poly_coords != '':
+        # Strip trailing comma
+        poly_coords = poly_coords[:-1]
+
+    results = session.execute('SELECT id, lat, lon FROM forts WHERE lat IS NOT NULL AND ST_CONTAINS(ST_GEOMFROMTEXT(\'POLYGON((' + poly_coords + '))\'), ST_POINTFROMTEXT(CONCAT(\'POINT(\', lon, \' \', lat, \')\')))')
+    session.commit()
+
+    for row in results:
+        forts.append(Fort(id=row[0],lat=row[1],lon=row[2]))
+
+    return forts
 
 def get_fort_ids_within_range(session, forts, range, lat, lon):
 
@@ -476,3 +503,24 @@ def get_fort_ids_within_range(session, forts, range, lat, lon):
     DBCache.fort_ids_within_range.append(cache_object)
 
     return ids
+
+def is_spatial_capable(session):
+    is_capable = False
+
+    if config.DB_ENGINE.startswith('mysql'):
+        is_capable = True
+    elif config.DB_ENGINE.startswith('postgres'):
+        results = session.execute('SELECT extversion FROM pg_catalog.pg_extension WHERE extname=\'postgis\'')
+        session.commit()
+        result = results.fetchone()
+        is_capable = True if result is not None else False
+
+    return is_capable
+
+def float_to_str(f):
+    """
+    Convert the given float to a string,
+    without resorting to scientific notation
+    """
+    d1 = ctx.create_decimal(repr(f))
+    return format(d1, 'f')

downloadfortimg.py

@@ -47,10 +47,10 @@ def main():
     if (config.MAP_START[0] == 0 and config.MAP_START[1] == 0) or (config.MAP_END[0] == 0 and config.MAP_END[1] == 0):
         check_boundary = False
     else:
-        north = max(MAP_START[0], MAP_END[0])
-        south = min(MAP_START[0], MAP_END[0])
-        east = max(MAP_START[1], MAP_END[1])
-        west = min(MAP_START[1], MAP_END[1])
+        north = max(config.MAP_START[0], config.MAP_END[0])
+        south = min(config.MAP_START[0], config.MAP_END[0])
+        east = max(config.MAP_START[1], config.MAP_END[1])
+        west = min(config.MAP_START[1], config.MAP_END[1])
 
     all_forts = [fort for fort in database.get_forts(session)]
     print('{} forts find in database. Start downloading.'.format(len(all_forts)))

raidscan.py

@@ -38,13 +38,24 @@ def __init__(self):
 
             LOG.info('Scan-Area is set! Getting Forts...')
             session = database.Session()
-            all_forts = database.get_forts(session)
+
+            is_spatial_capable = database.is_spatial_capable(session)
+            all_forts = []
+
+            if self.config.SCAN_AREA != 'All' and is_spatial_capable:
+                all_forts = database.get_forts_inside_scan_area(session)
+            else:
+                all_forts = database.get_forts(session)            
 
             all_forts_to_download = []
 
             session2 = database.Session()
 
+            i = 0;
+
             for fort in all_forts:
+                i = i + 1
+                LOG.info('Processing gym {} of {}'.format(i, len(all_forts)))
                 if fort.lat is not None and fort.lon is not None and self.config.SCAN_AREA == 'All':
                     all_forts_to_download.append(fort.id)
                     self.all_forts_inside.append(DBFort(fort.id, fort.lat, fort.lon, 0))