Merge pull request #78 from max-derous-mapotempo/feature/geocoder-her…

…e-v7-upgrade-with-docker-setup

Geocoder HERE v7 upgrade with docker setup

Gemfile

@@ -1,8 +1,7 @@
 source 'https://rubygems.org'
 ruby '~> 2.7.0'
 
-gem 'rack'
-gem 'rackup'
+gem 'rack', '2.2.8'
 gem 'rake'
 gem 'puma'
 gem 'rack-cors'
@@ -17,15 +16,16 @@ gem 'grape-swagger-entity'
 gem 'rack-contrib'
 gem 'rest-client'
 gem 'border_patrol'
-gem 'activesupport'
-gem 'actionpack'
+gem 'activesupport', '6.1.7.6'
+gem 'actionpack', '6.1.7.6'
 
 gem 'grape-erb'
-gem 'geocoder', git: 'https://github.com/Mapotempo/geocoder.git'
+gem 'geocoder', '>= 1.8' # For Geocoder Here v7 compatibility
 gem 'sqlite3', '< 1.5'
 gem 'redis-activesupport'
 
 gem 'sentry-raven'
+gem 'nokogiri', '1.14'
 
 group :test do
   gem 'rack-test'

Gemfile.lock

@@ -1,53 +1,37 @@
-GIT
-  remote: https://github.com/Mapotempo/geocoder.git
-  revision: c08ea9182060268c311dcad8fe910c364715acc5
-  specs:
-    geocoder (1.5.1)
-
 GEM
   remote: https://rubygems.org/
   specs:
-    actionpack (7.1.3)
-      actionview (= 7.1.3)
-      activesupport (= 7.1.3)
-      nokogiri (>= 1.8.5)
-      racc
-      rack (>= 2.2.4)
-      rack-session (>= 1.0.1)
+    actionpack (6.1.7.6)
+      actionview (= 6.1.7.6)
+      activesupport (= 6.1.7.6)
+      rack (~> 2.0, >= 2.0.9)
       rack-test (>= 0.6.3)
-      rails-dom-testing (~> 2.2)
-      rails-html-sanitizer (~> 1.6)
-    actionview (7.1.3)
-      activesupport (= 7.1.3)
+      rails-dom-testing (~> 2.0)
+      rails-html-sanitizer (~> 1.0, >= 1.2.0)
+    actionview (6.1.7.6)
+      activesupport (= 6.1.7.6)
       builder (~> 3.1)
-      erubi (~> 1.11)
-      rails-dom-testing (~> 2.2)
-      rails-html-sanitizer (~> 1.6)
-    activesupport (7.1.3)
-      base64
-      bigdecimal
+      erubi (~> 1.4)
+      rails-dom-testing (~> 2.0)
+      rails-html-sanitizer (~> 1.1, >= 1.2.0)
+    activesupport (6.1.7.6)
       concurrent-ruby (~> 1.0, >= 1.0.2)
-      connection_pool (>= 2.2.5)
-      drb
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
-      mutex_m
       tzinfo (~> 2.0)
+      zeitwerk (~> 2.3)
     ansi (1.5.0)
     ast (2.4.2)
     base64 (0.2.0)
-    bigdecimal (3.1.6)
     border_patrol (0.2.1)
       nokogiri (>= 1.4.3.1)
     builder (3.2.4)
     byebug (11.1.3)
     concurrent-ruby (1.2.3)
-    connection_pool (2.4.1)
     crass (1.0.6)
+    csv (3.3.0)
     docile (1.4.0)
     domain_name (0.6.20240107)
-    drb (2.2.0)
-      ruby2_keywords
     dry-core (1.0.0)
       concurrent-ruby (~> 1.0)
       zeitwerk (~> 2.6)
@@ -71,6 +55,9 @@ GEM
       faraday-net_http (>= 2.0, < 3.1)
       ruby2_keywords (>= 0.0.4)
     faraday-net_http (3.0.2)
+    geocoder (1.8.3)
+      base64 (>= 0.1.0)
+      csv (>= 3.0.0)
     grape (1.8.0)
       activesupport (>= 5)
       builder
@@ -107,7 +94,7 @@ GEM
     mime-types (3.5.2)
       mime-types-data (~> 3.2015)
     mime-types-data (3.2024.0206)
-    mini_portile2 (2.8.5)
+    mini_portile2 (2.8.7)
     minitest (5.22.1)
     minitest-focus (1.4.0)
       minitest (>= 4, < 6)
@@ -121,11 +108,10 @@ GEM
       ruby2_keywords (~> 0.0.1)
     mustermann-grape (1.0.2)
       mustermann (>= 1.0.0)
-    mutex_m (0.2.0)
     netrc (0.11.0)
     nio4r (2.7.0)
-    nokogiri (1.15.5)
-      mini_portile2 (~> 2.8.2)
+    nokogiri (1.14.0)
+      mini_portile2 (~> 2.8.0)
       racc (~> 1.4)
     parallel (1.24.0)
     parser (3.3.0.5)
@@ -134,7 +120,7 @@ GEM
     puma (6.4.2)
       nio4r (~> 2.0)
     racc (1.7.3)
-    rack (3.0.9)
+    rack (2.2.8)
     rack-accept (0.4.5)
       rack (>= 0.4)
     rack-contrib (2.4.0)
@@ -143,13 +129,8 @@ GEM
       rack (>= 2.0.0)
     rack-server-pages (0.1.0)
       rack
-    rack-session (2.0.0)
-      rack (>= 3.0.0)
     rack-test (2.1.0)
       rack (>= 1.3)
-    rackup (2.1.0)
-      rack (>= 3)
-      webrick (~> 1.8)
     rails-dom-testing (2.2.0)
       activesupport (>= 5.0.0)
       minitest
@@ -200,19 +181,18 @@ GEM
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
     unicode-display_width (2.5.0)
-    webrick (1.8.1)
     zeitwerk (2.6.13)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
-  actionpack
-  activesupport
+  actionpack (= 6.1.7.6)
+  activesupport (= 6.1.7.6)
   border_patrol
   byebug
   fakeredis
-  geocoder!
+  geocoder (>= 1.8)
   grape
   grape-entity
   grape-erb
@@ -222,13 +202,13 @@ DEPENDENCIES
   minitest
   minitest-focus
   minitest-reporters
+  nokogiri (= 1.14)
   puma
-  rack
+  rack (= 2.2.8)
   rack-contrib
   rack-cors
   rack-server-pages
   rack-test
-  rackup
   rake
   redis-activesupport
   rest-client

config/environments/development.rb

@@ -38,7 +38,7 @@ module GeocoderWrapper
   Geocoder::Configuration.always_raise = :all
   CACHE = CacheManager.new(ActiveSupport::Cache::FileStore.new(File.join(Dir.tmpdir, 'geocoder'), namespace: 'geocoder', expires_in: 60*60*24*1))
 
-  ADDOK_FRA = Wrappers::Addok.new(CACHE, 'localhost:7878', 'France', 'poly/france.kml')
+  ADDOK_FRA = Wrappers::Addok.new(CACHE, 'host.docker.internal:7878', 'France', 'poly/france.kml', PointInPolygon.new('./poly/france.sqlite'))
   OPENCAGEDATA = Wrappers::RubyGeocoderOpencagedata.new(CACHE)
   HERE = Wrappers::RubyGeocoderHere.new(CACHE)
   ESRI = Wrappers::Esri.new(nil, nil, CACHE)
@@ -77,7 +77,7 @@ module GeocoderWrapper
     },
     ruby_geocode: {
       # Set the appropriate authentication if required
-      here: ['APP_ID', 'APP_CODE'],
+      here: 'API_KEY',
       opencagedata: 'API_KEY'
     },
     addok_endpoint: '/search',

config/environments/production.rb

@@ -35,7 +35,7 @@ module GeocoderWrapper
   ActiveSupport::Cache.lookup_store :redis_store
   CACHE = CacheManager.new(ActiveSupport::Cache::RedisStore.new(host: ENV['REDIS_HOST'] || 'localhost', namespace: 'geocoder', expires_in: 60 * 60 * 24 * 1, raise_errors: true))
 
-  ADDOK_FR = Wrappers::Addok.new(CACHE, "#{ENV['ADDOK_FR_HOST'] || 'http://addok-fr'}:#{ENV['ADDOK_FR_PORT'] || '7878'}", 'France', 'poly/france.kml')
+  ADDOK_FR = Wrappers::Addok.new(CACHE, "#{ENV['ADDOK_FR_HOST'] || 'http://addok-fr'}:#{ENV['ADDOK_FR_PORT'] || '7878'}", 'France', 'poly/france.kml', PointInPolygon.new('./poly/france.sqlite'))
   ADDOK_LU = Wrappers::Addok.new(CACHE, "#{ENV['ADDOK_LU_HOST'] || 'http://addok-lu'}:#{ENV['ADDOK_LU_PORT'] || '7878'}", 'Luxemburg', 'poly/luxemburg.kml', PointInPolygon.new('./poly/luxemburg.sqlite'))
   HERE = Wrappers::RubyGeocoderHere.new(CACHE)
   DEMO = Wrappers::Demo.new(CACHE)
@@ -75,7 +75,7 @@ module GeocoderWrapper
     },
     ruby_geocode: {
       # Set the appropriate authentication if required
-      here: ['APP_ID', 'APP_CODE'],
+      here: 'API_KEY',
       opencagedata: 'API_KEY'
     },
     addok_endpoint: '/search2steps',

config/environments/test.rb

@@ -77,7 +77,7 @@ module GeocoderWrapper
     },
     ruby_geocode: {
       # Set the appropriate authentication if required
-      here: ['APP_ID', 'APP_CODE'],
+      here: 'API_KEY',
       opencagedata: 'API_KEY'
     },
     addok_endpoint: '/search',

docker-compose.yml

@@ -7,6 +7,20 @@ x-default-redis: &default-redis
     - ./redis:/data
 
 services:
+  api:
+    build:
+      context: .
+      dockerfile: docker/dev/Dockerfile
+    image: dev.example.com/mapotempo/geocoder-api
+    volumes:
+      - ${APP_HOME:-.}/poly:/srv/app/poly
+    ports:
+      - 8558:8558
+    environment:
+      PASSENGER_APP_ENV: test
+      REDIS_COUNT_HOST: redis-count
+      REDIS_HOST: redis-cache
+
   redis-cache:
     <<: *default-redis
     hostname: redis-cache

docker/README.md

@@ -8,6 +8,13 @@ mkdir -p ./docker/redis
 docker stack deploy -c ./docker-compose.yml ${PROJECT_NAME}
 ```
 
+## Quick start
+
+```
+docker compose -f docker-compose-build.yml build
+docker compose up -d
+```
+
 ## Data build
 
 After the first deployment, you need to initialize Addok database.

docker/addok/Dockerfile

@@ -15,6 +15,8 @@ ENV ADDOK_USAGE_NAME_BAN_FR_VERSION 1.0.0
 ENV ADDOK_USAGE_NAME_LU_VERSION 1.1.0
 ENV ADDOK_SCORE_PREPROCESSSED_QUERY_VERSION 1.0.0
 
+RUN echo "deb http://archive.debian.org/debian stretch main" > /etc/apt/sources.list
+
 RUN apt-get -q update && \
     apt-get -qy install redis-tools jq && \
     \

docker/dev/Dockerfile

@@ -0,0 +1,42 @@
+# 1.0.13 is the latest version containing bundler 2
+FROM phusion/passenger-ruby27:2.5.1
+
+VOLUME /srv/app/poly
+
+LABEL maintainer="Mapotempo <[email protected]>"
+
+ADD . /srv/app
+WORKDIR /srv/app
+
+# Install app
+ENV REDIS_HOST redis-cache
+
+RUN apt-get update && \
+  apt-get install -y git build-essential zlib1g-dev gdal-bin \
+          zlib1g libsqlite3-mod-spatialite libsqlite3-dev \
+          libspatialite-dev libnginx-mod-http-headers-more-filter && \
+  gem install bundler --version 2 && \
+  bundle install --path vendor/bundle --full-index -j $(nproc)
+
+USER root
+
+# Enable Nginx and Passenger
+RUN rm -f /etc/service/nginx/down
+RUN rm /etc/nginx/sites-enabled/default
+
+# Enable app
+ADD docker/env.d/* /etc/nginx/main.d/
+ADD docker/snippets/* /etc/nginx/snippets/
+ADD docker/webapp.conf /etc/nginx/sites-enabled/webapp.conf
+
+# Cleanup Debian packages
+RUN apt-get remove -y git build-essential zlib1g-dev gdal-bin \
+            libsqlite3-dev libspatialite-dev && \
+  apt-get autoremove -y && \
+  apt-get clean && \
+  echo -n > /var/lib/apt/extended_states && \
+  rm -rf /tmp/* /var/lib/apt/lists/* /var/tmp/*
+
+EXPOSE 8558
+
+CMD ["bundle", "exec", "puma", "-p", "8558"]

test/api/v01/bulk_test.rb

@@ -160,10 +160,10 @@ def test_use_quotas
     ]}
     assert_equal 429, last_response.status
     assert JSON.parse(last_response.body)['message'].include?('Too many monthly requests')
-    assert_equal({ "content-type" => "application/json; charset=UTF-8",
-                   "x-ratelimit-limit" => 2,
-                   "x-ratelimit-remaining" => 0,
-                   "x-ratelimit-reset" => Time.now.utc.to_date.next_month.to_time.to_i }, last_response.headers)
+    assert_equal({ "Content-Type" => "application/json; charset=UTF-8",
+                   "X-RateLimit-Limit" => 2,
+                   "X-RateLimit-Remaining" => 0,
+                   "X-RateLimit-Reset" => Time.now.utc.to_date.next_month.to_time.to_i }, last_response.headers)
   end
 
   def test_should_geocode_when_at_least_one_of_query_postcode_city_street

test/api/v01/unitary_test.rb

@@ -142,10 +142,10 @@ def test_use_quotas
     patch '/0.1/geocode', {api_key: 'bulk_limit', query: 'Place Pey Berland, Bordeaux', country: 'demo'}
     assert_equal 429, last_response.status
     assert JSON.parse(last_response.body)['message'].include?('Too many daily requests')
-    assert_equal({ "content-type" => "application/json; charset=UTF-8",
-                   "x-ratelimit-limit" => 1,
-                   "x-ratelimit-remaining" => 0,
-                   "x-ratelimit-reset" => Time.now.utc.to_date.next_day.to_time.to_i }, last_response.headers)
+    assert_equal({ "Content-Type" => "application/json; charset=UTF-8",
+                   "X-RateLimit-Limit" => 1,
+                   "X-RateLimit-Remaining" => 0,
+                   "X-RateLimit-Reset" => Time.now.utc.to_date.next_day.to_time.to_i }, last_response.headers)
   end
 
   def test_nil_quotas