Happy 2022!

.dockerignore

@@ -0,0 +1,7 @@
+venv/
+docs/
+corpora/
+README.md
+deploy.bash
+LICENSE
+*.egg-info

.gitignore

@@ -5,4 +5,6 @@ corpora/RoadFTS5_EU.NDS
 .idea
 __pycache__
 logs
-models/checkpoint
+models/checkpoint
+*.egg-info
+venv/

Dockerfile

@@ -0,0 +1,9 @@
+FROM python:3.6-buster
+
+RUN pip install uwsgi
+
+ADD requirements.txt /ds9/requirements.txt
+RUN cd /ds9 && pip install -r requirements.txt
+
+ADD . /ds9
+RUN cd /ds9 && pip install -e .

README.md

@@ -1,28 +1,41 @@
-deep-spell-v1.0rc
-=================
+Deep-Spell-9
+============
 
-The Deep-Spell-9 project contains implementations of various neural algorithms,
-which enable natural auto-suggestion via deep neural networks.
+## About
 
-## Dependencies
+Deep-Spell-9 enables natural auto-completion and matching of geographic queries via deep neural networks.
 
-Mandatory:
-* python 3.x
-* tensorflow >= 1.3
-* unidecode
-* flask
-* scipy
+![Figure 1](docs/figure.png)
 
-Optional:
-* pygtrie (for FTS5 baseline)
-* hat_trie (for generating Symspell DAWG)
-* DAWG (for generating/loading Symspell DAWG)
+## Web service
 
-## Scripts
+An interactive user interface is provided which allows
+you to run Deep-Spell-9 within an interactive browser UI.
+Use the `service.json` config file to configure the used models.
 
-### `python3 service.py`
+You have to options to launch the service:
 
-Launches the Deep Spell demonstrator web interface from the
-`modules/deepspell_service` flask module. Use the adjacent
-`service.json` config file to set the served port and models,
-among other options.
+1. Via a local Python setup: Clone the repo, and run
+
+   ```bash
+   cd [clone-directory]
+   pip install -e .
+   # Port defaults to 8091
+   ./serve.bash [port]
+   ```
+
+2. Via Docker:
+
+   ```bash
+   docker run --rm -it -p 8091:8091 \
+      "ghcr.io/klebert-engineering/ds9:latest" \
+      "//ds9/serve.bash"
+   ```
+
+   **Note:** The default `service.json` in the image does not
+   provide an FTS database to match a query to ranked results,
+   so you will only be able to see the classifier and completer
+   components in action.
+
+After the service is started, open localhost on the
+specified port in any browser ([localhost:8091](http://localhost:8091)) by default.

VERSION

@@ -0,0 +1 @@
+2022.1

deploy.bash

@@ -0,0 +1,59 @@
+#!/usr/bin/env bash
+
+image_name="ds9"
+version=$(head -n 1 "VERSION")
+push=""
+latest=""
+test=""
+
+while [[ $# -gt 0 ]]; do
+  case $1 in
+    -i|--image-name)
+      image_name=$2
+      shift
+      shift
+      ;;
+    -v|--version)
+      version=$2
+      shift
+      shift
+      ;;
+    -p|--push)
+      push="yes"
+      shift
+      ;;
+    -l|--latest)
+      latest="yes"
+      shift
+      ;;
+    -t|--test)
+      test="yes"
+      shift
+      ;;
+  esac
+done
+
+image_name="ghcr.io/klebert-engineering/$image_name"
+docker build -t "$image_name:$version" .
+
+if [[ -n "$latest" ]]; then
+  echo "Tagging latest."
+  docker tag "$image_name:$version" "$image_name:latest"
+fi
+
+if [[ -n "$push" ]]; then
+  echo "Pushing."
+  docker push "$image_name:$version"
+  if [[ -n "$latest" ]]; then
+    docker push "$image_name:latest"
+  fi
+fi
+
+if [[ -n "$test" ]]; then
+  echo "Running test."
+  docker run --rm -it \
+    -p 8091:8091 \
+    "$image_name:$version" \
+    "//ds9/serve.bash"
+fi
+py

match.py

@@ -35,7 +35,7 @@
 print("""
 =============================================================
   This is the `deepspell` LSTM DNN spell-match demo for NDS
-             (C) 2017 Klebert Engineering GmbH
+               (C) 2017 Klebert Engineering
 =============================================================
 
 Usage:

modules/deepspell_service/templates/index.html

@@ -77,7 +77,6 @@ <h6>Correction</h6>
         </div>
     </div>
     <script>
-        var host = "{{hostname}}";
         var withCorrection = "{{with_correction}}";
         var withFtsLookup = "{{with_ftslookup}}";
         var bestSuggestion = "";

modules/deepspell_service/views.py

@@ -45,7 +45,6 @@ def init(args):
     if args["fts_db"]:
         fts_lookup_db = DSFtsDatabaseConnection(**args["fts_db"])
     featureset = extrapolator_model.featureset
-    hostname = args["hostname"]+":"+str(args["port"])
     lowercase = args["lowercase"]
 
 

predict.py

@@ -22,7 +22,7 @@
 print("""
 =============================================================
 This is the `deepspell` LSTM DNN auto-completion demo for NDS
-             (C) 2017 Klebert Engineering GmbH
+              (C) 2017 Klebert Engineering
 =============================================================
 
 Usage:

requirements.txt

@@ -0,0 +1,5 @@
+tensorflow==1.9
+unidecode
+flask
+scipy
+DAWG

serve.bash

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+
+set -e
+
+export MY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+
+used_server_port="${1:-8091}"
+
+# Determine python virtual environment
+VIRTUAL_ENV_ARG=()
+if [[ -d "$VIRTUAL_ENV" ]]; then
+  VIRTUAL_ENV_ARG=(--virtualenv "$VIRTUAL_ENV")
+fi
+
+# Hand over to UWSGI HTTP server
+cd "$MY_DIR"
+exec uwsgi \
+    "${VIRTUAL_ENV_ARG[@]}" \
+    --processes 4 \
+    --enable-threads \
+    --http ":$used_server_port" \
+    --wsgi-file "service.py" \
+    --callable app

service.json

@@ -1,7 +1,4 @@
 {
-  "port": 5000,
-  "host": "0.0.0.0",
-  "hostname": "localhost",
   "__comment__": [
     "Use `discriminator` to specify the token category discrimination model:",
     "This may be any path pointing to a valid deepspell.DSLstmDiscriminator model, e.g.:",
@@ -29,10 +26,11 @@
   ],
   "discriminator": "models/deepsp_discr-v3_na-lower_lr003_dec50_bat3072_fw128-128_bw128.json",
   "extrapolator": "models/deepsp_extra-v2_na_lr003_dec50_bat3192_128-128-128.json",
-  "corrector": "models/deepsp_spell-v2_na-lower_lr003_dec70_bat2048_emb8_fw128_bw128_co256-256_dein256-256_drop75.json",
-  "corrector_files": "corpora/deepspell_data_north_america_nozip_v2.tsv",
-  "_fts_db": null,
-  "fts_db": {
+  "corrector": "",
+  "fts_db": null,
+  "_corrector": "models/deepsp_spell-v2_na-lower_lr003_dec70_bat2048_emb8_fw128_bw128_co256-256_dein256-256_drop75.json",
+  "_corrector_files": "corpora/deepspell_data_north_america_nozip_v2.tsv",
+  "_fts_db": {
     "path": "corpora/RoadFTS5_USA.nds",
     "fts_table_name": "nameFtsTable",
     "docid_column_name": "namedObjectId",

service.py

@@ -1,15 +1,12 @@
-
 import json
 import codecs
-import os
-import sys
-
-sys.path.append(os.path.dirname(os.path.realpath(__file__)) + "/modules")
-
 import deepspell_service
 
 with codecs.open("service.json") as config_file:
     params = json.load(config_file)
 
 deepspell_service.views.init(params)
-deepspell_service.app.run(host=params["host"], port=params["port"])
+app = deepspell_service.app
+
+if __name__ == "__main__":
+    deepspell_service.app.run()

setup.py

@@ -0,0 +1,24 @@
+import setuptools
+
+with open("requirements.txt", "r") as fh:
+    requirements = fh.readlines()
+
+with open("VERSION", "r") as v:
+    version = v.read().strip()
+
+setuptools.setup(
+    name="deep-spell-9",
+    version=version,
+    author="Klebert Engineering",
+    author_email="[email protected]",
+    description="Neural geographic query processor.",
+    url="https://github.com/klebert-engineering/deep-spell-9",
+    packages=setuptools.find_packages("modules"),
+    package_dir={'': 'modules'},
+    install_requires=[req for req in requirements],
+    classifiers=[
+        "Programming Language :: Python :: 3",
+        "Operating System :: OS Independent",
+    ],
+    python_requires='>=3.6',
+)