Refactor the Linker, Ranker, Recogniser and Pipeline

.github/workflows/ci.yml

@@ -53,4 +53,4 @@ jobs:
 
       - name: Test with pytest
         run: |
-          poetry run pytest
+          poetry run pytest tests

.github/workflows/docs.yml

@@ -1,67 +1,29 @@
-name: Docs2Pages
-
-on: [pull_request]
-
+name: docs2pages 
+on:
+  push:
+    branches:
+      - dev
+      - main
+permissions:
+  contents: write
 jobs:
-  run:
+  deploy:
     runs-on: ubuntu-latest
-
-    strategy:
-      matrix:
-        os: [ubuntu-latest]
-        python-version: [3.9]
-      fail-fast: false
-
     steps:
-      - uses: actions/checkout@v2
-        with:
-          fetch-depth: 2
-
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v1
-        with:
-          python-version: ${{ matrix.python-version }}
-
-      - name: Bootstrap poetry
-        shell: bash
-        run: |
-          python -m ensurepip
-          python -m pip install --upgrade pip
-          python -m pip install poetry
-
-      - name: Configure poetry
-        shell: bash
+      - uses: actions/checkout@v4
+      - name: Configure Git Credentials
         run: |
-          python -m poetry config virtualenvs.in-project true
-
-      # - name: Set up cache
-      #   uses: actions/cache@v2
-      #   id: cache
-      #   with:
-      #     path: .venv
-      #     key: venv-${{ runner.os }}-${{ steps.full-python-version.outputs.version }}-${{ hashFiles('**/poetry.lock') }}
-
-      # - name: Ensure cache is healthy
-      #   if: steps.cache.outputs.cache-hit == 'true'
-      #   shell: bash
-      #   run: timeout 10s python -m poetry run pip --version || rm -rf .venv
-
-      - name: Install dependencies
-        shell: bash
-        run: |
-          python -m poetry install --with docs
-
-      - name: Build documentation
-        run: |
-          mkdir gh-pages
-          touch gh-pages/.nojekyll
-          cd docs/
-          poetry run sphinx-build -b html source _build
-          cp -r _build/* ../gh-pages/
-
-      - name: Deploy documentation
-        # if: ${{ github.event_name == 'push' }}
-        uses: JamesIves/[email protected]
+          git config user.name github-actions[bot]
+          git config user.email 41898282+github-actions[bot]@users.noreply.github.com
+      - uses: actions/setup-python@v5
+        with:
+          python-version: 3.x
+      - run: echo "cache_id=$(date --utc '+%V')" >> $GITHUB_ENV 
+      - uses: actions/cache@v4
         with:
-          branch: gh-pages
-          folder: gh-pages
+          key: mkdocs-material-${{ env.cache_id }}
+          path: .cache
+          restore-keys: |
+            mkdocs-material-
+      - run: pip install mkdocs-material mkdocs-material-extensions
+      - run: mkdocs gh-deploy --force

.gitignore

@@ -55,6 +55,9 @@ coverage.xml
 *.mo
 *.pot
 
+# OS stuff:
+.DS_Store
+
 # Django stuff:
 *.log
 local_settings.py
@@ -150,4 +153,4 @@ preprocessing/toponymmatching/experiments/
 
 # Docs
 _build
-test.ipynb
+test.ipynb

app/app_template.py

@@ -1,35 +1,45 @@
+import importlib
 import os
 import sys
 import time
 from pathlib import Path
-from typing import Union, Optional, List
+from typing import List, Optional, Union
 
 import uvicorn
 from fastapi import FastAPI, Request
 from pydantic import BaseModel
 
-from config import CONFIG as pipeline_config
-
 from t_res.geoparser import pipeline
+from t_res.utils.dataclasses import SentenceMentions, Candidates
+
+os.environ["APP_CONFIG_NAME"] = "t-res_deezy_reldisamb-wpubl-wmtops"
+
+config_mod = importlib.import_module(
+    ".t-res_deezy_reldisamb-wpubl-wmtops", "app.configs"
+)
+pipeline_config = config_mod.CONFIG
 
 geoparser = pipeline.Pipeline(**pipeline_config)
 
 
-class APIQuery(BaseModel):
+class RecognitionAPIQuery(BaseModel):
     text: str
-    place: Optional[Union[str, None]] = None
-    place_wqid: Optional[Union[str, None]] = None
 
 
 class CandidatesAPIQuery(BaseModel):
-    toponyms: List[dict]
+    sentence_mentions: List[dict]
+    place_of_pub_wqid: Optional[str] = None
+    place_of_pub: Optional[str] = None
 
 
 class DisambiguationAPIQuery(BaseModel):
-    dataset: List[dict]
-    wk_cands: dict
-    place: Optional[Union[str, None]] = None
-    place_wqid: Optional[Union[str, None]] = None
+    candidates: dict
+
+
+class PipelineAPIQuery(BaseModel):
+    text: str
+    place_of_pub_wqid: Optional[str] = None
+    place_of_pub: Optional[str] = None
 
 
 app_config_name = os.environ["APP_CONFIG_NAME"]
@@ -38,74 +48,61 @@ class DisambiguationAPIQuery(BaseModel):
 
 @app.get("/")
 async def read_root(request: Request):
-    return {"Welcome to T-Res!": request.app.title}
-
-
-@app.get("/test")
-async def test_pipeline():
-    resolved = geoparser.run_sentence(
-        "Harvey, from London;Thomas and Elizabeth, Barnett.",
-        place="Manchester",
-        place_wqid="Q18125",
-    )
-
-    return resolved
-
-
-@app.get("/resolve_sentence")
-async def run_sentence(api_query: APIQuery, request_id: Union[str, None] = None):
-    place = "" if api_query.place is None else api_query.place
-    place_wqid = "" if api_query.place_wqid is None else api_query.place_wqid
-    resolved = geoparser.run_sentence(
-        api_query.text, place=place, place_wqid=place_wqid
-    )
-
-    return resolved
-
-
-@app.get("/resolve_full_text")
-async def run_text(api_query: APIQuery):
-
-    place = "" if api_query.place is None else api_query.place
-    place_wqid = "" if api_query.place_wqid is None else api_query.place_wqid
-    resolved = geoparser.run_text(api_query.text, place=place, place_wqid=place_wqid)
-
-    return resolved
-
+    return {
+        "Title": request.app.title,
+        "request.url": request.url,
+        "request.query_params": request.query_params,
+        "root_path": request.scope.get("root_path"),
+        "request.client": request.client,
+        "hostname": os.uname()[1],
+        "worker_id": os.getpid(),
+    }
 
 @app.get("/run_ner")
-async def run_ner(api_query: APIQuery):
-
-    place = "" if api_query.place is None else api_query.place
-    place_wqid = "" if api_query.place_wqid is None else api_query.place_wqid
+async def run_ner(api_query: RecognitionAPIQuery):
     ner_output = geoparser.run_text_recognition(
-        api_query.text, place=place, place_wqid=place_wqid
+        api_query.text
     )
-
     return ner_output
 
-
 @app.get("/run_candidate_selection")
 async def run_candidate_selection(cand_api_query: CandidatesAPIQuery):
-
-    wk_cands = geoparser.run_candidate_selection(cand_api_query.toponyms)
-    return wk_cands
-
+    sentence_mentions = SentenceMentions.from_json(cand_api_query.sentence_mentions)
+    candidates = geoparser.run_candidate_selection(
+        sentence_mentions,
+        place_of_pub_wqid=cand_api_query.place_of_pub_wqid,
+        place_of_pub=cand_api_query.place_of_pub,
+        )
+    return candidates
 
 @app.get("/run_disambiguation")
 async def run_disambiguation(api_query: DisambiguationAPIQuery):
-    place = "" if api_query.place is None else api_query.place
-    place_wqid = "" if api_query.place_wqid is None else api_query.place_wqid
-    disamb_output = geoparser.run_disambiguation(
-        api_query.dataset, api_query.wk_cands, place, place_wqid
+    candidates = Candidates.from_dict(api_query.candidates)
+    predictions = geoparser.run_disambiguation(candidates)
+    return predictions
+
+@app.get("/run_pipeline")
+async def run_pipeline(api_query: PipelineAPIQuery):
+    predictions = geoparser.run(
+        text=api_query.text,
+        place_of_pub_wqid=api_query.place_of_pub_wqid,
+        place_of_pub=api_query.place_of_pub,
     )
-    return disamb_output
+    return predictions
 
+@app.get("/test")
+async def test_pipeline():
+    predictions = geoparser.run(
+        "Harvey, from London;Thomas and Elizabeth, Barnett.",
+        place_of_pub_wqid="Q18125",
+        place_of_pub="Manchester",
+    )
+    return predictions
 
 @app.get("/health")
 async def healthcheck():
     return {"status": "ok"}
 
-
 if __name__ == "__main__":
-    uvicorn.run(app, host="0.0.0.0", port=8000)
+    # poetry run uvicorn app.run_local_app:app --port 8123
+    uvicorn.run(app, host="0.0.0.0", port=8123)

app/app_template_old.py

@@ -0,0 +1,111 @@
+import os
+import sys
+import time
+from pathlib import Path
+from typing import Union, Optional, List
+
+import uvicorn
+from fastapi import FastAPI, Request
+from pydantic import BaseModel
+
+from config import CONFIG as pipeline_config
+
+from t_res.geoparser import pipeline
+
+geoparser = pipeline.Pipeline(**pipeline_config)
+
+
+class APIQuery(BaseModel):
+    text: str
+    place: Optional[Union[str, None]] = None
+    place_wqid: Optional[Union[str, None]] = None
+
+
+class CandidatesAPIQuery(BaseModel):
+    toponyms: List[dict]
+
+
+class DisambiguationAPIQuery(BaseModel):
+    dataset: List[dict]
+    wk_cands: dict
+    place: Optional[Union[str, None]] = None
+    place_wqid: Optional[Union[str, None]] = None
+
+
+app_config_name = os.environ["APP_CONFIG_NAME"]
+app = FastAPI(title=f"Toponym Resolution Pipeline API ({app_config_name})")
+
+
+@app.get("/")
+async def read_root(request: Request):
+    return {"Welcome to T-Res!": request.app.title}
+
+
+@app.get("/test")
+async def test_pipeline():
+    resolved = geoparser.run_sentence_deprecated(
+        "Harvey, from London;Thomas and Elizabeth, Barnett.",
+        place="Manchester",
+        place_wqid="Q18125",
+    )
+
+    return resolved
+
+
+@app.get("/resolve_sentence")
+async def run_sentence(api_query: APIQuery, request_id: Union[str, None] = None):
+    place = "" if api_query.place is None else api_query.place
+    place_wqid = "" if api_query.place_wqid is None else api_query.place_wqid
+    resolved = geoparser.run_sentence_deprecated(
+        api_query.text, place=place, place_wqid=place_wqid
+    )
+
+    return resolved
+
+
+@app.get("/resolve_full_text")
+async def run_text(api_query: APIQuery):
+
+    place = "" if api_query.place is None else api_query.place
+    place_wqid = "" if api_query.place_wqid is None else api_query.place_wqid
+    resolved = geoparser.run_text_deprecated(api_query.text, place=place, place_wqid=place_wqid)
+
+    return resolved
+
+
+@app.get("/run_ner")
+async def run_ner(api_query: APIQuery):
+
+    place = "" if api_query.place is None else api_query.place
+    place_wqid = "" if api_query.place_wqid is None else api_query.place_wqid
+    ner_output = geoparser.run_text_recognition_deprecated(
+        api_query.text, place=place, place_wqid=place_wqid
+    )
+
+    return ner_output
+
+
+@app.get("/run_candidate_selection")
+async def run_candidate_selection(cand_api_query: CandidatesAPIQuery):
+
+    wk_cands = geoparser.run_candidate_selection_deprecated(cand_api_query.toponyms)
+    return wk_cands
+
+
+@app.get("/run_disambiguation")
+async def run_disambiguation(api_query: DisambiguationAPIQuery):
+    place = "" if api_query.place is None else api_query.place
+    place_wqid = "" if api_query.place_wqid is None else api_query.place_wqid
+    disamb_output = geoparser.run_disambiguation_deprecated(
+        api_query.dataset, api_query.wk_cands, place, place_wqid
+    )
+    return disamb_output
+
+
+@app.get("/health")
+async def healthcheck():
+    return {"status": "ok"}
+
+
+if __name__ == "__main__":
+    uvicorn.run(app, host="0.0.0.0", port=8000)