Convert from a Heavy Django app to lean and mean Flask API

.gitignore

@@ -30,6 +30,7 @@ wheels/
 #  before PyInstaller builds the exe, so as to inject date/other infos into it.
 *.manifest
 *.spec
+instance
 
 # Installer logs
 pip-log.txt
@@ -243,4 +244,6 @@ secrets/
 *.service
 temp/
 
-.env
+.env
+
+cba_server/run_sections*

MANIFEST.in

@@ -0,0 +1 @@
+.

cba_server/__init__.py

@@ -0,0 +1,31 @@
+import os
+from flask import Flask
+import cba_server.db as db
+import cba_server.api as api
+
+
+def create_app(test_config=None):
+    # create and configure the app
+    app = Flask(__name__, instance_relative_config=True)
+    app.config.from_mapping(
+        SECRET_KEY="dev",
+        DATABASE=os.path.join(app.instance_path, "flaskr.sqlite"),
+    )
+
+    if test_config is None:
+        # load the instance config, if it exists, when not testing
+        app.config.from_pyfile("config.py", silent=True)
+    else:
+        # load the test config if passed in
+        app.config.from_mapping(test_config)
+
+    # ensure the instance folder exists
+    try:
+        os.makedirs(app.instance_path)
+    except OSError:
+        pass
+
+    db.init_app(app)
+    app.register_blueprint(api.bp)
+
+    return app

cba_server/api.py

@@ -0,0 +1,71 @@
+from flask import (
+    Blueprint,
+    flash,
+    g,
+    redirect,
+    render_template,
+    request,
+    url_for,
+    jsonify,
+)
+from werkzeug.exceptions import abort
+import pandas as pd
+import json
+import sys
+import math
+
+from cba_server.db import get_db
+from cba_server.request_logging import log_request, log_response
+from roads_cba_py.section import Section, InvalidSection, parse_section
+from roads_cba_py.cba import CostBenefitAnalysisModel
+from roads_cba_py.utils import flatten, split_on_condition
+from roads_cba_py.config import Config
+
+
+bp = Blueprint("api", __name__)
+
+
+@bp.route("/run_sections", methods=["POST"])
+def run_sections():
+    request_id = log_request(request)
+    config = Config.parse_obj(request.json["config"])
+    assets = request.json["assets"]
+    valid_sections, invalid_sections, stats = split_assets_by_validity(assets)
+
+    cba = CostBenefitAnalysisModel(config)
+    results = [cba.compute_cba_for_section(s) for s in valid_sections]
+    results = sorted(results, key=lambda x: (x.work_year, -x.npv_cost))
+
+    problems = flatten([s.invalid_reason() for s in invalid_sections])
+    invalid_reasons = pd.DataFrame(data=problems, columns=["reason"])
+    invalid_reasons = invalid_reasons["reason"].value_counts().to_dict()
+
+    response = jsonify(
+        {
+            "stats": stats,
+            "invalids": invalid_reasons,
+            "data": [s.dict() for s in results],
+        }
+    )
+    log_response(response, request_id)
+
+    return response
+
+
+@bp.route("/evaluate_assets", methods=["POST"])
+def evaluate_assets():
+    _valid_sections, invalid_sections, stats = split_assets_by_validity(request.json)
+
+    problems = flatten([s.invalid_reason() for s in invalid_sections])
+    invalid_reasons = pd.DataFrame(data=problems, columns=["reason"])
+    invalid_reasons = invalid_reasons["reason"].value_counts().to_dict()
+    invalid_way_ids = [int(s.original_data["orma_way_id"]) for s in invalid_sections]
+    return jsonify({"stats": stats, "invalids": invalid_reasons, "invalid_way_ids": invalid_way_ids})
+
+
+def split_assets_by_validity(data):
+    sections = [parse_section(s) for s in data]
+    valid_sections, invalid_sections = split_on_condition(sections, lambda s: s.invalid_reason() is None)
+
+    stats = {"valid": len(valid_sections), "invalid": len(invalid_sections)}
+    return (valid_sections, invalid_sections, stats)

cba_server/db.py

@@ -0,0 +1,42 @@
+import sqlite3
+
+import click
+from flask import current_app, g
+from flask.cli import with_appcontext
+
+
+def get_db():
+    if 'db' not in g:
+        g.db = sqlite3.connect(
+            current_app.config['DATABASE'],
+            detect_types=sqlite3.PARSE_DECLTYPES
+        )
+        g.db.row_factory = sqlite3.Row
+
+    return g.db
+
+
+def close_db(e=None):
+    db = g.pop('db', None)
+
+    if db is not None:
+        db.close()
+
+def init_db():
+    db = get_db()
+
+    with current_app.open_resource('schema.sql') as f:
+        db.executescript(f.read().decode('utf8'))
+
+
+@click.command('init-db')
+@with_appcontext
+def init_db_command():
+    """Clear the existing data and create new tables."""
+    init_db()
+    click.echo('Initialized the database.')
+
+
+def init_app(app):
+    app.teardown_appcontext(close_db)
+    app.cli.add_command(init_db_command)

cba_server/request_logging.py

@@ -0,0 +1,36 @@
+import os
+import json
+import datetime
+from os.path import join, abspath, dirname
+
+
+def get_log_dir():
+    return abspath(join(dirname(__file__), "..", "log"))
+
+
+def get_request_dir():
+    remote_dir = join(get_log_dir(), "flask_requests")
+    if os.path.isdir(remote_dir):
+        return remote_dir
+    return None
+
+
+def log_request(request):
+    request_dir = get_request_dir()
+    if request_dir is None:
+        return None
+
+    timestamp = datetime.datetime.now().replace(microsecond=0).isoformat()
+    filename = join(request_dir, f"run_sections_request_{timestamp}")
+    with open(filename, "w") as f:
+        f.write(json.dumps(request.json))
+    return timestamp
+
+
+def log_response(response, id):
+    request_dir = get_request_dir()
+    if request_dir is None:
+        return None
+    filename = join(request_dir, f"run_sections_response_{id}")
+    with open(filename, "w") as f:
+        f.write(json.dumps(response.json))

cba_server/schema.sql

@@ -0,0 +1,17 @@
+DROP TABLE IF EXISTS user;
+DROP TABLE IF EXISTS post;
+
+CREATE TABLE user (
+  id INTEGER PRIMARY KEY AUTOINCREMENT,
+  username TEXT UNIQUE NOT NULL,
+  password TEXT NOT NULL
+);
+
+CREATE TABLE post (
+  id INTEGER PRIMARY KEY AUTOINCREMENT,
+  author_id INTEGER NOT NULL,
+  created TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  title TEXT NOT NULL,
+  body TEXT NOT NULL,
+  FOREIGN KEY (author_id) REFERENCES user (id)
+);

cba_server/wsgi.py

@@ -0,0 +1,9 @@
+import sys
+
+sys.path.append(".")
+from cba_server import create_app
+
+app = create_app()
+
+if __name__ == "__main__":
+    app.run()