diff --git a/.github/workflows/basic-tests.yml b/.github/workflows/basic-tests.yml index 5cddd078..62df4a43 100644 --- a/.github/workflows/basic-tests.yml +++ b/.github/workflows/basic-tests.yml @@ -1,6 +1,6 @@ -name: 'Basic (Unit) Tests' +# Install python and node. Run lint and unit tests. -# **What it does**: Setups up python dependencies and runs tests. +name: 'Basic (Unit) Tests' # **Why we have it**: Automatically run tests to ensure code doesn't introduce regressions. # **Who does it impact**: Python small-scale "unit" tests. @@ -22,13 +22,25 @@ jobs: steps: - name: Checkout uses: actions/checkout@v3 + + - name: Install Poetry + run: | + python -m pip install poetry==1.4.1 - name: Install Python3 uses: actions/setup-python@v4 with: - python-version: '3.9.x' - cache: 'pip' - cache-dependency-path: requirements.txt + python-version: '3.10.x' + cache: 'poetry' + + - name: Install dependencies and cache + uses: actions/cache@v2 + with: + path: ~/.cache/pypoetry + key: ${{ runner.os }}-poetry-${{ hashFiles('**/poetry.lock') }} + + - name: Install dependencies + run: make install-python - uses: actions/setup-node@v3 with: @@ -37,7 +49,6 @@ jobs: - name: Install dependencies run: | - make install-python npm ci - name: Lint @@ -56,20 +67,30 @@ jobs: - name: Checkout uses: actions/checkout@v3 + - name: Install Poetry + run: | + python -m pip install poetry==1.4.1 + - name: Install Python3 uses: actions/setup-python@v4 with: - python-version: '3.9.x' - cache: 'pip' + python-version: '3.10.x' + cache: 'poetry' + + - name: Install dependencies and cache + uses: actions/cache@v2 + with: + path: ~/.cache/pypoetry + key: ${{ runner.os }}-poetry-${{ hashFiles('**/poetry.lock') }} - name: Install dependencies - run: python3 -m pip install -r requirements.txt + run: make install-python - name: Run Tests run: make coverage - name: Check coverage - run: coverage report --fail-under=80 + run: make coverage-report run-js-tests: runs-on: ubuntu-22.04 diff --git a/.github/workflows/external-pr-open.yml b/.github/workflows/external-pr-open.yml index 5dea2f58..d3b76d31 100644 --- a/.github/workflows/external-pr-open.yml +++ b/.github/workflows/external-pr-open.yml @@ -1,7 +1,6 @@ # Ambuda workflow to build and publish docker image -name: Build and test incoming - +name: Basic image build and test env: AMBUDA_VERSION: v0.1 AMBUDA_HOST_IP: 127.0.0.1 @@ -19,7 +18,6 @@ jobs: build_and_test: name: Build external pr image runs-on: ubuntu-22.04 - environment: staging permissions: packages: write contents: read @@ -38,7 +36,7 @@ jobs: - name: Build and Test docker image id: build-to-test - uses: docker/build-push-action@v3.2.0 + uses: docker/build-push-action@v4.0.0 with: context: . file: build/containers/Dockerfile.final diff --git a/.github/workflows/rel-pr-create.yml b/.github/workflows/rel-pr-create.yml index 0b90ac38..9947a14d 100644 --- a/.github/workflows/rel-pr-create.yml +++ b/.github/workflows/rel-pr-create.yml @@ -1,14 +1,13 @@ -# Ambuda workflow to build and publish docker image - -name: Create Release PR on seeing new code +# Check for changes in "main". Push changes to "releases". +name: Push changes from main to releases env: GH_TOKEN: ${{ github.token }} # head/branch where current changes reside befor merge - PR_SOURCE_BRANCH: development + PR_SOURCE_BRANCH: main # base - branch intended to change once the proposed changes are meged. - PR_TARGET_BRANCH: releases + PR_TARGET_BRANCH: release on: workflow_dispatch: @@ -28,16 +27,16 @@ jobs: - uses: actions/checkout@v3 with: ref: ${{ env.PR_TARGET_BRANCH }} - - name: Find the recent PR merge on development + - name: Find the recent PR merge on ${{ env.PR_SOURCE_BRANCH }} id: find_pr run: | - LAST_RUNTIME=$(date +'%Y-%m-%dT%H:%M:%S' --date '-30000 min') + LAST_RUNTIME=$(date +'%Y-%m-%dT%H:%M:%S' --date '-1800 min') gh repo set-default ambuda-org/ambuda echo "PR_NUMBER=$(gh pr list --state merged --base ${{ env.PR_SOURCE_BRANCH }} --search "merged:>$LAST_RUNTIME" -L 1 --json number| jq '.[].number')" >> $GITHUB_OUTPUT create_pr: runs-on: ubuntu-22.04 - name: Create PR on releases branch + name: Create PR on release branch environment: staging permissions: packages: write @@ -61,4 +60,4 @@ jobs: base: ${{ env.PR_TARGET_BRANCH }} branch: ${{ env.PR_TARGET_BRANCH }}-${{ env.PR_NUMBER }} title: PR-${{ env.PR_NUMBER }} - merge - body: development/PR-${{ env.PR_NUMBER }} merge is open + body: ${{ env.PR_SOURCE_BRANCH }}/PR-${{ env.PR_NUMBER }} merge is open diff --git a/.github/workflows/rel-pr-merged.yml b/.github/workflows/rel-pr-merged.yml index e634645a..af3ca7cf 100644 --- a/.github/workflows/rel-pr-merged.yml +++ b/.github/workflows/rel-pr-merged.yml @@ -1,7 +1,6 @@ -# Ambuda workflow to build and publish docker image - -name: Teardown staging +# Cleanup staging. Usually occurs after a pr is merged or closed. +name: Teardown staging deployment env: AMBUDA_VERSION: v0.1 @@ -13,7 +12,7 @@ on: # - 'v*' pull_request: branches: - - 'releases' + - 'release' types: - closed diff --git a/.github/workflows/rel-pr-open.yml b/.github/workflows/rel-pr-open.yml index 2888db20..1b9eb173 100644 --- a/.github/workflows/rel-pr-open.yml +++ b/.github/workflows/rel-pr-open.yml @@ -1,7 +1,6 @@ -# Ambuda workflow to build and publish docker image - -name: Build publish and staging +# Build release image. Push image to ghcr.io. Deploy on staging environment. +name: Release image build & publish env: AMBUDA_VERSION: v0.1 AMBUDA_HOST_IP: 127.0.0.1 @@ -9,9 +8,10 @@ env: REGISTRY: ghcr.io on: + workflow_dispatch: pull_request: branches: - - 'releases' + - 'release' types: [opened, reopened, synchronize] jobs: @@ -60,7 +60,7 @@ jobs: - name: Build and push Docker images id: publish - uses: docker/build-push-action@v3.2.0 + uses: docker/build-push-action@v4.0.0 with: context: . file: build/containers/Dockerfile.final diff --git a/.gitignore b/.gitignore index 7700e201..0cc3a8a8 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,8 @@ # Large data files, including texts, parse data, and (in local dev) # image uploads. data/ +deploy/data_database +deploy/data_files # Autogenerated files (compiled CSS, testing, documentation, ...) .coverage diff --git a/Makefile b/Makefile index 4bb2e287..d50c401a 100644 --- a/Makefile +++ b/Makefile @@ -1,3 +1,34 @@ +# Environment. Valid values are: local, staging, and prod +AMBUDA_DEPLOYMENT_ENV=local +AMBUDA_HOST_IP=0.0.0.0 +AMBUDA_HOST_PORT=5000 + +# Control the verbosity of messages using a flag +ifdef mode + ifeq ("$(origin mode)", "command line") + BUILD_MODE = $(mode) + endif +else + BUILD_MODE = default +endif + +ifdef ($(BUILD_MODE),dev) + IO_REDIRECT = + DOCKER_VERBOSITY = + DOCKER_LOG_LEVEL = + DOCKER_DETACH = +else ifeq ($(BUILD_MODE),quiet) + IO_REDIRECT = &> /dev/null + DOCKER_VERBOSITY = -qq + DOCKER_LOG_LEVEL = --log-level ERROR + DOCKER_DETACH = --detach +else ifeq ($(BUILD_MODE),default) + IO_REDIRECT = + DOCKER_VERBOSITY = + DOCKER_LOG_LEVEsL = + DOCKER_DETACH = --detach +endif + # Needed because we have folders called "docs" and "test" that confuse `make`. .PHONY: docs test py-venv-check clean @@ -11,11 +42,6 @@ AMBUDA_NAME=ambuda AMBUDA_IMAGE=${AMBUDA_NAME}:${AMBUDA_VERSION}-${GITBRANCH}-${GITCOMMIT} AMBUDA_IMAGE_LATEST="$(AMBUDA_NAME)-rel:latest" -# Environment. Valid values are: local, staging, and prod -AMBUDA_DEPLOYMENT_ENV=local -AMBUDA_HOST_IP=0.0.0.0 -AMBUDA_HOST_PORT=5090 - py-venv-check: ifeq ("$(VIRTUAL_ENV)","") @echo "Error! Python venv not activated. Activate venv to proceed. Run: " @@ -26,6 +52,7 @@ endif DB_FILE = ${PWD}/deploy/data/database/database.db + # Setup commands # =============================================== @@ -39,11 +66,13 @@ install-frontend: npm install make css-prod js-prod -# Install Python dependencies. -install-python: +.PHONY: check-poetry install-python +check-poetry: + @command -v poetry > /dev/null || { echo >&2 "Poetry is not installed. Please visit https://python-poetry.org/docs/#installation for installation instructions."; exit 1; } + +install-python: check-poetry python3 -m venv env - . env/bin/activate; pip install --upgrade pip - . env/bin/activate; pip install -r requirements.txt + . env/bin/activate; poetry install # Fetch and build all i18n files. install-i18n: py-venv-check @@ -97,15 +126,30 @@ db-seed-all: py-venv-check python -m ambuda.seed.dictionaries.vacaspatyam -# Common development commands +# Local run commands +# =============================================== + +.PHONY: devserver celery + +# For Docker try `make mode=dev docker-start` +devserver: py-venv-check + ./node_modules/.bin/concurrently "flask run -h 0.0.0.0 -p 5000" "npx tailwindcss -i ambuda/static/css/style.css -o ambuda/static/gen/style.css --watch" "npx esbuild ambuda/static/js/main.js --outfile=ambuda/static/gen/main.js --bundle --watch" + +# Run a local Celery instance for background tasks. +celery: + celery -A ambuda.tasks worker --loglevel=INFO + + +# Docker commands # =============================================== +.PHONY: docker-setup-db docker-build docker-start docker-stop docker-logs # Start DB using Docker. docker-setup-db: docker-build ifneq ("$(wildcard $(DB_FILE))","") @echo "Ambuda using your existing database!" else - @docker --log-level ERROR compose -p ambuda-${AMBUDA_DEPLOYMENT_ENV} -f deploy/${AMBUDA_DEPLOYMENT_ENV}/docker-compose-dbsetup.yml up &> /dev/null + @docker ${DOCKER_LOG_LEVEL} compose -p ambuda-${AMBUDA_DEPLOYMENT_ENV} -f deploy/${AMBUDA_DEPLOYMENT_ENV}/docker-compose-dbsetup.yml up ${IO_REDIRECT} @echo "Ambuda Database : ✔ " endif @@ -114,12 +158,12 @@ endif docker-build: @echo "> Ambuda build is in progress. Expect it to take 2-5 minutes." @printf "%0.s-" {1..21} && echo - @docker build -q -t ${AMBUDA_IMAGE} -t ${AMBUDA_IMAGE_LATEST} -f build/containers/Dockerfile.final ${PWD} > /dev/null + @docker build ${DOCKER_VEBOSITY} -t ${AMBUDA_IMAGE} -t ${AMBUDA_IMAGE_LATEST} -f build/containers/Dockerfile.final ${PWD} ${IO_REDIRECT} @echo "Ambuda Image : ✔ (${AMBUDA_IMAGE}, ${AMBUDA_IMAGE_LATEST})" # Start Docker services. docker-start: docker-build docker-setup-db - @docker --log-level ERROR compose -p ambuda-${AMBUDA_DEPLOYMENT_ENV} -f deploy/${AMBUDA_DEPLOYMENT_ENV}/docker-compose.yml up --detach &> /dev/null + @docker ${DOCKER_LOG_LEVEL} compose -p ambuda-${AMBUDA_DEPLOYMENT_ENV} -f deploy/${AMBUDA_DEPLOYMENT_ENV}/docker-compose.yml up ${DOCKER_DETACH} ${IO_REDIRECT} @echo "Ambuda WebApp : ✔ " @echo "Ambuda URL : http://${AMBUDA_HOST_IP}:${AMBUDA_HOST_PORT}" @printf "%0.s-" {1..21} && echo @@ -127,41 +171,30 @@ docker-start: docker-build docker-setup-db # Stop docker services docker-stop: - @docker --log-level ERROR compose -p ambuda-${AMBUDA_DEPLOYMENT_ENV} -f deploy/${AMBUDA_DEPLOYMENT_ENV}/docker-compose.yml stop - @docker --log-level ERROR compose -p ambuda-${AMBUDA_DEPLOYMENT_ENV} -f deploy/${AMBUDA_DEPLOYMENT_ENV}/docker-compose.yml rm + @docker ${DOCKER_LOG_LEVEL} compose -p ambuda-${AMBUDA_DEPLOYMENT_ENV} -f deploy/${AMBUDA_DEPLOYMENT_ENV}/docker-compose.yml stop + @docker ${DOCKER_LOG_LEVEL} compose -p ambuda-${AMBUDA_DEPLOYMENT_ENV} -f deploy/${AMBUDA_DEPLOYMENT_ENV}/docker-compose.yml rm @echo "Ambuda URL stopped" # Show docker logs docker-logs: @docker compose -p ambuda-${AMBUDA_DEPLOYMENT_ENV} -f deploy/${AMBUDA_DEPLOYMENT_ENV}/docker-compose.yml logs -# Run a local Celery instance for background tasks. -celery: - celery -A ambuda.tasks worker --loglevel=INFO - -# Check imports in Python code -lint-isort: - @echo "Running Python isort to organize module imports" - @git ls-files '*.py' | xargs isort --check 2>&1 - -# Check formatting in Python code -lint-black: - @echo "Running Python Black to check formatting" - @git ls-files '*.py' | xargs black 2>&1 -# Check Python code complyies with PEP8 -lint-flake8: - @echo "Running Python flake8 to conform with PEP8" - @git ls-files '*.py' | xargs flake8 --config=./.flake8 2>&1 +# Lint commands +# =============================================== # Link checks on Python code -py-lint: py-venv-check lint-black lint-isort lint-flake8 - @echo "Python lint completed" +py-lint: py-venv-check + ruff . --fix + black . # Lint our Python and JavaScript code. Fail on any issues. -lint-check: js-lint py-lint +lint-check: js-lint black . --diff - @echo 'Lint completed' + + +# Test, coverage and documentation commands +# =============================================== # Run all Python unit tests. test: py-venv-check @@ -172,6 +205,9 @@ test: py-venv-check coverage: pytest --cov=ambuda --cov-report=html test/ +coverage-report: coverage + coverage report --fail-under=80 + # Generate Ambuda's technical documentation. # After the command completes, open "docs/_build/index.html". docs: py-venv-check @@ -221,7 +257,7 @@ js-check-types: # i18n and l10n commands # =============================================== -# Extract all translatable text from the application. +# Extract all translatable text from the application and save it in `messages.pot`. babel-extract: py-venv-check pybabel extract --mapping babel.cfg --keywords _l --output-file messages.pot . @@ -238,6 +274,9 @@ babel-update: py-venv-check babel-compile: py-venv-check pybabel compile -d ambuda/translations +# Clean up +# =============================================== + clean: @rm -rf deploy/data/ @rm -rf ambuda/translations/* diff --git a/README.md b/README.md index d5ac4b28..1096a56e 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,34 @@ ![Unit Tests](https://github.com/ambuda-org/ambuda/actions/workflows/basic-tests.yml/badge.svg) + Ambuda ====== -Ambuda is an online Sanskrit archive whose mission is to make the Sanskrit -tradition radically accessible. Our archive is hosted at https://ambuda.org. +Ambuda is an online Sanskrit library whose mission is to make Sanskrit +literature accessible to all. Our library is hosted at https://ambuda.org. + +This repository contains Ambuda's core code. It also includes *seed* scripts, +which will automatically pull external data sources and populate a development +database. + + +Quickstart +---------- + +(This setup process requires Docker. If you don't have Docker installed on your +machine, you can install it [here][docker].) -This repository contains Ambuda's core code. It also includes database seeding -scripts, which will automatically pull external data sources as needed. +To install and run Ambuda locally, please run the commands below: + +``` +$ git clone https://github.com/ambuda-org/ambuda.git +$ cd ambuda +$ make devserver +``` + +Then, navigate to `http://localhost:5000` in your web browser. + +[docker]: https://docs.docker.com/get-docker/ Documentation @@ -20,16 +41,9 @@ https://ambuda.readthedocs.io/en/latest/ It includes installation instructions, architecture notes, and other reference documentation about Ambuda's technical design. -Quick Installation ------------------- - -1. Clone git repo `$ git clone https://github.com/ambuda-org/ambuda.git` -2. Go to Ambuda code `$ cd ambuda` -3. Start Ambuda `$ make docker-start`. (Get [docker](https://docs.docker.com/get-docker/) if not installed on your computer.) -4. Open https://localhost:5090/ -How to contribute ------------------ +Contributing +------------ For details on how to contribute to Ambuda, see [`CONTRIBUTING.md`][CONTRIBUTING.md]. We also strongly recommend joining our [Discord channel][discord], where we have an diff --git a/ambuda/README.md b/ambuda/README.md index 900287bb..d2e4ec88 100644 --- a/ambuda/README.md +++ b/ambuda/README.md @@ -1 +1,15 @@ -See documentation at https://ambuda.readthedocs.io/en/latest/project-layout.html#core-code +This is the main application directly and contains all of our backend and +frontend code. + +The main folders are `models`, `static`, `templates`, and `views`: + +- `models` contains our database schemas and relations. +- `static` contains our CSS, JS, and other static assets. +- `templates` contains our HTML. +- `views` contains our application logic. + +Generally, you will need to work with files in all four of these directories to +implement a new feature. + +For detailed documentation, run `make docs` or use our pre-built documentation +at: https://ambuda.readthedocs.io/en/latest/project-layout.html#core-code diff --git a/ambuda/__init__.py b/ambuda/__init__.py index c5c62c4f..f7ae697b 100644 --- a/ambuda/__init__.py +++ b/ambuda/__init__.py @@ -103,7 +103,7 @@ def create_app(config_env: str): assert config_env == config_spec.AMBUDA_ENVIRONMENT if config_env != config.TESTING: with app.app_context(): - checks.check_database(config_spec.SQLALCHEMY_DATABASE_URI) + checks.check_database_uri(config_spec.SQLALCHEMY_DATABASE_URI) # Logger _initialize_logger(config_spec.LOG_LEVEL) @@ -111,13 +111,13 @@ def create_app(config_env: str): # Database _initialize_db_session(app, config_env) - # Extensions - babel = Babel(app) - - @babel.localeselector + # A custom Babel locale_selector. def get_locale(): return session.get("locale", config_spec.BABEL_DEFAULT_LOCALE) + # Extensions + Babel(app, locale_selector=get_locale) + login_manager = auth_manager.create_login_manager() login_manager.init_app(app) @@ -141,7 +141,7 @@ def get_locale(): app.register_blueprint(texts, url_prefix="/texts") # Debug-only routes for local development. - if app.debug: + if app.debug or config.TESTING: from ambuda.views.debug import bp as debug_bp app.register_blueprint(debug_bp, url_prefix="/debug") diff --git a/ambuda/admin.py b/ambuda/admin.py index f49eaa8a..68aefba7 100644 --- a/ambuda/admin.py +++ b/ambuda/admin.py @@ -16,10 +16,15 @@ def index(self): # Abort so that a malicious scraper can't infer that there's an # interesting page here. abort(404) - return super(AmbudaIndexView, self).index() + return super().index() class BaseView(sqla.ModelView): + """Base view for models. + + By default, only admins can see model data. + """ + def is_accessible(self): return current_user.is_admin @@ -28,6 +33,8 @@ def inaccessible_callback(self, name, **kw): class ModeratorBaseView(sqla.ModelView): + """Base view for models that moderators are allowed to access.""" + def is_accessible(self): return current_user.is_moderator @@ -37,18 +44,22 @@ def inaccessible_callback(self, name, **kw): class UserView(BaseView): column_list = form_columns = ["username", "email"] + can_delete = False class TextBlockView(BaseView): - column_list = form_columns = ["text_id", "slug", "xml"] + column_list = form_columns = ["text", "slug", "xml"] class TextView(BaseView): column_list = form_columns = ["slug", "title"] + form_widget_args = {"header": {"readonly": True}} + class ProjectView(BaseView): - column_list = form_columns = ["slug", "title", "creator_id"] + column_list = ["slug", "title", "creator"] + form_excluded_columns = ["creator", "board", "pages", "created_at", "updated_at"] class DictionaryView(BaseView): @@ -73,11 +84,12 @@ def create_admin_manager(app): index_view=AmbudaIndexView(), base_template="admin/master.html", ) + admin.add_view(DictionaryView(db.Dictionary, session)) admin.add_view(ProjectView(db.Project, session)) admin.add_view(TextBlockView(db.TextBlock, session)) admin.add_view(TextView(db.Text, session)) admin.add_view(UserView(db.User, session)) - admin.add_view(SponsorshipView(db.ProjectSponsorship, session)) + return admin diff --git a/ambuda/auth.py b/ambuda/auth.py index bb190bc9..030ea441 100644 --- a/ambuda/auth.py +++ b/ambuda/auth.py @@ -1,7 +1,6 @@ """Manages the auth/authentication data flow.""" from http import HTTPStatus -from typing import Optional from flask import abort, redirect, request, url_for from flask_login import LoginManager @@ -11,7 +10,7 @@ from ambuda.utils.user_mixins import AmbudaAnonymousUser -def _load_user(user_id: int) -> Optional[User]: +def _load_user(user_id: int) -> User | None: """Load a user from the database. Flask-Login uses this function to populate the `current_user` variable. diff --git a/ambuda/checks.py b/ambuda/checks.py index 6856aec5..9d317d76 100644 --- a/ambuda/checks.py +++ b/ambuda/checks.py @@ -4,11 +4,11 @@ from click import style from sqlalchemy import create_engine, inspect +from sqlalchemy.engine import Engine from sqlalchemy.schema import Column -from ambuda import consts +from ambuda import consts, enums from ambuda import database as db -from ambuda import enums from ambuda import queries as q from ambuda.models.base import Base @@ -43,7 +43,7 @@ def _check_column(app_col: Column, db_col: dict[str, str]) -> list[str]: return errors -def _check_app_schema_matches_db_schema(database_uri: str) -> list[str]: +def _check_app_schema_matches_db_schema(engine: Engine) -> list[str]: """Check that our application tables and database tables match. Currently, we apply the following checks: @@ -63,12 +63,11 @@ def _check_app_schema_matches_db_schema(database_uri: str) -> list[str]: If any check fails, exit gracefully and tell the user how to resolve the issue. - :param database_uri: + :param engine: the engine to check. We pass `engine` directly so that we + can test this logic more easily in unit tests. """ - engine = create_engine(database_uri) inspector = inspect(engine) - errors = [] for table_name, table in Base.metadata.tables.items(): @@ -142,8 +141,8 @@ def _check_bot_user(session) -> list[str]: return [f'Bot user "{username}" does not exist.'] -def check_database(database_uri: str): - errors = _check_app_schema_matches_db_schema(database_uri) +def _check_database_engine(engine: Engine): + errors = _check_app_schema_matches_db_schema(engine) session = q.get_session() errors += _check_lookup_tables(session) @@ -171,3 +170,8 @@ def check_database(database_uri: str): else: # Style the output to match Flask's styling. print(" * [OK] Ambuda database check has passed.", flush=True) + + +def check_database_uri(database_uri: str): + engine = create_engine(database_uri) + _check_database_engine(engine) diff --git a/ambuda/models/auth.py b/ambuda/models/auth.py index c4d7692b..96619ed6 100644 --- a/ambuda/models/auth.py +++ b/ambuda/models/auth.py @@ -1,3 +1,5 @@ +"""Models related to user authentication and authorization.""" + from datetime import datetime from sqlalchemy import Boolean, Column, DateTime, ForeignKey, Integer, String @@ -40,6 +42,13 @@ class User(AmbudaUserMixin, Base): #: All roles available for this user. roles = relationship("Role", secondary="user_roles") + def __str__(self): + return self.username + + def __repr__(self): + username = self.username + return f'' + def set_password(self, raw_password: str): """Hash and save the given password.""" self.password_hash = generate_password_hash(raw_password) diff --git a/ambuda/models/parse.py b/ambuda/models/parse.py index 948ebd4a..447997d8 100644 --- a/ambuda/models/parse.py +++ b/ambuda/models/parse.py @@ -1,4 +1,4 @@ -"""Models for parse data.""" +"""Models for parsed Sanskrit text data.""" from sqlalchemy import Column from sqlalchemy import Text as _Text @@ -19,5 +19,5 @@ class BlockParse(Base): block_id = foreign_key("text_blocks.id") #: The parse data as a semi-structured text blob. #: As Ambuda matures, we can make this field more structured and - #: searchable. + #: searchable. For now, it is just a 3-column TSV string. data = Column(_Text, nullable=False) diff --git a/ambuda/models/proofing.py b/ambuda/models/proofing.py index d159ec5f..e2dfebcd 100644 --- a/ambuda/models/proofing.py +++ b/ambuda/models/proofing.py @@ -1,3 +1,5 @@ +"""Models related to our proofing work.""" + from datetime import datetime from sqlalchemy import Column, DateTime, ForeignKey, Integer, String @@ -13,12 +15,16 @@ def string(): def text(): + """Create a non-nullable text column that defaults to the empty string.""" return Column(Text_, nullable=False, default="") class Project(Base): - """A proofreading project. Each project has exactly one book.""" + """A proofreading project. + + Each project corresponds to exactly one printed book. + """ __tablename__ = "proof_projects" @@ -67,7 +73,8 @@ class Page(Base): """A page in a proofreading project. - This corresponds to a specific page in a PDF.""" + This corresponds to a specific page in a PDF. + """ __tablename__ = "proof_pages" diff --git a/ambuda/models/site.py b/ambuda/models/site.py index ed29c7ba..7b97fd68 100644 --- a/ambuda/models/site.py +++ b/ambuda/models/site.py @@ -1,7 +1,8 @@ """Various site content unrelated to texts and proofing. -The idea is that a trusted user can edit site content directly without waiting -for a site deploy. +The idea is that a trusted user can edit site content by creating and modifyng +these objects. By doing so, they can update the site without waiting for a site +deploy. """ from sqlalchemy import Column, Integer, String @@ -22,7 +23,7 @@ class ProjectSponsorship(Base): sa_title = Column(String, nullable=False) #: English title. en_title = Column(String, nullable=False) - #: Description. + #: A short description of this project. description = Column(Text_, nullable=False) - #: The estimated cost of this project in INR. + #: The estimated cost of this project in Indian rupees (INR). cost_inr = Column(Integer, nullable=False) diff --git a/ambuda/models/talk.py b/ambuda/models/talk.py index ef5dc45c..45133103 100644 --- a/ambuda/models/talk.py +++ b/ambuda/models/talk.py @@ -1,4 +1,4 @@ -"""For discussion forums.""" +"""Models for discussion forums.""" from datetime import datetime diff --git a/ambuda/models/texts.py b/ambuda/models/texts.py index 464b8ee9..070c8547 100644 --- a/ambuda/models/texts.py +++ b/ambuda/models/texts.py @@ -1,4 +1,11 @@ -"""Models for text content.""" +"""Models for text documents in our library. + +We define texts with three different tables: + +- `Text` defines the text as a whole. +- `TextSection` defines ordered sections of a `Text`. +- `TextBlock` is typically a verse or paragraph within a `TextSection`. +""" from sqlalchemy import Column, Integer, String from sqlalchemy import Text as _Text @@ -24,6 +31,9 @@ class Text(Base): #: An ordered list of the sections contained within this text. sections = relationship("TextSection", backref="text", cascade="delete") + def __str__(self): + return self.slug + class TextSection(Base): @@ -43,6 +53,10 @@ class TextSection(Base): #: The text that contains this section. text_id = foreign_key("texts.id") #: Human-readable ID, which we display in the URL. + #: + #: Slugs are hierarchical, with different levels of the hierarchy separated + #: by "." characters. At serving time, we rely on this property to properly + #: organize a text into different sections. slug = Column(String, index=True, nullable=False) #: The title of this section. title = Column(String, nullable=False) @@ -69,7 +83,9 @@ class TextBlock(Base): section_id = foreign_key("text_sections.id") #: Human-readable ID, which we display in the URL. slug = Column(String, index=True, nullable=False) - #: Raw XMl content, which we translate into HTML at serving time. + #: Raw XML content, which we translate into HTML at serving time. xml = Column(_Text, nullable=False) #: (internal-only) Block A comes before block B iff A.n < B.n. n = Column(Integer, nullable=False) + + text = relationship("Text") diff --git a/ambuda/queries.py b/ambuda/queries.py index 1ca0f459..ec7e9122 100644 --- a/ambuda/queries.py +++ b/ambuda/queries.py @@ -5,7 +5,6 @@ """ import functools -from typing import Optional from flask import current_app from sqlalchemy import create_engine @@ -66,7 +65,7 @@ def page_statuses() -> list[db.PageStatus]: return session.query(db.PageStatus).all() -def text(slug: str) -> Optional[db.Text]: +def text(slug: str) -> db.Text | None: session = get_session() return ( session.query(db.Text) @@ -99,17 +98,17 @@ def text_meta(slug: str) -> db.Text: ) -def text_section(text_id: int, slug: str) -> Optional[db.TextSection]: +def text_section(text_id: int, slug: str) -> db.TextSection | None: session = get_session() return session.query(db.TextSection).filter_by(text_id=text_id, slug=slug).first() -def block(text_id: int, slug: str) -> Optional[db.TextBlock]: +def block(text_id: int, slug: str) -> db.TextBlock | None: session = get_session() return session.query(db.TextBlock).filter_by(text_id=text_id, slug=slug).first() -def block_parse(block_id: int) -> Optional[db.BlockParse]: +def block_parse(block_id: int) -> db.BlockParse | None: session = get_session() return session.query(db.BlockParse).filter_by(block_id=block_id).first() @@ -153,17 +152,17 @@ def projects() -> list[db.Project]: return session.query(db.Project).all() -def project(slug: str) -> Optional[db.Project]: +def project(slug: str) -> db.Project | None: session = get_session() return session.query(db.Project).filter(db.Project.slug == slug).first() -def thread(*, id: int) -> Optional[db.Thread]: +def thread(*, id: int) -> db.Thread | None: session = get_session() return session.query(db.Thread).filter_by(id=id).first() -def post(*, id: int) -> Optional[db.Post]: +def post(*, id: int) -> db.Post | None: session = get_session() return session.query(db.Post).filter_by(id=id).first() @@ -197,7 +196,7 @@ def create_post(*, board_id: int, thread: db.Thread, user_id: int, content: str) session.commit() -def page(project_id, page_slug: str) -> Optional[db.Page]: +def page(project_id, page_slug: str) -> db.Page | None: session = get_session() return ( session.query(db.Page) @@ -206,7 +205,7 @@ def page(project_id, page_slug: str) -> Optional[db.Page]: ) -def user(username: str) -> Optional[db.User]: +def user(username: str) -> db.User | None: session = get_session() return ( session.query(db.User) @@ -233,7 +232,7 @@ def create_user(*, username: str, email: str, raw_password: str) -> db.User: return user -def blog_post(slug: str) -> Optional[db.BlogPost]: +def blog_post(slug: str) -> db.BlogPost | None: """Fetch the given blog post.""" session = get_session() return session.query(db.BlogPost).filter_by(slug=slug).first() diff --git a/ambuda/scripts/analysis/dcs_utils.py b/ambuda/scripts/analysis/dcs_utils.py index f8d7fc51..ac828089 100644 --- a/ambuda/scripts/analysis/dcs_utils.py +++ b/ambuda/scripts/analysis/dcs_utils.py @@ -4,8 +4,8 @@ """ import re +from collections.abc import Iterator from dataclasses import dataclass -from typing import Iterator import conllu from indic_transliteration import sanscript diff --git a/ambuda/scripts/analysis/mahabharata.py b/ambuda/scripts/analysis/mahabharata.py index f0f43281..9fac8657 100644 --- a/ambuda/scripts/analysis/mahabharata.py +++ b/ambuda/scripts/analysis/mahabharata.py @@ -1,8 +1,8 @@ """Add the Mahabharata parse data from DCS.""" +from collections.abc import Iterator from pathlib import Path -from typing import Iterator import ambuda.scripts.analysis.dcs_utils as dcs from ambuda.scripts.analysis.ramayana import get_kanda_and_sarga, map_and_write diff --git a/ambuda/scripts/analysis/ramayana.py b/ambuda/scripts/analysis/ramayana.py index 02e5e75a..ee0382ab 100644 --- a/ambuda/scripts/analysis/ramayana.py +++ b/ambuda/scripts/analysis/ramayana.py @@ -1,8 +1,8 @@ """Add the Ramayana parse data from DCS.""" import xml.etree.ElementTree as ET +from collections.abc import Iterator from pathlib import Path -from typing import Iterator from indic_transliteration import sanscript from sqlalchemy.orm import Session diff --git a/ambuda/scripts/analysis/single_file_text.py b/ambuda/scripts/analysis/single_file_text.py index 4e1f4b30..d1ab1a73 100644 --- a/ambuda/scripts/analysis/single_file_text.py +++ b/ambuda/scripts/analysis/single_file_text.py @@ -1,8 +1,8 @@ """Add parse data from DCS for a simple text.""" import argparse +from collections.abc import Iterator from pathlib import Path -from typing import Iterator from sqlalchemy.orm import Session @@ -23,8 +23,7 @@ def iter_sections(dcs_text_name): / "files" / f"{dcs_text_name}-all.conllu" ) - for section in dcs.parse_file(text_path): - yield section + yield from dcs.parse_file(text_path) def iter_parsed_blocks(dcs_text_name) -> Iterator[tuple[str, str]]: diff --git a/ambuda/seed/dcs.py b/ambuda/seed/dcs.py index 920d7db9..26a9136b 100644 --- a/ambuda/seed/dcs.py +++ b/ambuda/seed/dcs.py @@ -11,7 +11,7 @@ DATA_DIR = PROJECT_DIR / "data" / "ambuda-dcs" -class UpdateException(Exception): +class UpdateError(Exception): pass @@ -80,7 +80,7 @@ def add_parse_data(text_slug: str, path: Path): with Session(engine) as session: text = session.query(db.Text).filter_by(slug=text_slug).first() if not text: - raise UpdateException() + raise UpdateError() drop_existing_parse_data(session, text.id) @@ -102,7 +102,7 @@ def run(): try: add_parse_data(path.stem, path) log(f"- Added {path.stem} parse data to the database.") - except UpdateException: + except UpdateError: log(f"- Skipped {path.stem}.") skipped.append(path.stem) diff --git a/ambuda/seed/dictionaries/amarakosha.py b/ambuda/seed/dictionaries/amarakosha.py index 9bf5412a..a7beb5f6 100644 --- a/ambuda/seed/dictionaries/amarakosha.py +++ b/ambuda/seed/dictionaries/amarakosha.py @@ -16,7 +16,7 @@ """ import re -from typing import Iterator +from collections.abc import Iterator import click from indic_transliteration import sanscript diff --git a/ambuda/seed/dictionaries/apte.py b/ambuda/seed/dictionaries/apte.py index a776173f..7e4607bb 100644 --- a/ambuda/seed/dictionaries/apte.py +++ b/ambuda/seed/dictionaries/apte.py @@ -55,7 +55,7 @@ def _make_compounds(first_word, groups): for group in groups: child = group[0] # Case 1: simple and well-formed - if re.fullmatch("\w+", child.text): + if re.fullmatch(r"\w+", child.text): samasa = sandhi_utils.apply(first_word, child.text) group[0].text = first_word + "\u2014" + group[0].text yield samasa, group diff --git a/ambuda/seed/dictionaries/apte_sanskrit_hindi.py b/ambuda/seed/dictionaries/apte_sanskrit_hindi.py index 4cedf1c8..9bde698e 100644 --- a/ambuda/seed/dictionaries/apte_sanskrit_hindi.py +++ b/ambuda/seed/dictionaries/apte_sanskrit_hindi.py @@ -24,7 +24,7 @@ """ import xml.etree.ElementTree as ET -from typing import Iterator +from collections.abc import Iterator import click from indic_transliteration import sanscript diff --git a/ambuda/seed/dictionaries/shabdartha_kaustubha.py b/ambuda/seed/dictionaries/shabdartha_kaustubha.py index 0632b98e..2aef213e 100644 --- a/ambuda/seed/dictionaries/shabdartha_kaustubha.py +++ b/ambuda/seed/dictionaries/shabdartha_kaustubha.py @@ -18,7 +18,7 @@ """ import re -from typing import Iterator +from collections.abc import Iterator import click from indic_transliteration import sanscript @@ -27,7 +27,7 @@ from ambuda.seed.utils.data_utils import create_db, fetch_text from ambuda.utils.dict_utils import standardize_key -RAW_URL = "https://raw.githubusercontent.com/indic-dict/stardict-sanskrit/raw/master/sa-head/other-indic-entries/shabdArtha_kaustubha/shabdArtha_kaustubha.babylon" +RAW_URL = "https://raw.githubusercontent.com/indic-dict/stardict-sanskrit/master/sa-head/other-indic-entries/shabdArtha_kaustubha/shabdArtha_kaustubha.babylon" def create_entries(key: str, body: str) -> Iterator[tuple[str, str]]: @@ -41,9 +41,9 @@ def create_entries(key: str, body: str) -> Iterator[tuple[str, str]]: body = re.sub(r"\[(.*)\]", r"\1", body) # Per Vishvas, '|' divides headwords. - for key in key.split("|"): - key = standardize_key(key) - yield key, f"{body}" + for k in key.split("|"): + k = standardize_key(k) + yield k, f"{body}" def sak_generator(dict_blob: str): @@ -74,6 +74,7 @@ def run(use_cache): engine = create_db() print(f"Fetching data from GitHub (use_cache = {use_cache})...") + print(RAW_URL) text_blob = fetch_text(RAW_URL, read_from_cache=use_cache) print("Adding items to database ...") diff --git a/ambuda/seed/lookup/__init__.py b/ambuda/seed/lookup/__init__.py index 3050d1a0..3c500771 100644 --- a/ambuda/seed/lookup/__init__.py +++ b/ambuda/seed/lookup/__init__.py @@ -11,6 +11,7 @@ def run(): create_bot_user.run() except Exception as ex: raise Exception( - "Error: Failed to create page statuses, " + "Error: Failed to create page statuses, " "create roles, and creat bot user." - f"Error: {ex}") + f"Error: {ex}" + ) from ex diff --git a/ambuda/seed/lookup/create_bot_user.py b/ambuda/seed/lookup/create_bot_user.py index c187eaa1..f89bd6e2 100644 --- a/ambuda/seed/lookup/create_bot_user.py +++ b/ambuda/seed/lookup/create_bot_user.py @@ -11,8 +11,10 @@ def _create_bot_user(session): try: password = os.environ["AMBUDA_BOT_PASSWORD"] - except KeyError: - raise ValueError("Please set the AMBUDA_BOT_PASSWORD environment variable.") + except KeyError as e: + raise ValueError( + "Please set the AMBUDA_BOT_PASSWORD environment variable." + ) from e user = db.User(username=consts.BOT_USERNAME, email="bot@ambuda.org") user.set_password(password) diff --git a/ambuda/seed/lookup/page_status.py b/ambuda/seed/lookup/page_status.py index c7fdc035..19760f87 100644 --- a/ambuda/seed/lookup/page_status.py +++ b/ambuda/seed/lookup/page_status.py @@ -14,9 +14,9 @@ def get_default_id(): return session.query(db.PageStatus).filter_by(name=SitePageStatus.R0).one() -def run(): +def run(engine=None): """Create page statuses iff they don't exist already.""" - engine = create_db() + engine = engine or create_db() logging.debug("Creating PageStatus rows ...") with Session(engine) as session: statuses = session.query(db.PageStatus).all() diff --git a/ambuda/seed/lookup/role.py b/ambuda/seed/lookup/role.py index 7d3abd98..c6c29910 100644 --- a/ambuda/seed/lookup/role.py +++ b/ambuda/seed/lookup/role.py @@ -7,13 +7,13 @@ from ambuda.seed.utils.data_utils import create_db -def run(): +def run(engine=None): """Create roles iff they don't exist already. NOTE: this script doesn't delete existing roles. """ - engine = create_db() + engine = engine or create_db() with Session(engine) as session: roles = session.query(db.Role).all() existing_names = {s.name for s in roles} diff --git a/ambuda/seed/texts/gretil.py b/ambuda/seed/texts/gretil.py index 1b67244e..11274de9 100644 --- a/ambuda/seed/texts/gretil.py +++ b/ambuda/seed/texts/gretil.py @@ -116,11 +116,10 @@ def run(): for spec in ALLOW: add_document(engine, spec) except Exception as ex: - raise Exception( - "Error: Failed to get latest from GRETIL. " - f"Error: {ex}") + raise Exception("Error: Failed to get latest from GRETIL.") from ex log("Done.") + if __name__ == "__main__": run() diff --git a/ambuda/seed/utils/itihasa_utils.py b/ambuda/seed/utils/itihasa_utils.py index 2db078e2..753f97d8 100644 --- a/ambuda/seed/utils/itihasa_utils.py +++ b/ambuda/seed/utils/itihasa_utils.py @@ -1,9 +1,9 @@ #!/usr/bin/env python3 """Database utility functions.""" +from collections.abc import Iterator from dataclasses import dataclass from pathlib import Path -from typing import Iterator from dotenv import load_dotenv from indic_transliteration import sanscript diff --git a/ambuda/static/css/style.css b/ambuda/static/css/style.css index d68d2821..d878e2c3 100644 --- a/ambuda/static/css/style.css +++ b/ambuda/static/css/style.css @@ -120,11 +120,20 @@ @apply text-green-600; } + /* Classes created by Flask-admin */ + .pagination { + @apply border rounded my-4; + } + .pagination ul { @apply flex; } + .pagination li.active { @apply font-bold text-white bg-sky-900; } + .pagination a { @apply py-2 px-4 block hover:bg-slate-200; } + /* heatmap */ .heatmap text { @apply text-xs; } .heatmap rect.l1 { @apply fill-slate-100; } .heatmap rect.l2 { @apply fill-sky-100; } .heatmap rect.l3 { @apply fill-sky-200; } .heatmap rect.l4 { @apply fill-sky-300; } + } diff --git a/ambuda/tasks/ocr.py b/ambuda/tasks/ocr.py index a5844e66..d165c843 100644 --- a/ambuda/tasks/ocr.py +++ b/ambuda/tasks/ocr.py @@ -1,6 +1,5 @@ """Background tasks for proofing projects.""" -from typing import Optional from celery import group from celery.result import GroupResult @@ -53,8 +52,10 @@ def _run_ocr_for_page_inner( version=0, author_id=bot_user.id, ) - except Exception: - raise ValueError(f'OCR failed for page "{project.slug}/{page.slug}".') + except Exception as e: + raise ValueError( + f'OCR failed for page "{project.slug}/{page.slug}".' + ) from e @app.task(bind=True) @@ -75,7 +76,7 @@ def run_ocr_for_page( def run_ocr_for_project( app_env: str, project: db.Project, -) -> Optional[GroupResult]: +) -> GroupResult | None: """Create a `group` task to run OCR on a project. Usage: diff --git a/ambuda/templates/about/people.html b/ambuda/templates/about/people.html index 387dc400..53723fa9 100644 --- a/ambuda/templates/about/people.html +++ b/ambuda/templates/about/people.html @@ -57,7 +57,7 @@

{{ _('People') }}

Ashwin has worked on a variety of projects around open source software development, digital humanities, cloud infrastructure and architecture, and cybersecurity. His interests in Sanskrit include Vedic texts and Vedantic -commentaries. Ashwin holds a B.S. in Computer Science in Stanford University +commentaries. Ashwin holds a B.S. in Computer Science from Stanford University and is currently pursuing a J.D. degree at Georgetown University Law Center, where he works on technology law and policy. {% endtrans %}

diff --git a/ambuda/templates/admin/model/edit.html b/ambuda/templates/admin/model/edit.html index 681c9ef3..5487732b 100644 --- a/ambuda/templates/admin/model/edit.html +++ b/ambuda/templates/admin/model/edit.html @@ -25,16 +25,23 @@ {% endblock %} {% block edit_form %} -
+ {% for field in form %} - {{ mf.field(field) }} +
+
{{ mf.label(field.label) }}
+
{{ field(class_="p-2 block w-full my-2") }}
+
{% endfor %} - {{ mf.submit("Save changes") }} +
+
 
+
{{ mf.submit("Save changes") }}
+
{% endblock %} {% block delete_form %} {% set id = request.args.get('id') %} +

Danger zone

diff --git a/ambuda/templates/admin/model/list.html b/ambuda/templates/admin/model/list.html index 85253335..512195c5 100644 --- a/ambuda/templates/admin/model/list.html +++ b/ambuda/templates/admin/model/list.html @@ -11,7 +11,6 @@ {% endblock %} {% block body %} -