diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 67f17d7c0..bdf860298 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -2,4 +2,4 @@ See our contributing guide at https://usepa.github.io/haztrak/development/contributing.html -Markdown file version of our documentation can be found [here](/docs/haztrak_book/src) +Markdown file version of our documentation can be found [here](/docs/guide/src) diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index f705c64f1..7abd5b2d7 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -23,7 +23,7 @@ jobs: runs-on: ubuntu-latest defaults: run: - working-directory: ./docs/haztrak_book + working-directory: ./docs/guide steps: - name: 'Checkout Repo' uses: actions/checkout@v2 @@ -39,7 +39,7 @@ jobs: - name: 'Upload Docs' uses: actions/upload-pages-artifact@v1 with: - path: ./docs/haztrak_book/book + path: ./docs/guide/book deploy_pages: name: 'Deploy Documentation' diff --git a/docs/README.md b/docs/README.md index 12b4231b9..dcb30cea0 100644 --- a/docs/README.md +++ b/docs/README.md @@ -2,4 +2,4 @@ See our full documentation at https://usepa.github.io/haztrak/ -The source for the documentation is found in the [haztrak_book](./haztrak_book) directory +The source for the documentation is found in the [haztrak_book](./guide) directory diff --git a/docs/haztrak_book/.gitignore b/docs/guide/.gitignore similarity index 100% rename from docs/haztrak_book/.gitignore rename to docs/guide/.gitignore diff --git a/docs/guide/book.toml b/docs/guide/book.toml new file mode 100644 index 000000000..09dd2d6f7 --- /dev/null +++ b/docs/guide/book.toml @@ -0,0 +1,17 @@ +[book] +authors = ["David Paul Graham"] +language = "en" +multilingual = false +src = "src" +title = "Haztrak Documention" +description = "An open-source web application for tracking hazardous materials shipments." + +[output.html] +default-theme = "light" +preferred-dark-theme = "navy" +git-repository-url = "https://github.com/USEPA/haztrak" +edit-url-template = "https://github.com/usepa/haztrak/edit/main/docs/guide/{path}" + +[output.html.fold] +enable = true +level = 1 diff --git a/docs/haztrak_book/src/SUMMARY.md b/docs/guide/src/SUMMARY.md similarity index 100% rename from docs/haztrak_book/src/SUMMARY.md rename to docs/guide/src/SUMMARY.md diff --git a/docs/haztrak_book/src/assets/erd.png b/docs/guide/src/assets/erd.png similarity index 100% rename from docs/haztrak_book/src/assets/erd.png rename to docs/guide/src/assets/erd.png diff --git a/docs/haztrak_book/src/assets/haztrak_google_cloud_diagram_library.excalidrawlib b/docs/guide/src/assets/haztrak_google_cloud_diagram_library.excalidrawlib similarity index 100% rename from docs/haztrak_book/src/assets/haztrak_google_cloud_diagram_library.excalidrawlib rename to docs/guide/src/assets/haztrak_google_cloud_diagram_library.excalidrawlib diff --git a/docs/haztrak_book/src/assets/images/156px-Warning.svg b/docs/guide/src/assets/images/156px-Warning.svg similarity index 100% rename from docs/haztrak_book/src/assets/images/156px-Warning.svg rename to docs/guide/src/assets/images/156px-Warning.svg diff --git a/docs/haztrak_book/src/assets/images/erd.png b/docs/guide/src/assets/images/erd.png similarity index 100% rename from docs/haztrak_book/src/assets/images/erd.png rename to docs/guide/src/assets/images/erd.png diff --git a/docs/haztrak_book/src/assets/images/haztrak-low-resolution-logo-teal-on-transparent-background.svg b/docs/guide/src/assets/images/haztrak-low-resolution-logo-teal-on-transparent-background.svg similarity index 100% rename from docs/haztrak_book/src/assets/images/haztrak-low-resolution-logo-teal-on-transparent-background.svg rename to docs/guide/src/assets/images/haztrak-low-resolution-logo-teal-on-transparent-background.svg diff --git a/docs/haztrak_book/src/assets/images/haztrak_on_gke_dark.svg b/docs/guide/src/assets/images/haztrak_on_gke_dark.svg similarity index 100% rename from docs/haztrak_book/src/assets/images/haztrak_on_gke_dark.svg rename to docs/guide/src/assets/images/haztrak_on_gke_dark.svg diff --git a/docs/haztrak_book/src/assets/images/haztrak_on_gke_light.svg b/docs/guide/src/assets/images/haztrak_on_gke_light.svg similarity index 100% rename from docs/haztrak_book/src/assets/images/haztrak_on_gke_light.svg rename to docs/guide/src/assets/images/haztrak_on_gke_light.svg diff --git a/docs/haztrak_book/src/assets/images/quicker_sign.png b/docs/guide/src/assets/images/quicker_sign.png similarity index 100% rename from docs/haztrak_book/src/assets/images/quicker_sign.png rename to docs/guide/src/assets/images/quicker_sign.png diff --git a/docs/guide/src/assets/images/search_mock_sites.png b/docs/guide/src/assets/images/search_mock_sites.png new file mode 100644 index 000000000..20b94e0ab Binary files /dev/null and b/docs/guide/src/assets/images/search_mock_sites.png differ diff --git a/docs/haztrak_book/src/deployment/configuration.md b/docs/guide/src/deployment/configuration.md similarity index 100% rename from docs/haztrak_book/src/deployment/configuration.md rename to docs/guide/src/deployment/configuration.md diff --git a/docs/haztrak_book/src/deployment/containerization.md b/docs/guide/src/deployment/containerization.md similarity index 100% rename from docs/haztrak_book/src/deployment/containerization.md rename to docs/guide/src/deployment/containerization.md diff --git a/docs/haztrak_book/src/deployment/deployment.md b/docs/guide/src/deployment/deployment.md similarity index 100% rename from docs/haztrak_book/src/deployment/deployment.md rename to docs/guide/src/deployment/deployment.md diff --git a/docs/haztrak_book/src/deployment/index.md b/docs/guide/src/deployment/index.md similarity index 100% rename from docs/haztrak_book/src/deployment/index.md rename to docs/guide/src/deployment/index.md diff --git a/docs/guide/src/design/browser-client.md b/docs/guide/src/design/browser-client.md new file mode 100644 index 000000000..0d512d8be --- /dev/null +++ b/docs/guide/src/design/browser-client.md @@ -0,0 +1,113 @@ +# Haztrak Client Architecture + +## Application Overview + +The browser client is a single page application (SPA) that uses the [React library and ecosystem](https://react.dev/) to client side render the user interface. It is responsible for guiding the +user during the creation, editing, and signing of electronic manifests. + +## Application Configuration + +The project was originally bootstrapped with `Create React App` but was transitioned to use the `Vite` +build tool. For more information on Vite, see the [Vite documentation](https://vitejs.dev/). The configuration is found in `./client/vite.config.ts`. + +The following tools have been configured to aid development: + +### ESLint + +The project uses ESLint to lint the codebase and prevent developers from making common/easy to fix mistakes and enforce consistency. For more information on ESLint, see the [ESLint documentation](https://eslint.org/). + +### TypeScript + +While ESLint is great for catching common little mistakes related to JavaScript, TypeScript is a superset of JavaScript that adds type checking. It's a great tool for catching more complex mistakes and preventing them from making it into production. For more information on TypeScript, see the [TypeScript documentation](https://www.typescriptlang.org/). TypeScript is configured in `./client/tsconfig.json`. + +### Prettier + +Prettier is a tool for formatting. If enforces a consistent style across the entire codebase and amongst multiple developers, thereby making the codebase more readable and maintainable. For more information on Prettier, see the [Prettier documentation](https://prettier.io/). + +## Style Guide + +### Clean Code + +A classic book on writing clean code is [Clean Code: A Handbook of Agile Software Craftsmanship](https://www.amazon.com/Clean-Code-Handbook-Software-Craftsmanship/dp/0132350882). It's a great read and highly recommended. + +### Naming Conventions + +Haztrak recommends reading this [naming convention guide](https://github.com/kettanaito/naming-cheatsheet) hosted on github that (conveniently) uses javascript. + +In addition, the project uses the following naming conventions for various items: + +#### Selectors + +Selectors are functions that take the state as an argument and return some data from the state. +They are used to encapsulate the state shape and allow us to change the state shape without having +to change all the places where we access the state. + +We use two naming conventions for selectors: + +1. `Selector` + +#### React Components + +Constructors that return a React component should be named with PascalCase. (e.g., `MyComponent`). In addition, the file they are located in should also be named with PascalCase (e.g., `MyComponent.tsx`). + +#### React features + +Do differentiate between react components and features, we use lowercase directory name. These directories often do not export a named constant that is identical to the directory name. + +```Typescript +import { ManifestDetails as Component } from './ManifestDetails'; + +export { Component }; +``` + +which will automatically be lazy loaded by the React router, which helps with code splitting and keeps are bundle sizes small. + +## Project Structure + +See the [source roadmap](./source-roadmap.md) for a high level overview of the project structure. + +## State Management + +### Application State + +The project employs `Redux` and `Redux toolkit` as a global state management library. The store is the single source of truth for the client application and stores information relevant to all/most/many subcomponents in the application. For more information on redux, see the [redux documentation](https://redux.js.org/). + +### Component State + +The project uses the `useState` and `useReducer` React hooks to manage component state. For more information on hooks, see the [react documentation](https://reactjs.org/docs/hooks-intro.html). This type of state is local to a component and can be passed down to child components via props or context. + +### Form State + +The project embraces use of uncontrolled form inputs where possible. As such, the project uses the `react-hook-form` library to manage form state. For more information on `react-hook-form`, see the [react-hook-form documentation](https://react-hook-form.com/). + +Since the manifest is a relatively large and complex schema, we do need to use controlled components when necessary. For example, the hazardous waste generator's state, which is used in other parts of the manifest form to decide what state waste codes are available for selection. This state is controlled in the ManifestForm component and passed down to the child components that need it via props (e.g., the generator form and the waste line forms). + +The `react-hook-form` library facilitates the mechanism we use to add one-to-many relationships via the +`useFieldArray` hook. This allows us to add/edit/delete multiple waste lines and transporters to a manifest. + +The forms are also integrated with a schema validation library, currently zod. This allows us to validate the form data before submitting it to the server. For more information on zod, see the [zod documentation](https://zod.dev). + +### URL state + +The project uses the `react-router` library to manage URL state. For more information on `react-router`, see the [react-router documentation](https://reactrouter.com/). Parameters are passed to components via the URL and are accessible via the `useParams` hook which allows users to share URLs with other users and bookmark URLs for later use. + +The complete URL tree can be viewed in the `routes.tsx` file in the root of the client directory. + +## Testing + +This project uses `vitest` as the test runner as it integrates its configs seamlessly with the `vite` build tool and exposes a `Jest` compatible API (which is generally a standard in the javascript ecosystem). For more information on `vitest`, see the [vitest documentation](https://vitest.dev/). + +### Tooling + +As previously mentioned the project uses `vitest`. In addition, for assertions, the project uses the `react-testing-library` library. For more information on `react-testing-library`, see the [react-testing-library documentation](https://testing-library.com/docs/react-testing-library/intro/). +We also use `msw` to mock API requests to ensure that test can be run offline with predictable results. For more information on `msw`, see the [msw documentation](https://mswjs.io/). + +### Unit Tests + +Unit test represent the bulk of our tests. These test a single unit or component and ensure that it behaves as expected. Unit tests are located in the same directory as the component they are testing and are named with the `.spec.tsx` extension. + +### Integration Tests + +Integration test look similar to unit test, but they test multiple components simultaneously and ensure they interact as expected. Like unit tests, integration tests are located in the same directory as the component they are testing and are named with the `.spec.tsx` extension. + +With integration test, we primarily focus on testing behavior of components to ensure the user interface is behaving as expected. We do not test the implementation details of the components, such as the internal state of the component. diff --git a/docs/haztrak_book/src/design/db-design.md b/docs/guide/src/design/db-design.md similarity index 73% rename from docs/haztrak_book/src/design/db-design.md rename to docs/guide/src/design/db-design.md index 9b8e1f5f4..9f5488849 100644 --- a/docs/haztrak_book/src/design/db-design.md +++ b/docs/guide/src/design/db-design.md @@ -4,7 +4,18 @@ ## Overview -ToDo +Haztrak depends on a relational database to persist its user data as well as +information synced with (pulled from) RCRAInfo. RCRAInfo/e-Manifest should +always be treated as the source of truth, however, the database provides users +the means to, for example, draft or update electronic manifests without submitting +the changes to RCRAInfo immediately. + +The database schema is maintained in version control via a series of 'migration' +scripts. This enables us to initiate a new database and scaffold the expected +schema quickly and consistently for local development, testing, and backup. + +The Haztrak project currently utilizes [PostgreSQL](https://www.postgresql.org/), +a widely used open-source object-relational database system known for reliability and performance. ## Django ORM diff --git a/docs/haztrak_book/src/design/http-server.md b/docs/guide/src/design/http-server.md similarity index 100% rename from docs/haztrak_book/src/design/http-server.md rename to docs/guide/src/design/http-server.md diff --git a/docs/haztrak_book/src/design/index.md b/docs/guide/src/design/index.md similarity index 100% rename from docs/haztrak_book/src/design/index.md rename to docs/guide/src/design/index.md diff --git a/docs/guide/src/design/sdd.md b/docs/guide/src/design/sdd.md new file mode 100644 index 000000000..5d6e4fe25 --- /dev/null +++ b/docs/guide/src/design/sdd.md @@ -0,0 +1,62 @@ +# Source Design Document + +This document's purpose is to outline Haztrak's system architecture and design. For more information on the project's specifications and requirements, see the [Software Requirements Specification](./srs.md). + +- [Architecture Overview](#architecture-overview) + - [Server](./http-server.md) + - [Database](./db-design.md) + - [Task Queue](./task-queue.md) + - [Task Scheduler](./task-queue.md#periodic-tasks) + - [Browser Client](./browser-client.md) + - [Admin Site](#admin-site) +- [Testing](./testing.md) +- [Versioning](#versioning) + +## Architecture Overview + +As a reference implementation, Haztrak follows a service oriented design however it would not be classified as a microservice architecture. The project is partitioned into a select number of containerized components that are deployed separately but closely work together, including: + +1. An [HTTP server](./http-server.md) that provides a RESTful API for the client and admin site. +2. A [relational database](./db-design.md) for persisting user data and data synced with RCRAInfo. +3. An in memory database (caching layer) +4. A [task queue](./task-queue.md) +5. A [task scheduler](./task-queue.md#periodic-tasks) +6. A [user interface for the browser](./browser-client.md) + +## Admin Site + +The Admin site provides a quick, model-centric interface where trusted +users can manage content. It's not intended to provide a process centric interface, +admin user's should not be, for example, signing manifests through the admin site. + +The admin interface is an out-of-the-box feature of the [Django framework](https://docs.djangoproject.com/en/4.1/ref/contrib/admin/). +It can be found by appending `/admin` to the URL of the host and port of HTTP server, for example `http://localhost:8000/admin` + +## In-memory Database + +The in-memory data store serves a couple purposes, + +- As a message broker for Haztrak's [task queue](./task-queue.md) +- A cache for the [http server](./http-server.md) + +As a cache, the in-memory data store is utilized to increase performance by allowing Haztrak to cut down on latency for recently used resources including recent database queries, and computed values. As a message broker, the data store provides a reliable way for the back end service to communicate which each other (e.g., launch background tasks). + +The Haztrak project currently uses [Redis](https://redis.io/) as both the message broker and in-memory data store. + +## Versioning + +Haztrak uses [semantic versioning](https://semver.org/) to keep track +of its software releases. Semantic versioning is a widely used versioning system that allows +developers to convey the nature of the changes in the software using +a version number. + +The Haztrak project stores versions are in +[Git tags](https://git-scm.com/book/en/v2/Git-Basics-Tagging). When a new +version of the software is released, a new Git tag is created to represent that version. +these tags are then used for the container images that are built and +released for that version. Since the Git tag and image tag correspond, +the source for a given container tag can always be easily found. Containers built +from non-release commits should use + +Haztrak is stored in a monorepo, the front-end and back-end +containers are built and released simultaneously with the same version number. diff --git a/docs/guide/src/design/source-roadmap.md b/docs/guide/src/design/source-roadmap.md new file mode 100644 index 000000000..51848cc9a --- /dev/null +++ b/docs/guide/src/design/source-roadmap.md @@ -0,0 +1,52 @@ +# Source Roadmap + +We know that finding your way around a larger codebase can be intimidating, where do you start? +To that end, this overview will hopefully get you started looking in the right directory. + +``` +├── /configs : Example configuration to be passsed as environment variables +├── runhaz.sh : shell script for aiding local development +├── /client : Root for the React Browser SPA client +│ ├── /public : Entry point and static assets +│ └── /src +│ ├── /components : React componets and building blocks +│ ├── /features : Higher level components +│ ├── /hooks : Custon hooks +│ ├── /services : Library for consuming web services +│ └── /store : Global store slices and logic +└── /server : Root for the Django http server + ├── /apps : Container for django 'apps' + │ ├── /core : Django app with core features used by all apps, such as auth + │ ├── /site : Django app encapsulating site and handler related functionality + │ └── /trak : Django app encapsulating hazardous waste manifest functionality + │ ├── /migrations : Database migration + │ ├── /models : Table/Model definitions for database persistence + │ ├── /serializers : DRF serializers for model to/from JSON representation + │ ├── /services : Business logic for our app + │ ├── /tasks : Celery tasks, used to asynch interface with RCRAInfo + │ ├── /tests : Our Django app specific tests + │ └── /views : Our Django (DRF) views + ├── /fixtures : Initial data loaded on start + └── /haztrak : Django setting module +``` + +## Notable Directories + +[`server/apps`](https://github.com/USEPA/haztrak/tree/main/server/apps/trak) +This directory houses all our Django apps, really just for organizational purposes. + +[`server/apps/trak`](https://github.com/USEPA/haztrak/tree/main/server/apps/trak) +A Django reusable app that contains models related to hazardous waste tracking (electronic manifests) +It depends on the `sites` app, the Handler model contains a foreign key to the RcraSite model (many-to-one relationship). + +[`server/apps/sites`](https://github.com/USEPA/haztrak/tree/main/server/apps/trak) +A Django reusable app that contains models related to RCRAInfo sites. + +[`client/src/features`](https://github.com/USEPA/haztrak/tree/main/client/src/features) +This directory, theoretically, contains the high level logic for +rendering haztrak components, consumes the redux store slices. Generally, features map to routes the user can take. A feature component only relies on global state/context and does not accept any props. + +[`client/src/components`](https://github.com/USEPA/haztrak/tree/main/client/src/components) +This directory (should) only contains basic components that are +primarily act as building blocks. Some of these components are +simple wrappers around [react-bootstrap](https://react-bootstrap.github.io/) components. diff --git a/docs/guide/src/design/srs.md b/docs/guide/src/design/srs.md new file mode 100644 index 000000000..90e899dbc --- /dev/null +++ b/docs/guide/src/design/srs.md @@ -0,0 +1,63 @@ +# Software Requirement Specification document + +## Purpose + +The purpose of this document is to provide a list of requirement specifications for the Haztrak project. This document is intended to be used as a reference for the design and implementation of the Haztrak project. It takes into account policies and recommendations from the United States Environmental Protection Agency (EPA) e-Manifest program. + +## Intended Audience + +This project is a reference implementation for a hazardous waste tracking system. It will be useful +for both technical and non-technical persons that are scoping out work for adding the ability to track hazardous waste electronically to an existing or new hazardous waste management system. + +### General + +Haztrak (hereafter referred to as the "project") will meet the following specifications: + +1. The project will be a web application that can be accessed via a web browser. +2. The project will be developed using modern web development technologies and best practices. +3. The project will be licensed in a way that allows it to be freely used, modified, and redistributed (for profit if desired) by others outside the United States Environmental Protection Agency (EPA). +4. The project source and supporting configs/files/data will be checked into a version control system (Git) that is publicly accessible. +5. The project will use Infrastructure as code (IaC) principles to allow third-parties to deploy using publicly available cloud services and tools using on the configurations provided. +6. The project will be designed to be scalable for a limited number (< 1000) of simultaneous users. +7. The project will be designed to be extensible to allow for additional functionality (possibly by others) to be added in the future. +8. The project will be designed to protect sensitive data such as third party API keys (e.g., RCRAInfo) and user passwords. +9. The project will be designed to be maintainable and provide a realistic working example without "cutting corners" that could not be used in secure production environments. +10. The project will document the design and implementation decisions made to ensure that others can understand the project and use it as a reference for their own projects. + +### Admin functionality + +the reference implementation will allow admins to... + +1. Create and manage users. +2. Store their RCRAInfo credentials securely. +3. Manage sites the admin has access to in RCRAInfo. +4. Manage the users that have access to a site. +5. Give haztrak users the ability to use the administrator's RCRAInfo API credentials to interface with RCRAInfo/e-Manifest web services. + +### User Authentication and Authorization + +1. The project will utilize authentication and authorization techniques to ensure that only + authorized users can access and modify the data. +2. The project will use role based access control (RBAC) to ensure that users can only access + the data that they are authorized to access. + +### User e-Manifest functionality + +The reference implementation will allow users to complete the following actions: + +1. Create and save draft electronic manifests to persistent storage + - The project will allow users save manifest to the haztrak database (with or without creating the electronic manifest in RCRAInfo/e-Manifest) + - The project should aid users while drafting a manifest in a manner that is consistent with the e-Manifest system. + - The project will validate user input at various points during data entry to ensure that the data will be accepted by RCRAInfo/e-Manifest. + - To ensure the accuracy and completeness of the data being collected, the +2. Edit electronic manifest and submit changes to RCRAInfo. + - The project will allow user to edit electronic manifests in a manner that is consistent with the e-Manifest system and prevent users from making changes that are not allowed by the e-Manifest system. +3. Sign electronic manifests using the Quicker Sign functionality and EPA's Remote Signer policy. + - If granted by the Haztrak admin, users that are not registered in RCRAInfo should be allowed to sign manifests using the Quicker Sign web service exposed by e-Manifest. +4. The project will allow users to view manifest details and the status of electronic manifests in RCRAInfo. + +### User Interface specification + +1. The project will present a user interface that meets the W3C consortium's Web Accessibility Initiative (WAI) Accessibility Guidelines. +2. The project will use employ the use of a modern front end framework that is widely used and supported by the web development community to build the user interface. +3. The user interface will guide users while drafting and editing electronic manifests and notify the user of any errors or warnings that occur during data entry and provide guidance to the user on how to fix the error or warning. diff --git a/docs/haztrak_book/src/design/task-queue.md b/docs/guide/src/design/task-queue.md similarity index 70% rename from docs/haztrak_book/src/design/task-queue.md rename to docs/guide/src/design/task-queue.md index 851a2f138..bcf2333c5 100644 --- a/docs/haztrak_book/src/design/task-queue.md +++ b/docs/guide/src/design/task-queue.md @@ -2,12 +2,27 @@ The distributed task queue serves as Haztrak's way to , asynchronously, perform operations that cannot give reasonable guarantees around the time necessary to complete. Sending computationally intensive or network dependant operation to the task queue allows Haztrak to remove long-lasting process from the HTTP request-response lifecycle to provide a 'smooth' user experience and allows haztrak to handle a higher volume of traffic. +For Haztrak, a large part of this is communicating with RCRAInfo via its [web services](https://github.com/USEPA/e-manifest). Offloading these external API calls keeps our user experience +feeling snappy and protects our [http server](#http-server) from network +errors cause by downtime in external systems. + +The task queue can be scaled horizontally to include additional workers as needed. +We also deploy [task schedulers](#task-scheduler) for periodic tasks. + ## Implementation Haztrak uses the [Celery](https://docs.celeryq.dev/en/stable/#) and Celery beat for asynchronous task management. Tasks can be scheduled to run at a specific time (via Celery Beat), or they can be triggered by events such as user actions or system events. Celery also provides support for task chaining and task result handling. Haztrak utilizes [Redis](https://redis.io/) as its Message Broker. ## Periodic Tasks +The scheduler kicks off tasks at regular intervals, that are then executed by available worker in the task queue. Only a single scheduler is running at any given point to avoid duplicating tasks. Periodic tasks can be scheduled by: + +- (Crontab)[https://crontab.guru/] +- Solar events (e.g., every day at sundown) +- Periodically (e.g., every 10 minutes) + +The Haztrak project uses Celery's [beat package](https://docs.celeryq.dev/en/stable/userguide/periodic-tasks.html) to schedule periodic tasks. + Haztrak comes preconfigured with an opinionated number of tasks that it believes should be executed periodically to ensure that it remains in sync with RCRAInfo. Many of these tasks are (to be implemented) related to keeping in sync with [RCRAInfo's 'Lookups']() (sorry, do not have link for this). Additional periodic tasks can be scheduled by a user with admin privileges through the Admin user interface. The Results of periodic tasks will be stored in the task results table, which is also accessible through the admin interface (see below). diff --git a/docs/haztrak_book/src/design/testing.md b/docs/guide/src/design/testing.md similarity index 100% rename from docs/haztrak_book/src/design/testing.md rename to docs/guide/src/design/testing.md diff --git a/docs/haztrak_book/src/development/cd.md b/docs/guide/src/development/cd.md similarity index 100% rename from docs/haztrak_book/src/development/cd.md rename to docs/guide/src/development/cd.md diff --git a/docs/haztrak_book/src/development/ci.md b/docs/guide/src/development/ci.md similarity index 100% rename from docs/haztrak_book/src/development/ci.md rename to docs/guide/src/development/ci.md diff --git a/docs/haztrak_book/src/development/contributing.md b/docs/guide/src/development/contributing.md similarity index 50% rename from docs/haztrak_book/src/development/contributing.md rename to docs/guide/src/development/contributing.md index 7b6afca96..5439f8431 100644 --- a/docs/haztrak_book/src/development/contributing.md +++ b/docs/guide/src/development/contributing.md @@ -9,20 +9,21 @@ tell you everything you need to know before your first pull request. # How to Contribute -There are a variety of ways you can contribute to Haztrak, from reporting issues -to making code changes. Here are some of the ways you can get involved: +We welcome all contributions! You don't need to know how to program to contribute, here are some of the ways you can get involved: -### Report an Issue +### Start a discussion or ask a question + +If you have a question about Haztrak, or want to start a discussion, please open an [ticket](https://github.com/USEPA/haztrak/issues).on our GitHub repository. We welcome all questions and feedback! + +### Report a bug You can help us improve Haztrak by reporting any bugs -or issues you encounter. You can do this by opening an [issue on our GitHub -repository](https://github.com/USEPA/haztrak/issues). +or issues you encounter. You can also do this by opening an [issue at our GitHub repository](https://github.com/USEPA/haztrak/issues). ### Suggest an Enhancement If you have an idea for how we could improve Haztrak, please -share it with us! You can open an issue on our GitHub repository to suggest an -enhancement. Please be as detailed as possible in your suggestion. +share it with us by opening an issue! (if you haven't noticed, we track everything through tickets on our issues page). Please be as detailed as possible in your suggestion. If you have an idea It's best to open an issue, before starting work or submitting a PR. we also welcome draft PRs. @@ -33,33 +34,18 @@ If you're looking to contribute code or documentation, here's the general proces - Ask to be assigned an [Issues](https://github.com/USEPA/haztrak/issues). - Fork this repo to your GitHub account (click 'Fork'). -- Clone your fork to your local workstation. - -```shell -git clone git@github.com//haztrak.git -``` - -- Install the [pre-commit](https://pre-commit.com/) hooks - - Once installed, will lint and format your changes to ensure we have a consistent style upon every commit. - - See [pre-commit installation docs](https://pre-commit.com/#installation) - -```shell -pre-commit install -``` - -- See our documentation on setting up a [local development environment](./local-development.md) -- Create a new branch, make and commit your changes - -```shell -git checkout -b "my_awesome_feature" -``` - -- If you are adding or changing haztrak's functionality, include a new test and - make sure the entire test suite passes. +- Clone your fork to your local workstation or a remote development environment. +- Set up a local development environment + - See our documentation on setting up a [local development environment](./local-development.md) +- Create a git feature branch branch, +- Write a test that fails (commit) - [See our documentation on Testing Haztrak](../design/testing.md) -- Submit a pull request to [USEPA/haztrak](https://github.com/USEPA/haztrak/pulls) - - Your PR should come from a new branch, not the default branch (usually 'main') and please leave "Allow edits from maintainers" checked. - - This way, if small edits need to be made, we can still do that. +- Write the code to make that test pass (commit some more) +- Run the test suite and ensure all tests pass (celebrate) +- refactor your code (commit often) +- clean up your commit history (squash commits) +- push your branch to your fork and open a PR from the feature branch + - please leave "Allow edits from maintainers" checked. ## Pull Request Guidelines diff --git a/docs/haztrak_book/src/development/index.md b/docs/guide/src/development/index.md similarity index 53% rename from docs/haztrak_book/src/development/index.md rename to docs/guide/src/development/index.md index 37f6f38d2..d3f8cafde 100644 --- a/docs/haztrak_book/src/development/index.md +++ b/docs/guide/src/development/index.md @@ -1,11 +1,9 @@ # Development -This chapter contains a practical introduction to working with Haztrak. -Things such as setting up a local development environment, contributing, our +This chapter contains a practical introduction to working on Haztrak, +such as setting up a local development environment, contributing, our CI/CD pipeline, and configuration. -For now, it also contains operational documentation on deploying as well. - ## Contents - [Contributing](contributing.md) diff --git a/docs/haztrak_book/src/development/local-development.md b/docs/guide/src/development/local-development.md similarity index 81% rename from docs/haztrak_book/src/development/local-development.md rename to docs/guide/src/development/local-development.md index 8c9e9fd56..7082fc647 100644 --- a/docs/haztrak_book/src/development/local-development.md +++ b/docs/guide/src/development/local-development.md @@ -50,6 +50,17 @@ that we've specified in the [docker-compose.yaml](https://github.com/USEPA/haztr - The admin user has superuser privileges and can also log in to the [django admin portal](https://docs.djangoproject.com/en/4.1/ref/contrib/admin/). +Fixtures will also load a couple of sites. + +1. `VATESTGEN001` - a preproduction site hazardous waste generator +2. `MOCKTRANS001` - a mock transporter site +3. `MOCKTRANS002` - a mock transporter site +4. `MOCKTSDF001` - a mock TSDF site + +By Default, `testuser1` only has access to `VATESTGEN001`, the mock sites can be used to draft electronic manifests. + +![Searching for mock sites without RCRAInfo API credentials](../assets/images/search_mock_sites.png) + ## RCRAInfo API credentials Haztrak's [docker-compose](/docker-compose.yaml) file will load fixtures, however this data is limited. @@ -79,12 +90,10 @@ style guide. Most popular IDEs have a plugin to support these configs. - pre-commit - [pre-commit](https://pre-commit.com/) hooks are set to run a number of linting and formatting checks before commits on any branch is accepted. - -```shell - pip install -r requirements_dev.txt - pre-commit install -``` - + ```shell + pip install -r requirements_dev.txt + pre-commit install + ``` - .editorconfig - Universal IDE configs for formatting files, most IDEs will have a plugin you can install that will apply these configs. @@ -99,9 +108,11 @@ style guide. Most popular IDEs have a plugin to support these configs. - [Black](https://black.readthedocs.io/en/stable/#) - Black is a Python formatter from the [Python Software Foundation](https://www.python.org/psf-landing/). It's very opinionated and largely unconfigurable. +- [Ruff](https://docs.astral.sh/ruff/) + - ruff is a Python linter and (recently) formatter. It provides a fast and pleasant developer experience. +- [MyPy](https://mypy-lang.org/) + - Haztrak is (currently in the process) incrementally adopting type hints. MyPy is a static type checker for Python. ## Local Development Without Docker -If you don't have a way to build and run containers, or you're a gluten for punishment, you can make use of the Django -management scripts and the npm scripts to set up a local -development environment, however it's not recommended. +While it is possible, local development without containerization (docker) is not supported. diff --git a/docs/haztrak_book/src/development/rcrainfo.md b/docs/guide/src/development/rcrainfo.md similarity index 100% rename from docs/haztrak_book/src/development/rcrainfo.md rename to docs/guide/src/development/rcrainfo.md diff --git a/docs/haztrak_book/src/e-Manifest.md b/docs/guide/src/e-Manifest.md similarity index 100% rename from docs/haztrak_book/src/e-Manifest.md rename to docs/guide/src/e-Manifest.md diff --git a/docs/haztrak_book/src/introduction.md b/docs/guide/src/introduction.md similarity index 100% rename from docs/haztrak_book/src/introduction.md rename to docs/guide/src/introduction.md diff --git a/docs/haztrak_book/src/references/demos.md b/docs/guide/src/references/demos.md similarity index 100% rename from docs/haztrak_book/src/references/demos.md rename to docs/guide/src/references/demos.md diff --git a/docs/haztrak_book/src/references/future-maintainers.md b/docs/guide/src/references/future-maintainers.md similarity index 100% rename from docs/haztrak_book/src/references/future-maintainers.md rename to docs/guide/src/references/future-maintainers.md diff --git a/docs/haztrak_book/src/references/index.md b/docs/guide/src/references/index.md similarity index 100% rename from docs/haztrak_book/src/references/index.md rename to docs/guide/src/references/index.md diff --git a/docs/haztrak_book/src/terminology.md b/docs/guide/src/terminology.md similarity index 100% rename from docs/haztrak_book/src/terminology.md rename to docs/guide/src/terminology.md diff --git a/docs/haztrak_book/book.toml b/docs/haztrak_book/book.toml deleted file mode 100644 index 489e7ae40..000000000 --- a/docs/haztrak_book/book.toml +++ /dev/null @@ -1,6 +0,0 @@ -[book] -authors = ["David Paul Graham"] -language = "en" -multilingual = false -src = "src" -title = "Haztrak Documention" diff --git a/docs/haztrak_book/src/design/back-end-errors.md b/docs/haztrak_book/src/design/back-end-errors.md deleted file mode 100644 index a03c81d86..000000000 --- a/docs/haztrak_book/src/design/back-end-errors.md +++ /dev/null @@ -1,12 +0,0 @@ -# Back end Error Handling - -Haztrak builds on top of Django and Django Rest Framework's built in error handling -capabilities. We map relevant django exceptions to DRF errors as described in -Hacksoftware django styleguide -[error handling section](https://github.com/HackSoftware/Django-Styleguide#errors--exception-handling). - -## API Error format - -ToDo (once finalized) - -## Celery Error format diff --git a/docs/haztrak_book/src/design/browser-client.md b/docs/haztrak_book/src/design/browser-client.md deleted file mode 100644 index f32da19ed..000000000 --- a/docs/haztrak_book/src/design/browser-client.md +++ /dev/null @@ -1,35 +0,0 @@ -# Browser Client Architecture - -To Do, document our front end's design :) - -## Redux Store - -The redux store is the single source of truth for the client application. It contains -state that's central to the operation of the application. For more information on redux, -see the [redux documentation](https://redux.js.org/). - -### Naming Conventions - -For consist naming conventions, we've adopted the following: - -#### Selectors - -Selectors are functions that take the state as an argument and return some data from the state. -They are used to encapsulate the state shape and allow us to change the state shape without having -to change all the places where we access the state. - -We use two naming conventions for selectors: - -1. `select` for selectors that return a single field from the state. -2. `Selector` for selectors that return a more complex data structure. Values are - memoized using the RTK's createSelector function. - -##### Examples - -```typescript -// Selects the `user` field from the state. -import { useAppSelector } from 'store/hooks'; -import { selectUserName } from 'store/userSlice'; - -const userName = useAppSelector(selectUserName); -``` diff --git a/docs/haztrak_book/src/design/sdd.md b/docs/haztrak_book/src/design/sdd.md deleted file mode 100644 index 00943a1f7..000000000 --- a/docs/haztrak_book/src/design/sdd.md +++ /dev/null @@ -1,231 +0,0 @@ -# Source Design Document - -This document provides a high-level overview of the Haztrak's system architecture and scope. You'll find the following -topics: - -- [introduction](#source-design-document) -- [Purpose](#purpose-of-haztrak) -- [scope](#scope) -- [Architecture](#architecture) - - [Front End Services](#front-end) - - [Back End Services](#back-end) -- [Testing](#testing) -- [Requirements](#requirements) -- [Versioning](#versioning) - -We also hope this document serves as food-for-thought for anyone scoping a project -that will need to interface with the U.S. Environmental Protection -Agency's [RCRAInfo](https://rcrainfo.epa.gov/rcrainfoprod/action/secured/login) -and [e-Manifest](https://www.epa.gov/e-manifest) systems. - -## Purpose of Haztrak - -Haztrak is, for lack of a better term, a proof of concept (POC) project that aims -to illustrate how third party system can leverage the resources exposed by the -U.S. Environmental Protection Agency's IT system, [RCRAInfo](https://rcrainfo.epa.gov). -More specifically, how these resources can be leveraged to electronically manifest -hazardous waste shipments to ensure proper management from cradle-to-grave instead -of the manual and paper intensive process that has been in place since the 1980's. - -Please keep in mind, Haztrak is a labor of love. There's aspects of Haztrak that -will likely never, truly, be ready for a production deployment. Our biggest constraint -is time, and our entire budget is our what we can give in our spare time. - -## Scope - -The scope of this project is to develop a web application that demonstrates -how hazardous waste management software can interface with the Environmental -Protection Agency's (EPA) [e-manifest system](../e-Manifest.md) to electronically track hazardous -waste and leverage resources exposed by the [RCRAInfo](https://rcrainfo.epa.gov/rcrainfoprod/action/secured/login) -to properly manage hazardous waste. - -The web application consists of a series of services such as a [single page application (SPA)](#front-end) and [HTTP server](#back-end) that allows users to input -information related to hazardous waste management, including information about -the waste generators, transporters, and disposal facilities. The web application -will then utilize the e-Manifest system's web services to electronically track -the hazardous waste throughout its lifecycle, from cradle-to-grave. - -To ensure the accuracy and completeness of the data being collected, the -application will incorporate data validation and verification techniques at -various points of data entry. This will include data formatting, data -type, and completeness checks. Additionally, the application will -utilize authentication and authorization techniques to ensure that only -authorized users can access and modify the data. - -The web application will be designed to be scalable. -The project will be developed using modern web development technologies and best -practices to ensure that the application is maintainable and provides a realistic working example. - -Overall, the Haztrak project will demonstrate the benefits of integrating hazardous -waste management software with the e-Manifest system to properly manage hazardous -waste and reduce the risk of environmental harm. The web application developed in -this project will serve as a proof of concept that can be used to convince -stakeholders to invest in integrating their hazardous waste management software -with the e-Manifest system. - -Haztrak does not offer a comprehensive suite of hazardous waste management -functionality but, instead, focuses on executing the electronic manifest -workflow. As such, we expect to support the following functionality. - -- [x] Draft new electronic manifests. -- [ ] Upload draft manifests to RCRAInfo to create electronic manifests. -- [ ] Edit electronic manifests (when appropriate) and upload changes to RCRAInfo. -- [x] Use EPA's Quicker Sign functionality to sign electronic manifests. -- [x] Control access to resources based on user's permissions from RCRAInfo. - -## Architecture - -![Architecture](../assets/images/haztrak_on_gke_light.svg) - -This section provides a high-level overview of how responsibilities of the system -are partitioned between system components/services. - -The Haztrak system can be described as a series of services. - -Throughout our documentation, you'll frequently see us place these services into -two categories, 'front end', and 'back end'. - -### Front End - -The Front End consist of two user interfaces. - -1. A user interface (client) -2. An Admin Site - -#### Client - -The client is, fundamentally, responsible for rendering the user interface and -presenting the user with Haztrak's available functionality. Haztrak comes -pre-equipped with a client for the browser, specifically a single page application (SPA). - -The browser client makes extensive use of the [React library and ecosystem](https://react.dev/). - -For more information, see our [chapter on the browser client](./browser-client.md) - -#### Admin Site - -The Admin site provides a quick, model-centric interface where trusted -users can manage content. It's not intended to provide a process centric interface, -admin user's should not be, for example, signing manifests through the admin site. - -The admin interface is an out-of-the-box feature of the [Django framework](https://docs.djangoproject.com/en/4.1/ref/contrib/admin/). -It can be found by appending `/admin` to the URL of the host and port of HTTP server, for example `http://localhost:8000/admin` - -### Back End - -The back end contains the following components: - -1. An HTTP server -2. A relational database -3. An in memory database -4. A task queue -5. A task scheduler - -#### Relational Database - -Haztrak depends on a relational database to persist its user data as well as -information synced with (pulled from) RCRAInfo. RCRAInfo/e-Manifest should -always be treated as the source of truth, however, the database provides users -the means to, for example, draft or update electronic manifests without submitting -the changes to RCRAInfo immediately. - -The database schema is maintained in version control via a series of 'migration' -scripts. This enables us to initiate a new database and scaffold the expected -schema quickly and consistently for local development, testing, and backup. - -The Haztrak project currently utilizes [PostgreSQL](https://www.postgresql.org/), -a widely used open-source object-relational database system known for reliability and performance. - -For more information, see our [chapter on database design](./db-design.md) - -#### In-memory Database - -The in-memory data store serves a couple purposes, - -- As a message broker for Haztrak's [task queue](#task-queue) -- A cache for the [http server](#http-server) - -As a cache, the in-memory data store is utilized to increase performance by allowing Haztrak to cut down on latency for recently used resources including recent database queries, and computed values. As a message broker, the data store provides a reliable way for the back end service to communicate which each other (e.g., launch background tasks). - -The Haztrak project currently uses [Redis](https://redis.io/) as both the message broker and in-memory data store. - -#### Task Queue - -The task queue is responsible for jobs/scripts/tasks/batch processing that should occur outside -the [http request-response cycle](https://en.wikipedia.org/wiki/Hypertext_Transfer_Protocol). -For Haztrak, a large part of this is communicating with RCRAInfo via -its [web services](https://github.com/USEPA/e-manifest), a well documented -RESTful API. Offloading these external API calls keeps our user experience -feeling snappy and protects our [http server](#http-server) from network -errors cause by downtime in external systems. - -The task queue can be scaled horizontally to include additional workers as needed. -We also deploy [task schedulers](#task-scheduler) for periodic tasks. - -The Haztrak project uses the distributed task queue, [Celery](https://docs.celeryq.dev/en/stable/) - -For more information, see our [chapter on the task queue](./task-queue.md) - -#### Task Scheduler - -The scheduler kicks off tasks at regular intervals, that are then executed by available -worker in the task queue. Only a single scheduler is running at any given point to avoid -duplicating tasks. Periodic tasks can be scheduled by: - -- (Crontab)[https://crontab.guru/] -- Solar events (e.g., every day at sundown) -- Periodically (e.g., every 10 minutes) - -The Haztrak project uses Celery's -[beat module](https://docs.celeryq.dev/en/stable/userguide/periodic-tasks.html) to schedule periodic tasks. - -For more information, see our [chapter on the task queue](./task-queue.md) - -#### HTTP server - -The RESTful API serves data to hydrate the client and handles user authentication. -It is client agnostic, so it's not tied to any specific client, whether it be a -browser or mobile application. The API does not directly communicate with RCRAInfo, -but instead manages tasks provided by the task queue and passes on any necessary parameters. - -The Haztrak Project makes extensive use of the -[Django framework,](https://www.djangoproject.com/) and it's ecosystem. - -For more information, see our [chapter on the HTTP server](./http-server.md) - -## Testing - -Haztrak employs a suite of automated tests for the HTTP server, the http client -(user interface), and the task queue. We rely heavily on these tests, as well as -our continuous integration pipeline to ensure that regressions do not enter the -base branch (often called 'main' or 'master'). - -We intentionally do not aim for 100% code coverage with our test suite because, -what inevitably happens, is the test suite contains a bunch of low quality tests. - -The Haztrak project uses the [pytest framework](https://docs.pytest.org/en/7.2.x/) -for testing the backend python services to write readable, [DRY](https://en.wikipedia.org/wiki/Don%27t_repeat_yourself) tests. - -For more information, see our [chapter on testing](./testing.md) - -## Requirements - -What dependencies and things needed to be able to deploy/build Haztrak. - -## Versioning - -Haztrak uses [semantic versioning](https://semver.org/) to keep track -of its software releases. Semantic versioning is a widely used versioning system that allows -developers to convey the nature of the changes in the software using -a version number. - -The Haztrak project stores versions are in -[Git tags](https://git-scm.com/book/en/v2/Git-Basics-Tagging). When a new -version of the software is released, a new Git tag is created to represent that version. -these tags are then used for the container images that are built and -released for that version. Since the Git tag and image tag correspond, -the source for a given container tag can always be easily found. Containers built -from non-release commits should use - -Haztrak is stored in a monorepo, the front-end and back-end -containers are built and released simultaneously with the same version number. diff --git a/docs/haztrak_book/src/design/source-roadmap.md b/docs/haztrak_book/src/design/source-roadmap.md deleted file mode 100644 index 59dbede3e..000000000 --- a/docs/haztrak_book/src/design/source-roadmap.md +++ /dev/null @@ -1,58 +0,0 @@ -# Source Roadmap - -We know that finding your way around a larger codebase can be intimidating, where do you -start? -To that end, this overview will hopefully get you started looking in the right -directory. - -``` -├── /configs : Example configuration -├── docker-compose.yaml : Docker compose file for starting -├── runhaz.sh : Convenience shell script -├── /client : Root for the React Browser SPA client -│ ├── /public : Entry point and static assets -│ └── /src -│ ├── /components : React componets and building blocks -│ ├── /features : Higher level components -│ ├── /hooks : Custon hooks -│ ├── /services : Library for consuming web services -│ ├── /store : Redux store slices and logic -│ ├── /test : tests utilties like MSW -│ └── /types : Project Type definitions -└── /server : Root for the Django http server - ├── /apps : Container for django 'apps' - │ ├── /core : Django related items used by all apps, such as auth - │ │ ├── exceptions.py : Universal custom error handlers - │ │ └── managemet : Custom Haztrak manage.py commands - │ └── /trak : HW tracking Django app - │ ├── migrations : Database migration - │ ├── models : Table/Model definitions for database persistence - │ ├── serializers : DRF serializers for model to/from JSON representation - │ ├── services : Business logic for our app - │ ├── tasks : Celery tasks, used to asynch interface with RCRAInfo - │ ├── tests : Our Django app specific tests - │ └── views : Our Django (DRF) views - ├── /fixtures : Initial data loaded on start - └── /haztrak : Django setting module -``` - -## Notable Directories - -[`server/apps`](https://github.com/USEPA/haztrak/tree/main/server/apps/trak) -This directory houses all our Django apps, really just for organizational purposes. - -[`server/apps/trak`](https://github.com/USEPA/haztrak/tree/main/server/apps/trak) -A Django reusable app that contains models related to hazardous waste tracking (electronic manifests) -It depends on the `sites` app, the Handler model contains a foreign key to the RcraSite model (many-to-one relationship). - -[`server/apps/sites`](https://github.com/USEPA/haztrak/tree/main/server/apps/trak) -A Django reusable app that contains models related to RCRAInfo sites. - -[`client/src/features`](https://github.com/USEPA/haztrak/tree/main/client/src/features) -This directory, theoretically, contains the high level logic for -rendering haztrak components, consumes the redux store slices. - -[`client/src/components`](https://github.com/USEPA/haztrak/tree/main/client/src/components) -This directory (should) only contains basic components that are -primarily act as building blocks. Some of these components are -simple wrappers around [react-bootstrap](https://react-bootstrap.github.io/) components.