feat: Allow developers to run API in docker, and select between mutable and immutable modes

.env.tpl

@@ -1,38 +1,40 @@
-# needs to be real so create a personal magic.link account or use the staging key
-MAGIC_SECRET_KEY=secret
 
-# Everything below this line can be left as-is for local development
+DATABASE_TOKEN=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpc3MiOiJzdXBhYmFzZSIsImlhdCI6MTYwMzk2ODgzNCwiZXhwIjoyNTUwNjUzNjM0LCJyb2xlIjoic2VydmljZV9yb2xlIn0.necIJaiP7X2T2QjGeV-FhpkizcNTX8HjDDBAxpgQTEI
+DATABASE_URL=http://post-rest:3000
 
-# Wrangler overrides
 DEBUG=true
+ENV=dev
+LOGTAIL_TOKEN=secret
 
-# API Secrets
-# the salt is literally secret, not a random string, just 'secret'
-SALT=secret
-MAILCHIMP_API_KEY=secret
-METAPLEX_AUTH_TOKEN=secret
 
-LOGTAIL_TOKEN=secret
-# Dev private key for ucan auth
-# You can use this one or create a new key with the ucan cli `ucan-storage keypair`
+# needs to be real so create a personal magic.link account or use the staging key
+MAGIC_SECRET_KEY=INSERT_MAGIC_SECRET_KEY
+
+MAILCHIMP_API_KEY=secret
+METAPLEX_AUTH_TOKEN=metaplex-test-token
 PRIVATE_KEY=xmbtWjE9eYuAxae9G65lQSkw36HV6H+0LSFq2aKqVwY=
+SALT=secret
 
 ## API Sentry
 SENTRY_DSN=https://[email protected]/00000
 SENTRY_TOKEN=secret
 SENTRY_UPLOAD=false
 
-## API PostgREST
-DATABASE_URL=http://localhost:3000
-DATABASE_TOKEN=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpc3MiOiJzdXBhYmFzZSIsImlhdCI6MTYwMzk2ODgzNCwiZXhwIjoyNTUwNjUzNjM0LCJyb2xlIjoic2VydmljZV9yb2xlIn0.necIJaiP7X2T2QjGeV-FhpkizcNTX8HjDDBAxpgQTEI
+## Flags
+MAINTENANCE_MODE=rw
+
+## Git
+VERSION=development
+COMMITHASH=development-hash
+BRANCH=development
 
 # Postgres Database
-DATABASE_CONNECTION=postgresql://postgres:postgres@localhost:5432/postgres
+DATABASE_CONNECTION=postgresql://postgres:postgres@db:5432/postgres
 
 # Cluster 
 CLUSTER_BASIC_AUTH_TOKEN = dGVzdDp0ZXN0
 CLUSTER_SERVICE = 
-CLUSTER_API_URL = http://localhost:9094
+CLUSTER_API_URL=http://ipfs-cluster:9094
 
 # Maintenance Mode
 MAINTENANCE_MODE = rw
@@ -42,4 +44,4 @@ S3_ENDPOINT = http://localhost:9095
 S3_REGION = test
 S3_ACCESS_KEY_ID = test
 S3_SECRET_ACCESS_KEY = test
-S3_BUCKET_NAME = test
+S3_BUCKET_NAME = test

.github/workflows/api.yml

@@ -21,33 +21,8 @@ jobs:
       - uses: actions/setup-node@v2
         with:
           node-version: '16'
-      - uses: bahmutov/npm-install@v1
-      - run: npx playwright install-deps
-      - run: yarn test:api
-        env:
-          DATABASE_URL: http://localhost:3000
-          DATABASE_TOKEN: eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpc3MiOiJzdXBhYmFzZSIsImlhdCI6MTYwMzk2ODgzNCwiZXhwIjoyNTUwNjUzNjM0LCJyb2xlIjoic2VydmljZV9yb2xlIn0.necIJaiP7X2T2QjGeV-FhpkizcNTX8HjDDBAxpgQTEI
-          DATABASE_CONNECTION: postgresql://postgres:postgres@localhost:5432/postgres
-  deploy-dev:
-    name: Deploy Dev
-    if: github.event_name == 'pull_request' && github.ref != 'refs/heads/main'
-    runs-on: ubuntu-latest
-    needs: test
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-node@v2
-        with:
-          node-version: '16'
-      - uses: bahmutov/npm-install@v1
-      - name: Publish app
-        if: env.CF_API_TOKEN
-        uses: cloudflare/[email protected]
-        env:
-          SENTRY_TOKEN: ${{secrets.SENTRY_TOKEN}}
-          CF_API_TOKEN: ${{secrets.CF_API_TOKEN}}
-        with:
-          apiToken: ${{secrets.CF_API_TOKEN }}
-          workingDirectory: 'packages/api'
+      - run: 'cp .env.tpl .env'
+      - run: yarn test:api:docker
   deploy-staging:
     name: Deploy Staging
     if: github.event_name == 'push' && github.ref == 'refs/heads/main'

.gitignore

@@ -42,4 +42,7 @@ site/public
 
 packages/website/public/robots.txt
 packages/website/public/sitemap.xml
-packages/api/docker/compose
+packages/api/docker/compose
+tmp/
+.envrc
+db_data

DEVELOPMENT.md

@@ -2,7 +2,7 @@
 
 This doc should contain everything you need to know to get a working development environment up and running. If it doesn't and you know what's missing, please open a PR or issue to update the guide!
 
-## Pre-requisites
+## Prerequisites
 
 You'll need at least the following:
 
@@ -25,8 +25,10 @@ We use `yarn` in this project and commit the `yarn.lock` file.
    ```
 2. Setup your local environment with a `.env` file. See [intructions](#local-environment-configuration).
 3. Run locally by starting the following processes.
-   1. API server (`yarn dev:api`).
-   2. Web server (`yarn dev:website`).
+   1. Web server (`yarn dev:website`).
+   2. API server (`yarn dev:api`).
+      - Note: This starts the API server in a Docker container, and does not persist database state between runs.
+      - If you would like to persist database state between runs, you can use the `dev:api:persist` command instead.
 
 The site should now be available at http://localhost:4000
 

package.json

@@ -4,11 +4,13 @@
     "packages/*"
   ],
   "scripts": {
-    "dev:api": "cd packages/api && yarn dev",
+    "dev:api": "yarn --cwd packages/api dev",
+    "dev:api:persist": "yarn --cwd packages/api dev:persist",
     "dev:website": "cd packages/website && yarn dev",
     "test": "run-s test:*",
     "test:client": "yarn --cwd packages/client test",
     "test:api": "yarn --cwd packages/api test",
+    "test:api:docker": "yarn --cwd packages/api test:docker",
     "test:website": "yarn --cwd packages/website test",
     "build:client:docs": "yarn --cwd packages/client typedoc",
     "build:website": "yarn --cwd packages/website build",

packages/api/.dockerignore

@@ -0,0 +1,2 @@
+dist/
+./docker/db_data/

packages/api/README.md

@@ -26,7 +26,7 @@ eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJkaWQ6ZXRocjoweDY1MDA3QTczOWFiN0F
 In the case you need to clean up docker after failed tests or debugging session you can just run the command below.
 
 ```bash
-yarn clean
+yarn dev:clean
 ```
 
 ### Dev CLI scripts

packages/api/db/cargo.testing.sql

@@ -106,3 +106,4 @@ INSERT INTO public.metric (name, value, updated_at)
 
 INSERT INTO public."user" (magic_link_id, github_id, name, email, public_address) VALUES ('did:ethr:0x65007A739ab7AC5c537161249b81250E49e2853C', 'github|000000', 'mock user', '[email protected]', '0x65007A739ab7AC5c537161249b81250E49e2853C');
 INSERT INTO public.auth_key (name, secret, user_id) VALUES ('main', 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJkaWQ6ZXRocjoweDY1MDA3QTczOWFiN0FDNWM1MzcxNjEyNDliODEyNTBFNDllMjg1M0MiLCJpc3MiOiJuZnQtc3RvcmFnZSIsImlhdCI6MTYzOTc1NDczNjYzOCwibmFtZSI6Im1haW4ifQ.wKwJIRXXHsgwVp8mOQp6r3_F4Lz5lnoAkgVP8wqwA_Y', 1);
+INSERT INTO public.user_tag (user_id, tag, value, reason) VALUES (1, 'HasPsaAccess', 'true', 'bc this is for testing');

packages/api/docker/Dockerfile

@@ -0,0 +1,24 @@
+FROM node:16-alpine3.12
+
+HEALTHCHECK --interval=5s --timeout=5s --retries=3 \
+  CMD curl -f http://localhost:8787/stats || exit 1
+
+# Install some utilities to make debugging easier
+RUN apk add --update bash curl vim tmux postgresql-client
+
+RUN npm i -g nodemon
+RUN mkdir -p /app
+
+WORKDIR /app
+
+# Make your docker builds 100x faster with this one trick :)
+COPY ./package.json .
+RUN yarn install
+
+COPY  ./tsconfig.json .
+COPY ./docker/scripts ./docker-scripts
+COPY ./scripts ./scripts
+COPY ./db ./db
+COPY ./src/ ./src
+
+ENTRYPOINT ["nodemon", "--watch", "src/", "--exec", "'./docker-scripts/run-miniflare.sh'" ]

packages/api/docker/docker-compose.dev.yml

@@ -0,0 +1,6 @@
+services:
+  api:
+    ports:
+      - '8787:8787'
+    volumes:
+      - ../src:/app/src

packages/api/docker/docker-compose.local-ports.yml

@@ -0,0 +1,10 @@
+services:
+  post-rest:
+    ports:
+      - '3000:3000'
+  db:
+    ports:
+      - '5432:5432'
+  ipfs-cluster:
+    ports:
+      - '9094:9094'

packages/api/docker/docker-compose.yml

@@ -1,57 +1,126 @@
-version: '3.6'
 services:
-  rest:
+  post-rest:
     image: postgrest/postgrest:v9.0.0
     depends_on:
-      - db
-    restart: always
-    ports:
-      - 3000:3000/tcp
+      - db-init
     environment:
-      PGRST_DB_URI: postgres://postgres:postgres@db:5432/postgres
+      PGRST_DB_URI: $DATABASE_CONNECTION
       PGRST_DB_SCHEMAS: public,cargo
       PGRST_DB_ANON_ROLE: postgres
       PGRST_JWT_SECRET: super-secret-jwt-token-with-at-least-32-characters-long
+  db-init:
+    build:
+      dockerfile: './docker/Dockerfile'
+      context: '../'
+    entrypoint: 'psql $DATABASE_CONNECTION -v "ON_ERROR_STOP=1" -f ./db/config.sql -f ./db/tables.sql -f ./db/cargo.testing.sql -f ./db/functions.sql'
+    depends_on:
+      db:
+        condition: service_healthy
+    environment:
+      - DATABASE_CONNECTION
+      - DATABASE_URL
+      - DATABASE_TOKEN
+  api:
+    restart: on-failure
+    build:
+      dockerfile: ./docker/Dockerfile
+      context: ../
+    environment:
+      - BRANCH
+      - CLUSTER_API_URL
+      - CLUSTER_BASIC_AUTH_TOKEN
+      - CLUSTER_SERVICE
+      - COMMITHASH
+      - DATABASE_CONNECTION
+      - DATABASE_TOKEN
+      - DATABASE_URL
+      - DEBUG
+      - ENV
+      - LOGTAIL_TOKEN
+      - MAGIC_SECRET_KEY
+      - MAILCHIMP_API_KEY
+      - MAINTENANCE_MODE
+      - METAPLEX_AUTH_TOKEN
+      - PRIVATE_KEY
+      - SALT
+      - SENTRY_DSN
+      - VERSION
+
+    depends_on:
+      - post-rest
+      - ipfs-cluster
   db:
+    restart: on-failure
+    healthcheck:
+      test:
+        [
+          'CMD',
+          'psql',
+          '$DATABASE_CONNECTION',
+          '-v',
+          'ON_ERROR_STOP=1',
+          '-c',
+          'SELECT 1',
+        ]
+      interval: '1s'
+      timeout: '1s'
+      retries: 100
     build:
       context: ./postgres
-    deploy:
-      resources:
-        limits:
-          cpus: '2'
-          memory: 2G
-        reservations:
-          cpus: '1'
-          memory: 1G
-    ports:
-      - 5432:5432
     environment:
+      DATABASE_CONNECTION: $DATABASE_CONNECTION
       POSTGRES_DB: postgres
       POSTGRES_USER: postgres
       POSTGRES_PASSWORD: postgres
       POSTGRES_PORT: 5432
-  ipfs0:
-    container_name: ipfs0
+  ipfs:
     image: ipfs/go-ipfs:v0.10.0 # update this when go-ipfs M1 macs https://github.com/ipfs/go-ipfs/issues/8645
-    volumes:
-      - ./compose/ipfs0:/data/ipfs
 
-  cluster0:
-    container_name: cluster0
-    image: ipfs/ipfs-cluster:v1.0.0-rc4
+  ipfs-cluster:
+    image: ipfs/ipfs-cluster:v1.0.0-rc4 # we need to use this image tag, as nft.storage expects CIDs to come back in a certain format
     depends_on:
-      - ipfs0
+      - ipfs
     environment:
-      CLUSTER_PEERNAME: cluster0
+      CLUSTER_PEERNAME: cluster
       CLUSTER_SECRET: ${CLUSTER_SECRET} # From shell variable if set
-      CLUSTER_IPFSHTTP_NODEMULTIADDRESS: /dns4/ipfs0/tcp/5001
+      CLUSTER_IPFSHTTP_NODEMULTIADDRESS: /dns4/ipfs/tcp/5001
       CLUSTER_CRDT_TRUSTEDPEERS: '*' # Trust all peers in Cluster
       CLUSTER_RESTAPI_HTTPLISTENMULTIADDRESS: /ip4/0.0.0.0/tcp/9094 # Expose API
       CLUSTER_RESTAPI_BASICAUTHCREDENTIALS: test:test
       CLUSTER_RESTAPI_CORSALLOWEDMETHODS: GET,POST,OPTIONS
       CLUSTER_RESTAPI_CORSALLOWEDHEADERS: authorization
       CLUSTER_MONITORPINGINTERVAL: 2s # Speed up peer discovery
-    ports:
-      - '127.0.0.1:9094:9094'
-    volumes:
-      - ./compose/cluster0:/data/ipfs-cluster
+  playwright:
+    profiles:
+      - test
+    depends_on:
+      api:
+        condition: service_healthy
+    build:
+      dockerfile: ./docker/test.Dockerfile
+      context: ../
+    environment:
+      - BRANCH
+      - CLUSTER_API_URL
+      - CLUSTER_BASIC_AUTH_TOKEN
+      - CLUSTER_SERVICE
+      - COMMITHASH
+      - DATABASE_CONNECTION
+      - DATABASE_TOKEN
+      - DATABASE_URL
+      - DEBUG
+      - ENV
+      - LOGTAIL_TOKEN
+      - MAGIC_SECRET_KEY
+      - MAILCHIMP_API_KEY
+      - MAINTENANCE_MODE
+      - METAPLEX_AUTH_TOKEN
+      - PRIVATE_KEY
+      - S3_ACCESS_KEY_ID
+      - S3_BUCKET_NAME
+      - S3_ENDPOINT
+      - S3_REGION
+      - S3_SECRET_ACCESS_KEY
+      - SALT
+      - SENTRY_DSN
+      - VERSION

packages/api/docker/postgres/Dockerfile

@@ -1,13 +1,7 @@
-FROM supabase/postgres:13.3.0
+FROM supabase/postgres:12.5.0
 
 COPY 00-initial-schema.sql /docker-entrypoint-initdb.d/00-initial-schema.sql
 
-# Run time values
-ENV POSTGRES_DB=postgres
-ENV POSTGRES_USER=postgres
-ENV POSTGRES_PASSWORD=postgres
-ENV POSTGRES_PORT=5432
-
 EXPOSE 5432
 
 # Enables cat /var/lib/postgresql/data/pg_log/postgresql.log within the container to debug queries