docker: add support for revision worker (bug 1787300)

WORK IN PROGRESS DO NOT LAND

Includes:
- various changes to local dev and revision worker
- add script to start up suite based on local repos
- clean up

bin/run-all.sh

@@ -0,0 +1,20 @@
+# !/bin/sh
+
+# Run docker-compose with local repos, excluding Bugzilla.
+# The following repos should be cloned in the parent directory:
+
+# conduit-autoland-hg
+# lando-api
+# lando-ui
+# phabricator
+# review
+
+docker-compose \
+-f docker-compose.yml \
+-f docker-compose.review.yml \
+-f docker-compose.lando-api.yml \
+-f docker-compose.lando-ui.yml \
+-f docker-compose.autolandhg.yml \
+-f docker-compose.phabricator.yml \
+-f docker-compose.override.yml \
+up --build -d

docker-compose.lando-api.yml

@@ -18,6 +18,7 @@ services:
       - ../lando-api:/app
       - ../lando-api/docker/dev-version.json:/version.json
       - ../lando-api/migrations:/migrations
+      - patches:/patches
 
   lando-api.landing-worker:
     build:
@@ -26,11 +27,14 @@ services:
       # Use ./Dockerfile to run using uwsgi, similar to remote environments
       dockerfile: ./Dockerfile-dev
     image: suite_lando-api
-    command: ["landing-worker"]
+    command: ["start-landing-worker"]
     environment:
       ENV: localdev
       # TREESTATUS_URL: "http://autoland.treestatus:8000/"
       DATABASE_URL: postgresql://postgres:[email protected]/postgres
+      PHABRICATOR_URL: http://phabricator.test/
+      PHABRICATOR_UNPRIVILEGED_API_KEY: api-lefsv24henzsbzpw337bhizawuyh
+      PHABRICATOR_ADMIN_API_KEY: api-lefsv24henzsbzpw337bhizawuyh
       SENTRY_DSN:
       CELERY_BROKER_URL: redis://redis.queue/0
       OIDC_IDENTIFIER: https://lando-api.test
@@ -53,6 +57,37 @@ services:
       # Prevent writing python cache to the host.
       - caches_cache:/app/.cache/
       - /repos
+      - patches:/patches
+    depends_on:
+      - lando-api.db
+      - redis.queue
+
+  lando-api.revision-worker:
+    build:
+      context: ../lando-api/
+      dockerfile: ./Dockerfile-dev
+    image: suite_lando-api
+    command: ["start-revision-worker"]
+    environment:
+      ENV: localdev
+      DATABASE_URL: postgresql://postgres:[email protected]/postgres
+      PHABRICATOR_URL: http://phabricator.test/
+      PHABRICATOR_UNPRIVILEGED_API_KEY: api-lefsv24henzsbzpw337bhizawuyh
+      PHABRICATOR_ADMIN_API_KEY: api-lefsv24henzsbzpw337bhizawuyh
+      SENTRY_DSN:
+      CELERY_BROKER_URL: redis://redis.queue/0
+      OIDC_IDENTIFIER: https://lando-api.test
+      LANDO_UI_URL: https://lando.test
+      REPO_CLONES_PATH: /repos
+      REPOS_TO_LAND: first-repo,second-repo,third-repo
+    user: root
+    volumes:
+      - ../lando-api:/app
+      - ../lando-api/migrations:/migrations
+      # Prevent writing python cache to the host.
+      - caches_cache:/app/.cache/
+      - /repos
+      - patches:/patches
     depends_on:
       - lando-api.db
       - redis.queue
@@ -68,3 +103,4 @@ services:
 
 volumes:
     caches_cache:
+    patches:

docker-compose.review.yml

@@ -8,3 +8,9 @@ services:
     command: bash -c "cd review && pip3 install -e . && cd ~ && bash"
     volumes: 
       - ../review:/home/phab/review
+
+  lando-api.revision-worker:
+    user: root
+    command: bash -c "cd /review && pip install -e . && cd ~ && bash"
+    volumes: 
+      - ../review:/review

docker-compose.yml

@@ -169,7 +169,7 @@ services:
       context: ../lando-api/
       dockerfile: ./Dockerfile
     image: suite_lando-api
-    command: ["landing-worker"]
+    command: ["start-landing-worker"]
     environment:
       - ENV=localdev
       - DATABASE_URL=postgresql://postgres:[email protected]/postgres
@@ -185,6 +185,37 @@ services:
       # Prevent writing python cache to the host.
       - caches_cache:/app/.cache/
     depends_on:
+      - autoland.hg
+      - lando-api.db
+      - redis.queue
+
+  lando-api.revision-worker:
+    build:
+      context: ../lando-api/
+      dockerfile: ./Dockerfile
+    image: suite_lando-api
+    command: ["start-revision-worker"]
+    environment:
+      ENV: localdev
+      DATABASE_URL: postgresql://postgres:[email protected]/postgres
+      PHABRICATOR_URL: http://phabricator.test/
+      PHABRICATOR_UNPRIVILEGED_API_KEY: api-lefsv24henzsbzpw337bhizawuyh
+      PHABRICATOR_PRIVILEGED_API_KEY: api-lefsv24henzsbzpw337bhizawuyh
+      SENTRY_DSN:
+      CELERY_BROKER_URL: redis://redis.queue/0
+      OIDC_IDENTIFIER: https://lando-api.test
+      LANDO_UI_URL: https://lando.test
+      REPO_CLONES_PATH: /repos
+      REPOS_TO_LAND: first-repo,second-repo,third-repo
+    user: root
+    volumes:
+      - ../lando-api:/app
+      - ../lando-api/migrations:/migrations
+      # Prevent writing python cache to the host.
+      - caches_cache:/app/.cache/
+      - /repos
+    depends_on:
+      - autoland.hg
       - lando-api.db
       - redis.queue
 
@@ -273,13 +304,6 @@ services:
       HTTP_ALLOWED: 1
     volumes:
       - local-dev:/home/phab
-    depends_on:
-      - tinyproxy
-      - phabricator.test
-      - bmo.test
-      - bmo.feed
-      - bmo.pushd
-      - autoland.hg.test
 
   ###########################
   # Proxy containers

docker/local-dev/Dockerfile

@@ -18,7 +18,6 @@ RUN apt-get install -y \
 RUN curl https://sh.rustup.rs -sSf | sh -s -- -y
 ENV PATH="/root/.cargo/bin:${PATH}"
 
-
 # copy configuration
 COPY arcrc .arcrc
 COPY hgrc .hgrc

docker/local-dev/clone_repositories.sh

@@ -39,4 +39,6 @@ cd third-repo
 echo -e "api-lefsv24henzsbzpw337bhizawuyh\n" | moz-phab install-certificate
 cd ..
 
+chmod a+rwx .
+
 git clone hg::http://hg.test/first-repo test-repo-cinnabar