From e43006236aca8fc18565910fa4f33f8721bca0da Mon Sep 17 00:00:00 2001 From: Ben Swift Date: Thu, 31 Oct 2024 17:34:10 +1100 Subject: [PATCH] WIP: in the process of deploying to fly.io --- .dockerignore | 45 +++++++++++ .github/workflows/fly-deploy.yml | 18 +++++ Dockerfile | 101 +++++++++++++++++++++++++ TODO.md | 3 + assets/package.json | 5 +- assets/pnpm-lock.yaml | 15 +++- fly.toml | 39 ++++++++++ lib/imaginative_restoration/release.ex | 28 +++++++ rel/env.sh.eex | 13 ++++ rel/overlays/bin/migrate | 5 ++ rel/overlays/bin/migrate.bat | 1 + rel/overlays/bin/server | 5 ++ rel/overlays/bin/server.bat | 2 + 13 files changed, 276 insertions(+), 4 deletions(-) create mode 100644 .dockerignore create mode 100644 .github/workflows/fly-deploy.yml create mode 100644 Dockerfile create mode 100644 fly.toml create mode 100644 lib/imaginative_restoration/release.ex create mode 100755 rel/env.sh.eex create mode 100755 rel/overlays/bin/migrate create mode 100755 rel/overlays/bin/migrate.bat create mode 100755 rel/overlays/bin/server create mode 100755 rel/overlays/bin/server.bat diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..61a7393 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,45 @@ +# This file excludes paths from the Docker build context. +# +# By default, Docker's build context includes all files (and folders) in the +# current directory. Even if a file isn't copied into the container it is still sent to +# the Docker daemon. +# +# There are multiple reasons to exclude files from the build context: +# +# 1. Prevent nested folders from being copied into the container (ex: exclude +# /assets/node_modules when copying /assets) +# 2. Reduce the size of the build context and improve build time (ex. /build, /deps, /doc) +# 3. Avoid sending files containing sensitive information +# +# More information on using .dockerignore is available here: +# https://docs.docker.com/engine/reference/builder/#dockerignore-file + +.dockerignore + +# Ignore git, but keep git HEAD and refs to access current commit hash if needed: +# +# $ cat .git/HEAD | awk '{print ".git/"$2}' | xargs cat +# d0b8727759e1e0e7aa3d41707d12376e373d5ecc +.git +!.git/HEAD +!.git/refs + +# Common development/test artifacts +/cover/ +/doc/ +/test/ +/tmp/ +.elixir_ls + +# Mix artifacts +/_build/ +/deps/ +*.ez + +# Generated on crash by the VM +erl_crash.dump + +# Static artifacts - These should be fetched and built inside the Docker image +/assets/node_modules/ +/priv/static/assets/ +/priv/static/cache_manifest.json diff --git a/.github/workflows/fly-deploy.yml b/.github/workflows/fly-deploy.yml new file mode 100644 index 0000000..b0c246e --- /dev/null +++ b/.github/workflows/fly-deploy.yml @@ -0,0 +1,18 @@ +# See https://fly.io/docs/app-guides/continuous-deployment-with-github-actions/ + +name: Fly Deploy +on: + push: + branches: + - main +jobs: + deploy: + name: Deploy app + runs-on: ubuntu-latest + concurrency: deploy-group # optional: ensure only one action runs at a time + steps: + - uses: actions/checkout@v4 + - uses: superfly/flyctl-actions/setup-flyctl@master + - run: flyctl deploy --remote-only + env: + FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }} diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..217d536 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,101 @@ +# Find eligible builder and runner images on Docker Hub. We use Ubuntu/Debian +# instead of Alpine to avoid DNS resolution issues in production. +# +# https://hub.docker.com/r/hexpm/elixir/tags?page=1&name=ubuntu +# https://hub.docker.com/_/ubuntu?tab=tags +# +# This file is based on these images: +# +# - https://hub.docker.com/r/hexpm/elixir/tags - for the build image +# - https://hub.docker.com/_/debian?tab=tags&page=1&name=bullseye-20241016-slim - for the release image +# - https://pkgs.org/ - resource for finding needed packages +# - Ex: hexpm/elixir:1.17.3-erlang-27.1.2-debian-bullseye-20241016-slim +# +ARG ELIXIR_VERSION=1.17.3 +ARG OTP_VERSION=27.1.2 +ARG DEBIAN_VERSION=bullseye-20241016-slim + +ARG BUILDER_IMAGE="hexpm/elixir:${ELIXIR_VERSION}-erlang-${OTP_VERSION}-debian-${DEBIAN_VERSION}" +ARG RUNNER_IMAGE="debian:${DEBIAN_VERSION}" + +FROM ${BUILDER_IMAGE} as builder + +# install build dependencies +RUN apt-get update -y && apt-get install -y build-essential git wget nodejs \ + && apt-get clean && rm -f /var/lib/apt/lists/*_* + +# prepare build dir +WORKDIR /app + +# install hex + rebar +RUN mix local.hex --force && \ + mix local.rebar --force + +# set build ENV +ENV MIX_ENV="prod" + +# install mix dependencies +COPY mix.exs mix.lock ./ +RUN mix deps.get --only $MIX_ENV +RUN mkdir config + +# copy compile-time config files before we compile dependencies +# to ensure any relevant config change will trigger the dependencies +# to be re-compiled. +COPY config/config.exs config/${MIX_ENV}.exs config/ +RUN mix deps.compile + +COPY priv priv + +COPY lib lib + +# install npm packages with pnpm +RUN wget -qO- https://get.pnpm.io/install.sh | ENV="$HOME/.bashrc" SHELL="$(which bash)" bash - + +COPY assets assets +RUN . /root/.bashrc && cd assets && pnpm install + +# compile assets +RUN mix assets.deploy + +# Compile the release +RUN mix compile + +# Changes to config/runtime.exs don't require recompiling the code +COPY config/runtime.exs config/ + +COPY rel rel +RUN mix release + +# start a new build stage so that the final image will only contain +# the compiled release and other runtime necessities +FROM ${RUNNER_IMAGE} + +RUN apt-get update -y && \ + apt-get install -y libstdc++6 openssl libncurses5 locales ca-certificates \ + && apt-get clean && rm -f /var/lib/apt/lists/*_* + +# Set the locale +RUN sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen && locale-gen + +ENV LANG en_US.UTF-8 +ENV LANGUAGE en_US:en +ENV LC_ALL en_US.UTF-8 + +WORKDIR "/app" +RUN chown nobody /app + +# set runner ENV +ENV MIX_ENV="prod" + +# Only copy the final release from the build stage +COPY --from=builder --chown=nobody:root /app/_build/${MIX_ENV}/rel/imaginative_restoration ./ + +USER nobody + +# If using an environment that doesn't automatically reap zombie processes, it is +# advised to add an init process such as tini via `apt-get install` +# above and adding an entrypoint. See https://github.com/krallin/tini for details +# ENTRYPOINT ["/tini", "--"] + +CMD ["/app/bin/server"] diff --git a/TODO.md b/TODO.md index 1db7a72..231667d 100644 --- a/TODO.md +++ b/TODO.md @@ -1,5 +1,8 @@ ## TODO +- create a new imgres app (maybe manually, not from launch, so that the mount + stuff doesn't get overridden) +- make sure the old volumes got deleted (both personal and socy org) - animate pipeline view - just use a couple of the clips - add an LLM in there to make the pictures conform to the scene a bit more diff --git a/assets/package.json b/assets/package.json index 9f2c704..a18fb88 100644 --- a/assets/package.json +++ b/assets/package.json @@ -4,6 +4,9 @@ "@thi.ng/color": "^5.7.8", "@thi.ng/geom": "^8.1.13", "@thi.ng/rstream": "^9.0.7", - "hash-grid": "link:@thi.ng/geom-accel/hash-grid" + "@thi.ng/geom-accel": "^4.1.10", + "@thi.ng/random": "^4.1.2", + "@thi.ng/timestep": "^0.5.67", + "@thi.ng/vectors": "^7.12.5" } } diff --git a/assets/pnpm-lock.yaml b/assets/pnpm-lock.yaml index 18b31e8..15708db 100644 --- a/assets/pnpm-lock.yaml +++ b/assets/pnpm-lock.yaml @@ -17,12 +17,21 @@ importers: '@thi.ng/geom': specifier: ^8.1.13 version: 8.1.13 + '@thi.ng/geom-accel': + specifier: ^4.1.10 + version: 4.1.10 + '@thi.ng/random': + specifier: ^4.1.2 + version: 4.1.2 '@thi.ng/rstream': specifier: ^9.0.7 version: 9.0.7 - hash-grid: - specifier: link:@thi.ng/geom-accel/hash-grid - version: link:@thi.ng/geom-accel/hash-grid + '@thi.ng/timestep': + specifier: ^0.5.67 + version: 0.5.67 + '@thi.ng/vectors': + specifier: ^7.12.5 + version: 7.12.5 packages: diff --git a/fly.toml b/fly.toml new file mode 100644 index 0000000..2192a21 --- /dev/null +++ b/fly.toml @@ -0,0 +1,39 @@ +# fly.toml app configuration file generated for imgres on 2024-10-31T17:22:22+11:00 +# +# See https://fly.io/docs/reference/configuration/ for information about how to use this file. +# + +app = 'imgres' +primary_region = 'syd' +kill_signal = 'SIGTERM' + +[build] + +[env] +DATABASE_PATH = '/mnt/imgres/imgres.db' +PHX_HOST = 'imgres.fly.dev' +PORT = '8080' + +[[mounts]] +source = 'imgres' +destination = '/mnt/imgres' +processes = ['app'] +initial_size = '5gb' + +[http_service] +internal_port = 8080 +force_https = true +auto_stop_machines = 'stop' +auto_start_machines = true +min_machines_running = 0 +processes = ['app'] + +[http_service.concurrency] +type = 'connections' +hard_limit = 1000 +soft_limit = 1000 + +[[vm]] +memory = '1gb' +cpu_kind = 'shared' +cpus = 1 diff --git a/lib/imaginative_restoration/release.ex b/lib/imaginative_restoration/release.ex new file mode 100644 index 0000000..ba7a5bc --- /dev/null +++ b/lib/imaginative_restoration/release.ex @@ -0,0 +1,28 @@ +defmodule ImaginativeRestoration.Release do + @moduledoc """ + Used for executing DB release tasks when run in production without Mix + installed. + """ + @app :imaginative_restoration + + def migrate do + load_app() + + for repo <- repos() do + {:ok, _, _} = Ecto.Migrator.with_repo(repo, &Ecto.Migrator.run(&1, :up, all: true)) + end + end + + def rollback(repo, version) do + load_app() + {:ok, _, _} = Ecto.Migrator.with_repo(repo, &Ecto.Migrator.run(&1, :down, to: version)) + end + + defp repos do + Application.fetch_env!(@app, :ecto_repos) + end + + defp load_app do + Application.load(@app) + end +end diff --git a/rel/env.sh.eex b/rel/env.sh.eex new file mode 100755 index 0000000..efeb7ff --- /dev/null +++ b/rel/env.sh.eex @@ -0,0 +1,13 @@ +#!/bin/sh + +# configure node for distributed erlang with IPV6 support +export ERL_AFLAGS="-proto_dist inet6_tcp" +export ECTO_IPV6="true" +export DNS_CLUSTER_QUERY="${FLY_APP_NAME}.internal" +export RELEASE_DISTRIBUTION="name" +export RELEASE_NODE="${FLY_APP_NAME}-${FLY_IMAGE_REF##*-}@${FLY_PRIVATE_IP}" + +# Uncomment to send crash dumps to stderr +# This can be useful for debugging, but may log sensitive information +# export ERL_CRASH_DUMP=/dev/stderr +# export ERL_CRASH_DUMP_BYTES=4096 diff --git a/rel/overlays/bin/migrate b/rel/overlays/bin/migrate new file mode 100755 index 0000000..4a23bf7 --- /dev/null +++ b/rel/overlays/bin/migrate @@ -0,0 +1,5 @@ +#!/bin/sh +set -eu + +cd -P -- "$(dirname -- "$0")" +exec ./imaginative_restoration eval ImaginativeRestoration.Release.migrate diff --git a/rel/overlays/bin/migrate.bat b/rel/overlays/bin/migrate.bat new file mode 100755 index 0000000..149ad49 --- /dev/null +++ b/rel/overlays/bin/migrate.bat @@ -0,0 +1 @@ +call "%~dp0\imaginative_restoration" eval ImaginativeRestoration.Release.migrate diff --git a/rel/overlays/bin/server b/rel/overlays/bin/server new file mode 100755 index 0000000..84080fa --- /dev/null +++ b/rel/overlays/bin/server @@ -0,0 +1,5 @@ +#!/bin/sh +set -eu + +cd -P -- "$(dirname -- "$0")" +PHX_SERVER=true exec ./imaginative_restoration start diff --git a/rel/overlays/bin/server.bat b/rel/overlays/bin/server.bat new file mode 100755 index 0000000..8353e24 --- /dev/null +++ b/rel/overlays/bin/server.bat @@ -0,0 +1,2 @@ +set PHX_SERVER=true +call "%~dp0\imaginative_restoration" start