From 673ef868a44b315911d4bc838a3bdb93c6a19613 Mon Sep 17 00:00:00 2001 From: MCGallaspy Date: Wed, 9 Oct 2024 13:55:18 -0700 Subject: [PATCH 1/6] Set up Docker env for local development --- .dockerignore | 35 ++++++++++++ .gitignore | 3 +- Dockerfile | 79 ++++++++++++++++++++++++++++ Dockerfile.replays | 5 ++ README.Docker.md | 22 ++++++++ compose.yaml | 24 +++++++++ package.json | 1 + src/schemas/dummydata/1_populate.sql | 0 src/schemas/replays.sql | 2 +- 9 files changed, 169 insertions(+), 2 deletions(-) create mode 100644 .dockerignore create mode 100644 Dockerfile create mode 100644 Dockerfile.replays create mode 100644 README.Docker.md create mode 100644 compose.yaml create mode 100644 src/schemas/dummydata/1_populate.sql diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..47c25f8 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,35 @@ +# Include any files or directories that you don't want to be copied to your +# container here (e.g., local build artifacts, temporary files, etc.). +# +# For more help, visit the .dockerignore file reference guide at +# https://docs.docker.com/go/build-context-dockerignore/ + +**/.build +**/.classpath +**/.dockerignore +**/.env +**/.git +**/.gitignore +**/.project +**/.settings +**/.toolstarget +**/.vs +**/.vscode +**/.next +**/.cache +**/*.*proj.user +**/*.dbmdl +**/*.jfm +**/charts +**/docker-compose* +**/compose.y*ml +**/Dockerfile* +**/node_modules +**/npm-debug.log +**/obj +**/secrets.dev.yaml +**/values.dev.yaml +**/build +**/dist +LICENSE +README.md diff --git a/.gitignore b/.gitignore index 1a8d12c..7204fa3 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,5 @@ node_modules .dist .eslintcache config/* -.DS_Store \ No newline at end of file +.DS_Store +.env \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..d2f945c --- /dev/null +++ b/Dockerfile @@ -0,0 +1,79 @@ +# syntax=docker/dockerfile:1 + +# Comments are provided throughout this file to help you get started. +# If you need more help, visit the Dockerfile reference guide at +# https://docs.docker.com/go/dockerfile-reference/ + +# Want to help us make this template better? Share your feedback here: https://forms.gle/ybq9Krt8jtBL3iCk7 + +ARG NODE_VERSION=20.10.0 + +################################################################################ +# Use node image for base image for all stages. +FROM node:${NODE_VERSION}-alpine as base + +# Set working directory for all build stages. +WORKDIR /usr/src/app + + +################################################################################ +# Create a stage for installing production dependecies. +FROM base as deps + +# Download dependencies as a separate step to take advantage of Docker's caching. +# Leverage a cache mount to /root/.npm to speed up subsequent builds. +# Leverage bind mounts to package.json and package-lock.json to avoid having to copy them +# into this layer. +RUN --mount=type=bind,source=package.json,target=package.json \ + --mount=type=bind,source=package-lock.json,target=package-lock.json \ + --mount=type=cache,target=/root/.npm \ + npm ci --omit=dev + +################################################################################ +# Create a stage for building the application. +FROM deps as build + +# Download additional development dependencies before building, as some projects require +# "devDependencies" to be installed to build. If you don't need this, remove this step. +RUN --mount=type=bind,source=package.json,target=package.json \ + --mount=type=bind,source=package-lock.json,target=package-lock.json \ + --mount=type=cache,target=/root/.npm \ + npm ci + +# Copy the rest of the source files into the image. +COPY . . + +# Run the build script. +RUN npm run build + +################################################################################ +# Create a new stage to run the application with minimal runtime dependencies +# where the necessary files are copied from the build stage. +FROM base as final + +# Use production node environment by default. +ENV NODE_ENV production + +# Run the application as a non-root user. +USER node + +# Copy package.json so that package manager commands can be used. +COPY package.json . + +# Copy the production dependencies from the build stage and also +# the built application from the build stage into the image. +COPY --from=build /usr/src/app/node_modules ./node_modules +COPY --from=build /usr/src/app/.dist ./.dist + +# Use default configuration files for loginserver +ADD config/pm2-example.js ./config/pm2.config.js +ADD config/config-example.js ./config/config.js + +# Add some necessary source files +ADD src/public ./src/public + +# Expose the port that the application listens on. +EXPOSE 8080 + +# Run the application. +CMD npm run start-on-docker && npx pm2 logs diff --git a/Dockerfile.replays b/Dockerfile.replays new file mode 100644 index 0000000..231adb3 --- /dev/null +++ b/Dockerfile.replays @@ -0,0 +1,5 @@ +FROM cockroachdb/cockroach:latest as base +COPY src/schemas/replays.sql /docker-entrypoint-initdb.d/replays.sql +# CockroachDB listens on port 26257. Overwrite default EXPOSE that also exposes 8080 +# EXPOSE 26257 +CMD ["start-single-node", "--insecure"] \ No newline at end of file diff --git a/README.Docker.md b/README.Docker.md new file mode 100644 index 0000000..5a7ae36 --- /dev/null +++ b/README.Docker.md @@ -0,0 +1,22 @@ +### Building and running your application + +When you're ready, start your application by running: +`docker compose up --build`. + +Your application will be available at http://localhost:8080. + +### Deploying your application to the cloud + +First, build your image, e.g.: `docker build -t myapp .`. +If your cloud uses a different CPU architecture than your development +machine (e.g., you are on a Mac M1 and your cloud provider is amd64), +you'll want to build the image for that platform, e.g.: +`docker build --platform=linux/amd64 -t myapp .`. + +Then, push it to your registry, e.g. `docker push myregistry.com/myapp`. + +Consult Docker's [getting started](https://docs.docker.com/go/get-started-sharing/) +docs for more detail on building and pushing. + +### References +* [Docker's Node.js guide](https://docs.docker.com/language/nodejs/) \ No newline at end of file diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 0000000..0bf392c --- /dev/null +++ b/compose.yaml @@ -0,0 +1,24 @@ +services: + pokemon-showdown-loginserver: + build: + context: . + environment: + NODE_ENV: production + ports: + - 8080:8080 + depends_on: + replay-db: + condition: service_started + develop: + watch: + - action: rebuild + path: . + replay-db: + restart: always + build: + context: . + dockerfile: Dockerfile.replays + develop: + watch: + - action: rebuild + path: src/schemas \ No newline at end of file diff --git a/package.json b/package.json index e71a900..d6a1b3f 100644 --- a/package.json +++ b/package.json @@ -9,6 +9,7 @@ "build": "npx tsc", "run": "npx tsc && node .dist/src/", "start": "npx tsc && npx pm2 start config/pm2.js", + "start-on-docker": "npx pm2 start config/pm2.config.js", "test": "npm run lint && npx tsc", "reload": "npx tsc && npx pm2 reload config/pm2.js", "stop": "npx pm2 stop config/pm2.js" diff --git a/src/schemas/dummydata/1_populate.sql b/src/schemas/dummydata/1_populate.sql new file mode 100644 index 0000000..e69de29 diff --git a/src/schemas/replays.sql b/src/schemas/replays.sql index fc256f7..a5ec731 100644 --- a/src/schemas/replays.sql +++ b/src/schemas/replays.sql @@ -31,5 +31,5 @@ CREATE TABLE public.replays ( INDEX private_uploadtime (private ASC, uploadtime ASC), INDEX private_formatid_uploadtime (private ASC, formatid ASC, uploadtime ASC), INDEX private_formatid_rating (private ASC, formatid ASC, rating ASC), - INVERTED INDEX log (log) + INVERTED INDEX log (log gin_trgm_ops) ); From 2a1ea84468d68a6cfee52691f0ca73cae995526c Mon Sep 17 00:00:00 2001 From: MCGallaspy Date: Thu, 17 Oct 2024 10:17:21 -0700 Subject: [PATCH 2/6] Add dummy data to replays database --- Dockerfile.replays | 6 +++--- compose.yaml | 10 +++++++++- src/schemas/dummydata/1_populate.sql | 0 3 files changed, 12 insertions(+), 4 deletions(-) delete mode 100644 src/schemas/dummydata/1_populate.sql diff --git a/Dockerfile.replays b/Dockerfile.replays index 231adb3..125934e 100644 --- a/Dockerfile.replays +++ b/Dockerfile.replays @@ -1,5 +1,5 @@ FROM cockroachdb/cockroach:latest as base -COPY src/schemas/replays.sql /docker-entrypoint-initdb.d/replays.sql -# CockroachDB listens on port 26257. Overwrite default EXPOSE that also exposes 8080 -# EXPOSE 26257 +COPY src/schemas/replays.sql /docker-entrypoint-initdb.d/0_replays.sql +COPY src/schemas/dummydata/replays.sql /docker-entrypoint-initdb.d/1_replays.sql +COPY src/schemas/dummydata/replays.csv src/schemas/dummydata/replays.csv CMD ["start-single-node", "--insecure"] \ No newline at end of file diff --git a/compose.yaml b/compose.yaml index 0bf392c..87bc87c 100644 --- a/compose.yaml +++ b/compose.yaml @@ -8,16 +8,24 @@ services: - 8080:8080 depends_on: replay-db: - condition: service_started + condition: service_healthy develop: watch: - action: rebuild path: . replay-db: restart: always + hostname: replaysdb build: context: . dockerfile: Dockerfile.replays + healthcheck: + test: curl --request GET --url 'http://localhost:8080/api/v2/health/?ready=True' + interval: 1m30s + timeout: 10s + retries: 3 + start_period: 40s + start_interval: 5s develop: watch: - action: rebuild diff --git a/src/schemas/dummydata/1_populate.sql b/src/schemas/dummydata/1_populate.sql deleted file mode 100644 index e69de29..0000000 From 3f3a91508f4a1c2c6aacebea5a030a8c7180d02c Mon Sep 17 00:00:00 2001 From: MCGallaspy Date: Thu, 17 Oct 2024 10:19:08 -0700 Subject: [PATCH 3/6] Change default replays db config These values work with the local dev setup using docker compose. --- config/config-example.js | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/config/config-example.js b/config/config-example.js index bcc1c3a..6300603 100644 --- a/config/config-example.js +++ b/config/config-example.js @@ -12,12 +12,13 @@ exports.mysql = { /** For the replay databases */ exports.replaysdb = { charset: "utf8", - database: "ps", + database: "defaultdb", password: "", - host: 'localhost', + host: 'replaysdb', user: "root", socketPath: '', - prefix: "ntbb_", + prefix: "", + port: 26257, }; /** @@ -107,8 +108,6 @@ exports.challengekeyid = 4; * DBs. */ /** @type {typeof exports.mysql | undefined}*/ -exports.replaysdb = undefined; -/** @type {typeof exports.mysql | undefined}*/ exports.ladderdb = undefined; /** From bb2fc4582c85cba7d756d010b924a47aff82bfdf Mon Sep 17 00:00:00 2001 From: MCGallaspy Date: Sat, 19 Oct 2024 15:36:04 -0700 Subject: [PATCH 4/6] Add replays batch endpoint --- src/actions.ts | 11 ++++++++++- src/replays.ts | 12 ++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/src/actions.ts b/src/actions.ts index 4f17cab..48064ff 100644 --- a/src/actions.ts +++ b/src/actions.ts @@ -904,7 +904,16 @@ export const actions: {[k: string]: QueryHandler} = { } return {password: pw}; }, - + async 'replays/batch.json'(params) { + if (!params.ids) { + throw new ActionError("Invalid batch replay request, must provide ids"); + } + const ids: string[] = params.ids.split(','); + const results = await Replays.getBatch(ids); + console.log(params, ids, results); + this.response.setHeader('Content-Type', 'application/json'); + return JSON.stringify(results); + }, // sent by ps server async 'smogon/validate'(params) { if (this.getIp() !== Config.restartip) { diff --git a/src/replays.ts b/src/replays.ts index 8f03c9d..f006c39 100644 --- a/src/replays.ts +++ b/src/replays.ts @@ -233,6 +233,18 @@ export const Replays = new class { SQL`uploadtime, id, format, players, rating` )`WHERE private = 0 ORDER BY uploadtime DESC LIMIT 51`.then(this.toReplays); } + + getBatch(ids: string[]) { + let idsForQuery = ''; + for (var i = 0; i < ids.length; i++) { + const id = ids[i]; + idsForQuery += `'${id}'`; + if (i !== ids.length - 1) idsForQuery += ","; + } + return replays.selectAll( + SQL`*` + )`WHERE private = 0 AND id IN (${idsForQuery}) LIMIT 51`.then(this.toReplays); + } }; export default Replays; From f4b316c782cba31cddae449fb03662a8222e8341 Mon Sep 17 00:00:00 2001 From: MCGallaspy Date: Sun, 20 Oct 2024 08:20:26 -0700 Subject: [PATCH 5/6] Include id list appropriately SQL magic can just take a list, and the right thing will happen. No need to construct a string. --- src/actions.ts | 1 - src/replays.ts | 10 ++-------- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/src/actions.ts b/src/actions.ts index 48064ff..156de60 100644 --- a/src/actions.ts +++ b/src/actions.ts @@ -910,7 +910,6 @@ export const actions: {[k: string]: QueryHandler} = { } const ids: string[] = params.ids.split(','); const results = await Replays.getBatch(ids); - console.log(params, ids, results); this.response.setHeader('Content-Type', 'application/json'); return JSON.stringify(results); }, diff --git a/src/replays.ts b/src/replays.ts index f006c39..a9e9fcc 100644 --- a/src/replays.ts +++ b/src/replays.ts @@ -235,16 +235,10 @@ export const Replays = new class { } getBatch(ids: string[]) { - let idsForQuery = ''; - for (var i = 0; i < ids.length; i++) { - const id = ids[i]; - idsForQuery += `'${id}'`; - if (i !== ids.length - 1) idsForQuery += ","; - } return replays.selectAll( SQL`*` - )`WHERE private = 0 AND id IN (${idsForQuery}) LIMIT 51`.then(this.toReplays); - } + )`WHERE private = 0 AND id IN (${ids}) LIMIT 51`.then(this.toReplays); + } }; export default Replays; From 564ff3fe24ba22c93f8030a148fcef5f7e3731c9 Mon Sep 17 00:00:00 2001 From: MCGallaspy Date: Sun, 20 Oct 2024 08:24:32 -0700 Subject: [PATCH 6/6] Only watch src dir Prevents rebuild when writing e.g. logs to base dir. --- compose.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose.yaml b/compose.yaml index 87bc87c..1239ff2 100644 --- a/compose.yaml +++ b/compose.yaml @@ -12,7 +12,7 @@ services: develop: watch: - action: rebuild - path: . + path: src replay-db: restart: always hostname: replaysdb