Merge pull request #8122 from Fitblip/ryan/mina-bp-stats

Add initial sidecar + ingest cloud function

automation/services/mina-bp-stats/.gitignore

@@ -0,0 +1 @@
+.idea/

automation/services/mina-bp-stats/ingest-lambda/README.md

@@ -0,0 +1,10 @@
+# Mina Block Producer Ingest Lambda
+
+This is a simple ingestion lambda that tags incoming stats data and lands things in a GCS bucket. 
+
+## Configuration
+
+This lambda takes in 2 environment variables that should be configured in the google console.
+
+- `TOKEN` - The token used to authenticate incoming requests
+- `GOOGLE_STORAGE_BUCKET` - The GCS bucket to store incoming data in

automation/services/mina-bp-stats/ingest-lambda/index.js

@@ -0,0 +1,44 @@
+const {Storage} = require('@google-cloud/storage');
+
+exports.handleRequest = async (req, res) => {
+  if (process.env.TOKEN === undefined){
+    return res.status(500).send("TOKEN envar not set")
+  }
+  if (process.env.GOOGLE_STORAGE_BUCKET === undefined){
+   return res.status(500).send("GOOGLE_STORAGE_BUCKET envar not set") 
+  }
+
+  if (!req.query.token || req.query.token !== process.env.TOKEN){
+    return res.status(401).send("Bad token")
+  }
+
+  const now = new Date()  
+  const dateStamp = now.toISOString().split('T')[0]
+
+  const ipAddress = req.headers['x-forwarded-for'] || req.connection.remoteAddress
+  const receivedAt = now.getTime()
+
+  const recvPayload = req.body
+
+  const bpKeys = recvPayload.daemonStatus.blockProductionKeys
+
+  if (bpKeys.length === 0){
+    return res.status(400).send("Invalid block production keys")
+  }
+
+  const payload = {
+    receivedAt,
+    receivedFrom: ipAddress,
+    blockProducerKey: bpKeys[0],
+    nodeData: recvPayload
+  }
+
+  // Upload to gstorage
+  const storage = new Storage()
+  const myBucket = storage.bucket(process.env.GOOGLE_STORAGE_BUCKET)
+  const file = myBucket.file(`${dateStamp}.${now.getTime()}.${recvPayload.blockHeight}.json`)
+  const contents = JSON.stringify(payload, null, 2)
+  await file.save(contents, {contentType: "application/json"})
+
+  return res.status(200).send("OK")
+};

automation/services/mina-bp-stats/ingest-lambda/package.json

@@ -0,0 +1,7 @@
+{
+  "name": "mina-bp-ingest",
+  "version": "1.0.0",
+  "dependencies": {
+    "@google-cloud/storage": "^5.8.1"
+  }
+}

automation/services/mina-bp-stats/sidecar/.gitignore

@@ -0,0 +1,2 @@
+*.deb
+deb_build

automation/services/mina-bp-stats/sidecar/Dockerfile

@@ -0,0 +1,5 @@
+FROM python:alpine
+
+COPY sidecar.py /opt/sidecar.py
+
+CMD python3 /opt/sidecar.py

automation/services/mina-bp-stats/sidecar/README.md

@@ -0,0 +1,154 @@
+# Mina Block Producer Metrics Sidecar
+
+This is a simple sidecar that communicates with Mina nodes to ship off uptime data for analysis.
+
+Unless you're a founding block producer, you shouldn't need to run this sidecar, and you'll need to talk with the Mina team to get a special URL to make it work properly.
+
+## Configuration
+
+The sidecar takes 2 approaches to configuration, a pair of envars, or a configuration file. 
+
+**Note**: Environment variables always take precedence, even if the config file is available and valid. 
+
+#### Envars
+- `MINA_BP_UPLOAD_URL` - The URL to upload block producer statistics to
+- `MINA_NODE_URL` - The URL that the sidecar will reach out to to get statistics from
+
+#### Config File
+The mina metrics sidecar will also look at `/etc/mina-sidecar.json` for its configuration variables, and the file should look like this:
+
+```
+{
+  "uploadURL": "https://your.upload.url.here?token=someToken", 
+  "nodeURL": "https://your.mina.node.here:4321"
+}
+```
+
+The `uploadURL` parameter should be given to you by the Mina engineers
+
+## Running with Docker
+Running in docker should be as straight forward as running any other docker image. 
+
+#### Pulling from dockerhub
+We push updates to `minaprotocol/mina-bp-stats-sidecar:latest` so you can simply run the following to pull the image down:
+
+```
+$ docker pull minaprotocol/mina-bp-stats-sidecar:latest
+```
+
+#### Building locally
+This is un-necessary if you use the version from dockerhub (which is recommended).
+
+If you want to build this image yourself though, you can run `docker build -t mina-sidecar .` in this folder to build the image while naming it "mina-sidecar". 
+
+You should then substitute that in lieu of `minaprotocol/mina-bp-stats-sidecar:latest` for the rest of the commands below.
+
+#### Running with envars
+```bash
+$ docker run --rm -it -e MINA_BP_UPLOAD_URL=https://some-url-here -e MINA_NODE_URL=https://localhost:4321 minaprotocol/mina-bp-stats-sidecar:latest
+```
+
+#### Running with a config file
+```bash
+$ docker run --rm -it -v $(pwd)/mina-sidecar.json:/etc/mina-sidecar.json minaprotocol/mina-bp-stats-sidecar:latest
+```
+#### You can even bake your own docker image with the config file already in it
+```bash
+# Copy the example and make edits
+$ cp mina-sidecar-example.json mina-sidecar.json
+$ vim mina-sidecar.json # Make edits to the config
+# Create custom Dockerfile
+$ cat <<EOF > Dockerfile.custom
+FROM minaprotocol/mina-bp-stats-sidecar:latest
+COPY your_custom_config.conf /etc/mina-sidecar.json
+EOF
+$ docker build -t your-custom-sidecar -f Dockerfile.custom .
+$ docker run --rm -it your-custom-sidecar
+```
+
+## Running with debian package
+
+Running the sidecar as a debian package is as simple as installing the package, editing the config file, and enabling the service.
+
+#### Installing the package
+
+This package will install 3 files:
+
+- `/usr/local/bin/mina-bp-stats-sidecar` (the mina sidecar program)
+- `/etc/mina-sidecar.json` (the config file for the mina sidecar)
+- `/etc/systemd/system/mina-bp-stats-sidecar.service` (the systemd config to run it as a service)
+
+Installing the deb directly should be done with `apt install`, which will install the dependencies along side the service:
+
+```
+$ apt install ./mina-bp-stats-sidecar.deb
+```
+
+If you prefer to use `dpkg`, you can do so after installing the dependencies:
+
+```
+$ apt-get update && apt-get install python3 python3-certifi
+$ dpkg -i ./mina-bp-stats-sidecar.deb
+```
+
+#### Configuring and Running
+
+See the [Configuration](#Configuration) section above for what should be in the `/etc/mina-sidecar.json` file.
+
+To (optionally) enable the service to run on reboot you can use:
+
+```
+$ systemctl enable mina-bp-stats-sidecar
+```
+
+Then to start the service itself:
+
+```
+$ service mina-bp-stats-sidecar start
+```
+
+From there you can check that it's running and see the most recent logs with `service mina-bp-stats-sidecar status`:
+
+```
+$ service mina-bp-stats-sidecar status
+● mina-bp-stats-sidecar.service - Mina Block Producer Stats Sidecar
+   Loaded: loaded (/etc/systemd/system/mina-bp-stats-sidecar.service; disabled; vendor preset: enabled)
+   Active: active (running) since Fri 2021-03-12 02:43:37 CET; 3s ago
+ Main PID: 1906 (python3)
+    Tasks: 1 (limit: 2300)
+   CGroup: /system.slice/mina-bp-stats-sidecar.service
+           └─1906 python3 /usr/local/bin/mina-bp-stats-sidecar
+
+INFO:root:Found /etc/mina-sidecar.json on the filesystem, using config file
+INFO:root:Starting Mina Block Producer Sidecar
+INFO:root:Fetching block 2136...
+INFO:root:Got block data
+INFO:root:Finished! New tip 2136...
+```
+
+#### Monitoring/Logging
+
+If you want to get logs from the sidecar service, you can use `journalctl`:
+
+```
+# Similar to "tail -f" for the sidecar service
+$ journalctl -f -u mina-bp-stats-sidecar.service
+```
+
+## Issues
+
+#### HTTP error 400 
+
+If you get a 400 while running your sidecar:
+
+```
+INFO:root:Fetching block 2136...
+INFO:root:Got block data
+ERROR:root:HTTP Error 400: Bad Request
+
+-- TRACEBACK --
+
+ERROR:root:Sleeping for 30s and trying again
+```
+
+It likely means you're shipping off data to the ingest pipeline without any block producer key configured on your Mina node - since your BP key is your identity we can't accept node data since we don't know who is submitting it!

automation/services/mina-bp-stats/sidecar/build.sh

@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+
+BUILDDIR="${BUILDDIR:-deb_build}"
+
+# Get CWD if run locally or run through "source"
+CURRENT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
+
+rm -rf "${BUILDDIR}"
+
+mkdir -p "${BUILDDIR}/DEBIAN"
+
+cat << EOF > "${BUILDDIR}/DEBIAN/control"
+Package: mina-bp-stats-sidecar
+Version: ${VERSION}
+License: Apache-2.0
+Vendor: none
+Architecture: all
+Maintainer: o(1)Labs <[email protected]>
+Installed-Size:
+Depends: python3, python3-certifi
+Section: base
+Priority: optional
+Homepage: https://minaprotocol.com/
+Description: A telemetry sidecar that ships stats about node status
+ back to Mina HQ for analysis.
+ Built from ${GITHASH} by ${BUILD_URL}
+EOF
+
+mkdir -p "${BUILDDIR}/usr/local/bin"
+mkdir -p "${BUILDDIR}/etc"
+mkdir -p "${BUILDDIR}/etc/systemd/system/"
+
+cp "${CURRENT_DIR}/sidecar.py" "${BUILDDIR}/usr/local/bin/mina-bp-stats-sidecar"
+cp "${CURRENT_DIR}/mina-sidecar-example.json" "${BUILDDIR}/etc/mina-sidecar.json"
+cp "${CURRENT_DIR}/mina-bp-stats-sidecar.service" "${BUILDDIR}/etc/systemd/system/mina-bp-stats-sidecar.service"
+
+fakeroot dpkg-deb --build "${BUILDDIR}" "mina-sidecar_${VERSION}.deb"
+
+rm -rf "${BUILDDIR}"

automation/services/mina-bp-stats/sidecar/mina-bp-stats-sidecar.service

@@ -0,0 +1,7 @@
+[Unit]
+Description=Mina Block Producer Stats Sidecar
+[Service]
+ExecStart=/usr/local/bin/mina-bp-stats-sidecar
+SuccessExitStatus=143
+[Install]
+WantedBy=multi-user.target

automation/services/mina-bp-stats/sidecar/mina-sidecar-example.json

@@ -0,0 +1,4 @@
+{
+  "uploadURL": "https://some-host.somewhere/some-endpoing?token=some-token",
+  "nodeURL": "https://some.node.somewhere:3085"
+}