diff --git a/automation/services/mina-bp-stats/.gitignore b/automation/services/mina-bp-stats/.gitignore new file mode 100644 index 00000000000..62c893550ad --- /dev/null +++ b/automation/services/mina-bp-stats/.gitignore @@ -0,0 +1 @@ +.idea/ \ No newline at end of file diff --git a/automation/services/mina-bp-stats/ingest-lambda/README.md b/automation/services/mina-bp-stats/ingest-lambda/README.md new file mode 100644 index 00000000000..15ccc99c224 --- /dev/null +++ b/automation/services/mina-bp-stats/ingest-lambda/README.md @@ -0,0 +1,10 @@ +# Mina Block Producer Ingest Lambda + +This is a simple ingestion lambda that tags incoming stats data and lands things in a GCS bucket. + +## Configuration + +This lambda takes in 2 environment variables that should be configured in the google console. + +- `TOKEN` - The token used to authenticate incoming requests +- `GOOGLE_STORAGE_BUCKET` - The GCS bucket to store incoming data in diff --git a/automation/services/mina-bp-stats/ingest-lambda/index.js b/automation/services/mina-bp-stats/ingest-lambda/index.js new file mode 100644 index 00000000000..4ddfbe466d9 --- /dev/null +++ b/automation/services/mina-bp-stats/ingest-lambda/index.js @@ -0,0 +1,44 @@ +const {Storage} = require('@google-cloud/storage'); + +exports.handleRequest = async (req, res) => { + if (process.env.TOKEN === undefined){ + return res.status(500).send("TOKEN envar not set") + } + if (process.env.GOOGLE_STORAGE_BUCKET === undefined){ + return res.status(500).send("GOOGLE_STORAGE_BUCKET envar not set") + } + + if (!req.query.token || req.query.token !== process.env.TOKEN){ + return res.status(401).send("Bad token") + } + + const now = new Date() + const dateStamp = now.toISOString().split('T')[0] + + const ipAddress = req.headers['x-forwarded-for'] || req.connection.remoteAddress + const receivedAt = now.getTime() + + const recvPayload = req.body + + const bpKeys = recvPayload.daemonStatus.blockProductionKeys + + if (bpKeys.length === 0){ + return res.status(400).send("Invalid block production keys") + } + + const payload = { + receivedAt, + receivedFrom: ipAddress, + blockProducerKey: bpKeys[0], + nodeData: recvPayload + } + + // Upload to gstorage + const storage = new Storage() + const myBucket = storage.bucket(process.env.GOOGLE_STORAGE_BUCKET) + const file = myBucket.file(`${dateStamp}.${now.getTime()}.${recvPayload.blockHeight}.json`) + const contents = JSON.stringify(payload, null, 2) + await file.save(contents, {contentType: "application/json"}) + + return res.status(200).send("OK") +}; diff --git a/automation/services/mina-bp-stats/ingest-lambda/package.json b/automation/services/mina-bp-stats/ingest-lambda/package.json new file mode 100644 index 00000000000..e07d3274999 --- /dev/null +++ b/automation/services/mina-bp-stats/ingest-lambda/package.json @@ -0,0 +1,7 @@ +{ + "name": "mina-bp-ingest", + "version": "1.0.0", + "dependencies": { + "@google-cloud/storage": "^5.8.1" + } +} diff --git a/automation/services/mina-bp-stats/sidecar/.gitignore b/automation/services/mina-bp-stats/sidecar/.gitignore new file mode 100644 index 00000000000..12d6a9c3220 --- /dev/null +++ b/automation/services/mina-bp-stats/sidecar/.gitignore @@ -0,0 +1,2 @@ +*.deb +deb_build diff --git a/automation/services/mina-bp-stats/sidecar/Dockerfile b/automation/services/mina-bp-stats/sidecar/Dockerfile new file mode 100644 index 00000000000..c87006499ff --- /dev/null +++ b/automation/services/mina-bp-stats/sidecar/Dockerfile @@ -0,0 +1,5 @@ +FROM python:alpine + +COPY sidecar.py /opt/sidecar.py + +CMD python3 /opt/sidecar.py \ No newline at end of file diff --git a/automation/services/mina-bp-stats/sidecar/README.md b/automation/services/mina-bp-stats/sidecar/README.md new file mode 100644 index 00000000000..9d34f7dfbbf --- /dev/null +++ b/automation/services/mina-bp-stats/sidecar/README.md @@ -0,0 +1,154 @@ +# Mina Block Producer Metrics Sidecar + +This is a simple sidecar that communicates with Mina nodes to ship off uptime data for analysis. + +Unless you're a founding block producer, you shouldn't need to run this sidecar, and you'll need to talk with the Mina team to get a special URL to make it work properly. + +## Configuration + +The sidecar takes 2 approaches to configuration, a pair of envars, or a configuration file. + +**Note**: Environment variables always take precedence, even if the config file is available and valid. + +#### Envars +- `MINA_BP_UPLOAD_URL` - The URL to upload block producer statistics to +- `MINA_NODE_URL` - The URL that the sidecar will reach out to to get statistics from + +#### Config File +The mina metrics sidecar will also look at `/etc/mina-sidecar.json` for its configuration variables, and the file should look like this: + +``` +{ + "uploadURL": "https://your.upload.url.here?token=someToken", + "nodeURL": "https://your.mina.node.here:4321" +} +``` + +The `uploadURL` parameter should be given to you by the Mina engineers + +## Running with Docker +Running in docker should be as straight forward as running any other docker image. + +#### Pulling from dockerhub +We push updates to `minaprotocol/mina-bp-stats-sidecar:latest` so you can simply run the following to pull the image down: + +``` +$ docker pull minaprotocol/mina-bp-stats-sidecar:latest +``` + +#### Building locally +This is un-necessary if you use the version from dockerhub (which is recommended). + +If you want to build this image yourself though, you can run `docker build -t mina-sidecar .` in this folder to build the image while naming it "mina-sidecar". + +You should then substitute that in lieu of `minaprotocol/mina-bp-stats-sidecar:latest` for the rest of the commands below. + +#### Running with envars +```bash +$ docker run --rm -it -e MINA_BP_UPLOAD_URL=https://some-url-here -e MINA_NODE_URL=https://localhost:4321 minaprotocol/mina-bp-stats-sidecar:latest +``` + +#### Running with a config file +```bash +$ docker run --rm -it -v $(pwd)/mina-sidecar.json:/etc/mina-sidecar.json minaprotocol/mina-bp-stats-sidecar:latest +``` +#### You can even bake your own docker image with the config file already in it +```bash +# Copy the example and make edits +$ cp mina-sidecar-example.json mina-sidecar.json +$ vim mina-sidecar.json # Make edits to the config +# Create custom Dockerfile +$ cat < Dockerfile.custom +FROM minaprotocol/mina-bp-stats-sidecar:latest +COPY your_custom_config.conf /etc/mina-sidecar.json +EOF +$ docker build -t your-custom-sidecar -f Dockerfile.custom . +$ docker run --rm -it your-custom-sidecar +``` + +## Running with debian package + +Running the sidecar as a debian package is as simple as installing the package, editing the config file, and enabling the service. + +#### Installing the package + +This package will install 3 files: + +- `/usr/local/bin/mina-bp-stats-sidecar` (the mina sidecar program) +- `/etc/mina-sidecar.json` (the config file for the mina sidecar) +- `/etc/systemd/system/mina-bp-stats-sidecar.service` (the systemd config to run it as a service) + +Installing the deb directly should be done with `apt install`, which will install the dependencies along side the service: + +``` +$ apt install ./mina-bp-stats-sidecar.deb +``` + +If you prefer to use `dpkg`, you can do so after installing the dependencies: + +``` +$ apt-get update && apt-get install python3 python3-certifi +$ dpkg -i ./mina-bp-stats-sidecar.deb +``` + +#### Configuring and Running + +See the [Configuration](#Configuration) section above for what should be in the `/etc/mina-sidecar.json` file. + +To (optionally) enable the service to run on reboot you can use: + +``` +$ systemctl enable mina-bp-stats-sidecar +``` + +Then to start the service itself: + +``` +$ service mina-bp-stats-sidecar start +``` + +From there you can check that it's running and see the most recent logs with `service mina-bp-stats-sidecar status`: + +``` +$ service mina-bp-stats-sidecar status +● mina-bp-stats-sidecar.service - Mina Block Producer Stats Sidecar + Loaded: loaded (/etc/systemd/system/mina-bp-stats-sidecar.service; disabled; vendor preset: enabled) + Active: active (running) since Fri 2021-03-12 02:43:37 CET; 3s ago + Main PID: 1906 (python3) + Tasks: 1 (limit: 2300) + CGroup: /system.slice/mina-bp-stats-sidecar.service + └─1906 python3 /usr/local/bin/mina-bp-stats-sidecar + +INFO:root:Found /etc/mina-sidecar.json on the filesystem, using config file +INFO:root:Starting Mina Block Producer Sidecar +INFO:root:Fetching block 2136... +INFO:root:Got block data +INFO:root:Finished! New tip 2136... +``` + +#### Monitoring/Logging + +If you want to get logs from the sidecar service, you can use `journalctl`: + +``` +# Similar to "tail -f" for the sidecar service +$ journalctl -f -u mina-bp-stats-sidecar.service +``` + +## Issues + +#### HTTP error 400 + +If you get a 400 while running your sidecar: + +``` +INFO:root:Fetching block 2136... +INFO:root:Got block data +ERROR:root:HTTP Error 400: Bad Request + +-- TRACEBACK -- + +ERROR:root:Sleeping for 30s and trying again +``` + +It likely means you're shipping off data to the ingest pipeline without any block producer key configured on your Mina node - since your BP key is your identity we can't accept node data since we don't know who is submitting it! \ No newline at end of file diff --git a/automation/services/mina-bp-stats/sidecar/build.sh b/automation/services/mina-bp-stats/sidecar/build.sh new file mode 100755 index 00000000000..b0119047030 --- /dev/null +++ b/automation/services/mina-bp-stats/sidecar/build.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash + +BUILDDIR="${BUILDDIR:-deb_build}" + +# Get CWD if run locally or run through "source" +CURRENT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" + +rm -rf "${BUILDDIR}" + +mkdir -p "${BUILDDIR}/DEBIAN" + +cat << EOF > "${BUILDDIR}/DEBIAN/control" +Package: mina-bp-stats-sidecar +Version: ${VERSION} +License: Apache-2.0 +Vendor: none +Architecture: all +Maintainer: o(1)Labs +Installed-Size: +Depends: python3, python3-certifi +Section: base +Priority: optional +Homepage: https://minaprotocol.com/ +Description: A telemetry sidecar that ships stats about node status + back to Mina HQ for analysis. + Built from ${GITHASH} by ${BUILD_URL} +EOF + +mkdir -p "${BUILDDIR}/usr/local/bin" +mkdir -p "${BUILDDIR}/etc" +mkdir -p "${BUILDDIR}/etc/systemd/system/" + +cp "${CURRENT_DIR}/sidecar.py" "${BUILDDIR}/usr/local/bin/mina-bp-stats-sidecar" +cp "${CURRENT_DIR}/mina-sidecar-example.json" "${BUILDDIR}/etc/mina-sidecar.json" +cp "${CURRENT_DIR}/mina-bp-stats-sidecar.service" "${BUILDDIR}/etc/systemd/system/mina-bp-stats-sidecar.service" + +fakeroot dpkg-deb --build "${BUILDDIR}" "mina-sidecar_${VERSION}.deb" + +rm -rf "${BUILDDIR}" \ No newline at end of file diff --git a/automation/services/mina-bp-stats/sidecar/mina-bp-stats-sidecar.service b/automation/services/mina-bp-stats/sidecar/mina-bp-stats-sidecar.service new file mode 100644 index 00000000000..d7fc212813c --- /dev/null +++ b/automation/services/mina-bp-stats/sidecar/mina-bp-stats-sidecar.service @@ -0,0 +1,7 @@ +[Unit] +Description=Mina Block Producer Stats Sidecar +[Service] +ExecStart=/usr/local/bin/mina-bp-stats-sidecar +SuccessExitStatus=143 +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/automation/services/mina-bp-stats/sidecar/mina-sidecar-example.json b/automation/services/mina-bp-stats/sidecar/mina-sidecar-example.json new file mode 100644 index 00000000000..179a5c8c708 --- /dev/null +++ b/automation/services/mina-bp-stats/sidecar/mina-sidecar-example.json @@ -0,0 +1,4 @@ +{ + "uploadURL": "https://some-host.somewhere/some-endpoing?token=some-token", + "nodeURL": "https://some.node.somewhere:3085" +} diff --git a/automation/services/mina-bp-stats/sidecar/sidecar.py b/automation/services/mina-bp-stats/sidecar/sidecar.py new file mode 100755 index 00000000000..fc77cf74d9c --- /dev/null +++ b/automation/services/mina-bp-stats/sidecar/sidecar.py @@ -0,0 +1,161 @@ +#!/usr/bin/env python3 + +import os +import json +import logging +import time +import math +import urllib.request +import urllib.parse + +logging.basicConfig(level=logging.INFO) + +MINA_CONFIG_FILE = '/etc/mina-sidecar.json' +MINA_BLOCK_PRODUCER_URL_ENVAR = 'MINA_BP_UPLOAD_URL' +MINA_NODE_URL_ENVAR = 'MINA_NODE_URL' + +FETCH_INTERVAL = 60 * 3 # Fetch updates every 3 mins +ERROR_SLEEP_INTERVAL = 30 # On errors, sleep for 30s before trying again +FINALIZATION_THRESHOLD = 12 # 12 blocks back is considered "finalized" + +SYNC_STATUS_GRAPHQL = ''' +query SyncStatus { + daemonStatus { + syncStatus + blockchainLength + } +} +''' + +FETCH_BLOCK_GRAPHQL = ''' +query FetchBlockData($blockID: Int!) { + version + daemonStatus { + blockchainLength + syncStatus + chainId + commitId + highestBlockLengthReceived + highestUnvalidatedBlockLengthReceived + stateHash + blockProductionKeys + uptimeSecs + } + block(height: $blockID) { + stateHash + } +} +''' + +upload_url, node_url = (None, None) + +if os.path.exists(MINA_CONFIG_FILE): + logging.info("Found {} on the filesystem, using config file".format(MINA_CONFIG_FILE)) + with open(MINA_CONFIG_FILE) as f: + config_file = f.read().strip() + parsed_config_file = json.loads(config_file) + upload_url = parsed_config_file['uploadURL'].rstrip('/') + node_url = parsed_config_file['nodeURL'].rstrip('/') + +if MINA_BLOCK_PRODUCER_URL_ENVAR in os.environ: + logging.info("Found {} in the environment, using envar".format(MINA_BLOCK_PRODUCER_URL_ENVAR)) + upload_url = os.environ[MINA_BLOCK_PRODUCER_URL_ENVAR] + +if MINA_NODE_URL_ENVAR in os.environ: + logging.info("Found {} in the environment, using envar".format(MINA_NODE_URL_ENVAR)) + node_url = os.environ[MINA_NODE_URL_ENVAR] + +if upload_url is None: + raise Exception("Could not find {} or {} environment variable is not set.".format(MINA_CONFIG_FILE, MINA_BLOCK_PRODUCER_URL_ENVAR)) + +if node_url is None: + raise Exception("Could not find {} or {} environment variable is not set.".format(MINA_CONFIG_FILE, MINA_NODE_URL_ENVAR)) + +def fetch_mina_status(): + url = node_url + '/graphql' + request = urllib.request.Request( + url, + headers={'Content-Type': 'application/json'}, + data=json.dumps({ + "query": SYNC_STATUS_GRAPHQL, + "variables": {}, + "operationName": "SyncStatus" + }).encode() + ) + response = urllib.request.urlopen(request) + response_body = response.read().decode('utf-8') + parsed_body = json.loads(response_body)['data'] + + return parsed_body['daemonStatus']['syncStatus'], parsed_body['daemonStatus']['blockchainLength'] + +def fetch_block(block_id): + url = node_url + '/graphql' + request = urllib.request.Request( + url, + headers={'Content-Type': 'application/json'}, + data=json.dumps({ + "query": FETCH_BLOCK_GRAPHQL, + "variables": {'blockID': block_id}, + "operationName": "FetchBlockData" + }).encode() + ) + + response = urllib.request.urlopen(request) + response_body = response.read().decode('utf-8') + response_data = json.loads(response_body)['data'] + if response_data is None: + raise Exception("Response seems to be an error! {}".format(response_body)) + + return response_data + +def send_update(block_data, block_height): + block_data.update({ + "retrievedAt": math.floor(time.time() * 1000), + "blockHeight": block_height + }) + request = urllib.request.Request( + upload_url, + headers={'Content-Type': 'application/json'}, + data=json.dumps(block_data).encode() + ) + + response = urllib.request.urlopen(request) + + assert response.getcode() == 200, "Non-200 from BP flush endpoint! [{}] - ".format(response.getcode(), response.read()) + +if __name__ == '__main__': + logging.info("Starting Mina Block Producer Sidecar") + # Go ensure the node is up and happy + while True: + try: + mina_sync_status, head_block_id = fetch_mina_status() + if mina_sync_status == "SYNCED" or mina_sync_status == "CATCHUP": + logging.debug("Mina sync status is acceptable ({}), continuing!".format(mina_sync_status)) + break + logging.info("Mina sync status is {}. Sleeping for 5s and trying again".format(mina_sync_status)) + except Exception as e: + logging.exception(e) + + time.sleep(5) + + # Go back FINALIZATION_THRESHOLD blocks from the tip to have a finalized block + current_finalized_tip = head_block_id - FINALIZATION_THRESHOLD + + # We're done with init to the point where we can start shipping off data + while True: + try: + logging.info("Fetching block {}...".format(current_finalized_tip)) + + block_data = fetch_block(current_finalized_tip) + + logging.info("Got block data ", block_data) + + send_update(block_data, current_finalized_tip) + + current_finalized_tip = block_data['daemonStatus']['blockchainLength'] - FINALIZATION_THRESHOLD # Go set a new finalized block + logging.info("Finished! New tip {}...".format(current_finalized_tip)) + time.sleep(FETCH_INTERVAL) + except Exception as e: + logging.exception(e) + logging.error("Sleeping for {}s and trying again".format(ERROR_SLEEP_INTERVAL)) + time.sleep(ERROR_SLEEP_INTERVAL) diff --git a/scripts/rebuild-deb.sh b/scripts/rebuild-deb.sh index 683be25f753..4f3cade5242 100755 --- a/scripts/rebuild-deb.sh +++ b/scripts/rebuild-deb.sh @@ -246,6 +246,10 @@ ls -lh mina*.deb #remove build dir to prevent running out of space on the host machine rm -rf "${BUILDDIR}" +# Build mina block producer sidecar +source ../automation/services/mina-bp-stats/sidecar/build.sh +ls -lh mina*.deb + # Export variables for use with downstream circle-ci steps (see buildkite/scripts/publish-deb.sh for BK DOCKER_DEPLOY_ENV) echo "export CODA_DEB_VERSION=$VERSION" >> /tmp/DOCKER_DEPLOY_ENV