From 12b7179559adbe863806582e2ee0943a2b724d89 Mon Sep 17 00:00:00 2001 From: danclay <18021681+danclay@users.noreply.github.com> Date: Sat, 16 Apr 2022 21:59:02 -0700 Subject: [PATCH 1/4] Updated central request handler Added bigint serialization --- src/sharding/Admiral.ts | 8 +++-- src/util/CentralRequestHandler.ts | 6 ++-- src/util/IPC.ts | 2 +- .../{ErrorHandler.ts => Serialization.ts} | 31 ++++++++++++++++++- 4 files changed, 40 insertions(+), 7 deletions(-) rename src/util/{ErrorHandler.ts => Serialization.ts} (52%) diff --git a/src/sharding/Admiral.ts b/src/sharding/Admiral.ts index ccbd517..44da463 100644 --- a/src/sharding/Admiral.ts +++ b/src/sharding/Admiral.ts @@ -11,7 +11,7 @@ import {Cluster} from "../clusters/Cluster"; import {Service} from "../services/Service"; import path from "path"; import { inspect } from "util"; -import { errorToJSON, reconstructError } from "../util/ErrorHandler"; +import { errorToJSON, parseJSON, reconstructError, stringifyJSON } from "../util/Serialization"; export interface ShardUpdate { shardID: number; @@ -803,7 +803,8 @@ export class Admiral extends EventEmitter { break; } case "centralApiRequest": { - this.centralApiRequest(worker, message.request.UUID, message.request.data); + const data = parseJSON(message.request.dataSerialized); + this.centralApiRequest(worker, message.request.UUID, data); break; } case "shardUpdate": { @@ -1662,12 +1663,13 @@ export class Admiral extends EventEmitter { private centralApiRequest(worker: master.Worker, UUID: string, data: {method: Eris.RequestMethod, url: string, auth?: boolean, body?: { [s: string]: unknown }, file?: Eris.FileContent, fileString?: string, _route?: string, short?: boolean}) { const reply = (resolved: boolean, value: unknown) => { + const valueSerialized = stringifyJSON(value); worker.send({ op: "centralApiResponse", id: UUID, value: { resolved, - value + valueSerialized } }); }; diff --git a/src/util/CentralRequestHandler.ts b/src/util/CentralRequestHandler.ts index 715cfb6..d5c8439 100644 --- a/src/util/CentralRequestHandler.ts +++ b/src/util/CentralRequestHandler.ts @@ -1,6 +1,6 @@ import { IPC } from "./IPC"; import crypto from "crypto"; -import { reconstructError } from "./ErrorHandler"; +import { parseJSON, reconstructError, stringifyJSON } from "./Serialization"; import Eris from "eris"; interface CentralRequestHandlerOptions { @@ -21,6 +21,7 @@ export class CentralRequestHandler { if (message.op === "centralApiResponse") { const request = this.requests.get(message.id); if (request) { + message.value.value = parseJSON(message.value.valueSerialized); request(message.value); } } @@ -38,8 +39,9 @@ export class CentralRequestHandler { } } const data = {method, url, auth, body, file, fileString, _route, short}; + const dataSerialized = stringifyJSON(data); - if (process.send) process.send({op: "centralApiRequest", request: {UUID, data}}); + if (process.send) process.send({op: "centralApiRequest", request: {UUID, dataSerialized}}); return new Promise((resolve, reject) => { // timeout diff --git a/src/util/IPC.ts b/src/util/IPC.ts index eaac16b..f9151f1 100644 --- a/src/util/IPC.ts +++ b/src/util/IPC.ts @@ -1,7 +1,7 @@ import {EventEmitter} from "events"; import {ClusterCollection, ServiceCollection, Stats, ReshardOptions} from "../sharding/Admiral"; import crypto from "crypto"; -import { errorToJSON } from "./ErrorHandler"; +import { errorToJSON } from "./Serialization"; import path from "path"; import { Collection } from "../util/Collection"; import { isMaster } from "cluster"; diff --git a/src/util/ErrorHandler.ts b/src/util/Serialization.ts similarity index 52% rename from src/util/ErrorHandler.ts rename to src/util/Serialization.ts index 74be3d2..7da977f 100644 --- a/src/util/ErrorHandler.ts +++ b/src/util/Serialization.ts @@ -27,4 +27,33 @@ const reconstructError = (data: NodeJS.ErrnoException): NodeJS.ErrnoException => return result as NodeJS.ErrnoException; }; -export {reconstructError, errorToJSON, SentError}; \ No newline at end of file +const stringifyJSON = (data: any) => { + return JSON.stringify(data, (key, value) => { + switch(typeof value) { + case "bigint": { + return `BIGINT::${value}`; + } + case "undefined": { + return "UNDEFINED::"; + } + default: { + return value; + } + } + }); +}; + +const parseJSON = (json: string) => { + return JSON.parse(json, (key, value) => { + if (typeof value === "string") { + if (value.startsWith("BIGINT::")) { + return BigInt(value.substring(8)); + } else if (value.startsWith("UNDEFINED::")) { + return undefined; + } + } + return value; + }); +}; + +export {reconstructError, errorToJSON, stringifyJSON, parseJSON, SentError}; \ No newline at end of file From 3b8bf074af4bccab5e21fea7ecd8992d1753927a Mon Sep 17 00:00:00 2001 From: danclay <18021681+danclay@users.noreply.github.com> Date: Sun, 17 Apr 2022 01:29:53 -0700 Subject: [PATCH 2/4] Update README.md - Added a little basic diagram - Fixed some links - Updated packages --- README.md | 27 ++++++++++---------- package.json | 2 +- yarn.lock | 70 ++++++++++++++++++++++++++-------------------------- 3 files changed, 49 insertions(+), 50 deletions(-) diff --git a/README.md b/README.md index b06f50d..d7dfa4b 100644 --- a/README.md +++ b/README.md @@ -38,6 +38,10 @@ eris-fleet currently supports Eris v0.16.x. - Can use a modified version of Eris - Concurrency support +A very basic diagram: + +![Basic diagram](https://cdn.discordapp.com/attachments/866590047436144641/965161462479859742/Untitled_Diagram.drawio_1.png) + ## Help If you still have questions, you can join the support server on Discord: [Discord server](https://discord.gg/tk2n3naJn3) @@ -96,9 +100,9 @@ if (isMaster) { Admiral.on('stats', m => console.log(m)); } ``` -This creates a new Admiral that will manage `bot.js` running in other processes. [More details](https://danclay.github.io/eris-fleet/classes/baseclusterworker.html) +This creates a new Admiral that will manage `bot.js` running in other processes. [More details](https://danclay.github.io/eris-fleet/classes/BaseClusterWorker.html) -The following is an example of `bot.js`. [Read the IPC docs](https://danclay.github.io/eris-fleet/classes/ipc.html) for what you can access and do with clusters. +The following is an example of `bot.js`. [Read the IPC docs](https://danclay.github.io/eris-fleet/classes/IPC.html) for what you can access and do with clusters. ```js const { BaseClusterWorker } = require('eris-fleet'); @@ -136,7 +140,7 @@ The bot above will respond with "Pong!" when it receives the command "!ping". ## Services -You can create services for your bot. Services are workers which do not interact directly with Eris. Services are useful for processing tasks, a central location to get the latest version of languages for your bot, custom statistics, and more! [Read the IPC docs](https://danclay.github.io/eris-fleet/classes/ipc.html) for what you can access and do with services. **Note that services always start before the clusters. Clusters will only start after all the services have started.** [More details](https://danclay.github.io/eris-fleet/classes/baseserviceworker.html) +You can create services for your bot. Services are workers which do not interact directly with Eris. Services are useful for processing tasks, a central location to get the latest version of languages for your bot, custom statistics, and more! [Read the IPC docs](https://danclay.github.io/eris-fleet/classes/IPC.html) for what you can access and do with services. **Note that services always start before the clusters. Clusters will only start after all the services have started.** [More details](https://danclay.github.io/eris-fleet/classes/BaseServiceWorker.html) To add a service, add the following to the options you pass to the fleet: @@ -266,24 +270,19 @@ Stats are given in [this](https://danclay.github.io/eris-fleet/interfaces/Stats. ## Using a specific version of Eris or a modified version of Eris -You can use an extended Eris client by passing it to the Options. (see the [options.customClient](https://danclay.github.io/eris-fleet/interfaces/Options.html#customclient) section). +You can use an extended Eris client by passing it to the Options. (see the [options.customClient](https://danclay.github.io/eris-fleet/interfaces/Options.html#customClient) section). Eris-fleet is able to use packages such as eris-additions if you desire. To do so, modify your bot file to match the following template: ```js -const { BaseClusterWorker } = require('eris-fleet'); - // Example using eris-additions +const { Fleet } = require("eris-fleet"); const Eris = require("eris-additions")(require("eris")); -class BotWorker extends BaseClusterWorker { - constructor(setup) { - super(setup); - // Your cool stuff - } +const options = { + // other options + customClient: Eris } - -// This export is needed for this to work. -module.exports = {BotWorker, Eris}; +const Admiral = new Fleet(options); ``` ## Using ES Modules diff --git a/package.json b/package.json index 44ed4a8..b30f9b2 100644 --- a/package.json +++ b/package.json @@ -15,7 +15,7 @@ "dotenv": "^10.0.0", "eris": "~0.16.0", "eslint": "^7.32.0", - "typedoc": "^0.22.9", + "typedoc": "^0.22.15", "typescript": "4.4.x" }, "scripts": { diff --git a/yarn.lock b/yarn.lock index 4a16daa..39fbdaa 100644 --- a/yarn.lock +++ b/yarn.lock @@ -249,6 +249,13 @@ brace-expansion@^1.1.7: balanced-match "^1.0.0" concat-map "0.0.1" +brace-expansion@^2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-2.0.1.tgz#1edc459e0f0c548486ecf9fc99f2221364b9a0ae" + integrity sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA== + dependencies: + balanced-match "^1.0.0" + braces@^3.0.1: version "3.0.2" resolved "https://registry.yarnpkg.com/braces/-/braces-3.0.2.tgz#3454e1a462ee8d599e236df336cd9ea4f8afe107" @@ -732,13 +739,6 @@ lodash.truncate@^4.4.2: resolved "https://registry.yarnpkg.com/lodash.truncate/-/lodash.truncate-4.4.2.tgz#5a350da0b1113b837ecfffd5812cbe58d6eae193" integrity sha1-WjUNoLERO4N+z//VgSy+WNbq4ZM= -lru-cache@^5.1.1: - version "5.1.1" - resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-5.1.1.tgz#1da27e6710271947695daf6848e847f01d84b920" - integrity sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w== - dependencies: - yallist "^3.0.2" - lru-cache@^6.0.0: version "6.0.0" resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-6.0.0.tgz#6d6fe6570ebd96aaf90fcad1dafa3b2566db3a94" @@ -751,10 +751,10 @@ lunr@^2.3.9: resolved "https://registry.yarnpkg.com/lunr/-/lunr-2.3.9.tgz#18b123142832337dd6e964df1a5a7707b25d35e1" integrity sha512-zTU3DaZaF3Rt9rhN3uBMGQD3dD2/vFQqnvZCDv4dl5iOzq2IZQqTxu90r4E5J+nP70J3ilqVCrbho2eWaeW8Ow== -marked@^3.0.8: - version "3.0.8" - resolved "https://registry.yarnpkg.com/marked/-/marked-3.0.8.tgz#2785f0dc79cbdc6034be4bb4f0f0a396bd3f8aeb" - integrity sha512-0gVrAjo5m0VZSJb4rpL59K1unJAMb/hm8HRXqasD8VeC8m91ytDPMritgFSlKonfdt+rRYYpP/JfLxgIX8yoSw== +marked@^4.0.12: + version "4.0.14" + resolved "https://registry.yarnpkg.com/marked/-/marked-4.0.14.tgz#7a3a5fa5c80580bac78c1ed2e3b84d7bd6fc3870" + integrity sha512-HL5sSPE/LP6U9qKgngIIPTthuxC0jrfxpYMZ3LdGDD3vTnLs59m2Z7r6+LNDR3ToqEQdkKd6YaaEfJhodJmijQ== merge2@^1.3.0: version "1.4.1" @@ -776,6 +776,13 @@ minimatch@^3.0.4: dependencies: brace-expansion "^1.1.7" +minimatch@^5.0.1: + version "5.0.1" + resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-5.0.1.tgz#fb9022f7528125187c92bd9e9b6366be1cf3415b" + integrity sha512-nLDxIFRyhDblz3qMuq+SoRZED4+miJ/G+tdDrjkkkRnjAsBexeGpgjLEQ0blJy7rHhR2b93rhQY4SvyWu9v03g== + dependencies: + brace-expansion "^2.0.1" + ms@2.1.2: version "2.1.2" resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.2.tgz#d09d1f357b443f493382a8eb3ccd183872ae6009" @@ -793,13 +800,6 @@ once@^1.3.0: dependencies: wrappy "1" -onigasm@^2.2.5: - version "2.2.5" - resolved "https://registry.yarnpkg.com/onigasm/-/onigasm-2.2.5.tgz#cc4d2a79a0fa0b64caec1f4c7ea367585a676892" - integrity sha512-F+th54mPc0l1lp1ZcFMyL/jTs2Tlq4SqIHKIXGZOR/VkHkF9A7Fr5rRr5+ZG/lWeRsyrClLYRq7s/yFQ/XhWCA== - dependencies: - lru-cache "^5.1.1" - optionator@^0.9.1: version "0.9.1" resolved "https://registry.yarnpkg.com/optionator/-/optionator-0.9.1.tgz#4f236a6373dae0566a6d43e1326674f50c291499" @@ -917,13 +917,13 @@ shebang-regex@^3.0.0: resolved "https://registry.yarnpkg.com/shebang-regex/-/shebang-regex-3.0.0.tgz#ae16f1644d873ecad843b0307b143362d4c42172" integrity sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A== -shiki@^0.9.12: - version "0.9.12" - resolved "https://registry.yarnpkg.com/shiki/-/shiki-0.9.12.tgz#70cbc8c1bb78ff7b356f84a7eecdb040efddd247" - integrity sha512-VXcROdldv0/Qu0w2XvzU4IrvTeBNs/Kj/FCmtcEXGz7Tic/veQzliJj6tEiAgoKianhQstpYmbPDStHU5Opqcw== +shiki@^0.10.1: + version "0.10.1" + resolved "https://registry.yarnpkg.com/shiki/-/shiki-0.10.1.tgz#6f9a16205a823b56c072d0f1a0bcd0f2646bef14" + integrity sha512-VsY7QJVzU51j5o1+DguUd+6vmCmZ5v/6gYu4vyYAhzjuNQU6P/vmSy4uQaOhvje031qQMiW0d2BwgMH52vqMng== dependencies: jsonc-parser "^3.0.0" - onigasm "^2.2.5" + vscode-oniguruma "^1.6.1" vscode-textmate "5.2.0" slash@^3.0.0: @@ -1032,16 +1032,16 @@ type-fest@^0.20.2: resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.20.2.tgz#1bf207f4b28f91583666cb5fbd327887301cd5f4" integrity sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ== -typedoc@^0.22.9: - version "0.22.9" - resolved "https://registry.yarnpkg.com/typedoc/-/typedoc-0.22.9.tgz#7e3f4f72586bbef4c2382f0afcea8e0f9270b712" - integrity sha512-84PjudoXVcap6bwdZFbYIUWlgdz/iLV09ZHwrCzhtHWXaDQG6mlosJ8te6DSThuRkRvQjp46HO+qY/P7Gpm78g== +typedoc@^0.22.15: + version "0.22.15" + resolved "https://registry.yarnpkg.com/typedoc/-/typedoc-0.22.15.tgz#c6ad7ed9d017dc2c3a06c9189cb392bd8e2d8c3f" + integrity sha512-CMd1lrqQbFvbx6S9G6fL4HKp3GoIuhujJReWqlIvSb2T26vGai+8Os3Mde7Pn832pXYemd9BMuuYWhFpL5st0Q== dependencies: glob "^7.2.0" lunr "^2.3.9" - marked "^3.0.8" - minimatch "^3.0.4" - shiki "^0.9.12" + marked "^4.0.12" + minimatch "^5.0.1" + shiki "^0.10.1" typescript@4.4.x: version "4.4.4" @@ -1060,6 +1060,11 @@ v8-compile-cache@^2.0.3: resolved "https://registry.yarnpkg.com/v8-compile-cache/-/v8-compile-cache-2.3.0.tgz#2de19618c66dc247dcfb6f99338035d8245a2cee" integrity sha512-l8lCEmLcLYZh4nbunNZvQCJc5pv7+RCwa8q/LdUx8u7lsWvPDKmpodJAJNwkAhJC//dFY48KuIEmjtd4RViDrA== +vscode-oniguruma@^1.6.1: + version "1.6.2" + resolved "https://registry.yarnpkg.com/vscode-oniguruma/-/vscode-oniguruma-1.6.2.tgz#aeb9771a2f1dbfc9083c8a7fdd9cccaa3f386607" + integrity sha512-KH8+KKov5eS/9WhofZR8M8dMHWN2gTxjMsG4jd04YhpbPR91fUj7rYQ2/XjeHCJWbg7X++ApRIU9NUwM2vTvLA== + vscode-textmate@5.2.0: version "5.2.0" resolved "https://registry.yarnpkg.com/vscode-textmate/-/vscode-textmate-5.2.0.tgz#01f01760a391e8222fe4f33fbccbd1ad71aed74e" @@ -1087,11 +1092,6 @@ ws@^8.2.3: resolved "https://registry.yarnpkg.com/ws/-/ws-8.3.0.tgz#7185e252c8973a60d57170175ff55fdbd116070d" integrity sha512-Gs5EZtpqZzLvmIM59w4igITU57lrtYVFneaa434VROv4thzJyV6UjIL3D42lslWlI+D4KzLYnxSwtfuiO79sNw== -yallist@^3.0.2: - version "3.1.1" - resolved "https://registry.yarnpkg.com/yallist/-/yallist-3.1.1.tgz#dbb7daf9bfd8bac9ab45ebf602b8cbad0d5d08fd" - integrity sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g== - yallist@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/yallist/-/yallist-4.0.0.tgz#9bb92790d9c0effec63be73519e11a35019a3a72" From c072406673fbff18bbb30166f45afe423c25a893 Mon Sep 17 00:00:00 2001 From: danclay <18021681+danclay@users.noreply.github.com> Date: Sun, 17 Apr 2022 01:32:16 -0700 Subject: [PATCH 3/4] Update Serialization.ts Fix types --- src/util/Serialization.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/util/Serialization.ts b/src/util/Serialization.ts index 7da977f..325cabc 100644 --- a/src/util/Serialization.ts +++ b/src/util/Serialization.ts @@ -27,7 +27,7 @@ const reconstructError = (data: NodeJS.ErrnoException): NodeJS.ErrnoException => return result as NodeJS.ErrnoException; }; -const stringifyJSON = (data: any) => { +const stringifyJSON = (data: unknown): any => { return JSON.stringify(data, (key, value) => { switch(typeof value) { case "bigint": { @@ -43,7 +43,7 @@ const stringifyJSON = (data: any) => { }); }; -const parseJSON = (json: string) => { +const parseJSON = (json: string): any => { return JSON.parse(json, (key, value) => { if (typeof value === "string") { if (value.startsWith("BIGINT::")) { From 1dbe872f9e4247aeb387597134b7b0e144f30cc7 Mon Sep 17 00:00:00 2001 From: danclay <18021681+danclay@users.noreply.github.com> Date: Sun, 17 Apr 2022 01:50:01 -0700 Subject: [PATCH 4/4] Clean up - remove useless code from examples - update version number for publish --- package.json | 2 +- test/basicUsingModules/index.mjs | 6 ------ test/withServiceUsingModules/index.mjs | 6 +++--- 3 files changed, 4 insertions(+), 10 deletions(-) diff --git a/package.json b/package.json index b30f9b2..cc9f648 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "eris-fleet", - "version": "1.0.1", + "version": "1.0.2", "dependencies": {}, "description": "Cluster management for Discord bots using the Eris library. Inspired by eris-sharder and @brussell98/megane.", "main": "dist/index.js", diff --git a/test/basicUsingModules/index.mjs b/test/basicUsingModules/index.mjs index fa19932..d6077c5 100644 --- a/test/basicUsingModules/index.mjs +++ b/test/basicUsingModules/index.mjs @@ -1,17 +1,11 @@ import {isMaster} from "cluster" import {Fleet} from "../../dist/index.js" -import path from "path" -import { fileURLToPath } from 'url'; -import { dirname } from 'path'; import {inspect} from "util" import {BotWorker} from "./bot.mjs" import dotenv from "dotenv" dotenv.config() -const __filename = fileURLToPath(import.meta.url); -const __dirname = dirname(__filename); - const options = { BotWorker: BotWorker, token: process.env.token, diff --git a/test/withServiceUsingModules/index.mjs b/test/withServiceUsingModules/index.mjs index bf4a76f..3b2424b 100644 --- a/test/withServiceUsingModules/index.mjs +++ b/test/withServiceUsingModules/index.mjs @@ -1,12 +1,12 @@ import { isMaster } from 'cluster'; import { Fleet } from '../../dist/index.js'; -import path from 'path'; import { inspect } from 'util'; -import dotenv from 'dotenv'; -dotenv.config(); import { ServiceWorker } from "./service.mjs"; import { BotWorker } from './bot.mjs'; +import dotenv from "dotenv" +dotenv.config() + const options = { BotWorker, token: process.env.token,