From 2827f85f3e3f89043bf69c870d1227525336dafe Mon Sep 17 00:00:00 2001 From: Igor Lins e Silva Date: Wed, 18 Jan 2023 14:20:56 -0300 Subject: [PATCH] v3.3.8 stable - fix readers stopping before having finished --- connections/state-history.ts | 14 +++- modules/master.ts | 42 ++++++++-- package-lock.json | 156 +++++++++++++++++------------------ package.json | 20 ++--- workers/state-reader.ts | 48 +++++++---- 5 files changed, 166 insertions(+), 114 deletions(-) diff --git a/connections/state-history.ts b/connections/state-history.ts index cabb9b69..1625a20f 100644 --- a/connections/state-history.ts +++ b/connections/state-history.ts @@ -6,6 +6,8 @@ export class StateHistorySocket { private ws; private readonly shipUrl; private readonly max_payload_mb; + retryOnDisconnect = true; + connected = false; constructor(ship_url, max_payload_mb) { this.shipUrl = ship_url; @@ -17,13 +19,13 @@ export class StateHistorySocket { } connect(onMessage, onDisconnect, onError, onConnected) { - debugLog(`Connecting to ${this.shipUrl}...`); this.ws = new WebSocket(this.shipUrl, { perMessageDeflate: false, maxPayload: this.max_payload_mb * 1024 * 1024, }); this.ws.on('open', () => { + this.connected = true; hLog('Websocket connected!'); if (onConnected) { onConnected(); @@ -31,15 +33,21 @@ export class StateHistorySocket { }); this.ws.on('message', (data) => onMessage(data)); this.ws.on('close', () => { + this.connected = false; hLog('Websocket disconnected!'); - onDisconnect(); + if(this.retryOnDisconnect) { + onDisconnect(); + } }); this.ws.on('error', (err) => { hLog(`${this.shipUrl} :: ${err.message}`); }); } - close() { + close(graceful: boolean) { + if(graceful) { + this.retryOnDisconnect = false; + } this.ws.close(); } diff --git a/modules/master.ts b/modules/master.ts index f134159b..22a1253f 100644 --- a/modules/master.ts +++ b/modules/master.ts @@ -234,6 +234,8 @@ export class HyperionMaster { tx: msg.trx_ids }); + this.lastProducedBlockNum = msg.block_num; + if (this.conf.settings.bp_monitoring && !this.conf.indexer.abi_scan_mode) { this.liveBlockQueue.push(msg).catch(reason => { hLog('Error handling consumed_block:', reason); @@ -326,17 +328,38 @@ export class HyperionMaster { if (end > this.head) { end = this.head; } - this.lastAssignedBlock += this.maxBatchSize; const def = { first_block: start, last_block: end }; + this.lastAssignedBlock = def.last_block; this.activeReadersCount++; messageAllWorkers(cluster, { event: 'new_range', target: msg.id, data: def }); + } else { + if (this.lastAssignedBlock >= this.head) { + hLog(`Parallel readers finished the requested range`); + const readers = this.workerMap.filter(value => value.worker_role === 'reader'); + this.workerMap = this.workerMap.filter(value => value.worker_role !== 'reader'); + readers.forEach(value => value.wref.kill()); + // for (let hyperionWorkerDef of this.workerMap) { + // if (hyperionWorkerDef.worker_role === 'reader') { + // hyperionWorkerDef.wref.kill(); + // } + // } + // for (let workersKey in cluster.workers) { + // const w = cluster.workers[workersKey]; + // console.log(w); + // if (w.id === parseInt(msg.id)) { + // const idx = this.workerMap.findIndex(value => value.worker_id === w.id); + // this.workerMap.splice(idx, 1); + // w.kill(); + // } + // } + } } } }, @@ -1461,6 +1484,12 @@ export class HyperionMaster { avg_consume_rate = consume_rate; } const log_msg = []; + + // print current head for live reading + if (this.lastProducedBlockNum > 0) { + log_msg.push(`#${this.lastProducedBlockNum}`); + } + log_msg.push(`W:${_workers}`); const _r = (this.pushedBlocks + this.livePushedBlocks) / tScale; @@ -1484,9 +1513,12 @@ export class HyperionMaster { this.metrics.indexingRate?.set(_ir); if (this.total_blocks < this.total_range && !this.conf.indexer.live_only_mode) { - const remaining = this.total_range - this.total_blocks; - const estimated_time = Math.round(remaining / avg_consume_rate); - const time_string = moment().add(estimated_time, 'seconds').fromNow(false); + let time_string = 'waiting for indexer'; + if (avg_consume_rate > 0) { + const remaining = this.total_range - this.total_blocks; + const estimated_time = Math.round(remaining / avg_consume_rate); + time_string = moment().add(estimated_time, 'seconds').fromNow(false); + } const pct_parsed = ((this.total_blocks / this.total_range) * 100).toFixed(1); const pct_read = ((this.total_read / this.total_range) * 100).toFixed(1); log_msg.push(`${this.total_blocks}/${this.total_read}/${this.total_range}`); @@ -1510,7 +1542,7 @@ export class HyperionMaster { hLog(log_msg.join(' | ')); } - if(this.liveConsumedBlocks > 0 && this.consumedBlocks === 0 && this.conf.indexer.abi_scan_mode) { + if (this.liveConsumedBlocks > 0 && this.consumedBlocks === 0 && this.conf.indexer.abi_scan_mode) { hLog('Warning: Live reading on ABI SCAN mode') } diff --git a/package-lock.json b/package-lock.json index e68a8dc9..ae53af77 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "hyperion-history", - "version": "3.3.7", + "version": "3.3.8", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "hyperion-history", - "version": "3.3.7", + "version": "3.3.8", "hasInstallScript": true, "license": "MIT", "dependencies": { @@ -14,8 +14,8 @@ "@eosrio/node-abieos": "^2.1.1", "@fastify/autoload": "4.0.1", "@fastify/cors": "7.0.0", - "@fastify/formbody": "^6.0.0", - "@fastify/rate-limit": "^6.0.0", + "@fastify/formbody": "^6.0.1", + "@fastify/rate-limit": "^6.0.1", "@fastify/redis": "^5.0.0", "@fastify/swagger": "6.1.0", "@pm2/io": "^5.0.0", @@ -35,26 +35,26 @@ "ioredis": "^4.28.5", "lodash": "^4.17.21", "moment": "^2.29.4", - "nodemailer": "^6.7.8", + "nodemailer": "^6.9.0", "pino-pretty": "^9.1.1", "portfinder": "^1.0.32", "socket.io": "4.5.4", "socket.io-client": "4.5.4", "socket.io-redis": "^6.1.1", - "telegraf": "^4.9.1", + "telegraf": "^4.11.2", "typescript": "^4.5.2", - "uWebSockets.js": "github:uNetworking/uWebSockets.js#v20.15.0", - "ws": "^8.11.0" + "uWebSockets.js": "github:uNetworking/uWebSockets.js#v20.19.0", + "ws": "^8.12.0" }, "devDependencies": { "@types/amqplib": "^0.8.2", "@types/async": "^3.2.16", "@types/global-agent": "^2.1.1", "@types/ioredis": "^4.28.10", - "@types/lodash": "^4.14.177", - "@types/node": "^18.11.9", - "@types/nodemailer": "^6.4.4", - "@types/ws": "^8.5.3" + "@types/lodash": "^4.14.191", + "@types/node": "^18.11.18", + "@types/nodemailer": "^6.4.7", + "@types/ws": "^8.5.4" }, "engines": { "node": "^18" @@ -148,17 +148,17 @@ "integrity": "sha512-wI3fpfDT0t7p8E6dA2eTECzzOd+bZsZCJ2Hcv+Onn2b7ZwK3RwD27uW2QDaMtQhAfWQQP+WNK7nKf0twLsBf9w==" }, "node_modules/@fastify/formbody": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/@fastify/formbody/-/formbody-6.0.0.tgz", - "integrity": "sha512-YzPTXJbB3CzDMqU5K9YGSBt/nc/RDFZSxSWZ4SoqA3T2VRJzCPd7sZFpggdmlBRWhEBlvl0EWW7EX33kfbbFlg==", + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/@fastify/formbody/-/formbody-6.0.1.tgz", + "integrity": "sha512-yIwCitoES4Sh0tPc6v+uHBqZEKw3CooSZ4kUvO9NC8Y7oRBCW+aC+pTgUZ8M3r2DOzRkNO+Pq0jMQkyL2k8jZQ==", "dependencies": { "fastify-plugin": "^3.0.0" } }, "node_modules/@fastify/rate-limit": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/@fastify/rate-limit/-/rate-limit-6.0.0.tgz", - "integrity": "sha512-EZ0TI7Mpo5aK/QaooSt3U0cAGSFjrN51vwj6osgBF1KrnLVSmRA7KPYhPimMoL9YFu9GH53aiNU0H1qTLh7s1Q==", + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/@fastify/rate-limit/-/rate-limit-6.0.1.tgz", + "integrity": "sha512-LSd6gIhvTMbt+xRG4vhfFKgbxy8x/namtIo9EcdUneE7LnsS6GSQ1NL2eo0Bg8Q61mNk812guS1xCLvO5kSTPQ==", "dependencies": { "fastify-plugin": "^3.0.1", "ms": "^2.1.3", @@ -424,20 +424,20 @@ } }, "node_modules/@types/lodash": { - "version": "4.14.177", - "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.14.177.tgz", - "integrity": "sha512-0fDwydE2clKe9MNfvXHBHF9WEahRuj+msTuQqOmAApNORFvhMYZKNGGJdCzuhheVjMps/ti0Ak/iJPACMaevvw==", + "version": "4.14.191", + "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.14.191.tgz", + "integrity": "sha512-BdZ5BCCvho3EIXw6wUCXHe7rS53AIDPLE+JzwgT+OsJk53oBfbSmZZ7CX4VaRoN78N+TJpFi9QPlfIVNmJYWxQ==", "dev": true }, "node_modules/@types/node": { - "version": "18.11.9", - "resolved": "https://registry.npmjs.org/@types/node/-/node-18.11.9.tgz", - "integrity": "sha512-CRpX21/kGdzjOpFsZSkcrXMGIBWMGNIHXXBVFSH+ggkftxg+XYP20TESbh+zFvFj3EQOl5byk0HTRn1IL6hbqg==" + "version": "18.11.18", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.11.18.tgz", + "integrity": "sha512-DHQpWGjyQKSHj3ebjFI/wRKcqQcdR+MoFBygntYOZytCqNfkd2ZC4ARDJ2DQqhjH5p85Nnd3jhUJIXrszFX/JA==" }, "node_modules/@types/nodemailer": { - "version": "6.4.4", - "resolved": "https://registry.npmjs.org/@types/nodemailer/-/nodemailer-6.4.4.tgz", - "integrity": "sha512-Ksw4t7iliXeYGvIQcSIgWQ5BLuC/mljIEbjf615svhZL10PE9t+ei8O9gDaD3FPCasUJn9KTLwz2JFJyiiyuqw==", + "version": "6.4.7", + "resolved": "https://registry.npmjs.org/@types/nodemailer/-/nodemailer-6.4.7.tgz", + "integrity": "sha512-f5qCBGAn/f0qtRcd4SEn88c8Fp3Swct1731X4ryPKqS61/A3LmmzN8zaEz7hneJvpjFbUUgY7lru/B/7ODTazg==", "dev": true, "dependencies": { "@types/node": "*" @@ -452,9 +452,9 @@ } }, "node_modules/@types/ws": { - "version": "8.5.3", - "resolved": "https://registry.npmjs.org/@types/ws/-/ws-8.5.3.tgz", - "integrity": "sha512-6YOoWjruKj1uLf3INHH7D3qTXwFfEsg1kf3c0uDdSBJwfa/llkwIjrAGV7j7mVgGNbzTQ3HiHKKDXl6bJPD97w==", + "version": "8.5.4", + "resolved": "https://registry.npmjs.org/@types/ws/-/ws-8.5.4.tgz", + "integrity": "sha512-zdQDHKUgcX/zBc4GrwsE/7dVdAD8JR4EuiAXiiUhhfyIJXXb2+PrGshFyeXWQPMmmZ2XxgaqclgpIC7eTXc1mg==", "dev": true, "dependencies": { "@types/node": "*" @@ -1940,9 +1940,9 @@ } }, "node_modules/nodemailer": { - "version": "6.7.8", - "resolved": "https://registry.npmjs.org/nodemailer/-/nodemailer-6.7.8.tgz", - "integrity": "sha512-2zaTFGqZixVmTxpJRCFC+Vk5eGRd/fYtvIR+dl5u9QXLTQWGIf48x/JXvo58g9sa0bU6To04XUv554Paykum3g==", + "version": "6.9.0", + "resolved": "https://registry.npmjs.org/nodemailer/-/nodemailer-6.9.0.tgz", + "integrity": "sha512-jFaCEGTeT3E/m/5R2MHWiyQH3pSARECRUDM+1hokOYc3lQAAG7ASuy+2jIsYVf+RVa9zePopSQwKNVFH8DKUpA==", "engines": { "node": ">=6.0.0" } @@ -2711,18 +2711,18 @@ } }, "node_modules/telegraf": { - "version": "4.9.1", - "resolved": "https://registry.npmjs.org/telegraf/-/telegraf-4.9.1.tgz", - "integrity": "sha512-MukWpKvAZ6/HpT3yHXz+jwUf2HsPa9TcsqPLQjJ+kHNGUS2PLgaNX690ExdWmWPuxjVjC4wNHmZ9JetO3C/tVA==", + "version": "4.11.2", + "resolved": "https://registry.npmjs.org/telegraf/-/telegraf-4.11.2.tgz", + "integrity": "sha512-RGEh+NXkHbq1KcSSbJeVYhHMrEN4rymd9DSe3SoIV0886bJPBHLzYCNrOqnk9aeZE2Idwh5uK0X/xbR6jScQKQ==", "dependencies": { "abort-controller": "^3.0.0", - "debug": "^4.3.3", + "debug": "^4.3.4", "mri": "^1.2.0", "node-fetch": "^2.6.7", "p-timeout": "^4.1.0", "safe-compare": "^1.1.4", "sandwich-stream": "^2.0.2", - "typegram": "^3.11.0" + "typegram": "^4.1.0" }, "bin": { "telegraf": "lib/cli.mjs" @@ -2769,9 +2769,9 @@ } }, "node_modules/typegram": { - "version": "3.11.0", - "resolved": "https://registry.npmjs.org/typegram/-/typegram-3.11.0.tgz", - "integrity": "sha512-4p6u+AFognlsDgBue8Hla2jO7Ax+UQXcLa27LC7xDdAeR9LTe+Cr4vJrYpoO1wgj/BFWgXTeboaH/+1YgWyfpA==" + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/typegram/-/typegram-4.2.0.tgz", + "integrity": "sha512-rObfoGpDlmWUhmggpWd2I/4xLsdGPDvfvrLpLxV4pBTBL2BBjm+7x7IOmTwJRV6Qe4UAkWdHq0ZYoTYAAPE5YA==" }, "node_modules/typescript": { "version": "4.5.2", @@ -2866,15 +2866,15 @@ "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=" }, "node_modules/ws": { - "version": "8.11.0", - "resolved": "https://registry.npmjs.org/ws/-/ws-8.11.0.tgz", - "integrity": "sha512-HPG3wQd9sNQoT9xHyNCXoDUa+Xw/VevmY9FoHyQ+g+rrMn4j6FB4np7Z0OhdTgjx6MgQLK7jwSy1YecU1+4Asg==", + "version": "8.12.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.12.0.tgz", + "integrity": "sha512-kU62emKIdKVeEIOIKVegvqpXMSTAMLJozpHZaJNDYqBjzlSYXQGviYwN1osDLJ9av68qHd4a2oSjd7yD4pacig==", "engines": { "node": ">=10.0.0" }, "peerDependencies": { "bufferutil": "^4.0.1", - "utf-8-validate": "^5.0.2" + "utf-8-validate": ">=5.0.2" }, "peerDependenciesMeta": { "bufferutil": { @@ -2986,17 +2986,17 @@ "integrity": "sha512-wI3fpfDT0t7p8E6dA2eTECzzOd+bZsZCJ2Hcv+Onn2b7ZwK3RwD27uW2QDaMtQhAfWQQP+WNK7nKf0twLsBf9w==" }, "@fastify/formbody": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/@fastify/formbody/-/formbody-6.0.0.tgz", - "integrity": "sha512-YzPTXJbB3CzDMqU5K9YGSBt/nc/RDFZSxSWZ4SoqA3T2VRJzCPd7sZFpggdmlBRWhEBlvl0EWW7EX33kfbbFlg==", + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/@fastify/formbody/-/formbody-6.0.1.tgz", + "integrity": "sha512-yIwCitoES4Sh0tPc6v+uHBqZEKw3CooSZ4kUvO9NC8Y7oRBCW+aC+pTgUZ8M3r2DOzRkNO+Pq0jMQkyL2k8jZQ==", "requires": { "fastify-plugin": "^3.0.0" } }, "@fastify/rate-limit": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/@fastify/rate-limit/-/rate-limit-6.0.0.tgz", - "integrity": "sha512-EZ0TI7Mpo5aK/QaooSt3U0cAGSFjrN51vwj6osgBF1KrnLVSmRA7KPYhPimMoL9YFu9GH53aiNU0H1qTLh7s1Q==", + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/@fastify/rate-limit/-/rate-limit-6.0.1.tgz", + "integrity": "sha512-LSd6gIhvTMbt+xRG4vhfFKgbxy8x/namtIo9EcdUneE7LnsS6GSQ1NL2eo0Bg8Q61mNk812guS1xCLvO5kSTPQ==", "requires": { "fastify-plugin": "^3.0.1", "ms": "^2.1.3", @@ -3234,20 +3234,20 @@ } }, "@types/lodash": { - "version": "4.14.177", - "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.14.177.tgz", - "integrity": "sha512-0fDwydE2clKe9MNfvXHBHF9WEahRuj+msTuQqOmAApNORFvhMYZKNGGJdCzuhheVjMps/ti0Ak/iJPACMaevvw==", + "version": "4.14.191", + "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.14.191.tgz", + "integrity": "sha512-BdZ5BCCvho3EIXw6wUCXHe7rS53AIDPLE+JzwgT+OsJk53oBfbSmZZ7CX4VaRoN78N+TJpFi9QPlfIVNmJYWxQ==", "dev": true }, "@types/node": { - "version": "18.11.9", - "resolved": "https://registry.npmjs.org/@types/node/-/node-18.11.9.tgz", - "integrity": "sha512-CRpX21/kGdzjOpFsZSkcrXMGIBWMGNIHXXBVFSH+ggkftxg+XYP20TESbh+zFvFj3EQOl5byk0HTRn1IL6hbqg==" + "version": "18.11.18", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.11.18.tgz", + "integrity": "sha512-DHQpWGjyQKSHj3ebjFI/wRKcqQcdR+MoFBygntYOZytCqNfkd2ZC4ARDJ2DQqhjH5p85Nnd3jhUJIXrszFX/JA==" }, "@types/nodemailer": { - "version": "6.4.4", - "resolved": "https://registry.npmjs.org/@types/nodemailer/-/nodemailer-6.4.4.tgz", - "integrity": "sha512-Ksw4t7iliXeYGvIQcSIgWQ5BLuC/mljIEbjf615svhZL10PE9t+ei8O9gDaD3FPCasUJn9KTLwz2JFJyiiyuqw==", + "version": "6.4.7", + "resolved": "https://registry.npmjs.org/@types/nodemailer/-/nodemailer-6.4.7.tgz", + "integrity": "sha512-f5qCBGAn/f0qtRcd4SEn88c8Fp3Swct1731X4ryPKqS61/A3LmmzN8zaEz7hneJvpjFbUUgY7lru/B/7ODTazg==", "dev": true, "requires": { "@types/node": "*" @@ -3262,9 +3262,9 @@ } }, "@types/ws": { - "version": "8.5.3", - "resolved": "https://registry.npmjs.org/@types/ws/-/ws-8.5.3.tgz", - "integrity": "sha512-6YOoWjruKj1uLf3INHH7D3qTXwFfEsg1kf3c0uDdSBJwfa/llkwIjrAGV7j7mVgGNbzTQ3HiHKKDXl6bJPD97w==", + "version": "8.5.4", + "resolved": "https://registry.npmjs.org/@types/ws/-/ws-8.5.4.tgz", + "integrity": "sha512-zdQDHKUgcX/zBc4GrwsE/7dVdAD8JR4EuiAXiiUhhfyIJXXb2+PrGshFyeXWQPMmmZ2XxgaqclgpIC7eTXc1mg==", "dev": true, "requires": { "@types/node": "*" @@ -4410,9 +4410,9 @@ "optional": true }, "nodemailer": { - "version": "6.7.8", - "resolved": "https://registry.npmjs.org/nodemailer/-/nodemailer-6.7.8.tgz", - "integrity": "sha512-2zaTFGqZixVmTxpJRCFC+Vk5eGRd/fYtvIR+dl5u9QXLTQWGIf48x/JXvo58g9sa0bU6To04XUv554Paykum3g==" + "version": "6.9.0", + "resolved": "https://registry.npmjs.org/nodemailer/-/nodemailer-6.9.0.tgz", + "integrity": "sha512-jFaCEGTeT3E/m/5R2MHWiyQH3pSARECRUDM+1hokOYc3lQAAG7ASuy+2jIsYVf+RVa9zePopSQwKNVFH8DKUpA==" }, "normalize-url": { "version": "6.1.0", @@ -5016,18 +5016,18 @@ "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==" }, "telegraf": { - "version": "4.9.1", - "resolved": "https://registry.npmjs.org/telegraf/-/telegraf-4.9.1.tgz", - "integrity": "sha512-MukWpKvAZ6/HpT3yHXz+jwUf2HsPa9TcsqPLQjJ+kHNGUS2PLgaNX690ExdWmWPuxjVjC4wNHmZ9JetO3C/tVA==", + "version": "4.11.2", + "resolved": "https://registry.npmjs.org/telegraf/-/telegraf-4.11.2.tgz", + "integrity": "sha512-RGEh+NXkHbq1KcSSbJeVYhHMrEN4rymd9DSe3SoIV0886bJPBHLzYCNrOqnk9aeZE2Idwh5uK0X/xbR6jScQKQ==", "requires": { "abort-controller": "^3.0.0", - "debug": "^4.3.3", + "debug": "^4.3.4", "mri": "^1.2.0", "node-fetch": "^2.6.7", "p-timeout": "^4.1.0", "safe-compare": "^1.1.4", "sandwich-stream": "^2.0.2", - "typegram": "^3.11.0" + "typegram": "^4.1.0" } }, "tiny-lru": { @@ -5056,9 +5056,9 @@ "integrity": "sha512-34R7HTnG0XIJcBSn5XhDd7nNFPRcXYRZrBB2O2jdKqYODldSzBAqzsWoZYYvduky73toYS/ESqxPvkDf/F0XMg==" }, "typegram": { - "version": "3.11.0", - "resolved": "https://registry.npmjs.org/typegram/-/typegram-3.11.0.tgz", - "integrity": "sha512-4p6u+AFognlsDgBue8Hla2jO7Ax+UQXcLa27LC7xDdAeR9LTe+Cr4vJrYpoO1wgj/BFWgXTeboaH/+1YgWyfpA==" + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/typegram/-/typegram-4.2.0.tgz", + "integrity": "sha512-rObfoGpDlmWUhmggpWd2I/4xLsdGPDvfvrLpLxV4pBTBL2BBjm+7x7IOmTwJRV6Qe4UAkWdHq0ZYoTYAAPE5YA==" }, "typescript": { "version": "4.5.2", @@ -5108,7 +5108,7 @@ }, "uWebSockets.js": { "version": "git+ssh://git@github.com/uNetworking/uWebSockets.js.git#806df48c9da86af7b3341f3e443388c7cd15c3de", - "from": "uWebSockets.js@github:uNetworking/uWebSockets.js#v20.15.0" + "from": "uWebSockets.js@github:uNetworking/uWebSockets.js#v20.19.0" }, "vary": { "version": "1.1.2", @@ -5135,9 +5135,9 @@ "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=" }, "ws": { - "version": "8.11.0", - "resolved": "https://registry.npmjs.org/ws/-/ws-8.11.0.tgz", - "integrity": "sha512-HPG3wQd9sNQoT9xHyNCXoDUa+Xw/VevmY9FoHyQ+g+rrMn4j6FB4np7Z0OhdTgjx6MgQLK7jwSy1YecU1+4Asg==", + "version": "8.12.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.12.0.tgz", + "integrity": "sha512-kU62emKIdKVeEIOIKVegvqpXMSTAMLJozpHZaJNDYqBjzlSYXQGviYwN1osDLJ9av68qHd4a2oSjd7yD4pacig==", "requires": {} }, "xmlhttprequest-ssl": { diff --git a/package.json b/package.json index 258bada4..6007034c 100644 --- a/package.json +++ b/package.json @@ -30,8 +30,8 @@ "@eosrio/node-abieos": "^2.1.1", "@fastify/autoload": "4.0.1", "@fastify/cors": "7.0.0", - "@fastify/formbody": "^6.0.0", - "@fastify/rate-limit": "^6.0.0", + "@fastify/formbody": "^6.0.1", + "@fastify/rate-limit": "^6.0.1", "@fastify/redis": "^5.0.0", "@fastify/swagger": "6.1.0", "@pm2/io": "^5.0.0", @@ -51,26 +51,26 @@ "ioredis": "^4.28.5", "lodash": "^4.17.21", "moment": "^2.29.4", - "nodemailer": "^6.7.8", + "nodemailer": "^6.9.0", "portfinder": "^1.0.32", "pino-pretty": "^9.1.1", "socket.io": "4.5.4", "socket.io-client": "4.5.4", "socket.io-redis": "^6.1.1", - "telegraf": "^4.9.1", + "telegraf": "^4.11.2", "typescript": "^4.5.2", - "uWebSockets.js": "github:uNetworking/uWebSockets.js#v20.15.0", - "ws": "^8.11.0" + "uWebSockets.js": "github:uNetworking/uWebSockets.js#v20.19.0", + "ws": "^8.12.0" }, "devDependencies": { "@types/amqplib": "^0.8.2", "@types/async": "^3.2.16", "@types/global-agent": "^2.1.1", "@types/ioredis": "^4.28.10", - "@types/lodash": "^4.14.177", - "@types/node": "^18.11.9", - "@types/nodemailer": "^6.4.4", - "@types/ws": "^8.5.3" + "@types/lodash": "^4.14.191", + "@types/node": "^18.11.18", + "@types/nodemailer": "^6.4.7", + "@types/ws": "^8.5.4" }, "optionalDependencies": { "bufferutil": "^4.0.7", diff --git a/workers/state-reader.ts b/workers/state-reader.ts index c4b8debf..c2b6fc8d 100644 --- a/workers/state-reader.ts +++ b/workers/state-reader.ts @@ -40,6 +40,7 @@ export default class StateReader extends HyperionWorker { private shipInitStatus: any; private forkedBlocks = new Map(); + private idle = true; constructor() { super(); @@ -122,7 +123,7 @@ export default class StateReader extends HyperionWorker { console.log('Message nacked!'); console.log(err.message); } else { - process.send({event: 'read_block', live: this.isLiveReader}); + process.send({event: 'read_block', live: this.isLiveReader, block_num: d.num}); } }); @@ -231,7 +232,7 @@ export default class StateReader extends HyperionWorker { case 'stop': { if (this.isLiveReader) { console.log('[LIVE READER] Closing Websocket'); - this.ship.close(); + this.ship.close(true); setTimeout(() => { console.log('[LIVE READER] Process killed'); process.exit(1); @@ -262,9 +263,7 @@ export default class StateReader extends HyperionWorker { pending++; } }); - if (pending === this.lastPendingCount && pending > 0) { - // console.log(`[${process.env['worker_id']}] Pending blocks: ${pending}`); - } else { + if (!(pending === this.lastPendingCount && pending > 0)) { this.lastPendingCount = pending; } } @@ -310,7 +309,7 @@ export default class StateReader extends HyperionWorker { if (!process.env.worker_role) { console.log("[FATAL ERROR] undefined role! Exiting now!"); - this.ship.close(); + this.ship.close(false); process.exit(1); return; } @@ -359,7 +358,7 @@ export default class StateReader extends HyperionWorker { } } else { - this.ship.close(); + this.ship.close(true); process.exit(1); } @@ -369,6 +368,7 @@ export default class StateReader extends HyperionWorker { const res = result[1]; if (res['this_block']) { + this.idle = false; const blk_num = res['this_block']['block_num']; const lib = res['last_irreversible']; const task_payload = {num: blk_num, content: data}; @@ -449,12 +449,15 @@ export default class StateReader extends HyperionWorker { } } } else { - hLog(`Reader is idle! - Head at: ${result[1].head.block_num}`); - this.ship.close(); - process.send({ - event: 'kill_worker', - id: process.env['worker_id'] - }); + this.idle = true; + // hLog(`Reader is idle! - Head at: ${result[1].head.block_num}`); + // this.ship.close(true); + // const queueSize = [this.stageOneDistQueue.length(), this.blockReadingQueue.length()]; + // hLog(queueSize); + // process.send({ + // event: 'kill_worker', + // id: process.env['worker_id'] + // }); } } @@ -514,8 +517,12 @@ export default class StateReader extends HyperionWorker { request.start_block_num = parseInt(first_block > 0 ? first_block.toString() : '1', 10); request.end_block_num = parseInt(last_block.toString(), 10); const reqType = 'get_blocks_request_' + this.shipRev; - debugLog(`Reader ${process.env.worker_id} sending ${reqType} from: ${request.start_block_num} to: ${request.end_block_num}`); - this.send([reqType, request]); + if (this.ship.connected) { + debugLog(`Reader ${process.env.worker_id} sending ${reqType} from: ${request.start_block_num} to: ${request.end_block_num}`); + this.send([reqType, request]); + } else { + hLog('[Warning] Request failed - SHIP is not online!'); + } } private send(req_data: (string | any)[]) { @@ -528,8 +535,12 @@ export default class StateReader extends HyperionWorker { this.local_block_num = request.start_block_num - 1; request.end_block_num = parseInt(last, 10); const reqType = 'get_blocks_request_' + this.shipRev; - debugLog(`Reader ${process.env.worker_id} sending ${reqType} from: ${request.start_block_num} to: ${request.end_block_num}`); - this.send([reqType, request]); + if (this.ship.connected) { + debugLog(`Reader ${process.env.worker_id} sending ${reqType} from: ${request.start_block_num} to: ${request.end_block_num}`); + this.send([reqType, request]); + } else { + hLog('[Warning] Request failed - SHIP is not online!'); + } } private async deleteForkedBlock(block_id: string) { @@ -577,6 +588,7 @@ export default class StateReader extends HyperionWorker { id: targetBlock }); targetBlock++; + console.log(blockData); if (blockData.body) { const targetBlockId = blockData.body._source.block_id; this.forkedBlocks.set(targetBlockId, Date.now()); @@ -659,7 +671,7 @@ export default class StateReader extends HyperionWorker { private handleLostConnection() { this.recovery = true; - this.ship.close(); + this.ship.close(false); hLog(`Retrying connection in 5 seconds... [attempt: ${this.reconnectCount + 1}]`); debugLog(`PENDING REQUESTS:', ${this.pendingRequest}`); debugLog(`LOCAL BLOCK:', ${this.local_block_num}`);