From 38d6dfed44d415156ce48550e37457e52437a7ae Mon Sep 17 00:00:00 2001 From: Matias Alejo Garcia Date: Wed, 24 Feb 2016 14:49:34 -0300 Subject: [PATCH 1/6] dead address cache --- app/models/Address.js | 45 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/app/models/Address.js b/app/models/Address.js index 780daf7ff..4553015d3 100644 --- a/app/models/Address.js +++ b/app/models/Address.js @@ -13,8 +13,13 @@ var TransactionDb = imports.TransactionDb || require('../../lib/TransactionDb'). var BlockDb = imports.BlockDb || require('../../lib/BlockDb').default(); var config = require('../../config/config'); var CONCURRENCY = 5; +var deadCache = {}; + +function Address(addrStr, deadCacheEnable) { + + if (deadCacheEnable && deadCache[addrStr]) + return deadCache[addrStr]; -function Address(addrStr) { this.balanceSat = 0; this.totalReceivedSat = 0; this.totalSentSat = 0; @@ -76,6 +81,23 @@ function Address(addrStr) { } + +Address.expireCache = function(addrStr) { + delete deadCache[addrStr]; +}; + + +Address.setCache = function() { + this.cached = true; + deadCache[this.addrStr] = this; + +console.log('[Address.js.94] setting DEAD cache for ', this.addrStr); //TODO +console.log('[Address.js.94] cache size:', _.keys(deadCache).length); //TODO + +// TODO expire it... +}; + + Address.prototype.getObj = function() { // Normalize json address return { @@ -165,6 +187,17 @@ Address.prototype.update = function(next, opts) { if (!('ignoreCache' in opts)) opts.ignoreCache = config.ignoreCache; + if (opts.onlyUnspend && opts.includeTxInfo) + return cb('Bad params'); + + if (!opts.ignoreCache && this.cached) { + if (opts.onlyUnspent && this.unspent) + return next(); + + if (opts.includeTxInfo && this.transactions) + return next(); + } + // should collect txList from address? var txList = opts.txLimit === 0 ? null : []; @@ -198,6 +231,10 @@ Address.prototype.update = function(next, opts) { confirmationsFromCache: !!x.isConfirmedCached, }; }), 'scriptPubKey');; + + if (deadCacheEnable && !self.unspent.length && txOut.length == 2) { + self.setCache(); + } return next(); }); } else { @@ -206,6 +243,12 @@ Address.prototype.update = function(next, opts) { }); if (txList) self.transactions = txList; + + + if (deadCacheEnable && !self.transactions.length == 2) { + self.setCache(); + } + return next(); } }); From 4e6b75c6bc77407b4b9e02ab2a0a84a339ca6dc6 Mon Sep 17 00:00:00 2001 From: Matias Alejo Garcia Date: Thu, 25 Feb 2016 08:33:16 -0300 Subject: [PATCH 2/6] REST sync for bitcoin 0.12 --- config/config.js | 1 + insight.js | 2 +- lib/BlockDb.js | 1 + lib/HistoricSync.js | 105 +++++++++++++++++++------------------------ lib/PeerSync.js | 11 +++-- lib/TransactionDb.js | 1 + package.json | 49 +++++++++++--------- 7 files changed, 82 insertions(+), 88 deletions(-) diff --git a/config/config.js b/config/config.js index 96402b409..0ad4ab671 100644 --- a/config/config.js +++ b/config/config.js @@ -75,6 +75,7 @@ var bitcoindConf = { port: process.env.BITCOIND_PORT || b_port, p2pPort: process.env.BITCOIND_P2P_PORT || p2p_port, p2pHost: process.env.BITCOIND_P2P_HOST || process.env.BITCOIND_HOST || '127.0.0.1', + restUrl: process.env.BITCOIND_REST_URL ||'http://127.0.0.1:18332/rest', dataDir: dataDir, // DO NOT CHANGE THIS! disableAgent: true diff --git a/insight.js b/insight.js index 52d673957..ab30652e2 100755 --- a/insight.js +++ b/insight.js @@ -105,7 +105,7 @@ if (!config.disableP2pSync) { // historic_sync process var historicSync = new HistoricSync({ - shouldBroadcastSync: true + shouldBroadcastSync: true, }); peerSync.historicSync = historicSync; diff --git a/lib/BlockDb.js b/lib/BlockDb.js index 41c5b86ea..f02fff66c 100644 --- a/lib/BlockDb.js +++ b/lib/BlockDb.js @@ -146,6 +146,7 @@ BlockDb.prototype.add = function(b, height, cb) { }); var dbScript = this._addBlockScript(b, height); + dbScript = dbScript.concat(this._addTxsScript(txs, b.hash, height)); this.txDb.addMany(b.tx, function(err) { if (err) return cb(err); diff --git a/lib/HistoricSync.js b/lib/HistoricSync.js index 505ce4c0b..8a72caf6c 100644 --- a/lib/HistoricSync.js +++ b/lib/HistoricSync.js @@ -3,6 +3,7 @@ var imports = require('soop').imports(); var util = require('util'); var async = require('async'); +var request = require('request'); var bitcore = require('bitcore'); var networks = bitcore.networks; @@ -43,15 +44,13 @@ function HistoricSync(opts) { this.rpc = new RpcClient(config.bitcoind); this.getBitcoinCoreVersion(function(bitcoinVersion) { - if (bitcoinVersion > 100000 && !config.forceRPCsync) { + if (bitcoinVersion < 100000) { info('-------------------------------------------------------'); - info('- Bitcoin Core version >0.10 only works with RPC sync -'); - info('- Set the env variable INSIGHT_FORCE_RPC_SYNC = 1 -'); + info('- Bitcoin Core version <0.12 not supported -'); info('-------------------------------------------------------'); process.exit(1); } else { info('Bitcoin Core version ', bitcoinVersion); - info('Using RPC sync '); } }); @@ -130,14 +129,37 @@ HistoricSync.prototype.updatePercentage = function() { if (this.syncPercentage > 100) this.syncPercentage = 100; }; -HistoricSync.prototype.getBlockFromRPC = function(cb) { + +HistoricSync.prototype.getBlockFromREST = function(cb) { var self = this; + if (!self.currentRESTHash) return cb(); + + var blockInfo; + var url = config.bitcoind.restUrl + '/block/' + self.currentRESTHash + '.json'; + console.log('GET ', url); //TODO + request(url, function(err, ret, body) { + if (err || ret.statusCode != 200) { + console.log('FAILED', err, ret ? ret.statusCode : ''); //TODO + setTimeout(function() { + self.getBlockFromREST(cb); + }, 2000); + }; + + var json = JSON.parse(body); + self.currentRESTHash = json.nextblockhash; + return cb(null, json); + }); +}; + +HistoricSync.prototype.getBlockFromRPC = function(cb) { + var self = this; if (!self.currentRpcHash) return cb(); var blockInfo; self.rpc.getBlock(self.currentRpcHash, function(err, ret) { if (err) return cb(err); + if (ret) { blockInfo = ret.result; // this is to match block retreived from file @@ -283,75 +305,37 @@ HistoricSync.prototype.updateStartBlock = function(opts, next) { } }; -HistoricSync.prototype.prepareFileSync = function(opts, next) { - var self = this; - - if (config.forceRPCsync) return next(); - - if (opts.forceRPC || !config.bitcoind.dataDir || - self.height > self.blockChainHeight * PERCENTAGE_TO_START_FROM_RPC) return next(); - - - try { - self.blockExtractor = new BlockExtractor(config.bitcoind.dataDir, config.network); - } catch (e) { - info(e.message + '. Disabling file sync.'); - return next(); - } - - self.getFn = self.getBlockFromFile; - self.allowReorgs = true; - self.sync.bDb.getLastFileIndex(function(err, idx) { - - if (opts.forceStartFile) - self.blockExtractor.currentFileIndex = opts.forceStartFile; - else if (idx) self.blockExtractor.currentFileIndex = idx; - - var h = self.genesis; - - info('Seeking file to:' + self.startBlock); - //forward till startBlock - async.whilst( - function() { - return h !== self.startBlock; - }, - function(w_cb) { - self.getBlockFromFile(function(err, b) { - if (!b) return w_cb('Could not find block ' + self.startBlock); - h = b.hash; - setImmediate(function() { - return w_cb(err); - }); - }); - }, function(err) { - console.log('\tFOUND Starting Block!'); - - // TODO SET HEIGHT - return next(err); - }); - }); -}; - //NOP HistoricSync.prototype.prepareRpcSync = function(opts, next) { var self = this; - if (self.blockExtractor) return next(); + if (self.restSync) return next(); self.getFn = self.getBlockFromRPC; self.allowReorgs = true; self.currentRpcHash = self.startBlock; return next(); }; +HistoricSync.prototype.prepareRESTSync = function(opts, next) { + var self = this; + self.getFn = self.getBlockFromREST; + self.allowReorgs = true; + self.restSync = true; + self.currentRESTHash = self.startBlock; + + return next(); +}; + + + HistoricSync.prototype.showSyncStartMessage = function() { var self = this; info('Got ' + self.height + ' blocks in current DB, out of ' + self.blockChainHeight + ' block at bitcoind'); - if (self.blockExtractor) { - info('bitcoind dataDir configured...importing blocks from .dat files'); - info('First file index: ' + self.blockExtractor.currentFileIndex); + if (self.restSync) { + info('syncing from REST'); } else { info('syncing from RPC (slow)'); } @@ -368,7 +352,7 @@ HistoricSync.prototype.setupSyncStatus = function() { if (step < 10) step = 10; self.step = step; - self.type = self.blockExtractor ? 'from .dat Files' : 'from RPC calls'; + self.type = self.restSync ? 'from REST' : 'from RPC calls'; self.status = 'syncing'; self.startTs = Date.now(); self.endTs = null; @@ -407,7 +391,7 @@ HistoricSync.prototype.prepareToSync = function(opts, next) { self.updateStartBlock(opts, s_c); }, function(s_c) { - self.prepareFileSync(opts, s_c); + self.prepareRESTSync(opts, s_c); }, function(s_c) { self.prepareRpcSync(opts, s_c); @@ -444,6 +428,7 @@ HistoricSync.prototype.start = function(opts, next) { if (err) return w_cb(self.setError(err)); if (blockInfo && blockInfo.hash && (!opts.stopAt || opts.stopAt !== blockInfo.hash)) { + self.sync.storeTipBlock(blockInfo, self.allowReorgs, function(err, height) { if (err) return w_cb(self.setError(err)); if (height >= 0) self.height = height; diff --git a/lib/PeerSync.js b/lib/PeerSync.js index d3e8d7c53..4f5840bfd 100644 --- a/lib/PeerSync.js +++ b/lib/PeerSync.js @@ -26,9 +26,7 @@ function PeerSync(opts) { this.verbose = opts.verbose || false; } -PeerSync.prototype.log = function() { - if (this.verbose) console.log(arguments); -}; +PeerSync.prototype.log = console.log.bind(); PeerSync.prototype.load_peers = function() { this.peerdb = [{ @@ -129,9 +127,10 @@ PeerSync.prototype.run = function() { this.peerman.on('connection', function(conn) { self.connected = true; - conn.on('inv', self.handleInv.bind(self)); - conn.on('block', self.handleBlock.bind(self)); - conn.on('tx', self.handleTx.bind(self)); + self.conn = conn; + self.conn.on('inv', self.handleInv.bind(self)); + self.conn.on('block', self.handleBlock.bind(self)); + self.conn.on('tx', self.handleTx.bind(self)); }); this.peerman.on('connect', self.handleConnected.bind(self)); diff --git a/lib/TransactionDb.js b/lib/TransactionDb.js index 4c2c17c99..cac87ef48 100644 --- a/lib/TransactionDb.js +++ b/lib/TransactionDb.js @@ -681,6 +681,7 @@ TransactionDb.prototype._addManyFromObjs = function(txs, next) { TransactionDb.prototype._addManyFromHashes = function(txs, next) { var self = this; var dbScript = []; + async.eachLimit(txs, CONCURRENCY, function(tx, each_cb) { if (tx === genesisTXID) return each_cb(); diff --git a/package.json b/package.json index 0b06e9880..64d4dbc24 100644 --- a/package.json +++ b/package.json @@ -3,25 +3,32 @@ "description": "An open-source bitcoin blockchain API. The Insight API provides you with a convenient, powerful and simple way to query and broadcast data on the bitcoin network and build your own services with it.", "version": "0.2.18", "repository": "git://github.com/bitpay/insight-api.git", - "contributors": [{ - "name": "Matias Alejo Garcia", - "email": "ematiu@gmail.com" - }, { - "name": "Manuel Araoz", - "email": "manuelaraoz@gmail.com" - }, { - "name": "Mario Colque", - "email": "colquemario@gmail.com" - }, { - "name": "Gustavo Cortez", - "email": "cmgustavo83@gmail.com" - }, { - "name": "Juan Ignacio Sosa Lopez", - "email": "bechilandia@gmail.com" - }, { - "name": "Ivan Socolsky", - "email": "jungans@gmail.com" - }], + "contributors": [ + { + "name": "Matias Alejo Garcia", + "email": "ematiu@gmail.com" + }, + { + "name": "Manuel Araoz", + "email": "manuelaraoz@gmail.com" + }, + { + "name": "Mario Colque", + "email": "colquemario@gmail.com" + }, + { + "name": "Gustavo Cortez", + "email": "cmgustavo83@gmail.com" + }, + { + "name": "Juan Ignacio Sosa Lopez", + "email": "bechilandia@gmail.com" + }, + { + "name": "Ivan Socolsky", + "email": "jungans@gmail.com" + } + ], "bugs": { "url": "https://github.com/bitpay/insight-api/issues" }, @@ -47,7 +54,6 @@ "async": "*", "base58-native": "0.1.2", "bignum": "*", - "morgan": "*", "bitcore": "git://github.com/bitpay/bitcore.git#51c53b16ced6bcaf4b78329955d6814579fe4ee9", "bufferput": "git://github.com/bitpay/node-bufferput.git", "buffertools": "*", @@ -62,8 +68,9 @@ "microtime": "^0.6.0", "mkdirp": "^0.5.0", "moment": "~2.5.0", + "morgan": "*", "preconditions": "^1.0.7", - "request": "^2.48.0", + "request": "^2.69.0", "socket.io": "1.0.6", "socket.io-client": "1.0.6", "soop": "=0.1.5", From ca79a6d3fdd31858f622ae917a34f38bb19a8bbb Mon Sep 17 00:00:00 2001 From: Matias Alejo Garcia Date: Thu, 25 Feb 2016 08:33:51 -0300 Subject: [PATCH 3/6] add version check --- lib/HistoricSync.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/HistoricSync.js b/lib/HistoricSync.js index 8a72caf6c..0c2c5c803 100644 --- a/lib/HistoricSync.js +++ b/lib/HistoricSync.js @@ -44,7 +44,7 @@ function HistoricSync(opts) { this.rpc = new RpcClient(config.bitcoind); this.getBitcoinCoreVersion(function(bitcoinVersion) { - if (bitcoinVersion < 100000) { + if (bitcoinVersion < 120000) { info('-------------------------------------------------------'); info('- Bitcoin Core version <0.12 not supported -'); info('-------------------------------------------------------'); From 5471d08ccae57a4fe5cd391d451a1fef8da0bdc1 Mon Sep 17 00:00:00 2001 From: Matias Alejo Garcia Date: Thu, 25 Feb 2016 12:05:57 -0300 Subject: [PATCH 4/6] cache expiration --- app/controllers/addresses.js | 5 +- app/models/Address.js | 17 ++++-- lib/HistoricSync.js | 2 +- lib/PeerSync.js | 16 ++--- lib/Sync.js | 111 +++++++++++++++++++---------------- 5 files changed, 84 insertions(+), 67 deletions(-) diff --git a/app/controllers/addresses.js b/app/controllers/addresses.js index ca2fc3a4e..1a7ea76ab 100644 --- a/app/controllers/addresses.js +++ b/app/controllers/addresses.js @@ -47,8 +47,9 @@ var getAddrs = function(req, res, next) { var addrStrs = req.param('addrs'); var s = addrStrs.split(','); if (s.length === 0) return as; + var enableDeadAddresses = s.length > 100; for (var i = 0; i < s.length; i++) { - var a = new Address(s[i]); + var a = new Address(s[i], enableDeadAddresses); as.push(a); } } catch (e) { @@ -244,7 +245,7 @@ exports.multitxs = function(req, res, next) { cache[addrStrs] = txs; // 5 min. just to purge memory. Cache is overwritten in from=0 requests. setTimeout(function(){ - console.log('Deleting cache'); + console.log('Deleting cache:', addrStrs.substr(0,20)); delete cache[addrStrs]; }, 5 * 60 * 1000); } diff --git a/app/models/Address.js b/app/models/Address.js index 4553015d3..8a1a8cd56 100644 --- a/app/models/Address.js +++ b/app/models/Address.js @@ -20,6 +20,8 @@ function Address(addrStr, deadCacheEnable) { if (deadCacheEnable && deadCache[addrStr]) return deadCache[addrStr]; + this.deadCacheEnable=deadCacheEnable; + this.balanceSat = 0; this.totalReceivedSat = 0; this.totalSentSat = 0; @@ -82,12 +84,15 @@ function Address(addrStr, deadCacheEnable) { } -Address.expireCache = function(addrStr) { - delete deadCache[addrStr]; +Address.deleteDeadCache = function(addrStr) { + if (deadCache[addrStr]) { + console.log('Deleting Dead Address Cache',addrStr); + delete deadCache[addrStr]; + } }; -Address.setCache = function() { +Address.prototype.setCache = function() { this.cached = true; deadCache[this.addrStr] = this; @@ -232,7 +237,9 @@ Address.prototype.update = function(next, opts) { }; }), 'scriptPubKey');; - if (deadCacheEnable && !self.unspent.length && txOut.length == 2) { + + console.log('Addr Data', self.deadCacheEnable, self.unspent.length, txOut.length); //TODO + if (self.deadCacheEnable && !self.unspent.length && (txOut.length == 0 || txOut.length == 2)) { self.setCache(); } return next(); @@ -245,7 +252,7 @@ Address.prototype.update = function(next, opts) { self.transactions = txList; - if (deadCacheEnable && !self.transactions.length == 2) { + if (self.deadCacheEnable && !self.transactions.length == 2) { self.setCache(); } diff --git a/lib/HistoricSync.js b/lib/HistoricSync.js index 0c2c5c803..403874b75 100644 --- a/lib/HistoricSync.js +++ b/lib/HistoricSync.js @@ -140,7 +140,7 @@ HistoricSync.prototype.getBlockFromREST = function(cb) { request(url, function(err, ret, body) { if (err || ret.statusCode != 200) { console.log('FAILED', err, ret ? ret.statusCode : ''); //TODO - setTimeout(function() { + return setTimeout(function() { self.getBlockFromREST(cb); }, 2000); }; diff --git a/lib/PeerSync.js b/lib/PeerSync.js index 4f5840bfd..38c61bbb5 100644 --- a/lib/PeerSync.js +++ b/lib/PeerSync.js @@ -26,7 +26,9 @@ function PeerSync(opts) { this.verbose = opts.verbose || false; } -PeerSync.prototype.log = console.log.bind(); +PeerSync.prototype.log = function() { + if (this.verbose) console.log(arguments); +}; PeerSync.prototype.load_peers = function() { this.peerdb = [{ @@ -66,10 +68,11 @@ PeerSync.prototype.handleTx = function(info) { tx.time = tx.time || Math.round(new Date().getTime() / 1000); this.sync.storeTx(tx, function(err, relatedAddrs) { +console.log('[PeerSync.js.70:relatedAddrs:]',relatedAddrs); //TODO if (err) { - self.log('[p2p_sync] Error in handle TX: ' + JSON.stringify(err)); + return self.log('[p2p_sync] Error in handle TX: ' + JSON.stringify(err)); } - else if (self.shouldBroadcast) { + if (self.shouldBroadcast) { sockets.broadcastTx(tx); self._broadcastAddr(tx.txid, relatedAddrs); } @@ -127,10 +130,9 @@ PeerSync.prototype.run = function() { this.peerman.on('connection', function(conn) { self.connected = true; - self.conn = conn; - self.conn.on('inv', self.handleInv.bind(self)); - self.conn.on('block', self.handleBlock.bind(self)); - self.conn.on('tx', self.handleTx.bind(self)); + conn.on('inv', self.handleInv.bind(self)); + conn.on('block', self.handleBlock.bind(self)); + conn.on('tx', self.handleTx.bind(self)); }); this.peerman.on('connect', self.handleConnected.bind(self)); diff --git a/lib/Sync.js b/lib/Sync.js index 36b0fa7df..9cfc82ab3 100644 --- a/lib/Sync.js +++ b/lib/Sync.js @@ -1,15 +1,17 @@ 'use strict'; -var imports = require('soop').imports(); +var imports = require('soop').imports(); -var config = imports.config || require('../config/config'); -var bitcore = require('bitcore'); -var networks = bitcore.networks; -var async = require('async'); +var config = imports.config || require('../config/config'); +var bitcore = require('bitcore'); +var networks = bitcore.networks; +var async = require('async'); +var _ = require('lodash'); +var Address = require('../app/models/Address'); var logger = require('./logger').logger; -var d = logger.log; -var info = logger.info; +var d = logger.log; +var info = logger.info; @@ -19,7 +21,7 @@ function Sync(opts) { this.id = syncId++; this.opts = opts || {}; this.bDb = require('./BlockDb').default(); - this.txDb = require('./TransactionDb').default(); + this.txDb = require('./TransactionDb').default(); this.network = config.network === 'testnet' ? networks.testnet : networks.livenet; this.cachedLastHash = null; } @@ -46,35 +48,35 @@ Sync.prototype.destroy = function(next) { }; /* - * Arrives a NEW block, which is the new TIP - * - * Case 0) Simple case - * A-B-C-D-E(TIP)-NEW - * - * Case 1) - * A-B-C-D-E(TIP) - * \ - * NEW - * - * 1) Declare D-E orphans (and possible invalidate TXs on them) - * - * Case 2) - * A-B-C-D-E(TIP) - * \ - * F-G-NEW - * 1) Set F-G as connected (mark TXs as valid) - * 2) Set new heights in F-G-NEW - * 3) Declare D-E orphans (and possible invalidate TXs on them) - * - * - * Case 3) - * - * A-B-C-D-E(TIP) ... NEW - * - * NEW is ignored (if allowReorgs is false) - * - * - */ + * Arrives a NEW block, which is the new TIP + * + * Case 0) Simple case + * A-B-C-D-E(TIP)-NEW + * + * Case 1) + * A-B-C-D-E(TIP) + * \ + * NEW + * + * 1) Declare D-E orphans (and possible invalidate TXs on them) + * + * Case 2) + * A-B-C-D-E(TIP) + * \ + * F-G-NEW + * 1) Set F-G as connected (mark TXs as valid) + * 2) Set new heights in F-G-NEW + * 3) Declare D-E orphans (and possible invalidate TXs on them) + * + * + * Case 3) + * + * A-B-C-D-E(TIP) ... NEW + * + * NEW is ignored (if allowReorgs is false) + * + * + */ Sync.prototype.storeTipBlock = function(b, allowReorgs, cb) { @@ -85,17 +87,18 @@ Sync.prototype.storeTipBlock = function(b, allowReorgs, cb) { if (!b) return cb(); var self = this; - if ( self.storingBlock ) { + if (self.storingBlock) { logger.debug('Storing a block already. Delaying storeTipBlock with:' + - b.hash); - return setTimeout( function() { + b.hash); + return setTimeout(function() { logger.debug('Retrying storeTipBlock with: ' + b.hash); - self.storeTipBlock(b,allowReorgs,cb); + self.storeTipBlock(b, allowReorgs, cb); }, 1000); } - self.storingBlock=1; - var oldTip, oldNext, oldHeight, needReorg = false, height = -1; + self.storingBlock = 1; + var oldTip, oldNext, oldHeight, needReorg = false, + height = -1; var newPrev = b.previousblockhash; async.series([ @@ -137,7 +140,7 @@ Sync.prototype.storeTipBlock = function(b, allowReorgs, cb) { return c(); }); }, - function(c) { + function(c) { if (!allowReorgs) return c(); if (needReorg) { info('NEW TIP: %s NEED REORG (old tip: %s #%d)', b.hash, oldTip, oldHeight); @@ -147,14 +150,13 @@ Sync.prototype.storeTipBlock = function(b, allowReorgs, cb) { height = h; return c(); }); - } - else { + } else { height = oldHeight + 1; return c(); } }, function(c) { - self.cachedLastHash = b.hash; // just for speed up. + self.cachedLastHash = b.hash; // just for speed up. self.bDb.add(b, height, c); }, function(c) { @@ -174,7 +176,7 @@ Sync.prototype.storeTipBlock = function(b, allowReorgs, cb) { if (err && err.toString().match(/WARN/)) { err = null; } - self.storingBlock=0; + self.storingBlock = 0; return cb(err, height); }); }; @@ -192,7 +194,7 @@ Sync.prototype.processReorg = function(oldTip, oldNext, newPrev, oldHeight, cb) // Case 3 + allowReorgs = true return c(new Error('Could not found block:' + newPrev)); } - if (height<0) return c(); + if (height < 0) return c(); newHeight = height + 1; info('Reorg Case 1) OldNext: %s NewHeight: %d', oldNext, newHeight); @@ -270,7 +272,7 @@ Sync.prototype.setBranchConnectedBackwards = function(fromHash, cb) { }); }, function() { - return hashInterator && yHeight<=0; + return hashInterator && yHeight <= 0; }, function() { info('\tFound yBlock: %s #%d', hashInterator, yHeight); @@ -286,14 +288,19 @@ Sync.prototype.setBranchConnectedBackwards = function(fromHash, cb) { }, function(err) { return cb(err, hashInterator, lastHash, heightIter); - }); + }); }); }; //Store unconfirmed TXs Sync.prototype.storeTx = function(tx, cb) { - this.txDb.add(tx, cb); + this.txDb.add(tx, function(err, relatedAddrs) { + _.each(relatedAddrs, function(v,addr) { + Address.deleteDeadCache(addr); + }); + + }, cb); }; From 2d8445f379fd17977df347e707719c54d45b4fcb Mon Sep 17 00:00:00 2001 From: Matias Alejo Garcia Date: Thu, 25 Feb 2016 12:22:54 -0300 Subject: [PATCH 5/6] rm deadAddress related code --- app/models/Address.js | 52 +------------------------------------------ 1 file changed, 1 insertion(+), 51 deletions(-) diff --git a/app/models/Address.js b/app/models/Address.js index 8a1a8cd56..780daf7ff 100644 --- a/app/models/Address.js +++ b/app/models/Address.js @@ -13,15 +13,8 @@ var TransactionDb = imports.TransactionDb || require('../../lib/TransactionDb'). var BlockDb = imports.BlockDb || require('../../lib/BlockDb').default(); var config = require('../../config/config'); var CONCURRENCY = 5; -var deadCache = {}; - -function Address(addrStr, deadCacheEnable) { - - if (deadCacheEnable && deadCache[addrStr]) - return deadCache[addrStr]; - - this.deadCacheEnable=deadCacheEnable; +function Address(addrStr) { this.balanceSat = 0; this.totalReceivedSat = 0; this.totalSentSat = 0; @@ -83,26 +76,6 @@ function Address(addrStr, deadCacheEnable) { } - -Address.deleteDeadCache = function(addrStr) { - if (deadCache[addrStr]) { - console.log('Deleting Dead Address Cache',addrStr); - delete deadCache[addrStr]; - } -}; - - -Address.prototype.setCache = function() { - this.cached = true; - deadCache[this.addrStr] = this; - -console.log('[Address.js.94] setting DEAD cache for ', this.addrStr); //TODO -console.log('[Address.js.94] cache size:', _.keys(deadCache).length); //TODO - -// TODO expire it... -}; - - Address.prototype.getObj = function() { // Normalize json address return { @@ -192,17 +165,6 @@ Address.prototype.update = function(next, opts) { if (!('ignoreCache' in opts)) opts.ignoreCache = config.ignoreCache; - if (opts.onlyUnspend && opts.includeTxInfo) - return cb('Bad params'); - - if (!opts.ignoreCache && this.cached) { - if (opts.onlyUnspent && this.unspent) - return next(); - - if (opts.includeTxInfo && this.transactions) - return next(); - } - // should collect txList from address? var txList = opts.txLimit === 0 ? null : []; @@ -236,12 +198,6 @@ Address.prototype.update = function(next, opts) { confirmationsFromCache: !!x.isConfirmedCached, }; }), 'scriptPubKey');; - - - console.log('Addr Data', self.deadCacheEnable, self.unspent.length, txOut.length); //TODO - if (self.deadCacheEnable && !self.unspent.length && (txOut.length == 0 || txOut.length == 2)) { - self.setCache(); - } return next(); }); } else { @@ -250,12 +206,6 @@ Address.prototype.update = function(next, opts) { }); if (txList) self.transactions = txList; - - - if (self.deadCacheEnable && !self.transactions.length == 2) { - self.setCache(); - } - return next(); } }); From 86be77fb752265de9d2e5a52747538fd8585caeb Mon Sep 17 00:00:00 2001 From: Matias Alejo Garcia Date: Thu, 25 Feb 2016 14:20:12 -0300 Subject: [PATCH 6/6] rm cache related changes --- app/controllers/addresses.js | 8 +-- lib/PeerSync.js | 1 - lib/Sync.js | 8 +-- test/test.CredentialStore.js | 101 ------------------------------- test/test.PublicProfile.js | 113 ----------------------------------- 5 files changed, 6 insertions(+), 225 deletions(-) delete mode 100644 test/test.CredentialStore.js delete mode 100644 test/test.PublicProfile.js diff --git a/app/controllers/addresses.js b/app/controllers/addresses.js index 1a7ea76ab..3c6965215 100644 --- a/app/controllers/addresses.js +++ b/app/controllers/addresses.js @@ -47,9 +47,8 @@ var getAddrs = function(req, res, next) { var addrStrs = req.param('addrs'); var s = addrStrs.split(','); if (s.length === 0) return as; - var enableDeadAddresses = s.length > 100; for (var i = 0; i < s.length; i++) { - var a = new Address(s[i], enableDeadAddresses); + var a = new Address(s[i]); as.push(a); } } catch (e) { @@ -132,7 +131,7 @@ var logtime = function(str, reset) { var cache = {}; exports.multitxs = function(req, res, next) { if (!checkSync(req, res)) return; - //logtime('Start', 1); + logtime('Start', 1); function processTxs(txs, from, to, cb) { var nbTxs = txs.length; @@ -205,7 +204,7 @@ exports.multitxs = function(req, res, next) { var addrStrs = req.param('addrs'); if (cache[addrStrs] && from > 0) { - //logtime('Cache hit'); + logtime('Cache hit'); txs =cache[addrStrs]; return processTxs(txs, from, to, function(err, transactions) { //logtime('After process Txs'); @@ -230,6 +229,7 @@ exports.multitxs = function(req, res, next) { }); }, function(err) { // finished callback if (err) return common.handleErrors(err, res); +console.log('[addresses.js.234:txs:]',txs); //TODO var MAX = 9999999999; txs = _.uniq(_.flatten(txs), 'txid'); diff --git a/lib/PeerSync.js b/lib/PeerSync.js index 38c61bbb5..bdd8e2cd1 100644 --- a/lib/PeerSync.js +++ b/lib/PeerSync.js @@ -68,7 +68,6 @@ PeerSync.prototype.handleTx = function(info) { tx.time = tx.time || Math.round(new Date().getTime() / 1000); this.sync.storeTx(tx, function(err, relatedAddrs) { -console.log('[PeerSync.js.70:relatedAddrs:]',relatedAddrs); //TODO if (err) { return self.log('[p2p_sync] Error in handle TX: ' + JSON.stringify(err)); } diff --git a/lib/Sync.js b/lib/Sync.js index 9cfc82ab3..e5abcd9a0 100644 --- a/lib/Sync.js +++ b/lib/Sync.js @@ -111,6 +111,7 @@ Sync.prototype.storeTipBlock = function(b, allowReorgs, cb) { // }); // }, function(c) { +console.log('[Sync.js.115:newPrev:]',newPrev); //TODO if (!allowReorgs || newPrev === self.cachedLastHash) return c(); self.bDb.has(newPrev, function(err, val) { // Genesis? no problem @@ -295,12 +296,7 @@ Sync.prototype.setBranchConnectedBackwards = function(fromHash, cb) { //Store unconfirmed TXs Sync.prototype.storeTx = function(tx, cb) { - this.txDb.add(tx, function(err, relatedAddrs) { - _.each(relatedAddrs, function(v,addr) { - Address.deleteDeadCache(addr); - }); - - }, cb); + this.txDb.add(tx, cb); }; diff --git a/test/test.CredentialStore.js b/test/test.CredentialStore.js deleted file mode 100644 index 608453d20..000000000 --- a/test/test.CredentialStore.js +++ /dev/null @@ -1,101 +0,0 @@ -'use strict'; - -var chai = require('chai'); -var assert = require('assert'); -var sinon = require('sinon'); -var should = chai.should; -var expect = chai.expect; - -describe('credentialstore test', function() { - - var globalConfig = require('../config/config'); - var leveldb_stub = sinon.stub(); - leveldb_stub.post = sinon.stub(); - leveldb_stub.get = sinon.stub(); - var plugin = require('../plugins/credentialstore'); - var express_mock = null; - var request = null; - var response = null; - - beforeEach(function() { - - express_mock = sinon.stub(); - express_mock.post = sinon.stub(); - express_mock.get = sinon.stub(); - - plugin.init(express_mock, {db: leveldb_stub}); - - request = sinon.stub(); - request.on = sinon.stub(); - request.param = sinon.stub(); - response = sinon.stub(); - response.send = sinon.stub(); - response.status = sinon.stub(); - response.json = sinon.stub(); - response.end = sinon.stub(); - }); - - it('initializes correctly', function() { - assert(plugin.db === leveldb_stub); - assert(express_mock.post.calledWith( - globalConfig.apiPrefix + '/credentials', plugin.post - )); - assert(express_mock.get.calledWith( - globalConfig.apiPrefix + '/credentials/:username', plugin.get - )); - }); - - it('writes a message correctly', function() { - - var data = 'username=1&secret=2&record=3'; - request.on.onFirstCall().callsArgWith(1, data); - request.on.onFirstCall().returnsThis(); - request.on.onSecondCall().callsArg(1); - leveldb_stub.put = sinon.stub(); - - leveldb_stub.put.onFirstCall().callsArg(2); - response.json.returnsThis(); - - plugin.post(request, response); - - assert(leveldb_stub.put.firstCall.args[0] === 'credentials-store-12'); - assert(leveldb_stub.put.firstCall.args[1] === '3'); - assert(response.json.calledWith({success: true})); - }); - - it('retrieves a message correctly', function() { - - request.param.onFirstCall().returns('username'); - request.param.onSecondCall().returns('secret'); - - var returnValue = '!@#$%'; - leveldb_stub.get.onFirstCall().callsArgWith(1, null, returnValue); - response.send.returnsThis(); - - plugin.get(request, response); - - assert(leveldb_stub.get.firstCall.args[0] === 'credentials-store-usernamesecret'); - assert(response.send.calledWith(returnValue)); - assert(response.end.calledOnce); - }); - - it('fails with messages that are too long', function() { - - response.writeHead = sinon.stub(); - request.connection = {}; - request.connection.destroy = sinon.stub(); - var data = 'blob'; - for (var i = 0; i < 2048; i++) { - data += '----'; - } - request.on.onFirstCall().callsArgWith(1, data); - request.on.onFirstCall().returnsThis(); - response.writeHead.returnsThis(); - - plugin.post(request, response); - - assert(response.writeHead.calledWith(413, {'Content-Type': 'text/plain'})); - assert(response.end.calledOnce); - assert(request.connection.destroy.calledOnce); - }); -}); diff --git a/test/test.PublicProfile.js b/test/test.PublicProfile.js deleted file mode 100644 index 5444b2e7a..000000000 --- a/test/test.PublicProfile.js +++ /dev/null @@ -1,113 +0,0 @@ -'use strict'; - -var chai = require('chai'); -var assert = require('assert'); -var sinon = require('sinon'); -var should = chai.should; -var expect = chai.expect; -var bitauth = require('bitauth'); - -describe('public profile test', function() { - - var globalConfig = require('../config/config'); - var leveldb_stub = sinon.stub(); - leveldb_stub.put = sinon.stub(); - leveldb_stub.get = sinon.stub(); - var plugin = require('../plugins/publicInfo/publicInfo.js'); - var express_mock = null; - var request = null; - var response = null; - - beforeEach(function() { - - express_mock = sinon.stub(); - express_mock.post = sinon.stub(); - express_mock.get = sinon.stub(); - - plugin.init(express_mock, {db: leveldb_stub}); - - request = sinon.stub(); - request.on = sinon.stub(); - request.param = sinon.stub(); - response = sinon.stub(); - response.send = sinon.stub(); - response.status = sinon.stub(); - response.json = sinon.stub(); - response.end = sinon.stub(); - }); - - it('initializes correctly', function() { - assert(plugin.db === leveldb_stub); - assert(express_mock.post.calledWith( - globalConfig.apiPrefix + '/public', plugin.post - )); - assert(express_mock.get.calledWith( - globalConfig.apiPrefix + '/public/:sin', plugin.get - )); - }); - - it('writes a message correctly', function(done) { - - var privateKey = bitauth.generateSin(); - var protocol = 'https'; - var dataToSign = protocol + '://hosturlSTUFF'; - var signature = bitauth.sign(dataToSign, privateKey.priv); - request.get = function() { return 'host'; }; - request.protocol = protocol; - request.url = 'url'; - request.headers = { - 'x-identity': privateKey.pub, - 'x-signature': signature - }; - request.on.onFirstCall().callsArgWith(1, 'STUFF'); - request.on.onFirstCall().returnsThis(); - request.on.onSecondCall().callsArg(1); - - leveldb_stub.put.onFirstCall().callsArg(2); - response.status.returns(response); - response.json.returns(response); - - request.testCallback = function() { - assert(leveldb_stub.put.firstCall.args[0] === 'public-info-' + privateKey.sin); - assert(leveldb_stub.put.firstCall.args[1] === 'STUFF'); - assert(response.json.calledOnce); - assert(response.end.calledOnce); - done(); - }; - - plugin.post(request, response); - }); - - it('fails if the signature is invalid', function() { - var data = 'uecord3'; - request.get = function() { return ''; }; - request.headers = {}; - request.on.onFirstCall().callsArgWith(1, data); - request.on.onFirstCall().returnsThis(); - request.on.onSecondCall().callsArg(1); - leveldb_stub.put = sinon.stub(); - - leveldb_stub.put.onFirstCall().callsArg(2); - response.json.returnsThis(); - response.status.returnsThis(); - - plugin.post(request, response); - - assert(response.end.calledOnce); - }); - - it('retrieves a message correctly', function() { - - request.param.onFirstCall().returns('SIN'); - - var returnValue = '!@#$%'; - leveldb_stub.get.onFirstCall().callsArgWith(1, null, returnValue); - response.send.returnsThis(); - - plugin.get(request, response); - - assert(leveldb_stub.get.firstCall.args[0] === 'public-info-SIN'); - assert(response.send.calledWith(returnValue)); - assert(response.end.calledOnce); - }); -});