Skip to content
New issue

Have a question about this project? # for a free GitHub account to open an issue and contact its maintainers and the community.

#

By clicking “#”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? # to your account

rework RemoteImporter #28

Merged
merged 1 commit into from
Jul 4, 2019
Merged

rework RemoteImporter #28

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
73 changes: 65 additions & 8 deletions lib/remote-importer/btccom-wrapper.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,11 +66,11 @@ class BtcComWrapper extends Wrapper {
* { address: <bitcoin_address>, txids: <txids>, ntx: <total_nb_txs>}
*/
async getAddress(address, filterAddr) {
// Extracts the scripthash from the bech32 address
// (btc.com api manages scripthashes, not bech32 addresses)
const scripthash = addrHelper.getScriptHashFromBech32(address)
const reqAddr = addrHelper.isBech32(address)
? addrHelper.getScriptHashFromBech32(address)
: address

const uri = `/address/${scripthash}`
const uri = `/address/${reqAddr}`
const result = await this._get(uri)

const ret = {
Expand All @@ -92,7 +92,7 @@ class BtcComWrapper extends Wrapper {
const listPages = Array.from(aPages, (val, idx) => idx + 1)

const results = await util.seriesCall(listPages, idx => {
return this._getTxsForAddress(scripthash, idx)
return this._getTxsForAddress(reqAddr, idx)
})

for (let txids of results)
Expand All @@ -103,15 +103,72 @@ class BtcComWrapper extends Wrapper {

/**
* Retrieve information for a given list of addresses
* @param {string} addresses - array of bitcoin addresses
* @param {string[]} addresses - array of bitcoin addresses
* @param {boolean} filterAddr - True if an upper bound should be used
* for #transactions associated to the address, False otherwise
* @returns {Promise} returns an array of objects
* { address: <bitcoin_address>, txids: <txids>, ntx: <total_nb_txs>}
*/
async getAddresses(addresses, filterAddr) {
// Not implemented for this api
throw "Not implemented"
const ret = []
const reqAddresses = []
const xlatedBech32Addr = {}

for (let a of addresses) {
if (addrHelper.isBech32(a)) {
const scriptHash = addrHelper.getScriptHashFromBech32(a)
reqAddresses.push(scriptHash)
xlatedBech32Addr[scriptHash] = a
} else {
reqAddresses.push(a)
}
}

// Send a batch request for all the addresses
const strReqAddresses = reqAddresses.join(',')
const uri = `/address/${strReqAddresses}`
const results = await this._get(uri)

const foundAddresses = Array.isArray(results.data)
? results.data
: [results.data]

for (let a of foundAddresses) {
if (a && a.tx_count > 0) {
// Translate bech32 address
const address = xlatedBech32Addr.hasOwnProperty(a.address)
? xlatedBech32Addr[a.address]
: a.address

if (a.tx_count <= 2) {
// Less than 3 transactions for this address
// all good
const retAddr = {
address: address,
ntx: a.tx_count,
txids: []
}

retAddr.txids = (a.tx_count == 1)
? [a.first_tx]
: [a.first_tx, a.last_tx]

ret.push(retAddr)

} else {
// More than 2 transactions for this address
// We need more requests to the API
if (filterAddr && a.tx_count > keys.addrFilterThreshold) {
Logger.info(` import of ${address} rejected (too many transactions - ${a.tx_count})`)
} else {
const retAddr = await this.getAddress(address)
ret.push(retAddr)
}
}
}
}

return ret
}

}
Expand Down
66 changes: 0 additions & 66 deletions lib/remote-importer/sources-mainnet.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,7 @@
*/
'use strict'

const addrHelper = require('../bitcoin/addresses-helper')
const network = require('../bitcoin/network')
const util = require('../util')
const Logger = require('../logger')
const keys = require('../../keys')[network.key]
const Sources = require('./sources')
Expand All @@ -24,8 +22,6 @@ class SourcesMainnet extends Sources {
*/
constructor() {
super()
// Initializes external source
this.source = null
this._initSource()
}

Expand All @@ -45,68 +41,6 @@ class SourcesMainnet extends Sources {
}
}

/**
* Retrieve information for a given address
* @param {string} address - bitcoin address
* @param {boolean} filterAddr - True if an upper bound should be used
* for #transactions associated to the address, False otherwise
* @returns {Promise} returns an object
* { address: <bitcoin_address>, txids: <txids>, ntx: <total_nb_txs>}
*/
async getAddress(address, filterAddr) {
const ret = {
address,
txids: [],
ntx: 0
}

try {
const result = await this.source.getAddress(address, filterAddr)

if (result.ntx)
ret.ntx = result.ntx
else if (result.txids)
ret.ntx = result.txids.length

if (result.txids)
ret.txids = result.txids

} catch(e) {
//Logger.error(e, `SourcesMainnet.getAddress() : ${address} from ${this.source.base}`)
Logger.error(null, `SourcesMainnet.getAddress() : ${address} from ${this.source.base}`)
} finally {
return ret
}
}

/**
* Retrieve information for a list of addresses
* @param {string[]} addresses - array of bitcoin address
* @param {boolean} filterAddr - True if an upper bound should be used
* for #transactions associated to the address, False otherwise
* @returns {Promise} returns an object
* { address: <bitcoin_address>, txids: <txids>, ntx: <total_nb_txs>}
*/
async getAddresses(addresses, filterAddr) {
const ret = []

try {
const results = await this.source.getAddresses(addresses, filterAddr)

for (let r of results) {
// Filter addresses with too many txs
if (!filterAddr || (r.ntx <= keys.addrFilterThreshold))
ret.push(r)
}

} catch(e) {
//Logger.error(e, `SourcesMainnet.getAddresses() : ${addresses} from ${this.source.base}`)
Logger.error(null, `SourcesMainnet.getAddresses() : ${addresses} from ${this.source.base}`)
} finally {
return ret
}
}

}

module.exports = SourcesMainnet
124 changes: 9 additions & 115 deletions lib/remote-importer/sources-testnet.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,19 +4,17 @@
*/
'use strict'

const addrHelper = require('../bitcoin/addresses-helper')
const network = require('../bitcoin/network')
const util = require('../util')
const Logger = require('../logger')
const keys = require('../../keys')[network.key]
const Sources = require('./sources')
const BitcoindWrapper = require('./bitcoind-wrapper')
const InsightWrapper = require('./insight-wrapper')
const BtcComWrapper = require('./btccom-wrapper')


/**
* Remote data sources for testnet polled round-robin to spread load
* Remote data sources for testnet
*/
class SourcesTestnet extends Sources {

Expand All @@ -25,126 +23,22 @@ class SourcesTestnet extends Sources {
*/
constructor() {
super()
this.sources = []
this.index = 0
this.sourceBech32 = null
this.isBitcoindActive = false
// Initializes external sources
this._initSources()
this._initSource()
}

/**
* Initialize the external data sources
* Initialize the external data source
*/
_initSources() {
_initSource() {
if (keys.explorers.bitcoind == 'active') {
// If local bitcoind option is activated
// we'll use the local node as our unique source
this.sourceBech32 = new BitcoindWrapper()
this.sources.push(this.sourceBech32)
this.isBitcoindActive = true
this.source = new BitcoindWrapper()
Logger.info('Activated Bitcoind as the data source for imports')
} else {
// Otherwise, we use a set of insight servers + btc.com for bech32 addresses
this.sourceBech32 = new BtcComWrapper(keys.explorers.btccom)
for (let url of keys.explorers.insight)
this.sources.push(new InsightWrapper(url))
this.isBitcoindActive = false
}
}

/**
* Get the next source index
* @returns {integer} returns the next source index
*/
nextIndex() {
this.index++
if (this.index >= this.sources.length)
this.index = 0
return this.index
}

/**
* Retrieve information for a given address
* @param {string} address - bitcoin address
* @param {boolean} filterAddr - True if an upper bound should be used
* for #transactions associated to the address, False otherwise
* @returns {Promise} returns an object
* { address: <bitcoin_address>, txids: <txids>, ntx: <total_nb_txs>}
*/
async getAddress(address, filterAddr) {
let source = ''

const isBech32 = addrHelper.isBech32(address)

const ret = {
address,
txids: [],
ntx: 0
}

try {
source = isBech32 ? this.sourceBech32 : this.sources[this.nextIndex()]
const result = await source.getAddress(address, filterAddr)

if (result.ntx)
ret.ntx = result.ntx
else if (result.txids)
ret.ntx = result.txids.length

if (result.txids)
ret.txids = result.txids

return ret

} catch(e) {
Logger.error(e, `SourcesTestnet.getAddress() : ${address} from ${source.base}`)
if (!isBech32 && this.sources.length > 1) {
// Try again with another source
return this.getAddress(address, filterAddr)
} else {
return ret
}
}
}

/**
* Retrieve information for a list of addresses
* @param {string[]} addresses - array of bitcoin address
* @param {boolean} filterAddr - True if an upper bound should be used
* for #transactions associated to the address, False otherwise
* @returns {Promise} returns an object
* { address: <bitcoin_address>, txids: <txids>, ntx: <total_nb_txs>}
*/
async getAddresses(addresses, filterAddr) {
const ret = []

try {
if (this.isBitcoindActive) {
const source = this.sources[0]
const results = await source.getAddresses(addresses, filterAddr)
for (let r of results) {
// Filter addresses with too many txs
if (!filterAddr || (r.ntx <= keys.addrFilterThreshold))
ret.push(r)
}
} else {
const lists = util.splitList(addresses, this.sources.length)
await util.seriesCall(lists, async list => {
const results = await Promise.all(list.map(a => {
return this.getAddress(a, filterAddr)
}))

for (let r of results) {
// Filter addresses with too many txs
if (!filterAddr || (r.ntx <= keys.addrFilterThreshold))
ret.push(r)
}
})
}
} catch (e) {
Logger.error(e, `SourcesTestnet.getAddresses() : Addr list = ${addresses}`)
} finally {
return ret
// Otherwise, we'll use the rest api provided by OXT
this.source = new BtcComWrapper(keys.explorers.btccom)
Logger.info('Activated BTC.COM API as the data source for imports')
}
}

Expand Down
Loading