From b25dc83fd663dc00c7fc48c04b34bafff76bb9c3 Mon Sep 17 00:00:00 2001 From: arnold Date: Sun, 17 Jul 2022 16:33:37 +0200 Subject: [PATCH 1/5] Batch anchor transactions. Add config option to batch anchor txs send through POST /hash. Service will send one anchor tx per block. --- src/config/config.service.ts | 12 +++------ src/config/data/default.schema.json | 18 +++++-------- src/hash/hash-listener.service.ts | 32 ++++++++++++++++++++++ src/hash/hash.controller.ts | 7 ++++- src/hash/hash.module.ts | 4 ++- src/hash/hash.service.ts | 41 +++++++++++++++++++++++++++++ src/index/index.events.ts | 4 ++- src/index/index.service.ts | 1 + src/node/node.service.ts | 37 +++++++++++++------------- 9 files changed, 115 insertions(+), 41 deletions(-) create mode 100644 src/hash/hash-listener.service.ts create mode 100644 src/hash/hash.service.ts diff --git a/src/config/config.service.ts b/src/config/config.service.ts index d9eef70..5c9943c 100644 --- a/src/config/config.service.ts +++ b/src/config/config.service.ts @@ -35,14 +35,6 @@ export class ConfigService { return this.config.get('auth.token'); } - getAnchorFee(): number { - return Number(this.config.get('fees.anchor')); - } - - getSponsorFee(): number { - return Number(this.config.get('fees.sponsor')); - } - getRedisClient(): string | string[] { return this.getRedisUrl() || this.getRedisCluster().split(';'); } @@ -114,4 +106,8 @@ export class ConfigService { isEip155IndexingEnabled(): boolean { return !!this.config.get('cross_chain.eip155.indexing'); } + + isAnchorBatched(): boolean { + return !!this.config.get('anchor.batch'); + } } diff --git a/src/config/data/default.schema.json b/src/config/data/default.schema.json index e953104..e5ce56b 100644 --- a/src/config/data/default.schema.json +++ b/src/config/data/default.schema.json @@ -75,6 +75,12 @@ "default": "none", "format": ["none", "trust", "all"], "env": "ANCHOR_INDEXING" + }, + "batch": { + "doc": "Batch hashes for anchor transaction", + "default": false, + "format": "Boolean", + "env": "ANCHOR_BATCH" } }, "stats": { @@ -98,18 +104,6 @@ "env": "STATS_INDEXING" } }, - "fees": { - "anchor": { - "doc": "Fees for anchor transactions", - "default": 35000000, - "env": "FEES_ANCHOR" - }, - "sponsor": { - "doc": "Fees for sponsor transactions", - "default": 500000000, - "env": "FEES_SPONSOR" - } - }, "redis": { "url": { "doc": "Redis database connection string", diff --git a/src/hash/hash-listener.service.ts b/src/hash/hash-listener.service.ts new file mode 100644 index 0000000..e812216 --- /dev/null +++ b/src/hash/hash-listener.service.ts @@ -0,0 +1,32 @@ +import { IndexEvent, IndexEventsReturnType } from '../index/index.events'; +import { EmitterService } from '../emitter/emitter.service'; +import { Injectable, OnModuleInit } from '@nestjs/common'; +import { HashService } from './hash.service'; +import { ConfigService } from '../config/config.service'; +import { LoggerService } from '../logger/logger.service'; + +@Injectable() +export class HashListenerService implements OnModuleInit { + constructor( + private readonly indexEmitter: EmitterService, + private readonly hashService: HashService, + private readonly config: ConfigService, + private readonly logger: LoggerService, + ) { } + + onModuleInit() { + if (!this.config.isAnchorBatched()) { + this.logger.debug(`hash-listener: Not batching anchors`); + return; + } + + this.onIndexBlock(); + } + + async onIndexBlock() { + this.indexEmitter.on( + IndexEvent.IndexBlock, + () => this.hashService.trigger(), + ); + } +} diff --git a/src/hash/hash.controller.ts b/src/hash/hash.controller.ts index 63998ad..b445220 100644 --- a/src/hash/hash.controller.ts +++ b/src/hash/hash.controller.ts @@ -48,7 +48,12 @@ export class HashController { try { const chainpoint = await this.node.anchor(hash, encoding); - res.status(200).json({ chainpoint }); + + if (!chainpoint) { + res.status(202); + } else { + res.status(200).json({chainpoint}); + } } catch (e) { this.logger.error(`hash-controller: failed to anchor '${e}'`, { stack: e.stack }); diff --git a/src/hash/hash.module.ts b/src/hash/hash.module.ts index 1e8caa3..61ff427 100644 --- a/src/hash/hash.module.ts +++ b/src/hash/hash.module.ts @@ -6,11 +6,13 @@ import { NodeModule } from '../node/node.module'; import { StorageModule } from '../storage/storage.module'; import { EncoderModule } from '../encoder/encoder.module'; import { AuthModule } from '../auth/auth.module'; +import { HashService } from './hash.service'; +import { HashListenerService } from './hash-listener.service'; export const HashModuleConfig = { imports: [LoggerModule, ConfigModule, NodeModule, StorageModule, EncoderModule, AuthModule], controllers: [HashController], - providers: [], + providers: [HashService, HashListenerService], }; @Module(HashModuleConfig) diff --git a/src/hash/hash.service.ts b/src/hash/hash.service.ts new file mode 100644 index 0000000..d1b69cf --- /dev/null +++ b/src/hash/hash.service.ts @@ -0,0 +1,41 @@ +import {Injectable} from '@nestjs/common'; +import {NodeService} from '../node/node.service'; +import {EncoderService} from '../encoder/encoder.service'; +import {ConfigService} from '../config/config.service'; + +@Injectable() +export class HashService +{ + private hashes; + + constructor( + private readonly config: ConfigService, + private readonly node: NodeService, + private readonly encoder: EncoderService, + ) { } + + async anchor( + hash: string, + encoding: string, + ): Promise<{ + '@context'; + type; + targetHash; + anchors; + } | null> { + if (this.config.isAnchorBatched()) { + this.hashes.push(this.encoder.base58Encode(this.encoder.decode(hash, encoding))); + return null; + } + + return await this.node.anchor(hash, encoding); + } + + async trigger(): Promise + { + if (this.hashes.length === 0) return; + + const chunks = [...Array(Math.ceil(this.hashes.length / 100))].map(_ => this.hashes.splice(0, 100)); + await Promise.all(chunks.map(chunk => this.node.anchorAll(...chunk))); + } +} diff --git a/src/index/index.events.ts b/src/index/index.events.ts index 2baff1d..1ab1b88 100644 --- a/src/index/index.events.ts +++ b/src/index/index.events.ts @@ -2,8 +2,10 @@ import { IndexDocumentType } from './model/index.model'; export interface IndexEventsReturnType { IndexTransaction: IndexDocumentType; + IndexBlock: number; } export const IndexEvent: { [P in keyof IndexEventsReturnType]: P } = { IndexTransaction: 'IndexTransaction', -} + IndexBlock: 'IndexBlock', +}; diff --git a/src/index/index.service.ts b/src/index/index.service.ts index 1aec8f0..c7e936d 100644 --- a/src/index/index.service.ts +++ b/src/index/index.service.ts @@ -24,6 +24,7 @@ export class IndexService { async index(index: IndexDocumentType): Promise { if (this.lastBlock !== index.blockHeight) { this.txCache = []; + this.event.emit(IndexEvent.IndexBlock, index.blockHeight); } this.lastBlock = index.blockHeight; diff --git a/src/node/node.service.ts b/src/node/node.service.ts index e0fb2b5..41dc071 100644 --- a/src/node/node.service.ts +++ b/src/node/node.service.ts @@ -3,7 +3,6 @@ import { NodeApiService } from './node-api.service'; import { LoggerService } from '../logger/logger.service'; import { EncoderService } from '../encoder/encoder.service'; import { StorageService } from '../storage/storage.service'; -import { ConfigService } from '../config/config.service'; import { Transaction } from '../transaction/interfaces/transaction.interface'; import { AxiosResponse } from 'axios'; @@ -35,7 +34,6 @@ export class NodeService { private readonly logger: LoggerService, private readonly encoder: EncoderService, private readonly storage: StorageService, - private readonly config: ConfigService, ) {} private async signAndBroadcastSponsor(type: 18 | 19, recipient: string): Promise { @@ -43,7 +41,7 @@ export class NodeService { version: 1, type, recipient, - fee: this.config.getSponsorFee(), + fee: 500000000, }); if (response instanceof Error) { @@ -97,15 +95,8 @@ export class NodeService { } const unconfirmed = response.data.filter(transaction => { - if (transaction.type !== 12) { - return false; - } - - if (transaction.data.find(data => data.value && data.value === `base64:${hash}`)) { - return true; - } - - return false; + return transaction.type === 12 && + !!transaction.data.find(data => data.value && data.value === `base64:${hash}`); }); if (unconfirmed.length === 0) { @@ -184,13 +175,13 @@ export class NodeService { return results.filter(result => !(result instanceof Error)); } - async createAnchorTransaction(senderAddress: string, hash: string): Promise { + async createAnchorTransaction(senderAddress: string, ...hashes: string[]): Promise { const response = await this.api.signAndBroadcastTransaction({ version: 1, type: 15, sender: senderAddress, - anchors: [hash], - fee: this.config.getAnchorFee(), + anchors: hashes, + fee: 25000000 + (hashes.length * 10000000), timestamp: Date.now(), }); @@ -230,6 +221,16 @@ export class NodeService { } } + async anchorAll(...hashes: string[]): Promise { + try { + const senderAddress = await this.getNodeWallet(); + await this.createAnchorTransaction(senderAddress, ...hashes); + } catch (e) { + this.logger.error(`hash: failed anchoring ${hashes.length} hashes`); + throw e; + } + } + async getTransactionByHash( hash: string, encoding?: string, @@ -273,9 +274,9 @@ export class NodeService { asChainPoint(hash: string, transactionId: string, blockHeight?: number, position?: number) { const result = { '@context': 'https://w3id.org/chainpoint/v2', - type: 'ChainpointSHA256v2', - targetHash: hash, - anchors: [ + 'type': 'ChainpointSHA256v2', + 'targetHash': hash, + 'anchors': [ { type: 'LTODataTransaction', sourceId: transactionId, From 4d0f1cd35fbafa758e4da436a1c6f4a409c087a7 Mon Sep 17 00:00:00 2001 From: arnold Date: Mon, 18 Jul 2022 02:15:28 +0200 Subject: [PATCH 2/5] Fix operation stats. Stats should be per day, just like tx stats. Increment by amount in redis and leveldb --- src/leveldb/classes/leveldb.connection.ts | 4 +- .../operations/operations.service.spec.ts | 29 +++++--- src/stats/operations/operations.service.ts | 24 ++----- src/stats/stats.controller.ts | 66 +++++++++++-------- src/storage/interfaces/storage.interface.ts | 2 +- src/storage/storage.service.spec.ts | 29 +++++--- src/storage/storage.service.ts | 16 +++-- src/storage/types/leveldb.storage.service.ts | 2 +- src/storage/types/redis.storage.service.ts | 4 +- 9 files changed, 102 insertions(+), 74 deletions(-) diff --git a/src/leveldb/classes/leveldb.connection.ts b/src/leveldb/classes/leveldb.connection.ts index 63d8ad0..eed50e8 100644 --- a/src/leveldb/classes/leveldb.connection.ts +++ b/src/leveldb/classes/leveldb.connection.ts @@ -50,12 +50,12 @@ export class LeveldbConnection { return this.connection.del(key); } - async incr(key): Promise { + async incr(key, amount = 1): Promise { await this.incrLock.acquireAsync(); try { const count = Number(await this.get(key)); - const result = await this.set(key, String(count + 1)); + const result = await this.set(key, String(count + amount)); return result; } finally { this.incrLock.release(); diff --git a/src/stats/operations/operations.service.spec.ts b/src/stats/operations/operations.service.spec.ts index f81d156..66b3359 100644 --- a/src/stats/operations/operations.service.spec.ts +++ b/src/stats/operations/operations.service.spec.ts @@ -18,16 +18,21 @@ describe('OperationsService', () => { function spy() { const anchor = { - getAnchorHashes: jest.spyOn(anchorService, 'getAnchorHashes').mockImplementation(() => ['hash_1', 'hash_2']) + getAnchorHashes: jest.spyOn(anchorService, 'getAnchorHashes').mockImplementation(() => ['hash_1', 'hash_2']), }; const transaction = { - getIdentifiersByType: jest.spyOn(transactionService, 'getIdentifiersByType').mockImplementation(() => ['all', 'transaction']) + getIdentifiersByType: jest.spyOn(transactionService, 'getIdentifiersByType').mockImplementation(() => ['all', 'transaction']), }; const storage = { incrOperationStats: jest.spyOn(storageService, 'incrOperationStats').mockImplementation(async () => {}), - getOperationStats: jest.spyOn(storageService, 'getOperationStats').mockImplementation(async () => '200'), + getOperationStats: jest.spyOn(storageService, 'getOperationStats').mockImplementation(async () => [ + { period: '2020-12-04 00:00:00', count: 300 }, + { period: '2020-12-05 00:00:00', count: 329 }, + { period: '2020-12-06 00:00:00', count: 402 }, + { period: '2020-12-07 00:00:00', count: 293 }, + ]), }; const logger = { @@ -68,10 +73,15 @@ describe('OperationsService', () => { test('should return the operation stats from storage', async () => { const spies = spy(); - const result = await operationsService.getOperationStats(); + const result = await operationsService.getOperationStats(18600, 18603); expect(spies.storage.getOperationStats.mock.calls.length).toBe(1); - expect(result).toBe('200'); + expect(result).toBe([ + { period: '2020-12-04 00:00:00', count: 300 }, + { period: '2020-12-05 00:00:00', count: 329 }, + { period: '2020-12-06 00:00:00', count: 402 }, + { period: '2020-12-07 00:00:00', count: 293 }, + ]); }); }); @@ -84,13 +94,15 @@ describe('OperationsService', () => { expect(spies.storage.incrOperationStats.mock.calls.length).toBe(3); expect(spies.logger.debug.mock.calls.length).toBe(1); - expect(spies.logger.debug.mock.calls[0][0]).toBe(`operation stats: 3 transfers: increase stats: ${transaction.id}`); + expect(spies.logger.debug.mock.calls[0][0]) + .toBe(`operation stats: 3 transfers: increase stats: ${transaction.id}`); }); test('should increase stats once if transaction has no transfer count', async () => { const spies = spy(); - // @ts-ignore (transfers is readonly) + // @ts-ignore + // noinspection JSConstantReassignment transaction.transfers = []; await operationsService.incrOperationStats(transaction); @@ -98,7 +110,8 @@ describe('OperationsService', () => { expect(spies.storage.incrOperationStats.mock.calls.length).toBe(1); expect(spies.logger.debug.mock.calls.length).toBe(1); - expect(spies.logger.debug.mock.calls[0][0]).toBe(`operation stats: 1 transfers: increase stats: ${transaction.id}`); + expect(spies.logger.debug.mock.calls[0][0]) + .toBe(`operation stats: 1 transfers: increase stats: ${transaction.id}`); }); test('should increase stats based on the anchor hashes', async () => { diff --git a/src/stats/operations/operations.service.ts b/src/stats/operations/operations.service.ts index e309077..8747052 100644 --- a/src/stats/operations/operations.service.ts +++ b/src/stats/operations/operations.service.ts @@ -15,31 +15,19 @@ export class OperationsService { private readonly transactionService: TransactionService, ) { } - async getOperationStats(): Promise { - return this.storage.getOperationStats(); + async getOperationStats(from: number, to: number): Promise<{ period: string; count: number }[]> { + return this.storage.getOperationStats(from, to); } async incrOperationStats(transaction: Transaction): Promise { const identifiers = this.transactionService.getIdentifiersByType(transaction.type); - if (identifiers.indexOf('anchor') >= 0) { - const anchorHashes = this.anchorService.getAnchorHashes(transaction); - - this.logger.debug(`operation stats: ${anchorHashes.length} anchors: increase stats: ${transaction.id}`); - - anchorHashes.forEach(async hash => { - await this.storage.incrOperationStats(); - }); - - return; - } - - const iterations = transaction.transfers?.length || 1; + const iterations = identifiers.indexOf('anchor') >= 0 + ? Math.min(this.anchorService.getAnchorHashes(transaction).length, 1) + : (transaction.transfers?.length || 1); this.logger.debug(`operation stats: ${iterations} transfers: increase stats: ${transaction.id}`); - for (let index = 0; index < iterations; index++) { - await this.storage.incrOperationStats(); - } + await this.storage.incrOperationStats(Math.floor(transaction.timestamp / 86400000), iterations); } } diff --git a/src/stats/stats.controller.ts b/src/stats/stats.controller.ts index f16c6d5..c8bff33 100644 --- a/src/stats/stats.controller.ts +++ b/src/stats/stats.controller.ts @@ -18,17 +18,41 @@ export class StatsController { private readonly transactions: TransactionService, ) { } - @Get('/operations') - @ApiOperation({ summary: 'Retrieves the operation stats' }) + private periodFromReq(req: Request) + { + const fromParam = req.params.from; + const toParam = req.params.to; + + const from = Math.floor(new Date(fromParam.match(/\D/) ? fromParam : Number(fromParam)).getTime() / 86400000); + const to = Math.floor(new Date(toParam.match(/\D/) ? toParam : Number(toParam)).getTime() / 86400000); + + if (Number.isNaN(from)) { + throw Error('invalid from date given'); + } + + if (Number.isNaN(to)) { + throw Error('invalid to date given'); + } + + if (to <= from || to - from > 100) { + throw Error('invalid period range given'); + } + + return {from, to}; + } + + @Get('/operations/:from/:to') + @ApiOperation({ summary: 'Get the operation count per day' }) @ApiResponse({ status: 200 }) @ApiResponse({ status: 400, description: 'failed to retrieve operation stats' }) async getOperationStats(@Req() req: Request, @Res() res: Response): Promise { try { - const operations = await this.operations.getOperationStats(); + const {from, to} = this.periodFromReq(req); + const stats = await this.operations.getOperationStats(from, to); - return res.status(200).json({ operations }); - } catch (error) { - return res.status(400).json({ error: 'failed to retrieve operation stats' }); + res.status(200).json(stats); + } catch (e) { + return res.status(400).send(e); } } @@ -89,35 +113,19 @@ export class StatsController { }) @ApiResponse({ status: 500, description: `failed to get transaction stats '[reason]'` }) async getTransactionStats(@Req() req: Request, @Res() res: Response): Promise { - const fromParam = req.params.from; - const toParam = req.params.to; - - let from = 0; - let to = 0; - - from = Math.floor(new Date(fromParam.match(/\D/) ? fromParam : Number(fromParam)).getTime() / 86400000); - to = Math.floor(new Date(toParam.match(/\D/) ? toParam : Number(toParam)).getTime() / 86400000); - - if (Number.isNaN(from)) { - return res.status(400).send('invalid from date given'); - } - - if (Number.isNaN(to)) { - return res.status(400).send('invalid to date given'); - } - const type = req.params.type; if (!this.transactions.hasIdentifier(type)) { return res.status(400).send('invalid type given'); } - if (to <= from || to - from > 100) { - return res.status(400).send('invalid period range given'); - } - - const stats = await this.transactions.getStats(type, from, to); + try { + const {from, to} = this.periodFromReq(req); + const stats = await this.transactions.getStats(type, from, to); - res.status(200).json(stats); + res.status(200).json(stats); + } catch (e) { + return res.status(400).send(e); + } } } diff --git a/src/storage/interfaces/storage.interface.ts b/src/storage/interfaces/storage.interface.ts index dd59cf0..4bc6995 100644 --- a/src/storage/interfaces/storage.interface.ts +++ b/src/storage/interfaces/storage.interface.ts @@ -4,7 +4,7 @@ export interface StorageInterface { getMultipleValues(keys: string[]): Promise; setValue(key: string, value: string): Promise; delValue(key: string): Promise; - incrValue(key: string): Promise; + incrValue(key: string, amount?: number): Promise; addObject(key: string, value: object): Promise; setObject(key: string, value: object): Promise; getObject(key: string): Promise; diff --git a/src/storage/storage.service.spec.ts b/src/storage/storage.service.spec.ts index 78f9821..ee1d2f1 100644 --- a/src/storage/storage.service.spec.ts +++ b/src/storage/storage.service.spec.ts @@ -461,22 +461,33 @@ describe('StorageService', () => { test('should increase the value of operation stats', async () => { const incrValue = jest.spyOn(redisStorageService, 'incrValue').mockImplementation(async () => {}); - await storageService.incrOperationStats(); + await storageService.incrOperationStats(800, 5); expect(incrValue.mock.calls.length).toBe(1); - expect(incrValue.mock.calls[0][0]).toBe(`lto:stats:operations`); + expect(incrValue.mock.calls[0][0]).toBe(`lto:stats:operations:800`); + expect(incrValue.mock.calls[0][1]).toBe(5); }); }); describe('getOperationStats()', () => { test('should fetch the value of operation stats', async () => { - const getValue = jest.spyOn(redisStorageService, 'getValue').mockImplementation(async () => '15'); - - const result = await storageService.getOperationStats(); - - expect(getValue.mock.calls.length).toBe(1); - expect(getValue.mock.calls[0][0]).toBe(`lto:stats:operations`); - expect(result).toBe('15'); + const getMultipleValues = jest.spyOn(redisStorageService, 'getMultipleValues') + .mockImplementation(async () => ['300', '329', '402', '293']); + + expect(await storageService.getOperationStats(18600, 18603)).toEqual([ + { period: '2020-12-04 00:00:00', count: 300 }, + { period: '2020-12-05 00:00:00', count: 329 }, + { period: '2020-12-06 00:00:00', count: 402 }, + { period: '2020-12-07 00:00:00', count: 293 }, + ]); + + expect(getMultipleValues.mock.calls.length).toBe(1); + expect(getMultipleValues.mock.calls[0][0]).toEqual([ + `lto:stats:transactions:18600`, + `lto:stats:transactions:18601`, + `lto:stats:transactions:18602`, + `lto:stats:transactions:18603`, + ]); }); }); }); diff --git a/src/storage/storage.service.ts b/src/storage/storage.service.ts index 454185d..ea4bae3 100644 --- a/src/storage/storage.service.ts +++ b/src/storage/storage.service.ts @@ -152,12 +152,20 @@ export class StorageService implements OnModuleInit, OnModuleDestroy { return this.storage.incrValue(`lto:stats:transactions:${type}:${day}`); } - async incrOperationStats(): Promise { - return this.storage.incrValue(`lto:stats:operations`); + async incrOperationStats(day: number, amount = 1): Promise { + return this.storage.incrValue(`lto:stats:operations:${day}`, amount); } - async getOperationStats(): Promise { - return this.storage.getValue(`lto:stats:operations`); + async getOperationStats(from: number, to: number): Promise<{ period: string; count: number }[]> { + const length = to - from + 1; + const keys = Array.from({ length }, (v, i) => `lto:stats:operations:${from + i}`); + const values = await this.storage.getMultipleValues(keys); + + const periods = Array.from({ length }, (v, i) => new Date((from + i) * 86400000)); + return periods.map((period: Date, index: number) => ({ + period: this.formatPeriod(period), + count: Number(values[index]), + })); } async getTxStats(type: string, from: number, to: number): Promise<{ period: string; count: number }[]> { diff --git a/src/storage/types/leveldb.storage.service.ts b/src/storage/types/leveldb.storage.service.ts index 4b137e7..7038396 100644 --- a/src/storage/types/leveldb.storage.service.ts +++ b/src/storage/types/leveldb.storage.service.ts @@ -49,7 +49,7 @@ export class LeveldbStorageService implements StorageInterface, OnModuleInit, On await this.connection.del(key); } - async incrValue(key: string): Promise { + async incrValue(key: string, amount = 1): Promise { await this.init(); await this.connection.incr(key); } diff --git a/src/storage/types/redis.storage.service.ts b/src/storage/types/redis.storage.service.ts index 333d10c..16baefd 100644 --- a/src/storage/types/redis.storage.service.ts +++ b/src/storage/types/redis.storage.service.ts @@ -52,9 +52,9 @@ export class RedisStorageService implements StorageInterface, OnModuleInit, OnMo return this.connection.del(key); } - async incrValue(key: string): Promise { + async incrValue(key: string, amount = 1): Promise { await this.init(); - await this.connection.incr(key); + await this.connection.incrby(key, amount); } async addObject(key: string, value: object): Promise { From b27549485a75342de1fd431a663092e630ab96bf Mon Sep 17 00:00:00 2001 From: arnold Date: Mon, 18 Jul 2022 02:37:06 +0200 Subject: [PATCH 3/5] Fix tests for stats and operations. --- src/node/node.service.spec.ts | 5 ++-- .../operations/operations.service.spec.ts | 2 +- src/stats/stats.controller.spec.ts | 28 +++++++++++-------- src/stats/stats.controller.ts | 4 +-- src/storage/storage.service.spec.ts | 8 +++--- 5 files changed, 26 insertions(+), 21 deletions(-) diff --git a/src/node/node.service.spec.ts b/src/node/node.service.spec.ts index 478e2ed..957ea3d 100644 --- a/src/node/node.service.spec.ts +++ b/src/node/node.service.spec.ts @@ -68,7 +68,6 @@ describe('NodeService', () => { }; const config = { - getSponsorFee: jest.spyOn(configService, 'getSponsorFee').mockImplementation(() => 5000), }; return { api, node, storage, config }; @@ -428,7 +427,7 @@ describe('NodeService', () => { version: 1, type: 18, recipient: 'some-recipient', - fee: 5000, + fee: 500000000, }); }); }); @@ -444,7 +443,7 @@ describe('NodeService', () => { version: 1, type: 19, recipient: 'some-recipient', - fee: 5000, + fee: 500000000, }); }); }); diff --git a/src/stats/operations/operations.service.spec.ts b/src/stats/operations/operations.service.spec.ts index 66b3359..6614648 100644 --- a/src/stats/operations/operations.service.spec.ts +++ b/src/stats/operations/operations.service.spec.ts @@ -76,7 +76,7 @@ describe('OperationsService', () => { const result = await operationsService.getOperationStats(18600, 18603); expect(spies.storage.getOperationStats.mock.calls.length).toBe(1); - expect(result).toBe([ + expect(result).toEqual([ { period: '2020-12-04 00:00:00', count: 300 }, { period: '2020-12-05 00:00:00', count: 329 }, { period: '2020-12-06 00:00:00', count: 402 }, diff --git a/src/stats/stats.controller.spec.ts b/src/stats/stats.controller.spec.ts index 32067d4..9b28432 100644 --- a/src/stats/stats.controller.spec.ts +++ b/src/stats/stats.controller.spec.ts @@ -17,7 +17,11 @@ describe('TrustNetworkController', () => { function spy() { const operations = { - getOperationStats: jest.spyOn(operationsService, 'getOperationStats').mockImplementation(async () => '200'), + getOperationStats: jest.spyOn(operationsService, 'getOperationStats').mockImplementation(async () => [ + { period: '2021-03-01 00:00:00', count: 4000 }, + { period: '2021-03-02 00:00:00', count: 5000 }, + { period: '2021-03-03 00:00:00', count: 6000 }, + ]), }; const supply = { @@ -51,21 +55,25 @@ describe('TrustNetworkController', () => { await module.close(); }); - describe('GET /stats/operations', () => { + describe('GET /stats/operations/:from/:to', () => { test('should return the operation stats from storage', async () => { const spies = spy(); const res = await request(app.getHttpServer()) - .get('/stats/operations') + .get('/stats/operations/2021-03-01/2021-03-03') .send(); expect(spies.operations.getOperationStats.mock.calls.length).toBe(1); + expect(spies.operations.getOperationStats.mock.calls[0][0]).toBe(18687); + expect(spies.operations.getOperationStats.mock.calls[0][1]).toBe(18689); expect(res.status).toBe(200); expect(res.header['content-type']).toBe('application/json; charset=utf-8'); - expect(res.body).toEqual({ - operations: '200' - }); + expect(res.body).toEqual([ + { period: '2021-03-01 00:00:00', count: 4000 }, + { period: '2021-03-02 00:00:00', count: 5000 }, + { period: '2021-03-03 00:00:00', count: 6000 }, + ]); }); test('should return error if service fails', async () => { @@ -74,15 +82,13 @@ describe('TrustNetworkController', () => { spies.operations.getOperationStats = jest.spyOn(operationsService, 'getOperationStats').mockRejectedValue('some error'); const res = await request(app.getHttpServer()) - .get('/stats/operations') + .get('/stats/operations/2021-03-01/2021-02-01') .send(); - expect(spies.operations.getOperationStats.mock.calls.length).toBe(1); + expect(spies.operations.getOperationStats.mock.calls.length).toBe(0); expect(res.status).toBe(400); - expect(res.body).toEqual({ - error: 'failed to retrieve operation stats' - }); + expect(res.body).toEqual({error: 'invalid period range given'}); }); }); diff --git a/src/stats/stats.controller.ts b/src/stats/stats.controller.ts index c8bff33..f1003f8 100644 --- a/src/stats/stats.controller.ts +++ b/src/stats/stats.controller.ts @@ -44,7 +44,7 @@ export class StatsController { @Get('/operations/:from/:to') @ApiOperation({ summary: 'Get the operation count per day' }) @ApiResponse({ status: 200 }) - @ApiResponse({ status: 400, description: 'failed to retrieve operation stats' }) + @ApiResponse({ status: 400, description: 'invalid period range given' }) async getOperationStats(@Req() req: Request, @Res() res: Response): Promise { try { const {from, to} = this.periodFromReq(req); @@ -52,7 +52,7 @@ export class StatsController { res.status(200).json(stats); } catch (e) { - return res.status(400).send(e); + return res.status(400).send({error: e.message}); } } diff --git a/src/storage/storage.service.spec.ts b/src/storage/storage.service.spec.ts index ee1d2f1..bde7796 100644 --- a/src/storage/storage.service.spec.ts +++ b/src/storage/storage.service.spec.ts @@ -483,10 +483,10 @@ describe('StorageService', () => { expect(getMultipleValues.mock.calls.length).toBe(1); expect(getMultipleValues.mock.calls[0][0]).toEqual([ - `lto:stats:transactions:18600`, - `lto:stats:transactions:18601`, - `lto:stats:transactions:18602`, - `lto:stats:transactions:18603`, + `lto:stats:operations:18600`, + `lto:stats:operations:18601`, + `lto:stats:operations:18602`, + `lto:stats:operations:18603`, ]); }); }); From 891c9ecc1a0585e6fa1ba80c04a54d706861a51e Mon Sep 17 00:00:00 2001 From: arnold Date: Mon, 18 Jul 2022 13:22:05 +0200 Subject: [PATCH 4/5] Fix tests for hash and indexing --- src/hash/hash.controller.spec.ts | 19 ++++++++------ src/hash/hash.controller.ts | 4 ++- src/hash/hash.module.ts | 3 ++- src/index/index.service.spec.ts | 6 +++-- src/index/index.service.ts | 2 +- .../operations/operations.service.spec.ts | 25 +++++++++++-------- src/stats/operations/operations.service.ts | 4 +-- 7 files changed, 38 insertions(+), 25 deletions(-) diff --git a/src/hash/hash.controller.spec.ts b/src/hash/hash.controller.spec.ts index 6c7f1c1..8436826 100644 --- a/src/hash/hash.controller.spec.ts +++ b/src/hash/hash.controller.spec.ts @@ -4,9 +4,11 @@ import request from 'supertest'; import { HashModuleConfig } from './hash.module'; import { NodeService } from '../node/node.service'; import { ConfigService } from '../config/config.service'; +import { HashService } from './hash.service'; describe('HashController', () => { let module: TestingModule; + let hashService: HashService; let nodeService: NodeService; let configService: ConfigService; let app: INestApplication; @@ -25,13 +27,15 @@ describe('HashController', () => { function spy() { const hash = { - anchor: jest.spyOn(nodeService, 'anchor') - .mockImplementation(async () => chainpoint), + anchor: jest.spyOn(hashService, 'anchor').mockImplementation(async () => chainpoint), + }; + + const node = { getTransactionByHash: jest.spyOn(nodeService, 'getTransactionByHash') .mockImplementation(async () => chainpoint), }; - return { hash }; + return { hash, node }; } beforeEach(async () => { @@ -39,6 +43,7 @@ describe('HashController', () => { app = module.createNestApplication(); await app.init(); + hashService = module.get(HashService); nodeService = module.get(NodeService); configService = module.get(ConfigService); }); @@ -122,8 +127,8 @@ describe('HashController', () => { expect(res.header['content-type']).toBe('application/json; charset=utf-8'); expect(res.body).toEqual({ chainpoint }); - expect(spies.hash.getTransactionByHash.mock.calls.length).toBe(1); - expect(spies.hash.getTransactionByHash.mock.calls[0][0]).toBe(hash); + expect(spies.node.getTransactionByHash.mock.calls.length).toBe(1); + expect(spies.node.getTransactionByHash.mock.calls[0][0]).toBe(hash); }); }); @@ -141,8 +146,8 @@ describe('HashController', () => { expect(res.header['content-type']).toBe('application/json; charset=utf-8'); expect(res.body).toEqual({ chainpoint }); - expect(spies.hash.getTransactionByHash.mock.calls.length).toBe(1); - expect(spies.hash.getTransactionByHash.mock.calls[0][0]).toBe(hash); + expect(spies.node.getTransactionByHash.mock.calls.length).toBe(1); + expect(spies.node.getTransactionByHash.mock.calls[0][0]).toBe(hash); }); }); }); diff --git a/src/hash/hash.controller.ts b/src/hash/hash.controller.ts index b445220..111bb1f 100644 --- a/src/hash/hash.controller.ts +++ b/src/hash/hash.controller.ts @@ -6,12 +6,14 @@ import { HashDto } from './dto/hash.dto'; import { NodeService } from '../node/node.service'; import { EncoderService } from '../encoder/encoder.service'; import { BearerAuthGuard } from '../auth/auth.guard'; +import { HashService } from './hash.service'; @Controller('hash') @ApiTags('hash') export class HashController { constructor( private readonly logger: LoggerService, + private readonly hash: HashService, private readonly node: NodeService, private readonly encoder: EncoderService, ) { } @@ -47,7 +49,7 @@ export class HashController { } try { - const chainpoint = await this.node.anchor(hash, encoding); + const chainpoint = await this.hash.anchor(hash, encoding); if (!chainpoint) { res.status(202); diff --git a/src/hash/hash.module.ts b/src/hash/hash.module.ts index 61ff427..768e9b2 100644 --- a/src/hash/hash.module.ts +++ b/src/hash/hash.module.ts @@ -8,9 +8,10 @@ import { EncoderModule } from '../encoder/encoder.module'; import { AuthModule } from '../auth/auth.module'; import { HashService } from './hash.service'; import { HashListenerService } from './hash-listener.service'; +import { EmitterModule } from '../emitter/emitter.module'; export const HashModuleConfig = { - imports: [LoggerModule, ConfigModule, NodeModule, StorageModule, EncoderModule, AuthModule], + imports: [LoggerModule, ConfigModule, NodeModule, StorageModule, EncoderModule, AuthModule, EmitterModule], controllers: [HashController], providers: [HashService, HashListenerService], }; diff --git a/src/index/index.service.spec.ts b/src/index/index.service.spec.ts index 6c3fe90..e8370c3 100644 --- a/src/index/index.service.spec.ts +++ b/src/index/index.service.spec.ts @@ -42,14 +42,16 @@ describe('IndexService', () => { test('should ignore genesis transactions', async () => { const spies = spy(); - const type = 'anchor'; const transaction = { id: 'fake_transaction', type: 1, sender: 'fake_sender' }; await indexService.index({ transaction: transaction as any, blockHeight: 1, position: 0}); - expect(spies.emitter.emit.mock.calls.length).toBe(1); + expect(spies.emitter.emit.mock.calls.length).toBe(2); + expect(spies.emitter.emit.mock.calls[0][0]).toBe('IndexBlock'); + expect(spies.emitter.emit.mock.calls[1][0]).toBe('IndexTransaction'); }); test('should index the anchor transaction', async () => { + indexService.lastBlock = 1; // Don't emit IndexBlock const spies = spy(); const type = 'anchor'; diff --git a/src/index/index.service.ts b/src/index/index.service.ts index c7e936d..9da9372 100644 --- a/src/index/index.service.ts +++ b/src/index/index.service.ts @@ -8,7 +8,7 @@ import { LoggerService } from '../logger/logger.service'; export class IndexService { public lastBlock: number; - public txCache: string[]; + public txCache: string[] = []; constructor( private readonly logger: LoggerService, diff --git a/src/stats/operations/operations.service.spec.ts b/src/stats/operations/operations.service.spec.ts index 6614648..f48f1bf 100644 --- a/src/stats/operations/operations.service.spec.ts +++ b/src/stats/operations/operations.service.spec.ts @@ -55,13 +55,13 @@ describe('OperationsService', () => { transaction = { type: 1, id: 'fake_transaction', - transfers: [{ - recipient: 'some_recipient', - }, { - recipient: 'some_recipient_2', - }, { - recipient: 'some_recipient_3', - }], + timestamp: new Date('2020-12-04 00:00:00+00:00').getTime(), + transfers: [ + {recipient: 'some_recipient'}, + {recipient: 'some_recipient_2'}, + {recipient: 'some_recipient_3'}, + ], + anchors: ['a', 'b'], } as Transaction; }); @@ -91,7 +91,9 @@ describe('OperationsService', () => { await operationsService.incrOperationStats(transaction); - expect(spies.storage.incrOperationStats.mock.calls.length).toBe(3); + expect(spies.storage.incrOperationStats.mock.calls.length).toBe(1); + expect(spies.storage.incrOperationStats.mock.calls[0][0]).toBe(18600); + expect(spies.storage.incrOperationStats.mock.calls[0][1]).toBe(3); expect(spies.logger.debug.mock.calls.length).toBe(1); expect(spies.logger.debug.mock.calls[0][0]) @@ -121,12 +123,13 @@ describe('OperationsService', () => { await operationsService.incrOperationStats(transaction); - expect(spies.anchor.getAnchorHashes.mock.calls.length).toBe(1); - expect(spies.storage.incrOperationStats.mock.calls.length).toBe(2); + expect(spies.storage.incrOperationStats.mock.calls.length).toBe(1); + expect(spies.storage.incrOperationStats.mock.calls[0][0]).toBe(18600); + expect(spies.storage.incrOperationStats.mock.calls[0][1]).toBe(2); expect(spies.transaction.getIdentifiersByType.mock.calls.length).toBe(1); expect(spies.logger.debug.mock.calls.length).toBe(1); - expect(spies.logger.debug.mock.calls[0][0]).toBe(`operation stats: 2 anchors: increase stats: ${transaction.id}`); + expect(spies.logger.debug.mock.calls[0][0]).toBe(`increase operation stats by 2: ${transaction.id}`); }); }); }); diff --git a/src/stats/operations/operations.service.ts b/src/stats/operations/operations.service.ts index 8747052..3f36b96 100644 --- a/src/stats/operations/operations.service.ts +++ b/src/stats/operations/operations.service.ts @@ -23,10 +23,10 @@ export class OperationsService { const identifiers = this.transactionService.getIdentifiersByType(transaction.type); const iterations = identifiers.indexOf('anchor') >= 0 - ? Math.min(this.anchorService.getAnchorHashes(transaction).length, 1) + ? Math.max(transaction.anchors.length, 1) : (transaction.transfers?.length || 1); - this.logger.debug(`operation stats: ${iterations} transfers: increase stats: ${transaction.id}`); + this.logger.debug(`increase operation stats by ${iterations}: ${transaction.id}`); await this.storage.incrOperationStats(Math.floor(transaction.timestamp / 86400000), iterations); } From b82ae2d04cad39d2697affc8b8abe00bcb81f14d Mon Sep 17 00:00:00 2001 From: arnold Date: Wed, 20 Jul 2022 02:43:37 +0200 Subject: [PATCH 5/5] Don't test debug logging in operations test --- src/stats/operations/operations.service.spec.ts | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/src/stats/operations/operations.service.spec.ts b/src/stats/operations/operations.service.spec.ts index f48f1bf..b05a80f 100644 --- a/src/stats/operations/operations.service.spec.ts +++ b/src/stats/operations/operations.service.spec.ts @@ -94,10 +94,6 @@ describe('OperationsService', () => { expect(spies.storage.incrOperationStats.mock.calls.length).toBe(1); expect(spies.storage.incrOperationStats.mock.calls[0][0]).toBe(18600); expect(spies.storage.incrOperationStats.mock.calls[0][1]).toBe(3); - - expect(spies.logger.debug.mock.calls.length).toBe(1); - expect(spies.logger.debug.mock.calls[0][0]) - .toBe(`operation stats: 3 transfers: increase stats: ${transaction.id}`); }); test('should increase stats once if transaction has no transfer count', async () => { @@ -110,10 +106,6 @@ describe('OperationsService', () => { await operationsService.incrOperationStats(transaction); expect(spies.storage.incrOperationStats.mock.calls.length).toBe(1); - - expect(spies.logger.debug.mock.calls.length).toBe(1); - expect(spies.logger.debug.mock.calls[0][0]) - .toBe(`operation stats: 1 transfers: increase stats: ${transaction.id}`); }); test('should increase stats based on the anchor hashes', async () => { @@ -127,9 +119,6 @@ describe('OperationsService', () => { expect(spies.storage.incrOperationStats.mock.calls[0][0]).toBe(18600); expect(spies.storage.incrOperationStats.mock.calls[0][1]).toBe(2); expect(spies.transaction.getIdentifiersByType.mock.calls.length).toBe(1); - - expect(spies.logger.debug.mock.calls.length).toBe(1); - expect(spies.logger.debug.mock.calls[0][0]).toBe(`increase operation stats by 2: ${transaction.id}`); }); }); });