Merge pull request #160 from ltonetwork/bundle-anchor

Ability to batch hash requests into a single anchor tx

src/config/config.service.ts

@@ -35,14 +35,6 @@ export class ConfigService {
     return this.config.get('auth.token');
   }
 
-  getAnchorFee(): number {
-    return Number(this.config.get('fees.anchor'));
-  }
-
-  getSponsorFee(): number {
-    return Number(this.config.get('fees.sponsor'));
-  }
-
   getRedisClient(): string | string[] {
     return this.getRedisUrl() || this.getRedisCluster().split(';');
   }
@@ -114,4 +106,8 @@ export class ConfigService {
   isEip155IndexingEnabled(): boolean {
     return !!this.config.get('cross_chain.eip155.indexing');
   }
+
+  isAnchorBatched(): boolean {
+    return !!this.config.get('anchor.batch');
+  }
 }

src/config/data/default.schema.json

@@ -75,6 +75,12 @@
       "default": "none",
       "format": ["none", "trust", "all"],
       "env": "ANCHOR_INDEXING"
+    },
+    "batch": {
+      "doc": "Batch hashes for anchor transaction",
+      "default": false,
+      "format": "Boolean",
+      "env": "ANCHOR_BATCH"
     }
   },
   "stats": {
@@ -98,18 +104,6 @@
       "env": "STATS_INDEXING"
     }
   },
-  "fees": {
-    "anchor": {
-      "doc": "Fees for anchor transactions",
-      "default": 35000000,
-      "env": "FEES_ANCHOR"
-    },
-    "sponsor": {
-      "doc": "Fees for sponsor transactions",
-      "default": 500000000,
-      "env": "FEES_SPONSOR"
-    }
-  },
   "redis": {
     "url": {
       "doc": "Redis database connection string",

src/hash/hash-listener.service.ts

@@ -0,0 +1,32 @@
+import { IndexEvent, IndexEventsReturnType } from '../index/index.events';
+import { EmitterService } from '../emitter/emitter.service';
+import { Injectable, OnModuleInit } from '@nestjs/common';
+import { HashService } from './hash.service';
+import { ConfigService } from '../config/config.service';
+import { LoggerService } from '../logger/logger.service';
+
+@Injectable()
+export class HashListenerService implements OnModuleInit {
+  constructor(
+    private readonly indexEmitter: EmitterService<IndexEventsReturnType>,
+    private readonly hashService: HashService,
+    private readonly config: ConfigService,
+    private readonly logger: LoggerService,
+  ) { }
+
+  onModuleInit() {
+    if (!this.config.isAnchorBatched()) {
+      this.logger.debug(`hash-listener: Not batching anchors`);
+      return;
+    }
+
+    this.onIndexBlock();
+  }
+
+  async onIndexBlock() {
+    this.indexEmitter.on(
+      IndexEvent.IndexBlock,
+      () => this.hashService.trigger(),
+    );
+  }
+}

src/hash/hash.controller.spec.ts

@@ -4,9 +4,11 @@ import request from 'supertest';
 import { HashModuleConfig } from './hash.module';
 import { NodeService } from '../node/node.service';
 import { ConfigService } from '../config/config.service';
+import { HashService } from './hash.service';
 
 describe('HashController', () => {
   let module: TestingModule;
+  let hashService: HashService;
   let nodeService: NodeService;
   let configService: ConfigService;
   let app: INestApplication;
@@ -25,20 +27,23 @@ describe('HashController', () => {
 
   function spy() {
     const hash = {
-      anchor: jest.spyOn(nodeService, 'anchor')
-        .mockImplementation(async () => chainpoint),
+      anchor: jest.spyOn(hashService, 'anchor').mockImplementation(async () => chainpoint),
+    };
+
+    const node = {
       getTransactionByHash: jest.spyOn(nodeService, 'getTransactionByHash')
         .mockImplementation(async () => chainpoint),
     };
 
-    return { hash };
+    return { hash, node };
   }
 
   beforeEach(async () => {
     module = await Test.createTestingModule(HashModuleConfig).compile();
     app = module.createNestApplication();
     await app.init();
 
+    hashService = module.get<HashService>(HashService);
     nodeService = module.get<NodeService>(NodeService);
     configService = module.get<ConfigService>(ConfigService);
   });
@@ -122,8 +127,8 @@ describe('HashController', () => {
       expect(res.header['content-type']).toBe('application/json; charset=utf-8');
       expect(res.body).toEqual({ chainpoint });
 
-      expect(spies.hash.getTransactionByHash.mock.calls.length).toBe(1);
-      expect(spies.hash.getTransactionByHash.mock.calls[0][0]).toBe(hash);
+      expect(spies.node.getTransactionByHash.mock.calls.length).toBe(1);
+      expect(spies.node.getTransactionByHash.mock.calls[0][0]).toBe(hash);
     });
   });
 
@@ -141,8 +146,8 @@ describe('HashController', () => {
       expect(res.header['content-type']).toBe('application/json; charset=utf-8');
       expect(res.body).toEqual({ chainpoint });
 
-      expect(spies.hash.getTransactionByHash.mock.calls.length).toBe(1);
-      expect(spies.hash.getTransactionByHash.mock.calls[0][0]).toBe(hash);
+      expect(spies.node.getTransactionByHash.mock.calls.length).toBe(1);
+      expect(spies.node.getTransactionByHash.mock.calls[0][0]).toBe(hash);
     });
   });
 });

src/hash/hash.controller.ts

@@ -6,12 +6,14 @@ import { HashDto } from './dto/hash.dto';
 import { NodeService } from '../node/node.service';
 import { EncoderService } from '../encoder/encoder.service';
 import { BearerAuthGuard } from '../auth/auth.guard';
+import { HashService } from './hash.service';
 
 @Controller('hash')
 @ApiTags('hash')
 export class HashController {
   constructor(
     private readonly logger: LoggerService,
+    private readonly hash: HashService,
     private readonly node: NodeService,
     private readonly encoder: EncoderService,
   ) { }
@@ -47,8 +49,13 @@ export class HashController {
     }
 
     try {
-      const chainpoint = await this.node.anchor(hash, encoding);
-      res.status(200).json({ chainpoint });
+      const chainpoint = await this.hash.anchor(hash, encoding);
+
+      if (!chainpoint) {
+        res.status(202);
+      } else {
+        res.status(200).json({chainpoint});
+      }
     } catch (e) {
       this.logger.error(`hash-controller: failed to anchor '${e}'`, { stack: e.stack });
 

src/hash/hash.module.ts

@@ -6,11 +6,14 @@ import { NodeModule } from '../node/node.module';
 import { StorageModule } from '../storage/storage.module';
 import { EncoderModule } from '../encoder/encoder.module';
 import { AuthModule } from '../auth/auth.module';
+import { HashService } from './hash.service';
+import { HashListenerService } from './hash-listener.service';
+import { EmitterModule } from '../emitter/emitter.module';
 
 export const HashModuleConfig = {
-  imports: [LoggerModule, ConfigModule, NodeModule, StorageModule, EncoderModule, AuthModule],
+  imports: [LoggerModule, ConfigModule, NodeModule, StorageModule, EncoderModule, AuthModule, EmitterModule],
   controllers: [HashController],
-  providers: [],
+  providers: [HashService, HashListenerService],
 };
 
 @Module(HashModuleConfig)

src/hash/hash.service.ts

@@ -0,0 +1,41 @@
+import {Injectable} from '@nestjs/common';
+import {NodeService} from '../node/node.service';
+import {EncoderService} from '../encoder/encoder.service';
+import {ConfigService} from '../config/config.service';
+
+@Injectable()
+export class HashService
+{
+    private hashes;
+
+    constructor(
+        private readonly config: ConfigService,
+        private readonly node: NodeService,
+        private readonly encoder: EncoderService,
+    ) { }
+
+    async anchor(
+        hash: string,
+        encoding: string,
+    ): Promise<{
+        '@context';
+        type;
+        targetHash;
+        anchors;
+    } | null> {
+        if (this.config.isAnchorBatched()) {
+            this.hashes.push(this.encoder.base58Encode(this.encoder.decode(hash, encoding)));
+            return null;
+        }
+
+        return await this.node.anchor(hash, encoding);
+    }
+
+    async trigger(): Promise<void>
+    {
+        if (this.hashes.length === 0) return;
+
+        const chunks = [...Array(Math.ceil(this.hashes.length / 100))].map(_ => this.hashes.splice(0, 100));
+        await Promise.all(chunks.map(chunk => this.node.anchorAll(...chunk)));
+    }
+}

src/index/index.events.ts

@@ -2,8 +2,10 @@ import { IndexDocumentType } from './model/index.model';
 
 export interface IndexEventsReturnType {
   IndexTransaction: IndexDocumentType;
+  IndexBlock: number;
 }
 
 export const IndexEvent: { [P in keyof IndexEventsReturnType]: P } = {
   IndexTransaction: 'IndexTransaction',
-}
+  IndexBlock: 'IndexBlock',
+};

src/index/index.service.spec.ts

@@ -42,14 +42,16 @@ describe('IndexService', () => {
     test('should ignore genesis transactions', async () => {
       const spies = spy();
 
-      const type = 'anchor';
       const transaction = { id: 'fake_transaction', type: 1, sender: 'fake_sender' };
       await indexService.index({ transaction: transaction as any, blockHeight: 1, position: 0});
 
-      expect(spies.emitter.emit.mock.calls.length).toBe(1);
+      expect(spies.emitter.emit.mock.calls.length).toBe(2);
+      expect(spies.emitter.emit.mock.calls[0][0]).toBe('IndexBlock');
+      expect(spies.emitter.emit.mock.calls[1][0]).toBe('IndexTransaction');
     });
 
     test('should index the anchor transaction', async () => {
+      indexService.lastBlock = 1; // Don't emit IndexBlock
       const spies = spy();
 
       const type = 'anchor';

src/index/index.service.ts

@@ -8,7 +8,7 @@ import { LoggerService } from '../logger/logger.service';
 export class IndexService {
 
   public lastBlock: number;
-  public txCache: string[];
+  public txCache: string[] = [];
 
   constructor(
     private readonly logger: LoggerService,
@@ -24,6 +24,7 @@ export class IndexService {
   async index(index: IndexDocumentType): Promise<boolean> {
     if (this.lastBlock !== index.blockHeight) {
       this.txCache = [];
+      this.event.emit(IndexEvent.IndexBlock, index.blockHeight);
     }
 
     this.lastBlock = index.blockHeight;

src/leveldb/classes/leveldb.connection.ts

@@ -50,12 +50,12 @@ export class LeveldbConnection {
     return this.connection.del(key);
   }
 
-  async incr(key): Promise<string> {
+  async incr(key, amount = 1): Promise<string> {
     await this.incrLock.acquireAsync();
 
     try {
       const count = Number(await this.get(key));
-      const result = await this.set(key, String(count + 1));
+      const result = await this.set(key, String(count + amount));
       return result;
     } finally {
       this.incrLock.release();

src/node/node.service.spec.ts

@@ -68,7 +68,6 @@ describe('NodeService', () => {
     };
 
     const config = {
-      getSponsorFee: jest.spyOn(configService, 'getSponsorFee').mockImplementation(() => 5000),
     };
 
     return { api, node, storage, config };
@@ -428,7 +427,7 @@ describe('NodeService', () => {
         version: 1,
         type: 18,
         recipient: 'some-recipient',
-        fee: 5000,
+        fee: 500000000,
       });
     });
   });
@@ -444,7 +443,7 @@ describe('NodeService', () => {
         version: 1,
         type: 19,
         recipient: 'some-recipient',
-        fee: 5000,
+        fee: 500000000,
       });
     });
   });