Issue #269, #272 & #166 - Implemented protocol versioning support with backward compatibility

README.md

@@ -12,9 +12,12 @@ See the [implementation document](docs/implementation.md) for the detailed descr
 
 1. Must pass `npm run test`.
 1. Must pass `npm run lint`.
-1. Must and only prefix the name of a "data structure interface" (interface that is without methods and act purely as data holders) with an `I`.
-1. Must and only export a class as a default export if the class name matches the file name.
-1. Must sort imports.
+1. Prefix an interface that require implementation with `I`. e.g. `ITransactionProcessor`.
+1. Suffix a data-holder interface (without definition of methods) with `Model`. e.g. `TransactionModel`.
+1. Use default export if class/interface name matches the file name.
+1. Sort imports.
 
 ## Docker
-The Sidetree components are also available via docker containers. Please see the [docker document](docs/docker.md) to find out details on building and running.
+> NOTE: 2019-08-13: docker-compose out-of-date, needs to be udpated.
+
+The Sidetree components are available via docker containers . Please see the [docker document](docs/docker.md) to find out details on building and running.

lib/bitcoin/BitcoinProcessor.ts

@@ -1,10 +1,10 @@
 import * as httpStatus from 'http-status';
 import MongoDbTransactionStore from '../common/MongoDbTransactionStore';
 import nodeFetch, { FetchError, Response, RequestInit } from 'node-fetch';
-import ErrorCode from '../common/ErrorCode';
-import ITransaction from '../common/ITransaction';
+import ErrorCode from '../common/SharedErrorCode';
 import ReadableStream from '../common/ReadableStream';
 import RequestError from './RequestError';
+import TransactionModel from '../common/models/TransactionModel';
 import TransactionNumber from './TransactionNumber';
 import { Address, Networks, PrivateKey, Script, Transaction } from 'bitcore-lib';
 import { IBitcoinConfig } from './IBitcoinConfig';
@@ -124,7 +124,7 @@ export default class BitcoinProcessor {
    * Initializes the Bitcoin processor
    */
   public async initialize () {
-    console.debug('Initializing TransactionStore');
+    console.debug('Initializing ITransactionStore');
     await this.transactionStore.initialize();
     const address = this.privateKey.toAddress();
     console.debug(`Checking if bitcoin contains a wallet for ${address}`);
@@ -194,7 +194,7 @@ export default class BitcoinProcessor {
    */
   public async transactions (since?: number, hash?: string): Promise<{
     moreTransactions: boolean,
-    transactions: ITransaction[]
+    transactions: TransactionModel[]
   }> {
     if ((since && !hash) ||
         (!since && hash)) {
@@ -215,7 +215,7 @@ export default class BitcoinProcessor {
         transactionTime: transaction.transactionTime,
         transactionTimeHash: transaction.transactionTimeHash,
         anchorFileHash: transaction.anchorFileHash
-      } as ITransaction;
+      } as TransactionModel;
     });
 
     return {
@@ -229,7 +229,7 @@ export default class BitcoinProcessor {
    * @param transactions List of transactions to check
    * @returns The first valid transaction, or undefined if none are valid
    */
-  public async firstValidTransaction (transactions: ITransaction[]): Promise<ITransaction | undefined> {
+  public async firstValidTransaction (transactions: TransactionModel[]): Promise<TransactionModel | undefined> {
     for (let index = 0; index < transactions.length; index++) {
       const transaction = transactions[index];
       const height = transaction.transactionTime;
@@ -516,7 +516,7 @@ export default class BitcoinProcessor {
         const data = Buffer.from(hexDataMatches[1], 'hex').toString();
         if (data.startsWith(this.sidetreePrefix)) {
           // we have found a sidetree transaction
-          const sidetreeTransaction: ITransaction = {
+          const sidetreeTransaction: TransactionModel = {
             transactionNumber: TransactionNumber.construct(block, transactionIndex),
             transactionTime: block,
             transactionTimeHash: blockHash,

lib/bitcoin/RequestError.ts

@@ -1,4 +1,3 @@
-import ErrorCode from '../common/ErrorCode';
 import Response, { ResponseStatus } from '../common/Response';
 
 /**
@@ -19,7 +18,7 @@ export default class RequestError extends Error {
     return this.code !== undefined;
   }
 
-  constructor (public readonly responseCode: ResponseStatus, public readonly code?: ErrorCode) {
+  constructor (public readonly responseCode: ResponseStatus, public readonly code?: string) {
     super(code ? JSON.stringify({ code }) : undefined);
 
     // NOTE: Extending 'Error' breaks prototype chain since TypeScript 2.1.

lib/common/MongoDbTransactionStore.ts

@@ -1,11 +1,11 @@
-import ITransaction from './ITransaction';
-import TransactionStore from '../core/interfaces/TransactionStore';
+import ITransactionStore from '../core/interfaces/ITransactionStore';
+import TransactionModel from './models/TransactionModel';
 import { Collection, Db, Long, MongoClient } from 'mongodb';
 
 /**
- * Implementation of TransactionStore that stores the transaction data in a MongoDB database.
+ * Implementation of ITransactionStore that stores the transaction data in a MongoDB database.
  */
-export default class MongoDbTransactionStore implements TransactionStore {
+export default class MongoDbTransactionStore implements ITransactionStore {
   /** Default database name used if not specified in constructor. */
   public static readonly defaultDatabaseName: string = 'sidetree';
   /** Collection name for transactions. */
@@ -44,7 +44,7 @@ export default class MongoDbTransactionStore implements TransactionStore {
     return transactionCount;
   }
 
-  public async getTransaction (transactionNumber: number): Promise<ITransaction | undefined> {
+  public async getTransaction (transactionNumber: number): Promise<TransactionModel | undefined> {
     const transactions = await this.transactionCollection!.find({ transactionNumber: Long.fromNumber(transactionNumber) }).toArray();
     if (transactions.length === 0) {
       return undefined;
@@ -54,7 +54,7 @@ export default class MongoDbTransactionStore implements TransactionStore {
     return transaction;
   }
 
-  public async getTransactionsLaterThan (transactionNumber: number | undefined, max: number): Promise<ITransaction[]> {
+  public async getTransactionsLaterThan (transactionNumber: number | undefined, max: number): Promise<TransactionModel[]> {
     let transactions = [];
 
     try {
@@ -82,7 +82,7 @@ export default class MongoDbTransactionStore implements TransactionStore {
     this.transactionCollection = await MongoDbTransactionStore.createTransactionCollectionIfNotExist(this.db!);
   }
 
-  async addTransaction (transaction: ITransaction): Promise<void> {
+  async addTransaction (transaction: TransactionModel): Promise<void> {
     try {
       const transactionInMongoDb = {
         anchorFileHash: transaction.anchorFileHash,
@@ -100,7 +100,7 @@ export default class MongoDbTransactionStore implements TransactionStore {
     }
   }
 
-  async getLastTransaction (): Promise<ITransaction | undefined> {
+  async getLastTransaction (): Promise<TransactionModel | undefined> {
     const lastTransactions = await this.transactionCollection!.find().limit(1).sort({ transactionNumber: -1 }).toArray();
     if (lastTransactions.length === 0) {
       return undefined;
@@ -110,8 +110,8 @@ export default class MongoDbTransactionStore implements TransactionStore {
     return lastProcessedTransaction;
   }
 
-  async getExponentiallySpacedTransactions (): Promise<ITransaction[]> {
-    const exponentiallySpacedTransactions: ITransaction[] = [];
+  async getExponentiallySpacedTransactions (): Promise<TransactionModel[]> {
+    const exponentiallySpacedTransactions: TransactionModel[] = [];
     const allTransactions = await this.transactionCollection!.find().sort({ transactionNumber: 1 }).toArray();
 
     let index = allTransactions.length - 1;
@@ -138,7 +138,7 @@ export default class MongoDbTransactionStore implements TransactionStore {
    * Gets the list of processed transactions.
    * Mainly used for test purposes.
    */
-  public async getTransactions (): Promise<ITransaction[]> {
+  public async getTransactions (): Promise<TransactionModel[]> {
     const transactions = await this.transactionCollection!.find().sort({ transactionNumber: 1 }).toArray();
     return transactions;
   }
@@ -147,7 +147,7 @@ export default class MongoDbTransactionStore implements TransactionStore {
    * Creates the `transaction` collection with indexes if it does not exists.
    * @returns The existing collection if exists, else the newly created collection.
    */
-  private static async createTransactionCollectionIfNotExist (db: Db): Promise<Collection<ITransaction>> {
+  private static async createTransactionCollectionIfNotExist (db: Db): Promise<Collection<TransactionModel>> {
     const collections = await db.collections();
     const collectionNames = collections.map(collection => collection.collectionName);
 

lib/common/Response.ts

@@ -1,7 +1,7 @@
 /**
  * Defines a Sidetree response object.
  */
-interface IResponse {
+interface ResponseModel {
   status: ResponseStatus;
   body?: any;
 }
@@ -38,4 +38,4 @@ export default class Response {
   }
 }
 
-export { IResponse, Response, ResponseStatus };
+export { Response, ResponseModel, ResponseStatus };

lib/common/SharedErrorCode.ts

@@ -0,0 +1,6 @@
+/**
+ * Common error codes used across services.
+ */
+export default {
+  InvalidTransactionNumberOrTimeHash: 'invalid_transaction_number_or_time_hash'
+};

lib/common/IFetchResult.ts → lib/common/models/FetchResult.ts

@@ -1,9 +1,9 @@
-import { FetchResultCode } from './FetchResultCode';
+import { FetchResultCode } from '../FetchResultCode';
 
 /**
  * Data structure representing the result of a content fetch from the Content Addressable Storage.
  */
-export default interface IFetchResult {
+export default interface FetchResult {
   /** Return code for the fetch. */
   code: FetchResultCode;
   content?: Buffer;

lib/common/ITransaction.ts → lib/common/models/TransactionModel.ts

@@ -1,7 +1,7 @@
 /**
  * Defines a Sidetree transaction.
  */
-export default interface ITransaction {
+export default interface TransactionModel {
   transactionNumber: number;
   transactionTime: number;
   transactionTimeHash: string;

lib/core/BatchScheduler.ts

@@ -0,0 +1,56 @@
+import IBatchWriter from './interfaces/IBatchWriter';
+import IBlockchain from './interfaces/IBlockchain';
+import timeSpan = require('time-span');
+
+/**
+ * Class that performs periodic writing of batches of Sidetree operations to CAS and blockchain.
+ */
+export default class BatchScheduler {
+  /**
+   * Flag indicating if this Batch Writer is currently processing a batch of operations.
+   */
+  private processing: boolean = false;
+
+  public constructor (
+    private getBatchWriter: (blockchainTime: number) => IBatchWriter,
+    private blockchain: IBlockchain,
+    private batchingIntervalInSeconds: number) {
+  }
+
+  /**
+   * The function that starts periodically anchoring operation batches to blockchain.
+   */
+  public startPeriodicBatchWriting () {
+    setInterval(async () => this.writeOperationBatch(), this.batchingIntervalInSeconds * 1000);
+  }
+
+  /**
+   * Processes the operations in the queue.
+   */
+  public async writeOperationBatch () {
+    const endTimer = timeSpan(); // For calcuating time taken to write operations.
+
+    // Wait until the next interval if the Batch Writer is still processing a batch.
+    if (this.processing) {
+      return;
+    }
+
+    try {
+      console.info('Start operation batch writing...');
+      this.processing = true;
+
+      // Get the correct version of the `BatchWriter`.
+      const currentTime = this.blockchain.approximateTime.time;
+      const batchWriter = this.getBatchWriter(currentTime);
+
+      await batchWriter.write();
+    } catch (error) {
+      console.error('Unexpected and unhandled error during batch writing, investigate and fix:');
+      console.error(error);
+    } finally {
+      this.processing = false;
+
+      console.info(`End batch writing. Duration: ${endTimer.rounded()} ms.`);
+    }
+  }
+}