Feat/improve get account transactions (#843)

* wip: refactor getTransactions

* feat: improved the sql statement for getAccountTransactions

* chore: minor clean up

* feat: adding auto creation of sqlite db folder

* feat: added automatic sqlite folder creation

* chore: some self-review cleanups

* chore: some self-review cleanups

resources/db/migration_h2_v2/V11__tx_index_sender_recipient.sql

@@ -0,0 +1 @@
+CREATE INDEX IF NOT EXISTS "transaction_recipient_id_sender_id_idx" ON "transaction" ("recipient_id", "sender_id");

resources/db/migration_mariadb/V11__tx_index_sender_recipient.sql

@@ -0,0 +1 @@
+CREATE INDEX IF NOT EXISTS transaction_recipient_id_sender_id_idx ON transaction(recipient_id, sender_id);

resources/db/migration_postgres/V11__tx_index_sender_recipient.sql

@@ -0,0 +1 @@
+CREATE INDEX IF NOT EXISTS transaction_recipient_id_sender_id_idx ON transaction (recipient_id, sender_id);

resources/db/migration_sqlite/V11__tx_index_sender_recipient.sql

@@ -0,0 +1 @@
+CREATE INDEX IF NOT EXISTS transaction_recipient_id_sender_id_idx ON "transaction" (recipient_id, sender_id);

src/brs/BlockchainProcessorImpl.java

@@ -704,7 +704,7 @@ private void processFork(Peer peer, final List<Block> forkBlocks, long forkBlock
           }
         } catch (Exception exception) {
           if (exception.toString().contains("[SQLITE_BUSY]") || exception.toString().contains("[SQLITE_BUSY_SNAPSHOT]")) {
-            logger.warn("SQLite deadlock, trying again later");
+            logger.warn("SQLite busy, trying again later...");
           } else {
             exception.printStackTrace();
             logger.error("Uncaught exception in blockImporterThread", exception);

src/brs/at/AtApiPlatformImpl.java

@@ -36,14 +36,6 @@ public static AtApiPlatformImpl getInstance() {
     return instance;
   }
 
-  // Version 1 - ok
-  // Pro Block die letzten 500 blocke der aktuell relevanten ATs in den Mem laden - 40 ms
-
-  // Version 2 - 80% ok
-  // Bei start alle tx der letzen 500 Bloecke in den Speicher laden - und (check)  - 400ms (1x)
-  // Jeden weiteren Block inkrementell die neuesten Tx an den Speicher anhaengen (check) - 2m  (Nx)
-  // --> Entferne alte Bloecke (aelter als 500 Bloecke)
-
 
   private static Long findTransaction(int startHeight, int endHeight, Long atID, int numOfTx, long minAmount) {
     ATProcessorCache cache = ATProcessorCache.getInstance();
@@ -55,21 +47,6 @@ private static Long findTransaction(int startHeight, int endHeight, Long atID, i
       }
     }
     return Signum.getStores().getAtStore().findTransaction(startHeight, endHeight, atID, numOfTx, minAmount);
-//
-//    long id = 0;
-//    long idOrig = Signum.getStores().getAtStore().findTransaction(startHeight, endHeight, atID, numOfTx, minAmount);
-//    try {
-//      id = ATProcessorCache.getInstance().findTransactionId(startHeight, endHeight, atID, numOfTx, minAmount);
-//    } catch (ATProcessorCache.CacheMissException e) {
-//      logger.debug("Cache miss");
-//      id = Signum.getStores().getAtStore().findTransaction(startHeight, endHeight, atID, numOfTx, minAmount);
-//      // no op
-//    }
-//    if(id != idOrig){
-//      logger.error("Cache mismatch: {} x {}", id, idOrig );
-//    }
-//
-//    return idOrig;
   }
 
   private static int findTransactionHeight(Long transactionId, int height, Long atID, long minAmount) {
@@ -82,20 +59,7 @@ private static int findTransactionHeight(Long transactionId, int height, Long at
       }
     }
     return Signum.getStores().getAtStore().findTransactionHeight(transactionId, height, atID, minAmount);
-//    int h;
-//    int hOrig =
-//      Signum.getStores().getAtStore().findTransactionHeight(transactionId, height, atID, minAmount);
-//    try {
-//      h = ATProcessorCache.getInstance().findTransactionHeight(transactionId, height, atID, minAmount);
-//    } catch (ATProcessorCache.CacheMissException e) {
-//      logger.debug("Cache miss");
-//      h = Signum.getStores().getAtStore().findTransactionHeight(transactionId, height, atID, minAmount);
-//    }
-//    if(h != hOrig){
-//      logger.error("Cache mismatch: {} - {}", h, hOrig );
-//    }
-//
-//    return hOrig;
+
   }
 
   @Override

src/brs/db/sql/Db.java

@@ -136,6 +136,7 @@ private static DSLContext getDSLContext() {
     Connection con = localConnection.get();
     Settings settings = new Settings();
     settings.setRenderSchema(Boolean.FALSE);
+
     SQLDialect dialect = databaseInstance.getDialect();
     if (con == null) {
       return DSL.using(databaseInstance.getDataSource(), dialect, settings);

src/brs/db/sql/SqlBlockchainStore.java

@@ -152,9 +152,11 @@ public long getAtBurnTotal() {
 
   @Override
   public Collection<Transaction> getTransactions(Account account, int numberOfConfirmations, byte type, byte subtype, int blockTimestamp, int from, int to, boolean includeIndirectIncoming) {
+
     int height = getHeightForNumberOfConfirmations(numberOfConfirmations);
     return Db.useDSLContext(ctx -> {
       ArrayList<Condition> conditions = new ArrayList<>();
+
       if (blockTimestamp > 0) {
         conditions.add(TRANSACTION.BLOCK_TIMESTAMP.ge(blockTimestamp));
       }
@@ -168,26 +170,26 @@ public Collection<Transaction> getTransactions(Account account, int numberOfConf
         conditions.add(TRANSACTION.HEIGHT.le(height));
       }
 
-      SelectOrderByStep<TransactionRecord> select = ctx.selectFrom(TRANSACTION).where(conditions).and(
-        account == null ? TRANSACTION.RECIPIENT_ID.isNull() :
-          TRANSACTION.RECIPIENT_ID.eq(account.getId()).and(
-            TRANSACTION.SENDER_ID.ne(account.getId())
-          )
-      ).unionAll(
-        account == null ? null :
-          ctx.selectFrom(TRANSACTION).where(conditions).and(
-            TRANSACTION.SENDER_ID.eq(account.getId())
-          )
-      );
+      Condition accountCondition = DSL.trueCondition();
+      if (account != null) {
+        accountCondition = TRANSACTION.RECIPIENT_ID.eq(account.getId())
+          .and(TRANSACTION.SENDER_ID.ne(account.getId()))
+          .or(TRANSACTION.SENDER_ID.eq(account.getId()));
 
-      if (includeIndirectIncoming) {
-        select = select.unionAll(ctx.selectFrom(TRANSACTION)
-          .where(conditions)
-          .and(TRANSACTION.ID.in(ctx.select(INDIRECT_INCOMING.TRANSACTION_ID).from(INDIRECT_INCOMING)
-            .where(INDIRECT_INCOMING.ACCOUNT_ID.eq(account.getId())))));
+        if (includeIndirectIncoming) {
+          accountCondition = accountCondition.or(
+            TRANSACTION.ID.in(
+              DSL.select(INDIRECT_INCOMING.TRANSACTION_ID)
+                .from(INDIRECT_INCOMING)
+                .where(INDIRECT_INCOMING.ACCOUNT_ID.eq(account.getId()))
+            )
+          );
+        }
       }
 
-      SelectQuery<TransactionRecord> selectQuery = select
+      SelectQuery<TransactionRecord> selectQuery = ctx.selectFrom(TRANSACTION)
+        .where(conditions)
+        .and(accountCondition)
         .orderBy(TRANSACTION.BLOCK_TIMESTAMP.desc(), TRANSACTION.ID.desc())
         .getQuery();
 
@@ -205,6 +207,7 @@ private static int getHeightForNumberOfConfirmations(int numberOfConfirmations)
     return height;
   }
 
+  // TODO: better introduce a dedicated bySender, byRecipient endpoint to reduce complexity
   @Override
   public Collection<Transaction> getTransactions(Long senderId, Long recipientId, int numberOfConfirmations, byte type, byte subtype, int blockTimestamp, int from, int to, boolean includeIndirectIncoming, boolean bidirectional) {
     int height = getHeightForNumberOfConfirmations(numberOfConfirmations);

src/brs/db/sql/dialects/DatabaseInstanceSqlite.java

@@ -7,27 +7,32 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.io.File;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+
 public class DatabaseInstanceSqlite extends DatabaseInstanceBaseImpl {
   private static final Logger logger = LoggerFactory.getLogger(DatabaseInstanceSqlite.class);
 
   protected DatabaseInstanceSqlite(PropertyService propertyService) {
     super(propertyService);
   }
 
-  private String getJournalMode(){
+  private String getJournalMode() {
     String journalMode = propertyService.getString(Props.DB_SQLITE_JOURNAL_MODE).toUpperCase();
-    if(
+    if (
       journalMode.equals("WAL") ||
-      journalMode.equals("TRUNCATE") ||
-      journalMode.equals("DELETE") ||
-      journalMode.equals("PERSIST")
-    ){
+        journalMode.equals("TRUNCATE") ||
+        journalMode.equals("DELETE") ||
+        journalMode.equals("PERSIST")
+    ) {
       return journalMode;
     }
     return "WAL";
   }
 
-  private String getSynchronousMode(){
+  private String getSynchronousMode() {
     String synchronous = propertyService.getString(Props.DB_SQLITE_SYNCHRONOUS).toUpperCase();
     switch (synchronous) {
       case "FULL":
@@ -40,12 +45,15 @@ private String getSynchronousMode(){
     }
   }
 
-  private int getCacheSize(){
+  private int getCacheSize() {
     return propertyService.getInt(Props.DB_SQLITE_CACHE_SIZE);
   }
 
   @Override
   protected HikariConfig configureImpl(HikariConfig config) {
+
+    ensureSqliteFolder();
+
     config.setMaximumPoolSize(10);
     config.setConnectionTestQuery("SELECT 1;");
     config.addDataSourceProperty("foreign_keys", "off");
@@ -68,6 +76,33 @@ public SQLDialect getDialect() {
     return SQLDialect.SQLITE;
   }
 
+  private static String extractSqliteFolderPath(String jdbcUrl) {
+    if (jdbcUrl == null || !jdbcUrl.startsWith("jdbc:sqlite:")) {
+      throw new IllegalArgumentException("Invalid SQLite JDBC URL");
+    }
+    String filePath = jdbcUrl.substring("jdbc:sqlite:file:".length());
+    Path path = Paths.get(filePath).toAbsolutePath().getParent();
+    return path != null ? path.toString() : null;
+  }
+
+  private void ensureSqliteFolder() {
+    String dbUrl = propertyService.getString(Props.DB_URL);
+    String folderPath = extractSqliteFolderPath(dbUrl);
+    if (folderPath != null) {
+      File dbFolder = new File(folderPath);
+      if (!dbFolder.exists()) {
+        logger.info("Creating SQLite DB folder(s): " + folderPath);
+        try{
+          Files.createDirectories(Paths.get(folderPath));
+        }catch(Exception e){
+          logger.error("Failed to create SQLite DB folder: " + folderPath);
+          throw new RuntimeException(e);
+        }
+      } else {
+        logger.warn("SQLite database folder path couldn't be found for " + dbUrl);
+      }
+    }
+  }
 
   @Override
   protected void onStartupImpl() {

src/brs/web/api/http/handler/GetAccountTransactions.java

@@ -94,6 +94,7 @@ protected JsonElement processRequest(HttpServletRequest req) throws SignumExcept
     int timestamp = ParameterParser.getTimestamp(req);
     int numberOfConfirmations = parameterService.getNumberOfConfirmations(req);
     boolean includeIndirect = parameterService.getIncludeIndirect(req);
+
     CollectionWithIndex<Transaction> accountTransactions = account != null
       ? blockchain.getTransactions(
       account,
@@ -116,7 +117,6 @@ protected JsonElement processRequest(HttpServletRequest req) throws SignumExcept
       includeIndirect,
       parameterService.getBidirectional(req));
 
-
     JsonArray transactions = new JsonArray();
     for (Transaction transaction : accountTransactions) {
       transactions.add(JSONData.transaction(transaction, blockchain.getHeight()));