From 442d93e11588dae50c92a7637b75968a1a4b5aab Mon Sep 17 00:00:00 2001 From: ohager Date: Sun, 1 Dec 2024 20:47:49 -0300 Subject: [PATCH] Feat/improve get account transactions (#843) * wip: refactor getTransactions * feat: improved the sql statement for getAccountTransactions * chore: minor clean up * feat: adding auto creation of sqlite db folder * feat: added automatic sqlite folder creation * chore: some self-review cleanups * chore: some self-review cleanups --- .../V11__tx_index_sender_recipient.sql | 1 + .../V11__tx_index_sender_recipient.sql | 1 + .../V11__tx_index_sender_recipient.sql | 1 + .../V11__tx_index_sender_recipient.sql | 1 + src/brs/BlockchainProcessorImpl.java | 2 +- src/brs/at/AtApiPlatformImpl.java | 38 +------------- src/brs/db/sql/Db.java | 1 + src/brs/db/sql/SqlBlockchainStore.java | 37 +++++++------- .../sql/dialects/DatabaseInstanceSqlite.java | 51 ++++++++++++++++--- .../http/handler/GetAccountTransactions.java | 2 +- 10 files changed, 71 insertions(+), 64 deletions(-) create mode 100644 resources/db/migration_h2_v2/V11__tx_index_sender_recipient.sql create mode 100644 resources/db/migration_mariadb/V11__tx_index_sender_recipient.sql create mode 100644 resources/db/migration_postgres/V11__tx_index_sender_recipient.sql create mode 100644 resources/db/migration_sqlite/V11__tx_index_sender_recipient.sql diff --git a/resources/db/migration_h2_v2/V11__tx_index_sender_recipient.sql b/resources/db/migration_h2_v2/V11__tx_index_sender_recipient.sql new file mode 100644 index 000000000..cbb0c75ee --- /dev/null +++ b/resources/db/migration_h2_v2/V11__tx_index_sender_recipient.sql @@ -0,0 +1 @@ +CREATE INDEX IF NOT EXISTS "transaction_recipient_id_sender_id_idx" ON "transaction" ("recipient_id", "sender_id"); diff --git a/resources/db/migration_mariadb/V11__tx_index_sender_recipient.sql b/resources/db/migration_mariadb/V11__tx_index_sender_recipient.sql new file mode 100644 index 000000000..ca5fca276 --- /dev/null +++ b/resources/db/migration_mariadb/V11__tx_index_sender_recipient.sql @@ -0,0 +1 @@ +CREATE INDEX IF NOT EXISTS transaction_recipient_id_sender_id_idx ON transaction(recipient_id, sender_id); diff --git a/resources/db/migration_postgres/V11__tx_index_sender_recipient.sql b/resources/db/migration_postgres/V11__tx_index_sender_recipient.sql new file mode 100644 index 000000000..6a8946f9c --- /dev/null +++ b/resources/db/migration_postgres/V11__tx_index_sender_recipient.sql @@ -0,0 +1 @@ +CREATE INDEX IF NOT EXISTS transaction_recipient_id_sender_id_idx ON transaction (recipient_id, sender_id); diff --git a/resources/db/migration_sqlite/V11__tx_index_sender_recipient.sql b/resources/db/migration_sqlite/V11__tx_index_sender_recipient.sql new file mode 100644 index 000000000..ba57d0a2d --- /dev/null +++ b/resources/db/migration_sqlite/V11__tx_index_sender_recipient.sql @@ -0,0 +1 @@ +CREATE INDEX IF NOT EXISTS transaction_recipient_id_sender_id_idx ON "transaction" (recipient_id, sender_id); diff --git a/src/brs/BlockchainProcessorImpl.java b/src/brs/BlockchainProcessorImpl.java index a154f73ed..1c212051b 100644 --- a/src/brs/BlockchainProcessorImpl.java +++ b/src/brs/BlockchainProcessorImpl.java @@ -704,7 +704,7 @@ private void processFork(Peer peer, final List forkBlocks, long forkBlock } } catch (Exception exception) { if (exception.toString().contains("[SQLITE_BUSY]") || exception.toString().contains("[SQLITE_BUSY_SNAPSHOT]")) { - logger.warn("SQLite deadlock, trying again later"); + logger.warn("SQLite busy, trying again later..."); } else { exception.printStackTrace(); logger.error("Uncaught exception in blockImporterThread", exception); diff --git a/src/brs/at/AtApiPlatformImpl.java b/src/brs/at/AtApiPlatformImpl.java index b154342fa..5cb761ecb 100644 --- a/src/brs/at/AtApiPlatformImpl.java +++ b/src/brs/at/AtApiPlatformImpl.java @@ -36,14 +36,6 @@ public static AtApiPlatformImpl getInstance() { return instance; } - // Version 1 - ok - // Pro Block die letzten 500 blocke der aktuell relevanten ATs in den Mem laden - 40 ms - - // Version 2 - 80% ok - // Bei start alle tx der letzen 500 Bloecke in den Speicher laden - und (check) - 400ms (1x) - // Jeden weiteren Block inkrementell die neuesten Tx an den Speicher anhaengen (check) - 2m (Nx) - // --> Entferne alte Bloecke (aelter als 500 Bloecke) - private static Long findTransaction(int startHeight, int endHeight, Long atID, int numOfTx, long minAmount) { ATProcessorCache cache = ATProcessorCache.getInstance(); @@ -55,21 +47,6 @@ private static Long findTransaction(int startHeight, int endHeight, Long atID, i } } return Signum.getStores().getAtStore().findTransaction(startHeight, endHeight, atID, numOfTx, minAmount); -// -// long id = 0; -// long idOrig = Signum.getStores().getAtStore().findTransaction(startHeight, endHeight, atID, numOfTx, minAmount); -// try { -// id = ATProcessorCache.getInstance().findTransactionId(startHeight, endHeight, atID, numOfTx, minAmount); -// } catch (ATProcessorCache.CacheMissException e) { -// logger.debug("Cache miss"); -// id = Signum.getStores().getAtStore().findTransaction(startHeight, endHeight, atID, numOfTx, minAmount); -// // no op -// } -// if(id != idOrig){ -// logger.error("Cache mismatch: {} x {}", id, idOrig ); -// } -// -// return idOrig; } private static int findTransactionHeight(Long transactionId, int height, Long atID, long minAmount) { @@ -82,20 +59,7 @@ private static int findTransactionHeight(Long transactionId, int height, Long at } } return Signum.getStores().getAtStore().findTransactionHeight(transactionId, height, atID, minAmount); -// int h; -// int hOrig = -// Signum.getStores().getAtStore().findTransactionHeight(transactionId, height, atID, minAmount); -// try { -// h = ATProcessorCache.getInstance().findTransactionHeight(transactionId, height, atID, minAmount); -// } catch (ATProcessorCache.CacheMissException e) { -// logger.debug("Cache miss"); -// h = Signum.getStores().getAtStore().findTransactionHeight(transactionId, height, atID, minAmount); -// } -// if(h != hOrig){ -// logger.error("Cache mismatch: {} - {}", h, hOrig ); -// } -// -// return hOrig; + } @Override diff --git a/src/brs/db/sql/Db.java b/src/brs/db/sql/Db.java index 4df2f35cb..23b305c6d 100644 --- a/src/brs/db/sql/Db.java +++ b/src/brs/db/sql/Db.java @@ -136,6 +136,7 @@ private static DSLContext getDSLContext() { Connection con = localConnection.get(); Settings settings = new Settings(); settings.setRenderSchema(Boolean.FALSE); + SQLDialect dialect = databaseInstance.getDialect(); if (con == null) { return DSL.using(databaseInstance.getDataSource(), dialect, settings); diff --git a/src/brs/db/sql/SqlBlockchainStore.java b/src/brs/db/sql/SqlBlockchainStore.java index bfce91a7a..7c26a3f84 100644 --- a/src/brs/db/sql/SqlBlockchainStore.java +++ b/src/brs/db/sql/SqlBlockchainStore.java @@ -152,9 +152,11 @@ public long getAtBurnTotal() { @Override public Collection getTransactions(Account account, int numberOfConfirmations, byte type, byte subtype, int blockTimestamp, int from, int to, boolean includeIndirectIncoming) { + int height = getHeightForNumberOfConfirmations(numberOfConfirmations); return Db.useDSLContext(ctx -> { ArrayList conditions = new ArrayList<>(); + if (blockTimestamp > 0) { conditions.add(TRANSACTION.BLOCK_TIMESTAMP.ge(blockTimestamp)); } @@ -168,26 +170,26 @@ public Collection getTransactions(Account account, int numberOfConf conditions.add(TRANSACTION.HEIGHT.le(height)); } - SelectOrderByStep select = ctx.selectFrom(TRANSACTION).where(conditions).and( - account == null ? TRANSACTION.RECIPIENT_ID.isNull() : - TRANSACTION.RECIPIENT_ID.eq(account.getId()).and( - TRANSACTION.SENDER_ID.ne(account.getId()) - ) - ).unionAll( - account == null ? null : - ctx.selectFrom(TRANSACTION).where(conditions).and( - TRANSACTION.SENDER_ID.eq(account.getId()) - ) - ); + Condition accountCondition = DSL.trueCondition(); + if (account != null) { + accountCondition = TRANSACTION.RECIPIENT_ID.eq(account.getId()) + .and(TRANSACTION.SENDER_ID.ne(account.getId())) + .or(TRANSACTION.SENDER_ID.eq(account.getId())); - if (includeIndirectIncoming) { - select = select.unionAll(ctx.selectFrom(TRANSACTION) - .where(conditions) - .and(TRANSACTION.ID.in(ctx.select(INDIRECT_INCOMING.TRANSACTION_ID).from(INDIRECT_INCOMING) - .where(INDIRECT_INCOMING.ACCOUNT_ID.eq(account.getId()))))); + if (includeIndirectIncoming) { + accountCondition = accountCondition.or( + TRANSACTION.ID.in( + DSL.select(INDIRECT_INCOMING.TRANSACTION_ID) + .from(INDIRECT_INCOMING) + .where(INDIRECT_INCOMING.ACCOUNT_ID.eq(account.getId())) + ) + ); + } } - SelectQuery selectQuery = select + SelectQuery selectQuery = ctx.selectFrom(TRANSACTION) + .where(conditions) + .and(accountCondition) .orderBy(TRANSACTION.BLOCK_TIMESTAMP.desc(), TRANSACTION.ID.desc()) .getQuery(); @@ -205,6 +207,7 @@ private static int getHeightForNumberOfConfirmations(int numberOfConfirmations) return height; } + // TODO: better introduce a dedicated bySender, byRecipient endpoint to reduce complexity @Override public Collection getTransactions(Long senderId, Long recipientId, int numberOfConfirmations, byte type, byte subtype, int blockTimestamp, int from, int to, boolean includeIndirectIncoming, boolean bidirectional) { int height = getHeightForNumberOfConfirmations(numberOfConfirmations); diff --git a/src/brs/db/sql/dialects/DatabaseInstanceSqlite.java b/src/brs/db/sql/dialects/DatabaseInstanceSqlite.java index 775183416..fa1421435 100644 --- a/src/brs/db/sql/dialects/DatabaseInstanceSqlite.java +++ b/src/brs/db/sql/dialects/DatabaseInstanceSqlite.java @@ -7,6 +7,11 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.File; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + public class DatabaseInstanceSqlite extends DatabaseInstanceBaseImpl { private static final Logger logger = LoggerFactory.getLogger(DatabaseInstanceSqlite.class); @@ -14,20 +19,20 @@ protected DatabaseInstanceSqlite(PropertyService propertyService) { super(propertyService); } - private String getJournalMode(){ + private String getJournalMode() { String journalMode = propertyService.getString(Props.DB_SQLITE_JOURNAL_MODE).toUpperCase(); - if( + if ( journalMode.equals("WAL") || - journalMode.equals("TRUNCATE") || - journalMode.equals("DELETE") || - journalMode.equals("PERSIST") - ){ + journalMode.equals("TRUNCATE") || + journalMode.equals("DELETE") || + journalMode.equals("PERSIST") + ) { return journalMode; } return "WAL"; } - private String getSynchronousMode(){ + private String getSynchronousMode() { String synchronous = propertyService.getString(Props.DB_SQLITE_SYNCHRONOUS).toUpperCase(); switch (synchronous) { case "FULL": @@ -40,12 +45,15 @@ private String getSynchronousMode(){ } } - private int getCacheSize(){ + private int getCacheSize() { return propertyService.getInt(Props.DB_SQLITE_CACHE_SIZE); } @Override protected HikariConfig configureImpl(HikariConfig config) { + + ensureSqliteFolder(); + config.setMaximumPoolSize(10); config.setConnectionTestQuery("SELECT 1;"); config.addDataSourceProperty("foreign_keys", "off"); @@ -68,6 +76,33 @@ public SQLDialect getDialect() { return SQLDialect.SQLITE; } + private static String extractSqliteFolderPath(String jdbcUrl) { + if (jdbcUrl == null || !jdbcUrl.startsWith("jdbc:sqlite:")) { + throw new IllegalArgumentException("Invalid SQLite JDBC URL"); + } + String filePath = jdbcUrl.substring("jdbc:sqlite:file:".length()); + Path path = Paths.get(filePath).toAbsolutePath().getParent(); + return path != null ? path.toString() : null; + } + + private void ensureSqliteFolder() { + String dbUrl = propertyService.getString(Props.DB_URL); + String folderPath = extractSqliteFolderPath(dbUrl); + if (folderPath != null) { + File dbFolder = new File(folderPath); + if (!dbFolder.exists()) { + logger.info("Creating SQLite DB folder(s): " + folderPath); + try{ + Files.createDirectories(Paths.get(folderPath)); + }catch(Exception e){ + logger.error("Failed to create SQLite DB folder: " + folderPath); + throw new RuntimeException(e); + } + } else { + logger.warn("SQLite database folder path couldn't be found for " + dbUrl); + } + } + } @Override protected void onStartupImpl() { diff --git a/src/brs/web/api/http/handler/GetAccountTransactions.java b/src/brs/web/api/http/handler/GetAccountTransactions.java index e89c56b6a..5e7eb448a 100644 --- a/src/brs/web/api/http/handler/GetAccountTransactions.java +++ b/src/brs/web/api/http/handler/GetAccountTransactions.java @@ -94,6 +94,7 @@ protected JsonElement processRequest(HttpServletRequest req) throws SignumExcept int timestamp = ParameterParser.getTimestamp(req); int numberOfConfirmations = parameterService.getNumberOfConfirmations(req); boolean includeIndirect = parameterService.getIncludeIndirect(req); + CollectionWithIndex accountTransactions = account != null ? blockchain.getTransactions( account, @@ -116,7 +117,6 @@ protected JsonElement processRequest(HttpServletRequest req) throws SignumExcept includeIndirect, parameterService.getBidirectional(req)); - JsonArray transactions = new JsonArray(); for (Transaction transaction : accountTransactions) { transactions.add(JSONData.transaction(transaction, blockchain.getHeight()));