diff --git a/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java b/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java index 2c86f0eb6c8..904a54eeb77 100644 --- a/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java +++ b/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java @@ -92,7 +92,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Optional; +import java.util.NoSuchElementException; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CopyOnWriteArraySet; @@ -213,6 +213,13 @@ public GetUpdatedDataRequest buildGetUpdatedDataRequest(NodeAddress senderNodeAd return new GetUpdatedDataRequest(senderNodeAddress, nonce, this.getKnownPayloadHashes()); } + /** + * Create the special key.

+ * + * For example: "1.3.4" encoded into a 20 byte array. + * + * @return the special key + */ private byte[] getSpecialKey() { byte[] result = new byte[20]; Arrays.fill(result, (byte) 0); @@ -220,12 +227,19 @@ private byte[] getSpecialKey() { return result; } - private String containsSpecialKey(Set collection) { - Optional result = collection.stream().map(byteArray -> new String(byteArray.bytes).trim()).filter(s -> s.matches("^[0-9]\\.[0-9]\\.[0-9]$")).findFirst(); - if (result.isPresent()) - return result.get(); - else - return ""; + /** + * See if the request contains a "special key". + * + * @param knownPayloadHashes + * @throws NoSuchElementException if there is no "special key" in the list + * @return the "special key" + */ + private String containsSpecialKey(Set knownPayloadHashes) { + return knownPayloadHashes.stream() + .map(byteArray -> new String(byteArray.bytes).trim()) + .filter(s -> s.matches("^[0-9]\\.[0-9]\\.[0-9]$")) + .findFirst() + .orElseThrow(); } /** @@ -238,14 +252,15 @@ private Set getKnownPayloadHashes() { // an object gets removed in between PreliminaryGetDataRequest and the GetUpdatedDataRequest and we would // miss that event if we do not load the full set or use some delta handling. Set excludedKeys; - if (seedNodeRepository != null && seedNodeRepository.isSeedNode(networkNode.getNodeAddress())) + if (seedNodeRepository != null && seedNodeRepository.isSeedNode(networkNode.getNodeAddress())) { excludedKeys = this.appendOnlyDataStoreService.getMap().keySet().stream() .map(e -> e.bytes) .collect(Collectors.toSet()); - else + } else { excludedKeys = this.appendOnlyDataStoreService.getMap("since " + Version.VERSION).keySet().stream() .map(e -> e.bytes) .collect(Collectors.toSet()); + } Set excludedKeysFromPersistedEntryMap = this.map.keySet() .stream() @@ -299,18 +314,19 @@ public GetDataResponse buildGetDataResponse( Set excludedKeysAsByteArray = P2PDataStorage.ByteArray.convertBytesSetToByteArraySet(getDataRequest.getExcludedKeys()); - String specialKey = containsSpecialKey(excludedKeysAsByteArray); - - Map tmp; - if ("".equals(specialKey)) - tmp = this.appendOnlyDataStoreService.getMap(); - else { - tmp = this.appendOnlyDataStoreService.getMap("since " + specialKey); + // In case we get a "new" data request, ie. with a "special key" like "1.3.4", we + // pre-filter the data. If there is no "special key", we use all data. + Map prefilteredData; + try { + String specialKey = containsSpecialKey(excludedKeysAsByteArray); + prefilteredData = this.appendOnlyDataStoreService.getMap("since " + specialKey); + } catch (NoSuchElementException e) { + prefilteredData = this.appendOnlyDataStoreService.getMap(); } Set filteredPersistableNetworkPayloads = filterKnownHashes( - tmp, + prefilteredData, Function.identity(), excludedKeysAsByteArray, peerCapabilities, diff --git a/p2p/src/main/java/bisq/network/p2p/storage/persistence/SplitStoreService.java b/p2p/src/main/java/bisq/network/p2p/storage/persistence/SplitStoreService.java index 0036c03e2e6..a67ea65a126 100644 --- a/p2p/src/main/java/bisq/network/p2p/storage/persistence/SplitStoreService.java +++ b/p2p/src/main/java/bisq/network/p2p/storage/persistence/SplitStoreService.java @@ -100,32 +100,72 @@ public Map getMap(String fi filter = filter.replace("since ", ""); if (!filter.equals(Version.VERSION)) { String finalFilter = filter; - history.entrySet().stream().filter(entry -> Integer.valueOf(entry.getKey().replace(".", "")) > Integer.valueOf(finalFilter.replace(".", ""))).forEach(entry -> result.putAll(entry.getValue().getMap())); + history.entrySet().stream() + .filter(entry -> parseSpecialKey(entry.getKey()) > parseSpecialKey(finalFilter)) + .forEach(entry -> result.putAll(entry.getValue().getMap())); } } return result; } + private int parseSpecialKey(String specialKey) { + return Integer.parseInt(specialKey.replace(".", "")); + } + + /** + * For the {@link SplitStoreService}s, we check if we already have all the historical data stores in our working + * directory. If we have, we can proceed loading the stores. If we do not, we have to transfer the fresh data stores + * from resources. + * + * @param postFix + */ @Override protected void readFromResources(String postFix) { // check Version.VERSION and see if we have latest r/o data store file in working directory - if (!new File(absolutePathOfStorageDir + File.separator + getFileName() + "_" + Version.VERSION).exists()) + if (!new File(absolutePathOfStorageDir, getFileName() + "_" + Version.VERSION).exists()) makeFileFromResourceFile(postFix); // if we have the latest file, we are good, else do stuff // TODO are we? else { - // load stores/storage + // if we have the r/o data stores in our working directory already, we can proceed on loading them. File dbDir = new File(absolutePathOfStorageDir); - List resourceFiles = Arrays.asList(dbDir.list((dir, name) -> name.startsWith(getFileName() + "_"))).stream().map(s -> new File(s)).collect(Collectors.toList()); + List resourceFiles = Arrays.asList(dbDir.list((dir, name) -> name.startsWith(getFileName() + "_"))) + .stream() + .map(s -> new File(s)) + .collect(Collectors.toList()); history = new HashMap<>(); store = readStore(getFileName()); resourceFiles.forEach(file -> { SplitStore tmp = readStore(file.getName().replace(postFix, "")); - history.put(file.getName().replace(postFix, "").replace(getFileName(), "").replace("_", ""), tmp); + history.put(extractSpecialKey(file.getName(), postFix), tmp); }); } } + /** + * Extracts the special key from a file name.

+ * + * Example: "TradeStatistics2Store_1.3.4_BTC_MAINNET" becomes "1.3.4" + * + * @param name the file name to begin with + * @param postFix the postfix eg. "BTC_MAINNET" + * @return the special key eg. "1.3.4" + */ + private String extractSpecialKey(String name, String postFix) { + return name.replace(postFix, "") // remove the postfix + .replace("_", "") // remove the spacings + .replace(getFileName(), ""); // remove the file name + } + + /** + * Bluntly copy and pasted from {@link StoreService} because: + *
  • The member function there is private
    • + *
  • it does not match our interface
    • + *
  • is a temporary solutions, until https://github.com/bisq-network/projects/issues/29
+ * + * @param name + * @return store + */ private T readStore(String name) { T store = storage.initAndGetPersistedWithFileName(name, 100); if (store != null) { @@ -139,6 +179,15 @@ private T readStore(String name) { return store; } + /** + * We compile a list of files available in resources and:
+ *
  • copy them to our working directory
    • + *
  • load the freshly copied stores
    • + *
  • remove the contents from our live data store
    • + *
  • persist the cleaned-up live store
+ * + * @param postFix + */ @Override protected void makeFileFromResourceFile(String postFix) { File dbDir = new File(absolutePathOfStorageDir); @@ -149,25 +198,25 @@ protected void makeFileFromResourceFile(String postFix) { List versions = new ArrayList<>(); versions.add(Version.VERSION); versions.addAll(Version.history); - List resourceFiles = versions.stream().map(s -> getFileName() + "_" + s + postFix).collect(Collectors.toList()); + List resourceFiles = versions.stream() + .map(s -> getFileName() + "_" + s + postFix) + .collect(Collectors.toList()); // if not, copy and split - resourceFiles.forEach(file -> { - final File destinationFile = new File(Paths.get(absolutePathOfStorageDir, file.replace(postFix, "")).toString()); - final String resourceFileName = file; + resourceFiles.forEach(resourceFileName -> { + final File destinationFile = new File(absolutePathOfStorageDir, resourceFileName.replace(postFix, "")); if (!destinationFile.exists()) { try { log.info("We copy resource to file: resourceFileName={}, destinationFile={}", resourceFileName, destinationFile); FileUtil.resourceToFile(resourceFileName, destinationFile); } catch (ResourceNotFoundException e) { - log.info("Could not find resourceFile " + resourceFileName + ". That is expected if none is provided yet."); + log.info("Could not find resourceFile {}. That is expected if none is provided yet.", resourceFileName); } catch (Throwable e) { - log.error("Could not copy resourceFile " + resourceFileName + " to " + - destinationFile.getAbsolutePath() + ".\n" + e.getMessage()); + log.error("Could not copy resourceFile {} to {}.\n{}", resourceFileName, destinationFile.getAbsolutePath(), e.getMessage()); e.printStackTrace(); } } else { - log.debug(file + " file exists already."); + log.debug(resourceFileName + " file exists already."); } }); @@ -177,7 +226,7 @@ protected void makeFileFromResourceFile(String postFix) { store = readStore(getFileName()); resourceFiles.forEach(file -> { SplitStore tmp = readStore(file.replace(postFix, "")); - history.put(file.replace(postFix, "").replace(getFileName(), "").replace("_", ""), tmp); + history.put(extractSpecialKey(file, postFix), tmp); // - subtract all that is in resource files store.getMap().keySet().removeAll(tmp.getMap().keySet()); });