Skip to content

Commit

Permalink
Refactoring and cleaning up
Browse files Browse the repository at this point in the history
  • Loading branch information
@freimair
freimair committed May 20, 2020
1 parent b059c44 commit 599110a
Show file tree
Hide file tree
Showing 2 changed files with 96 additions and 31 deletions.
50 changes: 33 additions & 17 deletions p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,7 @@
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.NoSuchElementException;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CopyOnWriteArraySet;
Expand Down Expand Up @@ -213,19 +213,33 @@ public GetUpdatedDataRequest buildGetUpdatedDataRequest(NodeAddress senderNodeAd
return new GetUpdatedDataRequest(senderNodeAddress, nonce, this.getKnownPayloadHashes());
}

/**
* Create the special key. <br><br>
*
* For example: "1.3.4" encoded into a 20 byte array.
*
* @return the special key
*/
private byte[] getSpecialKey() {
byte[] result = new byte[20];
Arrays.fill(result, (byte) 0);
System.arraycopy(Version.VERSION.getBytes(), 0, result, 0, Version.VERSION.length());
return result;
}

private String containsSpecialKey(Set<P2PDataStorage.ByteArray> collection) {
Optional<String> result = collection.stream().map(byteArray -> new String(byteArray.bytes).trim()).filter(s -> s.matches("^[0-9]\\.[0-9]\\.[0-9]$")).findFirst();
if (result.isPresent())
return result.get();
else
return "";
/**
* See if the request contains a "special key".
*
* @param knownPayloadHashes
* @throws NoSuchElementException if there is no "special key" in the list
* @return the "special key"
*/
private String containsSpecialKey(Set<P2PDataStorage.ByteArray> knownPayloadHashes) {
return knownPayloadHashes.stream()
.map(byteArray -> new String(byteArray.bytes).trim())
.filter(s -> s.matches("^[0-9]\\.[0-9]\\.[0-9]$"))
.findFirst()
.orElseThrow();
}

/**
Expand All @@ -238,14 +252,15 @@ private Set<byte[]> getKnownPayloadHashes() {
// an object gets removed in between PreliminaryGetDataRequest and the GetUpdatedDataRequest and we would
// miss that event if we do not load the full set or use some delta handling.
Set<byte[]> excludedKeys;
if (seedNodeRepository != null && seedNodeRepository.isSeedNode(networkNode.getNodeAddress()))
if (seedNodeRepository != null && seedNodeRepository.isSeedNode(networkNode.getNodeAddress())) {
excludedKeys = this.appendOnlyDataStoreService.getMap().keySet().stream()
.map(e -> e.bytes)
.collect(Collectors.toSet());
else
} else {
excludedKeys = this.appendOnlyDataStoreService.getMap("since " + Version.VERSION).keySet().stream()
.map(e -> e.bytes)
.collect(Collectors.toSet());
}

Set<byte[]> excludedKeysFromPersistedEntryMap = this.map.keySet()
.stream()
Expand Down Expand Up @@ -299,18 +314,19 @@ public GetDataResponse buildGetDataResponse(
Set<P2PDataStorage.ByteArray> excludedKeysAsByteArray =
P2PDataStorage.ByteArray.convertBytesSetToByteArraySet(getDataRequest.getExcludedKeys());

String specialKey = containsSpecialKey(excludedKeysAsByteArray);

Map<ByteArray, PersistableNetworkPayload> tmp;
if ("".equals(specialKey))
tmp = this.appendOnlyDataStoreService.getMap();
else {
tmp = this.appendOnlyDataStoreService.getMap("since " + specialKey);
// In case we get a "new" data request, ie. with a "special key" like "1.3.4", we
// pre-filter the data. If there is no "special key", we use all data.
Map<ByteArray, PersistableNetworkPayload> prefilteredData;
try {
String specialKey = containsSpecialKey(excludedKeysAsByteArray);
prefilteredData = this.appendOnlyDataStoreService.getMap("since " + specialKey);
} catch (NoSuchElementException e) {
prefilteredData = this.appendOnlyDataStoreService.getMap();
}

Set<PersistableNetworkPayload> filteredPersistableNetworkPayloads =
filterKnownHashes(
tmp,
prefilteredData,
Function.identity(),
excludedKeysAsByteArray,
peerCapabilities,
Expand Down
77 changes: 63 additions & 14 deletions p2p/src/main/java/bisq/network/p2p/storage/persistence/SplitStoreService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -100,32 +100,72 @@ public Map<P2PDataStorage.ByteArray, PersistableNetworkPayload> getMap(String fi
filter = filter.replace("since ", "");
if (!filter.equals(Version.VERSION)) {
String finalFilter = filter;
history.entrySet().stream().filter(entry -> Integer.valueOf(entry.getKey().replace(".", "")) > Integer.valueOf(finalFilter.replace(".", ""))).forEach(entry -> result.putAll(entry.getValue().getMap()));
history.entrySet().stream()
.filter(entry -> parseSpecialKey(entry.getKey()) > parseSpecialKey(finalFilter))
.forEach(entry -> result.putAll(entry.getValue().getMap()));
}
}

return result;
}

private int parseSpecialKey(String specialKey) {
return Integer.parseInt(specialKey.replace(".", ""));
}

/**
* For the {@link SplitStoreService}s, we check if we already have all the historical data stores in our working
* directory. If we have, we can proceed loading the stores. If we do not, we have to transfer the fresh data stores
* from resources.
*
* @param postFix
*/
@Override
protected void readFromResources(String postFix) {
// check Version.VERSION and see if we have latest r/o data store file in working directory
if (!new File(absolutePathOfStorageDir + File.separator + getFileName() + "_" + Version.VERSION).exists())
if (!new File(absolutePathOfStorageDir, getFileName() + "_" + Version.VERSION).exists())
makeFileFromResourceFile(postFix); // if we have the latest file, we are good, else do stuff // TODO are we?
else {
// load stores/storage
// if we have the r/o data stores in our working directory already, we can proceed on loading them.
File dbDir = new File(absolutePathOfStorageDir);
List<File> resourceFiles = Arrays.asList(dbDir.list((dir, name) -> name.startsWith(getFileName() + "_"))).stream().map(s -> new File(s)).collect(Collectors.toList());
List<File> resourceFiles = Arrays.asList(dbDir.list((dir, name) -> name.startsWith(getFileName() + "_")))
.stream()
.map(s -> new File(s))
.collect(Collectors.toList());

history = new HashMap<>();
store = readStore(getFileName());
resourceFiles.forEach(file -> {
SplitStore tmp = readStore(file.getName().replace(postFix, ""));
history.put(file.getName().replace(postFix, "").replace(getFileName(), "").replace("_", ""), tmp);
history.put(extractSpecialKey(file.getName(), postFix), tmp);
});
}
}

/**
* Extracts the special key from a file name.<br><br>
*
* Example: "TradeStatistics2Store_1.3.4_BTC_MAINNET" becomes "1.3.4"
*
* @param name the file name to begin with
* @param postFix the postfix eg. "BTC_MAINNET"
* @return the special key eg. "1.3.4"
*/
private String extractSpecialKey(String name, String postFix) {
return name.replace(postFix, "") // remove the postfix
.replace("_", "") // remove the spacings
.replace(getFileName(), ""); // remove the file name
}

/**
* Bluntly copy and pasted from {@link StoreService} because:
* <ul><li>The member function there is private</li>
* <li>it does not match our interface</li>
* <li>is a temporary solutions, until https://github.com/bisq-network/projects/issues/29</li></ul>
*
* @param name
* @return store
*/
private T readStore(String name) {
T store = storage.initAndGetPersistedWithFileName(name, 100);
if (store != null) {
Expand All @@ -139,6 +179,15 @@ private T readStore(String name) {
return store;
}

/**
* We compile a list of files available in resources and:<br>
* <ul><li>copy them to our working directory</li>
* <li>load the freshly copied stores</li>
* <li>remove the contents from our live data store</li>
* <li>persist the cleaned-up live store</li></ul>
*
* @param postFix
*/
@Override
protected void makeFileFromResourceFile(String postFix) {
File dbDir = new File(absolutePathOfStorageDir);
Expand All @@ -149,25 +198,25 @@ protected void makeFileFromResourceFile(String postFix) {
List<String> versions = new ArrayList<>();
versions.add(Version.VERSION);
versions.addAll(Version.history);
List<String> resourceFiles = versions.stream().map(s -> getFileName() + "_" + s + postFix).collect(Collectors.toList());
List<String> resourceFiles = versions.stream()
.map(s -> getFileName() + "_" + s + postFix)
.collect(Collectors.toList());

// if not, copy and split
resourceFiles.forEach(file -> {
final File destinationFile = new File(Paths.get(absolutePathOfStorageDir, file.replace(postFix, "")).toString());
final String resourceFileName = file;
resourceFiles.forEach(resourceFileName -> {
final File destinationFile = new File(absolutePathOfStorageDir, resourceFileName.replace(postFix, ""));
if (!destinationFile.exists()) {
try {
log.info("We copy resource to file: resourceFileName={}, destinationFile={}", resourceFileName, destinationFile);
FileUtil.resourceToFile(resourceFileName, destinationFile);
} catch (ResourceNotFoundException e) {
log.info("Could not find resourceFile " + resourceFileName + ". That is expected if none is provided yet.");
log.info("Could not find resourceFile {}. That is expected if none is provided yet.", resourceFileName);
} catch (Throwable e) {
log.error("Could not copy resourceFile " + resourceFileName + " to " +
destinationFile.getAbsolutePath() + ".\n" + e.getMessage());
log.error("Could not copy resourceFile {} to {}.\n{}", resourceFileName, destinationFile.getAbsolutePath(), e.getMessage());
e.printStackTrace();
}
} else {
log.debug(file + " file exists already.");
log.debug(resourceFileName + " file exists already.");
}
});

Expand All @@ -177,7 +226,7 @@ protected void makeFileFromResourceFile(String postFix) {
store = readStore(getFileName());
resourceFiles.forEach(file -> {
SplitStore tmp = readStore(file.replace(postFix, ""));
history.put(file.replace(postFix, "").replace(getFileName(), "").replace("_", ""), tmp);
history.put(extractSpecialKey(file, postFix), tmp);
// - subtract all that is in resource files
store.getMap().keySet().removeAll(tmp.getMap().keySet());
});
Expand Down

0 comments on commit 599110a

Please sign in to comment.