Skip to content

Commit

Permalink
Implement piped extractions.
Browse files Browse the repository at this point in the history 
Extractions can be piped by adding the command arguments --all or --until.
If --all is provided, the extractor will keep extracting until it reaches a format that it cannot handle.
If --until is provided, the extractor will keep extracting until it reaches the extension specified by this parameter, or until it reaches a format it cannot handle.

Example. src: dir/Test.jar.pack.xz dest: dir/output
/ze extract --all will extract all the jar contents into output.
/ze extract --until will extract Test.jar into output.
/ze extract will extract Test.jar.pack into output (non piped).

If the piped operation contains a format which cannot be scanned ahead of time, the destination directory must be empty. If it is not, the operation will be cancelled, even if --override is passed. An example of this is the ZIP format. We cannot scan for extractions without first having the source file in the ZIP format. The scan involed checking each ZIP entry, which we do not have access to until the file is in ZIP format. This is the same for JAR and RAR.
  • Loading branch information
@dscalzi
dscalzi committed May 8, 2019
1 parent 6bac652 commit b0421f0
Show file tree
Hide file tree
Showing 12 changed files with 345 additions and 78 deletions.
80 changes: 47 additions & 33 deletions ZipExtractor-Core/src/main/java/com/dscalzi/zipextractor/core/ZCompressor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,11 @@ public static void asyncCompress(ICommandSender sender, File src, File dest, boo

Deque<OpTuple> pDeque = new ArrayDeque<OpTuple>();
if(destExts.length < 2) {
pDeque.push(new OpTuple(src, dest));
TypeProvider provider = getApplicableProvider(src, dest, mm, sender);
if(provider == null) {
return;
}
pDeque.push(new OpTuple(src, dest, provider));
} else {
File dTemp = dest;
File sTemp = null;
Expand All @@ -72,7 +76,13 @@ public static void asyncCompress(ICommandSender sender, File src, File dest, boo
(srcExts.length > 0 ? !destExts[i].equalsIgnoreCase(srcExts[srcExts.length-1]) : true)) {
pth = pth.substring(0, pth.length()-destExts[i].length()-1);
sTemp = new File(pth);
pDeque.push(new OpTuple(i == 0 ? src : sTemp, dTemp));

TypeProvider provider = getApplicableProvider(sTemp, dTemp, mm, sender);
if(provider == null) {
return;
}

pDeque.push(new OpTuple(i == 0 ? src : sTemp, dTemp, provider));
dTemp = sTemp;
} else {
pDeque.peek().setSrc(src);
Expand All @@ -86,39 +96,25 @@ public static void asyncCompress(ICommandSender sender, File src, File dest, boo
boolean piped = false;
final Runnable[] pipes = new Runnable[pDeque.size()];
for (final OpTuple e : pDeque) {
for (final TypeProvider p : TypeProvider.getProviders()) {
if (p.destValidForCompression(e.getDest())) {
if (p.srcValidForCompression(e.getSrc())) {
final boolean interOp = c != pDeque.size()-1;

if (e.getDest().exists() && !override) {
if(!interOp) mm.destExists(sender);
else mm.destExistsPiped(sender, e.getDest());
return;
}

if(piped) {
pipes[c] = () -> {
p.compress(sender, e.getSrc(), e.getDest(), log, interOp);
e.getSrc().delete();
};
} else {
pipes[c] = () -> {
p.compress(sender, e.getSrc(), e.getDest(), log, interOp);
};
}
piped = true;
} else {
mm.invalidSourceForDest(sender, p.canCompressFrom(), p.canCompressTo());
return;
}
}
}
// If we can't process this phase, cancel the operation.
if(pipes[c] == null) {
mm.invalidCompressionExtension(sender);
final boolean interOp = c != pDeque.size()-1;

if (e.getDest().exists() && !override) {
if(!interOp) mm.destExists(sender);
else mm.destExistsPiped(sender, e.getDest());
return;
}

if(piped) {
pipes[c] = () -> {
e.getProvider().compress(sender, e.getSrc(), e.getDest(), log, interOp);
e.getSrc().delete();
};
} else {
pipes[c] = () -> {
e.getProvider().compress(sender, e.getSrc(), e.getDest(), log, interOp);
};
}
piped = true;
c++;
}

Expand All @@ -137,6 +133,24 @@ else if (result == 2)
mm.executorTerminated(sender, ZTask.COMPRESS);
}

private static TypeProvider getApplicableProvider(File src, File dest, MessageManager mm, ICommandSender sender) {
TypeProvider provider = null;
for(TypeProvider p : TypeProvider.getProviders()) {
if(p.destValidForCompression(dest)) {
if (p.srcValidForCompression(src)) {
provider = p;
break;
} else {
mm.invalidSourceForDest(sender, p.canCompressFrom(), p.canCompressTo());
}
}
}
if(provider == null) {
mm.invalidCompressionExtension(sender);
}
return provider;
}

public static List<String> supportedExtensions() {
if (SUPPORTED == null) {
SUPPORTED = new ArrayList<String>();
Expand Down
177 changes: 158 additions & 19 deletions ZipExtractor-Core/src/main/java/com/dscalzi/zipextractor/core/ZExtractor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,23 +19,29 @@
package com.dscalzi.zipextractor.core;

import java.io.File;
import java.nio.file.Path;
import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Deque;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.function.BooleanSupplier;

import com.dscalzi.zipextractor.core.managers.MessageManager;
import com.dscalzi.zipextractor.core.provider.TypeProvider;
import com.dscalzi.zipextractor.core.util.ICommandSender;
import com.dscalzi.zipextractor.core.util.OpTuple;
import com.dscalzi.zipextractor.core.util.PageList;

public class ZExtractor {

private static final Map<String, WarnData> WARNED = new HashMap<String, WarnData>();
private static List<String> SUPPORTED;
private static List<String> PIPED_RISKS;

public static void asyncExtract(ICommandSender sender, File src, File dest, boolean log, final boolean override) {
public static void asyncExtract(ICommandSender sender, File src, File dest, boolean log, final boolean override, final boolean pipe, String until) {
final MessageManager mm = MessageManager.inst();

// If the user was warned, clear it.
Expand All @@ -61,38 +67,159 @@ public static void asyncExtract(ICommandSender sender, File src, File dest, bool
return;
}
}

Deque<OpTuple> pDeque = new ArrayDeque<OpTuple>();

if(pipe) {

Path srcNorm = src.toPath().toAbsolutePath().normalize();
String[] srcSplit = srcNorm.getFileName().toString().split("\\.", 2);
String[] srcExts = srcSplit.length > 1 ? srcSplit[1].split("\\.") : new String[0];

if(srcExts.length < 2) {
TypeProvider p = getApplicableProvider(src);
if(p == null) {
mm.invalidExtractionExtension(sender);
return;
}
pDeque.add(new OpTuple(src, dest, p));
} else {

File tSrc = src;
String queue = srcNorm.toString();

for(int i=srcExts.length-1; i>=0; i--) {
if((until != null && srcExts[i].equalsIgnoreCase(until)) || !supportedExtensions().contains(srcExts[i].toLowerCase())) {
break;
}
TypeProvider p = getApplicableProvider(tSrc);
if(p == null) {
mm.invalidExtractionExtension(sender);
return;
}
pDeque.add(new OpTuple(tSrc, dest, p));
queue = queue.substring(0, queue.lastIndexOf('.'));
tSrc = new File(dest + File.separator + new File(queue).getName());

}

}

} else {
TypeProvider p = getApplicableProvider(src);
if(p == null) {
mm.invalidExtractionExtension(sender);
return;
}
pDeque.add(new OpTuple(src, dest, p));
}

// Ensure a proper scan can be performed with this piped extraction.
// This is only needed when the destination directory is not empty.
// There can never be a conflict with an empty destination.
if(pipe && dest.list().length > 0) {
// The first source can be fully scanned for conflicts since it is already
// in its final state on the disk.
boolean first = true;
for(final OpTuple op : pDeque) {
if(!first && !op.getProvider().canDetectPipedConflicts()) {
mm.pipedConflictRisk(sender);
return;
}
first = false;
}
}

// Fully scan the piped chain for extraction conflicts.
// This is so that we can detect ALL conflicts in every
// stage of the operation, and report the full list to the user.
if(!override && pipe) {
List<String> atRisk = new ArrayList<String>();
for(final OpTuple op : pDeque) {
atRisk.addAll(op.getProvider().scanForExtractionConflicts(sender, op.getSrc(), op.getDest()));
}
if(atRisk.size() > 0) {
WARNED.put(sender.getName(), new WarnData(src, dest, new PageList<String>(4, atRisk)));
mm.warnOfConflicts(sender, atRisk.size());
return;
}
}

// Prepare the tasks.
// We will still check for conflicts in this stage for added security.
// If any exist, the operation will be terminated.
Runnable task = null;
for (final TypeProvider p : TypeProvider.getProviders()) {
if (p.validForExtraction(src)) {
task = () -> {
int c = 0;
boolean piped = false;
final BooleanSupplier[] pipes = new BooleanSupplier[pDeque.size()];
for(final OpTuple op : pDeque) {
final boolean interOp = c != pDeque.size()-1;

if(piped) {
pipes[c] = () -> {
List<String> atRisk = new ArrayList<String>();
if (!override) {
atRisk = op.getProvider().scanForExtractionConflicts(sender, op.getSrc(), op.getDest());
}
if (atRisk.size() == 0 || override) {
op.getProvider().extract(sender, op.getSrc(), op.getDest(), log, interOp);
op.getSrc().delete();
return true;
} else {
WARNED.put(sender.getName(), new WarnData(op.getSrc(), op.getDest(), new PageList<String>(4, atRisk)));
mm.warnOfConflicts(sender, atRisk.size());
return false;
}
};
} else {
pipes[c] = () -> {
List<String> atRisk = new ArrayList<String>();
if (!override) {
atRisk = p.scanForExtractionConflicts(sender, src, dest);
atRisk = op.getProvider().scanForExtractionConflicts(sender, op.getSrc(), op.getDest());
}
if (atRisk.size() == 0 || override) {
p.extract(sender, src, dest, log);
op.getProvider().extract(sender, op.getSrc(), op.getDest(), log, interOp);
return true;
} else {
WARNED.put(sender.getName(), new WarnData(src, dest, new PageList<String>(4, atRisk)));
WARNED.put(sender.getName(), new WarnData(op.getSrc(), op.getDest(), new PageList<String>(4, atRisk)));
mm.warnOfConflicts(sender, atRisk.size());
return false;
}
};
break;
}

piped = true;
c++;
}
if (task != null) {
int result = ZServicer.getInstance().submit(task);
if (result == 0)
mm.addToQueue(sender, ZServicer.getInstance().getSize());
else if (result == 1)
mm.queueFull(sender, ZServicer.getInstance().getMaxQueueSize());
else if (result == 2)
mm.executorTerminated(sender, ZTask.EXTRACT);
} else {
mm.invalidExtractionExtension(sender);
}

task = () -> {
for(BooleanSupplier r : pipes) {
if(!r.getAsBoolean()) {
// Conflicts
break;
}
}
};

int result = ZServicer.getInstance().submit(task);
if (result == 0)
mm.addToQueue(sender, ZServicer.getInstance().getSize());
else if (result == 1)
mm.queueFull(sender, ZServicer.getInstance().getMaxQueueSize());
else if (result == 2)
mm.executorTerminated(sender, ZTask.EXTRACT);
}

private static TypeProvider getApplicableProvider(File src) {
TypeProvider provider = null;
for (final TypeProvider p : TypeProvider.getProviders()) {
if (p.validForExtraction(src)) {
provider = p;
}
}
return provider;
}

public static List<String> supportedExtensions() {
if (SUPPORTED == null) {
SUPPORTED = new ArrayList<String>();
Expand All @@ -102,6 +229,18 @@ public static List<String> supportedExtensions() {
}
return SUPPORTED;
}

public static List<String> pipedConflictRiskExtensions(){
if(PIPED_RISKS == null) {
PIPED_RISKS = new ArrayList<String>();
for(final TypeProvider p : TypeProvider.getProviders()) {
if(!p.canDetectPipedConflicts()) {
PIPED_RISKS.addAll(p.supportedExtractionTypes());
}
}
}
return PIPED_RISKS;
}

public static boolean wasWarned(String name, File src, File dest) {
if (WARNED.containsKey(name)) {
Expand Down
Loading

1 comment on commit b0421f0

@dscalzi
Copy link
Owner Author

@dscalzi dscalzi commented on b0421f0 May 8, 2019

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

There is a typo in the commit message. It should say

/ze extract --until jar will extract Test.jar into output.

Please sign in to comment.