Skip to content
This repository has been archived by the owner on Nov 25, 2020. It is now read-only.

Commit

Permalink
Commit to add tar binary support for compression and extraction of ta…
Browse files Browse the repository at this point in the history 
…r archive files. This allows the compression plugin to use the tar binary (if found) for these operations, which improves performance. It also supports and detects the parallel compression utilities: pigz, lbzip2, and pbzip2. If any of these are installed, then they will be used to compress or extract archive files while utilizing all available CPU threads on the system, increasing performance even further. This will work natively in nearly all flavours of Linux, and Windows is also supported as long as a usable tar binary is present in the command path.
  • Loading branch information
@snw35
snw35 committed Jul 20, 2016
1 parent c86f299 commit 6df570b
Showing 1 changed file with 146 additions and 50 deletions.
196 changes: 146 additions & 50 deletions core/src/plugins/action.compression/class.PluginCompression.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,32 @@ public function receiveAction($action, $httpVars, $fileVars)
$currentDirPath = AJXP_Utils::safeDirname($userSelection->getUniqueNode()->getPath());
$currentDirPath = rtrim($currentDirPath, "/") . "/";
$currentDirUrl = $userSelection->currentBaseUrl().$currentDirPath;

# Multi-platform function taken from stackoverflow
# http://stackoverflow.com/questions/12424787/how-to-check-if-a-shell-command-exists-from-php
# Check that a command exists on the system for use with shell_exec
function command_exists ($command) {
$whereIsCommand = (PHP_OS == 'WINNT') ? 'where' : 'which';
$process = proc_open(
"$whereIsCommand $command",
array(
0 => array("pipe", "r"), //STDIN
1 => array("pipe", "w"), //STDOUT
2 => array("pipe", "w"), //STDERR
),
$pipes
);
if ($process !== false) {
$stdout = stream_get_contents($pipes[1]);
$stderr = stream_get_contents($pipes[2]);
fclose($pipes[1]);
fclose($pipes[2]);
proc_close($process);
return $stdout != '';
}
return false;
}

if (empty($httpVars["compression_id"])) {
$compressionId = sha1(rand());
$httpVars["compression_id"] = $compressionId;
Expand Down Expand Up @@ -119,44 +145,85 @@ public function receiveAction($action, $httpVars, $fileVars)
file_put_contents($progressCompressionFileName, "Error : " . $messages["compression.17"]);
throw new AJXP_Exception($messages["compression.17"]);
}
try {
$tmpArchiveName = tempnam(AJXP_Utils::getAjxpTmpDir(), "tar-compression") . ".tar";
$archive = new PharData($tmpArchiveName);
} catch (Exception $e) {
file_put_contents($progressCompressionFileName, "Error : " . $e->getMessage());
throw $e;
}
$counterCompression = 0;
//THE TWO ARRAY ARE MERGED FOR THE FOREACH LOOP
$tabAllFiles = array_combine($tabAllRecursiveFiles, $tabFilesNames);
foreach ($tabAllFiles as $fullPath => $fileName) {
// Builds a shell command that changes to the workspace location in the filesystem, and then
// compresses with tar and any parallel compression utilities that are installed.
if (command_exists("tar")) {
$FileNames = array();
$FolderNames = array();
foreach ($nodes as $node) {
$nodeUrl = $node->getUrl();
if (is_file($nodeUrl) && filesize($nodeUrl) < $maxAuthorizedSize) {
array_push($FileNames, escapeshellarg(substr($nodeUrl, $currentDirUrlLength)));
}
if (is_dir($nodeUrl)) {
array_push($FolderNames, escapeshellarg(substr($nodeUrl, $currentDirUrlLength)));
}
}
if ($archiveFormat == ".tar.gz") {
if (command_exists("pigz")) {
$tarcommand = "tar -I pigz -cf ";
} else {
$tarcommand = "tar -czf ";
}
} elseif ($archiveFormat == ".tar.bz2") {
if (command_exists("lbzip2")) {
$tarcommand = "tar -I lbzip2 -cf ";
} elseif (command_exists("pbzip2")) {
$tarcommand = "tar -I pbzip2 -cf ";
} else {
$tarcommand = "tar -cjf ";
}
} elseif ($archiveFormat == ".tar") {
$tarcommand = "tar -cf ";
} else {
file_put_contents($progressExtractFileName, "Error : " . $messages["compression.15"]);
throw new AJXP_Exception($messages["compression.15"]);
}
$changedircommand = "cd " . AJXP_MetaStreamWrapper::getRealFSReference($currentDirUrl) . " && ";
$finalcommand = $changedircommand . $tarcommand . escapeshellarg($archiveName) . " " . implode(" ", $FolderNames) . " " . implode(" ", $FileNames);
shell_exec($finalcommand);
} else {
// If tar command is not found, compress with PHP Phar instead
try {
$archive->addFile(AJXP_MetaStreamWrapper::getRealFSReference($fullPath), $fileName);
$counterCompression++;
file_put_contents($progressCompressionFileName, sprintf($messages["compression.6"], round(($counterCompression / count($tabAllFiles)) * 100, 0, PHP_ROUND_HALF_DOWN) . " %"));
$tmpArchiveName = tempnam(AJXP_Utils::getAjxpTmpDir(), "tar-compression") . ".tar";
$archive = new PharData($tmpArchiveName);
} catch (Exception $e) {
unlink($tmpArchiveName);
file_put_contents($progressCompressionFileName, "Error : " . $e->getMessage());
throw $e;
}
}
$finalArchive = $tmpArchiveName;
if ($typeArchive != ".tar") {
$archiveTypeCompress = substr(strrchr($typeArchive, "."), 1);
file_put_contents($progressCompressionFileName, sprintf($messages["compression.7"], strtoupper($archiveTypeCompress)));
if ($archiveTypeCompress == "gz") {
$archive->compress(Phar::GZ);
} elseif ($archiveTypeCompress == "bz2") {
$archive->compress(Phar::BZ2);
$counterCompression = 0;
//THE TWO ARRAY ARE MERGED FOR THE FOREACH LOOP
$tabAllFiles = array_combine($tabAllRecursiveFiles, $tabFilesNames);
foreach ($tabAllFiles as $fullPath => $fileName) {
try {
$archive->addFile(AJXP_MetaStreamWrapper::getRealFSReference($fullPath), $fileName);
$counterCompression++;
file_put_contents($progressCompressionFileName, sprintf($messages["compression.6"], round(($counterCompression / count($tabAllFiles)) * 100, 0, PHP_ROUND_HALF_DOWN) . " %"));
} catch (Exception $e) {
unlink($tmpArchiveName);
file_put_contents($progressCompressionFileName, "Error : " . $e->getMessage());
throw $e;
}
}
$finalArchive = $tmpArchiveName . "." . $archiveTypeCompress;
}
$destArchive = AJXP_MetaStreamWrapper::getRealFSReference($currentDirUrl . $archiveName);
rename($finalArchive, $destArchive);
AJXP_Controller::applyHook("node.before_create", array($destArchive, filesize($destArchive)));
if (file_exists($tmpArchiveName)) {
unlink($tmpArchiveName);
unlink(substr($tmpArchiveName, 0, -4));
$finalArchive = $tmpArchiveName;
if ($typeArchive != ".tar") {
$archiveTypeCompress = substr(strrchr($typeArchive, "."), 1);
file_put_contents($progressCompressionFileName, sprintf($messages["compression.7"], strtoupper($archiveTypeCompress)));
if ($archiveTypeCompress == "gz") {
$archive->compress(Phar::GZ);
} elseif ($archiveTypeCompress == "bz2") {
$archive->compress(Phar::BZ2);
}
$finalArchive = $tmpArchiveName . "." . $archiveTypeCompress;
}
$destArchive = AJXP_MetaStreamWrapper::getRealFSReference($currentDirUrl . $archiveName);
rename($finalArchive, $destArchive);
AJXP_Controller::applyHook("node.before_create", array($destArchive, filesize($destArchive)));
if (file_exists($tmpArchiveName)) {
unlink($tmpArchiveName);
unlink(substr($tmpArchiveName, 0, -4));
}
// End of Phar compression section
}
$newNode = new AJXP_Node($currentDirUrl . $archiveName);
AJXP_Controller::applyHook("node.change", array(null, $newNode, false));
Expand Down Expand Up @@ -244,27 +311,55 @@ public function receiveAction($action, $httpVars, $fileVars)
}
mkdir($currentDirUrl . $onlyFileName, 0777, true);
chmod(AJXP_MetaStreamWrapper::getRealFSReference($currentDirUrl . $onlyFileName), 0777);
try {
$archive = new PharData(AJXP_MetaStreamWrapper::getRealFSReference($currentAllPydioPath));
$fichiersArchive = new RecursiveIteratorIterator(new RecursiveDirectoryIterator($pharCurrentAllPydioPath));
foreach ($fichiersArchive as $file) {
$fileGetPathName = $file->getPathname();
if($file->isDir()) {
continue;
// Builds a command that will use any parallel extraction utilities installed to extract an archive file though tar.
if (command_exists("tar")) {
if ($pathInfoCurrentAllPydioPath == "gz") {
if (command_exists("pigz")) {
$tarcommand = "tar -I pigz -xf ";
} else {
$tarcommand = "tar -xzf ";
}
$fileNameInArchive = substr(strstr($fileGetPathName, $fileArchive), strlen($fileArchive) + 1);
try {
$archive->extractTo(AJXP_MetaStreamWrapper::getRealFSReference($currentDirUrl . $onlyFileName), $fileNameInArchive, false);
} catch (Exception $e) {
file_put_contents($progressExtractFileName, "Error : " . $e->getMessage());
throw new AJXP_Exception($e);
} elseif ($pathInfoCurrentAllPydioPath == "bz2") {
if (command_exists("lbzip2")) {
$tarcommand = "tar -I lbzip2 -xf ";
} elseif (command_exists("pbzip2")) {
$tarcommand = "tar -I pbzip2 -xf ";
} else {
$tarcommand = "tar -xjf ";
}
} elseif ($pathInfoCurrentAllPydioPath == "tar") {
$tarcommand = "tar -xf ";
} else {
file_put_contents($progressExtractFileName, "Error : " . $messages["compression.15"]);
throw new AJXP_Exception($messages["compression.15"]);
}
$extractCommand = $tarcommand . escapeshellarg(AJXP_MetaStreamWrapper::getRealFSReference($currentDirUrl . $fileArchive)) . " -C " . escapeshellarg(AJXP_MetaStreamWrapper::getRealFSReference($currentDirUrl . $onlyFileName));
shell_exec($extractCommand);
} else {
// If tar command is not found, extract using PHP Phar instead
try {
$archive = new PharData(AJXP_MetaStreamWrapper::getRealFSReference($currentAllPydioPath));
$fichiersArchive = new RecursiveIteratorIterator(new RecursiveDirectoryIterator($pharCurrentAllPydioPath));
foreach ($fichiersArchive as $file) {
$fileGetPathName = $file->getPathname();
if($file->isDir()) {
continue;
}
$fileNameInArchive = substr(strstr($fileGetPathName, $fileArchive), strlen($fileArchive) + 1);
try {
$archive->extractTo(AJXP_MetaStreamWrapper::getRealFSReference($currentDirUrl . $onlyFileName), $fileNameInArchive, false);
} catch (Exception $e) {
file_put_contents($progressExtractFileName, "Error : " . $e->getMessage());
throw new AJXP_Exception($e);
}
$counterExtract++;
file_put_contents($progressExtractFileName, sprintf($messages["compression.13"], round(($counterExtract / $archive->count()) * 100, 0, PHP_ROUND_HALF_DOWN) . " %"));
}
$counterExtract++;
file_put_contents($progressExtractFileName, sprintf($messages["compression.13"], round(($counterExtract / $archive->count()) * 100, 0, PHP_ROUND_HALF_DOWN) . " %"));
} catch (Exception $e) {
file_put_contents($progressExtractFileName, "Error : " . $e->getMessage());
throw new AJXP_Exception($e);
}
} catch (Exception $e) {
file_put_contents($progressExtractFileName, "Error : " . $e->getMessage());
throw new AJXP_Exception($e);
// End of Phar extraction section
}
file_put_contents($progressExtractFileName, "SUCCESS");
$newNode = new AJXP_Node($currentDirUrl . $onlyFileName);
Expand Down Expand Up @@ -309,3 +404,4 @@ public function receiveAction($action, $httpVars, $fileVars)
}
}
}

0 comments on commit 6df570b

Please sign in to comment.