Skip to content

Commit

Permalink
feat: add command to scan external storages directly
Browse files Browse the repository at this point in the history
the main use case of this over simply scanning through is the ability to provide a username and/or password for cases where login credentials are used

Signed-off-by: Robin Appelman <[email protected]>
  • Loading branch information
icewind1991 committed Mar 6, 2024
1 parent 28c3e40 commit d7d8822
Show file tree
Hide file tree
Showing 6 changed files with 296 additions and 74 deletions.
1 change: 1 addition & 0 deletions apps/files_external/appinfo/info.xml
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ External storage can be configured using the GUI or at the command line. This se
<command>OCA\Files_External\Command\Backends</command>
<command>OCA\Files_External\Command\Verify</command>
<command>OCA\Files_External\Command\Notify</command>
<command>OCA\Files_External\Command\Scan</command>
</commands>

<settings>
Expand Down
2 changes: 2 additions & 0 deletions apps/files_external/composer/composer/autoload_classmap.php
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@
'OCA\\Files_External\\Command\\ListCommand' => $baseDir . '/../lib/Command/ListCommand.php',
'OCA\\Files_External\\Command\\Notify' => $baseDir . '/../lib/Command/Notify.php',
'OCA\\Files_External\\Command\\Option' => $baseDir . '/../lib/Command/Option.php',
'OCA\\Files_External\\Command\\Scan' => $baseDir . '/../lib/Command/Scan.php',
'OCA\\Files_External\\Command\\StorageAuthBase' => $baseDir . '/../lib/Command/StorageAuthBase.php',
'OCA\\Files_External\\Command\\Verify' => $baseDir . '/../lib/Command/Verify.php',
'OCA\\Files_External\\Config\\ConfigAdapter' => $baseDir . '/../lib/Config/ConfigAdapter.php',
'OCA\\Files_External\\Config\\ExternalMountPoint' => $baseDir . '/../lib/Config/ExternalMountPoint.php',
Expand Down
2 changes: 2 additions & 0 deletions apps/files_external/composer/composer/autoload_static.php
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,8 @@ class ComposerStaticInitFiles_External
'OCA\\Files_External\\Command\\ListCommand' => __DIR__ . '/..' . '/../lib/Command/ListCommand.php',
'OCA\\Files_External\\Command\\Notify' => __DIR__ . '/..' . '/../lib/Command/Notify.php',
'OCA\\Files_External\\Command\\Option' => __DIR__ . '/..' . '/../lib/Command/Option.php',
'OCA\\Files_External\\Command\\Scan' => __DIR__ . '/..' . '/../lib/Command/Scan.php',
'OCA\\Files_External\\Command\\StorageAuthBase' => __DIR__ . '/..' . '/../lib/Command/StorageAuthBase.php',
'OCA\\Files_External\\Command\\Verify' => __DIR__ . '/..' . '/../lib/Command/Verify.php',
'OCA\\Files_External\\Config\\ConfigAdapter' => __DIR__ . '/..' . '/../lib/Config/ConfigAdapter.php',
'OCA\\Files_External\\Config\\ExternalMountPoint' => __DIR__ . '/..' . '/../lib/Config/ExternalMountPoint.php',
Expand Down
80 changes: 6 additions & 74 deletions apps/files_external/lib/Command/Notify.php
Original file line number Diff line number Diff line change
Expand Up @@ -30,17 +30,13 @@
namespace OCA\Files_External\Command;

use Doctrine\DBAL\Exception\DriverException;
use OC\Core\Command\Base;
use OCA\Files_External\Lib\InsufficientDataForMeaningfulAnswerException;
use OCA\Files_External\Lib\StorageConfig;
use OCA\Files_External\Service\GlobalStoragesService;
use OCP\DB\QueryBuilder\IQueryBuilder;
use OCP\Files\Notify\IChange;
use OCP\Files\Notify\INotifyHandler;
use OCP\Files\Notify\IRenameChange;
use OCP\Files\Storage\INotifyStorage;
use OCP\Files\Storage\IStorage;
use OCP\Files\StorageNotAvailableException;
use OCP\IDBConnection;
use OCP\IUserManager;
use Psr\Log\LoggerInterface;
Expand All @@ -49,14 +45,14 @@
use Symfony\Component\Console\Input\InputOption;
use Symfony\Component\Console\Output\OutputInterface;

class Notify extends Base {
class Notify extends StorageAuthBase {
public function __construct(
private GlobalStoragesService $globalService,
private IDBConnection $connection,
private LoggerInterface $logger,
private IUserManager $userManager
GlobalStoragesService $globalService,
IUserManager $userManager,
) {
parent::__construct();
parent::__construct($globalService, $userManager);
}

protected function configure(): void {
Expand Down Expand Up @@ -97,71 +93,12 @@ protected function configure(): void {
parent::configure();
}

private function getUserOption(InputInterface $input): ?string {
if ($input->getOption('user')) {
return (string)$input->getOption('user');
}

return $_ENV['NOTIFY_USER'] ?? $_SERVER['NOTIFY_USER'] ?? null;
}

private function getPasswordOption(InputInterface $input): ?string {
if ($input->getOption('password')) {
return (string)$input->getOption('password');
}

return $_ENV['NOTIFY_PASSWORD'] ?? $_SERVER['NOTIFY_PASSWORD'] ?? null;
}

protected function execute(InputInterface $input, OutputInterface $output): int {
$mount = $this->globalService->getStorage($input->getArgument('mount_id'));
if (is_null($mount)) {
$output->writeln('<error>Mount not found</error>');
[$mount, $storage] = $this->createStorage($input, $output);
if ($storage === null) {
return self::FAILURE;
}
$noAuth = false;

$userOption = $this->getUserOption($input);
$passwordOption = $this->getPasswordOption($input);

// if only the user is provided, we get the user object to pass along to the auth backend
// this allows using saved user credentials
$user = ($userOption && !$passwordOption) ? $this->userManager->get($userOption) : null;

try {
$authBackend = $mount->getAuthMechanism();
$authBackend->manipulateStorageConfig($mount, $user);
} catch (InsufficientDataForMeaningfulAnswerException $e) {
$noAuth = true;
} catch (StorageNotAvailableException $e) {
$noAuth = true;
}

if ($userOption) {
$mount->setBackendOption('user', $userOption);
}
if ($passwordOption) {
$mount->setBackendOption('password', $passwordOption);
}

try {
$backend = $mount->getBackend();
$backend->manipulateStorageConfig($mount, $user);
} catch (InsufficientDataForMeaningfulAnswerException $e) {
$noAuth = true;
} catch (StorageNotAvailableException $e) {
$noAuth = true;
}

try {
$storage = $this->createStorage($mount);
} catch (\Exception $e) {
$output->writeln('<error>Error while trying to create storage</error>');
if ($noAuth) {
$output->writeln('<error>Login and/or password required</error>');
}
return self::FAILURE;
}
if (!$storage instanceof INotifyStorage) {
$output->writeln('<error>Mount of type "' . $mount->getBackend()->getText() . '" does not support active update notifications</error>');
return self::FAILURE;
Expand Down Expand Up @@ -189,11 +126,6 @@ protected function execute(InputInterface $input, OutputInterface $output): int
return self::SUCCESS;
}

private function createStorage(StorageConfig $mount): IStorage {
$class = $mount->getBackend()->getStorageClass();
return new $class($mount->getBackendOptions());
}

private function markParentAsOutdated($mountId, $path, OutputInterface $output, bool $dryRun): void {
$parent = ltrim(dirname($path), '/');
if ($parent === '.') {
Expand Down
155 changes: 155 additions & 0 deletions apps/files_external/lib/Command/Scan.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,155 @@
<?php

declare(strict_types=1);
/**
* @copyright Copyright (c) 2021 Robin Appelman <[email protected]>
*
* @license GNU AGPL version 3 or any later version
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
*/

namespace OCA\Files_External\Command;

use OC\Files\Cache\Scanner;
use OCA\Files_External\Service\GlobalStoragesService;
use OCP\IUserManager;
use Symfony\Component\Console\Helper\Table;
use Symfony\Component\Console\Input\InputArgument;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Input\InputOption;
use Symfony\Component\Console\Output\OutputInterface;

class Scan extends StorageAuthBase {
protected float $execTime = 0;
protected int $foldersCounter = 0;
protected int $filesCounter = 0;

public function __construct(
GlobalStoragesService $globalService,
IUserManager $userManager
) {
parent::__construct($globalService, $userManager);
}

protected function configure(): void {
$this
->setName('files_external:scan')
->setDescription('Scan an external storage for changed files')
->addArgument(
'mount_id',
InputArgument::REQUIRED,
'the mount id of the mount to scan'
)->addOption(
'user',
'u',
InputOption::VALUE_REQUIRED,
'The username for the remote mount (required only for some mount configuration that don\'t store credentials)'
)->addOption(
'password',
'p',
InputOption::VALUE_REQUIRED,
'The password for the remote mount (required only for some mount configuration that don\'t store credentials)'
)->addOption(
'path',
'',
InputOption::VALUE_OPTIONAL,
'The path in the storage to scan',
''
);
parent::configure();
}

protected function execute(InputInterface $input, OutputInterface $output): int {
[, $storage] = $this->createStorage($input, $output);
if ($storage === null) {
return 1;
}

$path = $input->getOption('path');

$this->execTime = -microtime(true);

/** @var Scanner $scanner */
$scanner = $storage->getScanner();

$scanner->listen('\OC\Files\Cache\Scanner', 'scanFile', function (string $path) use ($output) {
$output->writeln("\tFile\t<info>$path</info>", OutputInterface::VERBOSITY_VERBOSE);
++$this->filesCounter;
$this->abortIfInterrupted();
});

$scanner->listen('\OC\Files\Cache\Scanner', 'scanFolder', function (string $path) use ($output) {
$output->writeln("\tFolder\t<info>$path</info>", OutputInterface::VERBOSITY_VERBOSE);
++$this->foldersCounter;
$this->abortIfInterrupted();
});

$scanner->scan($path);

$this->presentStats($output);

return 0;
}

/**
* @param OutputInterface $output
*/
protected function presentStats(OutputInterface $output): void {
// Stop the timer
$this->execTime += microtime(true);

$headers = [
'Folders', 'Files', 'Elapsed time'
];

$this->showSummary($headers, [], $output);
}

/**
* Shows a summary of operations
*
* @param string[] $headers
* @param string[] $rows
* @param OutputInterface $output
*/
protected function showSummary(array $headers, array $rows, OutputInterface $output): void {
$niceDate = $this->formatExecTime();
if (!$rows) {
$rows = [
$this->foldersCounter,
$this->filesCounter,
$niceDate,
];
}
$table = new Table($output);
$table
->setHeaders($headers)
->setRows([$rows]);
$table->render();
}


/**
* Formats microtime into a human readable format
*
* @return string
*/
protected function formatExecTime(): string {
$secs = round($this->execTime);
# convert seconds into HH:MM:SS form
return sprintf('%02d:%02d:%02d', ($secs / 3600), ($secs / 60 % 60), $secs % 60);
}
}
Loading

0 comments on commit d7d8822

Please sign in to comment.