Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[stable10] Add group option to files:scan command #30615

Merged
merged 1 commit into from
Feb 26, 2018
Merged

[stable10] Add group option to files:scan command #30615

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
93 changes: 75 additions & 18 deletions apps/files/lib/Command/Scan.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
* @author Robin Appelman <[email protected]>
* @author Thomas Müller <[email protected]>
* @author Vincent Petry <[email protected]>
* @author Sujith Haridasan <[email protected]>
*
* @copyright Copyright (c) 2018, ownCloud GmbH
* @license AGPL-3.0
Expand Down Expand Up @@ -37,6 +38,7 @@
use OCP\Files\StorageNotAvailableException;
use OCP\IConfig;
use OCP\IDBConnection;
use OCP\IGroupManager;
use OCP\IUserManager;
use OCP\Lock\ILockingProvider;
use OCP\Lock\LockedException;
Expand All @@ -50,6 +52,8 @@ class Scan extends Base {

/** @var IUserManager $userManager */
private $userManager;
/** @var IGroupManager $groupManager */
private $groupManager;
/** @var ILockingProvider */
private $lockingProvider;
/** @var IMimeTypeLoader */
Expand All @@ -65,11 +69,13 @@ class Scan extends Base {

public function __construct(
IUserManager $userManager,
IGroupManager $groupManager,
ILockingProvider $lockingProvider,
IMimeTypeLoader $mimeTypeLoader,
IConfig $config
) {
$this->userManager = $userManager;
$this->groupManager = $groupManager;
$this->lockingProvider = $lockingProvider;
$this->mimeTypeLoader = $mimeTypeLoader;
$this->config = $config;
Expand All @@ -93,6 +99,12 @@ protected function configure() {
InputArgument::OPTIONAL,
'Limit rescan to this path, e.g., --path="/alice/files/Music", the user_id is determined by the path and the user_id parameter and --all are ignored.'
)
->addOption(
'groups',
'g',
InputArgument::OPTIONAL,
'Scan user(s) under the group(s). This option can be used as --groups=foo,bar to scan groups foo and bar'
)
->addOption(
'quiet',
'q',
Expand Down Expand Up @@ -235,12 +247,44 @@ protected function scanFiles($user, $path, $verbose, OutputInterface $output, $b
}
}

protected function getAllUsersFromGroup($group) {
$count = 0;
$users = [];
foreach ($this->groupManager->findUsersInGroup($group) as $user) {
array_push($users, $user->getUID());
$count++;
//Take 200 users at a time
if ($count > 199) {
yield $users;
$count = 1;
$users = [];
}
}
if (count($users) > 0) {
yield $users;
}
}

protected function execute(InputInterface $input, OutputInterface $output) {
$inputPath = $input->getOption('path');
$groups = $input->getOption('groups') ? explode(',', $input->getOption('groups')) : [];
$shouldRepairStoragesIndividually = (bool) $input->getOption('repair');

if ($inputPath) {
if (count($groups) >= 1) {
$users = [];
foreach ($groups as $group) {
if ($this->groupManager->groupExists($group) === false) {
$output->writeln("Group name $group doesn't exist");
return 1;
} else {
$users[$group] = [];
foreach ($this->getAllUsersFromGroup($group) as $users_array) {
$users[$group] = $users_array;
$this->processUserChunks($input, $output, $users, $inputPath, $shouldRepairStoragesIndividually, $group);
}
}
}
} else if ($inputPath) {
$inputPath = '/' . trim($inputPath, '/');
list (, $user,) = explode('/', $inputPath, 3);
$users = [$user];
Expand All @@ -264,6 +308,12 @@ protected function execute(InputInterface $input, OutputInterface $output) {
$users = $input->getArgument('user_id');
}

if (count($groups) === 0) {
$this->processUserChunks($input, $output, $users, $inputPath, $shouldRepairStoragesIndividually);
}
}

protected function processUserChunks($input, $output, $users, $inputPath, $shouldRepairStoragesIndividually, $group = null) {
# no messaging level option means: no full printout but statistics
# $quiet means no print at all
# $verbose means full printout including statistics
Expand All @@ -287,13 +337,35 @@ protected function execute(InputInterface $input, OutputInterface $output) {
$output->writeln("<error>Please specify the user id to scan, \"--all\" to scan for all users or \"--path=...\"</error>");
return;
} else {
if ($users_total > 1) {
$this->initTools();
if ($group !== null) {
$output->writeln("Scanning group $group");
$this->userScan($users[$group], $inputPath, $shouldRepairStoragesIndividually, $input, $output, $verbose);

} elseif ($users_total >= 1) {
$output->writeln("\nScanning files for $users_total users");
$this->userScan($users, $inputPath, $shouldRepairStoragesIndividually, $input, $output, $verbose);
}
}

$this->initTools();
# stat: printout statistics if $quiet was not set
if (!$quiet) {
$this->presentStats($output);
}
}

/**
* Initialises some useful tools for the Command
*/
protected function initTools() {
// Start the timer
$this->execTime = -microtime(true);
// Convert PHP errors to exceptions
set_error_handler([$this, 'exceptionErrorHandler'], E_ALL);
}

protected function userScan($users, $inputPath, $shouldRepairStoragesIndividually, $input, $output, $verbose) {
$users_total = count($users);
$user_count = 0;
foreach ($users as $user) {
if (is_object($user)) {
Expand All @@ -316,21 +388,6 @@ protected function execute(InputInterface $input, OutputInterface $output) {
break;
}
}

# stat: printout statistics if $quiet was not set
if (!$quiet) {
$this->presentStats($output);
}
}

/**
* Initialises some useful tools for the Command
*/
protected function initTools() {
// Start the timer
$this->execTime = -microtime(true);
// Convert PHP errors to exceptions
set_error_handler([$this, 'exceptionErrorHandler'], E_ALL);
}

/**
Expand Down
162 changes: 162 additions & 0 deletions apps/files/tests/Command/ScanTest.php
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,162 @@
<?php
/**
* @author Sujith Haridasan <[email protected]>
*
* @copyright Copyright (c) 2018, ownCloud GmbH
* @license AGPL-3.0
*
* This code is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License, version 3,
* as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License, version 3,
* along with this program. If not, see <http://www.gnu.org/licenses/>
*
*/

namespace OCA\Files\Tests\Command;

use OCA\Files\Command\Scan;
use Symfony\Component\Console\Tester\CommandTester;
use Test\TestCase;
use Test\Traits\UserTrait;

/**
* Class ScanTest
*
* @group DB
* @package OCA\Files\Tests\Command
*/
class ScanTest extends TestCase {
use UserTrait;

/** @var CommandTester */
private $commandTester;

private $groupsCreated = [];
protected function setUp() {
parent::setUp();
$command = new Scan(
\OC::$server->getUserManager(), \OC::$server->getGroupManager(),
\OC::$server->getLockingProvider(), \OC::$server->getMimeTypeLoader(),
\OC::$server->getConfig());

$this->commandTester = new CommandTester($command);
$user1 = $this->createUser('user1');
$this->createUser('user2');
\OC::$server->getGroupManager()->createGroup('group1');
\OC::$server->getGroupManager()->get('group1')->addUser($user1);
$this->groupsCreated[] = 'group1';
}

protected function tearDown() {
$this->tearDownUserTrait();
foreach ($this->groupsCreated as $group) {
\OC::$server->getGroupManager()->get($group)->delete();
}
parent::tearDown();
}

public function dataInput() {
return [
[['--groups' => 'haystack'], 'Group name haystack doesn\'t exist'],
[['--groups' => 'group1'], 'Starting scan for user 1 out of 1 (user1)'],
[['user_id' => ['user1']], 'Starting scan for user 1 out of 1 (user1)'],
[['user_id' => ['user2']], 'Starting scan for user 1 out of 1 (user2)']
];
}

/**
* @dataProvider dataInput
*/
public function testCommandInput($input, $expectedOutput) {
$this->commandTester->execute($input);
$output = $this->commandTester->getDisplay();
$this->assertContains($expectedOutput, $output);
}

public function userInputData() {
return [
[['--groups' => 'group1'], 'Starting scan for user 1 out of 200']
];
}

/**
* @dataProvider userInputData
* @param $input
* @param $expectedOutput
*/
public function testGroupPaginationForUsers($input, $expectedOutput) {
//First we populate the users
$user = 'user';
$numberOfUsersInGroup = 210;
for($i = 2; $i <= 210; $i++) {
$userObj = $this->createUser($user.$i);
\OC::$server->getGroupManager()->get('group1')->addUser($userObj);
}

$this->commandTester->execute($input);
$output = $this->commandTester->getDisplay();
$this->assertContains($expectedOutput, $output);
//If pagination works then below assert shouldn't fail
$this->assertNotContains('Starting scan for user 1 out of 210', $output);
}

public function multipleGroupTest() {
return [
[['--groups' => 'group1,group2'], ''],
[['--groups' => 'group1,group2,group3'], '']
];
}

/**
* @dataProvider multipleGroupTest
* @param $input
*/
public function testMultipleGroups($input) {
//Create 10 users in each group
$groups = explode(',', $input['--groups']);
$user = "user";
$userObj = [];
for ($i = 1; $i <= (10 * count($groups)); $i++ ) {
$userObj[] = $this->createUser($user.$i);
}

$userCount = 1;
foreach ($groups as $group) {
if (\OC::$server->getGroupManager()->groupExists($group) === false) {
\OC::$server->getGroupManager()->createGroup($group);
$this->groupsCreated[] = $group;
for ($i = $userCount; $i <= ($userCount + 9); $i++) {
$j = $i - 1;
\OC::$server->getGroupManager()->get($group)->addUser($userObj[$j]);
}
$userCount = $i;
} else {
for ($i = $userCount; $i <= ($userCount + 9); $i++) {
$j = $i - 1;
\OC::$server->getGroupManager()->get($group)->addUser($userObj[$j]);
}
$userCount = $i;
}
}

$this->commandTester->execute($input);
$output = $this->commandTester->getDisplay();
if (count($groups) === 2) {
$this->assertContains('Starting scan for user 1 out of 10 (user1)', $output);
$this->assertContains('Starting scan for user 1 out of 10 (user11)', $output);
}
if (count($groups) === 3) {
$this->assertContains('Starting scan for user 1 out of 10 (user1)', $output);
$this->assertContains('Starting scan for user 1 out of 10 (user11)', $output);
$this->assertContains('Starting scan for user 1 out of 10 (user21)', $output);
$this->assertContains('Starting scan for user 10 out of 10 (user30)', $output);
}
}
}