From 5d241f43cc18d062ad4fc1702eb7b60ba8879e54 Mon Sep 17 00:00:00 2001 From: Lennart Dohmann Date: Wed, 2 Oct 2024 09:36:55 +0200 Subject: [PATCH 1/7] Scope dependencies (#138) Scope dependencies --------- Co-authored-by: Matthias Simonis --- .devcontainer/postCreateCommands.sh | 10 ++- .github/workflows/release-app.yml | 5 ++ Makefile | 112 ++++++++++++----------- empty-skeleton.config.php | 1 + install.sh | 1 + lib/AppInfo/Application.php | 7 +- lib/AvirWrapper.php | 14 +-- memory.ini | 1 + scoper.inc.php | 135 ++++++++++++++++++++++++++++ 9 files changed, 219 insertions(+), 67 deletions(-) create mode 100644 memory.ini create mode 100644 scoper.inc.php diff --git a/.devcontainer/postCreateCommands.sh b/.devcontainer/postCreateCommands.sh index b7f0536c..1f574c89 100755 --- a/.devcontainer/postCreateCommands.sh +++ b/.devcontainer/postCreateCommands.sh @@ -6,16 +6,22 @@ sudo bash -c "docker completion bash > /usr/share/bash-completion/completions/do sudo bash -c "composer completion bash > /usr/share/bash-completion/completions/composer" sudo bash -c "npm completion > /usr/share/bash-completion/completions/npm" sudo cp xdebug.local.ini /usr/local/etc/php/conf.d/xdebug.ini +sudo cp memory.ini /usr/local/etc/php/conf.d/memory.ini sudo curl -sS https://webi.sh/gh | sh echo ". /usr/share/bash-completion/bash_completion" >> /home/vscode/.bashrc -NEXTCLOUD_VERSION=$(grep -oP "[0-9]+\.[0-9]+\.[0-9]+" install.sh) +NEXTCLOUD_VERSION=$(grep -oP -m 1 "[0-9]+\.[0-9]+\.[0-9]+" install.sh) + mkdir -p ~/.ssh/ ssh-keyscan -t rsa github.com >> ~/.ssh/known_hosts + +rm -rf nextcloud-server/ git clone --depth 1 --recurse-submodules --single-branch --branch v$NEXTCLOUD_VERSION git@github.com:nextcloud/server.git ./nextcloud-server cd nextcloud-server git submodule update --init cd - - +composer global require humbug/php-scoper +/home/vscode/.composer/vendor/bin/php-scoper completion bash >> /home/vscode/.bash_completion +echo 'export PATH=/home/vscode/.composer/vendor/bin/:$PATH' >>~/.bashrc ./install.sh \ No newline at end of file diff --git a/.github/workflows/release-app.yml b/.github/workflows/release-app.yml index 879439ca..5bff29a5 100644 --- a/.github/workflows/release-app.yml +++ b/.github/workflows/release-app.yml @@ -55,6 +55,11 @@ jobs: composer install ./vendor/bin/phpunit --bootstrap tests/unittests/bootstrap.php tests/unittests/ --testdox + - name: install php-scoper + run: | + composer global require humbug/php-scoper + echo $(composer config home --global) >> $GITHUB_PATH + - name: install nextcloud env: CLIENT_ID: ${{ secrets.VAAS_CLIENT_ID }} diff --git a/Makefile b/Makefile index 222cb8a7..d8abd510 100644 --- a/Makefile +++ b/Makefile @@ -40,7 +40,7 @@ app_directory_name=$(notdir $(CURDIR)) app_real_name=gdatavaas build_tools_directory=$(CURDIR)/build/tools -source_build_directory=$(CURDIR)/build/artifacts/source +source_build_directory=./build/artifacts/source source_package_name=$(source_build_directory)/$(app_directory_name) appstore_build_directory=$(CURDIR)/build/artifacts appstore_package_name=$(appstore_build_directory)/$(app_real_name) @@ -118,58 +118,62 @@ distclean: clean appstore: build rm -rf $(appstore_build_directory) mkdir -p $(appstore_build_directory) + php-scoper add-prefix --output-dir=$(source_build_directory) --force + mv $(source_build_directory)/vendor/netresearch/jsonmapper/src/JsonMapper/Exception.php $(source_build_directory)/vendor/netresearch/jsonmapper/src/JsonMapper/JsonMapper_Exception.php + mv $(source_build_directory)/vendor/netresearch/jsonmapper/src/JsonMapper.php $(source_build_directory)/vendor/netresearch/jsonmapper/src/JsonMapper/JsonMapper.php + composer dump-autoload --working-dir $(source_build_directory) --classmap-authoritative tar czf $(appstore_package_name).tar.gz \ - --transform s/$(app_directory_name)/$(app_real_name)/ \ + --transform s%$(source_build_directory)%$(app_real_name)% \ --exclude-vcs \ - --exclude="../$(app_directory_name)/opcache-disabled.ini" \ - --exclude="../$(app_directory_name)/opcache-blacklist.txt" \ - --exclude="../$(app_directory_name)/artifacts" \ - --exclude="../$(app_directory_name)/tmp*" \ - --exclude="../$(app_directory_name)/Dockerfile*" \ - --exclude="../$(app_directory_name)/nextcloud-server*" \ - --exclude="../$(app_directory_name)/compose-install.yaml" \ - --exclude="../$(app_directory_name)/empty-skeleton.config.php" \ - --exclude="../$(app_directory_name)/get-matrix.sh" \ - --exclude="../$(app_directory_name)/xdebug.*" \ - --exclude="../$(app_directory_name)/build" \ - --exclude="../$(app_directory_name)/tests" \ - --exclude="../$(app_directory_name)/Makefile" \ - --exclude="../$(app_directory_name)/*.log" \ - --exclude="../$(app_directory_name)/phpunit*xml" \ - --exclude="../$(app_directory_name)/composer.*" \ - --exclude="../$(app_directory_name)/node_modules" \ - --exclude="../$(app_directory_name)/js/node_modules" \ - --exclude="../$(app_directory_name)/js/tests" \ - --exclude="../$(app_directory_name)/js/test" \ - --exclude="../$(app_directory_name)/js/*.log" \ - --exclude="../$(app_directory_name)/js/package.json" \ - --exclude="../$(app_directory_name)/js/bower.json" \ - --exclude="../$(app_directory_name)/js/karma.*" \ - --exclude="../$(app_directory_name)/js/protractor.*" \ - --exclude="../$(app_directory_name)/package.json" \ - --exclude="../$(app_directory_name)/bower.json" \ - --exclude="../$(app_directory_name)/karma.*" \ - --exclude="../$(app_directory_name)/protractor\.*" \ - --exclude="../$(app_directory_name)/.*" \ - --exclude="../$(app_directory_name)/js/.*" \ - --exclude="../$(app_directory_name)/webpack.config.js" \ - --exclude="../$(app_directory_name)/stylelint.config.js" \ - --exclude="../$(app_directory_name)/CHANGELOG.md" \ - --exclude="../$(app_directory_name)/README.md" \ - --exclude="../$(app_directory_name)/package-lock.json" \ - --exclude="../$(app_directory_name)/LICENSES" \ - --exclude="../$(app_directory_name)/src" \ - --exclude="../$(app_directory_name)/babel.config.js" \ - --exclude="../$(app_directory_name)/devcontainer.yaml" \ - --exclude="../$(app_directory_name)/psalm.xml" \ - --exclude="../$(app_directory_name)/start-dev-environment.ps1" \ - --exclude="../$(app_directory_name)/start-dev-environment.sh" \ - --exclude="../$(app_directory_name)/dev-environment*" \ - --exclude="../$(app_directory_name)/install.sh" \ - --exclude="../$(app_directory_name)/renovate.json" \ - --exclude="../$(app_directory_name)/get-matrix.sh" \ - --exclude="../$(app_directory_name)/xdebug.ini" \ - --exclude="../$(app_directory_name)/compose-install.yaml" \ - --exclude="../$(app_directory_name)/Dockerfile.Nextcloud" \ - --exclude="../$(app_directory_name)/empty-skeleton.config.php" \ - ../$(app_directory_name) \ + --exclude="$(source_build_directory)/opcache-disabled.ini" \ + --exclude="$(source_build_directory)/opcache-blacklist.txt" \ + --exclude="$(source_build_directory)/artifacts" \ + --exclude="$(source_build_directory)/tmp*" \ + --exclude="$(source_build_directory)/Dockerfile*" \ + --exclude="$(source_build_directory)/nextcloud-server*" \ + --exclude="$(source_build_directory)/compose-install.yaml" \ + --exclude="$(source_build_directory)/empty-skeleton.config.php" \ + --exclude="$(source_build_directory)/get-matrix.sh" \ + --exclude="$(source_build_directory)/xdebug.*" \ + --exclude="$(source_build_directory)/build" \ + --exclude="$(source_build_directory)/tests" \ + --exclude="$(source_build_directory)/Makefile" \ + --exclude="$(source_build_directory)/*.log" \ + --exclude="$(source_build_directory)/phpunit*xml" \ + --exclude="$(source_build_directory)/composer.*" \ + --exclude="$(source_build_directory)/node_modules" \ + --exclude="$(source_build_directory)/js/node_modules" \ + --exclude="$(source_build_directory)/js/tests" \ + --exclude="$(source_build_directory)/js/test" \ + --exclude="$(source_build_directory)/js/*.log" \ + --exclude="$(source_build_directory)/js/package.json" \ + --exclude="$(source_build_directory)/js/bower.json" \ + --exclude="$(source_build_directory)/js/karma.*" \ + --exclude="$(source_build_directory)/js/protractor.*" \ + --exclude="$(source_build_directory)/package.json" \ + --exclude="$(source_build_directory)/bower.json" \ + --exclude="$(source_build_directory)/karma.*" \ + --exclude="$(source_build_directory)/protractor\.*" \ + --exclude="$(source_build_directory)/.*" \ + --exclude="$(source_build_directory)/js/.*" \ + --exclude="$(source_build_directory)/webpack.config.js" \ + --exclude="$(source_build_directory)/stylelint.config.js" \ + --exclude="$(source_build_directory)/CHANGELOG.md" \ + --exclude="$(source_build_directory)/README.md" \ + --exclude="$(source_build_directory)/package-lock.json" \ + --exclude="$(source_build_directory)/LICENSES" \ + --exclude="$(source_build_directory)/src" \ + --exclude="$(source_build_directory)/babel.config.js" \ + --exclude="$(source_build_directory)/devcontainer.yaml" \ + --exclude="$(source_build_directory)/psalm.xml" \ + --exclude="$(source_build_directory)/start-dev-environment.ps1" \ + --exclude="$(source_build_directory)/start-dev-environment.sh" \ + --exclude="$(source_build_directory)/dev-environment*" \ + --exclude="$(source_build_directory)/install.sh" \ + --exclude="$(source_build_directory)/renovate.json" \ + --exclude="$(source_build_directory)/get-matrix.sh" \ + --exclude="$(source_build_directory)/xdebug.ini" \ + --exclude="$(source_build_directory)/compose-install.yaml" \ + --exclude="$(source_build_directory)/Dockerfile.Nextcloud" \ + --exclude="$(source_build_directory)/empty-skeleton.config.php" \ + $(source_build_directory) \ diff --git a/empty-skeleton.config.php b/empty-skeleton.config.php index 30eef6ea..c445b487 100644 --- a/empty-skeleton.config.php +++ b/empty-skeleton.config.php @@ -2,4 +2,5 @@ $CONFIG = [ 'skeletondirectory' => '', + 'debug' => true, ]; diff --git a/install.sh b/install.sh index 21adaa04..36368be8 100755 --- a/install.sh +++ b/install.sh @@ -41,6 +41,7 @@ setup_nextcloud () { build_app () { echo "build app" + make distclean make appstore tar -xf ./build/artifacts/gdatavaas.tar.gz -C ./build/artifacts echo "build app finished" diff --git a/lib/AppInfo/Application.php b/lib/AppInfo/Application.php index 785e27ab..29aff2de 100644 --- a/lib/AppInfo/Application.php +++ b/lib/AppInfo/Application.php @@ -53,10 +53,9 @@ public function __construct() { * @return void */ public function register(IRegistrationContext $context): void { - $composerAutoloadFile = __DIR__ . '/../../vendor/autoload.php'; - if (file_exists($composerAutoloadFile)) { - require_once $composerAutoloadFile; - } + require_once file_exists(__DIR__.'/../../vendor/scoper-autoload.php') + ? __DIR__.'/../../vendor/scoper-autoload.php' + : __DIR__.'/../../vendor/autoload.php'; // Manually register TagService so that we can customize the DI used for $silentTagMapper $context->registerService(TagService::class, function ($c) { diff --git a/lib/AvirWrapper.php b/lib/AvirWrapper.php index 7dc3e4d7..fef698f5 100644 --- a/lib/AvirWrapper.php +++ b/lib/AvirWrapper.php @@ -111,7 +111,7 @@ public function writeStream(string $path, $stream, ?int $size = null): int { public function rename($source, $target) { if ($this->shouldWrap($source)) { // After the upload apps/dav/lib/Connector/Sabre/File.php calls moveFromStorage which calls rename - $this->logger->debug(sprintf("rename(%s, %s)", $source, $target)); + $this->logger->debug(sprintf('rename(%s, %s)', $source, $target)); $this->verdictService->onRename($this->getLocalFile($source), $this->getLocalFile($target)); } return parent::rename($source, $target); @@ -135,7 +135,7 @@ private function wrapSteam(string $path, $stream) { function () use ($path, $logger) { $localPath = $this->getLocalFile($path); $filesize = $this->filesize($path); - $logger->debug("Closing " . $localPath . " with size " . $filesize); + $logger->debug('Closing ' . $localPath . ' with size ' . $filesize); if (!$this->verdictService->isAllowedToScan($localPath)) { return; @@ -162,14 +162,14 @@ function () use ($path, $logger) { } catch (UploadFailedException|ServerException) { $this->logger->error("File $localPath could not be scanned on upload with GData VaaS because there was a temporary upstream server error"); } catch (VaasAuthenticationException) { - $this->logger->error("Authentication for VaaS scan failed. Please check your credentials."); + $this->logger->error('Authentication for VaaS scan failed. Please check your credentials.'); } catch (\Exception $e) { - $this->logger->error("Unexpected error while scanning file " . $localPath . " on upload: " . $e->getMessage()); + $this->logger->error('Unexpected error while scanning file ' . $localPath . ' on upload: ' . $e->getMessage()); } - $logger->debug("Verdict for " . $localPath . " is " . $verdict->Verdict->value); + $logger->debug('Verdict for ' . $localPath . ' is ' . $verdict->Verdict->value); if ($verdict->Verdict == Verdict::MALICIOUS) { - $logger->debug("Removing malicious file " . $localPath); + $logger->debug('Removing malicious file ' . $localPath); //prevent from going to trashbin if ($this->trashEnabled) { @@ -195,7 +195,7 @@ function () use ($path, $logger) { $activity = $this->activityManager->generateEvent(); $activity->setApp(Application::APP_ID) - ->setSubject(Provider::SUBJECT_VIRUS_DETECTED_UPLOAD, [$verdict->Detection ?? "no_detection_name"]) + ->setSubject(Provider::SUBJECT_VIRUS_DETECTED_UPLOAD, [$verdict->Detection ?? 'no_detection_name']) ->setMessage(Provider::MESSAGE_FILE_DELETED) ->setObject('', 0, $path) ->setAffectedUser($owner) diff --git a/memory.ini b/memory.ini new file mode 100644 index 00000000..b0fe7fef --- /dev/null +++ b/memory.ini @@ -0,0 +1 @@ +memory_limit=-1 diff --git a/scoper.inc.php b/scoper.inc.php new file mode 100644 index 00000000..06443fad --- /dev/null +++ b/scoper.inc.php @@ -0,0 +1,135 @@ + $fileInfo->getPathName(), +// iterator_to_array( +// Finder::create()->files()->in(__DIR__), +// false, +// ), +// ); +$excludedFiles = ['templates/admin.php']; + +return [ + // The prefix configuration. If a non-null value is used, a random prefix + // will be generated instead. + // + // For more see: https://github.com/humbug/php-scoper/blob/master/docs/configuration.md#prefix + 'prefix' => 'OCA\\GDataVaas\\Vendor', + + // The base output directory for the prefixed files. + // This will be overridden by the 'output-dir' command line option if present. + 'output-dir' => null, + + // By default when running php-scoper add-prefix, it will prefix all relevant code found in the current working + // directory. You can however define which files should be scoped by defining a collection of Finders in the + // following configuration key. + // + // This configuration entry is completely ignored when using Box. + // + // For more see: https://github.com/humbug/php-scoper/blob/master/docs/configuration.md#finders-and-paths + 'finders' => [ + Finder::create() + ->files() + ->ignoreVCS(true) + ->ignoreDotFiles(true) + ->exclude([ + 'build', + '.devcontainer', + 'nextcloud-server', + 'tests', + 'tmp', + 'test', + 'examples' + ]) + ->in(__DIR__), + ], + + // List of excluded files, i.e. files for which the content will be left untouched. + // Paths are relative to the configuration file unless if they are already absolute + // + // For more see: https://github.com/humbug/php-scoper/blob/master/docs/configuration.md#patchers + 'exclude-files' => [ + // 'src/an-excluded-file.php', + ...$excludedFiles, + ], + + // PHP version (e.g. `'7.2'`) in which the PHP parser and printer will be configured into. This will affect what + // level of code it will understand and how the code will be printed. + // If none (or `null`) is configured, then the host version will be used. + 'php-version' => null, + + // When scoping PHP files, there will be scenarios where some of the code being scoped indirectly references the + // original namespace. These will include, for example, strings or string manipulations. PHP-Scoper has limited + // support for prefixing such strings. To circumvent that, you can define patchers to manipulate the file to your + // heart contents. + // + // For more see: https://github.com/humbug/php-scoper/blob/master/docs/configuration.md#patchers + 'patchers' => [ + static function (string $filePath, string $prefix, string $contents): string { + // Change the contents here. + + if (str_ends_with($filePath, 'vendor/netresearch/jsonmapper/src/JsonMapper.php') === true) { + $contents = str_replace("namespace $prefix;", "namespace $prefix\\JsonMapper;", $contents); + } + + if (str_ends_with($filePath, 'vendor/netresearch/jsonmapper/src/JsonMapper/Exception.php') === true) { + $contents = str_replace("namespace $prefix;", "namespace $prefix\\JsonMapper;", $contents); + } + + if (str_ends_with($filePath, 'vendor/gdata/vaas/Vaas.php') === true) { + $contents = str_replace('use OCA\GDataVaas\Vendor\JsonMapper', 'use OCA\GDataVaas\Vendor\JsonMapper\JsonMapper', $contents); + } + + return $contents; + }, + ], + + // List of symbols to consider internal i.e. to leave untouched. + // + // For more information see: https://github.com/humbug/php-scoper/blob/master/docs/configuration.md#excluded-symbols + 'exclude-namespaces' => [ + 'Psr', + 'OCA', + 'OCP', + 'OC\Files', + 'OC\SystemTag', + 'Symfony', + 'Icewind' + ], + 'exclude-classes' => [ + 'OC', + ], + 'exclude-functions' => [ + ], + 'exclude-constants' => [ + // 'STDIN', + ], + + // List of symbols to expose. + // + // For more information see: https://github.com/humbug/php-scoper/blob/master/docs/configuration.md#exposed-symbols + 'expose-global-constants' => false, + 'expose-global-classes' => false, + 'expose-global-functions' => false, + 'expose-namespaces' => [ + // 'Acme\Foo' // The Acme\Foo namespace (and sub-namespaces) + // '~^PHPUnit\\\\Framework$~', // The whole namespace PHPUnit\Framework (but not sub-namespaces) + // '~^$~', // The root namespace only + // '', // Any namespace + ], + 'expose-classes' => [], + 'expose-functions' => [], + 'expose-constants' => [], +]; From 5f395721b5a61b8e0c3f3d8a1b862a68188a28e1 Mon Sep 17 00:00:00 2001 From: vaas-bot <108060048+ata-no-one@users.noreply.github.com> Date: Wed, 2 Oct 2024 12:36:56 +0200 Subject: [PATCH 2/7] chore(deps): update all non-major dependencies (#136) Co-authored-by: Renovate Bot --- composer.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/composer.json b/composer.json index d95ac738..e0413c78 100644 --- a/composer.json +++ b/composer.json @@ -15,9 +15,9 @@ "require-dev": { "nextcloud/ocp": "v30.0.0", "psalm/phar": "5.26.1", - "nextcloud/coding-standard": "v1.2.3", - "phpunit/phpunit": "10.5.34", - "symfony/console": "6.4.11" + "nextcloud/coding-standard": "v1.3.1", + "phpunit/phpunit": "10.5.35", + "symfony/console": "6.4.12" }, "autoload": { "psr-4": { From b93eb1715b914e6b145624e245e8360f68c32097 Mon Sep 17 00:00:00 2001 From: Lennart Dohmann Date: Thu, 3 Oct 2024 11:03:04 +0200 Subject: [PATCH 3/7] Hotfix for Postgres user --- lib/Db/DbFileMapper.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/Db/DbFileMapper.php b/lib/Db/DbFileMapper.php index 2de51ef3..aee90d0c 100644 --- a/lib/Db/DbFileMapper.php +++ b/lib/Db/DbFileMapper.php @@ -92,7 +92,7 @@ private function getPlatformSpecificCast(): string { $cast = 'CAST(' . 'o.objectid' . ' AS UNSIGNED)'; } elseif ($platform === 'sqlite') { $cast = 'CAST(' . 'o.objectid' . ' AS INTEGER)'; - } elseif ($platform === 'postgresql') { + } elseif ($platform === 'postgres') { $cast = 'CAST(' . 'o.objectid' . ' AS BIGINT)'; } else { throw new Exception('Unsupported database platform: ' . $platform); From 06d33a6134dd96a9f658f46cc046a34139360ebc Mon Sep 17 00:00:00 2001 From: Matthias Simonis Date: Wed, 16 Oct 2024 09:44:13 +0200 Subject: [PATCH 4/7] Use devcontainer in build (#142) * WIP * WIP magic * adjust scoper config * Resolve dependency hell * Make something * add build step for the nextcloud builder * add the checkout step * login to registry * use devcontainer as build-runner2 * skip the postCreate stuff when in CI * change default shell for docker build * extract some functions from the script to features * run poststartcommands in ci * fix filename * fix script for non vscode environments * remove extra install scoper step * add fail fast on install * use bash as default shell * make apt less noisy * try another fail * remove trap in distclean * add memory.ini to top * give the shell job control * just use bash * Scope dependencies (#138) Scope dependencies --------- Co-authored-by: Matthias Simonis * Fix if the file size is zero, the app evaluates the file as too large to scan (cherry picked from commit c37472d68d36007e3c28739015f9f78092ccb59a) * WIP * WIP magic * adjust scoper config * Resolve dependency hell * Make something * add fail fast on install * exit early in the install script * fix stuff * add composer path to github path * split postCreate and add to path * remove the scoper installation (done in the postCreateCommand) * add container user vscode * remove vscode user * add docker ps * replace hostname in ci * add debug stuff * use static ip * show json ps * try connect the builder container to the compose network * fix name * move connect to after installation * fix hostname * add hostname to trusted_domains array * fix smtp hostname * replace quote types * remove debugging stuff --------- Co-authored-by: Lennart Dohmann Co-authored-by: PT-ATA No One --- .devcontainer/devcontainer.json | 6 ++- .devcontainer/postCreateCommands.sh | 19 +++++--- .github/workflows/release-app.yml | 59 ++++++++++++++++-------- .gitignore | 3 +- Dockerfile.Nextcloud | 4 +- compose-install.yaml | 2 +- foobar.json | 0 install.sh | 10 ++-- tests/bats/.env-test | 14 ++++++ tests/bats/functionality-parallel.bats | 24 +++++----- tests/bats/functionality-sequential.bats | 42 ++++++++--------- 11 files changed, 117 insertions(+), 66 deletions(-) create mode 100644 foobar.json diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 384f245c..54fa65f7 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -31,6 +31,10 @@ "features": { "ghcr.io/devcontainers/features/node:1": {}, "ghcr.io/edouard-lopez/devcontainer-features/bats:0": {}, - "ghcr.io/devcontainers/features/docker-in-docker:2": {} + "ghcr.io/devcontainers/features/docker-in-docker:2": {}, + "ghcr.io/devcontainers/features/github-cli:1": {}, + "ghcr.io/wxw-matt/devcontainer-features/apt:latest": { + "packages": "bash-completion vim iputils-ping telnet" + } } } \ No newline at end of file diff --git a/.devcontainer/postCreateCommands.sh b/.devcontainer/postCreateCommands.sh index 1f574c89..71959e0e 100755 --- a/.devcontainer/postCreateCommands.sh +++ b/.devcontainer/postCreateCommands.sh @@ -1,13 +1,21 @@ #!/bin/bash -sudo apt-get update -sudo apt-get install -y bash-completion vim iputils-ping telnet +echo "setup php-scoper" +composer global require humbug/php-scoper +$(composer config home)/vendor/bin/php-scoper completion bash >> $HOME.bash_completion +echo "export PATH=$(composer config home)/vendor/bin/:\$PATH" >> $HOME/.bashrc +export PATH=$(composer config home)/vendor/bin/:$PATH + +sudo cp memory.ini /usr/local/etc/php/conf.d/memory.ini + +if [[ "$IS_CI" == "true" ]]; then + exit 0 +fi + sudo bash -c "docker completion bash > /usr/share/bash-completion/completions/docker" sudo bash -c "composer completion bash > /usr/share/bash-completion/completions/composer" sudo bash -c "npm completion > /usr/share/bash-completion/completions/npm" sudo cp xdebug.local.ini /usr/local/etc/php/conf.d/xdebug.ini -sudo cp memory.ini /usr/local/etc/php/conf.d/memory.ini -sudo curl -sS https://webi.sh/gh | sh echo ". /usr/share/bash-completion/bash_completion" >> /home/vscode/.bashrc @@ -21,7 +29,4 @@ git clone --depth 1 --recurse-submodules --single-branch --branch v$NEXTCLOUD_VE cd nextcloud-server git submodule update --init cd - -composer global require humbug/php-scoper -/home/vscode/.composer/vendor/bin/php-scoper completion bash >> /home/vscode/.bash_completion -echo 'export PATH=/home/vscode/.composer/vendor/bin/:$PATH' >>~/.bashrc ./install.sh \ No newline at end of file diff --git a/.github/workflows/release-app.yml b/.github/workflows/release-app.yml index 5bff29a5..99ed58ae 100644 --- a/.github/workflows/release-app.yml +++ b/.github/workflows/release-app.yml @@ -1,5 +1,8 @@ name: Build and Test +defaults: + run: + shell: bash on: push: branches: ["main"] @@ -17,23 +20,47 @@ jobs: - id: get-matrix run: echo "nextcloud_versions=$(./get-matrix.sh)" | tee -a "$GITHUB_OUTPUT" + build-devcontainer: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Pre-build dev container image + uses: devcontainers/ci@v0.3 + with: + imageName: ghcr.io/gdatasoftwareag/nextcloud/builder + cacheFrom: ghcr.io/gdatasoftwareag/nextcloud/builder + push: always + test: runs-on: ubuntu-latest - needs: define-matrix + container: + image: ghcr.io/gdatasoftwareag/nextcloud/builder:latest + env: + IS_CI: true + options: --name nextcloud-antivirus-build-container + needs: + - define-matrix + - build-devcontainer strategy: matrix: nextcloud_version: ${{fromJson(needs.define-matrix.outputs.nextcloud_versions)}} steps: - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - name: setup node - with: - node-version: 20 - - name: Setup BATS - uses: mig4/setup-bats@v1 - with: - bats-version: 1.11.0 + - name: postCreateCommands + run: | + source .devcontainer/postCreateCommands.sh + + - name: add composer bin to path + run: | + echo $(composer config home --global)/vendor/bin >> $GITHUB_PATH + echo "/home/vscode/bin/" >> $GITHUB_PATH - name: replace version in ./appinfo/info.xml if: startsWith(github.ref, 'refs/tags/') @@ -42,11 +69,6 @@ jobs: RELEASE_VERSION=${GITHUB_REF#refs/tags/} sed -i "s/0.0.0<\/version>/$RELEASE_VERSION<\/version>/g" ./appinfo/info.xml - - uses: docker-practice/actions-setup-docker@master - - uses: shivammathur/setup-php@v2 - with: - php-version: '8.3' - - name: unittests env: CLIENT_ID: ${{ secrets.VAAS_CLIENT_ID }} @@ -55,17 +77,16 @@ jobs: composer install ./vendor/bin/phpunit --bootstrap tests/unittests/bootstrap.php tests/unittests/ --testdox - - name: install php-scoper - run: | - composer global require humbug/php-scoper - echo $(composer config home --global) >> $GITHUB_PATH - - name: install nextcloud env: CLIENT_ID: ${{ secrets.VAAS_CLIENT_ID }} CLIENT_SECRET: ${{ secrets.VAAS_CLIENT_SECRET }} run: ./install.sh ${{ matrix.nextcloud_version }} 1 + - name: add builder into nextcloud network + run: | + docker network connect nextcloud-gdata-antivirus_nextcloud-network nextcloud-antivirus-build-container + - name: run tests id: bats-tests env: diff --git a/.gitignore b/.gitignore index 3d05ad75..4f69716d 100644 --- a/.gitignore +++ b/.gitignore @@ -69,4 +69,5 @@ core.1 nextcloud-server/ core-dump.zip apache/ -**/vendor/* \ No newline at end of file +**/vendor/* +test.sh \ No newline at end of file diff --git a/Dockerfile.Nextcloud b/Dockerfile.Nextcloud index 79b254c2..bde95499 100644 --- a/Dockerfile.Nextcloud +++ b/Dockerfile.Nextcloud @@ -3,7 +3,9 @@ ARG INSTALL_XDEBUG=1 FROM nextcloud:${NEXTCLOUD_VERSION} -RUN apt update && apt install -y \ +SHELL [ "/bin/bash", "-c" ] + +RUN apt-get -q update && apt-get -q install -y \ less vim telnet iputils-ping gdb libexpat1-dev libapr1-dev libaprutil1-dev devscripts debmake \ bison jdupes libbrotli-dev liblua5.4-dev libnghttp2-dev libssl-dev libxml2-dev libcurl4-openssl-dev libjansson-dev # RUN curl -L -o /tmp/apache2_2.4.61.orig.tar.gz https://launchpad.net/debian/+archive/primary/+sourcefiles/apache2/2.4.61-1/apache2_2.4.61.orig.tar.gz \ diff --git a/compose-install.yaml b/compose-install.yaml index eb5a2b00..6d46141e 100644 --- a/compose-install.yaml +++ b/compose-install.yaml @@ -21,7 +21,7 @@ services: - smtp restart: unless-stopped networks: - - nextcloud-network + nextcloud-network: smtp: image: rnwood/smtp4dev:v3 container_name: smtp diff --git a/foobar.json b/foobar.json new file mode 100644 index 00000000..e69de29b diff --git a/install.sh b/install.sh index 36368be8..31560fd8 100755 --- a/install.sh +++ b/install.sh @@ -1,4 +1,5 @@ #!/bin/bash +set -e export NEXTCLOUD_VERSION=${1:-30.0.0} export INSTALL_XDEBUG=${2:-1} @@ -12,7 +13,7 @@ setup_nextcloud () { docker compose -f compose-install.yaml rm --force --stop --volumes docker compose -f compose-install.yaml up --build --quiet-pull --wait -d --force-recreate --renew-anon-volumes --remove-orphans - docker exec -i nextcloud-container ulimit -c unlimited + docker exec -i nextcloud-container bash -c 'ulimit -c unlimited' docker exec -i nextcloud-container bash -c 'echo "/tmp/apache2-coredump/core-%e-%s-%u-%g-%p-%t" > /proc/sys/kernel/core_pattern' until docker exec --user www-data -i nextcloud-container php occ status | grep "installed: false" @@ -35,6 +36,7 @@ setup_nextcloud () { docker exec --user www-data -i nextcloud-container php occ app:disable firstrunwizard docker exec --user www-data -i nextcloud-container php occ app:disable weather_status docker exec --user www-data -i nextcloud-container php occ config:system:set trusted_domains 2 --value=192.168.5.80 + docker exec --user www-data -i nextcloud-container php occ config:system:set trusted_domains 3 --value=nextcloud-container echo "setup nextcloud finished" } @@ -43,7 +45,7 @@ build_app () { echo "build app" make distclean make appstore - tar -xf ./build/artifacts/gdatavaas.tar.gz -C ./build/artifacts + tar -xf ./build/artifacts/gdatavaas.tar.gz -C ./build/artifacts echo "build app finished" } @@ -53,8 +55,10 @@ if [ -z "$CLIENT_ID" ] || [ -z "$CLIENT_SECRET" ]; then fi setup_nextcloud & +setup_nextcloud_pid=$! build_app & -wait +wait %2 || exit 1 +wait %1 || exit 1 docker cp ./build/artifacts/gdatavaas nextcloud-container:/var/www/html/apps/ docker exec -i nextcloud-container chown -R www-data:www-data /var/www/html/apps/gdatavaas diff --git a/tests/bats/.env-test b/tests/bats/.env-test index 2f1ad484..795f1c58 100644 --- a/tests/bats/.env-test +++ b/tests/bats/.env-test @@ -1,3 +1,17 @@ +#!/bin/bash + +if [[ "$IS_CI" == "true" ]]; then + export HOSTNAME=nextcloud-container +else + export HOSTNAME=127.0.0.1 +fi + +if [[ "$IS_CI" == "true" ]]; then + export MAIL_HOSTNAME=smtp +else + export MAIL_HOSTNAME=127.0.0.1:8001 +fi + export FOLDER_PREFIX=./tmp/functionality-parallel export TESTUSER=testuser export TESTUSER_PASSWORD=myfancysecurepassword234 diff --git a/tests/bats/functionality-parallel.bats b/tests/bats/functionality-parallel.bats index 48a9f6d7..126d2183 100755 --- a/tests/bats/functionality-parallel.bats +++ b/tests/bats/functionality-parallel.bats @@ -16,7 +16,7 @@ setup_file() { @test "test admin eicar Upload" { EICAR_LENGTH=$(echo $EICAR_STRING | wc -c) - RESULT=$(echo $EICAR_STRING | curl -v -X PUT -d"$EICAR_STRING" -w "%{http_code}" -u admin:admin -T - http://127.0.0.1/remote.php/dav/files/admin/functionality-parallel.eicar.com.txt || echo "curl failed") + RESULT=$(echo $EICAR_STRING | curl -v -X PUT -d"$EICAR_STRING" -w "%{http_code}" -u admin:admin -T - http://$HOSTNAME/remote.php/dav/files/admin/functionality-parallel.eicar.com.txt || echo "curl failed") if [[ "$RESULT" =~ "curl failed" ]]; then echo "debugging stuff" @@ -38,12 +38,12 @@ setup_file() { fi echo "Actual: $RESULT" - curl --silent -q -u admin:admin -X DELETE http://127.0.0.1/remote.php/dav/files/admin/functionality-parallel.eicar.com.txt || echo "file not found" + curl --silent -q -u admin:admin -X DELETE http://$HOSTNAME/remote.php/dav/files/admin/functionality-parallel.eicar.com.txt || echo "file not found" [[ "$RESULT" =~ "Upload cannot be completed." ]] } @test "test admin clean upload" { - RESULT=$(echo $CLEAN_STRING | curl -w "%{http_code}" -u admin:admin -T - http://127.0.0.1/remote.php/dav/files/admin/functionality-parallel.clean.txt || echo "curl failed") + RESULT=$(echo $CLEAN_STRING | curl -w "%{http_code}" -u admin:admin -T - http://$HOSTNAME/remote.php/dav/files/admin/functionality-parallel.clean.txt || echo "curl failed") if [[ "$RESULT" =~ "curl failed" ]]; then echo "debugging stuff" @@ -61,36 +61,36 @@ setup_file() { fi echo "Actual: $RESULT" - curl --silent -q -u admin:admin -X DELETE http://127.0.0.1/remote.php/dav/files/admin/functionality-parallel.clean.txt || echo "file not found" + curl --silent -q -u admin:admin -X DELETE http://$HOSTNAME/remote.php/dav/files/admin/functionality-parallel.clean.txt || echo "file not found" [[ $RESULT -ge 200 && $RESULT -lt 300 ]] } @test "test admin pup Upload" { - RESULT=$(curl --silent -w "%{http_code}" -u admin:admin -T $FOLDER_PREFIX/pup.exe http://127.0.0.1/remote.php/dav/files/admin/functionality-parallel.pup.exe) + RESULT=$(curl --silent -w "%{http_code}" -u admin:admin -T $FOLDER_PREFIX/pup.exe http://$HOSTNAME/remote.php/dav/files/admin/functionality-parallel.pup.exe) echo "Actual: $RESULT" - curl --silent -q -u admin:admin -X DELETE http://127.0.0.1/remote.php/dav/files/admin/functionality-parallel.pup.exe || echo "file not found" + curl --silent -q -u admin:admin -X DELETE http://$HOSTNAME/remote.php/dav/files/admin/functionality-parallel.pup.exe || echo "file not found" [[ $RESULT -ge 200 && $RESULT -lt 300 ]] } @test "test testuser eicar Upload" { - RESULT=$(echo $EICAR_STRING | curl --silent -w "%{http_code}" -u $TESTUSER:$TESTUSER_PASSWORD -T - http://127.0.0.1/remote.php/dav/files/$TESTUSER/functionality-parallel.eicar.com.txt) + RESULT=$(echo $EICAR_STRING | curl --silent -w "%{http_code}" -u $TESTUSER:$TESTUSER_PASSWORD -T - http://$HOSTNAME/remote.php/dav/files/$TESTUSER/functionality-parallel.eicar.com.txt) echo "Actual: $RESULT" $DOCKER_EXEC_WITH_USER -i nextcloud-container php occ config:app:get gdatavaas clientSecret - curl --silent -q -u $TESTUSER:$TESTUSER_PASSWORD -X DELETE http://127.0.0.1/remote.php/dav/files/$TESTUSER/functionality-parallel.eicar.com.txt || echo "file not found" + curl --silent -q -u $TESTUSER:$TESTUSER_PASSWORD -X DELETE http://$HOSTNAME/remote.php/dav/files/$TESTUSER/functionality-parallel.eicar.com.txt || echo "file not found" [[ "$RESULT" =~ "Upload cannot be completed." ]] } @test "test testuser clean Upload" { - STATUS_CODE=$(echo $CLEAN_STRING | curl --silent -w "%{http_code}" -u $TESTUSER:$TESTUSER_PASSWORD -T - http://127.0.0.1/remote.php/dav/files/$TESTUSER/functionality-parallel.clean.txt) + STATUS_CODE=$(echo $CLEAN_STRING | curl --silent -w "%{http_code}" -u $TESTUSER:$TESTUSER_PASSWORD -T - http://$HOSTNAME/remote.php/dav/files/$TESTUSER/functionality-parallel.clean.txt) echo "Actual: $RESULT" - curl --silent -q -u $TESTUSER:$TESTUSER_PASSWORD -X DELETE http://127.0.0.1/remote.php/dav/files/$TESTUSER/functionality-parallel.clean.txt || echo "file not found" + curl --silent -q -u $TESTUSER:$TESTUSER_PASSWORD -X DELETE http://$HOSTNAME/remote.php/dav/files/$TESTUSER/functionality-parallel.clean.txt || echo "file not found" [[ $STATUS_CODE -ge 200 && $STATUS_CODE -lt 300 ]] || exit 1 } @test "test testuser pup Upload" { - RESULT=$(curl --silent -w "%{http_code}" -u $TESTUSER:$TESTUSER_PASSWORD -T $FOLDER_PREFIX/pup.exe http://127.0.0.1/remote.php/dav/files/$TESTUSER/functionality-parallel.pup.exe) + RESULT=$(curl --silent -w "%{http_code}" -u $TESTUSER:$TESTUSER_PASSWORD -T $FOLDER_PREFIX/pup.exe http://$HOSTNAME/remote.php/dav/files/$TESTUSER/functionality-parallel.pup.exe) echo "Actual: $RESULT" - curl --silent -q -u $TESTUSER:$TESTUSER_PASSWORD -X DELETE http://127.0.0.1/remote.php/dav/files/$TESTUSER/functionality-parallel.pup.exe || echo "file not found" + curl --silent -q -u $TESTUSER:$TESTUSER_PASSWORD -X DELETE http://$HOSTNAME/remote.php/dav/files/$TESTUSER/functionality-parallel.pup.exe || echo "file not found" [[ $RESULT -ge 200 && $RESULT -lt 300 ]] || exit 1 } diff --git a/tests/bats/functionality-sequential.bats b/tests/bats/functionality-sequential.bats index 44ec2fc5..352e496e 100644 --- a/tests/bats/functionality-sequential.bats +++ b/tests/bats/functionality-sequential.bats @@ -17,9 +17,9 @@ setup_file() { @test "test upload when vaas does not function" { $DOCKER_EXEC_WITH_USER nextcloud-container php occ config:app:set gdatavaas clientSecret --value="WRONG_PASSWORD" - RESULT=$(echo $EICAR_STRING | curl --silent -w "%{http_code}" -u admin:admin -T - http://127.0.0.1/remote.php/dav/files/admin/functionality-sequential.eicar.com.txt) + RESULT=$(echo $EICAR_STRING | curl --silent -w "%{http_code}" -u admin:admin -T - http://$HOSTNAME/remote.php/dav/files/admin/functionality-sequential.eicar.com.txt) $DOCKER_EXEC_WITH_USER nextcloud-container php occ config:app:set gdatavaas clientSecret --value="$CLIENT_SECRET" - curl --silent -q -u admin:admin -X DELETE http://127.0.0.1/remote.php/dav/files/admin/functionality-sequential.eicar.com.txt + curl --silent -q -u admin:admin -X DELETE http://$HOSTNAME/remote.php/dav/files/admin/functionality-sequential.eicar.com.txt echo "Actual: $RESULT" [[ $RESULT -ge 200 && $RESULT -lt 300 ]] @@ -27,9 +27,9 @@ setup_file() { @test "test croned scan for admin files" { $DOCKER_EXEC_WITH_USER nextcloud-container php occ config:app:set gdatavaas clientSecret --value="WRONG_PASSWORD" - echo $EICAR_STRING | curl --silent -w "%{http_code}" -u admin:admin -T - http://127.0.0.1/remote.php/dav/files/admin/admin.functionality-sequential.eicar.com.txt - curl --silent -w "%{http_code}" -u admin:admin -T $FOLDER_PREFIX/pup.exe http://127.0.0.1/remote.php/dav/files/admin/admin.pup.exe - echo $CLEAN_STRING | curl --silent -w "%{http_code}" -u admin:admin -T - http://127.0.0.1/remote.php/dav/files/admin/admin.functionality-sequential.clean.txt + echo $EICAR_STRING | curl --silent -w "%{http_code}" -u admin:admin -T - http://$HOSTNAME/remote.php/dav/files/admin/admin.functionality-sequential.eicar.com.txt + curl --silent -w "%{http_code}" -u admin:admin -T $FOLDER_PREFIX/pup.exe http://$HOSTNAME/remote.php/dav/files/admin/admin.pup.exe + echo $CLEAN_STRING | curl --silent -w "%{http_code}" -u admin:admin -T - http://$HOSTNAME/remote.php/dav/files/admin/admin.functionality-sequential.clean.txt $DOCKER_EXEC_WITH_USER nextcloud-container php occ config:app:set gdatavaas clientSecret --value="$CLIENT_SECRET" @@ -57,9 +57,9 @@ setup_file() { LOGS=$($DOCKER_EXEC_WITH_USER -i nextcloud-container tail -5000 data/nextcloud.log | egrep "admin.functionality-sequential.eicar.com.txt|admin.functionality-sequential.clean.txt|admin.pup.exe" ) - curl --silent -q -u admin:admin -X DELETE http://127.0.0.1/remote.php/dav/files/admin/admin.functionality-sequential.eicar.com.txt - curl --silent -q -u admin:admin -X DELETE http://127.0.0.1/remote.php/dav/files/admin/admin.pup.exe - curl --silent -q -u admin:admin -X DELETE http://127.0.0.1/remote.php/dav/files/admin/admin.functionality-sequential.clean.txt + curl --silent -q -u admin:admin -X DELETE http://$HOSTNAME/remote.php/dav/files/admin/admin.functionality-sequential.eicar.com.txt + curl --silent -q -u admin:admin -X DELETE http://$HOSTNAME/remote.php/dav/files/admin/admin.pup.exe + curl --silent -q -u admin:admin -X DELETE http://$HOSTNAME/remote.php/dav/files/admin/admin.functionality-sequential.clean.txt [[ $LOGS =~ ^.*admin.functionality-sequential.eicar.com.txt.*Verdict:.*Malicious ]] [[ $LOGS =~ ^.*admin.pup.exe.*Verdict:.*Pup ]] @@ -69,9 +69,9 @@ setup_file() { @test "test croned scan for testuser files" { $DOCKER_EXEC_WITH_USER -i nextcloud-container php occ config:app:set gdatavaas clientSecret --value="WRONG_PASSWORD" - echo $EICAR_STRING |curl --silent -w "%{http_code}" -u $TESTUSER:$TESTUSER_PASSWORD -T - http://127.0.0.1/remote.php/dav/files/$TESTUSER/$TESTUSER.functionality-sequential.eicar.com.txt - curl --silent -w "%{http_code}" -u $TESTUSER:$TESTUSER_PASSWORD -T $FOLDER_PREFIX/pup.exe http://127.0.0.1/remote.php/dav/files/$TESTUSER/$TESTUSER.pup.exe - echo $CLEAN_STRING |curl --silent -w "%{http_code}" -u $TESTUSER:$TESTUSER_PASSWORD -T - http://127.0.0.1/remote.php/dav/files/$TESTUSER/$TESTUSER.functionality-sequential.clean.txt + echo $EICAR_STRING |curl --silent -w "%{http_code}" -u $TESTUSER:$TESTUSER_PASSWORD -T - http://$HOSTNAME/remote.php/dav/files/$TESTUSER/$TESTUSER.functionality-sequential.eicar.com.txt + curl --silent -w "%{http_code}" -u $TESTUSER:$TESTUSER_PASSWORD -T $FOLDER_PREFIX/pup.exe http://$HOSTNAME/remote.php/dav/files/$TESTUSER/$TESTUSER.pup.exe + echo $CLEAN_STRING |curl --silent -w "%{http_code}" -u $TESTUSER:$TESTUSER_PASSWORD -T - http://$HOSTNAME/remote.php/dav/files/$TESTUSER/$TESTUSER.functionality-sequential.clean.txt $DOCKER_EXEC_WITH_USER -i nextcloud-container php occ config:app:set gdatavaas clientSecret --value="$CLIENT_SECRET" @@ -99,9 +99,9 @@ setup_file() { LOGS=$($DOCKER_EXEC_WITH_USER -i nextcloud-container tail -5000 data/nextcloud.log | egrep "$TESTUSER.functionality-sequential.eicar.com.txt|$TESTUSER.functionality-sequential.clean.txt|$TESTUSER.pup.exe") - curl --silent -q -u $TESTUSER:$TESTUSER_PASSWORD -X DELETE http://127.0.0.1/remote.php/dav/files/$TESTUSER/$TESTUSER.functionality-sequential.eicar.com.txt - curl --silent -q -u $TESTUSER:$TESTUSER_PASSWORD -X DELETE http://127.0.0.1/remote.php/dav/files/$TESTUSER/$TESTUSER.pup.exe - curl --silent -q -u $TESTUSER:$TESTUSER_PASSWORD -X DELETE http://127.0.0.1/remote.php/dav/files/$TESTUSER/$TESTUSER.functionality-sequential.clean.txt + curl --silent -q -u $TESTUSER:$TESTUSER_PASSWORD -X DELETE http://$HOSTNAME/remote.php/dav/files/$TESTUSER/$TESTUSER.functionality-sequential.eicar.com.txt + curl --silent -q -u $TESTUSER:$TESTUSER_PASSWORD -X DELETE http://$HOSTNAME/remote.php/dav/files/$TESTUSER/$TESTUSER.pup.exe + curl --silent -q -u $TESTUSER:$TESTUSER_PASSWORD -X DELETE http://$HOSTNAME/remote.php/dav/files/$TESTUSER/$TESTUSER.functionality-sequential.clean.txt # check for scans [[ $LOGS =~ ^.*$TESTUSER.functionality-sequential.eicar.com.txt.*Verdict:.*Malicious ]] @@ -113,8 +113,8 @@ setup_file() { $DOCKER_EXEC_WITH_USER -i nextcloud-container php occ config:app:set gdatavaas clientSecret --value="WRONG_PASSWORD" $DOCKER_EXEC_WITH_USER -i nextcloud-container php occ config:app:set gdatavaas disableUnscannedTag --value="true" - echo $EICAR_STRING |curl --silent -w "%{http_code}" -u $TESTUSER:$TESTUSER_PASSWORD -T - http://127.0.0.1/remote.php/dav/files/$TESTUSER/$TESTUSER.functionality-sequential.eicar.com.txt - echo $CLEAN_STRING |curl --silent -w "%{http_code}" -u $TESTUSER:$TESTUSER_PASSWORD -T - http://127.0.0.1/remote.php/dav/files/$TESTUSER/$TESTUSER.functionality-sequential.clean.txt + echo $EICAR_STRING |curl --silent -w "%{http_code}" -u $TESTUSER:$TESTUSER_PASSWORD -T - http://$HOSTNAME/remote.php/dav/files/$TESTUSER/$TESTUSER.functionality-sequential.eicar.com.txt + echo $CLEAN_STRING |curl --silent -w "%{http_code}" -u $TESTUSER:$TESTUSER_PASSWORD -T - http://$HOSTNAME/remote.php/dav/files/$TESTUSER/$TESTUSER.functionality-sequential.clean.txt $DOCKER_EXEC_WITH_USER -i nextcloud-container php occ config:app:set gdatavaas clientSecret --value="$CLIENT_SECRET" @@ -124,20 +124,20 @@ setup_file() { $DOCKER_EXEC_WITH_USER -i nextcloud-container php occ config:app:set gdatavaas disableUnscannedTag --value="false" - curl --silent -q -u $TESTUSER:$TESTUSER_PASSWORD -X DELETE http://127.0.0.1/remote.php/dav/files/$TESTUSER/$TESTUSER.functionality-sequential.eicar.com.txt - curl --silent -q -u $TESTUSER:$TESTUSER_PASSWORD -X DELETE http://127.0.0.1/remote.php/dav/files/$TESTUSER/$TESTUSER.functionality-sequential.clean.txt + curl --silent -q -u $TESTUSER:$TESTUSER_PASSWORD -X DELETE http://$HOSTNAME/remote.php/dav/files/$TESTUSER/$TESTUSER.functionality-sequential.eicar.com.txt + curl --silent -q -u $TESTUSER:$TESTUSER_PASSWORD -X DELETE http://$HOSTNAME/remote.php/dav/files/$TESTUSER/$TESTUSER.functionality-sequential.clean.txt } @test "test mailing on eicar upload" { - echo $EICAR_STRING | curl --silent -w "%{http_code}" -u admin:admin -T - http://127.0.0.1/remote.php/dav/files/admin/functionality-sequential.eicar.com.txt + echo $EICAR_STRING | curl --silent -w "%{http_code}" -u admin:admin -T - http://$HOSTNAME/remote.php/dav/files/admin/functionality-sequential.eicar.com.txt sleep 1 - RESULT=$(curl -X 'GET' 'http://127.0.0.1:8081/api/Messages/new?mailboxName=Default&pageSize=1' -H 'accept: application/json') + RESULT=$(curl -X 'GET' "http://$MAIL_HOSTNAME/api/Messages/new?mailboxName=Default&pageSize=1" -H 'accept: application/json') echo $RESULT [[ $RESULT =~ "Infected file upload" ]] - curl --silent -q -u admin:admin -X DELETE http://127.0.0.1/remote.php/dav/files/admin/functionality-sequential.eicar.com.txt + curl --silent -q -u admin:admin -X DELETE http://$HOSTNAME/remote.php/dav/files/admin/functionality-sequential.eicar.com.txt } tearddown_file() { From 8f214848e926207738d1c78a34d3461c1d8df7a5 Mon Sep 17 00:00:00 2001 From: vaas-bot <108060048+ata-no-one@users.noreply.github.com> Date: Wed, 16 Oct 2024 10:16:45 +0200 Subject: [PATCH 5/7] chore(deps): update dependency phpunit/phpunit to v10.5.36 (#148) Co-authored-by: Renovate Bot --- composer.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/composer.json b/composer.json index e0413c78..b9573f14 100644 --- a/composer.json +++ b/composer.json @@ -16,7 +16,7 @@ "nextcloud/ocp": "v30.0.0", "psalm/phar": "5.26.1", "nextcloud/coding-standard": "v1.3.1", - "phpunit/phpunit": "10.5.35", + "phpunit/phpunit": "10.5.36", "symfony/console": "6.4.12" }, "autoload": { From e26678369d35d270079ffec53a8e3db459210e09 Mon Sep 17 00:00:00 2001 From: pstadermann Date: Thu, 17 Oct 2024 15:42:55 +0200 Subject: [PATCH 6/7] Possibly expensive SQL Query on determining files to process #144 (#150) Possibly expensive SQL Query on determining files to process #144 Optimize getFileIdsWithoutTags, getFileIdsWithTags --------- Co-authored-by: Philip Stadermann Co-authored-by: Lennart Dohmann --- lib/Db/DbFileMapper.php | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/lib/Db/DbFileMapper.php b/lib/Db/DbFileMapper.php index aee90d0c..7f592582 100644 --- a/lib/Db/DbFileMapper.php +++ b/lib/Db/DbFileMapper.php @@ -8,8 +8,11 @@ use OCP\IDBConnection; class DbFileMapper extends QBMapper { + private string $stringType; + public function __construct(IDBConnection $db) { parent::__construct($db, 'filecache'); + $this->stringType = $this->getStringTypeDeclarationSQL(); } /** @@ -26,11 +29,15 @@ public function getFileIdsWithoutTags(array $excludedTagIds, int $limit, int $of $qb->select('f.fileid') ->from($this->getTableName(), 'f') - ->leftJoin('f', 'systemtag_object_mapping', 'o', $qb->expr()->eq('f.fileid', $qb->createFunction($this->getPlatformSpecificCast()))) + ->leftJoin('f', 'systemtag_object_mapping', 'o', $qb->expr()->eq('o.objectid', $qb->createFunction(sprintf('CAST(f.fileid AS %s)', $this->stringType)))) ->leftJoin('f', 'mimetypes', 'm', $qb->expr()->eq('f.mimetype', 'm.id')) ->where($qb->expr()->notIn('o.systemtagid', $qb->createNamedParameter($excludedTagIds, IQueryBuilder::PARAM_INT_ARRAY))) ->orWhere($qb->expr()->isNull('o.systemtagid')) ->andWhere($qb->expr()->notLike('m.mimetype', $qb->createNamedParameter('%unix-directory%'))) + ->andWhere($qb->expr()->orX( + $qb->expr()->eq('o.objecttype', $qb->createNamedParameter('files')), + $qb->expr()->isNull('o.objecttype') + )) ->andWhere($qb->expr()->orX( $qb->expr()->like('f.path', $qb->createNamedParameter('files/%')), $qb->expr()->like('f.path', $qb->createNamedParameter('__groupfolders/%')) @@ -61,10 +68,11 @@ public function getFileIdsWithTags(array $includedTagIds, int $limit, int $offse $qb->select('f.fileid') ->from($this->getTableName(), 'f') - ->leftJoin('f', 'systemtag_object_mapping', 'o', $qb->expr()->eq('f.fileid', $qb->createFunction($this->getPlatformSpecificCast()))) + ->leftJoin('f', 'systemtag_object_mapping', 'o', $qb->expr()->eq('o.objectid', $qb->createFunction(sprintf('CAST(f.fileid AS %s)', $this->stringType)))) ->leftJoin('f', 'mimetypes', 'm', $qb->expr()->eq('f.mimetype', 'm.id')) ->where($qb->expr()->in('o.systemtagid', $qb->createNamedParameter($includedTagIds, IQueryBuilder::PARAM_INT_ARRAY))) ->andWhere($qb->expr()->notLike('m.mimetype', $qb->createNamedParameter('%unix-directory%'))) + ->andWhere($qb->expr()->eq('o.objecttype', $qb->createNamedParameter('files'))) ->andWhere($qb->expr()->orX( $qb->expr()->like('f.path', $qb->createNamedParameter('files/%')), $qb->expr()->like('f.path', $qb->createNamedParameter('__groupfolders/%')) @@ -82,22 +90,18 @@ public function getFileIdsWithTags(array $includedTagIds, int $limit, int $offse } /** - * Create a platform-specific cast function - * @return string the database platform-specific cast function + * Get the DB type for a string + * @return string the database string type * @throws Exception if the database platform is not supported */ - private function getPlatformSpecificCast(): string { + private function getStringTypeDeclarationSQL(): string { $platform = $this->db->getDatabaseProvider(); - if ($platform === 'mysql') { - $cast = 'CAST(' . 'o.objectid' . ' AS UNSIGNED)'; - } elseif ($platform === 'sqlite') { - $cast = 'CAST(' . 'o.objectid' . ' AS INTEGER)'; - } elseif ($platform === 'postgres') { - $cast = 'CAST(' . 'o.objectid' . ' AS BIGINT)'; + if ($platform === 'mysql' || $platform === 'sqlite' || $platform === 'postgres') { + $stringType = 'VARCHAR(64)'; } else { throw new Exception('Unsupported database platform: ' . $platform); } - return $cast; + return $stringType; } /** From 7d8ae58c7db1f7605f108ca6fe65aa24a694bfb9 Mon Sep 17 00:00:00 2001 From: Philip Stadermann Date: Fri, 18 Oct 2024 11:14:01 +0200 Subject: [PATCH 7/7] Possibly expensive SQL Query on determining files to process #144 Fix cast for MySQL --- lib/Db/DbFileMapper.php | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/Db/DbFileMapper.php b/lib/Db/DbFileMapper.php index 7f592582..edac9a61 100644 --- a/lib/Db/DbFileMapper.php +++ b/lib/Db/DbFileMapper.php @@ -96,7 +96,10 @@ public function getFileIdsWithTags(array $includedTagIds, int $limit, int $offse */ private function getStringTypeDeclarationSQL(): string { $platform = $this->db->getDatabaseProvider(); - if ($platform === 'mysql' || $platform === 'sqlite' || $platform === 'postgres') { + if ($platform === "mysql") { + $stringType = 'CHAR(64)'; + } + else if ($platform === 'sqlite' || $platform === 'postgres') { $stringType = 'VARCHAR(64)'; } else { throw new Exception('Unsupported database platform: ' . $platform);