From 88d4e1b3f29ac8b6e8b73d03d1493fb866d8d607 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Pablo=20Villaf=C3=A1=C3=B1ez?= Date: Fri, 21 Oct 2022 10:39:55 +0200 Subject: [PATCH 1/3] Fix stream downloads --- lib/s3storage.php | 128 +++++++++++++++++++++++++++++----------------- 1 file changed, 81 insertions(+), 47 deletions(-) diff --git a/lib/s3storage.php b/lib/s3storage.php index a3baef3f..6fc1cf4e 100644 --- a/lib/s3storage.php +++ b/lib/s3storage.php @@ -31,6 +31,7 @@ use Aws\S3\Exception\S3Exception; use Aws\S3\ObjectUploader; use Aws\S3\S3Client; +use GuzzleHttp\Handler\StreamHandler; use GuzzleHttp\Handler\CurlMultiHandler; use GuzzleHttp\Middleware; use OC\ServiceUnavailableException; @@ -49,6 +50,9 @@ class S3Storage implements IObjectStore, IVersionedObjectStorage { */ private $connection; + /** @var S3Client|null */ + private $downConnection; + /** * @var array */ @@ -88,57 +92,20 @@ protected function init(): void { } $config = $this->params['options']; if ($useGuzzle5) { - /* - * Note: phan runs in CI with the latest core, which has Guzzle7 or later. - * So various things that phan reports for this Guzzle5 code have to be suppressed. - */ - /* @phan-suppress-next-line PhanUndeclaredClassMethod */ - $client = new \GuzzleHttp\Client(['handler' => new \GuzzleHttp\Ring\Client\CurlMultiHandler()]); - /* @phan-suppress-next-line PhanDeprecatedFunction */ - $emitter = $client->getEmitter(); - /* @phan-suppress-next-line PhanUndeclaredTypeParameter */ - $emitter->on('before', static function (\GuzzleHttp\Event\BeforeEvent $event) { - /* @phan-suppress-next-line PhanUndeclaredClassMethod */ - $request = $event->getRequest(); - if ($request->getMethod() !== 'PUT') { - return; - } - $body = $request->getBody(); - if ($body !== null && $body->getSize() !== 0) { - return; - } - if ($request->hasHeader('Content-Length')) { - return; - } - // force content length header on empty body - $request->setHeader('Content-Length', '0'); - }); - $h = new \Aws\Handler\GuzzleV5\GuzzleHandler($client); + $h = $this->getHandlerV5(false); // curlMultiHandler + $dh = $this->getHandlerV5(true); // streamHandler for downloads } else { - // Create a handler stack that has all of the default middlewares attached - $handler = \GuzzleHttp\HandlerStack::create(new CurlMultiHandler()); - // Push the handler onto the handler stack - $handler->push(Middleware::mapRequest(function (RequestInterface $request) { - if ($request->getMethod() !== 'PUT') { - return $request; - } - $body = $request->getBody(); - if ($body !== null && $body->getSize() !== 0) { - return $request; - } - if ($request->hasHeader('Content-Length')) { - return $request; - } - // force content length header on empty body - return $request->withHeader('Content-Length', '0'); - })); - // Inject the handler into the client - $client = new \GuzzleHttp\Client(['handler' => $handler]); - $h = new GuzzleHandler($client); + $h = $this->getHandlerV7(false); // curlMultiHandler + $dh = $this->getHandlerV7(true); // streamHandler for downloads } $config['http_handler'] = $h; /* @phan-suppress-next-line PhanDeprecatedFunction */ $this->connection = S3Client::factory($config); + + // replace the http_handler for the download connection + $config['http_handler'] = $dh; + /* @phan-suppress-next-line PhanDeprecatedFunction */ + $this->downConnection = S3Client::factory($config); try { $this->connection->listBuckets(); } catch (S3Exception $exception) { @@ -156,6 +123,73 @@ protected function init(): void { } } + private function getHandlerV5($isStream) { + /* + * Note: phan runs in CI with the latest core, which has Guzzle7 or later. + * So various things that phan reports for this Guzzle5 code have to be suppressed. + */ + if ($isStream) { + /* @phan-suppress-next-line PhanUndeclaredClassMethod */ + $client = new \GuzzleHttp\Client(['handler' => new \GuzzleHttp\Ring\Client\StreamHandler()]); + } else { + /* @phan-suppress-next-line PhanUndeclaredClassMethod */ + $client = new \GuzzleHttp\Client(['handler' => new \GuzzleHttp\Ring\Client\CurlMultiHandler()]); + } + + /* @phan-suppress-next-line PhanDeprecatedFunction */ + $emitter = $client->getEmitter(); + /* @phan-suppress-next-line PhanUndeclaredTypeParameter */ + $beforeEventFunc = static function (\GuzzleHttp\Event\BeforeEvent $event) { + /* @phan-suppress-next-line PhanUndeclaredClassMethod */ + $request = $event->getRequest(); + if ($request->getMethod() !== 'PUT') { + return; + } + $body = $request->getBody(); + if ($body !== null && $body->getSize() !== 0) { + return; + } + if ($request->hasHeader('Content-Length')) { + return; + } + // force content length header on empty body + $request->setHeader('Content-Length', '0'); + }; + $emitter->on('before', $beforeEventFunc); + $h = new \Aws\Handler\GuzzleV5\GuzzleHandler($client); + return $h; + } + + private function getHandlerV7($isStream) { + // Create a handler stack that has all of the default middlewares attached + if ($isStream) { + $handler = \GuzzleHttp\HandlerStack::create(new StreamHandler()); + } else { + $handler = \GuzzleHttp\HandlerStack::create(new CurlMultiHandler()); + } + + $requestFunc = function (RequestInterface $request) { + if ($request->getMethod() !== 'PUT') { + return $request; + } + $body = $request->getBody(); + if ($body !== null && $body->getSize() !== 0) { + return $request; + } + if ($request->hasHeader('Content-Length')) { + return $request; + } + // force content length header on empty body + return $request->withHeader('Content-Length', '0'); + }; + // Push the handler onto the handler stack + $handler->push(Middleware::mapRequest($requestFunc)); + // Inject the handler into the client + $client = new \GuzzleHttp\Client(['handler' => $handler]); + $h = new GuzzleHandler($client); + return $h; + } + /** * {@inheritDoc} */ @@ -228,7 +262,7 @@ public function readObject($urn) { $this->init(); try { $context = stream_context_create([ - 's3' => ['seekable' => true] + 's3' => ['seekable' => true, 'client' => $this->downConnection] ]); return \fopen($this->getUrl($urn), 'rb', false, $context); } catch (AwsException $ex) { From ab29b6660bd4408cad6cd3e886af5aeb89c04790 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Pablo=20Villaf=C3=A1=C3=B1ez?= Date: Fri, 21 Oct 2022 10:59:56 +0200 Subject: [PATCH 2/3] Fix phpstan with the class not found --- phpstan.neon | 1 + 1 file changed, 1 insertion(+) diff --git a/phpstan.neon b/phpstan.neon index 3a6e75d6..ae9906d6 100644 --- a/phpstan.neon +++ b/phpstan.neon @@ -6,6 +6,7 @@ parameters: - '#Method OCA\\Files_Primary_S3\\Panels\\Admin::getPanel\(\) should return OCP\\AppFramework\\Http\\TemplateResponse|OCP\\Template but returns null.#' - '#Comparison operation ">=" between 7 and 7 is always true.#' - '#Call to an undefined method GuzzleHttp\\Client::getEmitter\(\).#' + - '#Instantiated class GuzzleHttp\\Ring\\Client\\StreamHandler not found.#' - '#Instantiated class GuzzleHttp\\Ring\\Client\\CurlMultiHandler not found.#' - '#Parameter \$event of anonymous function has invalid typehint type GuzzleHttp\\Event\\BeforeEvent.#' - '#Call to method getRequest\(\) on an unknown class GuzzleHttp\\Event\\BeforeEvent.#' From 5cf1f97ca6da84b86f4bd5ded743680ce957af61 Mon Sep 17 00:00:00 2001 From: Kiran Parajuli Date: Fri, 14 Oct 2022 17:16:40 +0545 Subject: [PATCH 3/3] Bump oC10 server pins from 10.10.0-qa to 10.11.0-qa Signed-off-by: Kiran Parajuli --- .drone.star | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.drone.star b/.drone.star index 620b4715..04b27557 100644 --- a/.drone.star +++ b/.drone.star @@ -55,7 +55,7 @@ config = { ], "servers": [ "daily-master-qa", - "10.10.0-qa", + "10.11.0-qa", ], "databases": [ "sqlite", @@ -80,7 +80,7 @@ config = { ], "servers": [ "daily-master-qa", - "10.10.0-qa", + "10.11.0-qa", ], "scalityS3": { "config": "multibucket", @@ -103,7 +103,7 @@ config = { ], "servers": [ "daily-master-qa", - "10.10.0-qa", + "10.11.0-qa", ], "cephS3": True, "includeKeyInMatrixName": True,