-
Notifications
You must be signed in to change notification settings - Fork 63
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Added MinIO Storage adapter #51
base: main
Are you sure you want to change the base?
Changes from 1 commit
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change | ||||
---|---|---|---|---|---|---|
@@ -0,0 +1,119 @@ | ||||||
<?php | ||||||
|
||||||
namespace Utopia\Storage\Device; | ||||||
|
||||||
use Utopia\Storage\Device\S3; | ||||||
|
||||||
class MinIO extends S3 | ||||||
{ | ||||||
/** | ||||||
* Regions constants | ||||||
* | ||||||
*/ | ||||||
const EU_CENTRAL_1='eu-central-1'; | ||||||
const US_SOUTHEAST_1='us-southeast-1'; | ||||||
const US_EAST_1='us-east-1'; | ||||||
const AP_SOUTH_1='ap-south-1'; | ||||||
|
||||||
/** | ||||||
* Object Storage Constructor | ||||||
* | ||||||
* @param string $root | ||||||
* @param string $accessKey | ||||||
* @param string $secretKey | ||||||
* @param string $protocol | ||||||
* @param string $host | ||||||
* @param string $bucket | ||||||
* @param string $region | ||||||
* @param string $acl | ||||||
*/ | ||||||
public function __construct(string $root, string $accessKey, string $secretKey, string $protocol, string $host, string $bucket, string $region = self::EU_CENTRAL_1, string $acl = self::ACL_PRIVATE) | ||||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Suggested change
https://github.com/minio/minio-js/blob/master/src/main/minio.js#L70 There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Hi @shimonewman, thanks for the review. The problem is that in my setup MinIO is in a There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I think it might be better to adapt all adapters to use and endpoint instead of protocol and hostname for more flexibility. Hardcoding There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Would leave it for now and make it in another Pull Request There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. @groschi24, please create another PR with the proposed changes to the endpoint. |
||||||
{ | ||||||
parent::__construct($root, $accessKey, $secretKey, $bucket, $region, $acl); | ||||||
|
||||||
$this->protocol = $protocol; | ||||||
groschi24 marked this conversation as resolved.
Show resolved
Hide resolved
|
||||||
$this->headers['host'] = $host; | ||||||
} | ||||||
|
||||||
/** | ||||||
* Get list of objects in the given path. | ||||||
* | ||||||
* @param string $path | ||||||
* | ||||||
* @throws \Exception | ||||||
* | ||||||
* @return array | ||||||
*/ | ||||||
public function listObjects($prefix = '', $maxKeys = 1000, $continuationToken = '') | ||||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Suggested change
Please change S3 listObjects() access modifiers to protected in order for you to override it. |
||||||
{ | ||||||
$uri = '/' . $this->getRoot(); | ||||||
$this->headers['content-type'] = 'text/plain'; | ||||||
$this->headers['content-md5'] = \base64_encode(md5('', true)); | ||||||
|
||||||
$parameters = [ | ||||||
'list-type' => 2, | ||||||
'prefix' => $prefix, | ||||||
'max-keys' => $maxKeys, | ||||||
]; | ||||||
if(!empty($continuationToken)) { | ||||||
$parameters['continuation-token'] = $continuationToken; | ||||||
} | ||||||
$response = parent::call(self::METHOD_GET, $uri, '', $parameters); | ||||||
return $response->body; | ||||||
} | ||||||
|
||||||
/** | ||||||
* Delete files in given path, path must be a directory. Return true on success and false on failure. | ||||||
* | ||||||
* @param string $path | ||||||
* | ||||||
* @throws \Exception | ||||||
* | ||||||
* @return bool | ||||||
*/ | ||||||
public function deletePath(string $path): bool | ||||||
{ | ||||||
$uri = '/' . $this->getRoot(); | ||||||
$continuationToken = ''; | ||||||
do { | ||||||
$objects = $this->listObjects($path, continuationToken: $continuationToken); | ||||||
$count = (int) ($objects['KeyCount'] ?? 1); | ||||||
if($count < 1) { | ||||||
break; | ||||||
} | ||||||
$continuationToken = $objects['NextContinuationToken'] ?? ''; | ||||||
$body = '<Delete xmlns="http://s3.amazonaws.com/doc/2006-03-01/">'; | ||||||
|
||||||
if($count > 1) { | ||||||
foreach ($objects['Contents'] as $object) { | ||||||
$body .= "<Object><Key>{$object['Key']}</Key></Object>"; | ||||||
} | ||||||
} else { | ||||||
$body .= "<Object><Key>{$objects['Contents']['Key']}</Key></Object>"; | ||||||
} | ||||||
$body .= '<Quiet>true</Quiet>'; | ||||||
$body .= '</Delete>'; | ||||||
$this->amzHeaders['x-amz-content-sha256'] = \hash('sha256', $body); | ||||||
$this->headers['content-md5'] = \base64_encode(md5($body, true)); | ||||||
parent::call(self::METHOD_POST, $uri, $body, ['delete'=>'']); | ||||||
} while(!empty($continuationToken)); | ||||||
|
||||||
return true; | ||||||
} | ||||||
|
||||||
/** | ||||||
* @return string | ||||||
*/ | ||||||
public function getName(): string | ||||||
{ | ||||||
return 'MinIO Object Storage'; | ||||||
} | ||||||
|
||||||
/** | ||||||
* @return string | ||||||
*/ | ||||||
public function getDescription(): string | ||||||
{ | ||||||
return 'MinIO Object Storage'; | ||||||
} | ||||||
} |
Original file line number | Diff line number | Diff line change | ||||
---|---|---|---|---|---|---|
|
@@ -116,6 +116,7 @@ public function __construct(string $root, string $accessKey, string $secretKey, | |||||
$this->acl = $acl; | ||||||
$this->headers['host'] = $this->bucket . '.s3.'.$this->region.'.amazonaws.com'; | ||||||
$this->amzHeaders = []; | ||||||
$this->protocol = 'https'; | ||||||
} | ||||||
|
||||||
/** | ||||||
|
@@ -659,9 +660,9 @@ private function getSignatureV4(string $method, string $uri, array $parameters = | |||||
* | ||||||
* @return object | ||||||
*/ | ||||||
private function call(string $method, string $uri, string $data = '', array $parameters=[]) | ||||||
public function call(string $method, string $uri, string $data = '', array $parameters=[]) | ||||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Suggested change
Please avoid changing the access modifiers |
||||||
{ | ||||||
$url = 'https://' . $this->headers['host'] . $uri . '?' . \http_build_query($parameters, '', '&', PHP_QUERY_RFC3986); | ||||||
$url = $this->protocol . '://' . $this->headers['host'] . $uri . '?' . \http_build_query($parameters, '', '&', PHP_QUERY_RFC3986); | ||||||
$response = new \stdClass; | ||||||
$response->body = ''; | ||||||
$response->headers = []; | ||||||
|
Original file line number | Diff line number | Diff line change | ||||
---|---|---|---|---|---|---|
@@ -0,0 +1,32 @@ | ||||||
<?php | ||||||
|
||||||
namespace Utopia\Tests; | ||||||
|
||||||
use Utopia\Storage\Device\MinIO; | ||||||
use Utopia\Tests\S3Base; | ||||||
|
||||||
class MinIOTest extends S3Base | ||||||
{ | ||||||
protected function init(): void | ||||||
{ | ||||||
$this->root = 'minio-test-bucket'; | ||||||
$key = $_SERVER['MINIO_ACCESS_KEY'] ?? ''; | ||||||
$secret = $_SERVER['MINIO_SECRET'] ?? ''; | ||||||
$protocol = $_SERVER['MINIO_PROTOCOL'] ?? ''; | ||||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Suggested change
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Protocol has to be here, because MinIO can be hosted in an internal Network, where you don't have SSL |
||||||
$host = $_SERVER['MINIO_HOST'] ?? ''; | ||||||
$bucket = 'minio-test-bucket'; | ||||||
|
||||||
$this->object = new MinIO($this->root, $key, $secret, $protocol, $host, $bucket, MinIO::EU_CENTRAL_1); | ||||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Suggested change
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Protocol has to be here, because MinIO can be hosted in an internal Network, where you don't have SSL |
||||||
|
||||||
} | ||||||
|
||||||
protected function getAdapterName(): string | ||||||
{ | ||||||
return 'MinIO Object Storage'; | ||||||
} | ||||||
|
||||||
protected function getAdapterDescription(): string | ||||||
{ | ||||||
return 'MinIO Object Storage'; | ||||||
} | ||||||
} |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Protocol has to be here, because MinIO can be hosted in an internal Network, where you don't have SSL