TA#66782 [16.0][MIG] attachment_minio (#152)

Co-authored-by: Abdellatif Benzbiria <[email protected]>

attachment_minio/README.md

@@ -0,0 +1,70 @@
+# Use MinIO (or Amazon S3) for Attachment/filestore
+
+MinIO provides S3 API compatible storage to scale out without a shared filesystem like NFS.
+
+This module will store the bucket used in the attachment database object, thus allowing 
+you to retain read-only access to the filestore by simply overriding the bucket.
+
+## Setup details
+
+Before installing this app, you should add several System Parameters (the most important of
+which is `ir_attachment.location`), OR set them through the config file as described later.
+
+**The in database System Parameters will act as overrides to the Config File versions.**
+
+| Key                               | Example Value | Default Value |
+|-----------------------------------|---------------|---------------|
+| ir_attachment.location            | s3            |               |
+| ir_attachment.location.host       | minio:9000    |               |
+| ir_attachment.location.bucket     | odoo          |               |
+| ir_attachment.location.region     | us-west-1     | us-west-1     |
+| ir_attachment.location.access_key | minio         |               |
+| ir_attachment.location.secret_key | minio_secret  |               |
+| ir_attachment.location.secure     | 1             |               |
+
+**Config File:**
+
+```
+attachment_minio_host = minio:9000
+attachment_minio_region = us-west-1
+attachment_minio_access_key = minio
+attachment_minio_secret_key = minio_secret
+attachment_minio_bucket = odoo
+attachment_minio_secure = False
+```
+
+In general, they should all be specified other than "region" (if you are not using AWS S3) 
+and "secure" which should be set if the "host" needs to be accessed over SSL/TLS.
+
+Install `attachment_minio` and during the installation `base_attachment_object_storage` should move 
+your existing filestore attachment files into the database or object storage.
+
+For example, you can run a shell command like the following to set the parameter:
+
+```
+env['ir.config_parameter'].set_param('ir_attachment.location', 's3')
+# If already installed...
+# env['ir.attachment'].force_storage()
+env.cr.commit()
+```
+
+If `attachment_minio` is not already installed, you can then install it and the migration 
+should be noted in the logs.  **Ensure that the timeouts are long enough that the migration can finish.**
+
+### Base Setup
+
+This module utilizes `base_attachment_object_storage`
+
+The System Parameter `ir_attachment.storage.force.database` can be customized to 
+force storage of files in the database. See the documentation of the module 
+`base_attachment_object_storage`.
+
+Contributors
+------------
+* Camptocamp SA
+* Hibou Corp.
+* Numigi (tm) and all its contributors (https://bit.ly/numigiens)
+
+More information
+----------------
+* Meet us at https://bit.ly/numigi-com

attachment_minio/__init__.py

@@ -0,0 +1,6 @@
+# Copyright 2016 Camptocamp SA
+# Copyright 2020 Hibou Corp.
+# Copyright 2024-today Numigi and all its contributors (https://bit.ly/numigiens)
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
+
+from . import models

attachment_minio/__manifest__.py

@@ -0,0 +1,86 @@
+# Copyright 2016 Camptocamp SA
+# Copyright 2020 Hibou Corp.
+# Copyright 2024-today Numigi and all its contributors (https://bit.ly/numigiens)
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
+
+{
+    "name": "Attachment MinIO",
+    "version": "16.0.1.0.0",
+    "depends": [
+        "base_attachment_object_storage",
+    ],
+    "author": "Hibou Corp.",
+    "maintainer": "Numigi",
+    "license": "AGPL-3",
+    "description": """
+# Use MinIO (or Amazon S3) for Attachment/filestore
+
+MinIO provides S3 API compatible storage to scale out
+without a shared filesystem like NFS.
+
+This module will store the bucket used in the attachment database object, thus allowing
+you to retain read-only access to the filestore by simply overriding the bucket.
+
+## Setup details
+
+Before installing this app, you should add several System Parameters
+(the most important of which is `ir_attachment.location`),
+OR set them through the config file as described later.
+
+**The in database System Parameters will act as overrides to the Config File versions.**
+
+| Key                               | Example Value | Default Value |
+|-----------------------------------|---------------|---------------|
+| ir_attachment.location            | s3            |               |
+| ir_attachment.location.host       | minio:9000    |               |
+| ir_attachment.location.bucket     | odoo          |               |
+| ir_attachment.location.region     | us-west-1     | us-west-1     |
+| ir_attachment.location.access_key | minio         |               |
+| ir_attachment.location.secret_key | minio_secret  |               |
+| ir_attachment.location.secure     | 1             |               |
+
+**Config File:**
+
+```
+attachment_minio_host = minio:9000
+attachment_minio_region = us-west-1
+attachment_minio_access_key = minio
+attachment_minio_secret_key = minio_secret
+attachment_minio_bucket = odoo
+attachment_minio_secure = False
+```
+
+In general, they should all be specified other than "region"
+(if you are not using AWS S3)
+and "secure" which should be set if the "host" needs to be accessed over SSL/TLS.
+
+Install `attachment_minio` and during the installation `base_attachment_object_storage`
+should move your existing filestore attachment files into
+the database or object storage.
+
+For example, you can run a shell command like the following to set the parameter:
+
+```
+env['ir.config_parameter'].set_param('ir_attachment.location', 's3')
+# If already installed...
+# env['ir.attachment'].force_storage()
+env.cr.commit()
+```
+
+If `attachment_minio` is not already installed, you can then install it
+and the migration should be noted in the logs.
+**Ensure that the timeouts are long enough that the migration can finish.**
+
+    """,
+    "summary": "",
+    "website": "",
+    "category": "Tools",
+    "auto_install": False,
+    "installable": True,
+    "application": False,
+    "external_dependencies": {
+        "python": [
+            "minio",
+        ],
+    },
+}

attachment_minio/models/__init__.py

@@ -0,0 +1,6 @@
+# Copyright 2016 Camptocamp SA
+# Copyright 2020 Hibou Corp.
+# Copyright 2024-today Numigi and all its contributors (https://bit.ly/numigiens)
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
+
+from . import ir_attachment

attachment_minio/models/ir_attachment.py

@@ -0,0 +1,149 @@
+# Copyright 2016 Camptocamp SA
+# Copyright 2020 Hibou Corp.
+# Copyright 2024-today Numigi and all its contributors (https://bit.ly/numigiens)
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
+
+import io
+import logging
+from minio import Minio
+from minio import S3Error
+from odoo import api, exceptions, models, tools
+from ..s3uri import S3Uri
+
+_logger = logging.getLogger(__name__)
+
+
+class MinioAttachment(models.Model):
+    _inherit = "ir.attachment"
+
+    @api.model
+    def _get_minio_client(self):
+        config = tools.config
+        params = self.env["ir.config_parameter"].sudo()
+        host = params.get_param("ir_attachment.location.host") or config.get(
+            "attachment_minio_host"
+        )
+        region = params.get_param("ir_attachment.location.region") or config.get(
+            "attachment_minio_region", "us-west-1"
+        )
+        access_key = params.get_param(
+            "ir_attachment.location.access_key"
+        ) or config.get("attachment_minio_access_key")
+        secret_key = params.get_param(
+            "ir_attachment.location.secret_key"
+        ) or config.get("attachment_minio_secret_key")
+        secure = params.get_param("ir_attachment.location.secure") or config.get(
+            "attachment_minio_secure"
+        )
+        if not all((host, access_key, secret_key)):
+            raise exceptions.UserError("Incorrect configuration of attachment_minio.")
+        print("hostttttttttt", host)
+        return Minio(
+            host,
+            access_key=access_key,
+            secret_key=secret_key,
+            region=region,
+            secure=bool(secure),
+        )
+
+    @api.model
+    def _get_minio_bucket(self, client, name=None):
+        config = tools.config
+        params = self.env["ir.config_parameter"].sudo()
+        bucket = (
+            name
+            or params.get_param("ir_attachment.location.bucket")
+            or config.get("attachment_minio_bucket")
+        )
+        if not bucket:
+            raise exceptions.UserError(
+                "Incorrect configuration of attachment_minio -- Missing bucket."
+            )
+        if not client.bucket_exists(bucket):
+            region = (
+                params.get_param("ir_attachment.location.region", "us-west-1")
+                or config.get("attachment_minio_region", "us-west-1"))
+            client.make_bucket(bucket, region)
+        return bucket
+
+    @api.model
+    def _get_minio_key(self, sha):
+        # scatter files across 256 dirs
+        # This mirrors Odoo's own object storage so that it is easier to migrate
+        # to or from external storage.
+        fname = sha[:2] + "/" + sha
+        return fname
+
+    @api.model
+    def _get_minio_fname(self, bucket, key):
+        return "s3://%s/%s" % (bucket, key)
+
+    # core API methods from base_attachment_object_storage
+
+    def _get_stores(self):
+        res = super(MinioAttachment, self)._get_stores()
+        res.append("s3")
+        return res
+
+    @api.model
+    def _store_file_read(self, fname, bin_size=False):
+        if fname.startswith("s3://"):
+            client = self._get_minio_client()
+            s3uri = S3Uri(fname)
+            bucket = self._get_minio_bucket(client, name=s3uri.bucket())
+            try:
+                response = client.get_object(bucket, s3uri.item())
+                return response.read()
+            except S3Error as e:
+                if e.code == "NoSuchKey":
+                    _logger.info(
+                        'attachment "%s" missing from remote object storage', fname
+                    )
+                else:
+                    raise
+            return ""
+        return super(MinioAttachment, self)._store_file_read(fname, bin_size=bin_size)
+
+    @api.model
+    def _store_file_write(self, key, bin_data):
+        if self._storage() == "s3":
+            client = self._get_minio_client()
+            bucket = self._get_minio_bucket(client)
+            minio_key = self._get_minio_key(key)
+            with io.BytesIO(bin_data) as bin_data_io:
+                client.put_object(
+                    bucket,
+                    minio_key,
+                    bin_data_io,
+                    len(bin_data),
+                    content_type=self.mimetype,
+                )
+            return self._get_minio_fname(bucket, minio_key)
+        return super(MinioAttachment, self)._store_file_write(key, bin_data)
+
+    @api.model
+    def _store_file_delete(self, fname):
+        if fname.startswith("s3://"):
+            client = self._get_minio_client()
+            try:
+                s3uri = S3Uri(fname)
+            except ValueError:
+                # Cannot delete unparsable file
+                return True
+            bucket_name = s3uri.bucket()
+            if bucket_name == self._get_minio_bucket(client):
+                try:
+                    client.remove_object(bucket_name, s3uri.item())
+                except S3Error as e:
+                    if e.code == "NoSuchKey":
+                        _logger.info(
+                            'unable to remove missing attachment "%s" '
+                            "from remote object storage",
+                            fname,
+                        )
+                    else:
+                        raise
+            else:
+                _logger.info('skip delete "%s" because of bucket-mismatch', (fname,))
+            return
+        return super(MinioAttachment, self)._store_file_delete(fname)

attachment_minio/s3uri.py

@@ -0,0 +1,23 @@
+# Copyright 2016 Camptocamp SA
+# Copyright 2020 Hibou Corp.
+# Copyright 2024-today Numigi and all its contributors (https://bit.ly/numigiens)
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
+
+import re
+
+
+class S3Uri(object):
+
+    _url_re = re.compile("^s3:///*([^/]*)/?(.*)", re.IGNORECASE | re.UNICODE)
+
+    def __init__(self, uri):
+        match = self._url_re.match(uri)
+        if not match:
+            raise ValueError("%s: is not a valid S3 URI" % (uri,))
+        self._bucket, self._item = match.groups()
+
+    def bucket(self):
+        return self._bucket
+
+    def item(self):
+        return self._item

gitoo.yml

@@ -1,3 +1,7 @@
+- url: https://github.com/camptocamp/odoo-cloud-platform
+  branch: "16.0"
+  includes: ["base_attachment_object_storage"]
+
 - url: https://github.com/OCA/helpdesk
   branch: "16.0"
   includes: ["helpdesk_mgmt"]