Fix broken blobs in documents

castle/cms/browser/content/fc.py

@@ -37,7 +37,10 @@
 from zope.annotation.interfaces import IAnnotations
 from persistent.mapping import PersistentMapping
 from plone.namedfile.file import NamedBlobImage
-from ZODB.POSException import POSKeyError
+from ZODB.blob import FilesystemHelper
+from pkg_resources import resource_filename
+from shutil import copyfile
+import os
 from re import sub
 
 from plone.app.content.browser.vocabulary import (
@@ -190,59 +193,73 @@ def get_options(self):
 
 class PasteAsyncActionView(actions.ObjectPasteView):
 
+    def create_empty_blob(self, filename):
+        dirname = os.path.split(filename)[0]
+        if not os.path.isdir(dirname):
+            os.makedirs(dirname, 0o700)
+
+        source = resource_filename(
+            'castle.cms',
+            'static/images/placeholder.png',
+        )
+
+        try:
+            copyfile(source, filename)
+        except Exception as e:
+            logger.error("Error creating replacement blob: {}".format(e))
+
+        logger.info("Created missing blob for: %s", filename)
+
+
     def __call__(self):
         try:
             paste_data = get_paste_data(self.request)
         except CopyError:
             return self.copy_error()
 
-        # XXX: Experimental code block
-        # Can be removed in lieu of installing "experimental.gracefulblobmissing" package,
-        # which ignores missing blob errors
-
         site = getSite()
         for path in paste_data.get('paths'):
-            print(path)
-
             try:
                 obj = site.restrictedTraverse(path.strip('/'), None)
-                print(obj)
                 if obj is None:
                     logger.error("Object not found: '{}'".format(path))
                     return
             except Exception as e:
                 logger.error("Error retrieving object: {}".format(e))
                 return
 
+        # Set up FilesystemHelper to handle broken blob references, if any
+        conn = self.context._p_jar
+        storage = conn._storage
+        fshelper = storage.fshelper
+        base_dir = fshelper.base_dir
+        zeofshelper = FilesystemHelper(base_dir)
+        broken_items = []
+
         if IDocument.providedBy(obj):
             annotations = IAnnotations(obj)
-
-            # The ANNOTATIONS_KEY_PREFIX ignores tiles with images/blobs
             for key in annotations.keys():
                 data = annotations[key]
-
-                # PersistentMapping tiles contain the blobs we want
+                
+                # PersistentMapping tiles contain the blobs we need
                 if isinstance(data, PersistentMapping):
                     for item in data.values():
                         if isinstance(item, dict) and isinstance(item.get('data'), NamedBlobImage):
                             blobfile = item.get('data')
                             blob = blobfile._blob
                             try:
-                                # Check blob, opening may not be best approach
-                                f = blob.open('r')
-                                f.close()
-                            except POSKeyError:
+                                # Check blob
+                                blob._p_activate()
+                            except Exception as e:
                                 # Broken blob reference
-                                # XXX: Call save() on blob or reassign to generic fallback blob
-                                path_on_disk = blob._p_blob_committed
-                                pass
-
-        # XXX: end of block
-
+                                broken_items.append('/'.join(obj.getPhysicalPath()))
+                                filename = zeofshelper.getBlobFilename(blob._p_oid, blob._p_serial)
+                                if not os.path.exists(filename):
+                                    self.create_empty_blob(filename)     
 
         tasks.paste_items.delay(
             self.request.form['folder'], paste_data['op'],
-            paste_data['mdatas'])
+            paste_data['mdatas'], broken_items)
 
         return self.do_redirect(
             self.canonical_object_url,

castle/cms/tasks/content.py

@@ -39,7 +39,7 @@ def paste_error_handle(where, op, mdatas):
 
 
 @retriable(on_retry_exhausted=paste_error_handle)
-def _paste_items(where, op, mdatas):
+def _paste_items(where, op, mdatas, broken_items):
     logger.info('Copying a bunch of items')
     portal = api.portal.get()
     dest = portal.restrictedTraverse(str(where.lstrip('/')))
@@ -73,22 +73,40 @@ def _paste_items(where, op, mdatas):
     if email:
         name = user.getProperty('fullname') or user.getId()
         try:
-            utils.send_email(
-                recipients=email,
-                subject="Paste Operation Finished(Site: %s)" % (
+            subject="Paste Operation Finished(Site: %s)" % (
                     api.portal.get_registry_record('plone.site_title')),
-                html="""
+            html="""
     <p>Hi %s,</p>
 
     <p>The site has finished pasting items into /%s folder.</p>""" % (
-                    name, where.lstrip('/')))
+                    name, where.lstrip('/'))
+            if len(broken_items) > 0:
+                html+= """
+    <br />
+    <p>However, we detected broken file references
+        during the copy process. These references 
+        have been replaced with a blank placeholder image.
+    </p>
+    <br />
+    <p>Below are the pages containing these affected items:
+    </p>
+    <br />"""
+                for path in broken_items:
+                    html +="""
+    <p>%s</p>""" % (path)
+            utils.send_email(
+                recipients=email,
+                subject=subject,
+                html=html
+            )
+
         except Exception:
             logger.warn('Could not send status email ', exc_info=True)
 
 
 @task()
-def paste_items(where, op, mdatas):
-    _paste_items(where, op, mdatas)
+def paste_items(where, op, mdatas, broken_items=[]):
+    _paste_items(where, op, mdatas, broken_items)
 
 
 def delete_error_handle(where, op, mdatas):

travis.cfg

@@ -38,7 +38,6 @@ eggs +=
     Pillow
     Products.PloneKeywordManager
     plone.app.robotframework
-    experimental.gracefulblobmissing
 
 zcml =
     castle.cms-overrides

versions.cfg

@@ -57,8 +57,6 @@ z3c.jbot = 0.7.2
 castle.theme = 1.0.6
 Products.PloneKeywordManager = 2.2.1
 wildcard.hps = 1.4.0
-experimental.gracefulblobmissing = 2.0
-
 
 # dependency pins
 argon2_cffi = 16.3.0