Partial support for diskcache-style sconsign

This is a low-touch conversion to use an sqlite backend instead
of a flat "dblite" file for the sconsign database, without really
changing SCons logic - but includes some hacks where rather than full
detection, things are just commented out.

New SCons/sdiskcache.py module supports diskcache usage.

Definitely not ready to merge.

diskcache: https://pypi.org/project/diskcache/ and
https://grantjenks.com/docs/diskcache/

Experimental: with a --convert flag, sconsign utility will read existing
sconsign file and set up a sqlite-backed Diskcache.

=========================
TODO on the testing front:

test/Repository: fiddling the "writable" attribute in the framework
  doesn't help, sqlite db opened that way always unwritable,
  Get: OperationalError : attempt to write a readonly database
  Affects:
    test/Repository/Install-Local.py
    test/Repository/Local.py
    test/Repository/SharedLibrary.py
    test/Repository/StaticLibrary.py
    test/Repository/VariantDir.py
    test/Repository/link-object.py
    test/Repository/option-n.py
    test/Repository/variants.py

test/Configure/VariantDir-SConscript.py:
test/Configure/implicit-cache.py: hardcodes .dblite as the DB suffix.
test/SConsignFile/default.py: hardcodes .dblite as the DB suffix
test/SConsignFile/explicit-file.py: hardcodes .dblite as the DB suffix
test/option/hash-format.py: why does this do an assert, anyway?
  e.g. Missing files: `/tmp/scons/testcmd.305909.0g2s8ny7/.sconsign.dblite'

test/sconsign/corrupt.py: hardcodes .dblite
  Since it's not a flat file, corrupt is different. Expose Cache.check()?

test/sconsign/script/Configure.py: currently hitting unpickled data
test/sconsign/script/SConsignFile.py: currently hitting unpicked data
  Note some of the "hardcoded" tests would

test/sconsign/script/dblite.py: possibly should just not run,
  but maybe add parallel one for sdiskcache

test/option/option-n.py: see FW
testing/framework/TestSCons.py: get_sconsignname, unlink_sconsignfile
  don't work for sqlite db

Some tests should use unlink_sconsignfile (possibly renamed) that don't.
From a grep:
  test/Configure/VariantDir-SConscript.py
  test/MSVS/vs-10.0-exec.py
  test/MSVS/vs-10.0Exp-exec.py
  test/MSVS/vs-11.0-exec.py
  test/MSVS/vs-11.0Exp-exec.py
  test/MSVS/vs-14.0-exec.py
  test/MSVS/vs-14.0Exp-exec.py
  test/MSVS/vs-14.1-exec.py
  test/MSVS/vs-14.2-exec.py
  test/MSVS/vs-14.3-exec.py
  test/MSVS/vs-6.0-exec.py
  test/MSVS/vs-7.0-exec.py
  test/MSVS/vs-7.1-exec.py
  test/MSVS/vs-8.0-exec.py
  test/MSVS/vs-8.0Exp-exec.py
  test/MSVS/vs-9.0-exec.py
  test/MSVS/vs-9.0Exp-exec.py
  test/option/option-n.py
  test/option/hash-format.py

test/SConsignFile - Check - may need work, or may be unique to file-based.
   may need equivalent one for DB versions?

Signed-off-by: Mats Wichmann <[email protected]>

SCons/SConsign.py

@@ -30,10 +30,12 @@
 import time
 
 import SCons.dblite
+import SCons.sdiskcache
 import SCons.Warnings
 from SCons.compat import PICKLE_PROTOCOL
 from SCons.Util import print_time
 
+DEBUG = False
 
 def corrupt_dblite_warning(filename) -> None:
     SCons.Warnings.warn(
@@ -54,8 +56,9 @@ def corrupt_dblite_warning(filename) -> None:
 # "DB_Name" is the base name of the database file (minus any
 # extension the underlying DB module will add).
 DataBase = {}
-DB_Module = SCons.dblite
-DB_Name = None
+#DB_Module = SCons.dblite
+DB_Module = SCons.sdiskcache
+DB_Name = ".sconsign"
 DB_sync_list = []
 
 def current_sconsign_filename():
@@ -71,6 +74,17 @@ def current_sconsign_filename():
         return ".sconsign"
     return ".sconsign_" + current_hash_algorithm
 
+# a debugging decorator, not for production usage
+def dbwrap(func):
+    def wrapper(dnode):
+        db, mode = func(dnode)
+        if DEBUG:
+            print(f"Get_DataBase returns {db=}, {mode=}")
+        return db, mode
+
+    return wrapper
+
+@dbwrap
 def Get_DataBase(dir):
     global DB_Name
 
@@ -86,7 +100,8 @@ def Get_DataBase(dir):
                     return DataBase[d], mode
                 except KeyError:
                     path = d.entry_abspath(DB_Name)
-                    try: db = DataBase[d] = DB_Module.open(path, mode)
+                    try:
+                        db = DataBase[d] = DB_Module.open(path, mode)
                     except OSError:
                         pass
                     else:
@@ -126,13 +141,13 @@ def write() -> None:
         try:
             syncmethod = db.sync
         except AttributeError:
-            pass # Not all dbm modules have sync() methods.
+            pass  # Not all dbm modules have sync() methods.
         else:
             syncmethod()
         try:
             closemethod = db.close
         except AttributeError:
-            pass # Not all dbm modules have close() methods.
+            pass  # Not all dbm modules have close() methods.
         else:
             closemethod()
 
@@ -142,20 +157,21 @@ def write() -> None:
 
 
 class SConsignEntry:
-    """
-    Wrapper class for the generic entry in a .sconsign file.
+    """Wrapper class for the generic entry in an sconsign file.
+
     The Node subclass populates it with attributes as it pleases.
 
     XXX As coded below, we do expect a '.binfo' attribute to be added,
     but we'll probably generalize this in the next refactorings.
     """
+
     __slots__ = ("binfo", "ninfo", "__weakref__")
     current_version_id = 2
 
     def __init__(self) -> None:
         # Create an object attribute from the class attribute so it ends up
         # in the pickled data in the .sconsign file.
-        #_version_id = self.current_version_id
+        # _version_id = self.current_version_id
         pass
 
     def convert_to_sconsign(self) -> None:
@@ -199,15 +215,11 @@ def __init__(self) -> None:
         self.to_be_merged = {}
 
     def get_entry(self, filename):
-        """
-        Fetch the specified entry attribute.
-        """
+        """Fetch the specified entry attribute."""
         return self.entries[filename]
 
     def set_entry(self, filename, obj) -> None:
-        """
-        Set the entry.
-        """
+        """Set the entry."""
         self.entries[filename] = obj
         self.dirty = True
 
@@ -261,16 +273,21 @@ def __init__(self, dir) -> None:
         except KeyError:
             pass
         else:
-            try:
-                self.entries = pickle.loads(rawentries)
-                if not isinstance(self.entries, dict):
-                    self.entries = {}
-                    raise TypeError
-            except KeyboardInterrupt:
-                raise
-            except Exception as e:
-                SCons.Warnings.warn(SCons.Warnings.CorruptSConsignWarning,
-                                    "Ignoring corrupt sconsign entry : %s (%s)\n"%(self.dir.get_tpath(), e))
+            if DEBUG:
+                print(f"rawentries is a {type(rawentries)}")
+            if isinstance(rawentries, dict):
+                self.entries = rawentries
+            else:
+                try:
+                    self.entries = pickle.loads(rawentries)
+                    if not isinstance(self.entries, dict):
+                        self.entries = {}
+                        raise TypeError
+                except KeyboardInterrupt:
+                    raise
+                except Exception as e:
+                    SCons.Warnings.warn(SCons.Warnings.CorruptSConsignWarning,
+                                        "Ignoring corrupt sconsign entry : %s (%s)\n"%(self.dir.get_tpath(), e))
             for key, entry in self.entries.items():
                 entry.convert_from_sconsign(dir, key)
 
@@ -421,11 +438,9 @@ def write(self, sync: int=1) -> None:
 
 
 def File(name, dbm_module=None) -> None:
-    """
-    Arrange for all signatures to be stored in a global .sconsign.db*
-    file.
-    """
+    """Store the SCons signatures in a global .sconsign.* database."""
     global ForDirectory, DB_Name, DB_Module
+
     if name is None:
         ForDirectory = DirFile
         DB_Module = None

SCons/SConsignTests.py

@@ -300,13 +300,15 @@ def test_SConsignFile(self) -> None:
             assert SCons.SConsign.DB_Name == ".sconsign", SCons.SConsign.DB_Name
         else:
             assert SCons.SConsign.DB_Name == ".sconsign_{}".format(get_current_hash_algorithm_used()), SCons.SConsign.DB_Name
-        assert SCons.SConsign.DB_Module is SCons.dblite, SCons.SConsign.DB_Module
+        #assert SCons.SConsign.DB_Module is SCons.dblite, SCons.SConsign.DB_Module
+        assert SCons.SConsign.DB_Module is SCons.sdiskcache, SCons.SConsign.DB_Module
 
         SCons.SConsign.File(file)
 
         assert SCons.SConsign.DataBase == {}, SCons.SConsign.DataBase
         assert SCons.SConsign.DB_Name is file, SCons.SConsign.DB_Name
-        assert SCons.SConsign.DB_Module is SCons.dblite, SCons.SConsign.DB_Module
+        #assert SCons.SConsign.DB_Module is SCons.dblite, SCons.SConsign.DB_Module
+        assert SCons.SConsign.DB_Module is SCons.sdiskcache, SCons.SConsign.DB_Module
 
         SCons.SConsign.File(None)