diff --git a/SCons/SConsign.py b/SCons/SConsign.py index 1215c6a504..fce26c994e 100644 --- a/SCons/SConsign.py +++ b/SCons/SConsign.py @@ -30,10 +30,12 @@ import time import SCons.dblite +import SCons.sdiskcache import SCons.Warnings from SCons.compat import PICKLE_PROTOCOL from SCons.Util import print_time +DEBUG = False def corrupt_dblite_warning(filename) -> None: SCons.Warnings.warn( @@ -54,8 +56,9 @@ def corrupt_dblite_warning(filename) -> None: # "DB_Name" is the base name of the database file (minus any # extension the underlying DB module will add). DataBase = {} -DB_Module = SCons.dblite -DB_Name = None +#DB_Module = SCons.dblite +DB_Module = SCons.sdiskcache +DB_Name = ".sconsign" DB_sync_list = [] def current_sconsign_filename(): @@ -71,6 +74,17 @@ def current_sconsign_filename(): return ".sconsign" return ".sconsign_" + current_hash_algorithm +# a debugging decorator, not for production usage +def dbwrap(func): + def wrapper(dnode): + db, mode = func(dnode) + if DEBUG: + print(f"Get_DataBase returns {db=}, {mode=}") + return db, mode + + return wrapper + +@dbwrap def Get_DataBase(dir): global DB_Name @@ -86,7 +100,8 @@ def Get_DataBase(dir): return DataBase[d], mode except KeyError: path = d.entry_abspath(DB_Name) - try: db = DataBase[d] = DB_Module.open(path, mode) + try: + db = DataBase[d] = DB_Module.open(path, mode) except OSError: pass else: @@ -126,13 +141,13 @@ def write() -> None: try: syncmethod = db.sync except AttributeError: - pass # Not all dbm modules have sync() methods. + pass # Not all dbm modules have sync() methods. else: syncmethod() try: closemethod = db.close except AttributeError: - pass # Not all dbm modules have close() methods. + pass # Not all dbm modules have close() methods. else: closemethod() @@ -142,20 +157,21 @@ def write() -> None: class SConsignEntry: - """ - Wrapper class for the generic entry in a .sconsign file. + """Wrapper class for the generic entry in an sconsign file. + The Node subclass populates it with attributes as it pleases. XXX As coded below, we do expect a '.binfo' attribute to be added, but we'll probably generalize this in the next refactorings. """ + __slots__ = ("binfo", "ninfo", "__weakref__") current_version_id = 2 def __init__(self) -> None: # Create an object attribute from the class attribute so it ends up # in the pickled data in the .sconsign file. - #_version_id = self.current_version_id + # _version_id = self.current_version_id pass def convert_to_sconsign(self) -> None: @@ -199,15 +215,11 @@ def __init__(self) -> None: self.to_be_merged = {} def get_entry(self, filename): - """ - Fetch the specified entry attribute. - """ + """Fetch the specified entry attribute.""" return self.entries[filename] def set_entry(self, filename, obj) -> None: - """ - Set the entry. - """ + """Set the entry.""" self.entries[filename] = obj self.dirty = True @@ -261,16 +273,21 @@ def __init__(self, dir) -> None: except KeyError: pass else: - try: - self.entries = pickle.loads(rawentries) - if not isinstance(self.entries, dict): - self.entries = {} - raise TypeError - except KeyboardInterrupt: - raise - except Exception as e: - SCons.Warnings.warn(SCons.Warnings.CorruptSConsignWarning, - "Ignoring corrupt sconsign entry : %s (%s)\n"%(self.dir.get_tpath(), e)) + if DEBUG: + print(f"rawentries is a {type(rawentries)}") + if isinstance(rawentries, dict): + self.entries = rawentries + else: + try: + self.entries = pickle.loads(rawentries) + if not isinstance(self.entries, dict): + self.entries = {} + raise TypeError + except KeyboardInterrupt: + raise + except Exception as e: + SCons.Warnings.warn(SCons.Warnings.CorruptSConsignWarning, + "Ignoring corrupt sconsign entry : %s (%s)\n"%(self.dir.get_tpath(), e)) for key, entry in self.entries.items(): entry.convert_from_sconsign(dir, key) @@ -421,11 +438,9 @@ def write(self, sync: int=1) -> None: def File(name, dbm_module=None) -> None: - """ - Arrange for all signatures to be stored in a global .sconsign.db* - file. - """ + """Store the SCons signatures in a global .sconsign.* database.""" global ForDirectory, DB_Name, DB_Module + if name is None: ForDirectory = DirFile DB_Module = None diff --git a/SCons/SConsignTests.py b/SCons/SConsignTests.py index 84bc2000ba..8539582faf 100644 --- a/SCons/SConsignTests.py +++ b/SCons/SConsignTests.py @@ -300,13 +300,15 @@ def test_SConsignFile(self) -> None: assert SCons.SConsign.DB_Name == ".sconsign", SCons.SConsign.DB_Name else: assert SCons.SConsign.DB_Name == ".sconsign_{}".format(get_current_hash_algorithm_used()), SCons.SConsign.DB_Name - assert SCons.SConsign.DB_Module is SCons.dblite, SCons.SConsign.DB_Module + #assert SCons.SConsign.DB_Module is SCons.dblite, SCons.SConsign.DB_Module + assert SCons.SConsign.DB_Module is SCons.sdiskcache, SCons.SConsign.DB_Module SCons.SConsign.File(file) assert SCons.SConsign.DataBase == {}, SCons.SConsign.DataBase assert SCons.SConsign.DB_Name is file, SCons.SConsign.DB_Name - assert SCons.SConsign.DB_Module is SCons.dblite, SCons.SConsign.DB_Module + #assert SCons.SConsign.DB_Module is SCons.dblite, SCons.SConsign.DB_Module + assert SCons.SConsign.DB_Module is SCons.sdiskcache, SCons.SConsign.DB_Module SCons.SConsign.File(None) diff --git a/SCons/Utilities/sconsign.py b/SCons/Utilities/sconsign.py index 4cef4779bd..f022dbdb0a 100644 --- a/SCons/Utilities/sconsign.py +++ b/SCons/Utilities/sconsign.py @@ -39,10 +39,19 @@ import SCons.compat import SCons.SConsign +DEBUG = False -def my_whichdb(filename): - if filename[-7:] == ".dblite": + +def my_whichdb(filename: str) -> str: + """Try to detect sconsign db flavor, and return the type. + + We have a couple of quick local heuristics, else fall back + to dbm.whichdb, which can only recognize Python stdlib types. + """ + if filename.endswith('.dblite'): return "SCons.dblite" + elif filename.endswith('.sqlite') or os.path.isdir(filename + ".sqlite"): + return "SCons.sdiskcache" try: with open(filename + ".dblite", "rb"): return "SCons.dblite" @@ -52,11 +61,11 @@ def my_whichdb(filename): class Flagger: - default_value = 1 + default_value = True def __setitem__(self, item, value) -> None: self.__dict__[item] = value - self.default_value = 0 + self.default_value = False def __getitem__(self, item): return self.__dict__.get(item, self.default_value) @@ -66,21 +75,18 @@ def __getitem__(self, item): Print_Directories = [] Print_Entries = [] Print_Flags = Flagger() -Verbose = 0 -Readable = 0 +Verbose = False +Readable = False Warns = 0 +Convert_To_Diskcache = False -def default_mapper(entry, name): - """ - Stringify an entry that doesn't have an explicit mapping. +def default_mapper(entry, name) -> str: + """Stringify an entry that doesn't have an explicit mapping. Args: entry: entry name: field name - - Returns: str - """ try: val = eval("entry." + name) @@ -89,16 +95,12 @@ def default_mapper(entry, name): return str(val) -def map_action(entry, _): - """ - Stringify an action entry and signature. +def map_action(entry, _) -> str: + """Stringify an action entry and signature. Args: entry: action entry second argument is not used - - Returns: str - """ try: bact = entry.bact @@ -108,16 +110,12 @@ def map_action(entry, _): return '%s [%s]' % (bactsig, bact) -def map_timestamp(entry, _): - """ - Stringify a timestamp entry. +def map_timestamp(entry, _) -> str: + """Stringify a timestamp entry. Args: entry: timestamp entry second argument is not used - - Returns: str - """ try: timestamp = entry.timestamp @@ -129,16 +127,12 @@ def map_timestamp(entry, _): return str(timestamp) -def map_bkids(entry, _): - """ - Stringify an implicit entry. +def map_bkids(entry, _) -> str: + """Stringify an implicit entry. Args: entry: second argument is not used - - Returns: str - """ try: bkids = entry.bsources + entry.bdepends + entry.bimplicit @@ -252,6 +246,7 @@ def printentries(entries, location) -> None: print(nodeinfo_string(name, entry.ninfo)) printfield(name, entry.binfo) else: + #print(f"{__file__}/printentries: {entries=}") for name in sorted(entries.keys()): entry = entries[name] try: @@ -264,14 +259,20 @@ def printentries(entries, location) -> None: class Do_SConsignDB: - def __init__(self, dbm_name, dbm) -> None: + def __init__(self, dbm_name, dbm_type, dbm) -> None: self.dbm_name = dbm_name + self.dbm_type = dbm_type + if DEBUG: + print(f"DEBUG: Do_SConsignDB.__init__({dbm_name}, {dbm_type}, dbm)") self.dbm = dbm def __call__(self, fname): # The *dbm modules stick their own file suffixes on the names - # that are passed in. This causes us to jump through some - # hoops here. + # that are passed in, but the diskcache scheme does not + # (although by convention it uses a directory suffixed .d) + # This causes us to jump through some # hoops here. + if DEBUG: + print(f"DEBUG: Do_SConsignDB.__call__({fname}), type is {self.dbm_type}") try: # Try opening the specified file name. Example: # SPECIFIED OPENED BY self.dbm.open() @@ -314,6 +315,19 @@ def __call__(self, fname): sys.stderr.write("unrecognized pickle protocol.\n") return + if Convert_To_Diskcache and self.dbm_type != 'SCons.sdiskcache': + import SCons.sdiskcache + if fname.endswith('.dblite'): + dirname = fname[:-len('.dblite')] + else: + dirname = fname + dirname = dirname + '.sqlite' + newdb = SCons.sdiskcache.open(dirname, flags='n') + for d in sorted(db.keys()): + newdb[d] = pickle.loads(db[d]) + print(f"Converted {self.dbm_name} '{fname}' to diskcache in '{dirname}'") + return + if Print_Directories: for dir in Print_Directories: try: @@ -327,13 +341,15 @@ def __call__(self, fname): for dir in sorted(db.keys()): self.printentries(dir, db[dir]) - @staticmethod - def printentries(dir, val) -> None: + def printentries(self, dir, val) -> None: try: print('=== ' + dir + ':') except TypeError: print('=== ' + dir.decode() + ':') - printentries(pickle.loads(val), dir) + try: + printentries(pickle.loads(val), dir) + except TypeError: + printentries(val, dir) def Do_SConsignDir(name): @@ -364,6 +380,7 @@ def main() -> None: global args global Verbose global Readable + global Convert_To_Diskcache helpstr = """\ Usage: sconsign [OPTIONS] [FILE ...] @@ -371,6 +388,7 @@ def main() -> None: Options: -a, --act, --action Print build action information. -c, --csig Print content signature information. + --convert Create diskcache version of database. -d DIR, --dir=DIR Print only info about DIR. -e ENTRY, --entry=ENTRY Print only info about ENTRY. -f FORMAT, --format=FORMAT FILE is in the specified FORMAT. @@ -382,7 +400,6 @@ def main() -> None: -t, --timestamp Print timestamp information. -v, --verbose Verbose, describe each field. """ - try: opts, args = getopt.getopt( sys.argv[1:], @@ -390,6 +407,7 @@ def main() -> None: [ 'act', 'action', + 'convert', 'csig', 'dir=', 'entry=', @@ -410,9 +428,11 @@ def main() -> None: for o, a in opts: if o in ('-a', '--act', '--action'): - Print_Flags['action'] = 1 + Print_Flags['action'] = True elif o in ('-c', '--csig'): - Print_Flags['csig'] = 1 + Print_Flags['csig'] = True + elif o in ('--convert'): + Convert_To_Diskcache = True elif o in ('-d', '--dir'): Print_Directories.append(a) elif o in ('-e', '--entry'): @@ -420,60 +440,74 @@ def main() -> None: elif o in ('-f', '--format'): # Try to map the given DB format to a known module # name, that we can then try to import... - Module_Map = {'dblite': 'SCons.dblite', 'sconsign': None} - dbm_name = Module_Map.get(a, a) - if dbm_name: + Module_Map = { + 'dblite': 'SCons.dblite', + 'diskcache': 'SCons.sdiskcache', + 'sconsign': None, + } + dbm_type = Module_Map.get(a, a) + if dbm_type: + if DEBUG: + print(f"DEBUG: asked for {a}, which is {dbm_type}") try: - if dbm_name != "SCons.dblite": - dbm = importlib.import_module(dbm_name) - else: - import SCons.dblite - - dbm = SCons.dblite - # Ensure that we don't ignore corrupt DB files, + if dbm_type == "SCons.dblite": + import SCons.dblite as dbm + # Ensure that we don't ignore corrupt DB files SCons.dblite.IGNORE_CORRUPT_DBFILES = False + elif dbm_type == "SCons.sdiskcache": + import SCons.sdiskcache as dbm + else: + dbm = importlib.import_module(dbm_name) except ImportError: sys.stderr.write("sconsign: illegal file format `%s'\n" % a) print(helpstr) sys.exit(2) - Do_Call = Do_SConsignDB(a, dbm) + Do_Call = Do_SConsignDB(a, dbm_type, dbm) else: + if DEBUG: + print(f"DEBUG: asked for {dbm_type}") Do_Call = Do_SConsignDir elif o in ('-h', '--help'): print(helpstr) sys.exit(0) elif o in ('-i', '--implicit'): - Print_Flags['implicit'] = 1 + Print_Flags['implicit'] = True elif o in ('--raw',): nodeinfo_string = nodeinfo_raw elif o in ('-r', '--readable'): - Readable = 1 + Readable = True elif o in ('-s', '--size'): - Print_Flags['size'] = 1 + Print_Flags['size'] = True elif o in ('-t', '--timestamp'): - Print_Flags['timestamp'] = 1 + Print_Flags['timestamp'] = True elif o in ('-v', '--verbose'): - Verbose = 1 + Verbose = True if Do_Call: for a in args: Do_Call(a) else: if not args: - args = [".sconsign.dblite"] + #args = [".sconsign.dblite"] + args = [".sconsign"] for a in args: - dbm_name = my_whichdb(a) - if dbm_name: - Map_Module = {'SCons.dblite': 'dblite'} - if dbm_name != "SCons.dblite": - dbm = importlib.import_module(dbm_name) - else: - import SCons.dblite - - dbm = SCons.dblite + dbm_type = my_whichdb(a) + if DEBUG: + print(f"DEBUG: back from my_whichdb, dbm is {dbm_type}") + if dbm_type: + Map_Module = { + 'SCons.dblite': 'dblite', + 'SCons.sdiskcache': 'diskcache' + } + if dbm_type == "SCons.dblite": + import SCons.dblite as dbm # Ensure that we don't ignore corrupt DB files, SCons.dblite.IGNORE_CORRUPT_DBFILES = False - Do_SConsignDB(Map_Module.get(dbm_name, dbm_name), dbm)(a) + elif dbm_type == "SCons.sdiskcache": + import SCons.sdiskcache as dbm + else: + dbm = importlib.import_module(dbm_name) + Do_SConsignDB(Map_Module.get(dbm_type, dbm_type), dbm_type, dbm)(a) else: Do_SConsignDir(a) diff --git a/SCons/sdiskcache.py b/SCons/sdiskcache.py new file mode 100644 index 0000000000..abcc6d1bf9 --- /dev/null +++ b/SCons/sdiskcache.py @@ -0,0 +1,254 @@ +# MIT License +# +# Copyright The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +"""SConsign support for diskcache. """ + +import os +from collections.abc import KeysView, ValuesView, ItemsView +import diskcache + +DISKCACHE_SUFFIX = '.sqlite' +DEBUG = False + + +class _Diskcache: + """Processing of sconsign databases using diskcache. + + Derived from the SCons dblite module, but much simpler, because + there's no exit processing needed - diskcache keeps a consistent + sqlite database under the covers. We don't map prefectly, + since the dblite implementation leaks through into the model: + plenty of code expects the in-memory sconsign DB to not + be backed to disk _except_ on close. + + Most of this is a thin wrapper around a diskcache.Cache, + which is stored in the _dict attribute - the "in-memory" copy. + + We do want to add a few behaviors: some instances can be + read-only (e.g. if they are found in a repository we don't update); + to mirror the dbm/dblite behavior of open flags, "r" and "w" + expect the DB file to actually exist while "n" means it should + be emptied (that is, "new"); and we want to make sure there's + a keys method at least. + + The optional *flag* argument is as for :meth:`dbm.open`. + + +---------+---------------------------------------------------+ + | Value | Meaning | + +=========+===================================================+ + | ``'r'`` | Open existing database for reading only (default) | + +---------+---------------------------------------------------+ + | ``'w'`` | Open existing database for reading and writing | + +---------+---------------------------------------------------+ + | ``'c'`` | Open database for reading and writing, creating | + | | it if it doesn't exist | + +---------+---------------------------------------------------+ + | ``'n'`` | Always create a new, empty database, open for | + | | reading and writing | + +---------+---------------------------------------------------+ + + Arguments: + file_base_name: name of db, will get DISKCACHE_SUFFIX if not present + flag: opening mode + mode: UNIX-style mode of DB files (see dbm.open), unused here. + """ + + def __init__(self, file_base_name: str, flag: str = 'r', _: int = 0x000) -> None: + assert flag in ("r", "w", "c", "n") + + if file_base_name.endswith(DISKCACHE_SUFFIX): + # There's already a suffix on the file name, don't add one. + self._dir_name = file_base_name + else: + self._dir_name = file_base_name + DISKCACHE_SUFFIX + + if not os.path.isdir(self._dir_name) and flag in ('r', 'w'): + raise FileNotFoundError(f"No such sconsign database: {self._dir_name}") + self._dict = diskcache.Cache(self._dir_name) + self._writable: bool = flag not in ("r",) + if DEBUG: + print( + f"DEBUG: opened a Cache file {self._dir_name} (writable={self._writable})" + ) + if flag == "n": + self.clear() + + def check(self, fix=False, retry=False): + """Call disckache 'check' routine to verify cache.""" + self._dict.check(fix, retry) + + @staticmethod + def close() -> None: + """Close the Cache file. + + This exists in the SCons model because other sconsigns are + plain file backed, here it's a no-op. + """ + return + + def __getitem__(self, key): + """Return corresponding value for *key* from cache.""" + return self._dict[key] + + def __setitem__(self, key, value) -> None: + """Set correspoding *value* for *key* in cache. + + Cache can be in a read-only state, just skip if so. + TODO: raise error instead? + """ + if self._writable: + self._dict[key] = value + + def keys(self): + return KeysView(self._dict) + + def items(self): + return ItemsView(self._dict) + + def values(self): + return ValuesView(self._dict) + + def has_key(self, key) -> bool: + return key in self._dict + + def __contains__(self, key) -> bool: + """Return ``True`` if *key* is found in cache.""" + return key in self._dict + + __iter__ = keys + + def __len__(self) -> int: + """Count of items in cache including expired.""" + return len(self._dict) + + def clear(self): + """Remove all items from cache.""" + return self._dict.clear() + + def volume(self): + """Return estimated total size of cache on disk.""" + return self._dict.volume() + + def stats(self): + """Return cache statistics hits and misses.""" + return self._dict.stats() + + def expire(self, now=None, retry=False): + """Remove expired items from cache.""" + return self._dict.expire(now, retry) + + def cull(self, retry=False): + """Cull items from cache until volume is less than size limit.""" + return self._dict.cull(retry) + + def evict(self, tag, retry=False): + """Remove items with matching *tag* from cache.""" + return self._dict.evict(tag, retry) + + +def open( # pylint: disable=redefined-builtin + file: str, flags: str = "r", mode: int = 0o666 +) -> _Diskcache: + return _Diskcache(file, flags, mode) + + +def _exercise(): + import tempfile # pylint: disable=import-outside-toplevel + + with tempfile.TemporaryDirectory() as tmp: + # reading a nonexistent file with mode 'r' should fail + try: + test_cache = open(tmp + "_", "r") + except FileNotFoundError: + pass + else: + raise RuntimeError("FileNotFoundError exception expected") + + # create mode creates test_cache + test_cache = open(tmp, "c") + assert len(test_cache) == 0, len(test_cache) + test_cache["bar"] = "foo" + assert test_cache["bar"] == "foo" + assert len(test_cache) == 1, len(test_cache) + test_cache.close() + + # new database should be empty + test_cache = open(tmp, "n") + assert len(test_cache) == 0, len(test_cache) + test_cache["foo"] = "bar" + assert test_cache["foo"] == "bar" + assert len(test_cache) == 1, len(test_cache) + test_cache.close() + + # write mode is just normal + test_cache = open(tmp, "w") + assert len(test_cache) == 1, len(test_cache) + assert test_cache["foo"] == "bar" + test_cache["bar"] = "foo" + assert len(test_cache) == 2, len(test_cache) + assert test_cache["bar"] == "foo" + test_cache.close() + + # read-only database should silently fail to add + test_cache = open(tmp, "r") + assert len(test_cache) == 2, len(test_cache) + assert test_cache["foo"] == "bar" + assert test_cache["bar"] == "foo" + test_cache["ping"] = "pong" + assert len(test_cache) == 2, len(test_cache) + try: + test_cache["ping"] + except KeyError: + pass + else: + raise RuntimeError("KeyError exception expected") + test_cache.close() + + # test iterators + test_cache = open(tmp, 'w') + test_cache["foobar"] = "foobar" + assert len(test_cache) == 3, len(test_cache) + expected = {"foo": "bar", "bar": "foo", "foobar": "foobar"} + assert dict(test_cache) == expected, f"{test_cache} != {expected}" + key = sorted(test_cache.keys()) + exp = sorted(expected.keys()) + assert key == exp, f"{key} != {exp}" + key = sorted(test_cache.values()) + exp = sorted(expected.values()) + assert key == exp, f"{key} != {exp}" + key = sorted(test_cache.items()) + exp = sorted(expected.items()) + assert key == exp, f"{key} != {exp}" + test_cache.close() + + print("Completed _exercise()") + + +if __name__ == "__main__": + _exercise() + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: