From 9667aac8a5c4268139aa3d87a047fa7c30d3ff5e Mon Sep 17 00:00:00 2001 From: Valentin Kuznetsov Date: Tue, 4 Apr 2023 08:56:56 -0400 Subject: [PATCH 1/2] Replace imp by importlib --- bin/inject-to-config-cache | 19 +++++++++++-------- bin/outputmodules-from-config | 19 +++++++++++-------- bin/wmagent-mod-config | 5 +++-- src/python/PSetTweaks/PSetTweak.py | 5 +++-- src/python/WMCore/Agent/Configuration.py | 4 ---- src/python/WMCore/Configuration.py | 9 ++++----- 6 files changed, 32 insertions(+), 29 deletions(-) diff --git a/bin/inject-to-config-cache b/bin/inject-to-config-cache index 947ead7147..844348227f 100755 --- a/bin/inject-to-config-cache +++ b/bin/inject-to-config-cache @@ -7,8 +7,7 @@ Add a config and it's meta data to the config cache. import os import sys -import imp -import subprocess +import importlib from PSetTweaks.WMTweak import makeTweak from WMCore.Cache.WMConfigCache import ConfigCache @@ -23,15 +22,16 @@ def loadConfig(configPath): sys.stdout.flush() cfgBaseName = os.path.basename(configPath).replace(".py", "") cfgDirName = os.path.dirname(configPath) - modPath = imp.find_module(cfgBaseName, [cfgDirName]) - - loadedConfig = imp.load_module(cfgBaseName, modPath[0], - modPath[1], modPath[2]) + modSpecs = importlib.machinery.PathFinder().find_spec(cfgBaseName, [cfgDirName]) + module = modSpecs.loader.load_module() print("done.") - return loadedConfig + return module -if __name__ == "__main__": +def main(): + """ + Main function of the module + """ if len(sys.argv) != 8: print("Usage: %s couchUrl database_name user_name group_name input_file label description" % sys.argv[0]) sys.exit(1) @@ -54,3 +54,6 @@ if __name__ == "__main__": print(" DocID: %s" % configCache.document["_id"]) print(" Revision: %s" % configCache.document["_rev"]) sys.exit(0) + +if __name__ == "__main__": + main() diff --git a/bin/outputmodules-from-config b/bin/outputmodules-from-config index 3105a6f696..0c04af0f34 100755 --- a/bin/outputmodules-from-config +++ b/bin/outputmodules-from-config @@ -6,7 +6,7 @@ Pull output module metadata from a CMSSW config. """ import urllib.request -import imp +import importlib import os import sys import tempfile @@ -21,12 +21,9 @@ def loadConfig(configPath): """ cfgBaseName = os.path.basename(configPath).replace(".py", "") cfgDirName = os.path.dirname(configPath) - modPath = imp.find_module(cfgBaseName, [cfgDirName]) - - loadedConfig = imp.load_module(cfgBaseName, modPath[0], - modPath[1], modPath[2]) - - return loadedConfig + modSpecs = importlib.machinery.PathFinder().find_spec(cfgBaseName, [cfgDirName]) + module = modSpecs.loader.load_module() + return module def outputModulesFromConfig(configHandle): """ @@ -55,7 +52,10 @@ def outputModulesFromConfig(configHandle): return outputModules -if __name__ == "__main__": +def main(): + """ + Main function of the module + """ try: jsonDecoder = JSONDecoder() jsonEncoder = JSONEncoder() @@ -102,3 +102,6 @@ if __name__ == "__main__": except Exception as ex: print(ex) sys.exit(1) + +if __name__ == "__main__": + main() diff --git a/bin/wmagent-mod-config b/bin/wmagent-mod-config index da941f1c4b..757af8a07a 100755 --- a/bin/wmagent-mod-config +++ b/bin/wmagent-mod-config @@ -11,7 +11,7 @@ https://github.com/dmwm/deployment/blob/master/asyncstageout/manage#L246 """ import getopt -import imp +import importlib import os import socket import sys @@ -43,7 +43,8 @@ def importConfigTemplate(filename): Given filename, load it and grab the configuration object from it """ - mod = imp.load_module("wmcore_config_input", open(filename, 'r'), filename, (".py", "r", imp.PY_SOURCE)) + modSpecs = importlib.machinery.PathFinder().find_spec(filename) + mod = modSpecs.loader.load_module("wmcore_config_input") config = getattr(mod, 'config', None) if config is None: msg = "No config attribute found in %s" % filename diff --git a/src/python/PSetTweaks/PSetTweak.py b/src/python/PSetTweaks/PSetTweak.py index 46d42865ba..0d092aadc9 100644 --- a/src/python/PSetTweaks/PSetTweak.py +++ b/src/python/PSetTweaks/PSetTweak.py @@ -14,7 +14,7 @@ from past.builtins import basestring from future.utils import viewitems, viewvalues -import imp +import importlib import inspect import json import pickle @@ -401,7 +401,8 @@ def unpersist(self, filename, formatting=None): self.process.__dict__.update(unpickle.__dict__) if formatting == "python": - modRef = imp.load_source('tempTweak', filename) + modSpecs = importlib.util.spec_from_file_location('tempTweak', filename) + modRef = modSpecs.loader.load_module() lister = PSetLister() lister(modRef.process) for pset in lister.psets: diff --git a/src/python/WMCore/Agent/Configuration.py b/src/python/WMCore/Agent/Configuration.py index ff8fe41446..b2ada2f735 100644 --- a/src/python/WMCore/Agent/Configuration.py +++ b/src/python/WMCore/Agent/Configuration.py @@ -9,10 +9,6 @@ """ -import os -import imp -import types - from WMCore.Configuration import ConfigSection as BaseConfigSection from WMCore.Configuration import Configuration as BaseConfiguration from WMCore.Configuration import loadConfigurationFile as baseLoadConfigurationFile diff --git a/src/python/WMCore/Configuration.py b/src/python/WMCore/Configuration.py index d14d2a72fc..a807d10c5b 100644 --- a/src/python/WMCore/Configuration.py +++ b/src/python/WMCore/Configuration.py @@ -18,7 +18,7 @@ from Utils.PythonVersion import PY3 -import imp +import importlib _SimpleTypes = [ bool, @@ -596,12 +596,11 @@ def loadConfigurationFile(filename): cfgBaseName = os.path.basename(filename).replace(".py", "") cfgDirName = os.path.dirname(filename) if not cfgDirName: - modPath = imp.find_module(cfgBaseName) + modSpecs = importlib.machinery.PathFinder().find_spec(cfgBaseName) else: - modPath = imp.find_module(cfgBaseName, [cfgDirName]) + modSpecs = importlib.machinery.PathFinder().find_spec(cfgBaseName, [cfgDirName]) try: - modRef = imp.load_module(cfgBaseName, modPath[0], - modPath[1], modPath[2]) + modRef = modSpecs.loader.load_module() except Exception as ex: msg = "Unable to load Configuration File:\n" msg += "%s\n" % filename From 51cee1142f8e6a15c443b9701363579e1c2f694e Mon Sep 17 00:00:00 2001 From: Valentin Kuznetsov Date: Tue, 4 Apr 2023 08:57:10 -0400 Subject: [PATCH 2/2] Replace imp by importlib in unit tests --- test/python/Integration_t/RequestLifeCycleBase_t.py | 6 +++--- test/python/WMCore_t/Services_t/PyCurlRESTModel.py | 1 - .../WMCore_t/WMRuntime_t/Scripts_t/SetupCMSSWPset_t.py | 1 - 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/test/python/Integration_t/RequestLifeCycleBase_t.py b/test/python/Integration_t/RequestLifeCycleBase_t.py index 5e9cd625a2..174c19da7b 100644 --- a/test/python/Integration_t/RequestLifeCycleBase_t.py +++ b/test/python/Integration_t/RequestLifeCycleBase_t.py @@ -11,7 +11,7 @@ from nose.plugins.attrib import attr import time import os -import imp +import importlib from functools import wraps # decorator around tests - record errors @@ -70,8 +70,8 @@ def _configCacheId(self, label): configCache.addConfig(os.path.join(configDir, label + '.py')) configCache.setLabel(label) configCache.setDescription(label) - modPath = imp.find_module(label, [configDir]) - loadedConfig = imp.load_module(label, modPath[0], modPath[1], modPath[2]) + modSpecs = importlib.machinery.PathFinder().find_spec(label, [configDir]) + loadedConfig = modSpecs.loader.load_module() configCache.setPSetTweaks(makeTweak(loadedConfig.process).jsondictionary()) configCache.save() return configCache.getIDFromLabel(label) diff --git a/test/python/WMCore_t/Services_t/PyCurlRESTModel.py b/test/python/WMCore_t/Services_t/PyCurlRESTModel.py index 63d05d601a..d4b22207a3 100644 --- a/test/python/WMCore_t/Services_t/PyCurlRESTModel.py +++ b/test/python/WMCore_t/Services_t/PyCurlRESTModel.py @@ -7,7 +7,6 @@ import unittest import threading import cherrypy -import imp import os import uuid import tempfile diff --git a/test/python/WMCore_t/WMRuntime_t/Scripts_t/SetupCMSSWPset_t.py b/test/python/WMCore_t/WMRuntime_t/Scripts_t/SetupCMSSWPset_t.py index f9c54128b7..9dcfb59003 100644 --- a/test/python/WMCore_t/WMRuntime_t/Scripts_t/SetupCMSSWPset_t.py +++ b/test/python/WMCore_t/WMRuntime_t/Scripts_t/SetupCMSSWPset_t.py @@ -9,7 +9,6 @@ from builtins import zip -import imp import unittest import os import sys