Merge pull request #11530 from vkuznet/fix-issue-11434

Replace imp module by importlib

bin/inject-to-config-cache

@@ -7,8 +7,7 @@ Add a config and it's meta data to the config cache.
 
 import os
 import sys
-import imp
-import subprocess
+import importlib
 
 from PSetTweaks.WMTweak import makeTweak
 from WMCore.Cache.WMConfigCache import ConfigCache
@@ -23,15 +22,16 @@ def loadConfig(configPath):
     sys.stdout.flush()
     cfgBaseName = os.path.basename(configPath).replace(".py", "")
     cfgDirName = os.path.dirname(configPath)
-    modPath = imp.find_module(cfgBaseName, [cfgDirName])
-
-    loadedConfig = imp.load_module(cfgBaseName, modPath[0],
-                                   modPath[1], modPath[2])
+    modSpecs = importlib.machinery.PathFinder().find_spec(cfgBaseName, [cfgDirName])
+    module = modSpecs.loader.load_module()
 
     print("done.")
-    return loadedConfig
+    return module
 
-if __name__ == "__main__":
+def main():
+    """
+    Main function of the module
+    """
     if len(sys.argv) != 8:
         print("Usage: %s couchUrl database_name user_name group_name input_file label description" % sys.argv[0])
         sys.exit(1)
@@ -54,3 +54,6 @@ if __name__ == "__main__":
     print("  DocID:    %s" % configCache.document["_id"])
     print("  Revision: %s" % configCache.document["_rev"])
     sys.exit(0)
+
+if __name__ == "__main__":
+    main()

bin/outputmodules-from-config

@@ -6,7 +6,7 @@ Pull output module metadata from a CMSSW config.
 """
 import urllib.request
 
-import imp
+import importlib
 import os
 import sys
 import tempfile
@@ -21,12 +21,9 @@ def loadConfig(configPath):
     """
     cfgBaseName = os.path.basename(configPath).replace(".py", "")
     cfgDirName = os.path.dirname(configPath)
-    modPath = imp.find_module(cfgBaseName, [cfgDirName])
-
-    loadedConfig = imp.load_module(cfgBaseName, modPath[0],
-                                   modPath[1], modPath[2])
-
-    return loadedConfig
+    modSpecs = importlib.machinery.PathFinder().find_spec(cfgBaseName, [cfgDirName])
+    module = modSpecs.loader.load_module()
+    return module
 
 def outputModulesFromConfig(configHandle):
     """
@@ -55,7 +52,10 @@ def outputModulesFromConfig(configHandle):
 
     return outputModules
 
-if __name__ == "__main__":
+def main():
+    """
+    Main function of the module
+    """
     try:
         jsonDecoder = JSONDecoder()
         jsonEncoder = JSONEncoder()
@@ -102,3 +102,6 @@ if __name__ == "__main__":
     except Exception as ex:
         print(ex)
         sys.exit(1)
+
+if __name__ == "__main__":
+    main()

bin/wmagent-mod-config

@@ -11,7 +11,7 @@ https://github.com/dmwm/deployment/blob/master/asyncstageout/manage#L246
 """
 
 import getopt
-import imp
+import importlib
 import os
 import socket
 import sys
@@ -43,7 +43,8 @@ def importConfigTemplate(filename):
     Given filename, load it and grab the configuration object from it
 
     """
-    mod = imp.load_module("wmcore_config_input", open(filename, 'r'), filename, (".py", "r", imp.PY_SOURCE))
+    modSpecs = importlib.machinery.PathFinder().find_spec(filename)
+    mod = modSpecs.loader.load_module("wmcore_config_input")
     config = getattr(mod, 'config', None)
     if config is None:
         msg = "No config attribute found in %s" % filename

src/python/PSetTweaks/PSetTweak.py

@@ -14,7 +14,7 @@
 from past.builtins import basestring
 from future.utils import viewitems, viewvalues
 
-import imp
+import importlib
 import inspect
 import json
 import pickle
@@ -401,7 +401,8 @@ def unpersist(self, filename, formatting=None):
             self.process.__dict__.update(unpickle.__dict__)
 
         if formatting == "python":
-            modRef = imp.load_source('tempTweak', filename)
+            modSpecs = importlib.util.spec_from_file_location('tempTweak', filename)
+            modRef = modSpecs.loader.load_module()
             lister = PSetLister()
             lister(modRef.process)
             for pset in lister.psets:

src/python/WMCore/Agent/Configuration.py

@@ -9,10 +9,6 @@
 
 """
 
-import os
-import imp
-import types
-
 from WMCore.Configuration import ConfigSection as BaseConfigSection
 from WMCore.Configuration import Configuration as BaseConfiguration
 from WMCore.Configuration import loadConfigurationFile as baseLoadConfigurationFile

src/python/WMCore/Configuration.py

@@ -18,7 +18,7 @@
 
 from Utils.PythonVersion import PY3
 
-import imp
+import importlib
 
 _SimpleTypes = [
     bool,
@@ -596,12 +596,11 @@ def loadConfigurationFile(filename):
     cfgBaseName = os.path.basename(filename).replace(".py", "")
     cfgDirName = os.path.dirname(filename)
     if not cfgDirName:
-        modPath = imp.find_module(cfgBaseName)
+        modSpecs = importlib.machinery.PathFinder().find_spec(cfgBaseName)
     else:
-        modPath = imp.find_module(cfgBaseName, [cfgDirName])
+        modSpecs = importlib.machinery.PathFinder().find_spec(cfgBaseName, [cfgDirName])
     try:
-        modRef = imp.load_module(cfgBaseName, modPath[0],
-                                          modPath[1], modPath[2])
+        modRef = modSpecs.loader.load_module()
     except Exception as ex:
         msg = "Unable to load Configuration File:\n"
         msg += "%s\n" % filename

test/python/Integration_t/RequestLifeCycleBase_t.py

@@ -11,7 +11,7 @@
 from nose.plugins.attrib import attr
 import time
 import os
-import imp
+import importlib
 from functools import wraps
 
 # decorator around tests - record errors
@@ -70,8 +70,8 @@ def _configCacheId(self, label):
         configCache.addConfig(os.path.join(configDir, label + '.py'))
         configCache.setLabel(label)
         configCache.setDescription(label)
-        modPath = imp.find_module(label, [configDir])
-        loadedConfig = imp.load_module(label, modPath[0], modPath[1], modPath[2])
+        modSpecs = importlib.machinery.PathFinder().find_spec(label, [configDir])
+        loadedConfig = modSpecs.loader.load_module()
         configCache.setPSetTweaks(makeTweak(loadedConfig.process).jsondictionary())
         configCache.save()
         return configCache.getIDFromLabel(label)

test/python/WMCore_t/Services_t/PyCurlRESTModel.py

@@ -7,7 +7,6 @@
 import unittest
 import threading
 import cherrypy
-import imp
 import os
 import uuid
 import tempfile

test/python/WMCore_t/WMRuntime_t/Scripts_t/SetupCMSSWPset_t.py

@@ -9,7 +9,6 @@
 
 from builtins import zip
 
-import imp
 import unittest
 import os
 import sys