From d29ea99735597dd4d176b75fe26cb4a28250dc3a Mon Sep 17 00:00:00 2001 From: Malik Shahzad Muzaffar Date: Thu, 21 Nov 2024 12:17:16 +0100 Subject: [PATCH] Python3.12 support: Use importlib instead of deprecated imp module (#5350) --- src/python/CRABClient/Commands/SubCommand.py | 10 ++--- src/python/CRABClient/JobType/CMSSWConfig.py | 9 ++--- src/python/CRABClient/WMCoreConfiguration.py | 10 ++--- src/python/CRABClient/__init__.py | 38 +++++++++++++++++++ .../Commands_t/CRABRESTModelMock.py | 4 +- .../CRABClient_t/Commands_t/Commands_t.py | 4 +- 6 files changed, 52 insertions(+), 23 deletions(-) diff --git a/src/python/CRABClient/Commands/SubCommand.py b/src/python/CRABClient/Commands/SubCommand.py index e340d73ed..40e29a058 100644 --- a/src/python/CRABClient/Commands/SubCommand.py +++ b/src/python/CRABClient/Commands/SubCommand.py @@ -1,6 +1,5 @@ import os import re -import imp import json import types from ast import literal_eval @@ -11,7 +10,7 @@ from ServerUtilities import SERVICE_INSTANCES import CRABClient.Emulator -from CRABClient import __version__ +from CRABClient import __version__, find_module, load_module from CRABClient.ClientUtilities import colors from CRABClient.CRABOptParser import CRABCmdOptParser from CRABClient.CredentialInteractions import CredentialInteractions @@ -106,12 +105,9 @@ def _extractReason(self, configname, runErr): filename = os.path.abspath(configname) cfgBaseName = os.path.basename(filename).replace(".py", "") cfgDirName = os.path.dirname(filename) - if not cfgDirName: - modPath = imp.find_module(cfgBaseName) - else: - modPath = imp.find_module(cfgBaseName, [cfgDirName]) + modPath = find_module(cfgBaseName, cfgDirName) try: - imp.load_module(cfgBaseName, modPath[0], modPath[1], modPath[2]) + load_module(cfgBaseName, modPath) except Exception as ex: msg = str(ex) diff --git a/src/python/CRABClient/JobType/CMSSWConfig.py b/src/python/CRABClient/JobType/CMSSWConfig.py index 9822d8be3..0db3ad025 100644 --- a/src/python/CRABClient/JobType/CMSSWConfig.py +++ b/src/python/CRABClient/JobType/CMSSWConfig.py @@ -5,7 +5,6 @@ import re import os import sys -import imp import json import pickle import logging @@ -14,6 +13,7 @@ from ServerUtilities import BOOTSTRAP_CFGFILE_DUMP +from CRABClient import find_module, load_module, module_pathname from CRABClient.ClientExceptions import ConfigurationException, EnvironmentException from CRABClient.ClientUtilities import bootstrapDone, colors, BOOTSTRAP_CFGFILE_PKL,\ BOOTSTRAP_INFOFILE, LOGGERS, PKL_R_MODE, PKL_W_MODE @@ -59,8 +59,8 @@ def __init__(self, config, userConfig=None, logger=None): site.addsitedir(p) # details of user configuration file: - configFile, pathname, description = imp.find_module(cfgBaseName, [cfgDirName]) - cacheLine = (pathname, tuple(sys.argv)) + modRef = find_module(cfgBaseName, cfgDirName) + cacheLine = (module_pathname(modRef), tuple(sys.argv)) cwd=os.getcwd() if cwd not in sys.path: # cwd must be part of $PYTHONPATH for CMSSW config to work sys.path.append(cwd) @@ -84,11 +84,10 @@ def __init__(self, config, userConfig=None, logger=None): try: oldstdout = sys.stdout sys.stdout = open(logger.logfile, 'a') - self.fullConfig = imp.load_module(cfgBaseName, configFile, pathname, description) + self.fullConfig = load_module(cfgBaseName, modRef) finally: sys.stdout.close() sys.stdout = oldstdout - configFile.close() # need to turn sys.path into a static set of strings for using it as a cache key # otherwise is a pointer to a function and we can't use it to check for stability configurationCache[cacheLine] = { 'config' : self.fullConfig , 'path' : tuple(sys.path) } diff --git a/src/python/CRABClient/WMCoreConfiguration.py b/src/python/CRABClient/WMCoreConfiguration.py index 5a078d0c4..dfd2ef013 100644 --- a/src/python/CRABClient/WMCoreConfiguration.py +++ b/src/python/CRABClient/WMCoreConfiguration.py @@ -28,10 +28,10 @@ """ from __future__ import division -import imp import os import sys import traceback +from CRABClient import find_module, load_module PY3 = sys.version_info[0] == 3 @@ -334,13 +334,9 @@ def loadConfigurationFile(filename): cfgBaseName = os.path.basename(filename).replace(".py", "") cfgDirName = os.path.dirname(filename) - if not cfgDirName: - modPath = imp.find_module(cfgBaseName) - else: - modPath = imp.find_module(cfgBaseName, [cfgDirName]) + modPath = find_module(cfgBaseName, cfgDirName) try: - modRef = imp.load_module(cfgBaseName, modPath[0], - modPath[1], modPath[2]) + modRef = load_module(cfgBaseName, modPath) except Exception as ex: msg = "Unable to load Configuration File:\n" msg += "%s\n" % filename diff --git a/src/python/CRABClient/__init__.py b/src/python/CRABClient/__init__.py index 3605fa458..f12889c09 100644 --- a/src/python/CRABClient/__init__.py +++ b/src/python/CRABClient/__init__.py @@ -5,3 +5,41 @@ __version__ = "development" #the __version__ will be automatically be change according to rpm for production + +import sys +if (sys.version_info[0]*100+sys.version_info[1])>=312: + import importlib + def find_module(moduleName, moduleDir = None): + if moduleDir: + return importlib.machinery.PathFinder.find_spec(moduleName, [moduleDir]) + return importlib.machinery.PathFinder.find_spec(moduleName) + + def load_module(moduleName, moduleSpec): + moduleObj = importlib.util.module_from_spec(moduleSpec) + sys.modules[moduleName] = moduleObj + moduleSpec.loader.exec_module(moduleObj) + return moduleObj + + def load_source(moduleName, moduleFile): + moduleSpec = importlib.util.spec_from_file_location(moduleName, + moduleFile) + return load_module(moduleName, moduleSpec) + + def module_pathname(moduleObj): + return moduleObj.origin +else: + import imp + def find_module(moduleName, moduleDir = None): + if moduleDir: + return imp.find_module(moduleName, [moduleDir]) + return imp.find_module(moduleName) + + def load_module(moduleName, moduleObj): + return imp.load_module(moduleName, moduleObj[0], + moduleObj[1], moduleObj[2]) + + def load_source(moduleName, moduleFile): + return imp.load_source(moduleName, moduleFile) + + def module_pathname(moduleObj): + return moduleObj[1] diff --git a/test/python/CRABClient_t/Commands_t/CRABRESTModelMock.py b/test/python/CRABClient_t/Commands_t/CRABRESTModelMock.py index 6c7206907..61daccf33 100644 --- a/test/python/CRABClient_t/Commands_t/CRABRESTModelMock.py +++ b/test/python/CRABClient_t/Commands_t/CRABRESTModelMock.py @@ -1,9 +1,9 @@ +from CRABClient import load_source from WMCore.WebTools.RESTModel import RESTModel import WMCore import threading import cherrypy -import imp import os import uuid import tempfile @@ -21,7 +21,7 @@ class CRABRESTModelMock(RESTModel): def __init__(self, config={}): RESTModel.__init__(self, config) - self.mapme = imp.load_source('', os.path.join( os.path.dirname(__file__), "../../../data/mapper.py")) + self.mapme = load_source('', os.path.join( os.path.dirname(__file__), "../../../data/mapper.py")) self.defaulturi = self.mapme.defaulturi diff --git a/test/python/CRABClient_t/Commands_t/Commands_t.py b/test/python/CRABClient_t/Commands_t/Commands_t.py index e16a43d57..147255db5 100644 --- a/test/python/CRABClient_t/Commands_t/Commands_t.py +++ b/test/python/CRABClient_t/Commands_t/Commands_t.py @@ -1,6 +1,7 @@ import CRABRESTModelMock from FakeRESTServer import FakeRESTServer from WMCore.Configuration import Configuration +from WMCore import load_source from CRABClient.Commands.server_info import server_info from CRABClient.Commands.getoutput import getoutput from CRABClient.Commands.publish import publish @@ -19,7 +20,6 @@ import os import shutil import time -import imp from socket import error as SocketError class CommandTest(FakeRESTServer): @@ -34,7 +34,7 @@ def __init__(self, config): def setUp(self): #Dynamic import of the configuration which in principle is not in the PYTHONPATH - self.TestConfig = imp.load_source('TestConfig', os.path.join( os.path.dirname(__file__), "../../../data/TestConfig.py")) + self.TestConfig = load_source('TestConfig', os.path.join( os.path.dirname(__file__), "../../../data/TestConfig.py")) FakeRESTServer.setUp(self) if os.path.isdir("./crab_TestAnalysis"): shutil.rmtree("./crab_TestAnalysis")