Merge pull request #218 from fstagni/remoteLogging_test

Remote logging test -- and partial fixes.
DIRACGrid · Nov 9, 2023 · 91949a8 · 91949a8
2 parents b3e6e7a + 0259c4a
commit 91949a8
Show file tree

Hide file tree

Showing 7 changed files with 171 additions and 114 deletions.
diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml
@@ -8,21 +8,23 @@ jobs:
 
     strategy:
       matrix:
-        python: 
-          - 2.7.18
-          - 3.6.15
-          - 3.9.17
         dirac_version:
           - rel-v8r0
           - integration
         pilot_schema:
           - pilot_oldSchema.json
           - pilot_newSchema.json
-
-    container: python:${{ matrix.python }}-slim
+        VO: 
+          - dteam
+          - gridpp
 
     steps:
       - uses: actions/checkout@v4
+      - uses: cvmfs-contrib/github-action-cvmfs@v3
+
+      - name: Test CernVM-FS
+        run: ls /cvmfs/dirac.egi.eu
+
       - name: Retrieve the secret and decode it to a file
         env:
           HOSTCERT_BASE64: ${{ secrets.HOSTCERT_BASE64 }}
@@ -45,7 +47,10 @@ jobs:
           sed -i "s#VAR_CS#https://lbcertifdirac70.cern.ch:9135/Configuration/Server#g" pilot.json
           sed -i "s#VAR_USERDN#/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=alboyer/CN=819281/CN=Alexandre Franck Boyer#g" pilot.json
           sed -i "s#VAR_USERDN_GRIDPP#${DIRACUSERDN_GRIDPP}#g" pilot.json
-          python dirac-pilot.py --modules https://github.com/DIRACGrid/DIRAC.git:::DIRAC:::${{ matrix.dirac_version }} -M 1 -S DIRAC-Certification -N jenkins.cern.ch -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch --cert --certLocation=${GITHUB_WORKSPACE}/Pilot/etc/grid-security --debug
+          g_job="testintegrationworkflow${GITHUB_JOB//-/}"
+          pilotUUID="${g_job//_/}""${GITHUB_RUN_ID}"
+          pilotUUID=$(echo $pilotUUID | rev | cut -c 1-32 | rev)
+          python dirac-pilot.py --modules https://github.com/DIRACGrid/DIRAC.git:::DIRAC:::${{ matrix.dirac_version }} -M 1 -S DIRAC-Certification -N jenkins.cern.ch -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch --cert --certLocation=${GITHUB_WORKSPACE}/Pilot/etc/grid-security --wnVO=${{ matrix.VO }} --pilotUUID="${pilotUUID}" --debug
 
 
   integration-cvmfs:
@@ -93,7 +98,10 @@ jobs:
           sed -i "s#VAR_CS#https://lbcertifdirac70.cern.ch:9135/Configuration/Server#g" pilot.json
           sed -i "s#VAR_USERDN#/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=alboyer/CN=819281/CN=Alexandre Franck Boyer#g" pilot.json
           sed -i "s#VAR_USERDN_GRIDPP#${DIRACUSERDN_GRIDPP}#g" pilot.json
-          python dirac-pilot.py -M 1 -S DIRAC-Certification -N jenkins.cern.ch -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch --preinstalledEnvPrefix=/cvmfs/dirac.egi.eu/dirac --cert --certLocation=${GITHUB_WORKSPACE}/Pilot/etc/grid-security --wnVO=dteam --debug
+          g_job="testintegrationworkflow${GITHUB_JOB//-/}"
+          pilotUUID="${g_job//_/}""${GITHUB_RUN_ID}"
+          pilotUUID=$(echo $pilotUUID | rev | cut -c 1-32 | rev)
+          python dirac-pilot.py -M 1 -S DIRAC-Certification -N jenkins.cern.ch -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch --preinstalledEnvPrefix=/cvmfs/dirac.egi.eu/dirac --cert --certLocation=${GITHUB_WORKSPACE}/Pilot/etc/grid-security --wnVO=dteam --pilotUUID="${pilotUUID}" --debug
 
 
   integration-cvmfs_no_env:
@@ -128,7 +136,10 @@ jobs:
           sed -i "s#VAR_CS#https://lbcertifdirac70.cern.ch:9135/Configuration/Server#g" pilot.json
           sed -i "s#VAR_USERDN#/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=alboyer/CN=819281/CN=Alexandre Franck Boyer#g" pilot.json
           sed -i "s#VAR_USERDN_GRIDPP#${DIRACUSERDN_GRIDPP}#g" pilot.json
-          python dirac-pilot.py -M 1 -S DIRAC-Certification -N jenkins.cern.ch -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch --preinstalledEnvPrefix=/cvmfs/dirac.egi.eu/dirac --cert --certLocation=${GITHUB_WORKSPACE}/Pilot/etc/grid-security --wnVO=dteam -o cvmfsOnly --debug
+          g_job="testintegrationworkflow${GITHUB_JOB//-/}"
+          pilotUUID="${g_job//_/}""${GITHUB_RUN_ID}"
+          pilotUUID=$(echo $pilotUUID | rev | cut -c 1-32 | rev)
+          python dirac-pilot.py -M 1 -S DIRAC-Certification -N jenkins.cern.ch -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch --preinstalledEnvPrefix=/cvmfs/dirac.egi.eu/dirac --cert --certLocation=${GITHUB_WORKSPACE}/Pilot/etc/grid-security --wnVO=dteam -o cvmfsOnly --pilotUUID="${pilotUUID}" --debug
 
 
 
@@ -137,18 +148,17 @@ jobs:
 
     strategy:
       matrix:
-        python: 
-          - 2.7.18
-          - 3.6.15
-          - 3.9.17
         pilot_schema:
           - pilot_oldSchema.json
           - pilot_newSchema.json
 
-    container: python:${{ matrix.python }}-slim
-
     steps:
       - uses: actions/checkout@v4
+      - uses: cvmfs-contrib/github-action-cvmfs@v3
+
+      - name: Test CernVM-FS
+        run: ls /cvmfs/dirac.egi.eu
+
       - name: Retrieve the secret and decode it to a file
         env:
           HOSTCERT_BASE64: ${{ secrets.HOSTCERT_BASE64 }}
@@ -171,7 +181,10 @@ jobs:
           sed -i "s#VAR_CS#https://lbcertifdirac70.cern.ch:9135/Configuration/Server#g" pilot.json
           sed -i "s#VAR_USERDN#/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=alboyer/CN=819281/CN=Alexandre Franck Boyer#g" pilot.json
           sed -i "s#VAR_USERDN_GRIDPP#${DIRACUSERDN_GRIDPP}#g" pilot.json
-          python dirac-pilot.py --modules https://github.com/DIRACGrid/DIRAC.git:::DIRAC:::integration -M 1 -S DIRAC-Certification -N jenkins-full.cern.ch -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch --wnVO=dteam --cert --certLocation=${GITHUB_WORKSPACE}/Pilot/etc/grid-security --debug
+          g_job="testintegrationworkflow${GITHUB_JOB//-/}"
+          pilotUUID="${g_job//_/}""${GITHUB_RUN_ID}"
+          pilotUUID=$(echo $pilotUUID | rev | cut -c 1-32 | rev)
+          python dirac-pilot.py --modules https://github.com/DIRACGrid/DIRAC.git:::DIRAC:::integration -M 1 -S DIRAC-Certification -N jenkins-full.cern.ch -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch --wnVO=dteam --cert --certLocation=${GITHUB_WORKSPACE}/Pilot/etc/grid-security --pilotUUID="${pilotUUID}" --debug
 
 
   integration-cvmfs_matching:
@@ -215,7 +228,10 @@ jobs:
           sed -i "s#VAR_CS#https://lbcertifdirac70.cern.ch:9135/Configuration/Server#g" pilot.json
           sed -i "s#VAR_USERDN#/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=alboyer/CN=819281/CN=Alexandre Franck Boyer#g" pilot.json
           sed -i "s#VAR_USERDN_GRIDPP#${DIRACUSERDN_GRIDPP}#g" pilot.json
-          python dirac-pilot.py -M 1 -S DIRAC-Certification -N jenkins-full.cern.ch -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch --preinstalledEnvPrefix=/cvmfs/dirac.egi.eu/dirac --cert --certLocation=${GITHUB_WORKSPACE}/Pilot/etc/grid-security --wnVO=${{ matrix.VO }} -o cvmfsOnly --debug
+          g_job="testintegrationworkflow${GITHUB_JOB//-/}"
+          pilotUUID="${g_job//_/}""${GITHUB_RUN_ID}"
+          pilotUUID=$(echo $pilotUUID | rev | cut -c 1-32 | rev)
+          python dirac-pilot.py -M 1 -S DIRAC-Certification -N jenkins-full.cern.ch -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch --preinstalledEnvPrefix=/cvmfs/dirac.egi.eu/dirac --cert --certLocation=${GITHUB_WORKSPACE}/Pilot/etc/grid-security --wnVO=${{ matrix.VO }} -o cvmfsOnly --pilotUUID="${pilotUUID}" --debug
 
 
   ##################################
@@ -272,7 +288,10 @@ jobs:
           sed -i "s#VAR_CS#https://lbcertifdirac70.cern.ch:9135/Configuration/Server#g" pilot.json
           sed -i "s#VAR_USERDN#/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=alboyer/CN=819281/CN=Alexandre Franck Boyer#g" pilot.json
           sed -i "s#VAR_USERDN_GRIDPP#${DIRACUSERDN_GRIDPP}#g" pilot.json
-          python dirac-pilot.py -M 1 -S DIRAC-Certification -N jenkins-lhcb.cern.ch -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch -e LHCb -l LHCb -E LHCbPilot --cert --certLocation=${GITHUB_WORKSPACE}/Pilot/etc/grid-security --wnVO=LHCb  -o lbRunOnly --debug
+          g_job="testintegrationworkflow${GITHUB_JOB//-/}"
+          pilotUUID="${g_job//_/}""${GITHUB_RUN_ID}"
+          pilotUUID=$(echo $pilotUUID | rev | cut -c 1-32 | rev)
+          python dirac-pilot.py -M 1 -S DIRAC-Certification -N jenkins-lhcb.cern.ch -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch -e LHCb -l LHCb -E LHCbPilot --cert --certLocation=${GITHUB_WORKSPACE}/Pilot/etc/grid-security --wnVO=LHCb  -o lbRunOnly --pilotUUID="${pilotUUID}" --debug
 
   ext-lhcb_integration_dirac_installer_no_env:
     runs-on: ubuntu-latest
@@ -316,7 +335,10 @@ jobs:
           sed -i "s#VAR_CS#https://lbcertifdirac70.cern.ch:9135/Configuration/Server#g" pilot.json
           sed -i "s#VAR_USERDN#/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=alboyer/CN=819281/CN=Alexandre Franck Boyer#g" pilot.json
           sed -i "s#VAR_USERDN_GRIDPP#${DIRACUSERDN_GRIDPP}#g" pilot.json
-          python dirac-pilot.py -M 1 -S DIRAC-Certification -N ${{ matrix.ce }} -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch -e LHCb -l LHCb -E LHCbPilot --preinstalledEnvPrefix=/cvmfs/lhcb.cern.ch/lhcbdirac/ --cert --certLocation=${GITHUB_WORKSPACE}/Pilot/etc/grid-security --wnVO=LHCb -o cvmfsOnly --debug
+          g_job="testintegrationworkflow${GITHUB_JOB//-/}"
+          pilotUUID="${g_job//_/}""${GITHUB_RUN_ID}"
+          pilotUUID=$(echo $pilotUUID | rev | cut -c 1-32 | rev)
+          python dirac-pilot.py -M 1 -S DIRAC-Certification -N ${{ matrix.ce }} -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch -e LHCb -l LHCb -E LHCbPilot --preinstalledEnvPrefix=/cvmfs/lhcb.cern.ch/lhcbdirac/ --cert --certLocation=${GITHUB_WORKSPACE}/Pilot/etc/grid-security --wnVO=LHCb -o cvmfsOnly --pilotUUID="${pilotUUID}" --debug
 
   ext-lhcb_integration_no_env:
     runs-on: ubuntu-latest
@@ -360,4 +382,7 @@ jobs:
           sed -i "s#VAR_CS#https://lbcertifdirac70.cern.ch:9135/Configuration/Server#g" pilot.json
           sed -i "s#VAR_USERDN#/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=alboyer/CN=819281/CN=Alexandre Franck Boyer#g" pilot.json
           sed -i "s#VAR_USERDN_GRIDPP#${DIRACUSERDN_GRIDPP}#g" pilot.json
-          python dirac-pilot.py -M 1 -S DIRAC-Certification -N jenkins-lhcb.cern.ch -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch -e LHCb -l LHCb -E LHCbPilot --preinstalledEnvPrefix=/cvmfs/lhcb.cern.ch/lhcbdirac/ --cert --certLocation=${GITHUB_WORKSPACE}/Pilot/etc/grid-security --wnVO=LHCb -o cvmfsOnly --debug
+          g_job="testintegrationworkflow${GITHUB_JOB//-/}"
+          pilotUUID="${g_job//_/}""${GITHUB_RUN_ID}"
+          pilotUUID=$(echo $pilotUUID | rev | cut -c 1-32 | rev)
+          python dirac-pilot.py -M 1 -S DIRAC-Certification -N jenkins-lhcb.cern.ch -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch -e LHCb -l LHCb -E LHCbPilot --preinstalledEnvPrefix=/cvmfs/lhcb.cern.ch/lhcbdirac/ --cert --certLocation=${GITHUB_WORKSPACE}/Pilot/etc/grid-security --wnVO=LHCb -o cvmfsOnly --pilotUUID="${pilotUUID}" --debug
diff --git a/Pilot/pilotCommands.py b/Pilot/pilotCommands.py
@@ -206,6 +206,7 @@ def execute(self):
             )
             self.exitWithError(1)
 
+
 class InstallDIRAC(CommandBase):
     """ Source from CVMFS, or install locally
     """
@@ -498,7 +499,6 @@ def execute(self):
             # environment variables to add?
             if self.pp.userEnvVariables:
                 # User-requested environment variables (comma-separated, name and value separated by ":::")
-                # The case of dirac-install is handled in vanilla DIRAC Pilot
                 newEnvVars = dict(
                 name.split(":::", 1) for name in self.pp.userEnvVariables.replace(" ", "").split(",")
               )
@@ -526,7 +526,7 @@ class ConfigureBasics(CommandBase):
 
     It calls dirac-configure to:
 
-        * download, by default, the CAs
+        * (maybe) download the CAs
         * creates a standard or custom (defined by self.pp.localConfigFile) cfg file
           (by default 'pilot.cfg') to be used where all the pilot configuration is to be set, e.g.:
         * adds to it basic info like the version
@@ -605,37 +605,6 @@ def _getBasicsCFG(self):
         if self.pp.wnVO:
             self.cfg.append('-o "/Resources/Computing/CEDefaults/VirtualOrganization=%s"' % self.pp.wnVO)
 
-    def __checkSecurityDir(self, envName, dirName):
-
-        if envName in os.environ and safe_listdir(os.environ[envName]):
-            self.log.debug(
-                "%s is set in the host environment as %s, aligning installEnv to it"
-                % (envName, os.environ[envName])
-            )               
-            self.pp.installEnv[envName] = os.environ[envName]
-        else:
-            self.log.debug("%s is not set in the host environment" % envName)
-            # try and find it
-            for candidate in self.pp.CVMFS_locations:
-                candidateDir = os.path.join(candidate,
-                                            'etc/grid-security',
-                                            dirName)
-                self.log.debug(
-                    "Candidate directory for %s is %s"
-                    % (envName, candidateDir)
-                )
-                if safe_listdir(candidateDir):
-
-                    self.log.debug("Setting %s=%s" % (envName, candidateDir))
-                    self.pp.installEnv[envName] = candidateDir
-                    os.environ[envName] = candidateDir
-                    break
-                self.log.debug("%s not found or not a directory" % candidateDir)
-
-        if envName not in self.pp.installEnv:
-            self.log.error("Could not find/set %s" % envName)
-            sys.exit(1)
-
     def _getSecurityCFG(self):
         """ Sets security-related env variables, if needed
         """
@@ -645,17 +614,8 @@ def _getSecurityCFG(self):
             self.cfg.append("-o /DIRAC/Security/CertFile=%s/hostcert.pem" % self.pp.certsLocation)
             self.cfg.append("-o /DIRAC/Security/KeyFile=%s/hostkey.pem" % self.pp.certsLocation)
 
-        # If DIRAC (or its extension) is installed in CVMFS:
+        # If DIRAC (or its extension) is installed in CVMFS do not download VOMS and CAs
         if self.pp.preinstalledEnv:
-
-            self.__checkSecurityDir("X509_CERT_DIR", "certificates")
-            self.__checkSecurityDir("X509_VOMS_DIR", "vomsdir")
-            self.__checkSecurityDir("X509_VOMSES", "vomses")
-            # This is needed for the integration tests
-            self.pp.installEnv["DIRAC_VOMSES"] = self.pp.installEnv["X509_VOMSES"]
-            os.environ["DIRAC_VOMSES"] = os.environ["X509_VOMSES"]
-
-            # In any case do not download VOMS and CAs
             self.cfg.append("-DMH")