diff --git a/config/diracos.json b/config/diracos.json index 2998d40..34eed36 100644 --- a/config/diracos.json +++ b/config/diracos.json @@ -472,14 +472,17 @@ "name": "davix", "packages": [ { - "src": "https://diracos.web.cern.ch/diracos/SRPM/davix-0.7.5-1.el6.src.rpm", - "originalRepo": "https://dl.fedoraproject.org/pub/epel/6/SRPMS/Packages/", + "src": "https://diracos.web.cern.ch/diracos/SRPM/davix-0.7.6.200.3c48eab-1.el6.src.rpm", + "originalRepo": "https://storage-ci.web.cern.ch/storage-ci/davix/", "name": "davix" } ] }, { "name": "gfal2", + "RCRepos": "https://dmc-repo.web.cern.ch/dmc-repo/rc/", + "TestingRepos": "https://dmc-repo.web.cern.ch/dmc-repo/testing/", + "ProdRepos": "https://dmc-repo.web.cern.ch/dmc-repo/el6/x86_64/", "packages": [ { "src": "https://diracos.web.cern.ch/diracos/SRPM/epydoc-3.0.1-6.el6.src.rpm", @@ -536,8 +539,7 @@ "buildOnly": true }, { - "src": "https://diracos.web.cern.ch/diracos/SRPM/gfal2-2.17.1-1.el6_10.src.rpm", - "originalRepo": "https://dl.fedoraproject.org/pub/epel/6/SRPMS/Packages/", + "src": "https://diracos.web.cern.ch/diracos/SRPM/gfal2-2.19.0-1.el6.src.rpm", "excludePatterns": [ ".*-doc-.*", ".*-debuginfo-.*", @@ -546,11 +548,11 @@ "name": "gfal2" }, { - "src": "https://diracos.web.cern.ch/diracos/SRPM/gfal2-python-1.9.5-1.el6.src.rpm", + "src": "https://diracos.web.cern.ch/diracos/SRPM/gfal2-python-1.10.0-1.el6.src.rpm", "name": "gfal2-python" }, { - "src": "https://diracos.web.cern.ch/diracos/SRPM/gfal2-util-1.5.1-1.el6.src.rpm", + "src": "https://diracos.web.cern.ch/diracos/SRPM/gfal2-util-1.6.0-1.el6.src.rpm", "name": "gfal2-util" } ] diff --git a/diracos/diracoslib.py b/diracos/diracoslib.py index b34e7e8..ded24d3 100644 --- a/diracos/diracoslib.py +++ b/diracos/diracoslib.py @@ -360,17 +360,20 @@ def _buildFromSRPM(packageCfg): logging.info('Finished') -def _mockRebuild(srpmFile, mockConfigFile): +def _mockRebuild(srpmFile, mockConfigFile, noClean=False): """ Rebuild RPMs from a SRPM using mock :param srpmFile: path to the SRPM file :param mockConfigFile: path to the mock config file. Otherwise, mock will use the system one + :param noClean: if True, will not clean the mock environment before building """ cmd = ['/usr/bin/mock'] if mockConfigFile: cmd += ['-r', mockConfigFile] + if noClean: + cmd += ['--no-clean'] cmd += ['--rebuild', srpmFile] logging.debug("Rebuild SRPM with %s", cmd) diff --git a/docs/source/50_troubleshoot.rst b/docs/source/50_troubleshoot.rst index 97705c6..76f3765 100644 --- a/docs/source/50_troubleshoot.rst +++ b/docs/source/50_troubleshoot.rst @@ -86,6 +86,14 @@ prebuilt RPMs for an old ``golang`` version (split into removed. Singularity can then be built from source as is the case for other packages. +Davix (Cmake3) +-------------- + +Latest davix versions require ``CMake3``, which is available as RPM in the SLC6 +environment in which we build. To solve this, we download in the ``davix`` routine +a binary of ``CMake3`` that we then inject in the ``mock`` environment. We may have +to do similar tricks for other packages in the future... + About manualDependencies ------------------------ diff --git a/patches/davix.patch b/patches/davix.patch index a58b641..8506d90 100644 --- a/patches/davix.patch +++ b/patches/davix.patch @@ -1,36 +1,61 @@ +Only in modified/: davix-0.7.6.200.3c48eab-1.el6.py27.usc4.src.rpm diff -r -u original/davix.spec modified/davix.spec ---- original/davix.spec 2019-08-28 09:13:40.000000000 +0000 -+++ modified/davix.spec 2020-02-13 14:16:43.513053440 +0000 -@@ -22,10 +22,6 @@ +--- original/davix.spec 2020-11-10 03:21:20.000000000 +0000 ++++ modified/davix.spec 2021-01-28 08:09:13.207702045 +0000 +@@ -1,5 +1,5 @@ + %{!?_pkgdocdir: %global _pkgdocdir %{_docdir}/%{name}-%{version}} +-%global __cmake cmake3 ++%global cmake /usr/bin/cmake + + Name: davix + Version: 0.7.6.200.3c48eab +@@ -13,8 +13,6 @@ + #main lib dependencies + BuildRequires: python2 + BuildRequires: libuuid-devel +-BuildRequires: cmake +-BuildRequires: cmake3 + BuildRequires: doxygen + BuildRequires: libxml2-devel + BuildRequires: openssl-devel +@@ -25,10 +23,6 @@ # davix-copy dependencies BuildRequires: gsoap-devel - BuildRequires: libuuid-devel + -# unit tests and abi check -%if %{?fedora}%{!?fedora:0} >= 10 || %{?rhel}%{!?rhel:0} >= 6 -BuildRequires: abi-compliance-checker -%endif - + Requires: %{name}-libs%{?_isa} = %{version}-%{release} - Requires: libuuid -@@ -83,18 +79,14 @@ + +@@ -66,9 +60,9 @@ + with Http based protocols (WebDav, Amazon S3, ...). + + %package doc +-Summary: Documentation for %{name} ++Summary: Documentation for %{name} + %if %{?fedora}%{!?fedora:0} >= 10 || %{?rhel}%{!?rhel:0} >= 6 +-BuildArch: noarch ++BuildArch: noarch + %endif + + %description doc +@@ -89,7 +83,7 @@ + %cmake \ + -DDOC_INSTALL_DIR=%{_pkgdocdir} \ + -DENABLE_THIRD_PARTY_COPY=TRUE \ +--DENABLE_HTML_DOCS=TRUE \ ++-DCMAKE_INSTALL_PREFIX=/usr/ \ + . make %{?_smp_mflags} make doc - --%check --%if %{?fedora}%{!?fedora:0} >= 10 || %{?rhel}%{!?rhel:0} >= 6 --#make abi-check --%endif --ctest -V -T Test -- - - %install - rm -rf %{buildroot} - make DESTDIR=%{buildroot} install - --%ldconfig_scriptlets libs -+%post libs -p /sbin/ldconfig +@@ -129,7 +123,7 @@ + %files doc + %{_pkgdocdir}/LICENSE + %{_pkgdocdir}/RELEASE-NOTES.md +-%{_pkgdocdir}/html/ + -+%postun libs -p /sbin/ldconfig - - %files - %{_bindir}/* + + %files tests + %{_bindir}/davix-unit-tests diff --git a/patches/gfal2-util.patch b/patches/gfal2-util.patch new file mode 100644 index 0000000..4fec79f --- /dev/null +++ b/patches/gfal2-util.patch @@ -0,0 +1,39 @@ +--- original/gfal2-util.spec 2021-01-14 09:53:06.477321181 +0000 ++++ modified/gfal2-util.spec 2021-01-14 09:51:10.169742718 +0000 +@@ -14,7 +14,7 @@ + %endif + + %if %{with python2} +-%{!?python2_sitelib: %global python2_sitelib %(%{__python2} -c "from distutils.sysconfig import get_python_lib; print get_python_lib()")} ++%global python2_sitelib /usr/lib/python2.7/site-packages + %endif + + %if %{with python3} +@@ -83,13 +83,6 @@ + python3 setup.py install --root=%{buildroot} + %endif + +-%check +-%if %{with python2} +- python2 test/functional/test_all.py +-%endif +-%if %{with python3} +- python3 test/functional/test_all.py +-%endif + + %clean + rm -rf %{buildroot} +@@ -117,12 +110,9 @@ + + BuildRequires: gfal2-python >= 1.9.0 + BuildRequires: python2 +-BuildRequires: python2-rpm-macros +-BuildRequires: python2-future + Requires: gfal2-python >= 1.9.0 + Requires: gfal2-util-scripts = %{version}-%{release} + Requires: python2 +-Requires: python2-future + %if %{?fedora}%{!?fedora:0} < 26 || %{?rhel}%{!?rhel:0} < 7 + BuildRequires: python-argparse + Requires: python-argparse + diff --git a/patches/gfal2.patch b/patches/gfal2.patch deleted file mode 100644 index 761b200..0000000 --- a/patches/gfal2.patch +++ /dev/null @@ -1,14 +0,0 @@ -diff -r -u original/gfal2.spec modified/gfal2.spec ---- original/gfal2.spec 2019-12-13 11:06:42.000000000 +0000 -+++ modified/gfal2.spec 2020-02-13 15:12:26.529528637 +0000 -@@ -283,7 +283,9 @@ - make DESTDIR=%{buildroot} install - - --%ldconfig_scriptlets -+%post -p /sbin/ldconfig -+ -+%postun -p /sbin/ldconfig - - %files - %{_bindir}/gfal2_version diff --git a/routines/davix.py b/routines/davix.py new file mode 100644 index 0000000..e4f961a --- /dev/null +++ b/routines/davix.py @@ -0,0 +1,80 @@ +import logging +import shutil +import subprocess +from diracos import diracoslib +import os +import glob +import tarfile +import tempfile +from yum import rpmUtils + +# TODO: factorize with glite-ce-cream-cli.py, since it is essentialy the same thing + +# Original URL: https://github.com/Kitware/CMake/releases/download/v3.19.3/cmake-3.19.3-Linux-x86_64.sh +CMAKE3_SLC6_INSTALLER = 'https://diracos.web.cern.ch/diracos/bootstrap/cmake-3.19.3-Linux-x86_64.sh' + + +def execute(**kwargs): + """ This requires also patching the tar.gz""" + + logging.info("Executing xrootd routine") + logging.debug("Arguments %s", kwargs) + + srpmFile = kwargs['src'] + repository = kwargs['repo'] + mockConfig = kwargs['mockConfig'] + mockRoot = kwargs['mockRoot'] + patchDir = kwargs['patchDir'] + patchFile = None + + workDir = kwargs.get('workDir', '/tmp') + + logging.info("Building %s %s", srpmFile, "%s" % ("with mockConfig %s" % mockConfig + if mockConfig else "")) + + if not os.path.isfile(srpmFile): + logging.debug("SRPM file is an URL, download it first") + srpmFile = diracoslib._downloadFile(srpmFile, '/tmp') + + # get package name + pkgName, pkgVersion, _release, _epoch, _arch = rpmUtils.miscutils.splitFilename( + os.path.basename(srpmFile)) + + # If the src.rpm is already in the repo, do not rebuild it + # + existingBuild = glob.glob(os.path.join(repository, 'src', '%s-%s*' % (pkgName, pkgVersion))) + if existingBuild: + logging.info("The repo already contains a build, not rebuilding: %s", existingBuild) + return + + if not patchFile and patchDir: + + potentialPatchFile = os.path.join(patchDir, '%s.patch' % pkgName) + logging.debug("Checking existance of %s", potentialPatchFile) + if os.path.isfile(potentialPatchFile): + logging.debug("patch file found") + patchFile = potentialPatchFile + + if patchFile: + newSRPM = diracoslib._mock_patchAndRecreateSRPM(srpmFile, patchFile, mockConfig, mockRoot=mockRoot) + logging.debug("New SRPM file %s", newSRPM) + srpmFile = newSRPM + + # now comes the funky part ! + # Davix decided to use cmake3, which is not easily available on SLC6 as RPM. + # Luckily for us, there is a binary distribution by cmake themselves which works fine. + # So we just extract it in the mock environment, + # and build the RPM without cleaning ! + + installer = diracoslib._downloadFile(CMAKE3_SLC6_INSTALLER, workDir) + + mockUsrDir = os.path.join(mockRoot, 'root/usr/') + cmd = ['sh', installer, '--skip-license', '--prefix=%s' % mockUsrDir] + logging.info("Installing cmake3 in the mock environment %s", cmd) + subprocess.check_call(cmd) + + diracoslib._mockRebuild(srpmFile, mockConfig, noClean=True) + mockResultDir = os.path.join(mockRoot, 'result/') + diracoslib._copyRPMs(mockResultDir, repository, byRPMType=True) + diracoslib._createRepo(repository) + logging.info('Finished')