Skip to content
This repository has been archived by the owner on Feb 23, 2023. It is now read-only.

Gfal2.19 & Davix 0.7.6 #161

Merged
merged 2 commits into from
Jan 29, 2021
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 8 additions & 6 deletions config/diracos.json
Original file line number Diff line number Diff line change
@@ -472,14 +472,17 @@
"name": "davix",
"packages": [
{
"src": "https://diracos.web.cern.ch/diracos/SRPM/davix-0.7.5-1.el6.src.rpm",
"originalRepo": "https://dl.fedoraproject.org/pub/epel/6/SRPMS/Packages/",
"src": "https://diracos.web.cern.ch/diracos/SRPM/davix-0.7.6.200.3c48eab-1.el6.src.rpm",
"originalRepo": "https://storage-ci.web.cern.ch/storage-ci/davix/",
"name": "davix"
}
]
},
{
"name": "gfal2",
"RCRepos": "https://dmc-repo.web.cern.ch/dmc-repo/rc/",
"TestingRepos": "https://dmc-repo.web.cern.ch/dmc-repo/testing/",
"ProdRepos": "https://dmc-repo.web.cern.ch/dmc-repo/el6/x86_64/",
"packages": [
{
"src": "https://diracos.web.cern.ch/diracos/SRPM/epydoc-3.0.1-6.el6.src.rpm",
@@ -536,8 +539,7 @@
"buildOnly": true
},
{
"src": "https://diracos.web.cern.ch/diracos/SRPM/gfal2-2.17.1-1.el6_10.src.rpm",
"originalRepo": "https://dl.fedoraproject.org/pub/epel/6/SRPMS/Packages/",
"src": "https://diracos.web.cern.ch/diracos/SRPM/gfal2-2.19.0-1.el6.src.rpm",
"excludePatterns": [
".*-doc-.*",
".*-debuginfo-.*",
@@ -546,11 +548,11 @@
"name": "gfal2"
},
{
"src": "https://diracos.web.cern.ch/diracos/SRPM/gfal2-python-1.9.5-1.el6.src.rpm",
"src": "https://diracos.web.cern.ch/diracos/SRPM/gfal2-python-1.10.0-1.el6.src.rpm",
"name": "gfal2-python"
},
{
"src": "https://diracos.web.cern.ch/diracos/SRPM/gfal2-util-1.5.1-1.el6.src.rpm",
"src": "https://diracos.web.cern.ch/diracos/SRPM/gfal2-util-1.6.0-1.el6.src.rpm",
"name": "gfal2-util"
}
]
5 changes: 4 additions & 1 deletion diracos/diracoslib.py
Original file line number Diff line number Diff line change
@@ -360,17 +360,20 @@ def _buildFromSRPM(packageCfg):
logging.info('Finished')


def _mockRebuild(srpmFile, mockConfigFile):
def _mockRebuild(srpmFile, mockConfigFile, noClean=False):
"""
Rebuild RPMs from a SRPM using mock

:param srpmFile: path to the SRPM file
:param mockConfigFile: path to the mock config file. Otherwise, mock will use the system one
:param noClean: if True, will not clean the mock environment before building
"""

cmd = ['/usr/bin/mock']
if mockConfigFile:
cmd += ['-r', mockConfigFile]
if noClean:
cmd += ['--no-clean']
cmd += ['--rebuild', srpmFile]

logging.debug("Rebuild SRPM with %s", cmd)
8 changes: 8 additions & 0 deletions docs/source/50_troubleshoot.rst
Original file line number Diff line number Diff line change
@@ -86,6 +86,14 @@ prebuilt RPMs for an old ``golang`` version (split into
removed. Singularity can then be built from source as is the case for
other packages.

Davix (Cmake3)
--------------

Latest davix versions require ``CMake3``, which is available as RPM in the SLC6
environment in which we build. To solve this, we download in the ``davix`` routine
a binary of ``CMake3`` that we then inject in the ``mock`` environment. We may have
to do similar tricks for other packages in the future...

About manualDependencies
------------------------

75 changes: 50 additions & 25 deletions patches/davix.patch
Original file line number Diff line number Diff line change
@@ -1,36 +1,61 @@
Only in modified/: davix-0.7.6.200.3c48eab-1.el6.py27.usc4.src.rpm
diff -r -u original/davix.spec modified/davix.spec
--- original/davix.spec 2019-08-28 09:13:40.000000000 +0000
+++ modified/davix.spec 2020-02-13 14:16:43.513053440 +0000
@@ -22,10 +22,6 @@
--- original/davix.spec 2020-11-10 03:21:20.000000000 +0000
+++ modified/davix.spec 2021-01-28 08:09:13.207702045 +0000
@@ -1,5 +1,5 @@
%{!?_pkgdocdir: %global _pkgdocdir %{_docdir}/%{name}-%{version}}
-%global __cmake cmake3
+%global cmake /usr/bin/cmake

Name: davix
Version: 0.7.6.200.3c48eab
@@ -13,8 +13,6 @@
#main lib dependencies
BuildRequires: python2
BuildRequires: libuuid-devel
-BuildRequires: cmake
-BuildRequires: cmake3
BuildRequires: doxygen
BuildRequires: libxml2-devel
BuildRequires: openssl-devel
@@ -25,10 +23,6 @@
# davix-copy dependencies
BuildRequires: gsoap-devel
BuildRequires: libuuid-devel

-# unit tests and abi check
-%if %{?fedora}%{!?fedora:0} >= 10 || %{?rhel}%{!?rhel:0} >= 6
-BuildRequires: abi-compliance-checker
-%endif

Requires: %{name}-libs%{?_isa} = %{version}-%{release}
Requires: libuuid
@@ -83,18 +79,14 @@

@@ -66,9 +60,9 @@
with Http based protocols (WebDav, Amazon S3, ...).

%package doc
-Summary: Documentation for %{name}
+Summary: Documentation for %{name}
%if %{?fedora}%{!?fedora:0} >= 10 || %{?rhel}%{!?rhel:0} >= 6
-BuildArch: noarch
+BuildArch: noarch
%endif

%description doc
@@ -89,7 +83,7 @@
%cmake \
-DDOC_INSTALL_DIR=%{_pkgdocdir} \
-DENABLE_THIRD_PARTY_COPY=TRUE \
--DENABLE_HTML_DOCS=TRUE \
+-DCMAKE_INSTALL_PREFIX=/usr/ \
.
make %{?_smp_mflags}
make doc

-%check
-%if %{?fedora}%{!?fedora:0} >= 10 || %{?rhel}%{!?rhel:0} >= 6
-#make abi-check
-%endif
-ctest -V -T Test
-

%install
rm -rf %{buildroot}
make DESTDIR=%{buildroot} install

-%ldconfig_scriptlets libs
+%post libs -p /sbin/ldconfig
@@ -129,7 +123,7 @@
%files doc
%{_pkgdocdir}/LICENSE
%{_pkgdocdir}/RELEASE-NOTES.md
-%{_pkgdocdir}/html/
+
+%postun libs -p /sbin/ldconfig

%files
%{_bindir}/*

%files tests
%{_bindir}/davix-unit-tests
39 changes: 39 additions & 0 deletions patches/gfal2-util.patch
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
--- original/gfal2-util.spec 2021-01-14 09:53:06.477321181 +0000
+++ modified/gfal2-util.spec 2021-01-14 09:51:10.169742718 +0000
@@ -14,7 +14,7 @@
%endif

%if %{with python2}
-%{!?python2_sitelib: %global python2_sitelib %(%{__python2} -c "from distutils.sysconfig import get_python_lib; print get_python_lib()")}
+%global python2_sitelib /usr/lib/python2.7/site-packages
%endif

%if %{with python3}
@@ -83,13 +83,6 @@
python3 setup.py install --root=%{buildroot}
%endif

-%check
-%if %{with python2}
- python2 test/functional/test_all.py
-%endif
-%if %{with python3}
- python3 test/functional/test_all.py
-%endif

%clean
rm -rf %{buildroot}
@@ -117,12 +110,9 @@

BuildRequires: gfal2-python >= 1.9.0
BuildRequires: python2
-BuildRequires: python2-rpm-macros
-BuildRequires: python2-future
Requires: gfal2-python >= 1.9.0
Requires: gfal2-util-scripts = %{version}-%{release}
Requires: python2
-Requires: python2-future
%if %{?fedora}%{!?fedora:0} < 26 || %{?rhel}%{!?rhel:0} < 7
BuildRequires: python-argparse
Requires: python-argparse

14 changes: 0 additions & 14 deletions patches/gfal2.patch

This file was deleted.

80 changes: 80 additions & 0 deletions routines/davix.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
import logging
import shutil
import subprocess
from diracos import diracoslib
import os
import glob
import tarfile
import tempfile
from yum import rpmUtils

# TODO: factorize with glite-ce-cream-cli.py, since it is essentialy the same thing

# Original URL: https://github.com/Kitware/CMake/releases/download/v3.19.3/cmake-3.19.3-Linux-x86_64.sh
CMAKE3_SLC6_INSTALLER = 'https://diracos.web.cern.ch/diracos/bootstrap/cmake-3.19.3-Linux-x86_64.sh'


def execute(**kwargs):
""" This requires also patching the tar.gz"""

logging.info("Executing xrootd routine")
logging.debug("Arguments %s", kwargs)

srpmFile = kwargs['src']
repository = kwargs['repo']
mockConfig = kwargs['mockConfig']
mockRoot = kwargs['mockRoot']
patchDir = kwargs['patchDir']
patchFile = None

workDir = kwargs.get('workDir', '/tmp')

logging.info("Building %s %s", srpmFile, "%s" % ("with mockConfig %s" % mockConfig
if mockConfig else ""))

if not os.path.isfile(srpmFile):
logging.debug("SRPM file is an URL, download it first")
srpmFile = diracoslib._downloadFile(srpmFile, '/tmp')

# get package name
pkgName, pkgVersion, _release, _epoch, _arch = rpmUtils.miscutils.splitFilename(
os.path.basename(srpmFile))

# If the src.rpm is already in the repo, do not rebuild it
#
existingBuild = glob.glob(os.path.join(repository, 'src', '%s-%s*' % (pkgName, pkgVersion)))
if existingBuild:
logging.info("The repo already contains a build, not rebuilding: %s", existingBuild)
return

if not patchFile and patchDir:

potentialPatchFile = os.path.join(patchDir, '%s.patch' % pkgName)
logging.debug("Checking existance of %s", potentialPatchFile)
if os.path.isfile(potentialPatchFile):
logging.debug("patch file found")
patchFile = potentialPatchFile

if patchFile:
newSRPM = diracoslib._mock_patchAndRecreateSRPM(srpmFile, patchFile, mockConfig, mockRoot=mockRoot)
logging.debug("New SRPM file %s", newSRPM)
srpmFile = newSRPM

# now comes the funky part !
# Davix decided to use cmake3, which is not easily available on SLC6 as RPM.
# Luckily for us, there is a binary distribution by cmake themselves which works fine.
# So we just extract it in the mock environment,
# and build the RPM without cleaning !

installer = diracoslib._downloadFile(CMAKE3_SLC6_INSTALLER, workDir)

mockUsrDir = os.path.join(mockRoot, 'root/usr/')
cmd = ['sh', installer, '--skip-license', '--prefix=%s' % mockUsrDir]
logging.info("Installing cmake3 in the mock environment %s", cmd)
subprocess.check_call(cmd)

diracoslib._mockRebuild(srpmFile, mockConfig, noClean=True)
mockResultDir = os.path.join(mockRoot, 'result/')
diracoslib._copyRPMs(mockResultDir, repository, byRPMType=True)
diracoslib._createRepo(repository)
logging.info('Finished')