Make configure script work with pinned requirements #2070

Now ./configure will install dependencies from our "private" PyPI
using find-links

Signed-off-by: Abhishek Kumar <[email protected]>
Signed-off-by: Philippe Ombredanne <[email protected]>

configure

@@ -19,10 +19,11 @@ fi
 export TPP_DIR="thirdparty"
 
 # default configurations for dev
-CONF_DEFAULT="etc/conf/dev"
+CONF_DEFAULT="etc/conf" 
 
-if [[ "$CONF_DEFAULT" == "etc/conf/dev" ]]; then
-    export TPP_DIR_DEV="thirdparty/dev"
+if [[ "$CONF_DEFAULT" == "etc/conf" ]]; then
+    #export TPP_DIR_DEV="thirdparty/dev"
+    export TPP_DIR_links="https://github.com/Abhishek-Dev09/thirdparty/releases/tag/v2.0"
 fi
 
 

etc/conf/requirements.txt

@@ -0,0 +1,242 @@
+#
+# This file is autogenerated by pip-compile
+# To update, run:
+#
+#    pip-compile --allow-unsafe --find-links=/var/folders/61/wbzyy9553cxfykg4_9vrjrcm0000gn/T/tmpmowsb_wq --generate-hashes --output-file=requirements.txt --pip-args='--no-index'
+
+
+attrs==19.3.0 \
+    --hash=sha256:08a96c641c3a74e44eb59afb61a24f2cb9f4d7188748e76ba4bb5edfa3cb7d1c \
+    # via scancode-toolkit (setup.py)
+banal==0.3.3 \
+    --hash=sha256:63d979b8b63458dd538f10aca4ea5292b94729f60a08ad6b41b8129a9a9f716a \
+    # via normality
+beautifulsoup4==4.7.1 \
+    --hash=sha256:034740f6cb549b4e932ae1ab975581e6103ac8f942200a0e9759065984391858 \
+    # via scancode-toolkit (setup.py)
+binaryornot==0.4.4 \
+    --hash=sha256:b8b71173c917bddcd2c16070412e369c3ed7f0528926f70cac18a6c97fd563e4 \
+    # via scancode-toolkit (setup.py)
+bitarray==0.8.1 \
+    --hash=sha256:1e8612b7244ff0e311e36fd24108935461ebae63fc62fbcb098c2cadb9850892 \
+    --hash=sha256:2d33af83a66775d8f6e961ffb63c04a936ddd51d5383c10d34ebcdf737d373e5 \
+    --hash=sha256:7da501356e48a83c61f479393681c1bc4b94e5a34ace7e08cb29e7dd9290ab18 \
+    --hash=sha256:8db28f3166fa3b0f77b04f9378b5e9a249967cba1d2e487647d1e5ef2353cc27 \
+    --hash=sha256:dcae0e9839ca59ec383ba998318584c8cc0b126852e7f217475274a1a8734006 \
+    # via scancode-toolkit (setup.py)
+boolean.py==3.7 \
+    --hash=sha256:82ae181f9c85cb5c893a5a4daba9f24d60b538a7dd27fd0c6752a77eba4fbeff \
+    # via license-expression, scancode-toolkit (setup.py)
+certifi==2018.4.16 \
+    --hash=sha256:9fa520c1bacfb634fa7af20a76bcbd3d5fb390481724c597da32c719a7dca4b0 \
+    # via requests
+chardet==3.0.4 \
+    --hash=sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691 \
+    # via binaryornot, normality, requests, scancode-toolkit (setup.py)
+click==6.7 \
+    --hash=sha256:29f99fc6125fbc931b758dc053b3114e55c77a6e4c6c3a2674a2dc986016381d \
+    # via scancode-toolkit (setup.py)
+colorama==0.3.9 \
+    --hash=sha256:463f8483208e921368c9f306094eb6f725c6ca42b0f97e313cb5d5512459feda \
+    # via scancode-toolkit (setup.py)
+contextlib2==0.5.5 \
+    --hash=sha256:f5260a6e679d2ff42ec91ec5252f4eeffdcf21053db9113bd0a8e4d953769c00 \
+    # via scancode-toolkit (setup.py)
+dparse==0.4.1 \
+    --hash=sha256:00a5fdfa900629e5159bf3600d44905b333f4059a3366f28e0dbd13eeab17b19 \
+    --hash=sha256:a14ebcee6074d5614245cf6f7dc8d21526412c866d06305745010bd091fd630c \
+    # via scancode-toolkit (setup.py)
+extractcode-7z==16.2.4.1 \
+    --hash=sha256:3eeaa645732505d8bc36cf2d4a38ad3bed03c8da4f14e89217d9495edeecbd5a \
+    # via scancode-toolkit (setup.py)
+extractcode-libarchive==3.4.2.1 \
+    --hash=sha256:83fb3f9a101adbf2ea8ce046d13634c2cfd1e7297a33e55feb790cad9844fda5 \
+    --hash=sha256:dc41439b863ccf1f9ed092c870e5537aafa227c23d90bd335382a17e5de9343b \
+    # via scancode-toolkit (setup.py)
+fingerprints==0.6.5.1 \
+    --hash=sha256:a80555655fc249f103369792d8ea1bd5870434b0a10d3a92ade6ad3e908fe4aa \
+    # via scancode-toolkit (setup.py)
+future==0.16.0 \
+    --hash=sha256:c132ffad1ed08491003ee07fcdc6fab43fa99b02560ad4cdff8e52a141a79bfb \
+    # via pefile, scancode-toolkit (setup.py)
+html5lib==0.999999999 \
+    --hash=sha256:b8934484cf22f1db684c0fae27569a0db404d0208d20163fbf51cc537245d008 \
+    # via scancode-toolkit (setup.py)
+idna==2.6 \
+    --hash=sha256:8c7309c718f94b3a625cb648ace320157ad16ff131ae0af362c9f21b80ef6ec4 \
+    # via requests
+importlib-metadata==0.18 \
+    --hash=sha256:6dfd58dfe281e8d240937776065dd3624ad5469c835248219bd16cf2e12dbeb7 \
+    # via pluggy
+intbitset==2.3.0 \
+    --hash=sha256:01505d26ba2b99fe04d2a5b6d166d8ca92f489a02faa6aa825855af047971da8 \
+    --hash=sha256:589aff5117f2660767e04499dabb6288ef88e05566d3ac1d9fa3bb7cac4d5ef5 \
+    --hash=sha256:5a38b1b338479027b1a75351a39e7d126682a2ea75d210146ffb735d1a767b6f \
+    --hash=sha256:7dc2fac52a3888ab29592854511588bdb45ea2247378be78595a59a69afbd5c3 \
+    --hash=sha256:db1b85620aa3ad3e2b8d448cf9f088b61c30a4c75f1be46e07eca32fbb407409 \
+    # via scancode-toolkit (setup.py)
+isodate==0.6.0 \
+    --hash=sha256:aa4d33c06640f5352aca96e4b81afd8ab3b47337cc12089822d6f322ac772c81 \
+    # via rdflib
+jaraco.functools==2.0 \
+    --hash=sha256:e9e377644cee5f6f9128b4dab1631fca74981236e95a255f80e4292bcd2b5284 \
+    # via scancode-toolkit (setup.py), tempora, yg.lockfile
+javaproperties==0.5.1 \
+    --hash=sha256:8bfb757116ed0589d88b4e13ecdc7d05a22c647a4645abf43ec27059430b1468 \
+    # via scancode-toolkit (setup.py)
+jinja2==2.10 \
+    --hash=sha256:74c935a1b8bb9a3947c50a54766a969d4846290e1e788ea44c1392163723c3bd \
+    # via scancode-toolkit (setup.py)
+license-expression==1.2 \
+    --hash=sha256:6d97906380cecfc758a77f6d38c6760f2afade7e83d2b8295e234fe21f486fb8 \
+    # via scancode-toolkit (setup.py)
+lxml==4.2.1 \
+    --hash=sha256:0e3cd94c95d30ba9ca3cff40e9b2a14e1a10a4fd8131105b86c6b61648f57e4b \
+    --hash=sha256:28f0c6652c1b130f1e576b60532f84b19379485eb8da6185c29bd8c9c9bc97bf \
+    --hash=sha256:691f2cd97cf026c611df1ea5055755eec7f878f2d4f4330dc8686583de6fc5fd \
+    --hash=sha256:8f37627f16e026523fca326f1b5c9a43534862fede6c3e99c2ba6a776d75c1ab \
+    --hash=sha256:e2629cdbcad82b83922a3488937632a4983ecc0fed3e5cfbf430d069382eeb9b \
+    # via pymaven-patch, scancode-toolkit (setup.py)
+markupsafe==1.0 \
+    --hash=sha256:198e2d4a8bfcf9d59edd8e870454c21a7c1fc8e7e4dde85f20f1a033ac539a4e \
+    --hash=sha256:a6be69091dac236ea9c6bc7d012beab42010fa914c459791d627dad4910eb665 \
+    # via jinja2
+more-itertools==5.0.0 \
+    --hash=sha256:fe7a7cae1ccb57d33952113ff4fa1bc5f879963600ed74918f1236e212ee50b9 \
+    # via jaraco.functools
+nltk==3.2 \
+    --hash=sha256:76d756a0c3949796068eb8c29feb7dc7fd534a98a4d348ec39429032c652230f \
+    --hash=sha256:cdbf05a212c5ac8ad2fe0eb65a5399ec6979f97206ddfac5e358a0feecec60bc \
+    # via scancode-toolkit (setup.py)
+normality==0.5.12 \
+    --hash=sha256:063fd1b8319241d02d1960f7f1ab112925a10469fcf02231cbe953f021761cf1 \
+    # via fingerprints
+packageurl-python==0.9.0 \
+    --hash=sha256:cf53de6196a1b9d507b077604f9cead63537418634cdd096cc073df1c373ddf7 \
+    # via scancode-toolkit (setup.py)
+packaging==19.0 \
+    --hash=sha256:9e1cbf8c12b1f1ce0bb5344b8d7ecf66a6f8a6e91bcb0c84593ed6d3ab5c4ab3 \
+    # via dparse
+patch==1.16 \
+    --hash=sha256:f132f44ca01557adbd8407d8f1935d9ecc351f12a13f64119254c0358706fc91 \
+    # via scancode-toolkit (setup.py)
+pdfminer.six==20181108 \
+    --hash=sha256:f04d029d1d3e58c87da51bdefef2e9a1dbf2d7b63f727dd2a3e36054f5ae96ea \
+    # via scancode-toolkit (setup.py)
+pefile==2019.4.18 \
+    --hash=sha256:6702305f555558ed0d31c78a03d7c4fc8203b7be63874f7b21b6f779c8bd7b13 \
+    --hash=sha256:a5d6e8305c6b210849b47a6174ddf9c452b2888340b8177874b862ba6c207645 \
+    # via scancode-toolkit (setup.py)
+pkginfo==1.5.0.1 \
+    --hash=sha256:a6d9e40ca61ad3ebd0b72fbadd4fba16e4c0e4df0428c041e01e06eb6ee71f32 \
+    # via scancode-toolkit (setup.py)
+pluggy==0.12.0 \
+    --hash=sha256:b9817417e95936bf75d85d3f8767f7df6cdde751fc40aed3bb3074cbcb77757c \
+    # via scancode-toolkit (setup.py)
+ply==3.9 \
+    --hash=sha256:4c76e737669f494cea2040c8a94e89b43949f7c663c2e40686e9d82fca8767f7 \
+    # via spdx-tools
+publicsuffix2==2.20191221 \
+    --hash=sha256:786b5e36205b88758bd3518725ec8cfe7a8173f5269354641f581c6b80a99893 \
+    # via scancode-toolkit (setup.py), urlpy
+pyahocorasick==1.4.0 \
+    --hash=sha256:14f7c371b351196ace8b7cbc3be74363c627a606196fadc3ac0ea917aa58321c \
+    --hash=sha256:46aff0f15af1895daa18fc7e7cb77d4ec8041328c4fc66188de730b12605681e \
+    --hash=sha256:62114ed7d25bf31e44c232cc0c698575c02a05048ed82afd0ab962a21a5a435d \
+    --hash=sha256:90f27fe09eb884fa4fda42d40dccc74a1ad0db9957609f1e0c5b574f315e4e58 \
+    # via scancode-toolkit (setup.py)
+pycryptodome==3.6.1 \
+    --hash=sha256:15013007e393d0cc0e69f4329a47c4c8597b7f3d02c12c03f805405542f70c71 \
+    --hash=sha256:4b5a2680008da3ac0cef2d3661597e0cbf8a3eb19eed35b859fd67e2de63eb85 \
+    --hash=sha256:e51da4ef9d9e2695a04044152f380c2db17adc9fc6fad8e24d863ead9cd548ed \
+    --hash=sha256:e850e07f54dc3de9a1efdd59d227fcd1cb30cdd307dafdc647c79e8f30cf5032 \
+    # via pdfminer.six, scancode-toolkit (setup.py)
+pygments==2.2.0 \
+    --hash=sha256:78f3f434bcc5d6ee09020f92ba487f95ba50f1e3ef83ae96b9d5ffa1bab25c5d \
+    # via scancode-toolkit (setup.py)
+pymaven-patch==0.2.9 \
+    --hash=sha256:478f91bf236f07c6445f03518849936e95f5b1edf52029c323fcf609ec579a65 \
+    # via scancode-toolkit (setup.py)
+pyparsing==2.4.0 \
+    --hash=sha256:9b6323ef4ab914af344ba97510e966d64ba91055d6b9afa6b30799340e89cc03 \
+    # via packaging, rdflib
+pytz==2019.2 \
+    --hash=sha256:c894d57500a4cd2d5c71114aaab77dbab5eabd9022308ce5ac9bb93a60a6f0c7 \
+    # via scancode-toolkit (setup.py), tempora
+pyyaml==3.12 \
+    --hash=sha256:592766c6303207a20efc445587778322d7f73b161bd994f227adaa341ba212ab \
+    --hash=sha256:bb35246803f4e91ccb7e160d4c872dc8ea93cd664082033b6feacd72e25653d3 \
+    --hash=sha256:d154a918f5da478a1823d0a4190408e4ad450ce8965de8d4913ef37a58e56563 \
+    # via dparse, fingerprints, saneyaml, spdx-tools
+rdflib==4.2.1 \
+    --hash=sha256:a737fd5af85484b74a60aa25dccfb2abaa6032c687645845db360d1f850d8321 \
+    # via spdx-tools
+requests==2.18.4 \
+    --hash=sha256:6a1b267aa90cac58ac3a765d067950e7dbbf75b1da07e895d1f594193a40a38b \
+    # via pymaven-patch, scancode-toolkit (setup.py)
+saneyaml==0.1 \
+    --hash=sha256:ffef9eed0de856fc2a1d5807c962d22e2b9bbadc0eba2e0375cb64c145006c96 \
+    # via scancode-toolkit (setup.py)
+simplejson==3.15.0 \
+    --hash=sha256:3073b4a18355f6a6260b7bd21f57281d9ed3ff80b6d33ad29f64d5c04863f65f \
+    --hash=sha256:7d130c4dd66815ba5483dc620ad72f586e5e12afe6b94e3a7336dbe09a04d809 \
+    --hash=sha256:ad332f65d9551ceffc132d0a683f4ffd12e4bc7538681100190d577ced3473fb \
+    # via scancode-toolkit (setup.py)
+six==1.12.0 \
+    --hash=sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c \
+    # via dparse, fingerprints, html5lib, intbitset, javaproperties, more-itertools, normality, packaging, pdfminer.six, pymaven-patch, scancode-toolkit (setup.py), spdx-tools, tempora
+sortedcontainers==2.0.5 \
+    --hash=sha256:220bb2e3e1886297fd7cdd6d164cb5cf237be1cfae1a3a3e526d149c52816682 \
+    # via pdfminer.six
+soupsieve==1.9.2 \
+    --hash=sha256:e4e9c053d59795e440163733a7fec6c5972210e1790c507e4c7b051d6c5259de \
+    # via beautifulsoup4
+spdx-tools==0.6.1 \
+    --hash=sha256:45fa09d43534772ca63bff2dccf1fbde88e1fc9b9515fc7a34ae50540d7ad29c \
+    # via scancode-toolkit (setup.py)
+tempora==1.14.1 \
+    --hash=sha256:d28a03d2f64ee81aec6e6bff374127ef306fe00c1b7e27c7ff1618344221a699 \
+    # via scancode-toolkit (setup.py), yg.lockfile
+text-unidecode==1.2 \
+    --hash=sha256:801e38bd550b943563660a91de8d4b6fa5df60a542be9093f7abf819f86050cc \
+    # via scancode-toolkit (setup.py)
+toml==0.10.0 \
+    --hash=sha256:229f81c57791a41d65e399fc06bf0848bab550a9dfd5ed66df18ce5f05e73d5c \
+    --hash=sha256:235682dd292d5899d361a811df37e04a8828a5b1da3115886b73cf81ebc9100e \
+    # via scancode-toolkit (setup.py)
+typecode-libmagic==5.38.1 \
+    --hash=sha256:434ab7d8dcc9a1de5137dab1e0e3dc2959502815e30c0fd9920f3d3b58ca39dc \
+    --hash=sha256:5bc822547d9537cc67e8bb7472eaf7d02ae172f72f22162e743ec3548974c226 \
+    # via scancode-toolkit (setup.py)
+typing==3.6.4 \
+    --hash=sha256:3a887b021a77b292e151afb75323dea88a7bc1b3dfa92176cff8e44c8b68bddf \
+    # via scancode-toolkit (setup.py)
+unicodecsv==0.14.1 \
+    --hash=sha256:e0e43259d7656da259f264cfa369e66d0643dbd944ffcd001be4df9c21bcf731 \
+    # via scancode-toolkit (setup.py)
+urllib3==1.22 \
+    --hash=sha256:06330f386d6e4b195fbfc736b297f58c5a892e4440e54d294d7004e3a9bbea1b \
+    # via requests
+urlpy==0.5 \
+    --hash=sha256:841673d97e0dd7a4d7ba47abd49fa8e3a61709e189e40de1b04b150ce7c5ed9f \
+    # via scancode-toolkit (setup.py)
+webencodings==0.5 \
+    --hash=sha256:36b29f8752ec757b0a3181a104db8aa327ffd8f1e4996c4c755f97f7837acb05 \
+    # via html5lib
+xmltodict==0.11.0 \
+    --hash=sha256:add07d92089ff611badec526912747cf87afd4f9447af6661aca074eeaf32615 \
+    # via scancode-toolkit (setup.py), spdx-tools
+yg.lockfile==2.3 \
+    --hash=sha256:b3fe554a90a7ee97393bc0d2f27068894c401c91b7d92b76c04348fd13b86570 \
+    # via scancode-toolkit (setup.py)
+zc.lockfile==2.0 \
+    --hash=sha256:cc33599b549f0c8a248cb72f3bf32d77712de1ff7ee8814312eb6456b42c015f \
+    # via scancode-toolkit (setup.py), yg.lockfile
+zipp==0.5.1 \
+    --hash=sha256:8c1019c6aad13642199fbe458275ad6a84907634cc9f0989877ccc4a2840139d \
+    # via importlib-metadata
+
+# The following packages are considered to be unsafe in a requirements file:
+setuptools==41.2.0 \
+    --hash=sha256:4380abcf2a4ffd1a5ba22d687c6d690dce83b2b51c70e9c6d09f7e8c7e8040dc \
+    # via html5lib, zc.lockfile

etc/conf/requirements_base.txt

@@ -1 +1,2 @@
--e .
+-r requirements.txt
+#-e .

etc/configure.py

@@ -217,10 +217,13 @@ def build_pip_dirs_args(paths, root_dir, option='--extra-search-dir='):
     list of `paths` to directories.
     """
     for path in paths:
-        if not os.path.isabs(path):
-            path = os.path.join(root_dir, path)
-        if os.path.exists(path):
-            yield option + quote(path)
+        if path.startswith('https'):
+            yield option + '"{}"'.format(path)
+        else:        
+            if not os.path.isabs(path):
+                path = os.path.join(root_dir, path)
+            if os.path.exists(path):
+                yield option + quote(path)
 
 
 def create_virtualenv(std_python, root_dir, tpp_dirs=(), quiet=False):
@@ -246,6 +249,8 @@ def create_virtualenv(std_python, root_dir, tpp_dirs=(), quiet=False):
     # search the virtualenv.pyz app in the tpp_dirs. keep the first found
     venv_pyz = None
     for tpd in tpp_dirs:
+        if tpd.startswith('https'):
+            continue
         venv = os.path.join(root_dir, tpd, 'virtualenv.pyz')
         if os.path.exists(venv):
             venv_pyz = venv
@@ -286,6 +291,15 @@ def install_3pp(configs, root_dir, tpp_dirs, quiet=False):
     run_pip(requirements, root_dir, tpp_dirs, quiet)
 
 
+def install_local_package(root_dir, tpp_dirs, quiet=False):
+    """
+    Install the current local package with pip,
+    using the vendored components in `tpp_dirs`.
+    """
+    requirements = ['--editable', '.']
+    run_pip(requirements, root_dir, tpp_dirs, quiet)
+
+
 def run_pip(requirements, root_dir, tpp_dirs, quiet=False):
     """
     Install a list of `requirements` with pip,
@@ -540,26 +554,30 @@ def save_activate_this_py_script(activate_path):
     for envvar, path in os.environ.items():
         if not envvar.startswith('TPP_DIR'):
             continue
-        abs_path = path
-        if not os.path.isabs(path):
-            abs_path = os.path.join(root_dir, path)
-        if not os.path.exists(abs_path):
-            if not quiet:
-                print()
-                print(
-                    'WARNING: Third-party Python libraries directory does not exists:\n'
-                    '  %(path)r: %(abs_path)r\n'
-                    '  Provided by environment variable:\n'
-                    '  set %(envvar)s=%(path)r' % locals())
-                print()
-        else:
+        if path.startswith('https'):
             thirdparty_dirs.append(path)
+        else:        
+            abs_path = path
+            if not os.path.isabs(path):
+                abs_path = os.path.join(root_dir, path)
+            if not os.path.exists(abs_path):
+                if not quiet:
+                    print()
+                    print(
+                        'WARNING: Third-party Python libraries directory does not exists:\n'
+                        '  %(path)r: %(abs_path)r\n'
+                        '  Provided by environment variable:\n'
+                        '  set %(envvar)s=%(path)r' % locals())
+                    print()
+            else:
+                thirdparty_dirs.append(path)
 
     # Finally execute our three steps: venv, install and scripts
     if not os.path.exists(configured_python):
         create_virtualenv(standard_python, root_dir, thirdparty_dirs, quiet=quiet)
     activate(root_dir)
     install_3pp(configs, root_dir, thirdparty_dirs, quiet=quiet)
+    install_local_package(root_dir, thirdparty_dirs, quiet=quiet)
     run_scripts(configs, root_dir, configured_python, quiet=quiet)
 
     if not quiet: