Installing packages that references an git repository in requirements.txt fails.

[AraHaan]

Here is an example package's reuirements.txt and setup.py that is hosted on github only that fails every time it is attempted to be installed using pip.

Also happens when running setup.py directly without pip.

requirements.txt:

regex
aioredis
colorama
pycares
discord.py
aiohttp
PyNacl
cchardet
cffi
multidict
yarl
discord.webhooks
TinyURL3
consolechange
git+https://github.com/AraHaan/dbapi.git
youtube_dl

setup.py:

from setuptools import setup
from setuptools.extension import Extension
import sys


def get_requirements():
    """
    lists the requirements to install.
    """
    requirements = []
    try:
        with open('requirements.txt') as f:
            requirements = f.read().splitlines()
    except Exception as ex:
        with open('DecoraterBotUtils.egg-info\requires.txt') as f:
            requirements = f.read().splitlines()
    return requirements


def get_version():
    """
    returns version.
    """
    return '0.0.1'


def get_extensions():
    """
    lists the extensions to build with an compiler.
    """
    if sys.platform != 'cygwin':
        BotErrors = Extension(
            'DecoraterBotUtils.BotErrors', [
                'DecoraterBotUtils/BotErrors.c'])
    else:
        BotErrors = Extension(
            'DecoraterBotUtils.BotErrors',
            library_dirs=['/usr/local/bin'],
            sources=['DecoraterBotUtils/BotErrors.c'])
    return [BotErrors]


if not get_version():
    raise RuntimeError('version is not set')

try:
    with open('README.rst') as f:
        readme = f.read()
except FileNotFoundError:
    readme = ""

setup(name='DecoraterBotUtils',
      author='Decorater',
      author_email='[email protected]',
      url='https://github.com/DecoraterBot-devs/DecoraterBotUtils',
      bugtrack_url='https://github.com/DecoraterBot-devs/DecoraterBotUtils/issues',
      version=get_version(),
      packages=['DecoraterBotUtils'],
      ext_modules=get_extensions(),
      license='MIT',
      description='This package is for bringing various things to DecoraterBot.',
      long_description=readme,
      maintainer_email='[email protected]',
      download_url='https://github.com/DecoraterBot-devs/DecoraterBotUtils',
      include_package_data=True,
      install_requires=get_requirements(),
      platforms='Any',
      classifiers=[
        'Development Status :: 5 - Production/Stable',
        'Intended Audience :: Developers',
        'License :: OSI Approved :: MIT License',
        'Natural Language :: English',
        'Operating System :: OS Independent',
        'Programming Language :: C',
        'Programming Language :: Python :: 3',
        'Programming Language :: Python :: Implementation :: CPython',
        'Topic :: Software Development :: Libraries',
        'Topic :: Software Development :: Libraries :: Python Modules',
      ]
)

output with pip:

E:\Users\Elsword>cd E:\DecoraterBotUtils

E:\DecoraterBotUtils>python -m pip install --upgrade .
Processing e:\decoraterbotutils
    Complete output from command python setup.py egg_info:
    E:\python360\lib\distutils\dist.py:261: UserWarning: Unknown distribution op
tion: 'bugtrack_url'
      warnings.warn(msg)
    error in DecoraterBotUtils setup command: 'install_requires' must be a strin
g or list of strings containing valid project/version requirement specifiers; In
valid requirement, parse error at "'+https:/'"

    ----------------------------------------
Command "python setup.py egg_info" failed with error code 1 in E:\Users\Elsword\
AppData\Local\Temp\pip-7eq1kj18-build\

E:\DecoraterBotUtils>

output without pip:

E:\DecoraterBotUtils>python setup.py install .
E:\python360\lib\distutils\dist.py:261: UserWarning: Unknown distribution option
: 'bugtrack_url'
  warnings.warn(msg)
error in DecoraterBotUtils setup command: 'install_requires' must be a string or
 list of strings containing valid project/version requirement specifiers; Invali
d requirement, parse error at "'+https:/'"

When entering the get_requirements and executing it directly in python's interactive mode:

E:\DecoraterBotUtils>python
Python 3.6.1+ (heads/3.6:33a5568f69, Apr 27 2017, 00:35:08) [MSC v.1900 32 bit (
Intel)] on win32
Type "help", "copyright", "credits" or "license" for more information.
>>> def get_requirements():
...     """
...     lists the requirements to install.
...     """
...     requirements = []
...     try:
...         with open('requirements.txt') as f:
...             requirements = f.read().splitlines()
...     except Exception as ex:
...         with open('DecoraterBotUtils.egg-info\requires.txt') as f:
...             requirements = f.read().splitlines()
...     return requirements
...
>>> get_requirements()
['regex', 'aioredis', 'colorama', 'pycares', 'discord.py', 'aiohttp', 'PyNacl',
'cchardet', 'cffi', 'multidict', 'yarl', 'discord.webhooks', 'TinyURL3', 'consol
echange', 'git+https://github.com/AraHaan/dbapi.git', 'youtube_dl']
>>>

In interactive mode it works as expected (without executing setuptools) but in pip for some reason it fails when it should not fail. Unless setuptools cannot process the vcs's properly that pip gladly supports and is able to take in and process just fine.

[FRidh]

install-requires takes a list of names with optionally versions. If you want to pass in a link, you have to use dependency_links.

Furthermore, don't load a requirements.txt file. I know this is commonly done but this file is not for installation dependencies but instead for listing packages one wants in an environment.

[AraHaan]

As much as that is said I know of a few packages that use an requirements.txt file to install dependencies when you pip install it (discord.py being one of said packages).

I guess I could modify the code to where if it finds an git+ or whatever to remove it from the list and into another one. But it would have to know all possible svn's people can link or use on it.

Edit: here is my work arround setup.py.

from setuptools import setup
from setuptools.extension import Extension
import sys


def get_requirements(remove_links=True):
    """
    lists the requirements to install.
    """
    requirements = []
    try:
        with open('requirements.txt') as f:
            requirements = f.read().splitlines()
    except Exception as ex:
        with open('DecoraterBotUtils.egg-info\requires.txt') as f:
            requirements = f.read().splitlines()
    if remove_links:
        for requirement in requirements:
        # git repository url.
        if requirement.startswith("git+"):
            requirements.remove(requirement)
        # subversion repository url.
        if requirement.startswith("svn+"):
            requirements.remove(requirement)
        # mercurial repository url.
        if requirement.startswith("hg+"):
            requirements.remove(requirement)
    return requirements


def get_links():
    """
    gets URL Dependency links.
    """
    links_list = get_requirements(remove_links=False)
    for link in links_list:
        keep_link = False
        already_removed = False
        # git repository url.
        if not link.startswith("git+"):
            if not link.startswith("svn+"):
                if not link.startswith("hg+"):
                    links_list.remove(link)
                    already_removed = True
                else:
                    keep_link = True
                if not keep_link and not already_removed:
                    links_list.remove(link)
                    already_removed = True
            else:
                keep_link = True
            if not keep_link and not already_removed:
                links_list.remove(link)
    return links_list


def get_version():
    """
    returns version.
    """
    return '0.0.1'


def get_extensions():
    """
    lists the extensions to build with an compiler.
    """
    if sys.platform != 'cygwin':
        BotErrors = Extension(
            'DecoraterBotUtils.BotErrors', [
                'DecoraterBotUtils/BotErrors.c'])
    else:
        BotErrors = Extension(
            'DecoraterBotUtils.BotErrors',
            library_dirs=['/usr/local/bin'],
            sources=['DecoraterBotUtils/BotErrors.c'])
    return [BotErrors]


if not get_version():
    raise RuntimeError('version is not set')

try:
    with open('README.rst') as f:
        readme = f.read()
except FileNotFoundError:
    readme = ""

setup(name='DecoraterBotUtils',
      author='Decorater',
      author_email='[email protected]',
      url='https://github.com/DecoraterBot-devs/DecoraterBotUtils',
      bugtrack_url='https://github.com/DecoraterBot-devs/DecoraterBotUtils/issues',
      version=get_version(),
      packages=['DecoraterBotUtils'],
      ext_modules=get_extensions(),
      license='MIT',
      description='This package is for bringing various things to DecoraterBot.',
      long_description=readme,
      maintainer_email='[email protected]',
      download_url='https://github.com/DecoraterBot-devs/DecoraterBotUtils',
      include_package_data=True,
      install_requires=get_requirements(),
      dependency_links=get_links(),
      platforms='Any',
      classifiers=[
        'Development Status :: 5 - Production/Stable',
        'Intended Audience :: Developers',
        'License :: OSI Approved :: MIT License',
        'Natural Language :: English',
        'Operating System :: OS Independent',
        'Programming Language :: C',
        'Programming Language :: Python :: 3',
        'Programming Language :: Python :: Implementation :: CPython',
        'Topic :: Software Development :: Libraries',
        'Topic :: Software Development :: Libraries :: Python Modules',
      ]
)

[hnykda]

We were able to make it work only using pypa/pip#3610 (comment)

[hnykda]

We were able to make it work only using pypa/pip#3610 (comment)