From 4e1317d03a38738eeec8feb1253afa5857806c88 Mon Sep 17 00:00:00 2001 From: Elias Dorneles Date: Sun, 14 Jun 2015 00:13:37 -0300 Subject: [PATCH 1/6] fix tests for python3 using decode() on pipe output --- shub/utils.py | 36 ++++++++++++++++++------------------ tests/test_logout.py | 8 +++++--- 2 files changed, 23 insertions(+), 21 deletions(-) diff --git a/shub/utils.py b/shub/utils.py index f806cdaf..783fad68 100644 --- a/shub/utils.py +++ b/shub/utils.py @@ -1,3 +1,4 @@ +from __future__ import unicode_literals import imp import os import netrc @@ -71,30 +72,29 @@ def make_deploy_request(url, data, files, auth): raise ClickException("Deploy failed: {}".format(exc)) +def get_cmd_output(cmd): + return Popen(cmd, stdout=PIPE).communicate()[0].decode() + + def pwd_git_version(): - p = Popen(['git', 'describe', '--always'], stdout=PIPE) - d = p.communicate()[0].strip('\n') - if p.wait() != 0: - p = Popen(['git', 'rev-list', '--count', 'HEAD'], stdout=PIPE) - d = 'r%s' % p.communicate()[0].strip('\n') + process = Popen(['git', 'describe', '--always'], stdout=PIPE) + commit_id = process.communicate()[0].decode().strip('\n') + if process.wait() != 0: + commit_id = get_cmd_output(['git', 'rev-list', '--count', 'HEAD']).strip('\n') - p = Popen(['git', 'rev-parse', '--abbrev-ref', 'HEAD'], stdout=PIPE) - b = p.communicate()[0].strip('\n') - return '%s-%s' % (d, b) + branch = get_cmd_output(['git', 'rev-parse', '--abbrev-ref', 'HEAD']).strip('\n') + return '%s-%s' % (commit_id, branch) def pwd_hg_version(): - p = Popen(['hg', 'tip', '--template', '{rev}'], stdout=PIPE) - d = 'r%s' % p.communicate()[0] - p = Popen(['hg', 'branch'], stdout=PIPE) - b = p.communicate()[0].strip('\n') - return '%s-%s' % (d, b) + commit_id = 'r%s' % get_cmd_output(['hg', 'tip', '--template', '{rev}']) + + branch = get_cmd_output(['hg', 'branch']).strip('\n') + return '%s-%s' % (commit_id, branch) def pwd_bzr_version(): - p = Popen(['bzr', 'revno'], stdout=PIPE) - d = '%s' % p.communicate()[0].strip() - return d + return '%s' % get_cmd_output(['bzr', 'revno']).strip() def run(cmd): @@ -163,7 +163,7 @@ def _deploy_dependency_egg(shub_apikey, project_id): def _get_dependency_name(): - return run('python setup.py --name') + return run('python setup.py --name').decode() def _get_dependency_version(name): @@ -174,7 +174,7 @@ def _get_dependency_version(name): elif isdir('.bzr'): return pwd_bzr_version() - return "%s-%s" % (name, run('python setup.py --version')) + return "%s-%s" % (name, run('python setup.py --version').decode()) def _get_egg_info(name): diff --git a/tests/test_logout.py b/tests/test_logout.py index d5302cf1..fb9db0b8 100644 --- a/tests/test_logout.py +++ b/tests/test_logout.py @@ -1,6 +1,8 @@ -import unittest, os -from shub.logout import remove_sh_key +import os +import unittest from tempfile import NamedTemporaryFile +from shub.logout import remove_sh_key + class LogoutTestCase(unittest.TestCase): _netrc_file = None @@ -18,7 +20,7 @@ def test_was_key_removed_from_netrc(self): def _create_tmp_netrc(self): with NamedTemporaryFile(delete=False) as netrc: line = 'machine scrapinghub.com login ffffffffffffffffffffffffffffffff password ""' - netrc.write(line) + netrc.write(line.encode()) return netrc.name def _delete_tmp_netrc(self, netrc_file): From a562ece16c6009fca947208339b4cd0ec602474a Mon Sep 17 00:00:00 2001 From: Elias Dorneles Date: Sun, 21 Jun 2015 01:12:28 -0300 Subject: [PATCH 2/6] use sys.stdout.encoding with utf-8 fallback for pipes --- shub/utils.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/shub/utils.py b/shub/utils.py index 783fad68..2139b014 100644 --- a/shub/utils.py +++ b/shub/utils.py @@ -3,6 +3,7 @@ import os import netrc import subprocess +import sys from glob import glob from os.path import isdir @@ -19,6 +20,9 @@ OS_WIN = True if os.name == 'nt' else False NETRC_FILE = os.path.expanduser('~/_netrc') if OS_WIN else os.path.expanduser('~/.netrc') +FALLBACK_ENCODING = 'utf-8' +STDOUT_ENCODING = sys.stdout.encoding or FALLBACK_ENCODING + def missing_modules(*modules): """Receives a list of module names and returns those which are missing""" @@ -73,12 +77,12 @@ def make_deploy_request(url, data, files, auth): def get_cmd_output(cmd): - return Popen(cmd, stdout=PIPE).communicate()[0].decode() + return Popen(cmd, stdout=PIPE).communicate()[0].decode(STDOUT_ENCODING) def pwd_git_version(): process = Popen(['git', 'describe', '--always'], stdout=PIPE) - commit_id = process.communicate()[0].decode().strip('\n') + commit_id = process.communicate()[0].decode(STDOUT_ENCODING).strip('\n') if process.wait() != 0: commit_id = get_cmd_output(['git', 'rev-list', '--count', 'HEAD']).strip('\n') @@ -99,7 +103,7 @@ def pwd_bzr_version(): def run(cmd): output = subprocess.check_output(cmd, shell=True) - return output.strip() + return output.decode(STDOUT_ENCODING).strip() def decompress_egg_files(): @@ -163,7 +167,7 @@ def _deploy_dependency_egg(shub_apikey, project_id): def _get_dependency_name(): - return run('python setup.py --name').decode() + return run('python setup.py --name') def _get_dependency_version(name): @@ -174,7 +178,7 @@ def _get_dependency_version(name): elif isdir('.bzr'): return pwd_bzr_version() - return "%s-%s" % (name, run('python setup.py --version').decode()) + return "%s-%s" % (name, run('python setup.py --version')) def _get_egg_info(name): From ebec8dda47601a8b1ae9d0c172854141f0b4ced8 Mon Sep 17 00:00:00 2001 From: Elias Dorneles Date: Sun, 21 Jun 2015 01:13:00 -0300 Subject: [PATCH 3/6] use explicit ascii encoding in test code --- tests/test_logout.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_logout.py b/tests/test_logout.py index fb9db0b8..12b41769 100644 --- a/tests/test_logout.py +++ b/tests/test_logout.py @@ -20,7 +20,7 @@ def test_was_key_removed_from_netrc(self): def _create_tmp_netrc(self): with NamedTemporaryFile(delete=False) as netrc: line = 'machine scrapinghub.com login ffffffffffffffffffffffffffffffff password ""' - netrc.write(line.encode()) + netrc.write(line.encode('ascii')) return netrc.name def _delete_tmp_netrc(self, netrc_file): From 354c53bb3f55fcd4da8058438b29331b5bfc5951 Mon Sep 17 00:00:00 2001 From: Elias Dorneles Date: Sun, 21 Jun 2015 01:13:29 -0300 Subject: [PATCH 4/6] enable pypy and py3{3,4} test environments on Travis --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index 6b4e8e7b..0f850036 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,6 +2,9 @@ language: python python: 2.7 env: - TOXENV=py27 +- TOXENV=pypy +- TOXENV=py33 +- TOXENV=py34 install: - "./.travis-workarounds.sh" - pip install -U tox From 485ddd74369bab41e2d8a5fb5a040a49415efa9c Mon Sep 17 00:00:00 2001 From: Elias Dorneles Date: Sun, 21 Jun 2015 01:51:23 -0300 Subject: [PATCH 5/6] minor py3 compatibility fixes --- shub/login.py | 2 +- shub/tool.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/shub/login.py b/shub/login.py index c570d1bc..55ec73b5 100644 --- a/shub/login.py +++ b/shub/login.py @@ -10,7 +10,7 @@ def cli(context): if key and is_valid_key(key): descriptor = os.open( NETRC_FILE, - os.O_CREAT | os.O_RDWR | os.O_APPEND, 0600) + os.O_CREAT | os.O_RDWR | os.O_APPEND, 0o600) with os.fdopen(descriptor, 'a+') as out: line = 'machine scrapinghub.com login {0} password ""\n'.format(key) out.write(line) diff --git a/shub/tool.py b/shub/tool.py index 443ee79f..31c37a95 100644 --- a/shub/tool.py +++ b/shub/tool.py @@ -26,7 +26,7 @@ def cli(): "version": [], } -for command, modules in module_deps.iteritems(): +for command, modules in module_deps.items(): m = missing_modules(*modules) if m: cli.add_command(missingmod_cmd(m), command) From 8bfbd3c5f57a807b54e6dd4446699ac82eda1a37 Mon Sep 17 00:00:00 2001 From: Elias Dorneles Date: Sun, 21 Jun 2015 02:04:57 -0300 Subject: [PATCH 6/6] using print function in py2 --- shub/deploy_reqs.py | 1 + 1 file changed, 1 insertion(+) diff --git a/shub/deploy_reqs.py b/shub/deploy_reqs.py index 1274dc14..484e613a 100644 --- a/shub/deploy_reqs.py +++ b/shub/deploy_reqs.py @@ -1,3 +1,4 @@ +from __future__ import print_function import click import os import tempfile