diff --git a/core/src/cgcloud/core/__init__.py b/core/src/cgcloud/core/__init__.py index 6b46c29..31714dd 100644 --- a/core/src/cgcloud/core/__init__.py +++ b/core/src/cgcloud/core/__init__.py @@ -48,4 +48,3 @@ def command_classes( ): return __fail_deprecated( sorted( locals( ).values( ), key=lambda cls: cls.__name__ ) ) -test_namespace_suffix_length = 11 diff --git a/core/src/cgcloud/core/test/__init__.py b/core/src/cgcloud/core/test/__init__.py index b049c44..5720289 100644 --- a/core/src/cgcloud/core/test/__init__.py +++ b/core/src/cgcloud/core/test/__init__.py @@ -1,63 +1,25 @@ import os import sys -import time from contextlib import contextmanager from itertools import ifilter -from struct import pack from tempfile import mkstemp -from unittest import TestCase import subprocess32 -from bd2k.util.d64 import D64 from bd2k.util.iterables import concat -from boto.utils import get_instance_metadata, logging +from boto.utils import logging -from cgcloud.core import test_namespace_suffix_length from cgcloud.core.cli import main, CGCloud -from cgcloud.lib import aws_d64 -from cgcloud.lib.context import Context -from cgcloud.lib.ec2 import running_on_ec2 +from cgcloud.lib.test import CgcloudTestCase log = logging.getLogger( __name__ ) - -class CgcloudTestCase( TestCase ): - """ - A base class for CGCloud test cases. When run with CGCLOUD_NAMESPACE unset, a new test - namespace will be prepared during setup and cleaned up during teardown. Otherwise, - the configured namespace will be used and not - """ - cleanup = True - ctx = None - __namespace = None - +class CoreTestCase( CgcloudTestCase ): @classmethod def setUpClass( cls ): CGCloud.setup_logging( ) CGCloud.silence_boto_and_paramiko( ) - super( CgcloudTestCase, cls ).setUpClass( ) - if running_on_ec2( ): - os.environ.setdefault( 'CGCLOUD_ZONE', - get_instance_metadata( )[ 'placement' ][ 'availability-zone' ] ) - # Using the d64 of a binary string that starts with a 4-byte, big-endian time stamp - # yields compact names whose lexicographical sorting is consistent with the historical - # order. We add the process ID so we can run tests concurrently in child processes using - # the pytest-xdist plugin. - suffix = aws_d64.encode( pack( '>II', int( time.time( ) ), os.getpid( ) ) ) - assert len( suffix ) == test_namespace_suffix_length - cls.__namespace = '/test/%s/' % suffix - os.environ.setdefault( 'CGCLOUD_NAMESPACE', cls.__namespace ) - cls.ctx = Context( os.environ[ 'CGCLOUD_ZONE' ], os.environ[ 'CGCLOUD_NAMESPACE' ] ) - - @classmethod - def tearDownClass( cls ): - # Only cleanup if the context is using the default test namespace. If another namespace - # is configured, we can't assume that all resources were created by the test and that - # they can therefore be removed. - if cls.cleanup and cls.ctx.namespace == cls.__namespace: - cls.ctx.reset_namespace_security( ) - super( CgcloudTestCase, cls ).tearDownClass( ) + super( CoreTestCase, cls ).setUpClass( ) ssh_opts = ('-o', 'UserKnownHostsFile=/dev/null', '-o', 'StrictHostKeyChecking=no') @@ -107,8 +69,9 @@ def _cgcloud( cls, *args ): else: main( args ) + @contextmanager -def out_stderr(): +def out_stderr( ): with open( os.devnull, 'a' ) as f: f, sys.stderr = sys.stderr, f try: diff --git a/core/src/cgcloud/core/test/test_core.py b/core/src/cgcloud/core/test/test_core.py index 735dbae..17da377 100644 --- a/core/src/cgcloud/core/test/test_core.py +++ b/core/src/cgcloud/core/test/test_core.py @@ -4,12 +4,12 @@ from bd2k.util.exceptions import panic from cgcloud.core import roles -from cgcloud.core.test import CgcloudTestCase, out_stderr +from cgcloud.core.test import CoreTestCase, out_stderr log = logging.getLogger( __name__ ) -class CoreTests( CgcloudTestCase ): +class CoreTests( CoreTestCase ): """ Tests the typical life-cycle of instances and images """ diff --git a/jenkins/src/cgcloud/jenkins/cgcloud_jenkins_slave.py b/jenkins/src/cgcloud/jenkins/cgcloud_jenkins_slave.py index c7807a5..0cab01d 100644 --- a/jenkins/src/cgcloud/jenkins/cgcloud_jenkins_slave.py +++ b/jenkins/src/cgcloud/jenkins/cgcloud_jenkins_slave.py @@ -1,6 +1,6 @@ -from cgcloud.core import test_namespace_suffix_length from cgcloud.core.common_iam_policies import ec2_full_policy from cgcloud.core.ubuntu_box import Python27UpdateUbuntuBox +from cgcloud.lib import test_namespace_suffix_length from cgcloud.lib.util import abreviated_snake_case_class_name from cgcloud.jenkins.generic_jenkins_slaves import UbuntuTrustyGenericJenkinsSlave diff --git a/lib/src/cgcloud/lib/__init__.py b/lib/src/cgcloud/lib/__init__.py index 05d31ed..e632ee7 100644 --- a/lib/src/cgcloud/lib/__init__.py +++ b/lib/src/cgcloud/lib/__init__.py @@ -1,3 +1,5 @@ from bd2k.util.d64 import D64 aws_d64 = D64( '.-' ) # hopefully the dot is supported for all AWS resource names + +test_namespace_suffix_length = 11 diff --git a/lib/src/cgcloud/lib/test/__init__.py b/lib/src/cgcloud/lib/test/__init__.py new file mode 100644 index 0000000..3f66433 --- /dev/null +++ b/lib/src/cgcloud/lib/test/__init__.py @@ -0,0 +1,47 @@ +import os +import time +from struct import pack +from unittest import TestCase + +from boto.utils import get_instance_metadata + +from cgcloud.lib import aws_d64, test_namespace_suffix_length +from cgcloud.lib.context import Context +from cgcloud.lib.ec2 import running_on_ec2 + + +class CgcloudTestCase( TestCase ): + """ + A base class for CGCloud test cases. When run with CGCLOUD_NAMESPACE unset, a new test + namespace will be prepared during setup and cleaned up during teardown. Otherwise, + the configured namespace will be used but not cleaned up. + """ + __namespace = None + cleanup = True + ctx = None + + @classmethod + def setUpClass( cls ): + super( CgcloudTestCase, cls ).tearDownClass( ) + if running_on_ec2( ): + os.environ.setdefault( 'CGCLOUD_ZONE', + get_instance_metadata( )[ 'placement' ][ 'availability-zone' ] ) + # Using the d64 of a binary string that starts with a 4-byte, big-endian time stamp + # yields compact names whose lexicographical sorting is consistent with the historical + # order. We add the process ID so we can run tests concurrently in child processes using + # the pytest-xdist plugin. + suffix = aws_d64.encode( pack( '>II', int( time.time( ) ), os.getpid( ) ) ) + assert len( suffix ) == test_namespace_suffix_length + cls.__namespace = '/test/%s/' % suffix + os.environ.setdefault( 'CGCLOUD_NAMESPACE', cls.__namespace ) + cls.ctx = Context( availability_zone=os.environ[ 'CGCLOUD_ZONE' ], + namespace=os.environ[ 'CGCLOUD_NAMESPACE' ] ) + + @classmethod + def tearDownClass( cls ): + # Only cleanup if the context is using the default test namespace. If another namespace + # is configured, we can't assume that all resources were created by the test and that + # they can therefore be removed. + if cls.cleanup and cls.ctx.namespace == cls.__namespace: + cls.ctx.reset_namespace_security( ) + super( CgcloudTestCase, cls ).setUpClass( ) diff --git a/mesos/src/cgcloud/mesos/test/__init__.py b/mesos/src/cgcloud/mesos/test/__init__.py index 09f922d..5da04d1 100644 --- a/mesos/src/cgcloud/mesos/test/__init__.py +++ b/mesos/src/cgcloud/mesos/test/__init__.py @@ -1,10 +1,10 @@ import time -from cgcloud.core.test import CgcloudTestCase +from cgcloud.core.test import CoreTestCase from cgcloud.mesos.mesos_box import log_dir -class MesosTestCase( CgcloudTestCase ): +class MesosTestCase( CoreTestCase ): """ Common functionality between Toil and Mesos tests """ diff --git a/spark/src/cgcloud/spark/test/test_spark.py b/spark/src/cgcloud/spark/test/test_spark.py index 31a3aff..3f7acb8 100644 --- a/spark/src/cgcloud/spark/test/test_spark.py +++ b/spark/src/cgcloud/spark/test/test_spark.py @@ -5,7 +5,7 @@ import logging import unittest -from cgcloud.core.test import CgcloudTestCase +from cgcloud.core.test import CoreTestCase from cgcloud.spark.spark_box import install_dir, SparkBox, SparkMaster, SparkSlave log = logging.getLogger( __name__ ) @@ -17,7 +17,7 @@ num_slaves = 2 -class SparkClusterTests( CgcloudTestCase ): +class SparkClusterTests( CoreTestCase ): """ Covers the creation of a Spark cluster from scratch and running a simple Spark job on it. Also covers persistant HDFS between two cluster incarnations.