From 2577a624c0b1ce157a2ed52820dfa6a2db1662d4 Mon Sep 17 00:00:00 2001 From: Morris Jobke Date: Wed, 11 Feb 2015 01:06:07 +0100 Subject: [PATCH 1/7] [files_external] swift tests --- apps/files_external/tests/backends/swift.php | 9 ++- .../tests/env/start-swift-morrisjobke.sh | 66 +++++++++++++++++++ .../tests/env/stop-swift-morrisjobke.sh | 36 ++++++++++ 3 files changed, 106 insertions(+), 5 deletions(-) create mode 100755 apps/files_external/tests/env/start-swift-morrisjobke.sh create mode 100755 apps/files_external/tests/env/stop-swift-morrisjobke.sh diff --git a/apps/files_external/tests/backends/swift.php b/apps/files_external/tests/backends/swift.php index 2e6670f84f84..4ca2d1db7bd8 100644 --- a/apps/files_external/tests/backends/swift.php +++ b/apps/files_external/tests/backends/swift.php @@ -32,18 +32,17 @@ class Swift extends Storage { protected function setUp() { parent::setUp(); - $this->config = include('files_external/tests/config.php'); - if (!is_array($this->config) or !isset($this->config['swift']) - or !$this->config['swift']['run']) { + $this->config = include('files_external/tests/config.swift.php'); + if (!is_array($this->config) or !$this->config['run']) { $this->markTestSkipped('OpenStack Object Storage backend not configured'); } - $this->instance = new \OC\Files\Storage\Swift($this->config['swift']); + $this->instance = new \OC\Files\Storage\Swift($this->config); } protected function tearDown() { if ($this->instance) { $connection = $this->instance->getConnection(); - $container = $connection->getContainer($this->config['swift']['bucket']); + $container = $connection->getContainer($this->config['bucket']); $objects = $container->objectList(); while($object = $objects->next()) { diff --git a/apps/files_external/tests/env/start-swift-morrisjobke.sh b/apps/files_external/tests/env/start-swift-morrisjobke.sh new file mode 100755 index 000000000000..d6cd0d22e9da --- /dev/null +++ b/apps/files_external/tests/env/start-swift-morrisjobke.sh @@ -0,0 +1,66 @@ +#!/bin/bash +# +# ownCloud +# +# This script start a docker container to test the files_external tests +# against. It will also change the files_external config to use the docker +# container as testing environment. This is reverted in the stop step.W +# +# Set environment variable DEBUG to print config file +# +# @author Morris Jobke +# @copyright 2015 Morris Jobke +# + +if ! command -v docker >/dev/null 2>&1; then + echo "No docker executable found - skipped docker setup" + exit 0; +fi + +echo "Docker executable found - setup docker" + +echo "Fetch recent morrisjobke/docker-swift-onlyone docker image" +docker pull morrisjobke/docker-swift-onlyone + +# retrieve current folder to place the config in the parent folder +thisFolder=`echo $0 | replace "env/start-swift-morrisjobke.sh" ""` + +if [ -z "$thisFolder" ]; then + thisFolder="." +fi; + +container=`docker run -d -e SWIFT_SET_PASSWORDS=true morrisjobke/docker-swift-onlyone` + +host=`docker inspect $container | grep IPAddress | cut -d '"' -f 4` + + +echo "swift container: $container" + +# put container IDs into a file to drop them after the test run (keep in mind that multiple tests run in parallel on the same host) +echo $container >> $thisFolder/dockerContainerMorrisJobke.$EXECUTOR_NUMBER.swift + +# TODO find a way to determine the successful initialization inside the docker container +echo "Waiting 15 seconds for swift initialization ... " +sleep 15 + +user=test:tester +password=`docker logs $container | grep "user_test_tester " | cut -d " " -f3` + +cat > $thisFolder/config.swift.php <true, + 'url'=>'http://$host:8080/auth/v1.0', + 'user'=>'$user', + 'key'=>'$password', + 'bucket'=>'swift', + 'region' => 'DFW', +); + +DELIM + +if [ -n "$DEBUG" ]; then + cat $thisFolder/config.swift.php + cat $thisFolder/dockerContainerMorrisJobke.$EXECUTOR_NUMBER.swift +fi diff --git a/apps/files_external/tests/env/stop-swift-morrisjobke.sh b/apps/files_external/tests/env/stop-swift-morrisjobke.sh new file mode 100755 index 000000000000..f1660e65854a --- /dev/null +++ b/apps/files_external/tests/env/stop-swift-morrisjobke.sh @@ -0,0 +1,36 @@ +#!/bin/bash +# +# ownCloud +# +# This script stops the docker container the files_external tests were run +# against. It will also revert the config changes done in start step. +# +# @author Morris Jobke +# @copyright 2015 Morris Jobke +# + +if ! command -v docker >/dev/null 2>&1; then + echo "No docker executable found - skipped docker stop" + exit 0; +fi + +echo "Docker executable found - stop and remove docker containers" + +# retrieve current folder to remove the config from the parent folder +thisFolder=`echo $0 | replace "env/stop-swift-morrisjobke.sh" ""` + +if [ -z "$thisFolder" ]; then + thisFolder="." +fi; + +# stopping and removing docker containers +for container in `cat $thisFolder/dockerContainerMorrisJobke.$EXECUTOR_NUMBER.swift`; do + echo "Stopping and removing docker container $container" + # kills running container and removes it + docker rm -f $container +done; + +# cleanup +rm $thisFolder/config.swift.php +rm $thisFolder/dockerContainerMorrisJobke.$EXECUTOR_NUMBER.swift + From 0de9819da378590fee73c805b5e52661452eb141 Mon Sep 17 00:00:00 2001 From: Robin McCorkell Date: Wed, 9 Sep 2015 22:06:41 +0100 Subject: [PATCH 2/7] Replace Swift Docker test with Ceph image --- ...ift-morrisjobke.sh => start-swift-ceph.sh} | 50 ++++++++++++------- ...wift-morrisjobke.sh => stop-swift-ceph.sh} | 10 ++-- 2 files changed, 38 insertions(+), 22 deletions(-) rename apps/files_external/tests/env/{start-swift-morrisjobke.sh => start-swift-ceph.sh} (53%) rename apps/files_external/tests/env/{stop-swift-morrisjobke.sh => stop-swift-ceph.sh} (72%) diff --git a/apps/files_external/tests/env/start-swift-morrisjobke.sh b/apps/files_external/tests/env/start-swift-ceph.sh similarity index 53% rename from apps/files_external/tests/env/start-swift-morrisjobke.sh rename to apps/files_external/tests/env/start-swift-ceph.sh index d6cd0d22e9da..c2e7d4c1ae15 100755 --- a/apps/files_external/tests/env/start-swift-morrisjobke.sh +++ b/apps/files_external/tests/env/start-swift-ceph.sh @@ -9,8 +9,8 @@ # Set environment variable DEBUG to print config file # # @author Morris Jobke -# @copyright 2015 Morris Jobke -# +# @author Robin McCorkell +# @copyright 2015 ownCloud if ! command -v docker >/dev/null 2>&1; then echo "No docker executable found - skipped docker setup" @@ -19,48 +19,64 @@ fi echo "Docker executable found - setup docker" -echo "Fetch recent morrisjobke/docker-swift-onlyone docker image" -docker pull morrisjobke/docker-swift-onlyone +docker_image=xenopathic/ceph-keystone + +echo "Fetch recent ${docker_image} docker image" +docker pull ${docker_image} # retrieve current folder to place the config in the parent folder -thisFolder=`echo $0 | replace "env/start-swift-morrisjobke.sh" ""` +thisFolder=`echo $0 | replace "env/start-swift-ceph.sh" ""` if [ -z "$thisFolder" ]; then thisFolder="." fi; -container=`docker run -d -e SWIFT_SET_PASSWORDS=true morrisjobke/docker-swift-onlyone` +port=5001 + +user=test +pass=testing +tenant=testenant +region=testregion +service=testceph + +container=`docker run -d \ + -e KEYSTONE_PUBLIC_PORT=${port} \ + -e KEYSTONE_ADMIN_USER=${user} \ + -e KEYSTONE_ADMIN_PASS=${pass} \ + -e KEYSTONE_ADMIN_TENANT=${tenant} \ + -e KEYSTONE_ENDPOINT_REGION=${region} \ + -e KEYSTONE_SERVICE=${service} \ + ${docker_image}` host=`docker inspect $container | grep IPAddress | cut -d '"' -f 4` -echo "swift container: $container" +echo "${docker_image} container: $container" # put container IDs into a file to drop them after the test run (keep in mind that multiple tests run in parallel on the same host) -echo $container >> $thisFolder/dockerContainerMorrisJobke.$EXECUTOR_NUMBER.swift +echo $container >> $thisFolder/dockerContainerCeph.$EXECUTOR_NUMBER.swift # TODO find a way to determine the successful initialization inside the docker container -echo "Waiting 15 seconds for swift initialization ... " -sleep 15 - -user=test:tester -password=`docker logs $container | grep "user_test_tester " | cut -d " " -f3` +echo "Waiting 20 seconds for ceph initialization ... " +sleep 20 cat > $thisFolder/config.swift.php <true, - 'url'=>'http://$host:8080/auth/v1.0', + 'url'=>'http://$host:$port/v2.0', 'user'=>'$user', - 'key'=>'$password', + 'tenant'=>'$tenant', + 'password'=>'$pass', + 'service_name'=>'$service', 'bucket'=>'swift', - 'region' => 'DFW', + 'region' => '$region', ); DELIM if [ -n "$DEBUG" ]; then cat $thisFolder/config.swift.php - cat $thisFolder/dockerContainerMorrisJobke.$EXECUTOR_NUMBER.swift + cat $thisFolder/dockerContainerCeph.$EXECUTOR_NUMBER.swift fi diff --git a/apps/files_external/tests/env/stop-swift-morrisjobke.sh b/apps/files_external/tests/env/stop-swift-ceph.sh similarity index 72% rename from apps/files_external/tests/env/stop-swift-morrisjobke.sh rename to apps/files_external/tests/env/stop-swift-ceph.sh index f1660e65854a..edac1389a785 100755 --- a/apps/files_external/tests/env/stop-swift-morrisjobke.sh +++ b/apps/files_external/tests/env/stop-swift-ceph.sh @@ -6,8 +6,8 @@ # against. It will also revert the config changes done in start step. # # @author Morris Jobke -# @copyright 2015 Morris Jobke -# +# @author Robin McCorkell +# @copyright 2015 ownCloud if ! command -v docker >/dev/null 2>&1; then echo "No docker executable found - skipped docker stop" @@ -17,14 +17,14 @@ fi echo "Docker executable found - stop and remove docker containers" # retrieve current folder to remove the config from the parent folder -thisFolder=`echo $0 | replace "env/stop-swift-morrisjobke.sh" ""` +thisFolder=`echo $0 | replace "env/stop-swift-ceph.sh" ""` if [ -z "$thisFolder" ]; then thisFolder="." fi; # stopping and removing docker containers -for container in `cat $thisFolder/dockerContainerMorrisJobke.$EXECUTOR_NUMBER.swift`; do +for container in `cat $thisFolder/dockerContainerCeph.$EXECUTOR_NUMBER.swift`; do echo "Stopping and removing docker container $container" # kills running container and removes it docker rm -f $container @@ -32,5 +32,5 @@ done; # cleanup rm $thisFolder/config.swift.php -rm $thisFolder/dockerContainerMorrisJobke.$EXECUTOR_NUMBER.swift +rm $thisFolder/dockerContainerCeph.$EXECUTOR_NUMBER.swift From 70f4e9413fc4b7f676ecdb6b437751182a07dc56 Mon Sep 17 00:00:00 2001 From: Robin McCorkell Date: Wed, 23 Sep 2015 20:31:22 +0100 Subject: [PATCH 3/7] Make Swift test robust if container didn't exist --- apps/files_external/tests/backends/swift.php | 22 ++++++++++++-------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/apps/files_external/tests/backends/swift.php b/apps/files_external/tests/backends/swift.php index 4ca2d1db7bd8..83b27e6e82e4 100644 --- a/apps/files_external/tests/backends/swift.php +++ b/apps/files_external/tests/backends/swift.php @@ -41,16 +41,20 @@ protected function setUp() { protected function tearDown() { if ($this->instance) { - $connection = $this->instance->getConnection(); - $container = $connection->getContainer($this->config['bucket']); - - $objects = $container->objectList(); - while($object = $objects->next()) { - $object->setName(str_replace('#','%23',$object->getName())); - $object->delete(); + try { + $connection = $this->instance->getConnection(); + $container = $connection->getContainer($this->config['bucket']); + + $objects = $container->objectList(); + while($object = $objects->next()) { + $object->setName(str_replace('#','%23',$object->getName())); + $object->delete(); + } + + $container->delete(); + } catch (\Guzzle\Http\Exception\ClientErrorResponseException $e) { + // container didn't exist, so we don't need to delete it } - - $container->delete(); } parent::tearDown(); From 8467dbfa91921ccb0262161360edbf339f6b7daa Mon Sep 17 00:00:00 2001 From: Robin McCorkell Date: Wed, 23 Sep 2015 22:33:39 +0100 Subject: [PATCH 4/7] Detect Docker initialization properly Use netcat to check when the port opens, rather than an arbitrary timeout. Hard limit of 60 seconds in case something breaks --- .../tests/env/start-amazons3-ceph.sh | 18 ++++++++++--- .../tests/env/start-ftp-morrisjobke.sh | 19 +++++++++++--- .../tests/env/start-sftp-atmoz.sh | 20 +++++++++++--- .../tests/env/start-smb-silvershell.sh | 19 +++++++++++--- .../tests/env/start-swift-ceph.sh | 18 ++++++++++--- .../tests/env/start-webdav-ownCloud.sh | 26 +++++++++++++------ 6 files changed, 96 insertions(+), 24 deletions(-) diff --git a/apps/files_external/tests/env/start-amazons3-ceph.sh b/apps/files_external/tests/env/start-amazons3-ceph.sh index ad0fedba9890..f2beb0bd7140 100755 --- a/apps/files_external/tests/env/start-amazons3-ceph.sh +++ b/apps/files_external/tests/env/start-amazons3-ceph.sh @@ -49,9 +49,21 @@ echo "${docker_image} container: $container" # put container IDs into a file to drop them after the test run (keep in mind that multiple tests run in parallel on the same host) echo $container >> $thisFolder/dockerContainerCeph.$EXECUTOR_NUMBER.amazons3 -# TODO find a way to determine the successful initialization inside the docker container -echo "Waiting 20 seconds for ceph initialization ... " -sleep 20 +echo -n "Waiting for ceph initialization" +starttime=$(date +%s) +# support for GNU netcat and BSD netcat +while ! (nc -c -w 1 ${host} ${port} &/dev/null \ + || nc -w 1 ${host} ${port} &/dev/null); do + sleep 1 + echo -n '.' + if (( $(date +%s) > starttime + 60 )); then + echo + echo "[ERROR] Waited 60 seconds, no response" >&2 + exit 1 + fi +done +echo +sleep 1 echo "Create ceph user" docker exec $container radosgw-admin user create \ diff --git a/apps/files_external/tests/env/start-ftp-morrisjobke.sh b/apps/files_external/tests/env/start-ftp-morrisjobke.sh index 14112d7f8037..35e3eafb8390 100755 --- a/apps/files_external/tests/env/start-ftp-morrisjobke.sh +++ b/apps/files_external/tests/env/start-ftp-morrisjobke.sh @@ -54,12 +54,25 @@ echo "ftp container: $container" # put container IDs into a file to drop them after the test run (keep in mind that multiple tests run in parallel on the same host) echo $container >> $thisFolder/dockerContainerMorrisJobke.$EXECUTOR_NUMBER.ftp +echo -n "Waiting for ftp initialization" +starttime=$(date +%s) +# support for GNU netcat and BSD netcat +while ! (nc -c -w 1 ${host} 21 &/dev/null \ + || nc -w 1 ${host} 21 &/dev/null); do + sleep 1 + echo -n '.' + if (( $(date +%s) > starttime + 60 )); then + echo + echo "[ERROR] Waited 60 seconds, no response" >&2 + exit 1 + fi +done +echo +sleep 1 + if [ -n "$DEBUG" ]; then cat $thisFolder/config.ftp.php cat $thisFolder/dockerContainerMorrisJobke.$EXECUTOR_NUMBER.ftp fi -# TODO find a way to determine the successful initialization inside the docker container -echo "Waiting 5 seconds for ftp initialization ... " -sleep 5 diff --git a/apps/files_external/tests/env/start-sftp-atmoz.sh b/apps/files_external/tests/env/start-sftp-atmoz.sh index 91eb5747c54e..c4ef81b8357f 100755 --- a/apps/files_external/tests/env/start-sftp-atmoz.sh +++ b/apps/files_external/tests/env/start-sftp-atmoz.sh @@ -54,15 +54,27 @@ echo "sftp container: $container" # put container IDs into a file to drop them after the test run (keep in mind that multiple tests run in parallel on the same host) echo $container >> $thisFolder/dockerContainerAtmoz.$EXECUTOR_NUMBER.sftp +echo -n "Waiting for sftp initialization" +starttime=$(date +%s) +# support for GNU netcat and BSD netcat +while ! (nc -c -w 1 ${host} 22 &/dev/null \ + || nc -w 1 ${host} 22 &/dev/null); do + sleep 1 + echo -n '.' + if (( $(date +%s) > starttime + 60 )); then + echo + echo "[ERROR] Waited 60 seconds, no response" >&2 + exit 1 + fi +done +echo +sleep 1 + if [ -n "$DEBUG" ]; then cat $thisFolder/config.sftp.php cat $thisFolder/dockerContainerAtmoz.$EXECUTOR_NUMBER.sftp fi -# TODO find a way to determine the successful initialization inside the docker container -echo "Waiting 5 seconds for sftp initialization ... " -sleep 5 - # create folder "upload" with correct permissions docker exec $container bash -c "mkdir /home/$user/upload && chown $user:users /home/$user/upload" diff --git a/apps/files_external/tests/env/start-smb-silvershell.sh b/apps/files_external/tests/env/start-smb-silvershell.sh index c45807cc4c81..e2aa025c7119 100755 --- a/apps/files_external/tests/env/start-smb-silvershell.sh +++ b/apps/files_external/tests/env/start-smb-silvershell.sh @@ -52,12 +52,25 @@ echo "samba container: $container" # put container IDs into a file to drop them after the test run (keep in mind that multiple tests run in parallel on the same host) echo $container >> $thisFolder/dockerContainerSilvershell.$EXECUTOR_NUMBER.smb +echo -n "Waiting for samba initialization" +starttime=$(date +%s) +# support for GNU netcat and BSD netcat +while ! (nc -c -w 1 ${host} 445 &/dev/null \ + || nc -w 1 ${host} 445 &/dev/null); do + sleep 1 + echo -n '.' + if (( $(date +%s) > starttime + 60 )); then + echo + echo "[ERROR] Waited 60 seconds, no response" >&2 + exit 1 + fi +done +echo +sleep 1 + if [ -n "$DEBUG" ]; then cat $thisFolder/config.smb.php cat $thisFolder/dockerContainerSilvershell.$EXECUTOR_NUMBER.smb fi -# TODO find a way to determine the successful initialization inside the docker container -echo "Waiting 5 seconds for smbd initialization ... " -sleep 5 diff --git a/apps/files_external/tests/env/start-swift-ceph.sh b/apps/files_external/tests/env/start-swift-ceph.sh index c2e7d4c1ae15..a8305eee5a27 100755 --- a/apps/files_external/tests/env/start-swift-ceph.sh +++ b/apps/files_external/tests/env/start-swift-ceph.sh @@ -56,9 +56,21 @@ echo "${docker_image} container: $container" # put container IDs into a file to drop them after the test run (keep in mind that multiple tests run in parallel on the same host) echo $container >> $thisFolder/dockerContainerCeph.$EXECUTOR_NUMBER.swift -# TODO find a way to determine the successful initialization inside the docker container -echo "Waiting 20 seconds for ceph initialization ... " -sleep 20 +echo -n "Waiting for ceph initialization" +starttime=$(date +%s) +# support for GNU netcat and BSD netcat +while ! (nc -c -w 1 ${host} 80 &/dev/null \ + || nc -w 1 ${host} 80 &/dev/null); do + sleep 1 + echo -n '.' + if (( $(date +%s) > starttime + 60 )); then + echo + echo "[ERROR] Waited 60 seconds, no response" >&2 + exit 1 + fi +done +echo +sleep 1 cat > $thisFolder/config.swift.php <&/dev/null \ + || nc -w 1 ${host} 80 &/dev/null); do + sleep 1 + echo -n '.' + if (( $(date +%s) > starttime + 60 )); then + echo + echo "[ERROR] Waited 60 seconds, no response" >&2 + exit 1 + fi +done +echo +sleep 1 cat > $thisFolder/config.webdav.php <true, - 'host'=>'localhost:$port/owncloud/remote.php/webdav/', + 'host'=>'${host}:80/owncloud/remote.php/webdav/', 'user'=>'test', 'password'=>'test', 'root'=>'', From ad028c555d8517091b60bf9aff97fa0ab22a93ac Mon Sep 17 00:00:00 2001 From: Robin McCorkell Date: Wed, 23 Sep 2015 23:18:16 +0100 Subject: [PATCH 5/7] Don't attempt to run tests unless start file is successful --- .../tests/env/start-smb-windows.sh | 6 ++ autotest-external.sh | 81 +++++++++---------- 2 files changed, 46 insertions(+), 41 deletions(-) diff --git a/apps/files_external/tests/env/start-smb-windows.sh b/apps/files_external/tests/env/start-smb-windows.sh index 2143d7e74997..6779cdb2d567 100755 --- a/apps/files_external/tests/env/start-smb-windows.sh +++ b/apps/files_external/tests/env/start-smb-windows.sh @@ -19,6 +19,12 @@ user=smb-test password=!owncloud123 host=WIN-9GTFAS08C15 +if ! (nc -c -w 1 ${host} 445 &/dev/null \ + || nc -w 1 ${host} 445 &/dev/null); then + echo "[ERROR] Server not reachable" >&2 + exit 1 +fi + cat > $thisFolder/config.smb.php < Date: Thu, 24 Sep 2015 14:33:54 +0100 Subject: [PATCH 6/7] Skip Swift stat test --- apps/files_external/tests/backends/swift.php | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/apps/files_external/tests/backends/swift.php b/apps/files_external/tests/backends/swift.php index 83b27e6e82e4..07ee36043b18 100644 --- a/apps/files_external/tests/backends/swift.php +++ b/apps/files_external/tests/backends/swift.php @@ -59,4 +59,8 @@ protected function tearDown() { parent::tearDown(); } + + public function testStat() { + $this->markTestSkipped('Swift doesn\'t update the parents folder mtime'); + } } From d49fc9a5fcb993307ff7ef188fff3da8a3d03c69 Mon Sep 17 00:00:00 2001 From: Robin Appelman Date: Mon, 16 Nov 2015 21:51:10 +0100 Subject: [PATCH 7/7] fix ip detection for swift-ceph --- apps/files_external/tests/env/start-swift-ceph.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/files_external/tests/env/start-swift-ceph.sh b/apps/files_external/tests/env/start-swift-ceph.sh index a8305eee5a27..ea16e167af9a 100755 --- a/apps/files_external/tests/env/start-swift-ceph.sh +++ b/apps/files_external/tests/env/start-swift-ceph.sh @@ -48,7 +48,7 @@ container=`docker run -d \ -e KEYSTONE_SERVICE=${service} \ ${docker_image}` -host=`docker inspect $container | grep IPAddress | cut -d '"' -f 4` +host=`docker inspect --format="{{.NetworkSettings.IPAddress}}" $container` echo "${docker_image} container: $container"