From fb5cc5fcfd1ad834675a15be16fd75ffea14f9b8 Mon Sep 17 00:00:00 2001 From: mvousden Date: Fri, 22 Jul 2016 09:41:35 +0100 Subject: [PATCH 01/15] [ci skip] Fix spelling mistake and command mistake in documentation. --- docs/getting-started-user.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/getting-started-user.rst b/docs/getting-started-user.rst index da87a40..bb7a1f4 100644 --- a/docs/getting-started-user.rst +++ b/docs/getting-started-user.rst @@ -48,7 +48,7 @@ Your user will need to be in the docker group on the machine you are running on [#]_. After restarting your machine, command the following in an empty directory:: - docker run -ti virtualmicromagnetics/full:release /bin/bash + docker run -ti virtualmicromagnetics/lite:1.0.4 /bin/bash -l This command will download the Full Virtual Micromagnetics container image from the Internet to your computer, and start bash in your shell. @@ -58,5 +58,5 @@ besides the Full Virtual Micromagnetics environment. .. [#] Note that the docker group is root-equivalent, so you will likely need to own the machine to use Virtual Micromagnetics containers. To avoid this, - consider using a different provider that does not require root priveleges, + consider using a different provider that does not require root privileges, but beware, as Docker is the only container provider supported at present. From bec211020e61bbb9c7638e2803a3a76b4132c9db Mon Sep 17 00:00:00 2001 From: mvousden Date: Fri, 22 Jul 2016 13:31:48 +0100 Subject: [PATCH 02/15] BUG-049: Install FFTW and SUNDIALS from explicit scripts, as opposed to the full ubuntu installation script. --- roles/fidimag/tasks/make_fidimag.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/roles/fidimag/tasks/make_fidimag.yml b/roles/fidimag/tasks/make_fidimag.yml index 73f5fe3..51054f6 100644 --- a/roles/fidimag/tasks/make_fidimag.yml +++ b/roles/fidimag/tasks/make_fidimag.yml @@ -4,10 +4,13 @@ # SUNDIALS. # Install local dependencies. -- name: Install FFTW and SUNDIALS from the Fidimag installation script. - command: bash ubuntu_install_script.sh +- name: Install FFTW and SUNDIALS from Fidimag installation scripts. + command: bash {{ item }} args: chdir: "{{ FIDIMAG_INSTALL_PATH }}/bin" + with_items: + - install-fftw.sh + - install-sundials-2.5.sh sudo: yes # Build Fidimag. Makefile module? From 18f19c8cf0a5e41e53770a02a73891b10fe7597e Mon Sep 17 00:00:00 2001 From: mvousden Date: Fri, 22 Jul 2016 13:55:05 +0100 Subject: [PATCH 03/15] BUG-049: Remove explicit VM roles from container builds. --- jobs/provision_virtualmicromagnetics_fidimag.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/jobs/provision_virtualmicromagnetics_fidimag.yml b/jobs/provision_virtualmicromagnetics_fidimag.yml index 938ab69..e65e74b 100644 --- a/jobs/provision_virtualmicromagnetics_fidimag.yml +++ b/jobs/provision_virtualmicromagnetics_fidimag.yml @@ -13,8 +13,6 @@ - { role: rst_to_html, SOURCE_PATH: "/vagrant/extra_resources/welcome/welcome_{{ name }}.rst" } - fidimag - fidimag_examples - - xserver - - add_super_user - { role: xserver, when: type == "vm" } - { role: add_super_user, when: type == "vm" } - { role: copy_skeleton, TYPE: type } From b2651f2404ecf84d8880d06ca283facf570bcd14 Mon Sep 17 00:00:00 2001 From: mvousden Date: Fri, 22 Jul 2016 14:07:44 +0100 Subject: [PATCH 04/15] BUG-050: Configure container builds to not push by default. --- Makefile | 12 ++++++------ jobs/hook_container.yml | 1 + 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index 5cc7e39..88f2169 100755 --- a/Makefile +++ b/Makefile @@ -3,22 +3,22 @@ ## Container targets full-container: - ansible-playbook master.yml -c local -i localhost, -v -k --extra-vars="type=container container_name=full playbook=provision_virtualmicromagnetics_full.yml hookbook=hook_container.yml extra_resources_dir=guest_resources/" + ansible-playbook master.yml -c local -i localhost, -v -k --extra-vars="type=container container_name=full playbook=provision_virtualmicromagnetics_full.yml hookbook=hook_container.yml extra_resources_dir=guest_resources/ container_push=false" lite-container: - ansible-playbook master.yml -c local -i localhost, -v -k --extra-vars="type=container container_name=lite playbook=provision_virtualmicromagnetics_lite.yml hookbook=hook_container.yml extra_resources_dir=guest_resources/" + ansible-playbook master.yml -c local -i localhost, -v -k --extra-vars="type=container container_name=lite playbook=provision_virtualmicromagnetics_lite.yml hookbook=hook_container.yml extra_resources_dir=guest_resources/ container_push=false" oommf-container: - ansible-playbook master.yml -c local -i localhost, -v -k --extra-vars="type=container container_name=oommf playbook=provision_virtualmicromagnetics_oommf.yml hookbook=hook_container.yml extra_resources_dir=guest_resources/" + ansible-playbook master.yml -c local -i localhost, -v -k --extra-vars="type=container container_name=oommf playbook=provision_virtualmicromagnetics_oommf.yml hookbook=hook_container.yml extra_resources_dir=guest_resources/ container_push=false" magpar-container: - ansible-playbook master.yml -c local -i localhost, -v -k --extra-vars="type=container container_name=magpar playbook=provision_virtualmicromagnetics_magpar.yml hookbook=hook_container.yml extra_resources_dir=guest_resources/" + ansible-playbook master.yml -c local -i localhost, -v -k --extra-vars="type=container container_name=magpar playbook=provision_virtualmicromagnetics_magpar.yml hookbook=hook_container.yml extra_resources_dir=guest_resources/ container_push=false" nmag-container: - ansible-playbook master.yml -c local -i localhost, -v -k --extra-vars="type=container container_name=nmag playbook=provision_virtualmicromagnetics_nmag.yml hookbook=hook_container.yml extra_resources_dir=guest_resources/" + ansible-playbook master.yml -c local -i localhost, -v -k --extra-vars="type=container container_name=nmag playbook=provision_virtualmicromagnetics_nmag.yml hookbook=hook_container.yml extra_resources_dir=guest_resources/ container_push=false" fidimag-container: - ansible-playbook master.yml -c local -i localhost, -v -k --extra-vars="type=container container_name=fidimag playbook=provision_virtualmicromagnetics_fidimag.yml hookbook=hook_container.yml extra_resources_dir=guest_resources/" + ansible-playbook master.yml -c local -i localhost, -v -k --extra-vars="type=container container_name=fidimag playbook=provision_virtualmicromagnetics_fidimag.yml hookbook=hook_container.yml extra_resources_dir=guest_resources/ container_push=false" ## VM targets full-vm: full diff --git a/jobs/hook_container.yml b/jobs/hook_container.yml index 66cf174..e8df814 100644 --- a/jobs/hook_container.yml +++ b/jobs/hook_container.yml @@ -57,6 +57,7 @@ - name: Push the Docker image. command: docker push {{ output_image_name }} + when: "{{ container_push }}" - name: Destroy the container. command: vagrant destroy --force From 4f69e3293c34c28aaebab92a35f681723040675c Mon Sep 17 00:00:00 2001 From: mvousden Date: Fri, 22 Jul 2016 15:07:07 +0100 Subject: [PATCH 05/15] BUG-050: Remove unneccesary braces around variable. --- jobs/hook_container.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jobs/hook_container.yml b/jobs/hook_container.yml index e8df814..8a59667 100644 --- a/jobs/hook_container.yml +++ b/jobs/hook_container.yml @@ -57,7 +57,7 @@ - name: Push the Docker image. command: docker push {{ output_image_name }} - when: "{{ container_push }}" + when: container_push - name: Destroy the container. command: vagrant destroy --force From e492faa3025b78b7c4cfce86fe98829260425698 Mon Sep 17 00:00:00 2001 From: mvousden Date: Fri, 22 Jul 2016 15:25:59 +0100 Subject: [PATCH 06/15] FEAT-052: Push containers with tags equal to both the release version, and the git tag. --- jobs/hook_container.yml | 78 ++++++++++++++++++++++++++--------------- 1 file changed, 50 insertions(+), 28 deletions(-) diff --git a/jobs/hook_container.yml b/jobs/hook_container.yml index 8a59667..54a2b12 100644 --- a/jobs/hook_container.yml +++ b/jobs/hook_container.yml @@ -13,52 +13,74 @@ name={{ artefact_dir }} state=directory -- name: Obtain the current tag, (or version if untagged) of this repository. - shell: | - TAG=$(git name-rev --tags --name-only HEAD | cut -f1 -d^) - if [ "$(echo $TAG | tr -d [:space:])" == "undefined" ]; then - git rev-parse --short HEAD - else - echo $TAG - fi +- name: Obtain current tag of this repository, if any. + shell: git name-rev --tags --name-only HEAD | cut -f1 -d^ args: chdir: "{{ container_dir }}" executable: /bin/bash - register: current_version + register: tag -- name: Set artefact name. +- name: Obtain current version hash of this repository. + shell: git rev-parse --short HEAD + args: + chdir: "{{ container_dir }}" + executable: /bin/bash + register: version + +- name: Obtain current branch of this repository. + shell: git describe --all | grep -o [^/]*$ + args: + chdir: "{{ container_dir }}" + executable: /bin/bash + register: branch + +# Exporting container filesystem as tarball. + +- name: Set artefact name with tag, if we could find one. + set_fact: + artefact_name: "{{ container_name }}_{{ tag.stdout }}.tar" + when: tag.stdout != "undefined" + +- name: Set artefact name with version hash, if we could not find a tag. set_fact: - artefact_name: "{{ container_name }}_{{ current_version.stdout }}.tar" + artefact_name: "{{ container_name }}_{{ version.stdout }}.tar" + when: tag.stdout == "undefined" - name: Export the container filesystem as an artefact. command: docker export --output={{ artefact_dir }}{{ artefact_name }} {{ container_name }} args: creates: "{{ artefact_dir }}{{ artefact_name }}" -- name: Obtain the current tag (or branch if untagged) of this repository. - shell: | - TAG=$(git name-rev --tags --name-only HEAD | cut -f1 -d^) - if [ "$(echo $TAG | tr -d [:space:])" == "undefined" ]; then - git describe --all | grep -o [^/]*$ - else - echo $TAG - fi - args: - chdir: "{{ container_dir }}" - executable: /bin/bash - register: current_branch +# Storing container with tag=branch. -- name: Define the name used for the output Docker image +- name: Define the name used for the output Docker image using the branch. set_fact: - output_image_name: virtualmicromagnetics/{{ container_name }}:{{ current_branch.stdout }} + output_image_name_branch: virtualmicromagnetics/{{ container_name }}:{{ branch.stdout }} - name: Create a Docker image from our provisioned container. - command: docker commit {{ container_name }} {{ output_image_name }} + command: docker commit {{ container_name }} {{ output_image_name_branch }} -- name: Push the Docker image. - command: docker push {{ output_image_name }} +- name: Push the Docker image with tag equal to branch. + command: docker push {{ output_image_name_branch }} when: container_push +# Storing container with tag=tag. + +- name: Define the name used for the output Docker image using the branch. + set_fact: + output_image_name_tag: virtualmicromagnetics/{{ container_name }}:{{ tag.stdout }} + when: tag.stdout != "undefined" + +- name: Create a Docker image from our provisioned container. + command: docker commit {{ container_name }} {{ output_image_name_tag }} + when: tag.stdout != "undefined" + +- name: Push the Docker image with tag equal to branch. + command: docker push {{ output_image_name_tag }} + when: container_push and (tag.stdout != "undefined") + +# Remove the evidence. + - name: Destroy the container. command: vagrant destroy --force chdir={{ container_dir }} From 095be6fb9c19b81daf166800750fc1655a359f5f Mon Sep 17 00:00:00 2001 From: mvousden Date: Fri, 22 Jul 2016 16:07:53 +0100 Subject: [PATCH 07/15] FEAT-052: Obtain current branch sensibly. --- jobs/hook_container.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/jobs/hook_container.yml b/jobs/hook_container.yml index 54a2b12..45bd248 100644 --- a/jobs/hook_container.yml +++ b/jobs/hook_container.yml @@ -28,7 +28,7 @@ register: version - name: Obtain current branch of this repository. - shell: git describe --all | grep -o [^/]*$ + shell: git symbolic-ref HEAD | grep -o [^/]*$ args: chdir: "{{ container_dir }}" executable: /bin/bash @@ -66,7 +66,7 @@ # Storing container with tag=tag. -- name: Define the name used for the output Docker image using the branch. +- name: Define the name used for the output Docker image using the tag. set_fact: output_image_name_tag: virtualmicromagnetics/{{ container_name }}:{{ tag.stdout }} when: tag.stdout != "undefined" @@ -75,7 +75,7 @@ command: docker commit {{ container_name }} {{ output_image_name_tag }} when: tag.stdout != "undefined" -- name: Push the Docker image with tag equal to branch. +- name: Push the Docker image with tag equal to tag. command: docker push {{ output_image_name_tag }} when: container_push and (tag.stdout != "undefined") From 8294949656ff131ab914d5aebd769748181ba209 Mon Sep 17 00:00:00 2001 From: mvousden Date: Fri, 22 Jul 2016 16:08:11 +0100 Subject: [PATCH 08/15] FEAT-052: Fix push logic. --- jobs/hook_container.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jobs/hook_container.yml b/jobs/hook_container.yml index 45bd248..67488e9 100644 --- a/jobs/hook_container.yml +++ b/jobs/hook_container.yml @@ -77,7 +77,7 @@ - name: Push the Docker image with tag equal to tag. command: docker push {{ output_image_name_tag }} - when: container_push and (tag.stdout != "undefined") + when: (container_push == "true") and (tag.stdout != "undefined") # Remove the evidence. From 581d12c19db7b8c1de2fc77085e147a721239db8 Mon Sep 17 00:00:00 2001 From: mvousden Date: Sat, 23 Jul 2016 18:46:58 +0100 Subject: [PATCH 09/15] Allow Ansible to obtain the Git branch name when building on Jenkins (with a detached head). --- jobs/hook_container.yml | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/jobs/hook_container.yml b/jobs/hook_container.yml index 67488e9..f83da21 100644 --- a/jobs/hook_container.yml +++ b/jobs/hook_container.yml @@ -27,12 +27,23 @@ executable: /bin/bash register: version -- name: Obtain current branch of this repository. +- name: Obtain current branch of this repository. Does not work in detached-head state. shell: git symbolic-ref HEAD | grep -o [^/]*$ args: chdir: "{{ container_dir }}" executable: /bin/bash register: branch + when: ("{{ lookup('env','GIT_BRANCH') }}" == "") + +# Alternatively, we can obtain the branch in detached-head state using the +# environment variable GIT_BRANCH if it is set. This helps out Jenkins. +- name: Obtain current branch of this repository using GIT_BRANCH environment variable. + shell: echo $GIT_BRANCH + args: + chdir: "{{ container_dir }}" + executable: /bin/bash + register: branch + when: ("{{ lookup('env','GIT_BRANCH') }}" == "") # Exporting container filesystem as tarball. From 5a1ba3355ba8cc014d29ab04cea3437f297536bd Mon Sep 17 00:00:00 2001 From: mvousden Date: Sat, 23 Jul 2016 18:56:48 +0100 Subject: [PATCH 10/15] Specifically bring up containers with docker. --- roles/create_container/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/create_container/tasks/main.yml b/roles/create_container/tasks/main.yml index def0129..7da1e13 100644 --- a/roles/create_container/tasks/main.yml +++ b/roles/create_container/tasks/main.yml @@ -83,7 +83,7 @@ executable: /bin/bash - name: Start the container. Vagrant may attempt to install VirtualBox Guest Additions here. - shell: vagrant up --no-provision > {{ container_name }}.log + shell: vagrant up --no-provision --provider docker > {{ container_name }}.log chdir={{ container_dir }} - name: Provision the container. A log exists in machines/. From 126d21c66e6ab630561e51bec0835704959904ab Mon Sep 17 00:00:00 2001 From: mvousden Date: Sat, 23 Jul 2016 19:01:46 +0100 Subject: [PATCH 11/15] Try multiple attempts at extracting the VMDK from a vagrant box. This extraction appears to fail periodically. --- jobs/hook_vm.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/jobs/hook_vm.yml b/jobs/hook_vm.yml index 50aae27..9e28c28 100644 --- a/jobs/hook_vm.yml +++ b/jobs/hook_vm.yml @@ -73,6 +73,9 @@ chdir={{ artefact_dir }} creates={{ artefact_dir }}/box-disk1.vmdk when: not zip_status.stat.exists or (zip_status.stat.isreg and clobber_output) + register: result + until: result.rc == 0 + retries: 3 - name: Convert the virtual machine disk (VMDK) to a virtual hard disk (VHD). command: VBoxManage clonehd --format VHD box-disk1.vmdk {{ output_name }}.vhd From 65d6b31cc9a46813d2b4352727d38d1c361e955e Mon Sep 17 00:00:00 2001 From: mvousden Date: Sat, 23 Jul 2016 19:28:13 +0100 Subject: [PATCH 12/15] Fix syntax mistake. --- jobs/hook_container.yml | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/jobs/hook_container.yml b/jobs/hook_container.yml index f83da21..32944f3 100644 --- a/jobs/hook_container.yml +++ b/jobs/hook_container.yml @@ -27,23 +27,14 @@ executable: /bin/bash register: version -- name: Obtain current branch of this repository. Does not work in detached-head state. - shell: git symbolic-ref HEAD | grep -o [^/]*$ - args: - chdir: "{{ container_dir }}" - executable: /bin/bash - register: branch - when: ("{{ lookup('env','GIT_BRANCH') }}" == "") - # Alternatively, we can obtain the branch in detached-head state using the # environment variable GIT_BRANCH if it is set. This helps out Jenkins. -- name: Obtain current branch of this repository using GIT_BRANCH environment variable. - shell: echo $GIT_BRANCH +- name: Obtain current branch of this repository. + shell: if [ -z "$GIT_BRANCH" ]; then git symbolic-ref HEAD | grep -o [^/]*$; else echo "$GIT_BRANCH"; fi args: chdir: "{{ container_dir }}" executable: /bin/bash register: branch - when: ("{{ lookup('env','GIT_BRANCH') }}" == "") # Exporting container filesystem as tarball. From 5cf9b7f0e0726cbc84a520b306b557cd8645827a Mon Sep 17 00:00:00 2001 From: mvousden Date: Sat, 23 Jul 2016 21:46:43 +0100 Subject: [PATCH 13/15] Extract the VMDK file with the -z flag to ensure tar uses compression to do so. --- jobs/hook_vm.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jobs/hook_vm.yml b/jobs/hook_vm.yml index 9e28c28..096c7b0 100644 --- a/jobs/hook_vm.yml +++ b/jobs/hook_vm.yml @@ -69,7 +69,7 @@ when: (not vagrantfile.stat.exists) and (not box_status.stat.exists or (box_status.stat.isreg and clobber_output)) - name: Extract the virtual machine disk (VMDK) file from the box. - command: tar -xvf {{ output_box }} -C {{ artefact_dir }} ./box-disk1.vmdk + command: tar -xzvf {{ output_box }} -C {{ artefact_dir }} ./box-disk1.vmdk chdir={{ artefact_dir }} creates={{ artefact_dir }}/box-disk1.vmdk when: not zip_status.stat.exists or (zip_status.stat.isreg and clobber_output) From c53ea4669c2558ffacabac68fb119b9ec482674a Mon Sep 17 00:00:00 2001 From: mvousden Date: Sat, 23 Jul 2016 22:37:39 +0100 Subject: [PATCH 14/15] Process Jenkins' GIT_BRANCH environment variable correctly. --- jobs/hook_container.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jobs/hook_container.yml b/jobs/hook_container.yml index 32944f3..65585cd 100644 --- a/jobs/hook_container.yml +++ b/jobs/hook_container.yml @@ -30,7 +30,7 @@ # Alternatively, we can obtain the branch in detached-head state using the # environment variable GIT_BRANCH if it is set. This helps out Jenkins. - name: Obtain current branch of this repository. - shell: if [ -z "$GIT_BRANCH" ]; then git symbolic-ref HEAD | grep -o [^/]*$; else echo "$GIT_BRANCH"; fi + shell: if [ -z "$GIT_BRANCH" ]; then git symbolic-ref HEAD | grep -o [^/]*$; else echo "$GIT_BRANCH" | grep -o [^/][a-zA-Z0-9-]*$; fi args: chdir: "{{ container_dir }}" executable: /bin/bash From 261ed1a228795dd8e46d038665522b9e0875c840 Mon Sep 17 00:00:00 2001 From: mvousden Date: Thu, 28 Jul 2016 14:56:14 +0100 Subject: [PATCH 15/15] Update version number in documentation. --- docs/conf.py | 4 ++-- docs/developer-notes.rst | 2 +- docs/getting-started-user.rst | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index bdb2a21..b20f151 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -60,9 +60,9 @@ # built documents. # # The short X.Y version. -version = u'1.0.4' +version = u'1.1.0' # The full version, including alpha/beta/rc tags. -release = u'1.0.4' +release = u'1.1.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/docs/developer-notes.rst b/docs/developer-notes.rst index a0a4413..864d77c 100644 --- a/docs/developer-notes.rst +++ b/docs/developer-notes.rst @@ -1,6 +1,6 @@ .. _dev-notes: -Developer Notes (valid for version 1.0.4) +Developer Notes (valid for version 1.1.0) ========================================= In :ref:`getting-started-poweruser`, we created a virtual environment from diff --git a/docs/getting-started-user.rst b/docs/getting-started-user.rst index bb7a1f4..de59c69 100644 --- a/docs/getting-started-user.rst +++ b/docs/getting-started-user.rst @@ -48,7 +48,7 @@ Your user will need to be in the docker group on the machine you are running on [#]_. After restarting your machine, command the following in an empty directory:: - docker run -ti virtualmicromagnetics/lite:1.0.4 /bin/bash -l + docker run -ti virtualmicromagnetics/lite:1.1.0 /bin/bash -l This command will download the Full Virtual Micromagnetics container image from the Internet to your computer, and start bash in your shell.