From 32a4d082294f8eb19062e2f7709cac944d23b603 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miquel=20Sabat=C3=A9=20Sol=C3=A0?= Date: Thu, 15 Feb 2018 16:21:01 +0100 Subject: [PATCH 1/2] spec: nuked previous integration tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit They were buggy and we need to take another approach. Signed-off-by: Miquel Sabaté Solà --- .travis.yml | 2 +- db/seeds.rb | 22 -- lib/tasks/portus/test.rake | 41 -- spec/integration/client_spec.rb | 90 ----- spec/integration/fixtures/config.yml.erb | 23 -- spec/integration/fixtures/portus.crt | 32 -- spec/integration/helper.rb | 456 ----------------------- spec/integration/login_spec.rb | 150 -------- 8 files changed, 1 insertion(+), 815 deletions(-) delete mode 100644 lib/tasks/portus/test.rake delete mode 100644 spec/integration/client_spec.rb delete mode 100644 spec/integration/fixtures/config.yml.erb delete mode 100644 spec/integration/fixtures/portus.crt delete mode 100644 spec/integration/helper.rb delete mode 100644 spec/integration/login_spec.rb diff --git a/.travis.yml b/.travis.yml index c5857135c..01f6c495d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -44,7 +44,7 @@ script: - bundle exec rake portus:assets:compile # Ruby tests - - bundle exec rspec spec --tag ~integration + - bundle exec rspec spec # Style and security checks - bundle exec rubocop -V diff --git a/db/seeds.rb b/db/seeds.rb index 67b8686b2..c6ff944ed 100644 --- a/db/seeds.rb +++ b/db/seeds.rb @@ -14,25 +14,3 @@ email: "portus@portus.com", admin: true ) - -# Adding a user and a registry for integration tests. -if ENV["INTEGRATION_TESTS"] - Rails.logger.info "Adding user username31" - if User.find_by(username: "username31") - Rails.logger.fatal "User already exists. Please drop the database first" - exit(-1) - end - - User.create( - username: "username31", - email: "a1@b.com", - password: "test-password", - admin: true - ) - - Rails.logger.info "Adding registry portus.suse.example.com:5000" - Registry.create( - name: "portus.suse.example.com:5000", - hostname: "portus.suse.example.com:5000" - ) -end diff --git a/lib/tasks/portus/test.rake b/lib/tasks/portus/test.rake deleted file mode 100644 index 13fa0bb47..000000000 --- a/lib/tasks/portus/test.rake +++ /dev/null @@ -1,41 +0,0 @@ -# frozen_string_literal: true - -require "pty" - -# Spawn a new command and return its exit status. It will print to stdout on -# real time. -def spawn_cmd(cmd) - status = 0 - - PTY.spawn(cmd) do |stdout, _, pid| - begin - stdout.each { |line| print line } - rescue Errno::EIO - puts "EOI" - end - - Process.wait(pid) - status = $CHILD_STATUS.exitstatus - end - status -end - -namespace :portus do - desc "Properly test Portus" - task :test do |_, args| - tags = args.extras.map { |a| "--tag #{a}" } - tags << "--tag ~integration" if ENV["TRAVIS"] == "true" - - # Run normal tests + integration. - ENV["INTEGRATION_LDAP"] = nil - status = spawn_cmd("rspec spec #{tags.join(" ")}") - exit(status) if status != 0 - exit(0) if ENV["TRAVIS"] == "true" - - # Run LDAP integration tests. - ENV["INTEGRATION_LDAP"] = "t" - tags << "--tag integration" unless args.extras.include?("integration") - status = spawn_cmd("rspec spec #{tags.join(" ")}") - exit(status) - end -end diff --git a/spec/integration/client_spec.rb b/spec/integration/client_spec.rb deleted file mode 100644 index 6caa7ecce..000000000 --- a/spec/integration/client_spec.rb +++ /dev/null @@ -1,90 +0,0 @@ -# frozen_string_literal: true - -require "integration/helper" -require "portus/registry_client" - -def push_test_images - name = ldap? ? "johnldap" : "john" - email = "john@example.com" - password = "12341234" - create_user(name, email, password, true) - expect { login(name, password, email) }.not_to raise_error - - # Pulling images that we should already have (so we go faster :P). Then - # re-tag it so they can be pushed. The re-tag has another name, to avoid - # clashes with other tests. - name = "registry" - retag = "registre" - pulled_tags = ["2.3.1", "2.4.0", "latest"] - pulled_tags.each do |tag| - base = "#{name}:#{tag}" - tgt = "#{retag}:#{tag}" - img = "library/#{base}" - pull(img) - system("docker tag #{img} #{registry_hostname}/#{tgt}") - end - - expect(push("#{registry_hostname}/#{retag}")).to be_truthy - - # Wait until the registry has everything we need. - eventually_expect(3) do - tags = rails_exec("Tag.where(repository: Repository.find_by(name: '#{retag}')).to_json") - tags.size - end -end - -integration "Client" do - it "tells that a registry is reachable" do - client = Portus::RegistryClient.new(registry_hostname) - expect(client).to be_reachable - end - - it "fetches the catalog of pushed repositories" do - push_test_images - - client = Portus::RegistryClient.new(registry_hostname) - cat = client.catalog - expected = { "registre" => ["2.3.1", "2.4.0", "latest"] } - - cat.each do |r| - key = expected[r["name"]] - next if key.nil? - expect(key).to eq r["tags"] - end - - tags = client.tags("registre") - expect(tags).to eq(expected["registre"]) - end - - it "fetches the manifest of the given repo/tag" do - push_test_images - - client = Portus::RegistryClient.new(registry_hostname) - manifest = client.manifest("registre", "2.3.1") - - expect(manifest[0]).not_to be_empty - expect(manifest[1]).not_to be_empty - expect(manifest[2]["schemaVersion"]).to eq 2 - end - - it "supports deleting manifests" do - push_test_images - - client = Portus::RegistryClient.new(registry_hostname) - - # Provided digest did not match uploaded content - expect do - client.delete("registry", "itsdangeroustogoalonetakethis", "manifests") - end.to raise_error(RuntimeError) - - _, dig, = client.manifest("registre", "2.3.1") - - # Not found - expect do - client.delete("registry", dig, "manifests") - end.to raise_error(Portus::HttpHelpers::NotFoundError) - - client.delete("registre", dig, "manifests") - eventually_expect(2) { rails_exec("Tag.all.to_json").size } - end -end diff --git a/spec/integration/fixtures/config.yml.erb b/spec/integration/fixtures/config.yml.erb deleted file mode 100644 index f1f204257..000000000 --- a/spec/integration/fixtures/config.yml.erb +++ /dev/null @@ -1,23 +0,0 @@ -version: 0.1 -storage: - filesystem: - rootdirectory: /registry_data - delete: - enabled: true -http: - addr: 0.0.0.0:5000 - debug: - addr: 0.0.0.0:5001 -auth: - token: - realm: http://<%= @ip %>:3000/v2/token - service: <%= @ip %>:5000 - issuer: <%= @ip %> - rootcertbundle: /etc/docker/registry/portus.crt -notifications: - endpoints: - - name: portus - url: http://<%= @ip %>:3000/v2/webhooks/events - timeout: 500ms - threshold: 5 - backoff: 1s diff --git a/spec/integration/fixtures/portus.crt b/spec/integration/fixtures/portus.crt deleted file mode 100644 index ae7e52385..000000000 --- a/spec/integration/fixtures/portus.crt +++ /dev/null @@ -1,32 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIFejCCA2ICAQEwDQYJKoZIhvcNAQEFBQAwgYIxCzAJBgNVBAYTAkRFMRAwDgYD -VQQIDAdCYXZhcmlhMRMwEQYDVQQHDApOdWVyZW1iZXJnMQ0wCwYDVQQKDARTVVNF -MRgwFgYDVQQDDA9wb3J0dXMudGVzdC5sYW4xIzAhBgkqhkiG9w0BCQEWFHJvb3RA -cG9ydHVzLnRlc3QubGFuMB4XDTE1MDQxMjE1NTUwMloXDTI1MDQwOTE1NTUwMlow -gYIxCzAJBgNVBAYTAkRFMRAwDgYDVQQIDAdCYXZhcmlhMRMwEQYDVQQHDApOdWVy -ZW1iZXJnMQ0wCwYDVQQKDARTVVNFMRgwFgYDVQQDDA9wb3J0dXMudGVzdC5sYW4x -IzAhBgkqhkiG9w0BCQEWFHJvb3RAcG9ydHVzLnRlc3QubGFuMIICIjANBgkqhkiG -9w0BAQEFAAOCAg8AMIICCgKCAgEAyWA25/CoT+VsvCbwwx71KQ9YRy5gzadOufi3 -2t4NpP8O27tbemc4coIsEDLRBJSXxhBv97mTvfjAU4/nO0tJDgEHlrpl+p5IA6Up -3aYY2YqqY3riv+YI+e+RDcTau9Zd/ZxuB5OjpQocY16PGTP9dcUmn49oZ7xb3NUi -eoDHp2cS9UaTUzjNrxR+z6GrhjkLE9k5j1hi48v75/Ee/jL6W7rEiajJbuQDBkxc -mDmflalrrUAJnmCe1RpYRgbKEryBrFzUwBGsjqGwRnYwVNKc1CTnah986gj0Qx1O -FiPexIQrumCKY9Z7FwBrTm+8Ip0zdwfRMz7qZ6zfJqjcj5/1lNpXC/mBJ5k2HLgj -6eGSuQTBLHJNMu5S0dtG1vGnhQF6RjM1f/K+vwOAinrUJx6bSV/guwBdo8zg6m/o -krUvRAuP+l4ucyJP5T/JS53QXtJYSLNUdPVpec76EJOY1WrEBoyfdty2D3EHtnIF -GpTesW0hD9Jz0ofLXBA3UCd+Gi/Wr2A0wzpn3VfONDqFa6xiljpT2YgBKpa1eucC -+3JmVFRn6BY9jo76paC6Ygu/QzOfuF1nsv0aYdL9Lwdjf3HUBDFHUBJreJkh0QQ5 -yZMXMhdFI4yEKvYJLAiA7tUwAQ6xvDegy+JOsRMsEvDRNNfueczEk345FlrqRXI4 -KBPcdBsCAwEAATANBgkqhkiG9w0BAQUFAAOCAgEAGlCH3DFJJvOFrVO33Zp7lygq -XMd/XDMOLG1gwJ1cVvZPVaNGKgcB/v1Rjhf9R39fxum5uvw005ZX+APj1rtOgkO/ -fC9K0MA/kCIUjmU+NiH+UTDgcaChXXtVQ+PVAoWfKfEvwt6czcyQ4n+/hS0qJIjj -vOuFpnI9VBOxgN85tnjBAZ/7PPxg8FoUss51wtRXmML45rCW77Q2NiGH717Mo110 -xiue/+giTf7wP17Xl+Gvs4Fsm9rSDv0xhMYDjVbwU62ycQqXvDQVbbzkGjdNbKn2 -Fzo/C8bCQOYuPzUo18b3PoplEkO/b780Lv7t7m9lTHAB4X81MO0yg8vNPrISK2Af -VMJFDK4PsCdpGVFzY9Z+Jo5mGXV/n/nxRdaNujmANFeUl0Od1PuaDf+8w98GAuae -mKTlyV6C5cPMVjwgDeGMdGj0yz7Ht/PXwy4KltHSzSrfUww9sr5F3Kcpekh2mcb2 -NKXxXZ03b9AaWBPYEU2vD0N/MV7NwJqffW+/tLhMh/IVO991LTLFFKwZ31L+cHCj -ozJubbxDwix8wjTYw+Vj6dJyZrqb3IfLDgl2+ReaF1i80CKm4e+iikK+dmC88Av8 -FwdbTJL+QYEIxwHLz45cuHslqdD2josZYidrk1xBuQLMFN98jR+kwalmAT9dlSoA -vUZzjl/Is5XRXOjaJNE= ------END CERTIFICATE----- diff --git a/spec/integration/helper.rb b/spec/integration/helper.rb deleted file mode 100644 index e6b0499ad..000000000 --- a/spec/integration/helper.rb +++ /dev/null @@ -1,456 +0,0 @@ -# frozen_string_literal: true - -require "docker" -require "pty" -require "spec_helper" - -# The supported Docker Distribution versions. Make sure that these values -# correspond to valid tags on DockerHub. -# See: https://hub.docker.com/r/library/registry/tags/ -SUPPORTED_DISTRIBUTION_VERSIONS = ["2.3.1", "2.4.0", "latest"].freeze - -# Integration encapsulates an integration test. This method accepts one special -# tag: `distribution`. It can either be an Array of strings or a String, and it -# filters which versions of Docker distribution are to be tested. Useful when -# performing quick tests. -def integration(name, tags = {}, &blk) - tags[:integration] = true unless tags.key?(:integration) - - # Returns early if test has been explicitly marked as `skip`. - return if tags.key?(:skip) && tags[:skip] - return if ENV["TRAVIS"] - - WebMock.allow_net_connect! - VCR.turn_off! - - # Run the tests for the specified Distribution versions. - versions, rebuild = parameters_from_tags(tags) - versions.each { |version| setup_describe(name, version, rebuild, tags, &blk) } - - VCR.turn_on! - WebMock.disable_net_connect! -end - -# Setup the describe block for the given parameters. -# rubocop:disable Metrics/MethodLength -def setup_describe(name, version, rebuild, tags, &blk) - describe "#{name} (distribution: #{version}) (LDAP: #{ldap?})", tags do - before :all do - WebMock.allow_net_connect! - VCR.turn_off! - - # Set everything up. - once "setup" do - cleanup! - setup_db! - ldap? ? setup_ldap!("johnldap", "12341234") : setup_portus!(rebuild) - setup_templates! - end - - cleanup_distribution! - ensure_distribution!(version) - - VCR.turn_on! - WebMock.disable_net_connect! - end - - before do - WebMock.allow_net_connect! - VCR.turn_off! - - reset_db! - `docker restart integration_portus` - end - - after do - VCR.turn_on! - WebMock.disable_net_connect! - end - - instance_eval(&blk) - end -end -# rubocop:enable Metrics/MethodLength - -# Fetch some needed parameters from the given tags. It returns a two-sized -# array where the first item is an array of the docker distribution versions, -# and the last element is a boolean containing whether the Portus image has to -# be rebuilt or not. -def parameters_from_tags(tags) - # Pick the versions to be executed. - versions = SUPPORTED_DISTRIBUTION_VERSIONS - if tags.key?(:distribution) - filter = if tags[:distribution].is_a? Array - tags[:distribution] - else - [tags[:distribution]] - end - versions &= filter - end - - # Check if the Portus image has to be rebuilt. - rebuild = true - rebuild = tags[:rebuild] == true if tags.key?(:rebuild) - - [versions, rebuild] -end - -# Execute the given block just once. -def once(id) - yield if ENV["INTEGRATION_ONCE_BLOCK_#{id}"] != "t" - ENV["INTEGRATION_ONCE_BLOCK_#{id}"] = "t" -end - -# Returns whether LDAP has to be run on the current `integration` block. -def ldap? - ENV["INTEGRATION_LDAP"] == "t" || ENV["INTEGRATION_LDAP"] == "true" -end - -# Makes sure that there is a container running the given Docker distribution -# version. -def ensure_distribution!(version) - name = "portus_distribution_#{version.delete(".")}" - - src = __dir__ + "/" + "fixtures" - img = "registry:#{version}" - opts = { - "Volumes" => { src => { "/etc/docker/registry" => "ro,Z", "/registry_data" => "" } }, - "ExposedPorts" => { "5000/tcp" => {} }, - "HostConfig" => { - "Binds" => ["#{src}:/etc/docker/registry", "#{src}/data:/registry_data"], - "Links" => ["integration_portus"], - "PortBindings" => { "5000/tcp" => [{ "HostIP" => "0.0.0.0", "HostPort" => "5000" }] } - } - } - - start_container!(img, name, opts) -end - -def reset_db! - mysql_ping! - - # Migrate & seed - docker_exec("integration_portus", "rake db:drop") - docker_exec("integration_portus", "rake db:create") - docker_exec("integration_portus", "rake db:migrate:reset") - docker_exec("integration_portus", "rake db:seed") - - # Introduce the current registry. - cmd = "Registry.create(name: 'registry', hostname: '#{registry_hostname}', use_ssl: false)" - docker_exec("integration_portus", "rails runner \"#{cmd}\"") -end - -# Returns the hostname of the current registry. -def registry_hostname - "#{ip}:5000" -end - -# Start a container from the given image. It accepts a hash with extended -# options that you might want to pass it on creation time. -def start_container!(image, name, ext = {}) - # Remove old containers. - container = container_for(name) - container&.delete(force: true) - - opts = { "Image" => image }.merge(ext) - - begin - container = Docker::Container.create(opts) - rescue Docker::Error::NotFoundError - puts "Pulling from #{image}" - pull(image) - container = Docker::Container.create(opts) - end - - container.rename(name) - puts "Starting container #{name}" - container.start - - # Make sure that the container has actually started. Yes, the previous - # `start` method does not say anything about this (and even worse, we don't - # have a way in the client to tell why the container could not start). - begin - container.top - rescue Docker::Error::ServerError - raise StartError, "Could not start container '#{name}' from image '#{image}'." - end -end - -# Pulls a Docker image. -def pull(img) - Docker::Image.create("fromImage" => img) -end - -# Pushes an image onto the registry. An extra parameter can be passed telling -# this method whether a failure is expected or not. Returns a boolean telling -# whether the expection was met or not. -def push(img, succeed = true) - `docker restart integration_portus` - eventually_expect(succeed) { spawn_cmd("docker push #{img}") } -end - -# Returns a container object for the given identifier. If the container does -# not exist, it returns nil. -def container_for(name) - container = Docker::Container.get(name) - - # Check that the container is alive. It will raise a - # Docker::Error::ServerError if that's not the case. - container.top - container -rescue Docker::Error::ServerError - container.delete(force: true) - nil -rescue Docker::Error::NotFoundError - nil -end - -# Ping the mysql container within the portus container. It will raise an error -# if a ping attempt fails 5 times. -def mysql_ping! - conn = "--count=1 --connect-timeout=1" - cred = "--host=integration_db -u root --password=portus" - - success = docker_exec("integration_portus", "mysqladmin #{conn} #{cred} ping", 5) - raise StandardError, "Could not ping mysql database!" unless success -end - -# Setup the database container. -def setup_db! - start_container!( - "library/mariadb:10.0.23", - "integration_db", - "Env" => ["MYSQL_ROOT_PASSWORD=portus"] - ) -end - -# Setup the Portus container. -# - rebuild: whether the image has to be rebuilt or not. -# - env: a list of additional environment variables to be passed. -# - ip: the IP that the Portus container should have. -# rubocop:disable Metrics/MethodLength -def setup_portus!(rebuild = true, env = [], address = nil) - dir = File.expand_path(File.dirname(__FILE__) + "../../../") - - # Build Portus with the current code and run it in a container. - Dir.chdir(dir) do - if rebuild - # Docker::Image.build_from_dir fails spectacularly on this. Just execute - # the damn command. - puts "Building Portus from directory: #{dir}" - PTY.spawn("docker build -t integration_portus:latest .") do |stdout, _, _| - # rubocop:disable Lint/HandleExceptions - - stdout.each { |line| print line } - rescue Errno::EIO - # End of output - - # rubocop:enable Lint/HandleExceptions - end - end - - opts = { - "Env" => [ - "PORTUS_MACHINE_FQDN_VALUE=#{ip}", - "PORTUS_DB_HOST=integration_db", - "RAILS_ENV=test", - env - ].flatten, - "Volumes" => { Dir.pwd => { "/portus" => "" } }, - "Cmd" => "puma -b tcp://0.0.0.0:3000 -w 10".split(" "), - "ExposedPorts" => { "3000/tcp" => {} }, - "HostConfig" => { - "Binds" => ["#{Dir.pwd}:/portus"], - "Links" => ["integration_db"], - "PortBindings" => { "3000/tcp" => [{ "HostIP" => "0.0.0.0", "HostPort" => "3000" }] } - } - } - opts["NetworkSettings"] = { "IPAddress" => address } unless address.nil? - start_container!("integration_portus:latest", "integration_portus", opts) - end -end -# rubocop:enable Metrics/MethodLength - -def setup_templates! - # Render the template - @ip = ip - src = __dir__ + "/" + "fixtures" - tpl = File.read(File.join(src, "config.yml.erb")) - res = ERB.new(tpl, nil, "<>").result(binding) - - # Write it - output = File.join(src, "config.yml") - File.open(output, "w") { |file| file.write(res) } -end - -# Get the IP from docker0 -def ip - docker0 = `/sbin/ifconfig docker0` - ips = docker0.scan(/(([0-9]{1,3}[\.]){3}[0-9]{1,3})/) - ips.first.first -end - -# Execute a command on the given container. Returns a boolean specifying -# whether the command succeeded or not. -def docker_exec(container, cmd, repeat = 0) - success = spawn_cmd("docker exec -it #{container} #{cmd}") - - # Execute the failing command if we are allowed to do it. - if !success && repeat != 0 - puts "Command '#{cmd}' failed! Waiting 5 seconds..." - sleep 5 - docker_exec(container, cmd, repeat - 1) - else - success - end -end - -# Spawn a new command and return its exit status. It will print to stdout on -# real time. -def spawn_cmd(cmd) - success = true - - PTY.spawn(cmd) do |stdout, _, pid| - # rubocop:disable Lint/HandleExceptions - begin - stdout.each { |line| print line } - rescue Errno::EIO - # End of output - end - # rubocop:enable Lint/HandleExceptions - - Process.wait(pid) - success = $CHILD_STATUS.exitstatus == 0 - end - success -end - -# Sets up an LDAP instance. The LDAP instance can be tweaked with the given -# name and password, which will be used to create a read-only user. Note that -# it will restart the portus and the distribution containers, so it is a bit -# slow. -def setup_ldap!(name, password) - # Spin up the LDAP server. - opts = { - "Env" => [ - "LDAP_READONLY_USER=true", - "LDAP_READONLY_USER_USERNAME=#{name}", - "LDAP_READONLY_USER_PASSWORD=#{password}" - ] - } - cname = "integration_ldap" - start_container!("osixia/openldap:1.1.2", cname, opts) - - # And re-start the Portus container with the new LDAP config. - hostname = `docker inspect -f {{.NetworkSettings.IPAddress}} #{cname}`.strip - portus = `docker inspect -f {{.NetworkSettings.IPAddress}} integration_portus`.strip - setup_portus!(false, [ - "PORTUS_LDAP_ENABLED=true", - "PORTUS_LDAP_HOSTNAME=#{hostname}", - "PORTUS_LDAP_UID=cn", - "PORTUS_LDAP_BASE=dc=example,dc=org", - "PORTUS_LDAP_AUTHENTICATION_ENABLED=true", - "PORTUS_LDAP_AUTHENTICATION_BIND_DN=cn=admin,dc=example,dc=org", - "PORTUS_LDAP_AUTHENTICATION_PASSWORD=admin" - ], portus) -end - -# Execute the given command inside of the Portus container. Raises an ExecError -# on failure. -def portus_exec(cmd) - raise ExecError, "Failed to execute '#{cmd}'" unless docker_exec("integration_portus", cmd) -end - -# Create a new user. -# TODO: let it work in LDAP. -def create_user(name, email, password, admin = false) - return if ldap? - portus_exec("rake portus:create_user[#{name},#{email},#{password},#{admin}]") -end - -# Error raised when a container fails to start. -class StartError < StandardError; end - -# Error raised when a command fails inside of a Docker container. -class ExecError < StandardError; end - -# Error raised when the login attempt has not been successful. -class LoginError < StandardError; end - -# Error raised when an expectation is not met (e.g. `eventually_expect`). -class ExpectError < StandardError; end - -# Log in the given user. It raises a LoginError on failure. -def login(user, password, email) - eventually_expect true do - output = `docker login -u #{user} -e #{email} -p #{password} #{registry_hostname}` - output.include? "Login Succeeded" - end -rescue ExpectError - raise LoginError, "Login failed!" -end - -# Logout. -def logout! - `docker logout #{registry_hostname}` -end - -# Execute a command to the Rails CLI, while expecting a JSON response. Returns -# the JSON object already parsed. -def rails_exec(cmd) - cmd = "puts #{cmd}" - output = capture_stdout do - docker_exec("integration_portus", "rails runner \"#{cmd}\"") - end - - res = output.split("\n").last.strip - JSON.parse(res) -end - -# Capture the stdout of the given block. -def capture_stdout - stream = StringIO.new - orginal = $stdout - $stdout = stream - yield - stream.string -ensure - $stdout = orginal -end - -# Run the given block and check whether it meets the given expectation for a -# reasonable amount of time. If not, it will raise an `ExpectError` exception. -def eventually_expect(expect) - 15.times do - res = yield - return true if res == expect - puts "Expecting '#{expect}', got '#{res}'" - sleep 5 - end - raise ExpectError, "Eventual expectation failed" -end - -# Cleanup all the containers that might be running. -def cleanup! - %w[integration_db integration_portus integration_ldap].each do |container| - cleanup_container!(container) - end - cleanup_distribution! -end - -# Cleanup distribution versions. -def cleanup_distribution! - SUPPORTED_DISTRIBUTION_VERSIONS.map { |v| "portus_distribution_#{v.delete(".")}" }.each do |c| - cleanup_container!(c) - end -end - -# Forces the removal of the given container. -def cleanup_container!(container) - # rubocop:disable Lint/HandleExceptions - Docker::Container.get(container) - system("docker rm -f #{container}") -rescue Docker::Error::NotFoundError - # Container does not exist, moving on. - # rubocop:enable Lint/HandleExceptions -end diff --git a/spec/integration/login_spec.rb b/spec/integration/login_spec.rb deleted file mode 100644 index 5ad975c63..000000000 --- a/spec/integration/login_spec.rb +++ /dev/null @@ -1,150 +0,0 @@ -# frozen_string_literal: true - -require "integration/helper" - -integration "Login" do - let(:name) { ldap? ? "johnldap" : "john" } - let(:email) { "john@example.com" } - let(:password) { "12341234" } - - after do - `docker logout #{registry_hostname}` - end - - it "logs in a valid user" do - create_user(name, email, password, true) - - # Valid user. - expect { login(name, password, email) }.not_to raise_error - - # Invalid name. - expect { login(name + "o", password, email) }.to raise_error(LoginError) - - # Invalid password. - expect { login(name, password + "o", email) }.to raise_error(LoginError) - - create_user("mc!", email, password, true) - expect { login(name, password, email) }.not_to raise_error - end - - it "allows users to push images with multiple tags" do - create_user(name, email, password, true) - expect { login(name, password, email) }.not_to raise_error - - # Pulling images that we should already have (so we go faster :P). Then - # re-tag it so they can be pushed. We tag both a pusheable image and - # another one that cannot be pushed (namespace does not exist). - imagename = "registry" - pulled_tags = ["2.2.1", "2.3"] - pulled_tags.each do |tag| - base = "#{imagename}:#{tag}" - img = "library/#{base}" - - pull(img) - system("docker tag #{img} #{registry_hostname}/#{img}") - system("docker tag #{img} #{registry_hostname}/#{base}") - end - - # Pushing... - expect(push("#{registry_hostname}/#{imagename}")).to be_truthy - expect(push("#{registry_hostname}/library/#{imagename}", false)).to be_truthy - - # And finally let's see what's inside the DB. The registry might not be - # ready yet, so let's call it inside of `eventually_expect`. - repos = [] - tags = [] - eventually_expect(1) do - repos = rails_exec("Repository.all.to_json") - repos.size - end - eventually_expect(2) do - tags = rails_exec("Tag.all.to_json") - tags.size - end - - expect(repos.first["name"]).to eq "registry" - names = tags.map { |t| t["name"] } - expect(names).to eq pulled_tags - end - - it "handles the personal namespace properly" do - create_user(name, email, password, true) - expect { login(name, password, email) }.not_to raise_error - - # Push an image to the personal namespace. - img = "registry:2.3" - target = "#{registry_hostname}/#{name}/#{img}" - pull(img) - system("docker tag #{img} #{target}") - expect(push(target)).to be_truthy - - # Logout and try to pull/push it. It should fail. - logout! - expect(spawn_cmd("docker pull #{target}")).to be_falsey - expect(push(target, false)).to be_truthy - end - - it "handles viewers, contributors and owners accordingly" do - # TODO: once create_user works on LDAP, remove this guard. - unless ldap? - owner = "owner" - contributor = "contributor" - viewer = "viewer" - - create_user(owner, email, password, true) - create_user(contributor, "test1@email.com", password) - create_user(viewer, "test2@email.com", password) - - rb = <<~HERE - team = Team.new(name: 'team') - owner = User.find_by(username: '#{owner}') - contributor = User.find_by(username: '#{contributor}') - viewer = User.find_by(username: '#{viewer}') - team.owners = [owner] - team.contributors = [contributor] - team.viewers = [viewer] - team.save - Namespace.create(team: team.reload, name: 'namespace', registry: Registry.get) -HERE - docker_exec("integration_portus", "rails r \"#{rb}\"") - - expect { login(owner, password, email) }.not_to raise_error - - # Push an image to the team namespace. - img = "registry:2.3" - target = "#{registry_hostname}/namespace/#{img}" - pull(img) - system("docker tag #{img} #{target}") - - # Owner can push. - expect(push(target)).to be_truthy - logout! - - # Contributor can push. - expect { login(contributor, password, email) }.not_to raise_error - expect(push(target)).to be_truthy - logout! - - # Viewer cannot push. - expect { login(viewer, password, email) }.not_to raise_error - expect(push(target, false)).to be_truthy - end - end - - it "allows anonymous users to pull from the global namespace, but not to push" do - create_user(name, email, password, true) - expect { login(name, password, email) }.not_to raise_error - - # Push an image to the global namespace. - img = "registry:2.3" - target = "#{registry_hostname}/#{img}" - pull(img) - system("docker tag #{img} #{target}") - expect(push(target)).to be_truthy - - # Logout and try to pull/push it. - logout! - eventually_expect(true) { spawn_cmd("docker pull #{target}") } - expect(push(target, false)).to be_truthy - end -end From f297fd71618b5a22cb061bf3b401bb7e7d474842 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miquel=20Sabat=C3=A9=20Sol=C3=A0?= Date: Wed, 7 Mar 2018 16:44:00 +0100 Subject: [PATCH 2/2] Implemented from scratch integration tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit I've also fixed a couple of small nitpicks in the current testing setup. Fixes #1667 Signed-off-by: Miquel Sabaté Solà --- .dockerignore | 1 + .gitignore | 1 + .travis.yml | 31 +-- Dockerfile | 2 + Gemfile.lock | 2 +- README.md | 67 +++++++ app/models/registry.rb | 7 +- bin/ci.sh | 83 ++++++++ bin/integration/integration.rb | 182 ++++++++++++++++++ bin/test-integration.sh | 101 ++++++++++ config/secrets.yml | 9 +- docker-compose.yml | 2 +- .../compose => compose/clair}/clair.yml | 0 examples/compose/docker-compose.clair.yml | 120 ++++++++++++ examples/compose/docker-compose.insecure.yml | 2 + lib/portus/cmd.rb | 31 +++ lib/portus/test.rb | 28 +++ lib/tasks/test/integration.rake | 59 ++++++ spec/integration/README.md | 113 +++++++++++ spec/integration/events.bats | 50 +++++ spec/integration/helpers.bash | 79 ++++++++ spec/integration/helpers/delete.rb | 16 ++ spec/integration/helpers/eval.rb | 7 + spec/integration/helpers/wait_event_done.rb | 51 +++++ spec/integration/login.bats | 21 ++ spec/integration/profiles/full.rb | 30 +++ spec/integration/profiles/minimal.rb | 13 ++ spec/integration/profiles/shared.rb | 29 +++ spec/integration/push.bats | 82 ++++++++ 29 files changed, 1185 insertions(+), 34 deletions(-) create mode 100755 bin/ci.sh create mode 100644 bin/integration/integration.rb create mode 100755 bin/test-integration.sh rename examples/{development/compose => compose/clair}/clair.yml (100%) create mode 100644 examples/compose/docker-compose.clair.yml create mode 100644 lib/portus/cmd.rb create mode 100644 lib/portus/test.rb create mode 100644 lib/tasks/test/integration.rake create mode 100644 spec/integration/README.md create mode 100644 spec/integration/events.bats create mode 100644 spec/integration/helpers.bash create mode 100644 spec/integration/helpers/delete.rb create mode 100644 spec/integration/helpers/eval.rb create mode 100644 spec/integration/helpers/wait_event_done.rb create mode 100644 spec/integration/login.bats create mode 100644 spec/integration/profiles/full.rb create mode 100644 spec/integration/profiles/minimal.rb create mode 100644 spec/integration/profiles/shared.rb create mode 100644 spec/integration/push.bats diff --git a/.dockerignore b/.dockerignore index ba6ff0188..cf9ec52eb 100644 --- a/.dockerignore +++ b/.dockerignore @@ -4,3 +4,4 @@ vagrant packaging log/* .DS_Store +.bundle/config \ No newline at end of file diff --git a/.gitignore b/.gitignore index 05fd9e5b6..20d410c26 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,7 @@ /public/assets /config/config-local.yml /bin/portus +/bin/integration/init /docker-compose.overlay.yml /spec/integration/fixtures/config.yml /spec/integration/fixtures/data diff --git a/.travis.yml b/.travis.yml index 01f6c495d..12d3e5e8c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -31,6 +31,12 @@ before_install: - sudo apt-get install yarn - yarn install + # Install bats. + - git clone https://github.com/sstephenson/bats.git + - cd bats + - sudo ./install.sh /usr/local + - cd .. && rm -rf bats + # Intall Go, which is needed for git-validation - eval "$(curl -sL https://raw.githubusercontent.com/travis-ci/gimme/master/gimme | GIMME_GO_VERSION=1.8 bash)" - go get -u github.com/vbatts/git-validation @@ -40,30 +46,7 @@ before_script: - psql -c 'create database portus_test' -U postgres script: - # Compile assets - - bundle exec rake portus:assets:compile - - # Ruby tests - - bundle exec rspec spec - - # Style and security checks - - bundle exec rubocop -V - - bundle exec rubocop -F - - # Note: it ignores a couple of files which use ruby 2.5 syntax which brakeman - # does not know how to handle... - - bundle exec brakeman --skip-files lib/portus/background/sync.rb,lib/portus/registry_client.rb - - - bundle exec rake portus:annotate_and_exit - - # JavaScript tests - - yarn test - - # JavaScript style - - yarn eslint - - # Test commit messages - - bundle exec rake test:git + - chmod +x bin/ci.sh && ./bin/ci.sh env: global: diff --git a/Dockerfile b/Dockerfile index 50f0a42ec..8ec10163a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -24,7 +24,9 @@ RUN zypper ref && \ update-alternatives --install /usr/bin/bundler bundler /usr/bin/bundler.ruby2.5 3 && \ bundle install --retry=3 && \ go get -u github.com/vbatts/git-validation && \ + go get -u github.com/openSUSE/portusctl && \ mv /root/go/bin/git-validation /usr/local/bin/ && \ + mv /root/go/bin/portusctl /usr/local/bin/ && \ zypper -n rm wicked wicked-service autoconf automake \ binutils bison cpp cvs flex gdbm-devel gettext-tools \ libtool m4 make makeinfo && \ diff --git a/Gemfile.lock b/Gemfile.lock index 4799e8ae3..9cfec6df0 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -59,7 +59,7 @@ GEM bootstrap-sass (3.3.7) autoprefixer-rails (>= 5.2.1) sass (>= 3.3.4) - brakeman (4.1.1) + brakeman (4.2.0) builder (3.2.3) byebug (9.1.0) capybara (2.14.4) diff --git a/README.md b/README.md index 58a757f2a..d5367f2dd 100644 --- a/README.md +++ b/README.md @@ -103,6 +103,73 @@ For more information on development environments, check our feel free to explore the `examples` directory for a variety of ways in which you can deploy Portus. +### Testing + +#### Unit tests + +Unit tests are located in the `spec` directory. To run them, simply: + + $ bundle exec rspec spec + +Make sure to install [phantomjs](http://phantomjs.org/) from your Linux +distribution before running unit tests, since feature tests rely on PhantomJS +being installed. All the other ruby dependencies are already covered by our +`Gemfile`. + +We also have tests in the frontend. For this, you have to install +[yarn](https://yarnpkg.com/) from your Linux distribution and run: + + $ yarn test + +#### Integration tests + +Check [this +document](https://github.com/SUSE/Portus/blob/master/spec/integration/README.md) +in order to better understand how integration tests work. For development, +though, if you have already installed Docker, docker-composer and +[bats](https://github.com/sstephenson/bats), running the following should just +work: + +``` +$ chmod +x bin/test-integration.sh +$ ./bin/test-integration.sh +``` + +#### Other checks + +A common pitfall for developers is to forget about code style. For that, make +sure to run [rubocop](https://github.com/bbatsov/rubocop): + + $ bundle exec rubocop -a + +Note that the command above includes the `-a` flag. This flag will automatically +fix small issues for you. We also run a code style check for the frontend code: + + $ yarn eslint + +We also run [brakeman](https://brakemanscanner.org/) in order to detect security +vulnerabilities: + + $ bundle exec brakeman + +Last but not least, make sure that your git commit follows a proper style. To +ensure this, you can run the following task: + + $ bundle exec rake test:git + +#### The CI + +We use [Travis CI](https://travis-ci.org/) for continuous integration. You can +run what we run in Travis locally: + +``` +$ chmod +x bin/ci.sh +$ ./bin/ci.sh +``` + +This script simply executes all the tests and checks that we have presented +above. + ## Supported versions Docker technologies have a fast iteration pace. This is a good thing, but it diff --git a/app/models/registry.rb b/app/models/registry.rb index 3d7db0592..374947465 100644 --- a/app/models/registry.rb +++ b/app/models/registry.rb @@ -34,10 +34,9 @@ class Registry < ActiveRecord::Base # On create, make sure that all the needed namespaces are in place. after_create :create_namespaces! - # Today the data model supports many registries - # however Portus just supports on Registry - # therefore to avoid confusion, define just one way - # to ask for the registy + # Today the data model supports many registries however Portus just supports + # on Registry therefore to avoid confusion, define just one way to ask for the + # registy def self.get Registry.first end diff --git a/bin/ci.sh b/bin/ci.sh new file mode 100755 index 000000000..a4a7c0338 --- /dev/null +++ b/bin/ci.sh @@ -0,0 +1,83 @@ +#!/usr/bin/env bash + +set -e + +## +# Auxiliar functions + +PORTUS_DB_ADAPTER=${PORTUS_DB_ADAPTER:-mysql2} + +# Interacts with a daemon by taking two arguments: +# 1. The action (e.g. "start"). +# 2. The service (e.g. "mysql"). +# We do this to abstract the fact that Travis CI does not use systemd and we do. +function __daemon() { + if [[ -z "$CI" ]]; then + sudo systemctl $1 $2 + else + sudo service $2 $1 + fi +} + +# Performs systemctl calls to the current database adapter when used outside of +# a container. +function __database() { + if [[ -f /.dockerenv ]]; then + return + fi + + if [[ "$PORTUS_DB_ADAPTER" == "mysql2" ]]; then + __daemon $1 mysql + else + __daemon $1 postgresql + fi +} + +# Setup an insecure registry for the local docker. +function __docker_insecure() { + if [[ ! -z "$CI" ]]; then + sudo tee /etc/docker/daemon.json > /dev/null </dev/null 2>&1 || { echo >&2 "Bats is required. See https://github.com/sstephenson/bats"; exit 1; } + +## +# Set up the build directory. + +# Exported because it will be re-used by bats. +export ROOT_DIR="$( cd "$( dirname "$0" )/.." && pwd )" +export CNAME="integration_portus" +export RNAME="integration_registry" + +# Download the `init` script if possible. +if [ ! -f "$ROOT_DIR/bin/integration/init" ]; then + echo "[integration] Init file does not exist, downloading into '$ROOT_DIR/bin/integration/init'" + wget -O $ROOT_DIR/bin/integration/init https://raw.githubusercontent.com/openSUSE/docker-containers/master/derived_images/portus/init +fi +chmod +x $ROOT_DIR/bin/integration/init + +# Generate the build directory. +bundle exec rails runner $ROOT_DIR/bin/integration/integration.rb + +## +# Start containers. + +# It will kill and remove all containers related to integration testing. +cleanup_containers() { + pushd "$ROOT_DIR/build" + docker-compose kill + docker-compose rm -f + popd +} + +if [[ ! "$SKIP_ENV_TESTS" ]]; then + cleanup_containers + pushd "$ROOT_DIR/build" + docker-compose up -d + popd + + # We will wait 10 minutes until everything is properly set up. + TIMEOUT=600 + COUNT=0 + RETRY=1 + + while [ $RETRY -ne 0 ]; do + msg=$(SKIP_MIGRATION=1 docker exec $CNAME portusctl exec rails r /srv/Portus/bin/check_db.rb) + case $(echo "$msg" | grep DB) in + "DB_READY") + echo "Database ready" + break + ;; + *) + echo "Database is not ready yet:" + echo $msg + ;; + esac + + if [ "$COUNT" -ge "$TIMEOUT" ]; then + echo "[integration] Timeout reached, exiting with error" + cleanup_containers + exit 1 + fi + + sleep 5 + COUNT=$((COUNT+5)) + done + + echo "You may want to set the 'SKIP_ENV_TESTS' env. variable for successive runs..." + + # Travis oddities... + if [ ! -z "$CI" ]; then + sleep 10 + fi +fi + +# Run tests. +tests=() +if [[ -z "$TESTS" ]]; then + tests=($ROOT_DIR/spec/integration/*.bats) +else + for f in $TESTS; do + tests+=("$ROOT_DIR/spec/integration/$f.bats") + done +fi +set +e +echo "Running: ${tests[*]}" +bats -t ${tests[*]} +status=$? +set -e + +# Tear down +if [[ "$TEARDOWN_TESTS" ]]; then + cleanup_containers +fi + +exit $status diff --git a/config/secrets.yml b/config/secrets.yml index 8f720f49d..27a8311ed 100644 --- a/config/secrets.yml +++ b/config/secrets.yml @@ -11,9 +11,9 @@ # if you're sharing your code publicly. default: &default - secret_key_base: 8bc8ccc710eafd73d43cd59ac8881aadc89f7a6ab55f1ac11c97fb436a3931cc78c38e735e664958d9e793725f3d52178f4e2c376c346edbaca3936aebf66e27 + secret_key_base: <%= ENV["PORTUS_SECRET_KEY_BASE"] || "8bc8ccc710eafd73d43cd59ac8881aadc89f7a6ab55f1ac11c97fb436a3931cc78c38e735e664958d9e793725f3d52178f4e2c376c346edbaca3936aebf66e27" %> encryption_private_key_path: <%= ENV["PORTUS_KEY_PATH"] || "examples/development/vagrant/conf/ca_bundle/server.key" %> - portus_password: "portus1234" + portus_password: <%= ENV["PORTUS_PASSWORD"] || "portus1234" %> development: <<: *default @@ -24,8 +24,9 @@ staging: test: <<: *default -# Do not keep production secrets in the repository, -# instead read values from the environment. +# The only difference from production to the other environments, is that this +# environment forces data to be on environment variables, instead of taking a +# default. production: secret_key_base: <%= ENV["PORTUS_SECRET_KEY_BASE"] %> encryption_private_key_path: <%= ENV["PORTUS_KEY_PATH"] %> diff --git a/docker-compose.yml b/docker-compose.yml index 74702aeda..ec9d7c4e7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -66,7 +66,7 @@ services: - "6060-6061:6060-6061" volumes: - /tmp:/tmp - - ./examples/development/compose/clair.yml:/clair.yml + - ./examples/compose/clair/clair.yml:/clair.yml command: [-config, /clair.yml] db: diff --git a/examples/development/compose/clair.yml b/examples/compose/clair/clair.yml similarity index 100% rename from examples/development/compose/clair.yml rename to examples/compose/clair/clair.yml diff --git a/examples/compose/docker-compose.clair.yml b/examples/compose/docker-compose.clair.yml new file mode 100644 index 000000000..ebc78879e --- /dev/null +++ b/examples/compose/docker-compose.clair.yml @@ -0,0 +1,120 @@ +version: "2" + +services: + portus: + image: opensuse/portus:head + environment: + - PORTUS_MACHINE_FQDN_VALUE=${MACHINE_FQDN} + - PORTUS_SECURITY_CLAIR_SERVER=http://clair:6060 + + # DB. The password for the database should definitely not be here. You are + # probably better off with Docker Swarm secrets. + - PORTUS_DB_HOST=db + - PORTUS_DB_DATABASE=portus_production + - PORTUS_DB_PASSWORD=${DATABASE_PASSWORD} + - PORTUS_DB_POOL=5 + + # Secrets. It can possibly be handled better with Swarm's secrets. + - PORTUS_SECRET_KEY_BASE=${SECRET_KEY_BASE} + - PORTUS_KEY_PATH=/certificates/portus.key + - PORTUS_PASSWORD=${PORTUS_PASSWORD} + + # SSL + - PORTUS_CHECK_SSL_USAGE_ENABLED='false' + + # Since we have no nginx in insecure mode, portus have to + # serve the static files + - RAILS_SERVE_STATIC_FILES='true' + ports: + - 3000:3000 + depends_on: + - db + links: + - db + volumes: + - ./secrets:/certificates:ro + + background: + image: opensuse/portus:head + depends_on: + - portus + - db + environment: + # Theoretically not needed, but cconfig's been buggy on this... + - CCONFIG_PREFIX=PORTUS + - PORTUS_MACHINE_FQDN_VALUE=${MACHINE_FQDN} + - PORTUS_SECURITY_CLAIR_SERVER=http://clair:6060 + + # DB. The password for the database should definitely not be here. You are + # probably better off with Docker Swarm secrets. + - PORTUS_DB_HOST=db + - PORTUS_DB_DATABASE=portus_production + - PORTUS_DB_PASSWORD=${DATABASE_PASSWORD} + - PORTUS_DB_POOL=5 + + # Secrets. It can possibly be handled better with Swarm's secrets. + - PORTUS_SECRET_KEY_BASE=${SECRET_KEY_BASE} + - PORTUS_KEY_PATH=/certificates/portus.key + - PORTUS_PASSWORD=${PORTUS_PASSWORD} + + - PORTUS_BACKGROUND=true + links: + - db + volumes: + - ./secrets:/certificates:ro + + db: + image: library/mariadb:10.0.23 + command: mysqld --character-set-server=utf8 --collation-server=utf8_unicode_ci --init-connect='SET NAMES UTF8;' --innodb-flush-log-at-trx-commit=0 + environment: + - MYSQL_DATABASE=portus_production + + # Again, the password shouldn't be handled like this. + - MYSQL_ROOT_PASSWORD=${DATABASE_PASSWORD} + volumes: + - /var/lib/portus/mariadb:/var/lib/mysql + + registry: + image: library/registry:2.6 + environment: + # Authentication + REGISTRY_AUTH_TOKEN_REALM: http://${MACHINE_FQDN}:3000/v2/token + REGISTRY_AUTH_TOKEN_SERVICE: ${MACHINE_FQDN}:5000 + REGISTRY_AUTH_TOKEN_ISSUER: ${MACHINE_FQDN} + REGISTRY_AUTH_TOKEN_ROOTCERTBUNDLE: /secrets/portus.crt + + # Portus endpoint + REGISTRY_NOTIFICATIONS_ENDPOINTS: > + - name: portus + url: http://${MACHINE_FQDN}:3000/v2/webhooks/events + timeout: 2000ms + threshold: 5 + backoff: 1s + volumes: + - /var/lib/portus/registry:/var/lib/registry + - ./secrets:/secrets:ro + - ./registry/config.yml:/etc/docker/registry/config.yml:ro + ports: + - 5000:5000 + - 5001:5001 # required to access debug service + links: + - portus:portus + + clair: + image: quay.io/coreos/clair:v2.0.1 + restart: unless-stopped + depends_on: + - postgres + links: + - postgres + ports: + - "6060-6061:6060-6061" + volumes: + - /tmp:/tmp + - ./clair/clair.yml:/clair.yml + command: [-config, /clair.yml] + + postgres: + image: library/postgres:10-alpine + environment: + POSTGRES_PASSWORD: portus diff --git a/examples/compose/docker-compose.insecure.yml b/examples/compose/docker-compose.insecure.yml index c75f3a86c..7333d7e68 100644 --- a/examples/compose/docker-compose.insecure.yml +++ b/examples/compose/docker-compose.insecure.yml @@ -26,6 +26,8 @@ services: - RAILS_SERVE_STATIC_FILES='true' ports: - 3000:3000 + depends_on: + - db links: - db volumes: diff --git a/lib/portus/cmd.rb b/lib/portus/cmd.rb new file mode 100644 index 000000000..3853c6290 --- /dev/null +++ b/lib/portus/cmd.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +# :nocov: + +require "pty" + +module ::Portus + class Cmd + # Spawn a new command and return its exit status. It will print to stdout on + # real time. + def self.spawn(cmd) + success = true + + ::PTY.spawn(cmd) do |stdout, _, pid| + # rubocop:disable Lint/HandleExceptions + # rubocop:disable Rails/Output + begin + stdout.each { |line| print line } + rescue Errno::EIO + # End of output + end + # rubocop:enable Lint/HandleExceptions + # rubocop:enable Rails/Output + + Process.wait(pid) + success = $CHILD_STATUS.exitstatus.zero? + end + success + end + end +end diff --git a/lib/portus/test.rb b/lib/portus/test.rb new file mode 100644 index 000000000..8cf0de146 --- /dev/null +++ b/lib/portus/test.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +# :nocov: + +module ::Portus + # Test defines constants which are used in multiple places regarding + # integration tests. + class Test + LOCAL_IMAGE = "opensuse/portus:development" + HEAD_IMAGE = "opensuse/portus:head" + DEVELOPMENT_MATRIX = { + background: LOCAL_IMAGE, + db: "library/mariadb:10.0.23", + clair: "quay.io/coreos/clair:v2.0.1", + portus: LOCAL_IMAGE, + postgres: "library/postgres:10-alpine", + registry: "library/registry:2.6" + }.freeze + + # Returns true if the given image is allowed to fail in an integration + # test. Note that this image can be a string or a hash (with an element + # named :portus). + def self.allow_failure?(image) + return HEAD_IMAGE == image if image.is_a? String + image[:portus] == HEAD_IMAGE + end + end +end diff --git a/lib/tasks/test/integration.rake b/lib/tasks/test/integration.rake new file mode 100644 index 000000000..d0ca8af73 --- /dev/null +++ b/lib/tasks/test/integration.rake @@ -0,0 +1,59 @@ +# frozen_string_literal: true + +require "portus/cmd" +require "portus/test" + +# TODO: on a Travis PR, only test portus:development and registry:2.{5,6} +# TODO: otherwise, test with portus:2.3 and portus:head + +# Images for the supported registry versions. These versions will be applied +# through a cartesian product into the test matrix. +SUPPORTED_REGISTRIES = [ + "library/registry:2.5", + "library/registry:2.6" +].freeze + +MATRIX = [ + # Development + {}, + + # Stable release: 2.3 + { + background: "opensuse/portus:2.3", + portus: "opensuse/portus:2.3" + }, + + # Master. + { + background: "opensuse/portus:head", + portus: "opensuse/portus:head" + } +].product(SUPPORTED_REGISTRIES).map { |v| v.first.merge(registry: v.last) }.freeze + +namespace :test do + desc "Run the integration test suite" + task integration: :environment do + status = 0 + + MATRIX.each do |env| + str = env.map { |k, v| "#{k}##{v}" }.join(" ") + puts "[integration] Environment string: #{str}" + + # Don't build the image multiple times on Travis. + ENV["PORTUS_INTEGRATION_BUILD_IMAGE"] = "false" if ENV["CI"].present? + + ENV["PORTUS_TEST_INTEGRATION"] = str if str.present? + ENV["TEARDOWN_TESTS"] = "true" + + script = Rails.root.join("bin", "test-integration.sh") + success = ::Portus::Cmd.spawn("/bin/bash #{script}") + + unless success + status = 1 + puts "[integration] Allowed failure. Ignoring..." if ::Portus::Test.allow_failure?(env) + end + end + + exit status + end +end diff --git a/spec/integration/README.md b/spec/integration/README.md new file mode 100644 index 000000000..295811bfc --- /dev/null +++ b/spec/integration/README.md @@ -0,0 +1,113 @@ +## Integration tests + +### TL;DR + +Install docker, docker-compose and +[bats](https://github.com/sstephenson/bats). When developing, run: + + $ ./bin/test-integration.sh + +Before submitting a pull request, run: + + $ bundle exec rake test:integration + +This is already handled by the `bin/ci.sh` script though, so run that instead. + +### Dependencies + +The test suite expects the host to have installed all the Ruby dependencies for +development/test purposes as specified in the `Gemfile.lock` file. Thus, you +should perform (in the root of the project): + +``` +$ bundle +``` + +Besides this, you need Docker, docker-compose and +[bats](https://github.com/sstephenson/bats). + +### How to run integration tests + +The integration tests go through some stages before completing: + +1. A `build` directory is created with all the dependencies of the run + (i.e. `docker-compose.yml` for the specific versions, config files, etc.). +2. We then run `docker-compose` in the context of this newly created `build` + directory. +3. Finally we execute the `bats` tests located in `spec/integration` targeting + the created containers. + +All this can be accomplished with a simple command: + + $ bundle exec rake test:integration + +This is the command we use in Travis CI, and it executes all tests with the +following matrix: + +- The Portus and the background containers use a local build of the code + (i.e. `opensuse/portus:development`), the current stable release and the + current `head` tag. +- Each version of Portus being tested will use different supported versions of + the Docker registry. + +If all tests have passed... good! You are ready to go. That being said, if some +tests have passed, maybe it's because some image (most commonly `head`) is +currently broken, so don't be afraid and feel free to submit a PR for your +changes nonetheless. + +All that being said, when developing this can be *tedious*. There are lots of +tests to be run for lots of different combinations. So, instead of running the +rake test directly, you should be using the `bin/test-integration.sh` script +instead. In short, the rake task simply defines a matrix of combinations, and +then runs this script for each combo. By default, this script will run only the +development combination, so if you just want to perform a quick test on some +changes you are working on, you can simply perform: + + $ ./bin/test-integration.sh + +This script is quite flexible and it allows you to define the following +environment variables: + +- `SKIP_ENV_TESTS`: use this when running tests against a set of containers + which already exist (i.e. from a previous run). +- `TESTS`: a space-separated list of tests to be run. +- `TEARDOWN_TESTS`: set this to cleanup your host from running containers after + tests have finished. This is disabled by default so you can check the logs + after tests have finished, and to re-use these same containers on successive + runs with the `SKIP_ENV_TESTS` environment variables. + +As an example, this is how you'd run this script if you are just interesed in +the `spec/integration/push.bats` test: + + $ TESTS="push" ./bin/test-integration.sh + +This script will perform the stages described above. As a final note, the first +stage (setting up the `build` directory) is done by another script: +`bin/integration/integration.rb`. You don't need to know much from this script, +besides that it accepts some other environment variables besides the ones +described above and that might be useful on advanced uses: + +- `PORTUS_INTEGRATION_BUILD_IMAGE`: set to `false` if you don't want to re-build + the development image for every single run (i.e. you want to re-use an updated + image). +- `PORTUS_TEST_INTEGRATION`: a space-separated list of containers with their + versions. This is the environment variable touched by the rake task described + above, and it allows users to run variations of the default matrix. For + example, you can set it like this: + `PORTUS_TEST_INTEGRATION="portus#myimage:latest registry#registry:mine"`. + +### What you need to know when writing tests + +Tests are written with `bats`. You can think of it as bash with some extra +utilities for running tests. If you have to write a new file, pick another one +as an example. + +When writing tests make sure to use the functions defined in +`helpers.bash`. This file contains quite some useful functions that will prevent +from running into common pitfalls. As an overview: + +- Always use `sane_run` when you want to execute an external command + (e.g. instead of the default `run` function). +- Do *not* run `sane_run` when another helper would suffice. For example, do not + run `sane_run docker exec $CNAME portusctl exec mycommand` when + `portusctl_exec mycommand` would suffice. diff --git a/spec/integration/events.bats b/spec/integration/events.bats new file mode 100644 index 000000000..ba66bf8e0 --- /dev/null +++ b/spec/integration/events.bats @@ -0,0 +1,50 @@ +#!/usr/bin/env bats -t + +load helpers + +function setup() { + __setup full +} + +@test "updates the database after a successful push" { + docker_run login -u admin -p 12341234 172.17.0.1:5000 + [ $status -eq 0 ] + + docker_tag opensuse/portus:development 172.17.0.1:5000/portus:uniquetag + docker_run push 172.17.0.1:5000/portus:uniquetag + [ $status -eq 0 ] + + helper_runner wait_event_done.rb uniquetag + [ $status -eq 0 ] + + ruby_puts "Tag.count" + [[ "${lines[-1]}" =~ "1" ]] + + ruby_puts "Tag.first.name" + [[ "${lines[-1]}" =~ "uniquetag" ]] +} + +@test "updates the database after a successful delete" { + docker_run login -u admin -p 12341234 172.17.0.1:5000 + [ $status -eq 0 ] + + docker_tag opensuse/portus:development 172.17.0.1:5000/portus:uniquetag + docker_run push 172.17.0.1:5000/portus:uniquetag + [ $status -eq 0 ] + + helper_runner wait_event_done.rb uniquetag + [ $status -eq 0 ] + + ruby_puts "Tag.count" + [[ "${lines[-1]}" =~ "1" ]] + + # And now let's delete this tag. + helper_runner delete.rb portus uniquetag + [ $status -eq 0 ] + + helper_runner wait_event_done.rb uniquetag pickfirst + [ $status -eq 0 ] + + ruby_puts "Tag.count" + [[ "${lines[-1]}" =~ "0" ]] +} diff --git a/spec/integration/helpers.bash b/spec/integration/helpers.bash new file mode 100644 index 000000000..a3d0c5f75 --- /dev/null +++ b/spec/integration/helpers.bash @@ -0,0 +1,79 @@ +# Function taken from openSUSE/umoci. See: +# https://github.com/openSUSE/umoci/blob/57c73c27fe3c13d80e1fb7f82c9a046a2bc2b6f1/test/helpers.bash#L116-L125 +function sane_run() { + local cmd="$1" + shift + + run "$cmd" "$@" + + # Some debug information to make life easier. + echo "$(basename "$cmd") $@ (status=$status)" >&2 + echo "$output" >&2 +} + +# Wrapper for the docker command. +function docker_run() { + sane_run docker $@ +} + +# It accepts two parameters: the old tag and the new tag. This function will +# first remove any reference to the new tag, and then perform a docker tag. +function docker_tag() { + docker_run rmi -f $2 + docker_run tag $1 $2 +} + +# Performs a docker exec into the Portus container. +function docker_exec() { + docker_run exec $CNAME $@ +} + +# Performs a portusctl exec inside of the Portus container. +function portusctl_exec() { + docker_exec portusctl exec $@ +} + +# Run the given runner with the given arguments. +function helper_runner() { + local file="$1" + shift + + portusctl_exec rails runner /srv/Portus/spec/integration/helpers/$file $@ +} + +# Runs the `spec/integration/helpers/eval.rb` runner by passing the given +# argument. +function ruby_puts() { + helper_runner eval.rb $1 +} + +# Setup the database for each test case. It accepts an argument which can be +# used to determine the profile to be loaded on the database. +function __setup_db() { + portusctl_exec rails r /srv/Portus/spec/integration/profiles/$1.rb +} + +# Logout the current user. Perform this before each test. +function __logout() { + docker_run logout 172.17.0.1:5000 +} + +# Cleanup the data from the registry. +function __clear_registry() { + docker_run exec $RNAME rm -rf /var/lib/registry/docker/registry/v2/* +} + +# Restart the given services. +function __restart() { + pushd $ROOT_DIR/build + sane_run docker-compose restart $@ + popd +} + +# The main function to be called on `setup`. It accepts an argument, which will +# be directly passed to the `__setup_db` function. +function __setup() { + __setup_db $1 + __logout + __clear_registry +} diff --git a/spec/integration/helpers/delete.rb b/spec/integration/helpers/delete.rb new file mode 100644 index 000000000..ae97b2acf --- /dev/null +++ b/spec/integration/helpers/delete.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +# It accepts exactly two arguments: the repository name (full name) and the tag +# name. With these two things, it will simply issue a delete request to the +# Registry. + +require "portus/registry_client" + +REPOSITORY = ARGV.first.dup +TAG = ARGV.last.dup + +RegistryEvent.all.destroy_all + +client = Registry.get.client +_, digest, = client.manifest(REPOSITORY, TAG) +client.delete(REPOSITORY, digest, "manifests") diff --git a/spec/integration/helpers/eval.rb b/spec/integration/helpers/eval.rb new file mode 100644 index 000000000..32e0538a3 --- /dev/null +++ b/spec/integration/helpers/eval.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +# Outputs the evaluated object from the first argument. + +# rubocop:disable Security/Eval +puts eval(ARGV.first) +# rubocop:enable Security/Eval diff --git a/spec/integration/helpers/wait_event_done.rb b/spec/integration/helpers/wait_event_done.rb new file mode 100644 index 000000000..6ecd028b1 --- /dev/null +++ b/spec/integration/helpers/wait_event_done.rb @@ -0,0 +1,51 @@ +# frozen_string_literal: true + +# This runner waits until a registry event related to the given tag (first +# argument) is marked as done by the background process. This runner will +# timeout if it takes just too long. Last but not least, you can pass a final +# argument with the string "pickfirst", which will tell this runner to simply +# pick the first registry event. + +require "json" + +TAG = ARGV.first.dup + +# Returns true +def pick_first? + ARGV.last == "pickfirst" +end + +# Returns the first registry event that matches the required tag, or nil if none +# could be found. +def first_matching_tag + r = Registry.get + return unless r + + RegistryEvent.all.find_each do |event| + data = JSON.parse(event.data) + _, _, tag_name = r.get_namespace_from_event(data) + return event.dup if tag_name == TAG + end + + nil +end + +# Returns true if the event we were interested in has been processed. +def done? + re = pick_first? ? RegistryEvent.first : first_matching_tag + return false unless re + + re.status.to_i == RegistryEvent.statuses[:done].to_i +end + +SLEEP_TIME = 5.seconds +TIMEOUT = 1.minute + +current = 0 +while current < TIMEOUT + exit 0 if done? + sleep SLEEP_TIME + current += SLEEP_TIME +end + +exit 1 diff --git a/spec/integration/login.bats b/spec/integration/login.bats new file mode 100644 index 000000000..7d5de9bf1 --- /dev/null +++ b/spec/integration/login.bats @@ -0,0 +1,21 @@ +#!/usr/bin/env bats -t + +load helpers + +function setup() { + __setup minimal +} + +@test "proper user can run docker login" { + docker_run login -u admin -p 12341234 172.17.0.1:5000 + [ $status -eq 0 ] + # The first line is a warning because we are passing the password directly + # from the CLI. + [[ "${lines[1]}" =~ "Login Succeeded" ]] +} + +@test "unknown user cannot login" { + docker_run login -u user -p 12341234 172.17.0.1:5000 + [ $status -eq 1 ] + [[ "${lines[1]}" =~ "authentication required" ]] +} diff --git a/spec/integration/profiles/full.rb b/spec/integration/profiles/full.rb new file mode 100644 index 000000000..927295bb2 --- /dev/null +++ b/spec/integration/profiles/full.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +require_relative "shared" + +clean_db! +create_registry! + +admin = User.create!( + username: "admin", + password: "12341234", + email: "admin@example.local", + admin: true +) + +contributor = User.create!( + username: "user", + password: "12341234", + email: "user@example.local", + admin: false +) + +viewer = User.create!( + username: "viewer", + password: "12341234", + email: "viewer@example.local", + admin: false +) + +t = Team.create!(name: "team", owners: [admin], contributors: [contributor], viewers: [viewer]) +Namespace.create!(name: "namespace", team: t, registry: Registry.get) diff --git a/spec/integration/profiles/minimal.rb b/spec/integration/profiles/minimal.rb new file mode 100644 index 000000000..197770fb2 --- /dev/null +++ b/spec/integration/profiles/minimal.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +require_relative "shared" + +clean_db! +create_registry! + +User.create!( + username: "admin", + password: "12341234", + email: "admin@example.local", + admin: true +) diff --git a/spec/integration/profiles/shared.rb b/spec/integration/profiles/shared.rb new file mode 100644 index 000000000..1cdbce1ce --- /dev/null +++ b/spec/integration/profiles/shared.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +# It truncates the DB. Use this always on profiles. +def clean_db! + ActiveRecord::Base.establish_connection + ActiveRecord::Base.connection.execute("SET FOREIGN_KEY_CHECKS = 0") + ActiveRecord::Base.connection.tables.each do |table| + next if table == "schema_migrations" + + ActiveRecord::Base.connection.execute("TRUNCATE #{table}") + end + ActiveRecord::Base.connection.execute("SET FOREIGN_KEY_CHECKS = 1") +end + +# Creates a registry that works with the current setup, and it creates the +# Portus special user. +def create_registry! + # Hostname configurable so some tests can check wrong hostnames. + hostname = ENV["PORTUS_INTEGRATION_HOSTNAME"] || "172.17.0.1:5000" + Registry.create!(name: "registry", hostname: hostname, use_ssl: false) + ENV["PORTUS_INTEGRATION_HOSTNAME"] = nil + + User.create!( + username: "portus", + password: Rails.application.secrets.portus_password, + email: "portus@portus.com", + admin: true + ) +end diff --git a/spec/integration/push.bats b/spec/integration/push.bats new file mode 100644 index 000000000..d35b81f36 --- /dev/null +++ b/spec/integration/push.bats @@ -0,0 +1,82 @@ +#!/usr/bin/env bats -t + +load helpers + +function setup() { + __setup full +} + +## +# Pushing into the global namespace. + +@test "admin user can push to the global namespace" { + docker_run login -u admin -p 12341234 172.17.0.1:5000 + [ $status -eq 0 ] + + docker_tag opensuse/portus:development 172.17.0.1:5000/portus:development + docker_run push 172.17.0.1:5000/portus:development + + [ $status -eq 0 ] + [[ "${lines[-1]}" =~ "development: digest: sha256:" ]] +} + +@test "admin user can push multiple tags at once to the global namespace" { + docker_run login -u admin -p 12341234 172.17.0.1:5000 + [ $status -eq 0 ] + + docker_tag opensuse/portus:development 172.17.0.1:5000/portus:development + docker_tag opensuse/portus:development 172.17.0.1:5000/portus:development2 + docker_run push 172.17.0.1:5000/portus + + [ $status -eq 0 ] +} + +@test "regular user cannot push into the global namespace" { + docker_run login -u user -p 12341234 172.17.0.1:5000 + [ $status -eq 0 ] + + docker_tag opensuse/portus:development 172.17.0.1:5000/portus:development + docker_run push 172.17.0.1:5000/portus:development + + [ $status -eq 1 ] + [[ "${lines[-1]}" =~ "authentication required" ]] +} + +@test "push fails if the hostname of the registry is not properly set" { + export PORTUS_INTEGRATION_HOSTNAME="whatever:4000" + __setup_db full + + docker_run login -u user -p 12341234 172.17.0.1:5000 + + docker_tag opensuse/portus:development 172.17.0.1:5000/portus:development + docker_run push 172.17.0.1:5000/portus:development + [ $status -eq 1 ] +} + +## +# Pushing into namespace. + +@test "pushing in a namespace with contributors and viewers" { + docker_run login -u user -p 12341234 172.17.0.1:5000 + [ $status -eq 0 ] + + # Contributor can push and pull + + docker_tag opensuse/portus:development 172.17.0.1:5000/namespace/portus:development + docker_run push 172.17.0.1:5000/namespace/portus:development + [ $status -eq 0 ] + + docker_run pull 172.17.0.1:5000/namespace/portus:development + [ $status -eq 0 ] + + # Viewer can only push + + __logout + docker_run login -u viewer -p 12341234 172.17.0.1:5000 + + docker_run push 172.17.0.1:5000/namespace/portus:development + [ $status -eq 1 ] + + docker_run pull 172.17.0.1:5000/namespace/portus:development + [ $status -eq 0 ] +}