RFC: Resources v2

001-resources-v2/branch-gen.yml

@@ -0,0 +1,39 @@
+resources:
+- name: atc
+  type: git
+  source: {uri: https://github.com/concourse/atc}
+  space: master
+
+- name: atc-gen
+  type: git
+  source: {uri: "https://github.com/concourse/atc"}
+
+jobs:
+- name: gen
+  plan:
+  - get: atc
+    trigger: true
+  - task: gen
+    file: atc/ci/gen.yml
+    config:
+      platform: ...
+      image_resource: ...
+      outputs:
+      - name: generated-repo
+        # has the generated code committed to the repo
+      - name: branch-name
+        # has a 'name' file with `gen-(some deterministic hash)`
+  - put: atc-gen
+    params: {branch_name: branch-name/name, repository: generated-repo}
+
+- name: test
+  plan:
+  - get: atc
+    passed: [gen]
+    trigger: true
+  - get: atc-gen
+    passed: [gen]
+    spaces: [gen-*]
+    trigger: true
+  - task: test
+    file: atc/ci/test.yml

001-resources-v2/git-example/.gitignore

@@ -0,0 +1,2 @@
+check-repo
+dot

001-resources-v2/git-example/Gemfile

@@ -0,0 +1,4 @@
+source :rubygems
+
+gem 'rugged'
+gem 'pry'

001-resources-v2/git-example/Gemfile.lock

@@ -0,0 +1,19 @@
+GEM
+  remote: http://rubygems.org/
+  specs:
+    coderay (1.1.2)
+    method_source (0.9.0)
+    pry (0.11.3)
+      coderay (~> 1.1.0)
+      method_source (~> 0.9.0)
+    rugged (0.27.2)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  pry
+  rugged
+
+BUNDLED WITH
+   1.16.2

001-resources-v2/git-example/README.md

@@ -0,0 +1,13 @@
+# Git Resource v2
+
+This implementation is done in Ruby using the `git` gem. I chose Ruby
+over Bash because having a real language with more accessible data
+structures is probably going to be more important with this new
+interface, and Ruby feels pretty well suited (really just need a bit more
+than Bash).
+
+Please leave comments on parts you like/don't like! But bear in mind the
+goal here isn't necessarily the prettiness of the code, it's to see what
+kinds of things the resource has to do. I'll be using Ruby purely as a
+scripting language, hacking things together where needed in the interest
+of brevity.

001-resources-v2/git-example/artifact

@@ -0,0 +1,121 @@
+#!/usr/bin/env ruby
+
+require "json"
+require "rugged"
+require "pry"
+require "benchmark"
+
+$request = JSON.parse(STDIN.read, symbolize_names: true)
+
+def commit_version(c, s)
+  JSON.dump({
+    space: s,
+    version: {ref: c.oid},
+    metadata: [
+      {name: "author", value: enc(c, c.author[:name])},
+      {name: "author_date", value: c.author[:time].to_s},
+      {name: "committer", value: enc(c, c.committer[:name])},
+      {name: "committer_date", value: c.committer[:time].to_s},
+      {name: "message", value: enc(c, c.message)}
+    ]
+  })
+end
+
+def bench(label, &blk)
+  time = Benchmark.realtime(&blk)
+  $stderr.puts "#{label}: #{time}s"
+end
+
+def enc(c, str)
+  str = str.force_encoding("ISO-8859-1") unless c.header_field("Encoding")
+  str.encode("UTF-8")
+end
+
+case ARGV[0]
+when "check"
+  repo_dir = File.basename($request[:config][:uri])
+
+  repo =
+    if Dir.exists?(repo_dir)
+      Rugged::Repository.new(repo_dir).tap do |r|
+        r.fetch("origin")
+      end
+    else
+      Rugged::Repository.clone_at(
+        $request[:config][:uri],
+        repo_dir,
+        bare: true,
+        progress: lambda { |t| $stderr.print t })
+    end
+
+  spaces = []
+  default_branch = nil
+  file = File.new $request[:response_path], 'w'
+
+  repo.branches.each do |b|
+    unless b.remote?
+      # assume the only local branch is the default one
+      default_branch = b.name
+
+      file.puts JSON.dump({
+        default_space: default_branch
+      })
+      next
+    end
+
+    space_name = b.name.sub("#{b.remote_name}/", "")
+
+    commits = []
+
+    walker = Rugged::Walker.new(repo)
+    walker.sorting(Rugged::SORT_TOPO|Rugged::SORT_REVERSE)
+    walker.simplify_first_parent
+    walker.push(b.target)
+
+    from = $request[:from][space_name.to_sym]
+
+    if from && repo.include?(from[:ref])
+      commit = repo.lookup(from[:ref])
+      walker.hide(commit)
+
+      file.puts commit_version(commit, space_name)
+    end
+
+    bench("#{space_name} walk") do
+      walker.walk do |c|
+        # TODO: test if commit satisfies paths/ignore_paths
+        file.puts commit_version(c, space_name)
+      end
+    end
+
+    next if commits.empty?
+
+    $stderr.puts "#{space_name} commits: #{commits.size} (latest: #{has_latest})"
+  end
+
+  file.close
+
+when "get"
+  $request = {
+    config: {uri: "https://github.com/vito/booklit"},
+    space: "master",
+    version: {ref: "f828f2758256b0e93dc3c101f75604efe92ca07e"}
+  }
+
+  repo =
+    Rugged::Repository.clone_at(
+      $request[:config][:uri],
+      "dot", # TODO: irl this would be '.'
+      checkout_branch: $request[:space])
+
+  repo.checkout($request[:version][:ref])
+
+  # TODO: update/init submodules recursively
+
+  # TODO: draw the rest of the owl
+  #
+  # most of this is uninteresting.
+
+when "put"
+  puts "putting"
+end

001-resources-v2/git-example/collect-all-versions

@@ -0,0 +1,83 @@
+#!/usr/bin/env ruby
+
+require "json"
+require "subprocess"
+require "stringio"
+
+# concourse (~15 branches, ~8500 commits)
+$request = {
+  config: {uri: "https://github.com/concourse/concourse"},
+  from: {}
+}
+
+# booklit (~10 branches, ~200 commits)
+$request = {
+  config: {uri: "https://github.com/vito/booklit"},
+  from: {}
+}
+
+# rails (~36 branches, ~70k commits)
+$request = {
+  config: {uri: "https://github.com/rails/rails"},
+  from: {}
+}
+
+# linux (~2 branches, ~766k commits)
+$request = {
+  config: {uri: "https://github.com/torvalds/linux"},
+  from: {}
+}
+
+def check_all
+  all_start = Time.now
+  check_has_latest = false
+  done_last_check = false
+
+  total_versions = {}
+
+  while true
+    check_start = Time.now
+
+    Subprocess::Process.new(
+      ["bundle", "exec", "./artifact", "check"],
+      stdin: Subprocess::PIPE,
+      stdout: Subprocess::PIPE) do |check|
+      out, _ = check.communicate(JSON.dump($request))
+
+      done_last_check = true if check_has_latest
+
+      res = JSON.parse(out, symbolize_names: true)
+
+      res[:spaces].each do |s|
+        space = s[:space]
+        versions = s[:versions]
+
+        if versions.first[:version] == versions.last[:version]
+          puts "checked #{space}: #{versions.first[:version][:ref]}"
+        else
+          puts "checked #{space}: #{versions.first[:version][:ref]}..#{versions.last[:version][:ref]}"
+        end
+
+        total_versions[space] ||= 0
+        total_versions[space] += versions.size
+        if versions.first[:version] == $request[:from][space]
+          total_versions[space] -= 1
+        end
+
+        $request[:from][space] = versions.last[:version]
+      end
+
+      check_has_latest = res[:spaces].all? { |s| s[:has_latest] }
+    end
+
+    if done_last_check
+      puts "time for stable check: #{Time.now - check_start}s"
+      break
+    end
+  end
+
+  puts "total time for all checks: #{Time.now - all_start}s"
+  puts "total versions: #{total_versions.to_json}"
+end
+
+check_all

001-resources-v2/git-example/info

@@ -0,0 +1,12 @@
+#!/usr/bin/env ruby
+
+require "json"
+
+puts JSON.dump({
+  artifacts: {
+    api_version: "2.0",
+    check: "artifact check",
+    get: "artifact get",
+    put: "artifact put"
+  }
+})