retry download without s3 cache flag if cache miss (#1104)

* retry download without s3 cache flag if cache miss * changing abort on exception when fetching from internal cache --------- Signed-off-by: Justin Gruber <[email protected]>
chef · Feb 15, 2023 · aeec013 · aeec013
1 parent 72753d7
commit aeec013
Show file tree

Hide file tree

Showing 3 changed files with 15 additions and 4 deletions.
diff --git a/lib/omnibus/cli.rb b/lib/omnibus/cli.rb
@@ -85,7 +85,13 @@ def build(name)
       project = Project.load(name, manifest)
       say("Building #{project.name} #{project.build_version}...")
       Omnibus::S3Cache.populate if @options[:populate_s3_cache] && !Omnibus::S3Cache.fetch_missing.empty?
-      project.download
+      begin
+        project.download
+      rescue
+        Config.use_s3_caching(false) if Config.use_s3_caching
+        project = Project.load(name, nil)
+        project.download
+      end
       project.build
 
       if @options[:output_manifest]

diff --git a/lib/omnibus/project.rb b/lib/omnibus/project.rb
@@ -1062,7 +1062,10 @@ def built_manifest
     end
 
     def download
-      ThreadPool.new(Config.workers) do |pool|
+      # Setting abort_on_exception to false because it was causing
+      # errors by shutting down the main thread when encountering a cache miss
+      # in S3 and falling back to the internal source repo
+      ThreadPool.new(Config.workers, false) do |pool|
         softwares.each do |software|
           pool.schedule { software.fetch }
         end

diff --git a/lib/omnibus/thread_pool.rb b/lib/omnibus/thread_pool.rb
@@ -46,14 +46,16 @@ class ThreadPool
     #
     # @param [Integer] size
     #   the number of items to put in the thread pool
+    # @param [Boolean] abort_on_exception
+    #   if the thread should abort the main thread also on a failure
     #
-    def initialize(size)
+    def initialize(size, abort_on_exception = true)
       @size = size
       @jobs = Queue.new
 
       @pool = Array.new(@size) do |i|
         Thread.new do
-          Thread.abort_on_exception = true
+          Thread.abort_on_exception = abort_on_exception
           Thread.current[:id] = i
 
           catch(:exit) do