Fix tc main results path

integration_tests/utils/compute_mse.jl

@@ -80,6 +80,17 @@ function append_dict(filename::AbstractString, sym::Symbol, dict = Dict())
     return dict
 end
 
+function find_latest_dataset_folder(; dir = pwd())
+    matching_paths = String[]
+    for file in readdir(dir)
+        !ispath(joinpath(dir, file)) && continue
+        push!(matching_paths, joinpath(dir, file))
+    end
+    isempty(matching_paths) && return ""
+    # sort by timestamp
+    return pop!(sort(matching_paths))
+end
+
 function compute_mse_wrapper(
     case_name,
     best_mse,
@@ -93,10 +104,24 @@ function compute_mse_wrapper(
     # Note: cluster_data_prefix is also defined in utils/move_output.jl
     if haskey(ENV, "BUILDKITE_COMMIT")
         cluster_data_prefix = "/central/scratch/climaci/turbulenceconvection-main"
-        commit_sha = ENV["BUILDKITE_COMMIT"][1:7]
-        path = joinpath(cluster_data_prefix, commit_sha)
-        folder_name = "Output.$case_name.01" # TODO: make this more robust in case 01 changes
+        path = find_latest_dataset_folder(; dir = cluster_data_prefix)
+        folder_name = joinpath("Output.$case_name.01", "stats") # TODO: make this more robust in case 01 changes
+        @show folder_name
+        @show ispath(folder_name)
+        @show ispath(joinpath(path, folder_name))
+        all_files = [
+            joinpath(root, f)
+            for (root, dirs, files) in Base.Filesystem.walkdir(path)
+            for f in files
+        ]
+        for f in all_files
+            @show f
+        end
+        for p in readdir(joinpath(path))
+            @show p
+        end
         ds_tc_main_filename = joinpath(path, folder_name, "$case_name.nc")
+        @show isfile(ds_tc_main_filename)
     end
     # Note that we're using a closure over
     #  - PyCLES_output_dataset_path