Introduce index_files function not restricted to JLSO (#29)

* Change `index_checkpoint_files` to not filter to JLSO

* Introduce `index_files` function not restricted to JLSO

* Update test/indexing.jl

* Update test/indexing.jl

Project.toml

@@ -1,7 +1,7 @@
 name = "Checkpoints"
 uuid = "b4a3413d-e481-5afc-88ff-bdfbd6a50dce"
 authors = "Invenia Technical Computing Corporation"
-version = "0.3.9"
+version = "0.3.10"
 
 [deps]
 AWSS3 = "1c724243-ef5b-51ab-93f4-b0a88ac62a95"

src/Checkpoints.jl

@@ -20,7 +20,7 @@ using OrderedCollections
 export checkpoint, with_checkpoint_tags  # creating stuff
 export enabled_checkpoints
 # indexing stuff
-export IndexEntry, index_checkpoint_files
+export IndexEntry, index_checkpoint_files, index_files
 export checkpoint_name, checkpoint_path, prefixes, tags
 
 const LOGGER = getlogger(@__MODULE__)

src/indexing.jl

@@ -145,3 +145,17 @@ function index_checkpoint_files(dir::AbstractPath)
 end
 
 index_checkpoint_files(dir) = index_checkpoint_files(Path(dir))
+
+"""
+    index_files(dir)
+
+Constructs a index for all the files located within  `dir`.
+Same as [`index_checkpoint_files`] except not restricted to files created by Checkpoints.jl.
+"""
+function index_files(dir::AbstractPath)
+    map(Iterators.filter(isfile, walkpath(dir))) do path
+        return IndexEntry(path)
+    end
+end
+
+index_files(dir) = index_files(Path(dir))

test/indexing.jl

@@ -19,6 +19,19 @@
         end
     end
 
+    @testset "files not saved by Checkpoints.jl" begin
+        mktempdir(SystemPath) do path
+            Checkpoints.config("TestPkg.bar", path)
+            TestPkg.bar([1,2,3])
+            other_file = joinpath(path, "date=2021-01-01", "other_file.txt")
+            mkpath(dirname(other_file))
+            write(other_file, 1)
+            index = index_files(path)
+            @test length(index) == 2
+            @test other_file == only(checkpoint_path(entry) for entry in index if entry.date == "2021-01-01")
+        end
+    end
+
     @testset "clashing tags" begin
         mktempdir() do path
             Checkpoints.config("TestPkg.bar", path)