Locally caches tagged packages (#1292)

* locally caches tagged packages

* Prevents `nimble` to fail when reading a package

src/nimblepkg/nimblesat.nim

@@ -2,7 +2,7 @@ import sat/[sat, satvars]
 import version, packageinfotypes, download, packageinfo, packageparser, options, 
   sha1hashes, tools, downloadnim, cli
 
-import std/[tables, sequtils, algorithm, sets, strutils, options, strformat, os]
+import std/[tables, sequtils, algorithm, sets, strutils, options, strformat, os, json, jsonutils]
 
 
 type  
@@ -59,6 +59,23 @@ type
 
   GetPackageMinimal* = proc (pv: PkgTuple, options: Options): seq[PackageMinimalInfo]
 
+  TaggedPackageVersions = object
+    maxTaggedVersions: int # Maximum number of tags. When number changes, we invalidate the cache
+    versions: seq[PackageMinimalInfo]
+
+const TaggedVersionsFileName* = "tagged_versions.json"
+
+proc initFromJson*(dst: var PkgTuple, jsonNode: JsonNode, jsonPath: var string) =
+  dst = parseRequires(jsonNode.str)
+
+proc toJsonHook*(src: PkgTuple): JsonNode =
+  let ver = if src.ver.kind == verAny: "" else: $src.ver
+  case src.ver.kind
+  of verAny: newJString(src.name)
+  of verSpecial: newJString(src.name & ver)
+  else:
+    newJString(src.name & " " & ver)
+
 #From the STD as it is not available in older Nim versions
 func addUnique*[T](s: var seq[T], x: sink T) =
   ## Adds `x` to the container `s` if it is not already present. 
@@ -444,12 +461,39 @@ proc getAllNimReleases(options: Options): seq[PackageMinimalInfo] =
   for release in releases:
     result.add PackageMinimalInfo(name: "nim", version: release)
 
+proc getTaggedVersions*(repoDir: string, options: Options): Option[TaggedPackageVersions] =
+  let file = repoDir / TaggedVersionsFileName
+  if file.fileExists:
+    try:
+      let taggedVersions = file.readFile.parseJson().to(TaggedPackageVersions)
+      if taggedVersions.maxTaggedVersions != options.maxTaggedVersions:
+        return none(TaggedPackageVersions)
+      return some taggedVersions
+    except CatchableError as e:
+      displayWarning(&"Error reading tagged versions: {e.msg}", HighPriority)
+      return none(TaggedPackageVersions)
+  else:
+    none(TaggedPackageVersions)
+
+proc saveTaggedVersions*(repoDir: string, taggedVersions: TaggedPackageVersions) =
+  try:
+    let file = repoDir / TaggedVersionsFileName
+    file.writeFile((taggedVersions.toJson()).pretty)
+  except CatchableError as e:
+    displayWarning(&"Error saving tagged versions: {e.msg}", HighPriority)
+
 proc getPackageMinimalVersionsFromRepo*(repoDir, pkgName: string, downloadMethod: DownloadMethod, options: Options): seq[PackageMinimalInfo] =
   #This is expensive. We need to cache it. Potentially it could be also run in parallel
   # echo &"Discovering version for {pkgName}"
+  let taggedVersions = getTaggedVersions(repoDir, options)
+  if taggedVersions.isSome:
+    return taggedVersions.get.versions
   gitFetchTags(repoDir, downloadMethod)       
   #First package must be the current one
-  result.add getPkgInfo(repoDir, options).getMinimalInfo(options)
+  try:
+    result.add getPkgInfo(repoDir, options).getMinimalInfo(options)
+  except CatchableError as e:
+    displayWarning(&"Error getting package info for {pkgName}: {e.msg}", HighPriority)
   let tags = getTagsList(repoDir, downloadMethod).getVersionList()
   var checkedTags = 0
   for (ver, tag) in tags.pairs:    
@@ -467,6 +511,7 @@ proc getPackageMinimalVersionsFromRepo*(repoDir, pkgName: string, downloadMethod
     except CatchableError as e:
       displayWarning(&"Error reading tag {tag}: for package {pkgName}. This may not be relevant as it could be an old version of the package. \n {e.msg}", HighPriority)
 
+  saveTaggedVersions(repoDir, TaggedPackageVersions(maxTaggedVersions: options.maxTaggedVersions, versions: result))
 proc downloadMinimalPackage*(pv: PkgTuple, options: Options): seq[PackageMinimalInfo] =
   if pv.name == "": return newSeq[PackageMinimalInfo]()
   if pv.isNim and not options.disableNimBinaries: return getAllNimReleases(options)

tests/tsat.nim

@@ -6,16 +6,6 @@ import std/[tables, sequtils, json, jsonutils, strutils, times, options, strform
 import nimblepkg/[version, nimblesat, options, config, download, packageinfotypes, packageinfo]
 from nimblepkg/common import cd
 
-proc initFromJson*(dst: var PkgTuple, jsonNode: JsonNode, jsonPath: var string) =
-  dst = parseRequires(jsonNode.str)
-
-proc toJsonHook*(src: PkgTuple): JsonNode =
-  let ver = if src.ver.kind == verAny: "" else: $src.ver
-  case src.ver.kind
-  of verAny: newJString(src.name)
-  of verSpecial: newJString(src.name & ver)
-  else:
-    newJString(src.name & " " & ver)
 
 #Test utils:
 proc downloadAndStorePackageVersionTableFor(pkgName: string, options: Options) =
@@ -305,13 +295,42 @@ suite "SAT solver":
     let pv = parseRequires("nimfp >= 0.3.4")
     let repoDir = pv.downloadPkgFromUrl(options)[0].dir #This is just to setup the test. We need a git dir to work on
     let downloadMethod = DownloadMethod git
-
     let packageVersions = getPackageMinimalVersionsFromRepo(repoDir, pv[0], downloadMethod, options)
 
     #we know these versions are available
     let availableVersions = @["0.3.4", "0.3.5", "0.3.6", "0.4.5", "0.4.4"].mapIt(newVersion(it))
     for version in availableVersions:
       check version in packageVersions.mapIt(it.version)
+    check fileExists(repoDir / TaggedVersionsFileName)
+
+  test "should not use the cache when switching versions":
+    var options = initOptions()
+    options.maxTaggedVersions = 0 #all
+    options.nimBin = some options.makeNimBin("nim")
+    options.config.packageLists["official"] = PackageList(name: "Official", urls: @[
+    "https://raw.githubusercontent.com/nim-lang/packages/master/packages.json",
+    "https://nim-lang.org/nimble/packages.json"
+    ])
+    for dir in walkDir(".", true):
+      if dir.kind == PathComponent.pcDir and dir.path.startsWith("githubcom_vegansknimfp"):
+        echo "Removing dir", dir.path
+        removeDir(dir.path)
+
+    let pvPrev = parseRequires("nimfp >= 0.3.4")
+    let repoDirPrev = pvPrev.downloadPkgFromUrl(options)[0].dir 
+    discard getPackageMinimalVersionsFromRepo(repoDirPrev, pvPrev[0], DownloadMethod.git, options)
+    check fileExists(repoDirPrev / TaggedVersionsFileName)
+
+    let pv = parseRequires("nimfp >= 0.4.4")
+    let repoDir = pv.downloadPkgFromUrl(options)[0].dir 
+    check not fileExists(repoDir / TaggedVersionsFileName)
+
+    let packageVersions = getPackageMinimalVersionsFromRepo(repoDir, pv[0], DownloadMethod.git, options)
+    #we know these versions are available
+    let availableVersions = @["0.4.5", "0.4.4"].mapIt(newVersion(it))
+    for version in availableVersions:
+      check version in packageVersions.mapIt(it.version)
+    check fileExists(repoDir / TaggedVersionsFileName)
 
   test "if a dependency is unsatisfable, it should fallback to the previous version of the depency when available":
     let pkgVersionTable = {