Skip to content

Commit

Permalink
Include local existing .nupkg files in caching their referenced binar…
Browse files Browse the repository at this point in the history
…ies.
  • Loading branch information
robertZaufall committed Jul 6, 2023
1 parent d626041 commit c666588
Show file tree
Hide file tree
Showing 9 changed files with 102 additions and 69 deletions.
8 changes: 8 additions & 0 deletions ChocolateStoreCore/App.cs
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,14 @@ public async Task<int> Run(bool whatif = false)
_logger.LogInformation("found {downloads_Count} packages.", downloads.Count);

var neededPackages = _packageCacher.GetLastVersions(downloads, flattenDependencies: true);

var existingIds = neededPackages.Select(x => x.Id).ToList();
var nupkgs = _chocolateyHelper.GetPackagesInventory(sourcePath);

neededPackages.AddRange(
nupkgs.Where(x => !existingIds.Contains(x.Id))
.Select(x => new ChocolateyPackage { Id = x.Id, Version = x.Version }).ToList());

neededPackages.ForEach(x =>
{
_ = _packageCacher.CachePackage(x, sourcePath, targetPath, whatif);
Expand Down
6 changes: 3 additions & 3 deletions ChocolateStoreCore/ChocolateStoreCore.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,9 @@
<PropertyGroup>
<Product>ChocolateStoreCore</Product>
<Description>Download, modify and cache chocolatey packages locally</Description>
<Version>1.0.5</Version>
<FileVersion>1.0.5.30</FileVersion>
<AssemblyVersion>1.0.5.30</AssemblyVersion>
<Version>1.0.6</Version>
<FileVersion>1.0.6.1</FileVersion>
<AssemblyVersion>1.0.6.1</AssemblyVersion>
<Authors>Robert Zaufall</Authors>
<Company>Robert Zaufall IT Consulting</Company>
<Copyright>(c)2023 by Robert Zaufall</Copyright>
Expand Down
97 changes: 53 additions & 44 deletions ChocolateStoreCore/Helpers/ChocolateyHelper.cs
Original file line number Diff line number Diff line change
Expand Up @@ -29,14 +29,12 @@ public interface IChocolateyHelper

public class ChocolateyHelper : IChocolateyHelper
{
private readonly ISettings _settings;
private readonly ILogger<ChocolateyHelper> _logger;
private readonly IFileHelper _fileHelper;
private readonly IHttpHelper _httpHelper;

public ChocolateyHelper(ISettings config, IFileHelper fileHelper, IHttpHelper httpHelper, ILogger<ChocolateyHelper> logger)
public ChocolateyHelper(IFileHelper fileHelper, IHttpHelper httpHelper, ILogger<ChocolateyHelper> logger)
{
_settings = config ?? throw new ArgumentNullException(nameof(config));
_logger = logger ?? new Logger<ChocolateyHelper>(new NullLoggerFactory());
_fileHelper = fileHelper;
_httpHelper = httpHelper;
Expand All @@ -49,23 +47,26 @@ public ChocolateyHelper(ISettings config, IFileHelper fileHelper, IHttpHelper ht

var fileType = StringHelper.GetFileTypen(content);

var originalUrls = StringHelper.GetOriginalUrls(content, id, version);
var originalUrls = StringHelper.GetOriginalUrls(content, id, version, notToReplaceUrl: repo);
var transformedContent = Regex.Replace(content, StringHelper.RxUrlPattern.ToString(), new MatchEvaluator(m =>
{
var url = StringHelper.ReplaceTokens(m.Value, id, version);
var actualUrl = _httpHelper.CheckUrl(url);
var uri = new Uri(actualUrl ?? url);
var fileName = GetFileNameFromUrls(url, actualUrl);
var originalFileExtension = Path.GetExtension(fileName);

if (string.IsNullOrWhiteSpace(originalFileExtension) && !string.IsNullOrWhiteSpace(fileType))
fileName = fileName + "." + fileType;
if (string.IsNullOrEmpty(repo) || !m.Value.StartsWith(repo))
{
var url = StringHelper.ReplaceTokens(m.Value, id, version);
var actualUrl = _httpHelper.CheckUrl(url);
var uri = new Uri(actualUrl ?? url);
var fileName = GetFileNameFromUrls(url, actualUrl);
var originalFileExtension = Path.GetExtension(fileName);

var transformedUrl = repo + @"/" + folderName + @"/" + HttpUtility.UrlPathEncode(fileName);
if (string.IsNullOrWhiteSpace(originalFileExtension) && !string.IsNullOrWhiteSpace(fileType))
fileName = fileName + "." + fileType;

downloads.Add(new Download { Url = url, Path = Path.Combine(folder, fileName) });
var transformedUrl = repo + @"/" + folderName + @"/" + HttpUtility.UrlPathEncode(fileName);

return transformedUrl;
downloads.Add(new Download { Url = url, Path = Path.Combine(folder, fileName) });
return transformedUrl;
}
return m.Value;
}), RegexOptions.IgnoreCase);
return (transformedContent, downloads, fileType);
}
Expand Down Expand Up @@ -275,41 +276,49 @@ public ChocolateyPackage ParseMetadata(string originalId, string content)
XNamespace ns = "http://www.w3.org/2005/Atom";

var id = xml.Descendants(ns + "entry").Select(x => (string)x.Element(ns + "title")).FirstOrDefault();
var version = xml.Descendants(m + "properties").Select(x => (string)x.Element(d + "Version")).FirstOrDefault();
var downloadUrl = xml.Descendants(ns + "entry").Select(x => x.Element(ns + "content")).Select(x => x?.Attribute("src")?.Value).FirstOrDefault();
var xDependencies = xml.Descendants(m + "properties").Select(x => (string)x.Element(d + "Dependencies")).FirstOrDefault();
List<Dependency> dependencies = xDependencies?
.Split("|")
.ToList()
.Where(x => !string.IsNullOrWhiteSpace(x))
.Select(y =>
{
var depId = y?.Split(":")[0];
var parsed = VersionRange.TryParse(y?.Split(":")[1], out var depVersionRange);
var depVersion = parsed ? (
depVersionRange.HasLowerBound ?
depVersionRange.MinVersion.OriginalVersion :
depVersionRange.MaxVersion.OriginalVersion) :
null;
return new Dependency

if (id != null)
{
var version = xml.Descendants(m + "properties").Select(x => (string)x.Element(d + "Version")).FirstOrDefault();
var downloadUrl = xml.Descendants(ns + "entry").Select(x => x.Element(ns + "content")).Select(x => x?.Attribute("src")?.Value).FirstOrDefault();
var xDependencies = xml.Descendants(m + "properties").Select(x => (string)x.Element(d + "Dependencies")).FirstOrDefault();
List<Dependency> dependencies = xDependencies?
.Split("|")
.ToList()
.Where(x => !string.IsNullOrWhiteSpace(x))
.Select(y =>
{
Id = depId,
OriginalVersion = depVersion,
Version = (depVersion == null ? null : new NuGetVersion(depVersion)),
};
}).ToList();
var depId = y?.Split(":")[0];
var parsed = VersionRange.TryParse(y?.Split(":")[1], out var depVersionRange);
var depVersion = parsed ? (
depVersionRange.HasLowerBound ?
depVersionRange.MinVersion.OriginalVersion :
depVersionRange.MaxVersion.OriginalVersion) :
null;
return new Dependency
{
Id = depId,
OriginalVersion = depVersion,
Version = (depVersion == null ? null : new NuGetVersion(depVersion)),
};
}).ToList();

return new ChocolateyPackage
return new ChocolateyPackage
{
Id = id,
Version = new NuGetVersion(version),
Dependencies = dependencies,
DownloadUrl = downloadUrl,
};
}
else
{
Id = id,
Version = new NuGetVersion(version),
Dependencies = dependencies,
DownloadUrl = downloadUrl,
};
return new ChocolateyPackage { Id = originalId };
}
}
catch (Exception ex)
{
_logger.LogError(ex, "Error parsing metadata for {originalId}", originalId);
_logger.LogError(ex, "Error parsing metadata for '{originalId}'", originalId);
return null;
}
}
Expand Down
4 changes: 2 additions & 2 deletions ChocolateStoreCore/Helpers/HttpHelper.cs
Original file line number Diff line number Diff line change
Expand Up @@ -51,15 +51,15 @@ public string GetMetadataForPackageId(string packageId)
if (response.StatusCode == HttpStatusCode.OK)
{
var content = new StreamReader(response.Content.ReadAsStreamAsync().Result).ReadToEnd();
_logger.LogInformation("Download metadata for {packageId} done.", packageId);
_logger.LogInformation("Download metadata for '{packageId}' done.", packageId);
return content;
}
}
return null;
}
catch (Exception ex)
{
_logger.LogError(ex, "Failed to get metadata for {packageId}", packageId);
_logger.LogError(ex, "Failed to get metadata for '{packageId}'", packageId);
return null;
}
}
Expand Down
7 changes: 5 additions & 2 deletions ChocolateStoreCore/Helpers/StringHelper.cs
Original file line number Diff line number Diff line change
Expand Up @@ -29,13 +29,16 @@ public static string ReplaceTokens(string input, string id, string version)
;
}

public static List<string> GetOriginalUrls(string content, string id, string version)
public static List<string> GetOriginalUrls(string content, string id, string version, string notToReplaceUrl)
{
var downloads = new List<string>();
var uris = Regex.Replace(content, StringHelper.RxUrlPattern.ToString(), new MatchEvaluator(m =>
{
var url = ReplaceTokens(m.Value, id, version);
downloads.Add(url);
if (string.IsNullOrEmpty(notToReplaceUrl) || !url.StartsWith(notToReplaceUrl))
{
downloads.Add(url);
}
return url;
}), RegexOptions.IgnoreCase);
return downloads;
Expand Down
10 changes: 7 additions & 3 deletions ChocolateStoreCore/PackageCacher.cs
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ public interface IPackageCacher
List<ChocolateyPackage> GetLastVersions(List<string> downloads, bool flattenDependencies);
List<ChocolateyPackage> GetMissingFromDownloadsAndDependencies(List<string> downloads, string path, bool localOnly);
bool Purge(string path = null, bool whatif = false);
List<string> GetInventoryList(string path);
}

public class PackageCacher : IPackageCacher
Expand All @@ -32,6 +33,12 @@ public PackageCacher(ISettings settings, IFileHelper fileHelper, IHttpHelper htt
_logger = logger ?? new Logger<PackageCacher>(new NullLoggerFactory());
}

public List<string> GetInventoryList(string path)
{
var inventory = _chocolateyHelper.GetPackagesInventory(path);
return inventory?.Select(x => x.Id).ToList();
}

public bool Purge(string path = null, bool whatif = false)
{
var packageFolders = _fileHelper.GetDirectoryNames(path);
Expand Down Expand Up @@ -137,9 +144,6 @@ public bool CachePackage(ChocolateyPackage package, string sourcePath, string ta
var sourcePackagePath = Path.Combine(sourcePath, package.FileName);
var targetPackagePath = Path.Combine(targetPath ?? sourcePath, package.FileName);

if (_fileHelper.FileExists(sourcePackagePath))
return true;

if (!whatif && !_fileHelper.FileExists(sourcePackagePath))
{
_httpHelper.DownloadFile(package.DownloadUrl, targetPackagePath);
Expand Down
35 changes: 22 additions & 13 deletions ChocolateStoreCoreTests/ChocolateyHelperTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -15,17 +15,17 @@ public ChocolateyHelperTests(TestFixture fixture)
}

[Theory]
[InlineData("https://desktop.docker.com/win/main/amd64/82475/Docker%20Desktop%20Installer.exe", "exe", "Docker Desktop Installer.exe", "bla", "123")]
[InlineData("https://desktop.docker.com/win/main/amd64/82475/Docker%20Desktop%20Installer", "EXE", "Docker Desktop Installer.exe", "bla", "123")]
[InlineData("https://desktop.docker.com/win/main/amd64/82475/Docker%20Desktop%20Installer", "", "Docker Desktop Installer", "bla", "123")]
[InlineData("https://desktop.docker.com/win/main/amd64/82475/Docker%20Desktop%20Installer", "", "Docker Desktop Installer", "", "")]
[InlineData("if (\"https://static.rust-lang.org/dist/2023-01-10/rust-mingw-1.66.1-i686-pc-windows-gnu.tar.gz\" -ne \"\") {", "gz", "rust-mingw-1.66.1-i686-pc-windows-gnu.tar.gz", "", "")]
public void ExtractAndRewriteUrls(string originalUrl, string originalFileType, string fileNameExpected, string id, string version)
[InlineData("https://desktop.docker.com/win/main/amd64/82475/Docker%20Desktop%20Installer.exe", "exe", "Docker Desktop Installer.exe", "bla", "123", @"http://xyz", 1)]
[InlineData("https://desktop.docker.com/win/main/amd64/82475/Docker%20Desktop%20Installer", "EXE", "Docker Desktop Installer.exe", "bla", "123", @"http://xyz", 1)]
[InlineData("https://desktop.docker.com/win/main/amd64/82475/Docker%20Desktop%20Installer", "", "Docker Desktop Installer", "bla", "123", @"http://xyz", 1)]
[InlineData("https://desktop.docker.com/win/main/amd64/82475/Docker%20Desktop%20Installer", "", "Docker Desktop Installer", "", "", @"http://xyz", 1)]
[InlineData("if (\"https://static.rust-lang.org/dist/2023-01-10/rust-mingw-1.66.1-i686-pc-windows-gnu.tar.gz\" -ne \"\") {", "gz", "rust-mingw-1.66.1-i686-pc-windows-gnu.tar.gz", "", "", @"http://xyz", 1)]
[InlineData("https://123.xyz/test.zip", "zip", "test.zip", "bla", "123", @"https://123.xyz", 0)]
public void ExtractAndRewriteUrls(string originalUrl, string originalFileType, string fileNameExpected, string id, string version, string repo, int cntDownloads)
{
// Arrange
var fixture = TestFixture.GetFixture();

string repo = @"http://xyz";
string folder = @"c:\folder";
string folderName = Path.GetFileName(folder);
var originalContent = new StringBuilder();
Expand All @@ -42,12 +42,21 @@ public void ExtractAndRewriteUrls(string originalUrl, string originalFileType, s

// Assert
content.Should().NotBeNull();
downloads.Should().HaveCount(1);
var download = downloads[0];
string fileName = Path.GetFileName(download.Path);
fileName.Should().Be(fileNameExpected);
content.ToLower().Should().Contain($"{repo.ToLower()}/{folderName.ToLower()}/{System.Web.HttpUtility.UrlPathEncode(fileName.ToLower())}");
download.Path.ToLower().Should().Be($"{folder.ToLower()}\\{fileName.ToLower()}");
content.ToLower().Should().Contain($"{repo.ToLower()}");
downloads.Should().HaveCount(cntDownloads);
if (cntDownloads > 0)
{
var download = downloads[0];
string fileName = Path.GetFileName(download.Path);
fileName.Should().Be(fileNameExpected);
content.ToLower().Should().Contain($"{repo.ToLower()}/{folderName.ToLower()}/{System.Web.HttpUtility.UrlPathEncode(fileName.ToLower())}");
download.Path.ToLower().Should().Be($"{folder.ToLower()}\\{fileName.ToLower()}");
}
else
{
originalUrl.ToLower().Should().Contain($"{repo.ToLower()}");
content.ToLower().Should().Contain($"{originalUrl.ToLower()}");
}
}

[Theory]
Expand Down
2 changes: 1 addition & 1 deletion ChocolateStoreCoreTests/StringHelperTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ public void GetOriginalUrlPatterns(string originalUrl, string urlExpected, strin
// Arrange

// Act
var downloads = StringHelper.GetOriginalUrls(originalUrl, id, version);
var downloads = StringHelper.GetOriginalUrls(originalUrl, id, version, "###");

// Assert
downloads.Should().NotBeNull();
Expand Down
2 changes: 1 addition & 1 deletion version.txt
Original file line number Diff line number Diff line change
@@ -1 +1 @@
1.0.5.30
1.0.6.1

0 comments on commit c666588

Please sign in to comment.