Skip to content

Commit

Permalink
(GH-1397) Reduce traffic - choco queries
Browse files Browse the repository at this point in the history 
Ensure all Chocolatey queries are sped up by using better selection on
the front part of the call to ensure what is returned is the result that
is being requested without a huge amount of traffic to get to that value.

Previously, a call to find a package or even a version for upgrade might
send queries to remote sources to provide all of the versions of a package
starting with the oldest version to the latest. This results in terrible
performance, especially as a source continues to have more and more versions
of a package, the performance gets worse. Instead, for install/upgrade/outdated
scenarios where choco is querying service-based url, it should make one call
ensuring it receives the version it is looking for.
  • Loading branch information
@ferventcoder
ferventcoder committed Oct 2, 2018
1 parent 419ed4a commit 7788964
Show file tree
Hide file tree
Showing 2 changed files with 142 additions and 110 deletions.
16 changes: 8 additions & 8 deletions src/chocolatey/infrastructure.app/nuget/NugetList.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,15 +47,15 @@ private static IQueryable<IPackage> execute_package_search(ChocolateyConfigurati
// 2. Does it support prerelease in a straight-forward way?
// Choco previously dealt with this by taking the path of least resistance and manually filtering out and sort unwanted packages
// This result in blocking operations that didn't let service based repositories, like OData, take care of heavy lifting on the server.
bool isRemote;
bool isServiceBased;
var aggregateRepo = packageRepository as AggregateRepository;
if (aggregateRepo != null)
{
isRemote = aggregateRepo.Repositories.All(repo => repo is IServiceBasedRepository);
isServiceBased = aggregateRepo.Repositories.All(repo => repo is IServiceBasedRepository);
}
else
{
isRemote = packageRepository is IServiceBasedRepository;
isServiceBased = packageRepository is IServiceBasedRepository;
}

IQueryable<IPackage> results = packageRepository.Search(searchTermLower, configuration.Prerelease);
Expand All @@ -72,21 +72,21 @@ private static IQueryable<IPackage> execute_package_search(ChocolateyConfigurati

if (configuration.ListCommand.ByIdOnly)
{
results = isRemote ?
results = isServiceBased ?
results.Where(p => p.Id.ToLower().Contains(searchTermLower))
: results.Where(p => p.Id.contains(searchTermLower, StringComparison.OrdinalIgnoreCase));
}

if (configuration.ListCommand.ByTagOnly)
{
results = isRemote
results = isServiceBased
? results.Where(p => p.Tags.Contains(searchTermLower))
: results.Where(p => p.Tags.contains(searchTermLower, StringComparison.InvariantCultureIgnoreCase));
}

if (configuration.ListCommand.IdStartsWith)
{
results = isRemote ?
results = isServiceBased ?
results.Where(p => p.Id.ToLower().StartsWith(searchTermLower))
: results.Where(p => p.Id.StartsWith(searchTermLower, StringComparison.OrdinalIgnoreCase));
}
Expand All @@ -108,7 +108,7 @@ private static IQueryable<IPackage> execute_package_search(ChocolateyConfigurati

if (configuration.AllVersions || !string.IsNullOrWhiteSpace(configuration.Version))
{
if (isRemote)
if (isServiceBased)
{
return results.OrderBy(p => p.Id).ThenByDescending(p => p.Version);
}
Expand All @@ -127,7 +127,7 @@ private static IQueryable<IPackage> execute_package_search(ChocolateyConfigurati
results = results.Where(p => p.IsLatestVersion);
}

if (!isRemote)
if (!isServiceBased)
{
results =
results
Expand Down
236 changes: 134 additions & 102 deletions src/chocolatey/infrastructure.app/services/NugetService.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -336,7 +336,7 @@ public virtual void push_run(ChocolateyConfiguration config)
NugetPush.push_package(config, _fileSystem.get_full_path(nupkgFilePath));


if (config.RegularOutput && (config.Sources.is_equal_to(ApplicationParameters.ChocolateyCommunityFeedPushSource) ||config.Sources.is_equal_to(ApplicationParameters.ChocolateyCommunityFeedPushSourceOld)))
if (config.RegularOutput && (config.Sources.is_equal_to(ApplicationParameters.ChocolateyCommunityFeedPushSource) || config.Sources.is_equal_to(ApplicationParameters.ChocolateyCommunityFeedPushSourceOld)))
{
this.Log().Warn(ChocolateyLoggers.Important, () => @"
Expand Down Expand Up @@ -434,6 +434,7 @@ public virtual ConcurrentDictionary<string, PackageResult> install_run(Chocolate
uninstallSuccessAction: null,
addUninstallHandler: true);

bool repositoryIsServiceBased = repository_is_service_based(packageManager.SourceRepository);

foreach (string packageName in packageNames.or_empty_list_if_null())
{
Expand Down Expand Up @@ -466,17 +467,17 @@ public virtual ConcurrentDictionary<string, PackageResult> install_run(Chocolate
continue;
}

IPackage availablePackage = packageManager.SourceRepository.FindPackage(packageName, version, config.Prerelease, allowUnlisted: false);
IPackage availablePackage = find_package(packageName, version, config, packageManager.SourceRepository, repositoryIsServiceBased);
if (availablePackage == null)
{
var logMessage = @"{0} not installed. The package was not found with the source(s) listed.
Source(s): '{1}'
NOTE: When you specify explicit sources, it overrides default sources.
If the package version is a prerelease and you didn't specify `--pre`,
the package may not be found.{2}{3}".format_with(packageName, config.Sources, string.IsNullOrWhiteSpace(config.Version) ? String.Empty :
the package may not be found.{2}{3}".format_with(packageName, config.Sources, string.IsNullOrWhiteSpace(config.Version) ? String.Empty :
@"
Version was specified as '{0}'. It is possible that version
does not exist for '{1}' at the source specified.".format_with(config.Version.to_string(), packageName),
does not exist for '{1}' at the source specified.".format_with(config.Version.to_string(), packageName),
@"
Please see https://chocolatey.org/docs/troubleshooting for more
assistance.");
Expand Down Expand Up @@ -604,6 +605,8 @@ public virtual ConcurrentDictionary<string, PackageResult> upgrade_run(Chocolate
uninstallSuccessAction: null,
addUninstallHandler: false);

bool repositoryIsServiceBased = repository_is_service_based(packageManager.SourceRepository);

var configIgnoreDependencies = config.IgnoreDependencies;
set_package_names_if_all_is_specified(config, () => { config.IgnoreDependencies = true; });
config.IgnoreDependencies = configIgnoreDependencies;
Expand Down Expand Up @@ -677,7 +680,7 @@ public virtual ConcurrentDictionary<string, PackageResult> upgrade_run(Chocolate
config.Prerelease = true;
}

IPackage availablePackage = packageManager.SourceRepository.FindPackage(packageName, version, config.Prerelease, allowUnlisted: false);
IPackage availablePackage = find_package(packageName, version, config, packageManager.SourceRepository, repositoryIsServiceBased);
config.Prerelease = originalPrerelease;

if (availablePackage == null)
Expand Down Expand Up @@ -857,103 +860,132 @@ public virtual ConcurrentDictionary<string, PackageResult> upgrade_run(Chocolate
return packageInstalls;
}

public ConcurrentDictionary<string, PackageResult> get_outdated(ChocolateyConfiguration config)
{
var packageManager = NugetCommon.GetPackageManager(
config,
_nugetLogger,
_packageDownloader,
installSuccessAction: null,
uninstallSuccessAction: null,
addUninstallHandler: false);

var outdatedPackages = new ConcurrentDictionary<string, PackageResult>();

set_package_names_if_all_is_specified(config, () => { config.IgnoreDependencies = true; });
var packageNames = config.PackageNames.Split(new[] { ApplicationParameters.PackageNamesSeparator }, StringSplitOptions.RemoveEmptyEntries).or_empty_list_if_null().ToList();

var repository = packageManager.SourceRepository;

bool isRemote = repository_is_remote(repository);

foreach (var packageName in packageNames)
{
IPackage latestPackage;

if (isRemote)
{
latestPackage = repository.GetPackages().Where(x => x.Id.ToLower() == packageName && x.IsLatestVersion).ToList().SingleOrDefault();
}
else
{
latestPackage = repository.FindPackage(packageName);
}

if (latestPackage == null)
{
this.Log().Warn(packageName + " doesn't exist in the source.");
continue;
}

var installedPackage = packageManager.LocalRepository.FindPackage(packageName);

if (latestPackage.Version > installedPackage.Version)
{
var packageResult = outdatedPackages.GetOrAdd(packageName, new PackageResult(latestPackage, _fileSystem.combine_paths(ApplicationParameters.PackagesLocation, latestPackage.Id)));

var pkgInfo = _packageInfoService.get_package_information(installedPackage);
bool isPinned = pkgInfo.IsPinned;

if (config.OutdatedCommand.IgnorePinned && isPinned)
{
string logMessage = "{0} is pinned. Skipping pinned package.".format_with(packageName);
packageResult.Messages.Add(new ResultMessage(ResultType.Debug, logMessage));
packageResult.Messages.Add(new ResultMessage(ResultType.Inconclusive, logMessage));
if (config.RegularOutput)
this.Log().Warn(ChocolateyLoggers.Important, logMessage);
}
else
{
string logMessage = "You have {0} v{1} installed. Version {2} is available based on your source(s).".format_with(installedPackage.Id, installedPackage.Version, latestPackage.Version);
packageResult.Messages.Add(new ResultMessage(ResultType.Note, logMessage));

if (config.RegularOutput)
{
this.Log().Warn("{0}{1}".format_with(Environment.NewLine, logMessage));
}
else
{
this.Log().Info("{0}|{1}|{2}|{3}".format_with(installedPackage.Id, installedPackage.Version, latestPackage.Version, isPinned.to_string().to_lower()));
}
}
}
}

return outdatedPackages;
}

private bool repository_is_remote(IPackageRepository repository)
{
bool isRemote;
var aggregateRepo = repository as AggregateRepository;
if (aggregateRepo != null)
{
isRemote = aggregateRepo.Repositories.All(repo => repo is IServiceBasedRepository);
}
else
{
isRemote = repository is IServiceBasedRepository;
}

return isRemote;
}

/// <summary>
/// Sets the configuration for the package upgrade
/// </summary>
/// <param name="config">The configuration.</param>
/// <param name="packageInfo">The package information.</param>
/// <returns>The original unmodified configuration, so it can be reset after upgrade</returns>
public ConcurrentDictionary<string, PackageResult> get_outdated(ChocolateyConfiguration config)
{
var packageManager = NugetCommon.GetPackageManager(
config,
_nugetLogger,
_packageDownloader,
installSuccessAction: null,
uninstallSuccessAction: null,
addUninstallHandler: false);

var repository = packageManager.SourceRepository;
bool repositoryIsServiceBased = repository_is_service_based(repository);

var outdatedPackages = new ConcurrentDictionary<string, PackageResult>();

set_package_names_if_all_is_specified(config, () => { config.IgnoreDependencies = true; });
var packageNames = config.PackageNames.Split(new[] { ApplicationParameters.PackageNamesSeparator }, StringSplitOptions.RemoveEmptyEntries).or_empty_list_if_null().ToList();

var originalConfig = config.deep_copy();

foreach (var packageName in packageNames)
{
// set original config back each time through
config = originalConfig;

var installedPackage = packageManager.LocalRepository.FindPackage(packageName);
var pkgInfo = _packageInfoService.get_package_information(installedPackage);
bool isPinned = pkgInfo.IsPinned;

// if the package is pinned and we are skipping pinned,
// move on quickly
if (isPinned && config.OutdatedCommand.IgnorePinned)
{
string pinnedLogMessage = "{0} is pinned. Skipping pinned package.".format_with(packageName);
var pinnedPackageResult = outdatedPackages.GetOrAdd(packageName, new PackageResult(installedPackage, _fileSystem.combine_paths(ApplicationParameters.PackagesLocation, installedPackage.Id)));
pinnedPackageResult.Messages.Add(new ResultMessage(ResultType.Debug, pinnedLogMessage));
pinnedPackageResult.Messages.Add(new ResultMessage(ResultType.Inconclusive, pinnedLogMessage));

continue;
}

if (installedPackage != null && !string.IsNullOrWhiteSpace(installedPackage.Version.SpecialVersion) && !config.UpgradeCommand.ExcludePrerelease)
{
// this is a prerelease - opt in for newer prereleases.
config.Prerelease = true;
}

var latestPackage = find_package(packageName, null, config, repository, repositoryIsServiceBased);

if (latestPackage == null)
{
if (config.Features.IgnoreUnfoundPackagesOnUpgradeOutdated) continue;

string unfoundLogMessage = "{0} was not found with the source(s) listed.{1} Source(s): \"{2}\"".format_with(packageName, Environment.NewLine, config.Sources);
var unfoundResult = outdatedPackages.GetOrAdd(packageName, new PackageResult(installedPackage, _fileSystem.combine_paths(ApplicationParameters.PackagesLocation, installedPackage.Id)));
unfoundResult.Messages.Add(new ResultMessage(ResultType.Warn, unfoundLogMessage));
unfoundResult.Messages.Add(new ResultMessage(ResultType.Inconclusive, unfoundLogMessage));

this.Log().Warn("{0}|{1}|{1}|{2}".format_with(installedPackage.Id, installedPackage.Version, isPinned.to_string().to_lower()));
continue;
}

if (latestPackage.Version <= installedPackage.Version) continue;

var packageResult = outdatedPackages.GetOrAdd(packageName, new PackageResult(latestPackage, _fileSystem.combine_paths(ApplicationParameters.PackagesLocation, latestPackage.Id)));

string logMessage = "You have {0} v{1} installed. Version {2} is available based on your source(s).{3} Source(s): \"{4}\"".format_with(installedPackage.Id, installedPackage.Version, latestPackage.Version, Environment.NewLine, config.Sources);
packageResult.Messages.Add(new ResultMessage(ResultType.Note, logMessage));

this.Log().Info("{0}|{1}|{2}|{3}".format_with(installedPackage.Id, installedPackage.Version, latestPackage.Version, isPinned.to_string().to_lower()));
}

return outdatedPackages;
}

private IPackage find_package(string packageName, SemanticVersion version, ChocolateyConfiguration config, IPackageRepository repository, bool isRepositoryServiceBased)
{
// find the package based on version
if (version != null) return repository.FindPackage(packageName, version, config.Prerelease, allowUnlisted: false);

IQueryable<IPackage> results = repository.GetPackages().Where(x => x.Id == packageName);

if (config.Prerelease && repository.SupportsPrereleasePackages)
{
results = results.Where(p => p.IsAbsoluteLatestVersion);
}
else
{
results = results.Where(p => p.IsLatestVersion);
}

if (!isRepositoryServiceBased)
{
results = results
.Where(PackageExtensions.IsListed)
.Where(p => config.Prerelease || p.IsReleaseVersion())
.distinct_last(PackageEqualityComparer.Id, PackageComparer.Version)
.AsQueryable();
}

// get only one result, should be the latest
return results.ToList().OrderByDescending(x => x.Version).FirstOrDefault();
}

private bool repository_is_service_based(IPackageRepository repository)
{
bool isRemote;
var aggregateRepo = repository as AggregateRepository;
if (aggregateRepo != null)
{
isRemote = aggregateRepo.Repositories.All(repo => repo is IServiceBasedRepository);
}
else
{
isRemote = repository is IServiceBasedRepository;
}

return isRemote;
}

/// <summary>
/// Sets the configuration for the package upgrade
/// </summary>
/// <param name="config">The configuration.</param>
/// <param name="packageInfo">The package information.</param>
/// <returns>The original unmodified configuration, so it can be reset after upgrade</returns>
protected virtual ChocolateyConfiguration set_package_config_for_upgrade(ChocolateyConfiguration config, ChocolateyPackageInformation packageInfo)
{
if (!config.Features.UseRememberedArgumentsForUpgrades || string.IsNullOrWhiteSpace(packageInfo.Arguments)) return config;
Expand Down

0 comments on commit 7788964

Please sign in to comment.