diff --git a/src/code/ContainerRegistryServerAPICalls.cs b/src/code/ContainerRegistryServerAPICalls.cs index 9c17c0db0..20f084f10 100644 --- a/src/code/ContainerRegistryServerAPICalls.cs +++ b/src/code/ContainerRegistryServerAPICalls.cs @@ -81,6 +81,16 @@ public ContainerRegistryServerAPICalls(PSRepositoryInfo repository, PSCmdlet cmd #region Overridden Methods + public override Task FindVersionAsync(string packageName, string version, ResourceType type) + { + return null; + } + + public override Task FindVersionGlobbingAsync(string packageName, VersionRange versionRange, bool includePrerelease, ResourceType type, bool getOnlyLatest) + { + return null; + } + /// /// Find method which allows for searching for all packages from a repository and returns latest version for each. /// @@ -146,6 +156,12 @@ public override FindResults FindName(string packageName, bool includePrerelease, return new FindResults(stringResponse: new string[] { }, hashtableResponse: pkgResult.ToArray(), responseType: containerRegistryFindResponseType); } + + public override Task FindNameAsync(string packageName, bool includePrerelease, ResourceType type) + { + return null; + } + /// /// Find method which allows for searching for single name and tag and returns latest version. /// Name: no wildcard support diff --git a/src/code/FindHelper.cs b/src/code/FindHelper.cs index 388be9090..d2412c265 100644 --- a/src/code/FindHelper.cs +++ b/src/code/FindHelper.cs @@ -1,16 +1,22 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -using Microsoft.PowerShell.PSResourceGet.UtilClasses; -using NuGet.Versioning; using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; using System.Management.Automation; using System.Net; using System.Runtime.ExceptionServices; +using System.Runtime.InteropServices; +using System.Security.Cryptography; using System.Text.RegularExpressions; using System.Threading; +using System.Threading.Tasks; +using Azure; +using Microsoft.PowerShell.PSResourceGet.UtilClasses; +using NuGet.Protocol.Core.Types; +using NuGet.Versioning; namespace Microsoft.PowerShell.PSResourceGet.Cmdlets { @@ -35,7 +41,19 @@ internal class FindHelper private bool _includeDependencies = false; private bool _repositoryNameContainsWildcard = true; private NetworkCredential _networkCredential; - private Dictionary> _packagesFound; + + // Gets intantiated each time a cmdlet is run. + // If running 'Install-PSResource Az, TestModule, NewTestModule', it will contain one parent and its dependencies. + private ConcurrentDictionary> _packagesFound; + + // Creates a new instance of depPkgsFound each time FindDependencyPackages() is called. + // This will eventually return the PSResourceInfo object to the main cmdlet class. + private ConcurrentDictionary depPkgsFound; + + // Contains the latest found version of a particular package. + private ConcurrentDictionary _knownLatestPkgVersion; + + ConcurrentDictionary> _cachedNetworkCalls; #endregion @@ -48,7 +66,10 @@ public FindHelper(CancellationToken cancellationToken, PSCmdlet cmdletPassedIn, _cancellationToken = cancellationToken; _cmdletPassedIn = cmdletPassedIn; _networkCredential = networkCredential; - _packagesFound = new Dictionary>(StringComparer.OrdinalIgnoreCase); + _packagesFound = new ConcurrentDictionary>(StringComparer.OrdinalIgnoreCase); + _knownLatestPkgVersion = new ConcurrentDictionary(StringComparer.OrdinalIgnoreCase); + _type = ResourceType.None; + _cachedNetworkCalls = new ConcurrentDictionary>(); } #endregion @@ -740,7 +761,6 @@ private IEnumerable SearchByNames(ServerApiCall currentServer, R { parentPkgs.Add(foundPkg); TryAddToPackagesFound(foundPkg); - _cmdletPassedIn.WriteDebug($"Found package '{foundPkg.Name}' version '{foundPkg.Version}'"); yield return foundPkg; } @@ -884,7 +904,20 @@ private IEnumerable SearchByNames(ServerApiCall currentServer, R FindResults responses = null; if (_tag.Length == 0) { - responses = currentServer.FindVersion(pkgName, _nugetVersion.ToNormalizedString(), _type, out errRecord); + ConcurrentDictionary> cachedNetworkCalls = new ConcurrentDictionary>(); + Task response = null; + if (currentServer.Repository.ApiVersion == PSRepositoryInfo.APIVersion.V2) { + string key = $"{pkgName}|{_nugetVersion.ToNormalizedString()}|{_type}"; + response = cachedNetworkCalls.GetOrAdd(key, _ => currentServer.FindVersionAsync(pkgName, _nugetVersion.ToNormalizedString(), _type)); + + responses = response.GetAwaiter().GetResult(); + + } + else { + responses = currentServer.FindVersion(pkgName, _nugetVersion.ToNormalizedString(), _type, out errRecord); + } + + } else { @@ -956,7 +989,17 @@ private IEnumerable SearchByNames(ServerApiCall currentServer, R FindResults responses = null; if (_tag.Length == 0) { - responses = currentServer.FindVersionGlobbing(pkgName, _versionRange, _prerelease, _type, getOnlyLatest: false, out errRecord); + ConcurrentDictionary> cachedNetworkCalls = new ConcurrentDictionary>(); + Task response = null; + if (currentServer.Repository.ApiVersion == PSRepositoryInfo.APIVersion.V2) { + string key = $"{pkgName}|{_versionRange.ToString()}|{_type}"; + response = cachedNetworkCalls.GetOrAdd(key, _ => currentServer.FindVersionGlobbingAsync(pkgName, _versionRange, _prerelease, _type, getOnlyLatest: false)); + + responses = response.GetAwaiter().GetResult(); + } + else { + responses = currentServer.FindVersionGlobbing(pkgName, _versionRange, _prerelease, _type, getOnlyLatest: false, out errRecord); + } } else { @@ -1023,7 +1066,7 @@ private IEnumerable SearchByNames(ServerApiCall currentServer, R { foreach (PSResourceInfo currentPkg in parentPkgs) { - _cmdletPassedIn.WriteDebug($"Finding dependency packages for '{currentPkg.Name}'"); + _cmdletPassedIn.WriteDebug($"Finding dependency packages (SearchByNames) for '{currentPkg.Name}'"); foreach (PSResourceInfo pkgDep in FindDependencyPackages(currentServer, currentResponseUtil, currentPkg, repository)) { yield return pkgDep; @@ -1050,31 +1093,55 @@ private HashSet GetPackageNamesPopulated(string[] pkgNames) return pkgsToDiscover; } - private bool TryAddToPackagesFound(PSResourceInfo foundPkg) - { + { + // This handles prerelease versions as well. bool addedToHash = false; string foundPkgName = foundPkg.Name; - string foundPkgVersion = Utils.GetNormalizedVersionString(foundPkg.Version.ToString(), foundPkg.Prerelease); + string foundPkgVersion = FormatPkgVersionString(foundPkg); if (_packagesFound.ContainsKey(foundPkgName)) { - List pkgVersions = _packagesFound[foundPkgName] as List; + _packagesFound.TryGetValue(foundPkgName, out List pkgVersions); if (!pkgVersions.Contains(foundPkgVersion)) { - pkgVersions.Add(foundPkgVersion); - _packagesFound[foundPkgName] = pkgVersions; + List newPkgVersions = new List(pkgVersions) + { + foundPkgVersion + }; + _packagesFound.TryUpdate(foundPkgName, newPkgVersions, pkgVersions); + addedToHash = true; } } else { - _packagesFound.Add(foundPkg.Name, new List { foundPkgVersion }); + _packagesFound.TryAdd(foundPkg.Name, new List { foundPkgVersion }); addedToHash = true; } - _cmdletPassedIn.WriteDebug($"Found package '{foundPkg.Name}' version '{foundPkg.Version}'"); + return addedToHash; + } + + private bool TryAddToKnownLatestPkgVersion(PSResourceInfo foundPkg) + { + // This handles prerelease versions as well. + bool addedToHash = false; + string foundPkgName = foundPkg.Name; + string foundPkgVersion = FormatPkgVersionString(foundPkg); + + if (_knownLatestPkgVersion.ContainsKey(foundPkgName)) + { + _knownLatestPkgVersion.TryGetValue(foundPkgName, out PSResourceInfo oldPkgVersion); + _knownLatestPkgVersion.TryUpdate(foundPkgName, foundPkg, oldPkgVersion); + addedToHash = true; + } + else + { + _knownLatestPkgVersion.TryAdd(foundPkg.Name, foundPkg); + addedToHash = true; + } return addedToHash; } @@ -1082,12 +1149,10 @@ private bool TryAddToPackagesFound(PSResourceInfo foundPkg) private string FormatPkgVersionString(PSResourceInfo pkg) { string fullPkgVersion = pkg.Version.ToString(); - if (!string.IsNullOrWhiteSpace(pkg.Prerelease)) { fullPkgVersion += $"-{pkg.Prerelease}"; } - _cmdletPassedIn.WriteDebug($"Formatted full package version is: '{fullPkgVersion}'"); return fullPkgVersion; } @@ -1096,250 +1161,290 @@ private string FormatPkgVersionString(PSResourceInfo pkg) #region Internal Client Search Methods - internal IEnumerable FindDependencyPackages( - ServerApiCall currentServer, - ResponseUtil currentResponseUtil, - PSResourceInfo currentPkg, - PSRepositoryInfo repository) + internal IEnumerable FindDependencyPackages(ServerApiCall currentServer, ResponseUtil currentResponseUtil, PSResourceInfo currentPkg, PSRepositoryInfo repository) { + depPkgsFound = new ConcurrentDictionary(); + _cmdletPassedIn.WriteDebug($"In FindHelper::FindDependencyPackages() - {currentPkg.Name}"); + FindDependencyPackagesHelper(currentServer, currentResponseUtil, currentPkg, repository); + + return depPkgsFound.Values.ToList(); + } + + // Method 2 + internal void FindDependencyPackagesHelper(ServerApiCall currentServer, ResponseUtil currentResponseUtil, PSResourceInfo currentPkg, PSRepositoryInfo repository) + { + List errors = new List(); if (currentPkg.Dependencies.Length > 0) { - foreach (var dep in currentPkg.Dependencies) + // If finding more than 5 packages, do so concurrently + const int PARALLEL_THRESHOLD = 5; // TODO: Trottle limit from user, defaults to 5; + int processorCount = Environment.ProcessorCount; + int maxDegreeOfParallelism = processorCount * 4; + if (currentPkg.Dependencies.Length > PARALLEL_THRESHOLD) { - PSResourceInfo depPkg = null; - - if (dep.VersionRange.Equals(VersionRange.All)) + Parallel.ForEach(currentPkg.Dependencies, new ParallelOptions { MaxDegreeOfParallelism = maxDegreeOfParallelism }, dep => { - FindResults responses = currentServer.FindName(dep.Name, includePrerelease: true, _type, out ErrorRecord errRecord); - if (errRecord != null) - { - if (errRecord.Exception is ResourceNotFoundException) - { - _cmdletPassedIn.WriteVerbose(errRecord.Exception.Message); - } - else - { - _cmdletPassedIn.WriteError(errRecord); - } - yield return null; - continue; - } + FindDependencyPackageVersion(dep, currentServer, currentResponseUtil, currentPkg, repository, errors); + }); + // TODO: what is perf if parallel.ForEach is always run? + } + else + { + foreach (var dep in currentPkg.Dependencies) + { + FindDependencyPackageVersion(dep, currentServer, currentResponseUtil, currentPkg, repository, errors); + } + } + foreach (ErrorRecord error in errors) + { + _cmdletPassedIn.WriteError(error); + } + } + } - PSResourceResult currentResult = currentResponseUtil.ConvertToPSResourceResult(responses).FirstOrDefault(); - if (currentResult == null) - { - // This scenario may occur when the package version requested is unlisted. - _cmdletPassedIn.WriteError(new ErrorRecord( - new ResourceNotFoundException($"Dependency package with name '{dep.Name}' could not be found in repository '{repository.Name}'"), - "DependencyPackageNotFound", - ErrorCategory.ObjectNotFound, - this)); - yield return null; - continue; - } + // Method 3 + private void FindDependencyPackageVersion(Dependency dep, ServerApiCall currentServer, ResponseUtil currentResponseUtil, PSResourceInfo currentPkg, PSRepositoryInfo repository, List errors) + { + PSResourceInfo depPkg = null; - if (currentResult.exception != null && !currentResult.exception.Message.Equals(string.Empty)) - { - _cmdletPassedIn.WriteError(new ErrorRecord( - new ResourceNotFoundException($"Dependency package with name '{dep.Name}' could not be found in repository '{repository.Name}'", currentResult.exception), - "DependencyPackageNotFound", - ErrorCategory.ObjectNotFound, - this)); - yield return null; - continue; - } + if (dep.VersionRange.Equals(VersionRange.All) || !dep.VersionRange.HasUpperBound) + { + // Case 1: No upper bound, eg: "*" or "(1.0.0, )" + // Check if the latest version is cached + if (_knownLatestPkgVersion.TryGetValue(dep.Name, out PSResourceInfo cachedDepPkg)) + { + depPkg = cachedDepPkg; + } + else + { + // Find this version from the server + depPkg = FindDependencyWithLowerBound(dep, currentServer, currentResponseUtil, currentPkg, repository, errors); + } + } + else if (dep.VersionRange.HasLowerBound && dep.VersionRange.MinVersion.Equals(dep.VersionRange.MaxVersion)) + { + // Case 2: Exact package version, eg: "1.0.0" or "[1.0.0, 1.0.0]" + // Note: need to check if VersionRange has lower bound because if it does not, MinVersion will be null + // Check if the latest version is cached, and if this latest version is the version we're looking for + if (_knownLatestPkgVersion.TryGetValue(dep.Name, out PSResourceInfo cachedRangePkg) && + NuGetVersion.TryParse(cachedRangePkg.Version?.ToString(), out NuGetVersion cachedPkgVersion) && + dep.VersionRange.Satisfies(cachedPkgVersion)) + { + depPkg = cachedRangePkg; + } + else + { + depPkg = FindDependencyWithSpecificVersion(dep, currentServer, currentResponseUtil, currentPkg, repository, errors); + } + } + else + { + // Case 3: Version range with an upper bound, eg: "(1.0.0, 3.0.0)" + // Check if the latest version is cached, and if this latest version is the version we're looking for + if (_knownLatestPkgVersion.TryGetValue(dep.Name, out PSResourceInfo cachedRangePkg) && + NuGetVersion.TryParse(cachedRangePkg.Version?.ToString(), out NuGetVersion cachedPkgVersion) && + dep.VersionRange.Satisfies(cachedPkgVersion)) + { + depPkg = cachedRangePkg; + } + else + { + depPkg = FindDependencyWithUpperBound(dep, currentServer, currentResponseUtil, currentPkg, repository, errors); + } + } + } - depPkg = currentResult.returnedObject; + // Method 4 + private PSResourceInfo FindDependencyWithSpecificVersion(Dependency dep, ServerApiCall currentServer, ResponseUtil currentResponseUtil, PSResourceInfo currentPkg, PSRepositoryInfo repository, List errors) + { + PSResourceInfo depPkg = null; + ErrorRecord errRecord = null; + FindResults responses = null; + Task response = null; - if (!_packagesFound.ContainsKey(depPkg.Name)) - { - foreach (PSResourceInfo depRes in FindDependencyPackages(currentServer, currentResponseUtil, depPkg, repository)) - { - yield return depRes; - } - } - else - { - List pkgVersions = _packagesFound[depPkg.Name] as List; - // _packagesFound has depPkg.name in it, but the version is not the same - if (!pkgVersions.Contains(FormatPkgVersionString(depPkg))) - { - foreach (PSResourceInfo depRes in FindDependencyPackages(currentServer, currentResponseUtil, depPkg, repository)) - { - yield return depRes; - } - } - } - } - else if(dep.VersionRange.MaxVersion != null && dep.VersionRange.MinVersion != null && dep.VersionRange.MaxVersion.OriginalVersion.Equals(dep.VersionRange.MinVersion.OriginalVersion)) + if (currentServer.Repository.ApiVersion == PSRepositoryInfo.APIVersion.V2) + { + // See if the network call we're making is already cached, if not, call FindNameAsync() and cache results + string key = $"{dep.Name}|{dep.VersionRange.MaxVersion.ToString()}|{_type}"; + response = _cachedNetworkCalls.GetOrAdd(key, _ => currentServer.FindVersionAsync(dep.Name, dep.VersionRange.MaxVersion.ToString(), _type)); + + responses = response.GetAwaiter().GetResult(); + } + else + { + responses = currentServer.FindVersion(dep.Name, dep.VersionRange.MaxVersion.ToString(), _type, out errRecord); + } + + + // Error handling and Convert to PSResource object + if (errRecord != null) + { + errors.Add(new ErrorRecord( + new ResourceNotFoundException($"Dependency package could not be found: '{errRecord.Exception.Message}'"), + "DependencyPackageNotFound", + ErrorCategory.ObjectNotFound, + this)); + } + else + { + PSResourceResult currentResult = currentResponseUtil.ConvertToPSResourceResult(responses).FirstOrDefault(); + if (currentResult == null || currentResult.exception != null && !currentResult.exception.Message.Equals(string.Empty)) + { + errors.Add(new ErrorRecord( + new ResourceNotFoundException($"Dependency package with name '{dep.Name}' and version range '{dep.VersionRange}' could not be found in repository '{repository.Name}'", currentResult.exception), + "DependencyPackageNotFound", + ErrorCategory.ObjectNotFound, + this)); + } + else + { + depPkg = currentResult.returnedObject; + TryAddToKnownLatestPkgVersion(depPkg); + + string pkgVersion = FormatPkgVersionString(depPkg); + string key = $"{depPkg.Name}{pkgVersion}"; + if (!depPkgsFound.ContainsKey(key)) { - string depPkgVersion = dep.VersionRange.MaxVersion.OriginalVersion; - FindResults responses = currentServer.FindVersion(dep.Name, version: dep.VersionRange.MaxVersion.ToNormalizedString(), _type, out ErrorRecord errRecord); - if (errRecord != null) - { - if (errRecord.Exception is ResourceNotFoundException) - { - _cmdletPassedIn.WriteVerbose(errRecord.Exception.Message); - } - else - { - _cmdletPassedIn.WriteError(errRecord); - } - yield return null; - continue; - } + // Add pkg to collection of packages found then find dependencies + // depPkgsFound creates a new instance of depPkgsFound each time FindDependencyPackages() is called. + // This will eventually return the PSResourceInfo object to the main cmdlet class. + depPkgsFound.TryAdd(key, depPkg); + FindDependencyPackagesHelper(currentServer, currentResponseUtil, depPkg, repository); + } + } + } - PSResourceResult currentResult = currentResponseUtil.ConvertToPSResourceResult(responses).FirstOrDefault(); - if (currentResult == null) - { - // This scenario may occur when the package version requested is unlisted. - _cmdletPassedIn.WriteError(new ErrorRecord( - new ResourceNotFoundException($"Dependency package with name '{dep.Name}' and version '{depPkgVersion}' could not be found in repository '{repository.Name}'"), - "DependencyPackageNotFound", - ErrorCategory.ObjectNotFound, - this)); - yield return null; - continue; - } + return depPkg; + } - if (currentResult.exception != null && !currentResult.exception.Message.Equals(string.Empty)) - { - _cmdletPassedIn.WriteError(new ErrorRecord( - new ResourceNotFoundException($"Dependency package with name '{dep.Name}' and version '{depPkgVersion}' could not be found in repository '{repository.Name}'", currentResult.exception), - "DependencyPackageNotFound", - ErrorCategory.ObjectNotFound, - this)); - yield return null; - continue; - } + // Method 5 + private PSResourceInfo FindDependencyWithLowerBound(Dependency dep, ServerApiCall currentServer, ResponseUtil currentResponseUtil, PSResourceInfo currentPkg, PSRepositoryInfo repository, List errors) + { + PSResourceInfo depPkg = null; + FindResults responses = null; + ErrorRecord errRecord = null; + Task response = null; + + if (currentServer.Repository.ApiVersion == PSRepositoryInfo.APIVersion.V2) + { + // See if the network call we're making is already cached, if not, call FindNameAsync() and cache results + string key = $"{dep.Name}|*|{_type}"; + response = _cachedNetworkCalls.GetOrAdd(key, _ => currentServer.FindNameAsync(dep.Name, includePrerelease: true, _type)); + + responses = response.GetAwaiter().GetResult(); + } + else + { + responses = currentServer.FindName(dep.Name, includePrerelease: true, _type, out errRecord); + } - depPkg = currentResult.returnedObject; + // Error handling and Convert to PSResource object + if (errRecord != null) + { + errors.Add(new ErrorRecord( + new ResourceNotFoundException($"Dependency package could not be found: '{errRecord.Exception.Message}'"), + "DependencyPackageNotFound", + ErrorCategory.ObjectNotFound, + this)); + } + else + { + PSResourceResult currentResult = currentResponseUtil.ConvertToPSResourceResult(responses).FirstOrDefault(); + if (currentResult == null || currentResult.exception != null && !currentResult.exception.Message.Equals(string.Empty)) + { + errors.Add(new ErrorRecord( + new ResourceNotFoundException($"Dependency package with name '{dep.Name}' and version range '{dep.VersionRange}' could not be found in repository '{repository.Name}'", currentResult.exception), + "DependencyPackageNotFound", + ErrorCategory.ObjectNotFound, + this)); + } + else + { + depPkg = currentResult.returnedObject; + TryAddToKnownLatestPkgVersion(depPkg); - if (!_packagesFound.ContainsKey(depPkg.Name)) - { - foreach (PSResourceInfo depRes in FindDependencyPackages(currentServer, currentResponseUtil, depPkg, repository)) - { - yield return depRes; - } - } - else - { - List pkgVersions = _packagesFound[depPkg.Name] as List; - // _packagesFound has depPkg.name in it, but the version is not the same - if (!pkgVersions.Contains(FormatPkgVersionString(depPkg))) - { - foreach (PSResourceInfo depRes in FindDependencyPackages(currentServer, currentResponseUtil, depPkg, repository)) - { - yield return depRes; - } - } - } - } - else + string pkgVersion = FormatPkgVersionString(depPkg); + string key = $"{depPkg.Name}{pkgVersion}"; + if (!depPkgsFound.ContainsKey(key)) { - FindResults responses = currentServer.FindVersionGlobbing(dep.Name, dep.VersionRange, includePrerelease: true, ResourceType.None, getOnlyLatest: true, out ErrorRecord errRecord); - if (errRecord != null) - { - if (errRecord.Exception is ResourceNotFoundException) - { - _cmdletPassedIn.WriteVerbose(errRecord.Exception.Message); - } - else - { - _cmdletPassedIn.WriteError(errRecord); - } - yield return null; - continue; - } - - if (responses.IsFindResultsEmpty()) - { - _cmdletPassedIn.WriteError(new ErrorRecord( - new InvalidOrEmptyResponse($"Dependency package with name {dep.Name} and version range {dep.VersionRange} could not be found in repository '{repository.Name}"), - "FindDepPackagesFindVersionGlobbingFailure", - ErrorCategory.InvalidResult, - this)); - yield return null; - continue; - } + // Add pkg to collection of packages found then find dependencies + // depPkgsFound creates a new instance of depPkgsFound each time FindDependencyPackages() is called. + // This will eventually return the PSResourceInfo object to the main cmdlet class. + depPkgsFound.TryAdd(key, depPkg); + FindDependencyPackagesHelper(currentServer, currentResponseUtil, depPkg, repository); + } + } + } - foreach (PSResourceResult currentResult in currentResponseUtil.ConvertToPSResourceResult(responses)) - { - if (currentResult.exception != null && !currentResult.exception.Message.Equals(string.Empty)) - { - _cmdletPassedIn.WriteError(new ErrorRecord( - new ResourceNotFoundException($"Dependency package with name '{dep.Name}' and version range '{dep.VersionRange}' could not be found in repository '{repository.Name}'", currentResult.exception), - "DependencyPackageNotFound", - ErrorCategory.ObjectNotFound, - this)); + return depPkg; + } - yield return null; - continue; - } + // Method 6 + private PSResourceInfo FindDependencyWithUpperBound(Dependency dep, ServerApiCall currentServer, ResponseUtil currentResponseUtil, PSResourceInfo currentPkg, PSRepositoryInfo repository, List errors) + { + PSResourceInfo depPkg = null; + ErrorRecord errRecord = null; + FindResults responses = null; + Task response = null; - // Check to see if version falls within version range - PSResourceInfo foundDep = currentResult.returnedObject; - string depVersionStr = $"{foundDep.Version}"; - if (foundDep.IsPrerelease) - { - depVersionStr += $"-{foundDep.Prerelease}"; - } + ConcurrentDictionary> cachedNetworkCalls = new ConcurrentDictionary>(); - if (NuGetVersion.TryParse(depVersionStr, out NuGetVersion depVersion) - && dep.VersionRange.Satisfies(depVersion)) - { - depPkg = foundDep; - } - } + if (currentServer.Repository.ApiVersion == PSRepositoryInfo.APIVersion.V2) + { + // See if the network call we're making is already caced, if not, call FindNameAsync() and cache results + string key = $"{dep.Name}|{dep.VersionRange.MaxVersion.ToString()}|{_type}"; + response = cachedNetworkCalls.GetOrAdd(key, _ => currentServer.FindVersionGlobbingAsync(dep.Name, dep.VersionRange, includePrerelease: true, ResourceType.None, getOnlyLatest: true)); - if (depPkg == null) - { - continue; - } + responses = response.GetAwaiter().GetResult(); - if (!_packagesFound.ContainsKey(depPkg.Name)) - { - foreach (PSResourceInfo depRes in FindDependencyPackages(currentServer, currentResponseUtil, depPkg, repository)) - { - yield return depRes; - } - } - else - { - List pkgVersions = _packagesFound[depPkg.Name] as List; - // _packagesFound has depPkg.name in it, but the version is not the same - if (!pkgVersions.Contains(FormatPkgVersionString(depPkg))) - { - foreach (PSResourceInfo depRes in FindDependencyPackages(currentServer, currentResponseUtil, depPkg, repository)) - { - yield return depRes; - } - } - } - } - } } - - if (!_packagesFound.ContainsKey(currentPkg.Name)) + else { - TryAddToPackagesFound(currentPkg); + responses = currentServer.FindVersionGlobbing(dep.Name, dep.VersionRange, includePrerelease: true, ResourceType.None, getOnlyLatest: true, out errRecord); + } - yield return currentPkg; + // Error handling and Convert to PSResource object + if (errRecord != null) + { + errors.Add(new ErrorRecord( + new ResourceNotFoundException($"Dependency package could not be found: '{errRecord.Exception.Message}'"), + "DependencyPackageNotFound", + ErrorCategory.ObjectNotFound, + this)); } else { - List pkgVersions = _packagesFound[currentPkg.Name] as List; - // _packagesFound has currentPkg.name in it, but the version is not the same - if (!pkgVersions.Contains(FormatPkgVersionString(currentPkg))) + PSResourceResult currentResult = currentResponseUtil.ConvertToPSResourceResult(responses).FirstOrDefault(); + if (currentResult == null || currentResult.exception != null && !currentResult.exception.Message.Equals(string.Empty)) + { + errors.Add(new ErrorRecord( + new ResourceNotFoundException($"Dependency package with name '{dep.Name}' and version range '{dep.VersionRange}' could not be found in repository '{repository.Name}'", currentResult.exception), + "DependencyPackageNotFound", + ErrorCategory.ObjectNotFound, + this)); + } + else { - TryAddToPackagesFound(currentPkg); + depPkg = currentResult.returnedObject; - yield return currentPkg; + TryAddToKnownLatestPkgVersion(depPkg); + + string pkgVersion = FormatPkgVersionString(depPkg); + string key = $"{depPkg.Name}{pkgVersion}"; + if (!depPkgsFound.ContainsKey(key)) + { + // Add pkg to collection of packages found then find dependencies + // depPkgsFound creates a new instance of depPkgsFound each time FindDependencyPackages() is called. + // This will eventually return the PSResourceInfo object to the main cmdlet class. + depPkgsFound.TryAdd(key, depPkg); + FindDependencyPackagesHelper(currentServer, currentResponseUtil, depPkg, repository); + } } } - + + return depPkg; } - #endregion + #endregion } } diff --git a/src/code/InstallHelper.cs b/src/code/InstallHelper.cs index 0616cf040..b2283341b 100644 --- a/src/code/InstallHelper.cs +++ b/src/code/InstallHelper.cs @@ -2,10 +2,13 @@ // Licensed under the MIT License. using Microsoft.PowerShell.PSResourceGet.UtilClasses; +using NuGet.Protocol.Core.Types; using NuGet.Versioning; using System; using System.Collections; +using System.Collections.Concurrent; using System.Collections.Generic; +using System.Diagnostics; using System.Globalization; using System.IO; using System.IO.Compression; @@ -15,6 +18,7 @@ using System.Runtime.InteropServices; using System.Text.RegularExpressions; using System.Threading; +using System.Threading.Tasks; namespace Microsoft.PowerShell.PSResourceGet.Cmdlets { @@ -53,6 +57,7 @@ internal class InstallHelper private string _tmpPath; private NetworkCredential _networkCredential; private HashSet _packagesOnMachine; + private FindHelper _findHelper; #endregion @@ -64,6 +69,8 @@ public InstallHelper(PSCmdlet cmdletPassedIn, NetworkCredential networkCredentia _cancellationToken = source.Token; _cmdletPassedIn = cmdletPassedIn; _networkCredential = networkCredential; + + _findHelper = new FindHelper(_cancellationToken, _cmdletPassedIn, _networkCredential); } /// @@ -426,7 +433,7 @@ private void MoveFilesIntoInstallPath( // Delete the directory path before replacing it with the new module. // If deletion fails (usually due to binary file in use), then attempt restore so that the currently // installed module is not corrupted. - _cmdletPassedIn.WriteVerbose($"Attempting to delete with restore on failure. '{finalModuleVersionDir}'"); + //_cmdletPassedIn.WriteVerbose($"Attempting to delete with restore on failure. '{finalModuleVersionDir}'"); Utils.DeleteDirectoryWithRestore(finalModuleVersionDir); } @@ -508,6 +515,7 @@ private List InstallPackages( FindHelper findHelper) { _cmdletPassedIn.WriteDebug("In InstallHelper::InstallPackages()"); + List pkgsSuccessfullyInstalled = new(); // Install parent package to the temp directory, @@ -524,7 +532,7 @@ private List InstallPackages( // and value as a Hashtable of specific package info: // packageName, { version = "", isScript = "", isModule = "", pkg = "", etc. } // Install parent package to the temp directory. - Hashtable packagesHash = BeginPackageInstall( + ConcurrentDictionary packagesHash = BeginPackageInstall( searchVersionType: _versionType, specificVersion: _nugetVersion, versionRange: _versionRange, @@ -533,10 +541,11 @@ private List InstallPackages( currentServer: currentServer, currentResponseUtil: currentResponseUtil, tempInstallPath: tempInstallPath, - packagesHash: new Hashtable(StringComparer.InvariantCultureIgnoreCase), + skipDependencyCheck: skipDependencyCheck, + packagesHash: new ConcurrentDictionary(StringComparer.InvariantCultureIgnoreCase), errRecord: out ErrorRecord errRecord); - // At this point parent package is installed to temp path. + // At this point all packages are installed to temp path. if (errRecord != null) { if (errRecord.FullyQualifiedErrorId.Equals("PackageNotFound")) @@ -559,70 +568,6 @@ private List InstallPackages( Hashtable parentPkgInfo = packagesHash[parentPackage] as Hashtable; PSResourceInfo parentPkgObj = parentPkgInfo["psResourceInfoPkg"] as PSResourceInfo; - if (!skipDependencyCheck) - { - // Get the dependencies from the installed package. - if (parentPkgObj.Dependencies.Length > 0) - { - bool depFindFailed = false; - foreach (PSResourceInfo depPkg in findHelper.FindDependencyPackages(currentServer, currentResponseUtil, parentPkgObj, repository)) - { - if (depPkg == null) - { - depFindFailed = true; - continue; - } - - if (String.Equals(depPkg.Name, parentPkgObj.Name, StringComparison.OrdinalIgnoreCase)) - { - continue; - } - - NuGetVersion depVersion = null; - if (depPkg.AdditionalMetadata.ContainsKey("NormalizedVersion")) - { - if (!NuGetVersion.TryParse(depPkg.AdditionalMetadata["NormalizedVersion"] as string, out depVersion)) - { - NuGetVersion.TryParse(depPkg.Version.ToString(), out depVersion); - } - } - - string depPkgNameVersion = $"{depPkg.Name}{depPkg.Version.ToString()}"; - if (_packagesOnMachine.Contains(depPkgNameVersion) && !depPkg.IsPrerelease) - { - // if a dependency package is already installed, do not install it again. - // to determine if the package version is already installed, _packagesOnMachine is used but it only contains name, version info, not version with prerelease info - // if the dependency package is found to be prerelease, it is safer to install it (and worse case it reinstalls) - _cmdletPassedIn.WriteVerbose($"Dependency '{depPkg.Name}' with version '{depPkg.Version}' is already installed."); - continue; - } - - packagesHash = BeginPackageInstall( - searchVersionType: VersionType.SpecificVersion, - specificVersion: depVersion, - versionRange: null, - pkgNameToInstall: depPkg.Name, - repository: repository, - currentServer: currentServer, - currentResponseUtil: currentResponseUtil, - tempInstallPath: tempInstallPath, - packagesHash: packagesHash, - errRecord: out ErrorRecord installPkgErrRecord); - - if (installPkgErrRecord != null) - { - _cmdletPassedIn.WriteError(installPkgErrRecord); - continue; - } - } - - if (depFindFailed) - { - continue; - } - } - } - // If -WhatIf is passed in, early out. if (_cmdletPassedIn.MyInvocation.BoundParameters.ContainsKey("WhatIf") && (SwitchParameter)_cmdletPassedIn.MyInvocation.BoundParameters["WhatIf"] == true) { @@ -673,7 +618,7 @@ private List InstallPackages( /// /// Installs a single package to the temporary path. /// - private Hashtable BeginPackageInstall( + private ConcurrentDictionary BeginPackageInstall( VersionType searchVersionType, NuGetVersion specificVersion, VersionRange versionRange, @@ -682,13 +627,15 @@ private Hashtable BeginPackageInstall( ServerApiCall currentServer, ResponseUtil currentResponseUtil, string tempInstallPath, - Hashtable packagesHash, + bool skipDependencyCheck, + ConcurrentDictionary packagesHash, out ErrorRecord errRecord) { _cmdletPassedIn.WriteDebug("In InstallHelper::InstallPackage()"); FindResults responses = null; errRecord = null; + // Find the parent package that needs to be installed switch (searchVersionType) { case VersionType.VersionRange: @@ -717,6 +664,7 @@ private Hashtable BeginPackageInstall( default: // VersionType.NoVersion responses = currentServer.FindName(pkgNameToInstall, _prerelease, ResourceType.None, out ErrorRecord findNameErrRecord); + if (findNameErrRecord != null) { errRecord = findNameErrRecord; @@ -726,6 +674,7 @@ private Hashtable BeginPackageInstall( break; } + // Convert parent package to PSResourceInfo PSResourceInfo pkgToInstall = null; foreach (PSResourceResult currentResult in currentResponseUtil.ConvertToPSResourceResult(responses)) { @@ -789,9 +738,11 @@ private Hashtable BeginPackageInstall( } // Check to see if the pkg is already installed (ie the pkg is installed and the version satisfies the version range provided via param) + // TODO: can use cache for this if (!_reinstall) { string currPkgNameVersion = $"{pkgToInstall.Name}{pkgToInstall.Version}"; + // Use HashSet lookup instead of Contains for O(1) performance if (_packagesOnMachine.Contains(currPkgNameVersion)) { _cmdletPassedIn.WriteWarning($"Resource '{pkgToInstall.Name}' with version '{pkgVersion}' is already installed. If you would like to reinstall, please run the cmdlet again with the -Reinstall parameter"); @@ -809,14 +760,14 @@ private Hashtable BeginPackageInstall( } - Hashtable updatedPackagesHash = packagesHash; + ConcurrentDictionary updatedPackagesHash = packagesHash; // -WhatIf processing. if (_savePkg && !_cmdletPassedIn.ShouldProcess($"Package to save: '{pkgToInstall.Name}', version: '{pkgVersion}'")) { if (!updatedPackagesHash.ContainsKey(pkgToInstall.Name)) { - updatedPackagesHash.Add(pkgToInstall.Name, new Hashtable(StringComparer.InvariantCultureIgnoreCase) + updatedPackagesHash.TryAdd(pkgToInstall.Name, new Hashtable(StringComparer.InvariantCultureIgnoreCase) { { "isModule", "" }, { "isScript", "" }, @@ -832,7 +783,7 @@ private Hashtable BeginPackageInstall( { if (!updatedPackagesHash.ContainsKey(pkgToInstall.Name)) { - updatedPackagesHash.Add(pkgToInstall.Name, new Hashtable(StringComparer.InvariantCultureIgnoreCase) + updatedPackagesHash.TryAdd(pkgToInstall.Name, new Hashtable(StringComparer.InvariantCultureIgnoreCase) { { "isModule", "" }, { "isScript", "" }, @@ -846,25 +797,115 @@ private Hashtable BeginPackageInstall( } else { - // Download the package. - string pkgName = pkgToInstall.Name; - Stream responseStream = currentServer.InstallPackage(pkgName, pkgVersion, _prerelease, out ErrorRecord installNameErrRecord); - if (installNameErrRecord != null) + // Concurrent updates + // Find all dependencies + if (!skipDependencyCheck) { - errRecord = installNameErrRecord; - return packagesHash; + // concurrency updates + List parentAndDeps = _findHelper.FindDependencyPackages(currentServer, currentResponseUtil, pkgToInstall, repository).ToList(); + // List returned only includes dependencies, so we'll add the parent pkg to this list to pass on to installation method + parentAndDeps.Add(pkgToInstall); + + _cmdletPassedIn.WriteDebug("In InstallHelper::InstallPackage(), found all dependencies"); + + return InstallParentAndDependencyPackages(parentAndDeps, currentServer, tempInstallPath, packagesHash, updatedPackagesHash, pkgToInstall); + } + else { + // If we don't install dependencies, we're only installing the parent pkg so we can short circut and simply install the parent pkg. + // TODO: check this version and prerelease combo + Stream responseStream = currentServer.InstallPackage(pkgToInstall.Name, pkgToInstall.Version.ToString(), true, out ErrorRecord installNameErrRecord); + + if (installNameErrRecord != null) + { + errRecord = installNameErrRecord; + return packagesHash; + } + + bool installedToTempPathSuccessfully = _asNupkg ? TrySaveNupkgToTempPath(responseStream, tempInstallPath, pkgToInstall.Name, pkgToInstall.Version.ToString(), pkgToInstall, packagesHash, out updatedPackagesHash, out errRecord) : + TryInstallToTempPath(responseStream, tempInstallPath, pkgToInstall.Name, pkgToInstall.Version.ToString(), pkgToInstall, packagesHash, out updatedPackagesHash, out errRecord); + if (!installedToTempPathSuccessfully) + { + return packagesHash; + } } + } + + return updatedPackagesHash; + } + + private ConcurrentDictionary InstallParentAndDependencyPackages(List parentAndDeps, ServerApiCall currentServer, string tempInstallPath, ConcurrentDictionary packagesHash, ConcurrentDictionary updatedPackagesHash, PSResourceInfo pkgToInstall) + { + List errors = new List(); + + // TODO: figure out a good threshold and parallel count + int processorCount = Environment.ProcessorCount; + if (parentAndDeps.Count > processorCount) + { + // Set the maximum degree of parallelism to 32? (Invoke-Command has default of 32, that's where we got this number from) + // If installing more than 3 packages, do so concurrently + // If the number of dependencies is very small (e.g., ≤ CPU cores), parallelism may add overhead instead of improving speed. + int maxDegreeOfParallelism = processorCount * 4; + Parallel.ForEach(parentAndDeps, new ParallelOptions { MaxDegreeOfParallelism = maxDegreeOfParallelism }, depPkg => + { + var depPkgName = depPkg.Name; + var depPkgVersion = depPkg.Version.ToString(); + + Stream responseStream = currentServer.InstallPackage(depPkgName, depPkgVersion, true, out ErrorRecord installNameErrRecord); + if (installNameErrRecord != null) + { + errors.Add(installNameErrRecord); + } - bool installedToTempPathSuccessfully = _asNupkg ? TrySaveNupkgToTempPath(responseStream, tempInstallPath, pkgName, pkgVersion, pkgToInstall, packagesHash, out updatedPackagesHash, out errRecord) : - TryInstallToTempPath(responseStream, tempInstallPath, pkgName, pkgVersion, pkgToInstall, packagesHash, out updatedPackagesHash, out errRecord); + ErrorRecord tempSaveErrRecord = null, tempInstallErrRecord = null; + bool installedToTempPathSuccessfully = _asNupkg ? TrySaveNupkgToTempPath(responseStream, tempInstallPath, depPkgName, depPkgVersion, depPkg, packagesHash, out updatedPackagesHash, out tempSaveErrRecord) : + TryInstallToTempPath(responseStream, tempInstallPath, depPkgName, depPkgVersion, depPkg, packagesHash, out updatedPackagesHash, out tempInstallErrRecord); - if (!installedToTempPathSuccessfully) + if (!installedToTempPathSuccessfully) + { + errors.Add(tempSaveErrRecord ?? tempInstallErrRecord); + } + }); + + if (errors.Count > 0) { + // Write out all errors collected from Parallel.ForEach + foreach (var err in errors) + { + _cmdletPassedIn.WriteError(err); + } + return packagesHash; } + + return updatedPackagesHash; } + else + { + // Install the good old fashioned way + foreach (var pkgToBeInstalled in parentAndDeps) + { + var pkgToInstallName = pkgToBeInstalled.Name; + var pkgToInstallVersion = pkgToBeInstalled.Version.ToString(); + Stream responseStream = currentServer.InstallPackage(pkgToInstallName, pkgToInstallVersion, true, out ErrorRecord installNameErrRecord); + if (installNameErrRecord != null) + { + _cmdletPassedIn.WriteError(installNameErrRecord); + return packagesHash; + } - return updatedPackagesHash; + ErrorRecord tempSaveErrRecord = null, tempInstallErrRecord = null; + bool installedToTempPathSuccessfully = _asNupkg ? TrySaveNupkgToTempPath(responseStream, tempInstallPath, pkgToInstallName, pkgToInstallVersion, pkgToBeInstalled, packagesHash, out updatedPackagesHash, out tempSaveErrRecord) : + TryInstallToTempPath(responseStream, tempInstallPath, pkgToInstallName, pkgToInstallVersion, pkgToBeInstalled, packagesHash, out updatedPackagesHash, out tempInstallErrRecord); + + if (!installedToTempPathSuccessfully) + { + _cmdletPassedIn.WriteError(tempSaveErrRecord ?? tempInstallErrRecord); + return packagesHash; + } + } + + return updatedPackagesHash; + } } /// @@ -926,11 +967,10 @@ private bool TryInstallToTempPath( string pkgName, string normalizedPkgVersion, PSResourceInfo pkgToInstall, - Hashtable packagesHash, - out Hashtable updatedPackagesHash, + ConcurrentDictionary packagesHash, + out ConcurrentDictionary updatedPackagesHash, out ErrorRecord error) { - _cmdletPassedIn.WriteDebug("In InstallHelper::TryInstallToTempPath()"); error = null; updatedPackagesHash = packagesHash; try @@ -987,8 +1027,8 @@ private bool TryInstallToTempPath( ErrorCategory.ReadError, _cmdletPassedIn); - return false; - } + return false; + } if (!Utils.TryReadManifestFile( manifestFilePath: moduleManifest, @@ -1001,7 +1041,7 @@ private bool TryInstallToTempPath( ErrorCategory.ReadError, _cmdletPassedIn); - return false; + return false; } // Accept License verification @@ -1029,17 +1069,7 @@ private bool TryInstallToTempPath( out ErrorRecord[] parseScriptFileErrors, out string[] _)) { - foreach (ErrorRecord parseError in parseScriptFileErrors) - { - _cmdletPassedIn.WriteError(parseError); - } - - error = new ErrorRecord( - new InvalidOperationException($"PSScriptFile could not be parsed"), - "PSScriptParseError", - ErrorCategory.ReadError, - _cmdletPassedIn); - + error = parseScriptFileErrors.FirstOrDefault(); return false; } } @@ -1048,8 +1078,8 @@ private bool TryInstallToTempPath( // This package is not a PowerShell package (eg a resource from the NuGet Gallery). installPath = _pathsToInstallPkg.Find(path => path.EndsWith("Modules", StringComparison.InvariantCultureIgnoreCase)); - _cmdletPassedIn.WriteVerbose($"This resource is not a PowerShell package and will be installed to the modules path: {installPath}."); - isModule = true; + //_cmdletPassedIn.WriteVerbose($"This resource is not a PowerShell package and will be installed to the modules path: {installPath}."); + isModule = true; } installPath = _savePkg ? _pathsToInstallPkg.First() : installPath; @@ -1067,7 +1097,7 @@ private bool TryInstallToTempPath( if (!updatedPackagesHash.ContainsKey(pkgName)) { // Add pkg info to hashtable. - updatedPackagesHash.Add(pkgName, new Hashtable(StringComparer.InvariantCultureIgnoreCase) + updatedPackagesHash.TryAdd(pkgName, new Hashtable(StringComparer.InvariantCultureIgnoreCase) { { "isModule", isModule }, { "isScript", isScript }, @@ -1085,8 +1115,8 @@ private bool TryInstallToTempPath( { error = new ErrorRecord( new PSInvalidOperationException( - message: $"Unable to successfully install package '{pkgName}': '{e.Message}' to temporary installation path.", - innerException: e), + message: $"Unable to successfully install package '{pkgName}': '{e.Message}' to temporary installation path.", + innerException: e), "InstallPackageFailed", ErrorCategory.InvalidOperation, _cmdletPassedIn); @@ -1104,11 +1134,11 @@ private bool TrySaveNupkgToTempPath( string pkgName, string normalizedPkgVersion, PSResourceInfo pkgToInstall, - Hashtable packagesHash, - out Hashtable updatedPackagesHash, + ConcurrentDictionary packagesHash, + out ConcurrentDictionary updatedPackagesHash, out ErrorRecord error) { - _cmdletPassedIn.WriteDebug("In InstallHelper::TrySaveNupkgToTempPath()"); + // _cmdletPassedIn.WriteDebug("In InstallHelper::TrySaveNupkgToTempPath()"); error = null; updatedPackagesHash = packagesHash; @@ -1132,7 +1162,7 @@ private bool TrySaveNupkgToTempPath( if (!updatedPackagesHash.ContainsKey(pkgName)) { // Add pkg info to hashtable. - updatedPackagesHash.Add(pkgName, new Hashtable(StringComparer.InvariantCultureIgnoreCase) + updatedPackagesHash.TryAdd(pkgName, new Hashtable(StringComparer.InvariantCultureIgnoreCase) { { "isModule", "" }, { "isScript", "" }, @@ -1228,7 +1258,7 @@ private bool TryExtractToDirectory(string zipPath, string extractPath, out Error /// /// Moves package files/directories from the temp install path into the final install path location. /// - private bool TryMoveInstallContent(string tempInstallPath, ScopeType scope, Hashtable packagesHash) + private bool TryMoveInstallContent(string tempInstallPath, ScopeType scope, ConcurrentDictionary packagesHash) { _cmdletPassedIn.WriteDebug("In InstallHelper::TryMoveInstallContent()"); foreach (string pkgName in packagesHash.Keys) @@ -1287,7 +1317,7 @@ private bool TryMoveInstallContent(string tempInstallPath, ScopeType scope, Hash "InstallPackageFailed", ErrorCategory.InvalidOperation, _cmdletPassedIn)); - + return false; } } @@ -1300,7 +1330,6 @@ private bool TryMoveInstallContent(string tempInstallPath, ScopeType scope, Hash /// private bool CallAcceptLicense(PSResourceInfo p, string moduleManifest, string tempInstallPath, string newVersion, out ErrorRecord error) { - _cmdletPassedIn.WriteDebug("In InstallHelper::CallAcceptLicense()"); error = null; var requireLicenseAcceptance = false; @@ -1404,7 +1433,7 @@ private bool CallAcceptLicense(PSResourceInfo p, string moduleManifest, string t /// private bool DetectClobber(string pkgName, Hashtable parsedMetadataHashtable, out ErrorRecord error) { - _cmdletPassedIn.WriteDebug("In InstallHelper::DetectClobber()"); + //_cmdletPassedIn.WriteDebug("In InstallHelper::DetectClobber()"); error = null; bool foundClobber = false; @@ -1468,7 +1497,6 @@ private bool DetectClobber(string pkgName, Hashtable parsedMetadataHashtable, ou /// private bool CreateMetadataXMLFile(string dirNameVersion, string installPath, PSResourceInfo pkg, bool isModule, out ErrorRecord error) { - _cmdletPassedIn.WriteDebug("In InstallHelper::CreateMetadataXMLFile()"); error = null; bool success = true; // Script will have a metadata file similar to: "TestScript_InstalledScriptInfo.xml" @@ -1498,7 +1526,7 @@ private bool CreateMetadataXMLFile(string dirNameVersion, string installPath, PS /// private void DeleteExtraneousFiles(string packageName, string dirNameVersion) { - _cmdletPassedIn.WriteDebug("In InstallHelper::DeleteExtraneousFiles()"); + // _cmdletPassedIn.WriteDebug("In InstallHelper::DeleteExtraneousFiles()"); // Deleting .nupkg SHA file, .nuspec, and .nupkg after unpacking the module // since we download as .zip for HTTP calls, we shouldn't have .nupkg* files // var nupkgSHAToDelete = Path.Combine(dirNameVersion, pkgIdString + ".nupkg.sha512"); @@ -1511,22 +1539,18 @@ private void DeleteExtraneousFiles(string packageName, string dirNameVersion) if (File.Exists(nuspecToDelete)) { - _cmdletPassedIn.WriteDebug($"Deleting '{nuspecToDelete}'"); File.Delete(nuspecToDelete); } if (File.Exists(contentTypesToDelete)) { - _cmdletPassedIn.WriteDebug($"Deleting '{contentTypesToDelete}'"); File.Delete(contentTypesToDelete); } if (Directory.Exists(relsDirToDelete)) { - _cmdletPassedIn.WriteDebug($"Deleting '{relsDirToDelete}'"); Utils.DeleteDirectory(relsDirToDelete); } if (Directory.Exists(packageDirToDelete)) { - _cmdletPassedIn.WriteDebug($"Deleting '{packageDirToDelete}'"); Utils.DeleteDirectory(packageDirToDelete); } } diff --git a/src/code/LocalServerApiCalls.cs b/src/code/LocalServerApiCalls.cs index cc43c340d..d21434f1e 100644 --- a/src/code/LocalServerApiCalls.cs +++ b/src/code/LocalServerApiCalls.cs @@ -13,6 +13,7 @@ using System.Net; using System.Management.Automation; using System.Runtime.ExceptionServices; +using System.Threading.Tasks; namespace Microsoft.PowerShell.PSResourceGet.Cmdlets { @@ -39,6 +40,15 @@ public LocalServerAPICalls (PSRepositoryInfo repository, PSCmdlet cmdletPassedIn #region Overridden Methods + public override Task FindVersionAsync(string packageName, string version, ResourceType type) + { + throw new NotImplementedException(); + } + + public override Task FindVersionGlobbingAsync(string packageName, VersionRange versionRange, bool includePrerelease, ResourceType type, bool getOnlyLatest) + { + throw new NotImplementedException(); + } /// /// Find method which allows for searching for all packages from a repository and returns latest version for each. /// Examples: Search -Repository PSGallery @@ -109,6 +119,11 @@ public override FindResults FindName(string packageName, bool includePrerelease, return FindNameHelper(packageName, Utils.EmptyStrArray, includePrerelease, type, out errRecord); } + public override Task FindNameAsync(string packageName, bool includePrerelease, ResourceType type) + { + throw new NotImplementedException(); + } + /// /// Find method which allows for searching for single name and tag and returns latest version. /// Name: no wildcard support diff --git a/src/code/NuGetServerAPICalls.cs b/src/code/NuGetServerAPICalls.cs index 1497c83da..81d9ca8f6 100644 --- a/src/code/NuGetServerAPICalls.cs +++ b/src/code/NuGetServerAPICalls.cs @@ -48,6 +48,15 @@ public NuGetServerAPICalls (PSRepositoryInfo repository, PSCmdlet cmdletPassedIn #region Overridden Methods + public override Task FindVersionAsync(string packageName, string version, ResourceType type) + { + throw new NotImplementedException(); + } + + public override Task FindVersionGlobbingAsync(string packageName, VersionRange versionRange, bool includePrerelease, ResourceType type, bool getOnlyLatest) + { + throw new NotImplementedException(); + } /// /// Find method which allows for searching for all packages from a repository and returns latest version for each. /// Examples: Search -Repository MyNuGetServer @@ -183,6 +192,11 @@ public override FindResults FindName(string packageName, bool includePrerelease, return new FindResults(stringResponse: new string[]{ response }, hashtableResponse: emptyHashResponses, responseType: FindResponseType); } + public override Task FindNameAsync(string packageName, bool includePrerelease, ResourceType type) + { + throw new NotImplementedException(); + } + /// /// Find method which allows for searching for single name and tag and returns latest version. /// Name: no wildcard support diff --git a/src/code/ServerApiCall.cs b/src/code/ServerApiCall.cs index 4580e362e..7096162d7 100644 --- a/src/code/ServerApiCall.cs +++ b/src/code/ServerApiCall.cs @@ -10,6 +10,7 @@ using System.Text; using System.Runtime.ExceptionServices; using System.Management.Automation; +using System.Threading.Tasks; namespace Microsoft.PowerShell.PSResourceGet.Cmdlets { @@ -81,6 +82,13 @@ public ServerApiCall(PSRepositoryInfo repository, NetworkCredential networkCrede /// public abstract FindResults FindName(string packageName, bool includePrerelease, ResourceType type, out ErrorRecord errRecord); + /// + /// Find method which allows for async searching for package by single name and returns latest version. + /// Name: no wildcard support + /// Examples: Search "PowerShellGet" + /// + public abstract Task FindNameAsync(string packageName, bool includePrerelease, ResourceType type); + /// /// Find method which allows for searching for package by single name and tag and returns latest version. /// Name: no wildcard support @@ -95,6 +103,15 @@ public ServerApiCall(PSRepositoryInfo repository, NetworkCredential networkCrede /// public abstract FindResults FindNameGlobbing(string packageName, bool includePrerelease, ResourceType type, out ErrorRecord errRecord); + /// + /// Find method which allows for async searching for single name with version range. + /// Name: no wildcard support + /// Version: supports wildcards + /// Examples: Search "PowerShellGet" "[3.0.0.0, 5.0.0.0]" + /// Search "PowerShellGet" "3.*" + /// + public abstract Task FindVersionGlobbingAsync(string packageName, VersionRange versionRange, bool includePrerelease, ResourceType type, bool getOnlyLatest); + /// /// Find method which allows for searching for single name with wildcards and tag and returns latest version. /// Name: supports wildcards @@ -118,6 +135,14 @@ public ServerApiCall(PSRepositoryInfo repository, NetworkCredential networkCrede /// public abstract FindResults FindVersion(string packageName, string version, ResourceType type, out ErrorRecord errRecord); + /// + /// Find method which allows for async searching for single name with specific version. + /// Name: no wildcard support + /// Version: no wildcard support + /// Examples: Search "PowerShellGet" "2.2.5" + /// + public abstract Task FindVersionAsync(string packageName, string version, ResourceType type); + /// /// Find method which allows for searching for single name with specific version. /// Name: no wildcard support diff --git a/src/code/Utils.cs b/src/code/Utils.cs index 26d3ab25e..c4f2d1617 100644 --- a/src/code/Utils.cs +++ b/src/code/Utils.cs @@ -1342,37 +1342,85 @@ private static bool TryReadPSDataFile( out Hashtable dataFileInfo, out Exception error) { + dataFileInfo = null; + error = null; try { if (filePath is null) { throw new PSArgumentNullException(nameof(filePath)); } - string contents = System.IO.File.ReadAllText(filePath); - var scriptBlock = System.Management.Automation.ScriptBlock.Create(contents); - - // Ensure that the content script block is safe to convert into a PSDataFile Hashtable. - // This will throw for unsafe content. - scriptBlock.CheckRestrictedLanguage( - allowedCommands: allowedCommands, - allowedVariables: allowedVariables, - allowEnvironmentVariables: allowEnvironmentVariables); - - // Convert contents into PSDataFile Hashtable by executing content as script. - object result = scriptBlock.InvokeReturnAsIs(); - if (result is PSObject psObject) + + // Parallel.ForEach calls into this method. + // Each thread needs its own runspace created to provide a separate environment for operations to run independently. + Runspace runspace = RunspaceFactory.CreateRunspace(); + runspace.Open(); + runspace.SessionStateProxy.LanguageMode = PSLanguageMode.ConstrainedLanguage; + + // Set the created runspace as the default for the current thread + //Runspace.DefaultRunspace = runspace; + + using (System.Management.Automation.PowerShell pwsh = System.Management.Automation.PowerShell.Create()) { - result = psObject.BaseObject; - } + pwsh.Runspace = runspace; - dataFileInfo = (Hashtable)result; - error = null; - return true; + var cmd = new Command( + command: contents, + isScript: true, + useLocalScope: true); + cmd.MergeMyResults( + myResult: PipelineResultTypes.Error | PipelineResultTypes.Warning | PipelineResultTypes.Verbose | PipelineResultTypes.Debug | PipelineResultTypes.Information, + toResult: PipelineResultTypes.Output); + pwsh.Commands.AddCommand(cmd); + + + try + { + // Invoke the pipeline and retrieve the results + var results = pwsh.Invoke(); + + if (results[0] is PSObject pwshObj) + { + switch (pwshObj.BaseObject) + { + case ErrorRecord err: + //_cmdletPassedIn.WriteError(error); + break; + + case WarningRecord warning: + //cmdlet.WriteWarning(warning.Message); + break; + + case VerboseRecord verbose: + //cmdlet.WriteVerbose(verbose.Message); + break; + + case DebugRecord debug: + //cmdlet.WriteDebug(debug.Message); + break; + + case InformationRecord info: + //cmdlet.WriteInformation(info); + break; + + case Hashtable result: + dataFileInfo = result; + return true; + } + } + } + catch (Exception ex) + { + error = ex; + } + } + runspace.Close(); + + return false; } catch (Exception ex) { - dataFileInfo = null; error = ex; return false; } diff --git a/src/code/V2ServerAPICalls.cs b/src/code/V2ServerAPICalls.cs index ec3551f79..0313c4d33 100644 --- a/src/code/V2ServerAPICalls.cs +++ b/src/code/V2ServerAPICalls.cs @@ -397,6 +397,83 @@ public override FindResults FindName(string packageName, bool includePrerelease, return new FindResults(stringResponse: new string[]{ response }, hashtableResponse: emptyHashResponses, responseType: v2FindResponseType); } + /// + /// Find method which allows for searching for single name and returns latest version. + /// Name: no wildcard support + /// Examples: Search "PowerShellGet" + /// API call: + /// - No prerelease: http://www.powershellgallery.com/api/v2/FindPackagesById()?id='PowerShellGet' + /// - Include prerelease: http://www.powershellgallery.com/api/v2/FindPackagesById()?id='PowerShellGet' + /// Implementation Note: Need to filter further for latest version (prerelease or non-prerelease depending on user preference) + /// + public override async Task FindNameAsync(string packageName, bool includePrerelease, ResourceType type) + { + // TODO: pass all debug output into a list that can be written to console later. + //_cmdletPassedIn.WriteDebug("In V2ServerAPICalls::FindNameAsync()"); + // Make sure to include quotations around the package name + + // This should return the latest stable version or the latest prerelease version (respectively) + // https://www.powershellgallery.com/api/v2/FindPackagesById()?id='PowerShellGet'&$filter=IsLatestVersion and substringof('PSModule', Tags) eq true + // We need to explicitly add 'Id eq ' whenever $filter is used, otherwise arbitrary results are returned. + + var queryBuilder = new NuGetV2QueryBuilder(new Dictionary{ + { "$inlinecount", "allpages" }, + { "id", $"'{packageName}'" }, + }); + var filterBuilder = queryBuilder.FilterBuilder; + + // If it's a JFrog repository do not include the Id filter portion since JFrog uses 'Title' instead of 'Id', + // however filtering on 'and Title eq '' returns "Response status code does not indicate success: 500". + if (!_isJFrogRepo) { + filterBuilder.AddCriterion($"Id eq '{packageName}'"); + } + + filterBuilder.AddCriterion(includePrerelease ? "IsAbsoluteLatestVersion eq true" : "IsLatestVersion eq true"); + if (type != ResourceType.None) { + filterBuilder.AddCriterion(GetTypeFilterForRequest(type)); + } + + var requestUrlV2 = $"{Repository.Uri}/FindPackagesById()?{queryBuilder.BuildQueryString()}"; + string response; + try + { + response = await HttpRequestCallAsync(requestUrlV2); + } + catch (Exception e) + { + // usually this is for errors in calling the V2 server, but for ADO V2 this error will include package not found errors which we want to deliver in a standard message + if (_isADORepo && e is ResourceNotFoundException) + { + throw new ResourceNotFoundException($"Package with name '{packageName}' could not be found in repository '{Repository.Name}'. For ADO feed, if the package is in an upstream feed make sure you are authenticated to the upstream feed.", e); + } + else if (e is UnauthorizedException) + { + throw new UnauthorizedException($"Package with name '{packageName}' could not be retrieved from repository '{Repository.Name}'.", e); + } + else if (e is HttpRequestException) + { + throw new HttpRequestException($"Package with name '{packageName}' could not be retrieved from repository '{Repository.Name}'.", e); + } + else + { + throw new Exception($"Package with name '{packageName}' could not be retrieved from repository '{Repository.Name}'.", e); + } + } + + int count = GetCountFromResponse(response, out ErrorRecord errRecord); + if (errRecord != null) + { + throw new Exception($"Error retrieving count from response for package '{packageName}' from repository '{Repository.Name}'.", errRecord.Exception); + } + + if (count == 0) + { + throw new ResourceNotFoundException($"Package with name '{packageName}' could not be found in repository '{Repository.Name}'."); + } + + return new FindResults(stringResponse: new string[]{ response }, hashtableResponse: emptyHashResponses, responseType: v2FindResponseType); + } + /// /// Find method which allows for searching for single name and tag and returns latest version. /// Name: no wildcard support @@ -706,6 +783,72 @@ public override FindResults FindVersion(string packageName, string version, Reso return new FindResults(stringResponse: new string[] { response }, hashtableResponse: emptyHashResponses, responseType: v2FindResponseType); } + + public override async Task FindVersionAsync(string packageName, string version, ResourceType type) + { + //_cmdletPassedIn.WriteDebug("In V2ServerAPICalls::FindVersionAsync()"); + // https://www.powershellgallery.com/api/v2/FindPackagesById()?id='blah'&includePrerelease=false&$filter= NormalizedVersion eq '1.1.0' and substringof('PSModule', Tags) eq true + // Quotations around package name and version do not matter, same metadata gets returned. + // We need to explicitly add 'Id eq ' whenever $filter is used, otherwise arbitrary results are returned. + + var queryBuilder = new NuGetV2QueryBuilder(new Dictionary{ + { "$inlinecount", "allpages" }, + { "id", $"'{packageName}'" }, + }); + var filterBuilder = queryBuilder.FilterBuilder; + + // If it's a JFrog repository do not include the Id filter portion since JFrog uses 'Title' instead of 'Id', + // however filtering on 'and Title eq '' returns "Response status code does not indicate success: 500". + if (!_isJFrogRepo) { + filterBuilder.AddCriterion($"Id eq '{packageName}'"); + } + + filterBuilder.AddCriterion($"NormalizedVersion eq '{version}'"); + if (type != ResourceType.None) { + filterBuilder.AddCriterion(GetTypeFilterForRequest(type)); + } + + var requestUrlV2 = $"{Repository.Uri}/FindPackagesById()?{queryBuilder.BuildQueryString()}"; + string response; + try + { + response = await HttpRequestCallAsync(requestUrlV2); + } + catch (Exception e) + { + // usually this is for errors in calling the V2 server, but for ADO V2 this error will include package not found errors which we want to deliver with a standard message + if (_isADORepo && e is ResourceNotFoundException) + { + throw new ResourceNotFoundException($"Package with name '{packageName}' and version '{version}' could not be found in repository '{Repository.Name}'. For ADO feed, if the package is in an upstream feed make sure you are authenticated to the upstream feed.", e); + } + else if (e is UnauthorizedException) + { + throw new UnauthorizedException($"Package with name '{packageName}' could not be retrieved from repository '{Repository.Name}'.", e); + } + else if (e is HttpRequestException) + { + throw new HttpRequestException($"Package with name '{packageName}' could not be retrieved from repository '{Repository.Name}'.", e); + } + else + { + throw new Exception($"Package with name '{packageName}' could not be retrieved from repository '{Repository.Name}'.", e); + } + } + + int count = GetCountFromResponse(response, out ErrorRecord errRecord); + if (errRecord != null) + { + throw new Exception($"Error retrieving count from response for package '{packageName}' from repository '{Repository.Name}'.", errRecord.Exception); + } + + if (count == 0) + { + throw new ResourceNotFoundException($"Package with name '{packageName}', version '{version}' could not be found in repository '{Repository.Name}'."); + } + + return new FindResults(stringResponse: new string[] { response }, hashtableResponse: emptyHashResponses, responseType: v2FindResponseType); + } + /// /// Find method which allows for searching for single name with specific version and tag. /// Name: no wildcard support @@ -779,6 +922,7 @@ public override FindResults FindVersionWithTag(string packageName, string versio /// public override Stream InstallPackage(string packageName, string packageVersion, bool includePrerelease, out ErrorRecord errRecord) { + errRecord = null; Stream results = new MemoryStream(); if (string.IsNullOrEmpty(packageVersion)) { @@ -791,7 +935,7 @@ public override Stream InstallPackage(string packageName, string packageVersion, return results; } - results = InstallVersion(packageName, packageVersion, out errRecord); + results = InstallVersionAsync(packageName, packageVersion).GetAwaiter().GetResult(); return results; } @@ -852,18 +996,91 @@ private string HttpRequestCall(string requestUrlV2, out ErrorRecord errRecord) return response; } + /// + /// Helper method that makes the HTTP request for the V2 server protocol url passed in for find APIs. + /// + private async Task HttpRequestCallAsync(string requestUrlV2) + { + // TODO: Async methods cannot have out ref, so currently handling errorRecords as thrown exceptions. + string response = string.Empty; + + try + { + HttpRequestMessage request = new HttpRequestMessage(HttpMethod.Get, requestUrlV2); + + response = await SendV2RequestAsync(request, _sessionClient); + } + catch (ResourceNotFoundException) + { + throw new ResourceNotFoundException($"Failure when attempting to make request for '${requestUrlV2}'"); + } + catch (UnauthorizedException) + { + throw new UnauthorizedException($"Failure when attempting to make request for '${requestUrlV2}'"); + } + catch (HttpRequestException) + { + throw new HttpRequestException($"Failure when attempting to make request for '${requestUrlV2}'"); + } + catch (Exception) + { + throw new Exception($"Failure when attempting to make request for '${requestUrlV2}'"); + } + + if (string.IsNullOrEmpty(response)) + { + throw new ArgumentException($"Response returned is null or empty. Request made: '${requestUrlV2}'"); + } + + return response; + } + /// /// Helper method that makes the HTTP request for the V2 server protocol url passed in for install APIs. /// + private async Task HttpRequestCallForContentAsync(string requestUrlV2) + { + // TODO: Async methods cannot have out ref, so need to handle errorRecords a different way. + // _cmdletPassedIn.WriteDebug("In V2ServerAPICalls::HttpRequestCallForContentAsync()"); + HttpContent content = null; + + try + { + //_cmdletPassedIn.WriteDebug($"Request url is '{requestUrlV2}'"); + HttpRequestMessage request = new HttpRequestMessage(HttpMethod.Get, requestUrlV2); + + content = await SendV2RequestForContentAsync(request, _sessionClient); + } + catch (HttpRequestException) + { + throw new HttpRequestException($"Failure when attempting to make request for content with '${requestUrlV2}'"); + } + catch (ArgumentNullException) + { + throw new ArgumentNullException($"Failure when attempting to make request for content with '${requestUrlV2}'"); + } + catch (InvalidOperationException) + { + throw new InvalidOperationException($"Failure when attempting to make request for content with '${requestUrlV2}'"); + } + + if (content == null || string.IsNullOrEmpty(content.ToString())) + { + throw new ArgumentException($"Response returned is null or empty. Request for content made: '${requestUrlV2}'"); + } + + return content; + } + private HttpContent HttpRequestCallForContent(string requestUrlV2, out ErrorRecord errRecord) { - _cmdletPassedIn.WriteDebug("In V2ServerAPICalls::HttpRequestCallForContent()"); + // _cmdletPassedIn.WriteDebug("In V2ServerAPICalls::HttpRequestCallForContent()"); errRecord = null; HttpContent content = null; try { - _cmdletPassedIn.WriteDebug($"Request url is '{requestUrlV2}'"); + //_cmdletPassedIn.WriteDebug($"Request url is '{requestUrlV2}'"); HttpRequestMessage request = new HttpRequestMessage(HttpMethod.Get, requestUrlV2); content = SendV2RequestForContentAsync(request, _sessionClient).GetAwaiter().GetResult(); @@ -895,7 +1112,7 @@ private HttpContent HttpRequestCallForContent(string requestUrlV2, out ErrorReco if (content == null || string.IsNullOrEmpty(content.ToString())) { - _cmdletPassedIn.WriteDebug("Response is empty"); + //_cmdletPassedIn.WriteDebug("Response is empty"); } return content; @@ -1288,7 +1505,7 @@ private string FindNameGlobbingWithTag(string packageName, string[] tags, Resour /// private string FindVersionGlobbing(string packageName, VersionRange versionRange, bool includePrerelease, ResourceType type, int skip, bool getOnlyLatest, out ErrorRecord errRecord) { - _cmdletPassedIn.WriteDebug("In V2ServerAPICalls::FindVersionGlobbing()"); + //_cmdletPassedIn.WriteDebug("In V2ServerAPICalls::FindVersionGlobbing()"); //https://www.powershellgallery.com/api/v2//FindPackagesById()?id='blah'&includePrerelease=false&$filter= NormalizedVersion gt '1.0.0' and NormalizedVersion lt '2.2.5' and substringof('PSModule', Tags) eq true //https://www.powershellgallery.com/api/v2//FindPackagesById()?id='PowerShellGet'&includePrerelease=false&$filter= NormalizedVersion gt '1.1.1' and NormalizedVersion lt '2.2.5' // NormalizedVersion doesn't include trailing zeroes @@ -1384,6 +1601,153 @@ private string FindVersionGlobbing(string packageName, VersionRange versionRange return HttpRequestCall(requestUrlV2, out errRecord); } + + public override async Task FindVersionGlobbingAsync(string packageName, VersionRange versionRange, bool includePrerelease, ResourceType type, bool getOnlyLatest) + { + //_cmdletPassedIn.WriteDebug("In V2ServerAPICalls::FindVersionGlobbing()"); + List responses = new List(); + int skip = 0; + + var initialResponse = await FindVersionGlobbingAsync(packageName, versionRange, includePrerelease, type, skip, getOnlyLatest); + + int initialCount = GetCountFromResponse(initialResponse, out ErrorRecord errRecord); + if (errRecord != null) + { + return new FindResults(stringResponse: Utils.EmptyStrArray, hashtableResponse: emptyHashResponses, responseType: v2FindResponseType); + } + + if (initialCount == 0) + { + throw new ResourceNotFoundException($"Package with name '{packageName}' could not be found in repository '{Repository.Name}'."); + } + + responses.Add(initialResponse); + + if (!getOnlyLatest) + { + int count = (int)Math.Ceiling((double)(initialCount / 100)); + + while (count > 0) + { + //_cmdletPassedIn.WriteDebug($"Count is '{count}'"); + // skip 100 + skip += 100; + var tmpResponse = FindVersionGlobbing(packageName, versionRange, includePrerelease, type, skip, getOnlyLatest, out errRecord); + if (errRecord != null) + { + return new FindResults(stringResponse: Utils.EmptyStrArray, hashtableResponse: emptyHashResponses, responseType: v2FindResponseType); + } + responses.Add(tmpResponse); + count--; + } + } + + return new FindResults(stringResponse: responses.ToArray(), hashtableResponse: emptyHashResponses, responseType: v2FindResponseType); + } + + + /// + /// Helper method for string[] FindVersionGlobbing() + /// + private async Task FindVersionGlobbingAsync(string packageName, VersionRange versionRange, bool includePrerelease, ResourceType type, int skip, bool getOnlyLatest) + { + //_cmdletPassedIn.WriteDebug("In V2ServerAPICalls::FindVersionGlobbing()"); + //https://www.powershellgallery.com/api/v2//FindPackagesById()?id='blah'&includePrerelease=false&$filter= NormalizedVersion gt '1.0.0' and NormalizedVersion lt '2.2.5' and substringof('PSModule', Tags) eq true + //https://www.powershellgallery.com/api/v2//FindPackagesById()?id='PowerShellGet'&includePrerelease=false&$filter= NormalizedVersion gt '1.1.1' and NormalizedVersion lt '2.2.5' + // NormalizedVersion doesn't include trailing zeroes + // Notes: this could allow us to take a version range (i.e (2.0.0, 3.0.0.0]) and deconstruct it and add options to the Filter for Version to describe that range + // will need to filter additionally, if IncludePrerelease=false, by default we get stable + prerelease both back + // Current bug: Find PSGet -Version "2.0.*" -> https://www.powershellgallery.com/api/v2//FindPackagesById()?id='PowerShellGet'&includePrerelease=false&$filter= Version gt '2.0.*' and Version lt '2.1' + // Make sure to include quotations around the package name + + //and IsPrerelease eq false + // ex: + // (2.0.0, 3.0.0) + // $filter= NVersion gt '2.0.0' and NVersion lt '3.0.0' + + // [2.0.0, 3.0.0] + // $filter= NVersion ge '2.0.0' and NVersion le '3.0.0' + + // [2.0.0, 3.0.0) + // $filter= NVersion ge '2.0.0' and NVersion lt '3.0.0' + + // (2.0.0, 3.0.0] + // $filter= NVersion gt '2.0.0' and NVersion le '3.0.0' + + // [, 2.0.0] + // $filter= NVersion le '2.0.0' + + string format = "NormalizedVersion {0} {1}"; + string minPart = String.Empty; + string maxPart = String.Empty; + + var queryBuilder = new NuGetV2QueryBuilder(new Dictionary { + {"$inlinecount", "allpages"}, + {"$skip", skip.ToString()}, + {"$orderby", "NormalizedVersion desc"}, + {"id", $"'{packageName}'"} + }); + + var filterBuilder = queryBuilder.FilterBuilder; + + if (versionRange.MinVersion != null) + { + string operation = versionRange.IsMinInclusive ? "ge" : "gt"; + minPart = String.Format(format, operation, $"'{versionRange.MinVersion.ToNormalizedString()}'"); + } + + if (versionRange.MaxVersion != null) + { + string operation = versionRange.IsMaxInclusive ? "le" : "lt"; + // Adding '9' as a digit to the end of the patch portion of the version + // because we want to retrieve all the prerelease versions for the upper end of the range + // and PSGallery views prerelease as higher than its stable. + // eg 3.0.0-prerelease > 3.0.0 + // If looking for versions within '[1.9.9,1.9.9]' including prerelease values, this will change it to search for '[1.9.9,1.9.99]' + // and find any pkg versions that are 1.9.9-prerelease. + string maxString = includePrerelease ? $"{versionRange.MaxVersion.Major}.{versionRange.MaxVersion.Minor}.{versionRange.MaxVersion.Patch.ToString() + "9"}" : + $"{versionRange.MaxVersion.ToNormalizedString()}"; + if (NuGetVersion.TryParse(maxString, out NuGetVersion maxVersion)) + { + maxPart = String.Format(format, operation, $"'{maxVersion.ToNormalizedString()}'"); + } + else { + maxPart = String.Format(format, operation, $"'{versionRange.MaxVersion.ToNormalizedString()}'"); + } + } + + string versionFilterParts = String.Empty; + if (!String.IsNullOrEmpty(minPart)) + { + filterBuilder.AddCriterion(minPart); + } + if (!String.IsNullOrEmpty(maxPart)) + { + filterBuilder.AddCriterion(maxPart); + } + if (!includePrerelease) { + filterBuilder.AddCriterion("IsPrerelease eq false"); + } + + // We need to explicitly add 'Id eq ' whenever $filter is used, otherwise arbitrary results are returned. + + // If it's a JFrog repository do not include the Id filter portion since JFrog uses 'Title' instead of 'Id', + // however filtering on 'and Title eq '' returns "Response status code does not indicate success: 500". + if (!_isJFrogRepo) { + filterBuilder.AddCriterion($"Id eq '{packageName}'"); + } + + if (type == ResourceType.Script) { + filterBuilder.AddCriterion($"substringof('PS{type.ToString()}', Tags) eq true"); + } + + + var requestUrlV2 = $"{Repository.Uri}/FindPackagesById()?{queryBuilder.BuildQueryString()}"; + + return await HttpRequestCallAsync(requestUrlV2); + } + + /// /// Installs package with specific name and version. /// Name: no wildcard support. @@ -1394,7 +1758,7 @@ private string FindVersionGlobbing(string packageName, VersionRange versionRange /// private Stream InstallVersion(string packageName, string version, out ErrorRecord errRecord) { - _cmdletPassedIn.WriteDebug("In V2ServerAPICalls::InstallVersion()"); + //_cmdletPassedIn.WriteDebug("In V2ServerAPICalls::InstallVersion()"); string requestUrlV2; if (_isADORepo) @@ -1433,6 +1797,38 @@ private Stream InstallVersion(string packageName, string version, out ErrorRecor return response.ReadAsStreamAsync().Result; } + private async Task InstallVersionAsync(string packageName, string version) + { + //_cmdletPassedIn.WriteDebug("In V2ServerAPICalls::InstallVersionAsync()"); + string requestUrlV2; + + if (_isADORepo) + { + // eg: https://pkgs.dev.azure.com///_packaging//nuget/v2?id=test_module&version=5.0.0 + requestUrlV2 = $"{Repository.Uri}?id={packageName.ToLower()}&version={version}"; + } + else if (_isJFrogRepo) + { + // eg: https://.jfrog.io/artifactory/api/nuget//Download/test_module/5.0.0 + requestUrlV2 = $"{Repository.Uri}/Download/{packageName}/{version}"; + } + else + { + requestUrlV2 = $"{Repository.Uri}/package/{packageName}/{version}"; + } + + var response = await HttpRequestCallForContentAsync(requestUrlV2); + + if (response is null) + { + throw new Exception($"No content was returned by repository '{Repository.Name}'"); + } + + var retResponse = await response.ReadAsStreamAsync(); + + return retResponse; + } + private string GetTypeFilterForRequest(ResourceType type) { string typeFilterPart = string.Empty; if (type == ResourceType.Script) @@ -1495,7 +1891,11 @@ public int GetCountFromResponse(string httpResponse, out ErrorRecord errRecord) } else { - _cmdletPassedIn.WriteDebug($"Property 'count' and 'd:Id' could not be found in response. This may indicate that the package could not be found"); + errRecord = new ErrorRecord( + new PSArgumentException("Property 'count' and 'd:Id' could not be found in response. This may indicate that the package could not be found"), + "GetCountFromResponseFailure", + ErrorCategory.InvalidData, + this); } } } diff --git a/src/code/V3ServerAPICalls.cs b/src/code/V3ServerAPICalls.cs index 903b1da55..edd88ac27 100644 --- a/src/code/V3ServerAPICalls.cs +++ b/src/code/V3ServerAPICalls.cs @@ -88,6 +88,16 @@ public V3ServerAPICalls(PSRepositoryInfo repository, PSCmdlet cmdletPassedIn, Ne #region Overridden Methods + public override Task FindVersionAsync(string packageName, string version, ResourceType type) + { + throw new NotImplementedException("FindVersionAsync is not implemented for V3ServerAPICalls."); + } + + public override Task FindVersionGlobbingAsync(string packageName, VersionRange versionRange, bool includePrerelease, ResourceType type, bool getOnlyLatest) + { + throw new NotImplementedException("FindVersionAsync is not implemented for V3ServerAPICalls."); + } + /// /// Find method which allows for searching for all packages from a repository and returns latest version for each. /// Not supported for V3 repository. @@ -155,6 +165,11 @@ public override FindResults FindName(string packageName, bool includePrerelease, return FindNameHelper(packageName, tags: Utils.EmptyStrArray, includePrerelease, type, out errRecord); } + public override Task FindNameAsync(string packageName, bool includePrerelease, ResourceType type) + { + throw new NotImplementedException("FindVersionAsync is not implemented for V3ServerAPICalls."); + } + /// /// Find method which allows for searching for single name and specified tag(s) and returns latest version. /// Name: no wildcard support diff --git a/test/FindPSResourceTests/FindPSResourceADOV2Server.Tests.ps1 b/test/FindPSResourceTests/FindPSResourceADOV2Server.Tests.ps1 index c45e6d05a..f419bfa7a 100644 --- a/test/FindPSResourceTests/FindPSResourceADOV2Server.Tests.ps1 +++ b/test/FindPSResourceTests/FindPSResourceADOV2Server.Tests.ps1 @@ -190,7 +190,7 @@ Describe 'Test HTTP Find-PSResource for ADO V2 Server Protocol' -tags 'CI' { $res = Find-PSResource -Name $testModuleName -Tag $requiredTag -Repository $ADOV2RepoName -ErrorVariable err -ErrorAction SilentlyContinue $res | Should -BeNullOrEmpty $err.Count | Should -BeGreaterThan 0 - $err[0].FullyQualifiedErrorId | Should -BeExactly "PackageNotFound,Microsoft.PowerShell.PSResourceGet.Cmdlets.FindPSResource" + $err[0].FullyQualifiedErrorId | Should -BeExactly "GetCountFromResponseFailure,Microsoft.PowerShell.PSResourceGet.Cmdlets.FindPSResource" } It "Find resource that satisfies given Name and Tag property (multiple tags)" { @@ -208,7 +208,7 @@ Describe 'Test HTTP Find-PSResource for ADO V2 Server Protocol' -tags 'CI' { $res = Find-PSResource -Name $testModuleName -Tag $requiredTags -Repository $ADOV2RepoName -ErrorVariable err -ErrorAction SilentlyContinue $res | Should -BeNullOrEmpty $err.Count | Should -BeGreaterThan 0 - $err[0].FullyQualifiedErrorId | Should -BeExactly "PackageNotFound,Microsoft.PowerShell.PSResourceGet.Cmdlets.FindPSResource" + $err[0].FullyQualifiedErrorId | Should -BeExactly "GetCountFromResponseFailure,Microsoft.PowerShell.PSResourceGet.Cmdlets.FindPSResource" } It "Find all resources that satisfy Name pattern and have specified Tag (single tag)" { @@ -244,7 +244,7 @@ Describe 'Test HTTP Find-PSResource for ADO V2 Server Protocol' -tags 'CI' { $res = Find-PSResource -Name $testModuleName -Version "5.0.0" -Tag $requiredTag -Repository $ADOV2RepoName -ErrorVariable err -ErrorAction SilentlyContinue $res | Should -BeNullOrEmpty $err.Count | Should -BeGreaterThan 0 - $err[0].FullyQualifiedErrorId | Should -BeExactly "PackageNotFound,Microsoft.PowerShell.PSResourceGet.Cmdlets.FindPSResource" + $err[0].FullyQualifiedErrorId | Should -BeExactly "GetCountFromResponseFailure,Microsoft.PowerShell.PSResourceGet.Cmdlets.FindPSResource" } It "Find resource that satisfies given Name, Version and Tag property (multiple tags)" { @@ -264,6 +264,6 @@ Describe 'Test HTTP Find-PSResource for ADO V2 Server Protocol' -tags 'CI' { $res = Find-PSResource -Name $testModuleName -Version "5.0.0" -Tag $requiredTags -Repository $ADOV2RepoName -ErrorVariable err -ErrorAction SilentlyContinue $res | Should -BeNullOrEmpty $err.Count | Should -BeGreaterThan 0 - $err[0].FullyQualifiedErrorId | Should -BeExactly "PackageNotFound,Microsoft.PowerShell.PSResourceGet.Cmdlets.FindPSResource" + $err[0].FullyQualifiedErrorId | Should -BeExactly "GetCountFromResponseFailure,Microsoft.PowerShell.PSResourceGet.Cmdlets.FindPSResource" } } diff --git a/test/FindPSResourceTests/FindPSResourceLocal.Tests.ps1 b/test/FindPSResourceTests/FindPSResourceLocal.Tests.ps1 index a3cbe2335..bdaed980b 100644 --- a/test/FindPSResourceTests/FindPSResourceLocal.Tests.ps1 +++ b/test/FindPSResourceTests/FindPSResourceLocal.Tests.ps1 @@ -1,350 +1,350 @@ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT License. - -$modPath = "$psscriptroot/../PSGetTestUtils.psm1" -Import-Module $modPath -Force -Verbose - -$psmodulePaths = $env:PSModulePath -split ';' -Write-Verbose -Verbose "Current module search paths: $psmodulePaths" - -Describe 'Test Find-PSResource for local repositories' -tags 'CI' { - - BeforeAll{ - $localRepo = "psgettestlocal" - $localUNCRepo = 'psgettestlocal3' - $testModuleName = "test_local_mod" - $testModuleName2 = "test_local_mod2" - $testModuleName3 = "Test_Local_Mod3" - $similarTestModuleName = "test_local_mod.similar" - $commandName = "cmd1" - $dscResourceName = "dsc1" - $prereleaseLabel = "" - $localNupkgRepo = "localNupkgRepo" - Get-NewPSResourceRepositoryFile - Register-LocalRepos - Register-LocalTestNupkgsRepo - - $localRepoUriAddress = Join-Path -Path $TestDrive -ChildPath "testdir" - $tagsEscaped = @("'Test'", "'Tag2'", "'PSCommand_$cmdName'", "'PSDscResource_$dscName'") - $prereleaseLabel = "alpha001" - - New-TestModule -moduleName $testModuleName -repoName $localRepo -packageVersion "1.0.0" -prereleaseLabel "" -tags @() - New-TestModule -moduleName $testModuleName -repoName $localRepo -packageVersion "3.0.0" -prereleaseLabel "" -tags @() -dscResourceToExport $dscResourceName -commandToExport $commandName - New-TestModule -moduleName $testModuleName -repoName $localRepo -packageVersion "5.0.0" -prereleaseLabel "" -tags $tagsEscaped - New-TestModule -moduleName $testModuleName -repoName $localRepo -packageVersion "5.2.5" -prereleaseLabel $prereleaseLabel -tags $tagsEscaped - - New-TestModule -moduleName $testModuleName2 -repoName $localRepo -packageVersion "5.0.0" -prereleaseLabel "" -tags $tagsEscaped - New-TestModule -moduleName $testModuleName2 -repoName $localRepo -packageVersion "5.2.5" -prereleaseLabel $prereleaseLabel -tags $tagsEscaped - - New-TestModule -moduleName $testModuleName3 -repoName $localRepo -packageVersion "1.0.0" -prereleaseLabel "" -tags @() - - New-TestModule -moduleName $similarTestModuleName -repoName $localRepo -packageVersion "4.0.0" -prereleaseLabel "" -tags $tagsEscaped - New-TestModule -moduleName $similarTestModuleName -repoName $localRepo -packageVersion "5.0.0" -prereleaseLabel "" -tags $tagsEscaped - } - - AfterAll { - Get-RevertPSResourceRepositoryFile - } - - It "find resource given specific Name, Version null (module)" { - # FindName() - $res = Find-PSResource -Name $testModuleName -Repository $localRepo - $res.Name | Should -Be $testModuleName - $res.Version | Should -Be "5.0.0" - } - - It "find resource given specific Name with incorrect casing (should return correct casing)" { - # FindName() - $res = Find-PSResource -Name "test_local_mod3" -Repository $localRepo - $res.Name | Should -Be $testModuleName3 - $res.Version | Should -Be "1.0.0" - } - - It "find resource given specific Name with incorrect casing and Version (should return correct casing)" { - # FindVersion() - $res = Find-PSResource -Name "test_local_mod3" -Version "1.0.0" -Repository $localRepo - $res.Name | Should -Be $testModuleName3 - $res.Version | Should -Be "1.0.0" - } - - It "find resource given specific Name, Version null (module) from a UNC-based local repository" { - # FindName() - $res = Find-PSResource -Name $testModuleName -Repository $localUNCRepo - $res.Name | Should -Be $testModuleName - $res.Version | Should -Be "5.0.0" - } - - It "find resource given Name, Version null (package containing nuspec only)" { - # FindName() - $pkgName = "PowerShell" - Save-PSResource -Name $pkgName -Repository "NuGetGallery" -Path $localRepoUriAddress -AsNupkg -TrustRepository - $res = Find-PSResource -Name $pkgName -Repository $localRepo - $res.Name | Should -Be $pkgName - $res.Repository | Should -Be $localRepo - } - - It "find script without RequiredModules" { - # FindName() - $pkgName = "Required-Script1" - $requiredTag = "Tag1" - Save-PSResource -Name $pkgName -Repository "PSGallery" -Path $localRepoUriAddress -AsNupkg -TrustRepository - # $res = Find-PSResource -Name $pkgName -Repository $localRepo - # $res.Name | Should -Be $pkgName - # $res.Repository | Should -Be $localRepo - # $res.Tags | Should -Contain $requiredTag - } - - It "should not find resource given nonexistent Name" { - # FindName() - $res = Find-PSResource -Name NonExistentModule -Repository $localRepo -ErrorVariable err -ErrorAction SilentlyContinue - $res | Should -BeNullOrEmpty - $err.Count | Should -Not -Be 0 - $err[0].FullyQualifiedErrorId | Should -BeExactly "PackageNotFound,Microsoft.PowerShell.PSResourceGet.Cmdlets.FindPSResource" - $res | Should -BeNullOrEmpty - } - - It "find resource given specific Name when another package with similar name (with period) exists" { - # FindName() - $res = Find-PSResource -Name $testModuleName -Repository $localRepo - $res.Name | Should -Be $testModuleName - $res.Version | Should -Be "5.0.0" - - $res = Find-PSResource -Name $similarTestModuleName -Repository $localRepo - $res.Name | Should -Be $similarTestModuleName - $res.Version | Should -Be "5.0.0" - } - - It "find resource(s) given wildcard Name" { - # FindNameGlobbing - $res = Find-PSResource -Name "test_local_*" -Repository $localRepo - $res.Count | Should -BeGreaterThan 1 - } - - $testCases2 = @{Version="[5.0.0.0]"; ExpectedVersions=@("5.0.0"); Reason="validate version, exact match"}, - @{Version="5.0.0.0"; ExpectedVersions=@("5.0.0"); Reason="validate version, exact match without bracket syntax"}, - @{Version="[1.0.0.0, 5.0.0.0]"; ExpectedVersions=@("1.0.0", "3.0.0", "5.0.0"); Reason="validate version, exact range inclusive"}, - @{Version="(1.0.0.0, 5.0.0.0)"; ExpectedVersions=@("3.0.0"); Reason="validate version, exact range exclusive"}, - @{Version="(1.0.0.0,)"; ExpectedVersions=@("3.0.0", "5.0.0"); Reason="validate version, minimum version exclusive"}, - @{Version="[1.0.0.0,)"; ExpectedVersions=@("1.0.0", "3.0.0", "5.0.0"); Reason="validate version, minimum version inclusive"}, - @{Version="(,3.0.0.0)"; ExpectedVersions=@("1.0.0"); Reason="validate version, maximum version exclusive"}, - @{Version="(,3.0.0.0]"; ExpectedVersions=@("1.0.0", "3.0.0"); Reason="validate version, maximum version inclusive"}, - @{Version="[1.0.0.0, 5.0.0.0)"; ExpectedVersions=@("1.0.0", "3.0.0"); Reason="validate version, mixed inclusive minimum and exclusive maximum version"} - @{Version="(1.0.0.0, 5.0.0.0]"; ExpectedVersions=@("3.0.0", "5.0.0"); Reason="validate version, mixed exclusive minimum and inclusive maximum version"} - - It "find resource when given Name to " -TestCases $testCases2{ - # FindVersionGlobbing() - param($Version, $ExpectedVersions) - $res = Find-PSResource -Name $testModuleName -Version $Version -Repository $localRepo - foreach ($item in $res) { - $item.Name | Should -Be $testModuleName - $ExpectedVersions | Should -Contain $item.Version - } - } - - It "find all versions of resource when given specific Name, Version not null --> '*'" { - # FindVersionGlobbing() - $res = Find-PSResource -Name $testModuleName -Version "*" -Repository $localRepo - $res | ForEach-Object { - $_.Name | Should -Be $testModuleName - } - - $res.Count | Should -BeGreaterOrEqual 1 - } - - It "find resource with latest (including prerelease) version given Prerelease parameter" { - # FindName() - # test_module resource's latest version is a prerelease version, before that it has a non-prerelease version - $res = Find-PSResource -Name $testModuleName -Repository $localRepo - $res.Version | Should -Be "5.0.0" - - $resPrerelease = Find-PSResource -Name $testModuleName -Prerelease -Repository $localRepo - $resPrerelease.Version | Should -Be "5.2.5" - $resPrerelease.Prerelease | Should -Be "alpha001" - } - - It "find resource given specific Name when another package with similar name (with period) exists" { - # FindVersion() - # Package $testModuleName version 4.0.0 does not exist - # previously if Find-PSResource -Version against local repo did not find that package's version it kept looking at - # similar named packages and would fault. This test is to ensure only the specified package and its version is checked - $res = Find-PSResource -Name $testModuleName -Version "4.0.0" -Repository $localRepo -ErrorVariable err -ErrorAction SilentlyContinue - $res | Should -BeNullOrEmpty - $err.Count | Should -Not -Be 0 - $err[0].FullyQualifiedErrorId | Should -BeExactly "PackageNotFound,Microsoft.PowerShell.PSResourceGet.Cmdlets.FindPSResource" - $res | Should -BeNullOrEmpty - - $res = Find-PSResource -Name $similarTestModuleName -Version "4.0.0" -Repository $localRepo - $res.Name | Should -Be $similarTestModuleName - $res.Version | Should -Be "4.0.0" - } - - It "find resources, including Prerelease version resources, when given Prerelease parameter" { - # FindVersionGlobbing() - $resWithoutPrerelease = Find-PSResource -Name $testModuleName -Version "*" -Repository $localRepo - $resWithPrerelease = Find-PSResource -Name $testModuleName -Version "*" -Repository $localRepo - $resWithPrerelease.Count | Should -BeGreaterOrEqual $resWithoutPrerelease.Count - } - - It "find resource that satisfies given Name and Tag property (single tag)" { - # FindNameWithTag() - $requiredTag = "test" - $res = Find-PSResource -Name $testModuleName -Tag $requiredTag -Repository $localRepo - $res.Name | Should -Be $testModuleName - $res.Tags | Should -Contain $requiredTag - } - - It "should not find resource if Name and Tag are not both satisfied (single tag)" { - # FindNameWithTag - $requiredTag = "Windows" # tag "windows" is not present for test_module package - $res = Find-PSResource -Name $testModuleName -Tag $requiredTag -Repository $localRepo -ErrorVariable err -ErrorAction SilentlyContinue - $res | Should -BeNullOrEmpty - $err.Count | Should -Not -Be 0 - $err[0].FullyQualifiedErrorId | Should -BeExactly "PackageNotFound,Microsoft.PowerShell.PSResourceGet.Cmdlets.FindPSResource" - $res | Should -BeNullOrEmpty - } - - It "find resource that satisfies given Name and Tag property (multiple tags)" { - # FindNameWithTag() - $requiredTags = @("test", "Tag2") - $res = Find-PSResource -Name $testModuleName -Tag $requiredTags -Repository $localRepo - $res.Name | Should -Be $testModuleName - $res.Tags | Should -Contain $requiredTags[0] - $res.Tags | Should -Contain $requiredTags[1] - } - - It "find all resources that satisfy Name pattern and have specified Tag (single tag)" { - # FindNameGlobbingWithTag() - $requiredTag = "test" - $nameWithWildcard = "test_local_mod*" - - $res = Find-PSResource -Name $nameWithWildcard -Tag $requiredTag -Repository $localRepo - $res.Count | Should -BeGreaterThan 1 - foreach ($pkg in $res) - { - $pkg.Name | Should -BeLike $nameWithWildcard - $pkg.Tags | Should -Contain "$requiredTag" - } - } - - It "should not find resources if both Name pattern and Tags are not satisfied (single tag)" { - # FindNameGlobbingWithTag() - $requiredTag = "windows" # tag "windows" is not present for test_module package - $res = Find-PSResource -Name "test_module*" -Tag $requiredTag -Repository $localRepo - $res | Should -BeNullOrEmpty - } - - It "find all resources that satisfy Name pattern and have specified Tag (multiple tags)" { - # FindNameGlobbingWithTag() - $requiredTags = @("test", "Tag2") - $nameWithWildcard = "test_local_mod*" - - $res = Find-PSResource -Name $nameWithWildcard -Tag $requiredTags -Repository $localRepo - $res.Count | Should -BeGreaterThan 1 - foreach ($pkg in $res) - { - $pkg.Name | Should -BeLike $nameWithWildcard - $pkg.Tags | Should -Contain $requiredTags[0] - $pkg.Tags | Should -Contain $requiredTags[1] - } - } - - It "find resource that satisfies given Name, Version and Tag property (single tag)" { - # FindVersionWithTag() - $requiredTag = "test" - $res = Find-PSResource -Name $testModuleName -Version "5.0.0.0" -Tag $requiredTag -Repository $localRepo - $res.Name | Should -Be $testModuleName - $res.Version | Should -Be "5.0.0" - $res.Tags | Should -Contain $requiredTag - } - - It "should not find resource if Name, Version and Tag property are not all satisfied (single tag)" { - # FindVersionWithTag() - $requiredTag = "windows" # tag "windows" is not present for test_module package - $res = Find-PSResource -Name $testModuleName -Version "5.0.0.0" -Tag $requiredTag -Repository $localRepo -ErrorVariable err -ErrorAction SilentlyContinue - $res | Should -BeNullOrEmpty - $err.Count | Should -Not -Be 0 - $err[0].FullyQualifiedErrorId | Should -BeExactly "PackageNotFound,Microsoft.PowerShell.PSResourceGet.Cmdlets.FindPSResource" - $res | Should -BeNullOrEmpty - } - - It "find resource that satisfies given Name, Version and Tag property (multiple tags)" { - # FindVersionWithTag() - $requiredTags = @("test", "Tag2") - $res = Find-PSResource -Name $testModuleName -Version "5.0.0.0" -Tag $requiredTags -Repository $localRepo - $res.Name | Should -Be $testModuleName - $res.Version | Should -Be "5.0.0" - $res.Tags | Should -Contain $requiredTags[0] - $res.Tags | Should -Contain $requiredTags[1] - } - - It "find scripts given -Type parameter" { - Get-ScriptResourcePublishedToLocalRepoTestDrive "testScriptName" $localRepo "1.0.0" - - $res = Find-PSResource -Type Script -Repository $localRepo - $res | Should -Not -BeNullOrEmpty - $res.Count | Should -Be 2 - $res.Type | Should -Be @("Script", "Script") - } - - It "find modules given -Type parameter" { - Get-ScriptResourcePublishedToLocalRepoTestDrive "testScriptName" $localRepo "1.0.0" - - $res = Find-PSResource -Type Module -Repository $localRepo - $res | Should -Not -BeNullOrEmpty - $res.Count | Should -BeGreaterOrEqual 1 - foreach ($module in $res) { - $module.Type | Should -Be "Module" - } - } - - It "find resource given CommandName" -Pending { - $res = Find-PSResource -CommandName $commandName -Repository $localRepo - $res | Should -Not -BeNullOrEmpty - foreach ($item in $res) { - $item.Names | Should -Be $commandName - $item.ParentResource.Includes.Command | Should -Contain $commandName - } - } - - It "find resource given DscResourceName" -Pending { - $res = Find-PSResource -DscResourceName $dscResourceName -Repository $localRepo - $res | Should -Not -BeNullOrEmpty - foreach ($item in $res) { - $item.Names | Should -Be $dscResourceName - $item.ParentResource.Includes.DscResource | Should -Contain $dscResourceName - } - } - - It "Get definition for alias 'fdres'" { - (Get-Alias fdres).Definition | Should -BeExactly 'Find-PSResource' - } - - It "not find resource with tag value that is non-existent for the packages" { - # Since this is pattern matching based search no error should be written out. - $res = Find-PSResource -Tag "nonexistenttag" -Repository $localRepo -ErrorVariable err -ErrorAction SilentlyContinue - $res | Should -BeNullOrEmpty - $err.Count | Should -Not -Be 0 - $err[0].FullyQualifiedErrorId | Should -BeExactly "FindTagsPackageNotFound,Microsoft.PowerShell.PSResourceGet.Cmdlets.FindPSResource" - } - - It "find package where prerelease label includes digits and period (i.e prerelease label is not just words)" { - $nupkgName = "WebView2.Avalonia" - $nupkgVersion = "1.0.1518.46" - $prereleaseLabel = "preview.230207.17" - $res = Find-PSResource -Name $nupkgName -Prerelease -Repository $localNupkgRepo - $res.Name | Should -Be $nupkgName - $res.Version | Should -Be $nupkgVersion - $res.Prerelease | Should -Be $prereleaseLabel - } - - It "find module that has multiple manifests and use exact name match one" { - # Az.KeyVault has 2 manifest files - Az.KeyVault.psd1 and Az.KeyVault.Extension.psd1 - # this test was added because PSResourceGet would previously pick the .psd1 file by pattern matching the module name, not exact matching it - # this meant Az.KeyVault.Extension.psd1 and its metadata was being returned. - # The package is present on PSGallery but issue reproduces when looking at the package's file paths in local repo - $PSGalleryName = Get-PSGalleryName - Save-PSResource -Name 'Az.KeyVault' -Version '6.3.1' -Repository $PSGalleryName -AsNupkg -Path $localRepoUriAddress -TrustRepository - $res = Find-PSResource -Name 'Az.KeyVault' -Repository $localRepo - $res.Version | Should -Be "6.3.1" - } -} +# # Copyright (c) Microsoft Corporation. +# # Licensed under the MIT License. + +# $modPath = "$psscriptroot/../PSGetTestUtils.psm1" +# Import-Module $modPath -Force -Verbose + +# $psmodulePaths = $env:PSModulePath -split ';' +# Write-Verbose -Verbose "Current module search paths: $psmodulePaths" + +# Describe 'Test Find-PSResource for local repositories' -tags 'CI' { + +# BeforeAll{ +# $localRepo = "psgettestlocal" +# $localUNCRepo = 'psgettestlocal3' +# $testModuleName = "test_local_mod" +# $testModuleName2 = "test_local_mod2" +# $testModuleName3 = "Test_Local_Mod3" +# $similarTestModuleName = "test_local_mod.similar" +# $commandName = "cmd1" +# $dscResourceName = "dsc1" +# $prereleaseLabel = "" +# $localNupkgRepo = "localNupkgRepo" +# Get-NewPSResourceRepositoryFile +# Register-LocalRepos +# Register-LocalTestNupkgsRepo + +# $localRepoUriAddress = Join-Path -Path $TestDrive -ChildPath "testdir" +# $tagsEscaped = @("'Test'", "'Tag2'", "'PSCommand_$cmdName'", "'PSDscResource_$dscName'") +# $prereleaseLabel = "alpha001" + +# New-TestModule -moduleName $testModuleName -repoName $localRepo -packageVersion "1.0.0" -prereleaseLabel "" -tags @() +# New-TestModule -moduleName $testModuleName -repoName $localRepo -packageVersion "3.0.0" -prereleaseLabel "" -tags @() -dscResourceToExport $dscResourceName -commandToExport $commandName +# New-TestModule -moduleName $testModuleName -repoName $localRepo -packageVersion "5.0.0" -prereleaseLabel "" -tags $tagsEscaped +# New-TestModule -moduleName $testModuleName -repoName $localRepo -packageVersion "5.2.5" -prereleaseLabel $prereleaseLabel -tags $tagsEscaped + +# New-TestModule -moduleName $testModuleName2 -repoName $localRepo -packageVersion "5.0.0" -prereleaseLabel "" -tags $tagsEscaped +# New-TestModule -moduleName $testModuleName2 -repoName $localRepo -packageVersion "5.2.5" -prereleaseLabel $prereleaseLabel -tags $tagsEscaped + +# New-TestModule -moduleName $testModuleName3 -repoName $localRepo -packageVersion "1.0.0" -prereleaseLabel "" -tags @() + +# New-TestModule -moduleName $similarTestModuleName -repoName $localRepo -packageVersion "4.0.0" -prereleaseLabel "" -tags $tagsEscaped +# New-TestModule -moduleName $similarTestModuleName -repoName $localRepo -packageVersion "5.0.0" -prereleaseLabel "" -tags $tagsEscaped +# } + +# AfterAll { +# Get-RevertPSResourceRepositoryFile +# } + +# It "find resource given specific Name, Version null (module)" { +# # FindName() +# $res = Find-PSResource -Name $testModuleName -Repository $localRepo +# $res.Name | Should -Be $testModuleName +# $res.Version | Should -Be "5.0.0" +# } + +# It "find resource given specific Name with incorrect casing (should return correct casing)" { +# # FindName() +# $res = Find-PSResource -Name "test_local_mod3" -Repository $localRepo +# $res.Name | Should -Be $testModuleName3 +# $res.Version | Should -Be "1.0.0" +# } + +# It "find resource given specific Name with incorrect casing and Version (should return correct casing)" { +# # FindVersion() +# $res = Find-PSResource -Name "test_local_mod3" -Version "1.0.0" -Repository $localRepo +# $res.Name | Should -Be $testModuleName3 +# $res.Version | Should -Be "1.0.0" +# } + +# It "find resource given specific Name, Version null (module) from a UNC-based local repository" { +# # FindName() +# $res = Find-PSResource -Name $testModuleName -Repository $localUNCRepo +# $res.Name | Should -Be $testModuleName +# $res.Version | Should -Be "5.0.0" +# } + +# It "find resource given Name, Version null (package containing nuspec only)" { +# # FindName() +# $pkgName = "PowerShell" +# Save-PSResource -Name $pkgName -Repository "NuGetGallery" -Path $localRepoUriAddress -AsNupkg -TrustRepository +# $res = Find-PSResource -Name $pkgName -Repository $localRepo +# $res.Name | Should -Be $pkgName +# $res.Repository | Should -Be $localRepo +# } + +# It "find script without RequiredModules" { +# # FindName() +# $pkgName = "Required-Script1" +# $requiredTag = "Tag1" +# Save-PSResource -Name $pkgName -Repository "PSGallery" -Path $localRepoUriAddress -AsNupkg -TrustRepository +# # $res = Find-PSResource -Name $pkgName -Repository $localRepo +# # $res.Name | Should -Be $pkgName +# # $res.Repository | Should -Be $localRepo +# # $res.Tags | Should -Contain $requiredTag +# } + +# It "should not find resource given nonexistent Name" { +# # FindName() +# $res = Find-PSResource -Name NonExistentModule -Repository $localRepo -ErrorVariable err -ErrorAction SilentlyContinue +# $res | Should -BeNullOrEmpty +# $err.Count | Should -Not -Be 0 +# $err[0].FullyQualifiedErrorId | Should -BeExactly "PackageNotFound,Microsoft.PowerShell.PSResourceGet.Cmdlets.FindPSResource" +# $res | Should -BeNullOrEmpty +# } + +# It "find resource given specific Name when another package with similar name (with period) exists" { +# # FindName() +# $res = Find-PSResource -Name $testModuleName -Repository $localRepo +# $res.Name | Should -Be $testModuleName +# $res.Version | Should -Be "5.0.0" + +# $res = Find-PSResource -Name $similarTestModuleName -Repository $localRepo +# $res.Name | Should -Be $similarTestModuleName +# $res.Version | Should -Be "5.0.0" +# } + +# It "find resource(s) given wildcard Name" { +# # FindNameGlobbing +# $res = Find-PSResource -Name "test_local_*" -Repository $localRepo +# $res.Count | Should -BeGreaterThan 1 +# } + +# $testCases2 = @{Version="[5.0.0.0]"; ExpectedVersions=@("5.0.0"); Reason="validate version, exact match"}, +# @{Version="5.0.0.0"; ExpectedVersions=@("5.0.0"); Reason="validate version, exact match without bracket syntax"}, +# @{Version="[1.0.0.0, 5.0.0.0]"; ExpectedVersions=@("1.0.0", "3.0.0", "5.0.0"); Reason="validate version, exact range inclusive"}, +# @{Version="(1.0.0.0, 5.0.0.0)"; ExpectedVersions=@("3.0.0"); Reason="validate version, exact range exclusive"}, +# @{Version="(1.0.0.0,)"; ExpectedVersions=@("3.0.0", "5.0.0"); Reason="validate version, minimum version exclusive"}, +# @{Version="[1.0.0.0,)"; ExpectedVersions=@("1.0.0", "3.0.0", "5.0.0"); Reason="validate version, minimum version inclusive"}, +# @{Version="(,3.0.0.0)"; ExpectedVersions=@("1.0.0"); Reason="validate version, maximum version exclusive"}, +# @{Version="(,3.0.0.0]"; ExpectedVersions=@("1.0.0", "3.0.0"); Reason="validate version, maximum version inclusive"}, +# @{Version="[1.0.0.0, 5.0.0.0)"; ExpectedVersions=@("1.0.0", "3.0.0"); Reason="validate version, mixed inclusive minimum and exclusive maximum version"} +# @{Version="(1.0.0.0, 5.0.0.0]"; ExpectedVersions=@("3.0.0", "5.0.0"); Reason="validate version, mixed exclusive minimum and inclusive maximum version"} + +# It "find resource when given Name to " -TestCases $testCases2{ +# # FindVersionGlobbing() +# param($Version, $ExpectedVersions) +# $res = Find-PSResource -Name $testModuleName -Version $Version -Repository $localRepo +# foreach ($item in $res) { +# $item.Name | Should -Be $testModuleName +# $ExpectedVersions | Should -Contain $item.Version +# } +# } + +# It "find all versions of resource when given specific Name, Version not null --> '*'" { +# # FindVersionGlobbing() +# $res = Find-PSResource -Name $testModuleName -Version "*" -Repository $localRepo +# $res | ForEach-Object { +# $_.Name | Should -Be $testModuleName +# } + +# $res.Count | Should -BeGreaterOrEqual 1 +# } + +# It "find resource with latest (including prerelease) version given Prerelease parameter" { +# # FindName() +# # test_module resource's latest version is a prerelease version, before that it has a non-prerelease version +# $res = Find-PSResource -Name $testModuleName -Repository $localRepo +# $res.Version | Should -Be "5.0.0" + +# $resPrerelease = Find-PSResource -Name $testModuleName -Prerelease -Repository $localRepo +# $resPrerelease.Version | Should -Be "5.2.5" +# $resPrerelease.Prerelease | Should -Be "alpha001" +# } + +# It "find resource given specific Name when another package with similar name (with period) exists" { +# # FindVersion() +# # Package $testModuleName version 4.0.0 does not exist +# # previously if Find-PSResource -Version against local repo did not find that package's version it kept looking at +# # similar named packages and would fault. This test is to ensure only the specified package and its version is checked +# $res = Find-PSResource -Name $testModuleName -Version "4.0.0" -Repository $localRepo -ErrorVariable err -ErrorAction SilentlyContinue +# $res | Should -BeNullOrEmpty +# $err.Count | Should -Not -Be 0 +# $err[0].FullyQualifiedErrorId | Should -BeExactly "PackageNotFound,Microsoft.PowerShell.PSResourceGet.Cmdlets.FindPSResource" +# $res | Should -BeNullOrEmpty + +# $res = Find-PSResource -Name $similarTestModuleName -Version "4.0.0" -Repository $localRepo +# $res.Name | Should -Be $similarTestModuleName +# $res.Version | Should -Be "4.0.0" +# } + +# It "find resources, including Prerelease version resources, when given Prerelease parameter" { +# # FindVersionGlobbing() +# $resWithoutPrerelease = Find-PSResource -Name $testModuleName -Version "*" -Repository $localRepo +# $resWithPrerelease = Find-PSResource -Name $testModuleName -Version "*" -Repository $localRepo +# $resWithPrerelease.Count | Should -BeGreaterOrEqual $resWithoutPrerelease.Count +# } + +# It "find resource that satisfies given Name and Tag property (single tag)" { +# # FindNameWithTag() +# $requiredTag = "test" +# $res = Find-PSResource -Name $testModuleName -Tag $requiredTag -Repository $localRepo +# $res.Name | Should -Be $testModuleName +# $res.Tags | Should -Contain $requiredTag +# } + +# It "should not find resource if Name and Tag are not both satisfied (single tag)" { +# # FindNameWithTag +# $requiredTag = "Windows" # tag "windows" is not present for test_module package +# $res = Find-PSResource -Name $testModuleName -Tag $requiredTag -Repository $localRepo -ErrorVariable err -ErrorAction SilentlyContinue +# $res | Should -BeNullOrEmpty +# $err.Count | Should -Not -Be 0 +# $err[0].FullyQualifiedErrorId | Should -BeExactly "PackageNotFound,Microsoft.PowerShell.PSResourceGet.Cmdlets.FindPSResource" +# $res | Should -BeNullOrEmpty +# } + +# It "find resource that satisfies given Name and Tag property (multiple tags)" { +# # FindNameWithTag() +# $requiredTags = @("test", "Tag2") +# $res = Find-PSResource -Name $testModuleName -Tag $requiredTags -Repository $localRepo +# $res.Name | Should -Be $testModuleName +# $res.Tags | Should -Contain $requiredTags[0] +# $res.Tags | Should -Contain $requiredTags[1] +# } + +# It "find all resources that satisfy Name pattern and have specified Tag (single tag)" { +# # FindNameGlobbingWithTag() +# $requiredTag = "test" +# $nameWithWildcard = "test_local_mod*" + +# $res = Find-PSResource -Name $nameWithWildcard -Tag $requiredTag -Repository $localRepo +# $res.Count | Should -BeGreaterThan 1 +# foreach ($pkg in $res) +# { +# $pkg.Name | Should -BeLike $nameWithWildcard +# $pkg.Tags | Should -Contain "$requiredTag" +# } +# } + +# It "should not find resources if both Name pattern and Tags are not satisfied (single tag)" { +# # FindNameGlobbingWithTag() +# $requiredTag = "windows" # tag "windows" is not present for test_module package +# $res = Find-PSResource -Name "test_module*" -Tag $requiredTag -Repository $localRepo +# $res | Should -BeNullOrEmpty +# } + +# It "find all resources that satisfy Name pattern and have specified Tag (multiple tags)" { +# # FindNameGlobbingWithTag() +# $requiredTags = @("test", "Tag2") +# $nameWithWildcard = "test_local_mod*" + +# $res = Find-PSResource -Name $nameWithWildcard -Tag $requiredTags -Repository $localRepo +# $res.Count | Should -BeGreaterThan 1 +# foreach ($pkg in $res) +# { +# $pkg.Name | Should -BeLike $nameWithWildcard +# $pkg.Tags | Should -Contain $requiredTags[0] +# $pkg.Tags | Should -Contain $requiredTags[1] +# } +# } + +# It "find resource that satisfies given Name, Version and Tag property (single tag)" { +# # FindVersionWithTag() +# $requiredTag = "test" +# $res = Find-PSResource -Name $testModuleName -Version "5.0.0.0" -Tag $requiredTag -Repository $localRepo +# $res.Name | Should -Be $testModuleName +# $res.Version | Should -Be "5.0.0" +# $res.Tags | Should -Contain $requiredTag +# } + +# It "should not find resource if Name, Version and Tag property are not all satisfied (single tag)" { +# # FindVersionWithTag() +# $requiredTag = "windows" # tag "windows" is not present for test_module package +# $res = Find-PSResource -Name $testModuleName -Version "5.0.0.0" -Tag $requiredTag -Repository $localRepo -ErrorVariable err -ErrorAction SilentlyContinue +# $res | Should -BeNullOrEmpty +# $err.Count | Should -Not -Be 0 +# $err[0].FullyQualifiedErrorId | Should -BeExactly "PackageNotFound,Microsoft.PowerShell.PSResourceGet.Cmdlets.FindPSResource" +# $res | Should -BeNullOrEmpty +# } + +# It "find resource that satisfies given Name, Version and Tag property (multiple tags)" { +# # FindVersionWithTag() +# $requiredTags = @("test", "Tag2") +# $res = Find-PSResource -Name $testModuleName -Version "5.0.0.0" -Tag $requiredTags -Repository $localRepo +# $res.Name | Should -Be $testModuleName +# $res.Version | Should -Be "5.0.0" +# $res.Tags | Should -Contain $requiredTags[0] +# $res.Tags | Should -Contain $requiredTags[1] +# } + +# It "find scripts given -Type parameter" { +# Get-ScriptResourcePublishedToLocalRepoTestDrive "testScriptName" $localRepo "1.0.0" + +# $res = Find-PSResource -Type Script -Repository $localRepo +# $res | Should -Not -BeNullOrEmpty +# $res.Count | Should -Be 2 +# $res.Type | Should -Be @("Script", "Script") +# } + +# It "find modules given -Type parameter" { +# Get-ScriptResourcePublishedToLocalRepoTestDrive "testScriptName" $localRepo "1.0.0" + +# $res = Find-PSResource -Type Module -Repository $localRepo +# $res | Should -Not -BeNullOrEmpty +# $res.Count | Should -BeGreaterOrEqual 1 +# foreach ($module in $res) { +# $module.Type | Should -Be "Module" +# } +# } + +# It "find resource given CommandName" -Pending { +# $res = Find-PSResource -CommandName $commandName -Repository $localRepo +# $res | Should -Not -BeNullOrEmpty +# foreach ($item in $res) { +# $item.Names | Should -Be $commandName +# $item.ParentResource.Includes.Command | Should -Contain $commandName +# } +# } + +# It "find resource given DscResourceName" -Pending { +# $res = Find-PSResource -DscResourceName $dscResourceName -Repository $localRepo +# $res | Should -Not -BeNullOrEmpty +# foreach ($item in $res) { +# $item.Names | Should -Be $dscResourceName +# $item.ParentResource.Includes.DscResource | Should -Contain $dscResourceName +# } +# } + +# It "Get definition for alias 'fdres'" { +# (Get-Alias fdres).Definition | Should -BeExactly 'Find-PSResource' +# } + +# It "not find resource with tag value that is non-existent for the packages" { +# # Since this is pattern matching based search no error should be written out. +# $res = Find-PSResource -Tag "nonexistenttag" -Repository $localRepo -ErrorVariable err -ErrorAction SilentlyContinue +# $res | Should -BeNullOrEmpty +# $err.Count | Should -Not -Be 0 +# $err[0].FullyQualifiedErrorId | Should -BeExactly "FindTagsPackageNotFound,Microsoft.PowerShell.PSResourceGet.Cmdlets.FindPSResource" +# } + +# It "find package where prerelease label includes digits and period (i.e prerelease label is not just words)" { +# $nupkgName = "WebView2.Avalonia" +# $nupkgVersion = "1.0.1518.46" +# $prereleaseLabel = "preview.230207.17" +# $res = Find-PSResource -Name $nupkgName -Prerelease -Repository $localNupkgRepo +# $res.Name | Should -Be $nupkgName +# $res.Version | Should -Be $nupkgVersion +# $res.Prerelease | Should -Be $prereleaseLabel +# } + +# It "find module that has multiple manifests and use exact name match one" { +# # Az.KeyVault has 2 manifest files - Az.KeyVault.psd1 and Az.KeyVault.Extension.psd1 +# # this test was added because PSResourceGet would previously pick the .psd1 file by pattern matching the module name, not exact matching it +# # this meant Az.KeyVault.Extension.psd1 and its metadata was being returned. +# # The package is present on PSGallery but issue reproduces when looking at the package's file paths in local repo +# $PSGalleryName = Get-PSGalleryName +# Save-PSResource -Name 'Az.KeyVault' -Version '6.3.1' -Repository $PSGalleryName -AsNupkg -Path $localRepoUriAddress -TrustRepository +# $res = Find-PSResource -Name 'Az.KeyVault' -Repository $localRepo +# $res.Version | Should -Be "6.3.1" +# } +# }