diff --git a/src/code/ContainerRegistryServerAPICalls.cs b/src/code/ContainerRegistryServerAPICalls.cs index b7af3b98d..cd8c4c8be 100644 --- a/src/code/ContainerRegistryServerAPICalls.cs +++ b/src/code/ContainerRegistryServerAPICalls.cs @@ -82,14 +82,40 @@ public ContainerRegistryServerAPICalls(PSRepositoryInfo repository, PSCmdlet cmd #region Overridden Methods + /// + /// Async find method which allows for searching for single name with specific version. + /// Name: no wildcard support + /// Version: no wildcard support + /// This is the concurrent (parallel) counterpart of FindVersion(). + /// public override Task FindVersionAsync(string packageName, string version, ResourceType type, ConcurrentQueue errorMsgs, ConcurrentQueue warningMsgs, ConcurrentQueue debugMsgs, ConcurrentQueue verboseMsgs) { - throw new NotImplementedException("FindVersionAsync is not implemented for ContainerRegistryServerAPICalls."); + debugMsgs.Enqueue("In ContainerRegistryServerAPICalls::FindVersionAsync()"); + FindResults findResponse = FindVersion(packageName, version, type, out ErrorRecord errRecord); + if (errRecord != null) + { + errorMsgs.Enqueue(errRecord); + } + + return Task.FromResult(findResponse); } + /// + /// Async find method which allows for searching for single name with version range. + /// Name: no wildcard support + /// Version: supports wildcards + /// This is the concurrent (parallel) counterpart of FindVersionGlobbing(). + /// public override Task FindVersionGlobbingAsync(string packageName, VersionRange versionRange, bool includePrerelease, ResourceType type, bool getOnlyLatest, ConcurrentQueue errorMsgs, ConcurrentQueue warningMsgs, ConcurrentQueue debugMsgs, ConcurrentQueue verboseMsgs) { - throw new NotImplementedException("FindVersionGlobbingAsync is not implemented for ContainerRegistryServerAPICalls."); + debugMsgs.Enqueue("In ContainerRegistryServerAPICalls::FindVersionGlobbingAsync()"); + FindResults findResponse = FindVersionGlobbing(packageName, versionRange, includePrerelease, type, getOnlyLatest, out ErrorRecord errRecord); + if (errRecord != null) + { + errorMsgs.Enqueue(errRecord); + } + + return Task.FromResult(findResponse); } /// @@ -158,9 +184,21 @@ public override FindResults FindName(string packageName, bool includePrerelease, } + /// + /// Async find method which allows for searching for single name and returns latest version. + /// Name: no wildcard support + /// This is the concurrent (parallel) counterpart of FindName(). + /// public override Task FindNameAsync(string packageName, bool includePrerelease, ResourceType type, ConcurrentQueue errorMsgs, ConcurrentQueue warningMsgs, ConcurrentQueue debugMsgs, ConcurrentQueue verboseMsgs) { - throw new NotImplementedException("FindNameAsync is not implemented for ContainerRegistryServerAPICalls."); + debugMsgs.Enqueue("In ContainerRegistryServerAPICalls::FindNameAsync()"); + FindResults findResponse = FindName(packageName, includePrerelease, type, out ErrorRecord errRecord); + if (errRecord != null) + { + errorMsgs.Enqueue(errRecord); + } + + return Task.FromResult(findResponse); } /// @@ -329,7 +367,22 @@ public override Stream InstallPackage(string packageName, string packageVersion, /// public override Task InstallPackageAsync(string packageName, string packageVersion, bool includePrerelease, ConcurrentQueue errorMsgs, ConcurrentQueue warningMsgs, ConcurrentQueue debugMsgs, ConcurrentQueue verboseMsgs) { - throw new NotImplementedException("FindNameAsync is not implemented for ContainerRegistryServerAPICalls."); + debugMsgs.Enqueue("In ContainerRegistryServerAPICalls::InstallPackageAsync()"); + Stream results = new MemoryStream(); + if (string.IsNullOrEmpty(packageVersion)) + { + errorMsgs.Enqueue(new ErrorRecord( + exception: new ArgumentNullException($"Package version could not be found for {packageName}"), + "PackageVersionNullOrEmptyError", + ErrorCategory.InvalidArgument, + _cmdletPassedIn)); + + return Task.FromResult(results); + } + + string packageNameForInstall = PrependMARPrefix(packageName); + results = InstallVersionAsync(packageNameForInstall, packageVersion, errorMsgs, debugMsgs, verboseMsgs); + return Task.FromResult(results); } /// @@ -400,6 +453,76 @@ private Stream InstallVersion( return responseContent.ReadAsStreamAsync().Result; } + /// + /// Installs a package with version specified using concurrent queues for output instead of cmdlet streams. + /// Used by the async install path to avoid cross-thread cmdlet stream writes. + /// + private Stream InstallVersionAsync( + string packageName, + string packageVersion, + ConcurrentQueue errorMsgs, + ConcurrentQueue debugMsgs, + ConcurrentQueue verboseMsgs) + { + debugMsgs.Enqueue("In ContainerRegistryServerAPICalls::InstallVersionAsync()"); + string packageNameLowercase = packageName.ToLower(); + string tempPath = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString()); + try + { + Directory.CreateDirectory(tempPath); + } + catch (Exception e) + { + errorMsgs.Enqueue(new ErrorRecord( + exception: e, + "InstallVersionTempDirCreationError", + ErrorCategory.InvalidResult, + _cmdletPassedIn)); + + return null; + } + + string containerRegistryAccessToken = GetContainerRegistryAccessToken(needCatalogAccess: false, isPushOperation: false, out ErrorRecord errRecord); + if (errRecord != null) + { + errorMsgs.Enqueue(errRecord); + return null; + } + + verboseMsgs.Enqueue($"Getting manifest for {packageNameLowercase} - {packageVersion}"); + var manifest = GetContainerRegistryRepositoryManifest(packageNameLowercase, packageVersion, containerRegistryAccessToken, out errRecord); + if (errRecord != null) + { + errorMsgs.Enqueue(errRecord); + return null; + } + string digest = GetDigestFromManifest(manifest, out errRecord); + if (errRecord != null) + { + errorMsgs.Enqueue(errRecord); + return null; + } + + verboseMsgs.Enqueue($"Downloading blob for {packageNameLowercase} - {packageVersion}"); + HttpContent responseContent; + try + { + responseContent = GetContainerRegistryBlobAsync(packageNameLowercase, digest, containerRegistryAccessToken).Result; + } + catch (Exception e) + { + errorMsgs.Enqueue(new ErrorRecord( + exception: e, + "InstallVersionGetContainerRegistryBlobAsyncError", + ErrorCategory.InvalidResult, + _cmdletPassedIn)); + + return null; + } + + return responseContent.ReadAsStreamAsync().Result; + } + #endregion #region Authentication and Token Methods diff --git a/src/code/FindHelper.cs b/src/code/FindHelper.cs index 1a074b67e..e4c9dd8d5 100644 --- a/src/code/FindHelper.cs +++ b/src/code/FindHelper.cs @@ -46,10 +46,6 @@ internal class FindHelper // If running 'Install-PSResource Az, TestModule, NewTestModule', it will contain one parent and its dependencies. private ConcurrentDictionary> _packagesFound; - // Creates a new instance of depPkgsFound each time FindDependencyPackages() is called. - // This will eventually return the PSResourceInfo object to the main cmdlet class. - private ConcurrentDictionary depPkgsFound; - // Contains the latest found version of a particular package. private ConcurrentDictionary _knownLatestPkgVersion; @@ -904,25 +900,18 @@ private IEnumerable SearchByNames(ServerApiCall currentServer, R // Example: Find-PSResource -Name "Az" -Version "3.0.0.0" // Example: Find-PSResource -Name "Az" -Version "3.0.0.0" -Tag "Windows" _cmdletPassedIn.WriteDebug("Exact version and package name are specified"); - + string key = string.Empty; FindResults responses = null; if (_tag.Length == 0) { - - ConcurrentDictionary> cachedNetworkCalls = new ConcurrentDictionary>(); Task response = null; - if (currentServer.Repository.ApiVersion == PSRepositoryInfo.APIVersion.V2) { - string key = $"{pkgName}|{_nugetVersion.ToNormalizedString()}|{_type}"; - response = cachedNetworkCalls.GetOrAdd(key, _ => currentServer.FindVersionAsync(pkgName, _nugetVersion.ToNormalizedString(), _type, errorMsgs, warningMsgs, debugMsgs, verboseMsgs)); - - responses = response.GetAwaiter().GetResult(); + key = $"{pkgName}|{_nugetVersion.ToNormalizedString()}|{_type}"; + response = cachedNetworkCalls.GetOrAdd(key, _ => currentServer.FindVersionAsync(pkgName, _nugetVersion.ToNormalizedString(), _type, errorMsgs, warningMsgs, debugMsgs, verboseMsgs)); + + responses = response.GetAwaiter().GetResult(); - Utils.WriteOutConcurrentQueue(_cmdletPassedIn, errorMsgs, warningMsgs, debugMsgs, verboseMsgs); - } - else { - responses = currentServer.FindVersion(pkgName, _nugetVersion.ToNormalizedString(), _type, out errRecord); - } + Utils.WriteOutConcurrentQueue(_cmdletPassedIn, errorMsgs, warningMsgs, debugMsgs, verboseMsgs); } else { @@ -991,20 +980,18 @@ private IEnumerable SearchByNames(ServerApiCall currentServer, R // Example: Find-PSResource -Name "Az" -Version "[1.0.0.0, 3.0.0.0]" _cmdletPassedIn.WriteDebug("Version range and package name are specified"); + errRecord = null; FindResults responses = null; if (_tag.Length == 0) { ConcurrentDictionary> cachedNetworkCalls = new ConcurrentDictionary>(); Task response = null; - if (currentServer.Repository.ApiVersion == PSRepositoryInfo.APIVersion.V2) { - string key = $"{pkgName}|{_versionRange.ToString()}|{_type}"; - response = cachedNetworkCalls.GetOrAdd(key, _ => currentServer.FindVersionGlobbingAsync(pkgName, _versionRange, _prerelease, _type, getOnlyLatest: false, errorMsgs, warningMsgs, debugMsgs, verboseMsgs)); - - responses = response.GetAwaiter().GetResult(); - } - else { - responses = currentServer.FindVersionGlobbing(pkgName, _versionRange, _prerelease, _type, getOnlyLatest: false, out errRecord); - } + string key = $"{pkgName}|{_versionRange.ToString()}|{_type}"; + response = cachedNetworkCalls.GetOrAdd(key, _ => currentServer.FindVersionGlobbingAsync(pkgName, _versionRange, _prerelease, _type, getOnlyLatest: false, errorMsgs, warningMsgs, debugMsgs, verboseMsgs)); + + responses = response.GetAwaiter().GetResult(); + + Utils.WriteOutConcurrentQueue(_cmdletPassedIn, errorMsgs, warningMsgs, debugMsgs, verboseMsgs); } else { @@ -1069,13 +1056,36 @@ private IEnumerable SearchByNames(ServerApiCall currentServer, R // After retrieving all packages find their dependencies if (_includeDependencies) { - foreach (PSResourceInfo currentPkg in parentPkgs) + // Resolving each parent package's dependency closure is independent work, so do it concurrently. + // yield return cannot be used inside Parallel.ForEach, so collect results into a thread-safe bag first. + ConcurrentBag dependencyPkgs = new ConcurrentBag(); + int processorCount = Environment.ProcessorCount; + int maxDegreeOfParallelism = processorCount * 4; + if (parentPkgs.Count > processorCount) + { + Parallel.ForEach(parentPkgs, new ParallelOptions { MaxDegreeOfParallelism = maxDegreeOfParallelism }, currentPkg => + { + foreach (PSResourceInfo pkgDep in FindDependencyPackages(currentServer, currentResponseUtil, currentPkg, repository)) + { + dependencyPkgs.Add(pkgDep); + } + }); + } + else { - foreach (PSResourceInfo pkgDep in FindDependencyPackages(currentServer, currentResponseUtil, currentPkg, repository)) + foreach (PSResourceInfo currentPkg in parentPkgs) { - yield return pkgDep; + foreach (PSResourceInfo pkgDep in FindDependencyPackages(currentServer, currentResponseUtil, currentPkg, repository)) + { + dependencyPkgs.Add(pkgDep); + } } } + + foreach (PSResourceInfo pkgDep in dependencyPkgs) + { + yield return pkgDep; + } } } @@ -1167,15 +1177,17 @@ private string FormatPkgVersionString(PSResourceInfo pkg) internal IEnumerable FindDependencyPackages(ServerApiCall currentServer, ResponseUtil currentResponseUtil, PSResourceInfo currentPkg, PSRepositoryInfo repository) { - depPkgsFound = new ConcurrentDictionary(); + // Use a local instance so multiple parent packages can resolve their dependency closures concurrently + // without racing on shared state. + ConcurrentDictionary depPkgsFound = new ConcurrentDictionary(); _cmdletPassedIn.WriteDebug($"In FindHelper::FindDependencyPackages() - {currentPkg.Name}"); - FindDependencyPackagesHelper(currentServer, currentResponseUtil, currentPkg, repository); + FindDependencyPackagesHelper(currentServer, currentResponseUtil, currentPkg, repository, depPkgsFound); return depPkgsFound.Values.ToList(); } // Method 2 - internal void FindDependencyPackagesHelper(ServerApiCall currentServer, ResponseUtil currentResponseUtil, PSResourceInfo currentPkg, PSRepositoryInfo repository) + internal void FindDependencyPackagesHelper(ServerApiCall currentServer, ResponseUtil currentResponseUtil, PSResourceInfo currentPkg, PSRepositoryInfo repository, ConcurrentDictionary depPkgsFound) { ConcurrentQueue errorMsgs = new ConcurrentQueue(); ConcurrentQueue verboseMsgs = new ConcurrentQueue(); @@ -1189,12 +1201,12 @@ internal void FindDependencyPackagesHelper(ServerApiCall currentServer, Response //const int PARALLEL_THRESHOLD = 5; // TODO: Trottle limit from user, defaults to 5; int processorCount = Environment.ProcessorCount; int maxDegreeOfParallelism = processorCount * 4; - if (currentServer.Repository.ApiVersion == PSRepositoryInfo.APIVersion.V2 && currentPkg.Dependencies.Length > processorCount) + if (currentPkg.Dependencies.Length > processorCount) { Parallel.ForEach(currentPkg.Dependencies, new ParallelOptions { MaxDegreeOfParallelism = maxDegreeOfParallelism }, dep => { debugMsgs.Enqueue($"Finding dependency '{dep.Name}' version range '{dep.VersionRange}'"); - FindDependencyPackageVersion(dep, currentServer, currentResponseUtil, currentPkg, repository, errorMsgs, warningMsgs, debugMsgs, verboseMsgs); + FindDependencyPackageVersion(dep, currentServer, currentResponseUtil, currentPkg, repository, depPkgsFound, errorMsgs, warningMsgs, debugMsgs, verboseMsgs); }); } else @@ -1202,7 +1214,7 @@ internal void FindDependencyPackagesHelper(ServerApiCall currentServer, Response foreach (var dep in currentPkg.Dependencies) { debugMsgs.Enqueue($"Finding dependency '{dep.Name}' version range '{dep.VersionRange}'"); - FindDependencyPackageVersion(dep, currentServer, currentResponseUtil, currentPkg, repository, errorMsgs, warningMsgs, debugMsgs, verboseMsgs); + FindDependencyPackageVersion(dep, currentServer, currentResponseUtil, currentPkg, repository, depPkgsFound, errorMsgs, warningMsgs, debugMsgs, verboseMsgs); } } @@ -1217,6 +1229,7 @@ private void FindDependencyPackageVersion( ResponseUtil currentResponseUtil, PSResourceInfo currentPkg, PSRepositoryInfo repository, + ConcurrentDictionary depPkgsFound, ConcurrentQueue errorMsgs, ConcurrentQueue warningMsgs, ConcurrentQueue debugMsgs, @@ -1237,7 +1250,7 @@ private void FindDependencyPackageVersion( else { // Find this version from the server - depPkg = FindDependencyWithLowerBound(dep, currentServer, currentResponseUtil, currentPkg, repository, errorMsgs, warningMsgs, debugMsgs, verboseMsgs); + depPkg = FindDependencyWithLowerBound(dep, currentServer, currentResponseUtil, currentPkg, repository, depPkgsFound, errorMsgs, warningMsgs, debugMsgs, verboseMsgs); } } else if (dep.VersionRange.HasLowerBound && dep.VersionRange.MinVersion.Equals(dep.VersionRange.MaxVersion)) @@ -1254,7 +1267,7 @@ private void FindDependencyPackageVersion( } else { - depPkg = FindDependencyWithSpecificVersion(dep, currentServer, currentResponseUtil, currentPkg, repository, errorMsgs, warningMsgs, debugMsgs, verboseMsgs); + depPkg = FindDependencyWithSpecificVersion(dep, currentServer, currentResponseUtil, currentPkg, repository, depPkgsFound, errorMsgs, warningMsgs, debugMsgs, verboseMsgs); } } else @@ -1270,7 +1283,7 @@ private void FindDependencyPackageVersion( } else { - depPkg = FindDependencyWithUpperBound(dep, currentServer, currentResponseUtil, currentPkg, repository, errorMsgs, warningMsgs, debugMsgs, verboseMsgs); + depPkg = FindDependencyWithUpperBound(dep, currentServer, currentResponseUtil, currentPkg, repository, depPkgsFound, errorMsgs, warningMsgs, debugMsgs, verboseMsgs); } } } @@ -1282,6 +1295,7 @@ private PSResourceInfo FindDependencyWithSpecificVersion( ResponseUtil currentResponseUtil, PSResourceInfo currentPkg, PSRepositoryInfo repository, + ConcurrentDictionary depPkgsFound, ConcurrentQueue errorMsgs, ConcurrentQueue warningMsgs, ConcurrentQueue debugMsgs, @@ -1290,23 +1304,40 @@ private PSResourceInfo FindDependencyWithSpecificVersion( PSResourceInfo depPkg = null; ErrorRecord errRecord = null; FindResults responses = null; - Task response = null; debugMsgs.Enqueue("In FindHelper::FindDependencyWithSpecificVersion()"); - if (currentServer.Repository.ApiVersion == PSRepositoryInfo.APIVersion.V2) + ConcurrentQueue operationErrorMsgs = new ConcurrentQueue(); + ConcurrentQueue operationWarningMsgs = new ConcurrentQueue(); + ConcurrentQueue operationDebugMsgs = new ConcurrentQueue(); + ConcurrentQueue operationVerboseMsgs = new ConcurrentQueue(); + + // Call FindVersionAsync() for dependency with specific version. + responses = currentServer.FindVersionAsync(dep.Name, dep.VersionRange.MaxVersion.ToString(), _type, operationErrorMsgs, operationWarningMsgs, operationDebugMsgs, operationVerboseMsgs).GetAwaiter().GetResult(); + + while (operationErrorMsgs.TryDequeue(out ErrorRecord queuedError)) { - // See if the network call we're making is already cached, if not, call FindNameAsync() and cache results - string key = $"{dep.Name}|{dep.VersionRange.MaxVersion.ToString()}|{_type}"; - debugMsgs.Enqueue("Checking if network call is cached."); - response = _cachedNetworkCalls.GetOrAdd(key, _ => currentServer.FindVersionAsync(dep.Name, dep.VersionRange.MaxVersion.ToString(), _type, errorMsgs, warningMsgs, debugMsgs, verboseMsgs)); - - responses = response.GetAwaiter().GetResult(); + if (errRecord == null) + { + errRecord = queuedError; + } + + errorMsgs.Enqueue(queuedError); } - else + + while (operationWarningMsgs.TryDequeue(out string queuedWarning)) { - responses = currentServer.FindVersion(dep.Name, dep.VersionRange.MaxVersion.ToString(), _type, out errRecord); + warningMsgs.Enqueue(queuedWarning); } + while (operationDebugMsgs.TryDequeue(out string queuedDebug)) + { + debugMsgs.Enqueue(queuedDebug); + } + + while (operationVerboseMsgs.TryDequeue(out string queuedVerbose)) + { + verboseMsgs.Enqueue(queuedVerbose); + } // Error handling and Convert to PSResource object if (errRecord != null) @@ -1343,7 +1374,7 @@ private PSResourceInfo FindDependencyWithSpecificVersion( // This will eventually return the PSResourceInfo object to the main cmdlet class. debugMsgs.Enqueue($"Adding'{key}' to list of dependency packages found"); depPkgsFound.TryAdd(key, depPkg); - FindDependencyPackagesHelper(currentServer, currentResponseUtil, depPkg, repository); + FindDependencyPackagesHelper(currentServer, currentResponseUtil, depPkg, repository, depPkgsFound); } } } @@ -1358,6 +1389,7 @@ private PSResourceInfo FindDependencyWithLowerBound( ResponseUtil currentResponseUtil, PSResourceInfo currentPkg, PSRepositoryInfo repository, + ConcurrentDictionary depPkgsFound, ConcurrentQueue errorMsgs, ConcurrentQueue warningMsgs, ConcurrentQueue debugMsgs, @@ -1369,19 +1401,12 @@ private PSResourceInfo FindDependencyWithLowerBound( Task response = null; debugMsgs.Enqueue("In FindHelper::FindDependencyWithLowerBound()"); - if (currentServer.Repository.ApiVersion == PSRepositoryInfo.APIVersion.V2) - { - // See if the network call we're making is already cached, if not, call FindNameAsync() and cache results - string key = $"{dep.Name}|*|{_type}"; - debugMsgs.Enqueue("Checking if network call is cached."); - response = _cachedNetworkCalls.GetOrAdd(key, _ => currentServer.FindNameAsync(dep.Name, includePrerelease: true, _type, errorMsgs, warningMsgs, debugMsgs, verboseMsgs)); - - responses = response.GetAwaiter().GetResult(); - } - else - { - responses = currentServer.FindName(dep.Name, includePrerelease: true, _type, out errRecord); - } + // See if the network call we're making is already cached, if not, call FindNameAsync() and cache results + string key = $"{dep.Name}|*|{_type}"; + debugMsgs.Enqueue("Checking if network call is cached."); + response = _cachedNetworkCalls.GetOrAdd(key, _ => currentServer.FindNameAsync(dep.Name, includePrerelease: true, _type, errorMsgs, warningMsgs, debugMsgs, verboseMsgs)); + + responses = response.GetAwaiter().GetResult(); // Error handling and Convert to PSResource object if (errRecord != null) @@ -1410,7 +1435,7 @@ private PSResourceInfo FindDependencyWithLowerBound( string pkgVersion = FormatPkgVersionString(depPkg); debugMsgs.Enqueue($"Found dependency '{depPkg.Name}' version '{pkgVersion}'"); - string key = $"{depPkg.Name}{pkgVersion}"; + key = $"{depPkg.Name}{pkgVersion}"; if (!depPkgsFound.ContainsKey(key)) { // Add pkg to collection of packages found then find dependencies @@ -1418,7 +1443,7 @@ private PSResourceInfo FindDependencyWithLowerBound( // This will eventually return the PSResourceInfo object to the main cmdlet class. debugMsgs.Enqueue($"Adding'{key}' to list of dependency packages found"); depPkgsFound.TryAdd(key, depPkg); - FindDependencyPackagesHelper(currentServer, currentResponseUtil, depPkg, repository); + FindDependencyPackagesHelper(currentServer, currentResponseUtil, depPkg, repository, depPkgsFound); } } } @@ -1433,6 +1458,7 @@ private PSResourceInfo FindDependencyWithUpperBound( ResponseUtil currentResponseUtil, PSResourceInfo currentPkg, PSRepositoryInfo repository, + ConcurrentDictionary depPkgsFound, ConcurrentQueue errorMsgs, ConcurrentQueue warningMsgs, ConcurrentQueue debugMsgs, @@ -1445,21 +1471,13 @@ private PSResourceInfo FindDependencyWithUpperBound( ConcurrentDictionary> cachedNetworkCalls = new ConcurrentDictionary>(); debugMsgs.Enqueue("In FindHelper::FindDependencyWithUpperBound()"); + // See if the network call we're making is already cached, if not, call FindNameAsync() and cache results + string key = $"{dep.Name}|{dep.VersionRange.MaxVersion.ToString()}|{_type}"; + debugMsgs.Enqueue("Checking if network call is cached."); + response = cachedNetworkCalls.GetOrAdd(key, _ => currentServer.FindVersionGlobbingAsync(dep.Name, dep.VersionRange, includePrerelease: true, ResourceType.None, getOnlyLatest: true, errorMsgs, warningMsgs, debugMsgs, verboseMsgs)); - if (currentServer.Repository.ApiVersion == PSRepositoryInfo.APIVersion.V2) - { - // See if the network call we're making is already caced, if not, call FindNameAsync() and cache results - string key = $"{dep.Name}|{dep.VersionRange.MaxVersion.ToString()}|{_type}"; - debugMsgs.Enqueue("Checking if network call is cached."); - response = cachedNetworkCalls.GetOrAdd(key, _ => currentServer.FindVersionGlobbingAsync(dep.Name, dep.VersionRange, includePrerelease: true, ResourceType.None, getOnlyLatest: true, errorMsgs, warningMsgs, debugMsgs, verboseMsgs)); + responses = response.GetAwaiter().GetResult(); - responses = response.GetAwaiter().GetResult(); - - } - else - { - responses = currentServer.FindVersionGlobbing(dep.Name, dep.VersionRange, includePrerelease: true, ResourceType.None, getOnlyLatest: true, out errRecord); - } // Error handling and Convert to PSResource object if (errRecord != null) @@ -1489,7 +1507,7 @@ private PSResourceInfo FindDependencyWithUpperBound( string pkgVersion = FormatPkgVersionString(depPkg); debugMsgs.Enqueue($"Found dependency '{depPkg.Name}' version '{pkgVersion}'"); - string key = $"{depPkg.Name}{pkgVersion}"; + key = $"{depPkg.Name}{pkgVersion}"; if (!depPkgsFound.ContainsKey(key)) { // Add pkg to collection of packages found then find dependencies @@ -1497,7 +1515,7 @@ private PSResourceInfo FindDependencyWithUpperBound( // This will eventually return the PSResourceInfo object to the main cmdlet class. debugMsgs.Enqueue($"Adding'{key}' to list of dependency packages found"); depPkgsFound.TryAdd(key, depPkg); - FindDependencyPackagesHelper(currentServer, currentResponseUtil, depPkg, repository); + FindDependencyPackagesHelper(currentServer, currentResponseUtil, depPkg, repository, depPkgsFound); } } } diff --git a/src/code/InstallHelper.cs b/src/code/InstallHelper.cs index c89ef0ee9..fe01209be 100644 --- a/src/code/InstallHelper.cs +++ b/src/code/InstallHelper.cs @@ -516,73 +516,123 @@ private List InstallPackages( FindHelper findHelper) { _cmdletPassedIn.WriteDebug("In InstallHelper::InstallPackages()"); - + List pkgsSuccessfullyInstalled = new(); - // Install parent package to the temp directory, - // Get the dependencies from the installed package, - // Install all dependencies to temp directory. - // If a single dependency fails to install, roll back by deleting the temp directory. + // ---------- Phase 1 (pipeline thread): resolve each parent package and evaluate ShouldProcess ---------- + // ShouldProcess / -WhatIf / -Confirm and Write* must run on the pipeline thread, so all gating happens + // here, before any parallel download work begins. Packages that pass the gate become work items. + List workItems = new(); foreach (var parentPackage in pkgNamesToInstall) { - string tempInstallPath = CreateInstallationTempPath(); - - try + PSResourceInfo pkgToInstall = FindParentPackage( + searchVersionType: _versionType, + specificVersion: _nugetVersion, + versionRange: _versionRange, + pkgNameToInstall: parentPackage, + repository: repository, + currentServer: currentServer, + currentResponseUtil: currentResponseUtil, + pkgVersion: out string pkgVersion, + errRecord: out ErrorRecord findErrRecord); + + if (findErrRecord != null) { - // Hashtable has the key as the package name - // and value as a Hashtable of specific package info: - // packageName, { version = "", isScript = "", isModule = "", pkg = "", etc. } - // Install parent package to the temp directory. - ConcurrentDictionary packagesHash = BeginPackageInstall( - searchVersionType: _versionType, - specificVersion: _nugetVersion, - versionRange: _versionRange, - pkgNameToInstall: parentPackage, - repository: repository, - currentServer: currentServer, - currentResponseUtil: currentResponseUtil, - tempInstallPath: tempInstallPath, - skipDependencyCheck: skipDependencyCheck, - packagesHash: new ConcurrentDictionary(StringComparer.InvariantCultureIgnoreCase), - warning: out string warning, - errRecord: out ErrorRecord errRecord); - - // At this point all packages are installed to temp path. - if (errRecord != null) + if (findErrRecord.FullyQualifiedErrorId.Equals("PackageNotFound")) { - if (errRecord.FullyQualifiedErrorId.Equals("PackageNotFound")) - { - _cmdletPassedIn.WriteVerbose(errRecord.Exception.Message); - } - else - { - _cmdletPassedIn.WriteError(errRecord); - } - - continue; + _cmdletPassedIn.WriteVerbose(findErrRecord.Exception.Message); } - if (warning != null) + else { - _cmdletPassedIn.WriteWarning(warning); + _cmdletPassedIn.WriteError(findErrRecord); } - if (packagesHash.Count == 0) - { - continue; - } + continue; + } + + if (pkgToInstall == null) + { + continue; + } + + // Check to see if the pkg is already installed (unless -Reinstall was specified). + if (!_reinstall && _packagesOnMachine.Contains($"{pkgToInstall.Name}{pkgVersion}")) + { + _cmdletPassedIn.WriteWarning($"Resource '{pkgToInstall.Name}' with version '{pkgVersion}' is already installed. If you would like to reinstall, please run the cmdlet again with the -Reinstall parameter"); + + // Remove from tracking list of packages to install. + _pkgNamesToInstall.RemoveAll(x => x.Equals(pkgToInstall.Name, StringComparison.InvariantCultureIgnoreCase)); + + continue; + } + + // ShouldProcess gate (handles -WhatIf / -Confirm) on the pipeline thread. + string shouldProcessTarget = _savePkg + ? $"Package to save: '{pkgToInstall.Name}', version: '{pkgVersion}'" + : $"Package to install: '{pkgToInstall.Name}', version: '{pkgVersion}'"; + if (!_cmdletPassedIn.ShouldProcess(shouldProcessTarget)) + { + continue; + } + + workItems.Add(new ParentInstallWorkItem + { + PkgToInstall = pkgToInstall, + PkgVersion = pkgVersion, + TempInstallPath = CreateInstallationTempPath() + }); + } - Hashtable parentPkgInfo = packagesHash[parentPackage] as Hashtable; - PSResourceInfo parentPkgObj = parentPkgInfo["psResourceInfoPkg"] as PSResourceInfo; + if (workItems.Count == 0) + { + return pkgsSuccessfullyInstalled; + } + + // ---------- Phase 2 (parallel): download each parent + its dependencies to its own temp path ---------- + // This is network-bound work. No pipeline-thread calls are made here; all host messages are queued + // per work item and drained in Phase 3. Each work item downloads into its own temp path and its own + // packagesHash, so there is no shared mutable state between parents. + int processorCount = Environment.ProcessorCount; + if (workItems.Count > 1) + { + int maxDegreeOfParallelism = processorCount * 4; + Parallel.ForEach(workItems, new ParallelOptions { MaxDegreeOfParallelism = maxDegreeOfParallelism }, workItem => + { + workItem.PackagesHash = DownloadParentAndDeps( + workItem.PkgToInstall, workItem.PkgVersion, workItem.TempInstallPath, repository, + currentServer, currentResponseUtil, skipDependencyCheck, + workItem.ErrorMsgs, workItem.WarningMsgs, workItem.DebugMsgs, workItem.VerboseMsgs, + out bool succeeded); + workItem.Succeeded = succeeded; + }); + } + else + { + ParentInstallWorkItem workItem = workItems[0]; + workItem.PackagesHash = DownloadParentAndDeps( + workItem.PkgToInstall, workItem.PkgVersion, workItem.TempInstallPath, repository, + currentServer, currentResponseUtil, skipDependencyCheck, + workItem.ErrorMsgs, workItem.WarningMsgs, workItem.DebugMsgs, workItem.VerboseMsgs, + out bool succeeded); + workItem.Succeeded = succeeded; + } - // If -WhatIf is passed in, early out. - if (_cmdletPassedIn.MyInvocation.BoundParameters.ContainsKey("WhatIf") && (SwitchParameter)_cmdletPassedIn.MyInvocation.BoundParameters["WhatIf"] == true) + // ---------- Phase 3 (pipeline thread): drain messages, move content to final location, record results ---------- + // If a single dependency fails to install, roll back that parent by deleting its temp directory. + foreach (ParentInstallWorkItem workItem in workItems) + { + try + { + Utils.WriteOutConcurrentQueue(_cmdletPassedIn, workItem.ErrorMsgs, workItem.WarningMsgs, workItem.DebugMsgs, workItem.VerboseMsgs); + + if (!workItem.Succeeded || workItem.PackagesHash == null || workItem.PackagesHash.Count == 0) { - return pkgsSuccessfullyInstalled; + continue; } // Parent package and dependencies are now installed to temp directory. // Try to move all package directories from temp directory to final destination. - if (!TryMoveInstallContent(tempInstallPath, scope, packagesHash)) + if (!TryMoveInstallContent(workItem.TempInstallPath, scope, workItem.PackagesHash)) { _cmdletPassedIn.WriteError(new ErrorRecord( new InvalidOperationException(), @@ -592,9 +642,9 @@ private List InstallPackages( } else { - foreach (string pkgName in packagesHash.Keys) + foreach (string pkgName in workItem.PackagesHash.Keys) { - Hashtable pkgInfo = packagesHash[pkgName] as Hashtable; + Hashtable pkgInfo = workItem.PackagesHash[pkgName] as Hashtable; pkgsSuccessfullyInstalled.Add(pkgInfo["psResourceInfoPkg"] as PSResourceInfo); // Add each pkg to _packagesOnMachine (ie pkgs fully installed on the machine). @@ -610,11 +660,11 @@ private List InstallPackages( ErrorCategory.InvalidOperation, _cmdletPassedIn)); - throw e; + throw; } finally { - DeleteInstallationTempPath(tempInstallPath); + DeleteInstallationTempPath(workItem.TempInstallPath); } } @@ -622,9 +672,27 @@ private List InstallPackages( } /// - /// Installs a single package to the temporary path. + /// Tracks the per-parent state used to parallelize parent-package installation across the three phases. + /// Each parent has its own temp path, result hash, and message queues so there is no shared mutable state + /// during the parallel download phase. + /// + private sealed class ParentInstallWorkItem + { + public PSResourceInfo PkgToInstall; + public string PkgVersion; + public string TempInstallPath; + public ConcurrentDictionary PackagesHash; + public bool Succeeded; + public readonly ConcurrentQueue ErrorMsgs = new(); + public readonly ConcurrentQueue WarningMsgs = new(); + public readonly ConcurrentQueue DebugMsgs = new(); + public readonly ConcurrentQueue VerboseMsgs = new(); + } + + /// + /// Resolves the parent package to install (find + version selection). Must run on the pipeline thread. /// - private ConcurrentDictionary BeginPackageInstall( + private PSResourceInfo FindParentPackage( VersionType searchVersionType, NuGetVersion specificVersion, VersionRange versionRange, @@ -632,15 +700,12 @@ private ConcurrentDictionary BeginPackageInstall( PSRepositoryInfo repository, ServerApiCall currentServer, ResponseUtil currentResponseUtil, - string tempInstallPath, - bool skipDependencyCheck, - ConcurrentDictionary packagesHash, - out string warning, + out string pkgVersion, out ErrorRecord errRecord) { - _cmdletPassedIn.WriteDebug("In InstallHelper::BeginPackageInstall()"); + _cmdletPassedIn.WriteDebug("In InstallHelper::FindParentPackage()"); FindResults responses = null; - warning = null; + pkgVersion = null; errRecord = null; // Find the parent package that needs to be installed @@ -652,7 +717,7 @@ private ConcurrentDictionary BeginPackageInstall( if (findVersionGlobbingErrRecord != null || responses.IsFindResultsEmpty()) { errRecord = findVersionGlobbingErrRecord; - return packagesHash; + return null; } break; @@ -664,7 +729,7 @@ private ConcurrentDictionary BeginPackageInstall( if (findVersionErrRecord != null) { errRecord = findVersionErrRecord; - return packagesHash; + return null; } break; @@ -675,7 +740,7 @@ private ConcurrentDictionary BeginPackageInstall( if (findNameErrRecord != null) { errRecord = findNameErrRecord; - return packagesHash; + return null; } break; @@ -721,11 +786,11 @@ private ConcurrentDictionary BeginPackageInstall( if (pkgToInstall == null) { - return packagesHash; + return null; } pkgToInstall.RepositorySourceLocation = repository.Uri.ToString(); - pkgToInstall.AdditionalMetadata.TryGetValue("NormalizedVersion", out string pkgVersion); + pkgToInstall.AdditionalMetadata.TryGetValue("NormalizedVersion", out pkgVersion); if (pkgVersion == null) { // Not all NuGet providers (e.g. Artifactory, possibly others) send NormalizedVersion in NuGet package responses. @@ -745,117 +810,67 @@ private ConcurrentDictionary BeginPackageInstall( } // Check to see if the pkg is already installed (ie the pkg is installed and the version satisfies the version range provided via param) - // TODO: can use cache for this - if (!_reinstall) - { - string currPkgNameVersion = $"{pkgToInstall.Name}{pkgVersion}"; - // Use HashSet lookup instead of Contains for O(1) performance - if (_packagesOnMachine.Contains(currPkgNameVersion)) - { - _cmdletPassedIn.WriteWarning($"Resource '{pkgToInstall.Name}' with version '{pkgVersion}' is already installed. If you would like to reinstall, please run the cmdlet again with the -Reinstall parameter"); - - // Remove from tracking list of packages to install. - _pkgNamesToInstall.RemoveAll(x => x.Equals(pkgToInstall.Name, StringComparison.InvariantCultureIgnoreCase)); - - return packagesHash; - } - } - - if (packagesHash.ContainsKey(pkgToInstall.Name)) - { - return packagesHash; - } + // Note: the already-installed check and ShouldProcess gate are handled by the caller on the pipeline thread. + return pkgToInstall; + } + /// + /// Downloads the parent package and its dependencies to the temporary path. Safe to run on worker threads: + /// all host messages are routed to the provided queues and drained by the caller on the pipeline thread. + /// + private ConcurrentDictionary DownloadParentAndDeps( + PSResourceInfo pkgToInstall, + string pkgVersion, + string tempInstallPath, + PSRepositoryInfo repository, + ServerApiCall currentServer, + ResponseUtil currentResponseUtil, + bool skipDependencyCheck, + ConcurrentQueue errorMsgs, + ConcurrentQueue warningMsgs, + ConcurrentQueue debugMsgs, + ConcurrentQueue verboseMsgs, + out bool success) + { + debugMsgs.Enqueue("In InstallHelper::DownloadParentAndDeps()"); + ConcurrentDictionary packagesHash = new ConcurrentDictionary(StringComparer.InvariantCultureIgnoreCase); - ConcurrentDictionary updatedPackagesHash = packagesHash; - - // -WhatIf processing. - if (_savePkg && !_cmdletPassedIn.ShouldProcess($"Package to save: '{pkgToInstall.Name}', version: '{pkgVersion}'")) - { - updatedPackagesHash.TryAdd(pkgToInstall.Name, new Hashtable(StringComparer.InvariantCultureIgnoreCase) - { - { "isModule", "" }, - { "isScript", "" }, - { "psResourceInfoPkg", pkgToInstall }, - { "tempDirNameVersionPath", tempInstallPath }, - { "pkgVersion", "" }, - { "scriptPath", "" }, - { "installPath", "" } - }); - } - else if (!_cmdletPassedIn.ShouldProcess($"Package to install: '{pkgToInstall.Name}', version: '{pkgVersion}'")) + List parentAndDeps = new List(); + if (!skipDependencyCheck) { - if (!updatedPackagesHash.ContainsKey(pkgToInstall.Name)) - { - updatedPackagesHash.TryAdd(pkgToInstall.Name, new Hashtable(StringComparer.InvariantCultureIgnoreCase) - { - { "isModule", "" }, - { "isScript", "" }, - { "psResourceInfoPkg", pkgToInstall }, - { "tempDirNameVersionPath", tempInstallPath }, - { "pkgVersion", "" }, - { "scriptPath", "" }, - { "installPath", "" } - }); - } + // List returned only includes dependencies, so we'll add the parent pkg to this list to pass on to installation method. + parentAndDeps.AddRange(_findHelper.FindDependencyPackages(currentServer, currentResponseUtil, pkgToInstall, repository)); + debugMsgs.Enqueue("In InstallHelper::DownloadParentAndDeps(), found all dependencies"); } - else - { - // Concurrent updates, currently only implemented for v2 server repositories - // Find all dependencies - if (!skipDependencyCheck) - { - // concurrency updates - List parentAndDeps = _findHelper.FindDependencyPackages(currentServer, currentResponseUtil, pkgToInstall, repository).ToList(); - // List returned only includes dependencies, so we'll add the parent pkg to this list to pass on to installation method - parentAndDeps.Add(pkgToInstall); - _cmdletPassedIn.WriteDebug("In InstallHelper::BeginPackageInstall(), found all dependencies"); + parentAndDeps.Add(pkgToInstall); - return InstallParentAndDependencyPackages(parentAndDeps, currentServer, tempInstallPath, packagesHash, updatedPackagesHash, pkgToInstall); - } - else { - // If we don't install dependencies, we're only installing the parent pkg so we can short circut and simply install the parent pkg. - Stream responseStream = currentServer.InstallPackage(pkgToInstall.Name, pkgVersion, true, out ErrorRecord installNameErrRecord); - - if (installNameErrRecord != null) - { - errRecord = installNameErrRecord; - return packagesHash; - } - ConcurrentQueue errorMsgs = new ConcurrentQueue(); - ConcurrentQueue warningMsgs = new ConcurrentQueue(); - ConcurrentQueue debugMsgs = new ConcurrentQueue(); - ConcurrentQueue verboseMsgs = new ConcurrentQueue(); - - bool installedToTempPathSuccessfully = _asNupkg ? TrySaveNupkgToTempPath(responseStream, tempInstallPath, pkgToInstall.Name, pkgVersion, pkgToInstall, packagesHash, out updatedPackagesHash, errorMsgs, warningMsgs, debugMsgs, verboseMsgs) : - TryInstallToTempPath(responseStream, tempInstallPath, pkgToInstall.Name, pkgVersion, pkgToInstall, packagesHash, out updatedPackagesHash, errorMsgs, warningMsgs, debugMsgs, verboseMsgs); - - Utils.WriteOutConcurrentQueue(_cmdletPassedIn, errorMsgs, warningMsgs, debugMsgs, verboseMsgs); - if (!installedToTempPathSuccessfully) - { - return packagesHash; - } - } - } + ConcurrentDictionary updatedPackagesHash = InstallParentAndDependencyPackages( + parentAndDeps, currentServer, tempInstallPath, packagesHash, packagesHash, pkgToInstall, + errorMsgs, warningMsgs, debugMsgs, verboseMsgs); + success = errorMsgs.IsEmpty; return updatedPackagesHash; } - private ConcurrentDictionary InstallParentAndDependencyPackages(List parentAndDeps, ServerApiCall currentServer, string tempInstallPath, ConcurrentDictionary packagesHash, ConcurrentDictionary updatedPackagesHash, PSResourceInfo pkgToInstall) + private ConcurrentDictionary InstallParentAndDependencyPackages( + List parentAndDeps, + ServerApiCall currentServer, + string tempInstallPath, + ConcurrentDictionary packagesHash, + ConcurrentDictionary updatedPackagesHash, + PSResourceInfo pkgToInstall, + ConcurrentQueue errorMsgs, + ConcurrentQueue warningMsgs, + ConcurrentQueue debugMsgs, + ConcurrentQueue verboseMsgs) { - string warning = string.Empty; - ConcurrentQueue errorMsgs = new ConcurrentQueue(); - ConcurrentQueue verboseMsgs = new ConcurrentQueue(); - ConcurrentQueue debugMsgs = new ConcurrentQueue(); - ConcurrentQueue warningMsgs = new ConcurrentQueue(); - // TODO: figure out a good threshold and parallel count int processorCount = Environment.ProcessorCount; - _cmdletPassedIn.WriteDebug($"parentAndDeps.Count is {parentAndDeps.Count}, processor count is: {processorCount}"); - if (currentServer.Repository.ApiVersion == PSRepositoryInfo.APIVersion.V2 && parentAndDeps.Count > processorCount) + debugMsgs.Enqueue($"parentAndDeps.Count is {parentAndDeps.Count}, processor count is: {processorCount}"); + if (parentAndDeps.Count > processorCount) { - _cmdletPassedIn.WriteDebug($"parentAndDeps.Count is greater than processor count"); + debugMsgs.Enqueue($"parentAndDeps.Count is greater than processor count"); // Set the maximum degree of parallelism to 32? (Invoke-Command has default of 32, that's where we got this number from) // If installing more than 3 packages, do so concurrently // If the number of dependencies is very small (e.g., ≤ CPU cores), parallelism may add overhead instead of improving speed. @@ -891,8 +906,7 @@ private ConcurrentDictionary InstallParentAndDependencyPackag } }); - Utils.WriteOutConcurrentQueue(_cmdletPassedIn, errorMsgs, warningMsgs, debugMsgs, verboseMsgs); - if (errorMsgs.Count > 0) + if (!errorMsgs.IsEmpty) { return packagesHash; } @@ -910,16 +924,13 @@ private ConcurrentDictionary InstallParentAndDependencyPackag if (installNameErrRecord != null) { - _cmdletPassedIn.WriteError(installNameErrRecord); + errorMsgs.Enqueue(installNameErrRecord); return packagesHash; } - //ErrorRecord tempSaveErrRecord = null, tempInstallErrRecord = null; bool installedToTempPathSuccessfully = _asNupkg ? TrySaveNupkgToTempPath(responseStream, tempInstallPath, pkgToInstallName, pkgToInstallVersion, pkgToBeInstalled, packagesHash, out updatedPackagesHash, errorMsgs, warningMsgs, debugMsgs, verboseMsgs) : TryInstallToTempPath(responseStream, tempInstallPath, pkgToInstallName, pkgToInstallVersion, pkgToBeInstalled, packagesHash, out updatedPackagesHash, errorMsgs, warningMsgs, debugMsgs, verboseMsgs); - Utils.WriteOutConcurrentQueue(_cmdletPassedIn, errorMsgs, warningMsgs, debugMsgs, verboseMsgs); - if (!installedToTempPathSuccessfully) { return packagesHash; diff --git a/src/code/LocalServerApiCalls.cs b/src/code/LocalServerApiCalls.cs index a8e505acb..1ebb72dcb 100644 --- a/src/code/LocalServerApiCalls.cs +++ b/src/code/LocalServerApiCalls.cs @@ -41,14 +41,40 @@ public LocalServerAPICalls (PSRepositoryInfo repository, PSCmdlet cmdletPassedIn #region Overridden Methods + /// + /// Async find method which allows for searching for single name with specific version. + /// Name: no wildcard support + /// Version: no wildcard support + /// This is the concurrent (parallel) counterpart of FindVersion(). + /// public override Task FindVersionAsync(string packageName, string version, ResourceType type, ConcurrentQueue errorMsgs, ConcurrentQueue warningMsgs, ConcurrentQueue debugMsgs, ConcurrentQueue verboseMsgs) { - throw new NotImplementedException(); + debugMsgs.Enqueue("In LocalServerApiCalls::FindVersionAsync()"); + FindResults findResponse = FindVersionHelper(packageName, version, tags: Utils.EmptyStrArray, type, out ErrorRecord errRecord); + if (errRecord != null) + { + errorMsgs.Enqueue(errRecord); + } + + return Task.FromResult(findResponse); } + /// + /// Async find method which allows for searching for single name with version range. + /// Name: no wildcard support + /// Version: supports wildcards + /// This is the concurrent (parallel) counterpart of FindVersionGlobbing(). + /// public override Task FindVersionGlobbingAsync(string packageName, VersionRange versionRange, bool includePrerelease, ResourceType type, bool getOnlyLatest, ConcurrentQueue errorMsgs, ConcurrentQueue warningMsgs, ConcurrentQueue debugMsgs, ConcurrentQueue verboseMsgs) { - throw new NotImplementedException(); + debugMsgs.Enqueue("In LocalServerApiCalls::FindVersionGlobbingAsync()"); + FindResults findResponse = FindVersionGlobbing(packageName, versionRange, includePrerelease, type, getOnlyLatest, out ErrorRecord errRecord); + if (errRecord != null) + { + errorMsgs.Enqueue(errRecord); + } + + return Task.FromResult(findResponse); } /// /// Find method which allows for searching for all packages from a repository and returns latest version for each. @@ -124,9 +150,21 @@ public override FindResults FindName(string packageName, bool includePrerelease, return FindNameHelper(packageName, Utils.EmptyStrArray, includePrerelease, type, out errRecord); } + /// + /// Async find method which allows for searching for single name and returns latest version. + /// Name: no wildcard support + /// This is the concurrent (parallel) counterpart of FindName(). + /// public override Task FindNameAsync(string packageName, bool includePrerelease, ResourceType type, ConcurrentQueue errorMsgs, ConcurrentQueue warningMsgs, ConcurrentQueue debugMsgs, ConcurrentQueue verboseMsgs) { - throw new NotImplementedException(); + debugMsgs.Enqueue("In LocalServerApiCalls::FindNameAsync()"); + FindResults findResponse = FindNameHelper(packageName, tags: Utils.EmptyStrArray, includePrerelease, type, out ErrorRecord errRecord); + if (errRecord != null) + { + errorMsgs.Enqueue(errRecord); + } + + return Task.FromResult(findResponse); } /// @@ -278,7 +316,26 @@ public override Stream InstallPackage(string packageName, string packageVersion, /// public override Task InstallPackageAsync(string packageName, string packageVersion, bool includePrerelease, ConcurrentQueue errorMsgs, ConcurrentQueue warningMsgs, ConcurrentQueue debugMsgs, ConcurrentQueue verboseMsgs) { - throw new NotImplementedException("InstallPackageAsync is not implemented for LocalServerAPICalls."); + debugMsgs.Enqueue("In LocalServerApiCalls::InstallPackageAsync()"); + Stream results = new MemoryStream(); + if (string.IsNullOrEmpty(packageVersion)) + { + errorMsgs.Enqueue(new ErrorRecord( + exception: new ArgumentNullException($"Package version could not be found for {packageName}"), + "PackageVersionNullOrEmptyError", + ErrorCategory.InvalidArgument, + _cmdletPassedIn)); + + return Task.FromResult(results); + } + + results = InstallVersion(packageName, packageVersion, out ErrorRecord errRecord); + if (errRecord != null) + { + errorMsgs.Enqueue(errRecord); + } + + return Task.FromResult(results); } #endregion diff --git a/src/code/NuGetServerAPICalls.cs b/src/code/NuGetServerAPICalls.cs index 1c6bb2828..d8c6b5ffe 100644 --- a/src/code/NuGetServerAPICalls.cs +++ b/src/code/NuGetServerAPICalls.cs @@ -49,14 +49,51 @@ public NuGetServerAPICalls (PSRepositoryInfo repository, PSCmdlet cmdletPassedIn #region Overridden Methods + /// + /// Async find method which allows for searching for single name with specific version. + /// Name: no wildcard support + /// Version: no wildcard support + /// This is the concurrent (parallel) counterpart of FindVersion(). + /// public override Task FindVersionAsync(string packageName, string version, ResourceType type, ConcurrentQueue errorMsgs, ConcurrentQueue warningMsgs, ConcurrentQueue debugMsgs, ConcurrentQueue verboseMsgs) { - throw new NotImplementedException("FindVersionAsync is not implemented for NuGetServerAPICalls."); + debugMsgs.Enqueue("In NuGetServerAPICalls::FindVersionAsync()"); + var queryBuilder = new NuGetV2QueryBuilder(new Dictionary{ + { "id", $"'{packageName}'" }, + }); + var filterBuilder = queryBuilder.FilterBuilder; + + // We need to explicitly add 'Id eq ' whenever $filter is used, otherwise arbitrary results are returned. + filterBuilder.AddCriterion($"Id eq '{packageName}'"); + filterBuilder.AddCriterion($"NormalizedVersion eq '{packageName}'"); + + var requestUrl = $"{Repository.Uri}/FindPackagesById()?{queryBuilder.BuildQueryString()}"; + string response = HttpRequestCallAsync(requestUrl, debugMsgs, out ErrorRecord errRecord); + FindResults findResponse = new FindResults(stringResponse: new string[] { response }, hashtableResponse: emptyHashResponses, responseType: FindResponseType); + if (errRecord != null) + { + errorMsgs.Enqueue(errRecord); + } + + return Task.FromResult(findResponse); } + /// + /// Async find method which allows for searching for single name with version range. + /// Name: no wildcard support + /// Version: supports wildcards + /// This is the concurrent (parallel) counterpart of FindVersionGlobbing(). + /// public override Task FindVersionGlobbingAsync(string packageName, VersionRange versionRange, bool includePrerelease, ResourceType type, bool getOnlyLatest, ConcurrentQueue errorMsgs, ConcurrentQueue warningMsgs, ConcurrentQueue debugMsgs, ConcurrentQueue verboseMsgs) { - throw new NotImplementedException("FindVersionGlobbingAsync is not implemented for NuGetServerAPICalls."); + debugMsgs.Enqueue("In NuGetServerAPICalls::FindVersionGlobbingAsync()"); + FindResults findResponse = FindVersionGlobbing(packageName, versionRange, includePrerelease, type, getOnlyLatest, out ErrorRecord errRecord); + if (errRecord != null) + { + errorMsgs.Enqueue(errRecord); + } + + return Task.FromResult(findResponse); } /// /// Find method which allows for searching for all packages from a repository and returns latest version for each. @@ -193,9 +230,21 @@ public override FindResults FindName(string packageName, bool includePrerelease, return new FindResults(stringResponse: new string[]{ response }, hashtableResponse: emptyHashResponses, responseType: FindResponseType); } + /// + /// Async find method which allows for searching for single name and returns latest version. + /// Name: no wildcard support + /// This is the concurrent (parallel) counterpart of FindName(). + /// public override Task FindNameAsync(string packageName, bool includePrerelease, ResourceType type, ConcurrentQueue errorMsgs, ConcurrentQueue warningMsgs, ConcurrentQueue debugMsgs, ConcurrentQueue verboseMsgs) { - throw new NotImplementedException("FindNameAsync is not implemented for NuGetServerAPICalls."); + debugMsgs.Enqueue("In NuGetServerAPICalls::FindNameAsync()"); + FindResults findResponse = FindName(packageName, includePrerelease, type, out ErrorRecord errRecord); + if (errRecord != null) + { + errorMsgs.Enqueue(errRecord); + } + + return Task.FromResult(findResponse); } /// @@ -471,7 +520,14 @@ public override Stream InstallPackage(string packageName, string packageVersion, /// public override Task InstallPackageAsync(string packageName, string packageVersion, bool includePrerelease, ConcurrentQueue errorMsgs, ConcurrentQueue warningMsgs, ConcurrentQueue debugMsgs, ConcurrentQueue verboseMsgs) { - throw new NotImplementedException("InstallPackageAsync is not implemented for NuGetServerAPICalls."); + debugMsgs.Enqueue("In NuGetServerAPICalls::InstallPackageAsync()"); + Stream results = InstallPackage(packageName, packageVersion, includePrerelease, out ErrorRecord errRecord); + if (errRecord != null) + { + errorMsgs.Enqueue(errRecord); + } + + return Task.FromResult(results); } /// @@ -572,6 +628,55 @@ private HttpContent HttpRequestCallForContent(string requestUrl, out ErrorRecord return content; } + /// + /// Helper method that makes the HTTP request for the NuGet server protocol url passed in for async find APIs. + /// This helper writes diagnostics to the provided debug queue and avoids cmdlet stream writes. + /// + private string HttpRequestCallAsync(string requestUrl, ConcurrentQueue debugMsgs, out ErrorRecord errRecord) + { + debugMsgs.Enqueue("In NuGetServerAPICalls::HttpRequestCallAsync()"); + errRecord = null; + string response = string.Empty; + + try + { + debugMsgs.Enqueue($"Request url is: '{requestUrl}'"); + HttpRequestMessage request = new HttpRequestMessage(HttpMethod.Get, requestUrl); + response = SendRequestAsync(request, _sessionClient).GetAwaiter().GetResult(); + } + catch (HttpRequestException e) + { + errRecord = new ErrorRecord( + exception: e, + "HttpRequestFallFailure", + ErrorCategory.ConnectionError, + this); + } + catch (ArgumentNullException e) + { + errRecord = new ErrorRecord( + exception: e, + "HttpRequestFallFailure", + ErrorCategory.ConnectionError, + this); + } + catch (InvalidOperationException e) + { + errRecord = new ErrorRecord( + exception: e, + "HttpRequestFallFailure", + ErrorCategory.ConnectionError, + this); + } + + if (string.IsNullOrEmpty(response)) + { + debugMsgs.Enqueue("Response is empty"); + } + + return response; + } + #endregion #region Private Methods diff --git a/src/code/V2ServerAPICalls.cs b/src/code/V2ServerAPICalls.cs index 6f2c5c3e1..963ed5ed7 100644 --- a/src/code/V2ServerAPICalls.cs +++ b/src/code/V2ServerAPICalls.cs @@ -1074,7 +1074,6 @@ private string HttpRequestCall(string requestUrlV2, out ErrorRecord errRecord) /// private async Task HttpRequestCallAsync(string requestUrlV2, ConcurrentQueue errorMsgs, ConcurrentQueue warningMsgs, ConcurrentQueue debugMsgs, ConcurrentQueue verboseMsgs) { - // TODO: Async methods cannot have out ref, so currently handling errorRecords as thrown exceptions. debugMsgs.Enqueue("In V2ServerAPICalls::HttpRequestCallAsync()"); string response = string.Empty; @@ -1131,7 +1130,6 @@ private async Task HttpRequestCallAsync(string requestUrlV2, ConcurrentQ /// private async Task HttpRequestCallForContentAsync(string requestUrlV2, ConcurrentQueue errorMsgs, ConcurrentQueue warningMsgs, ConcurrentQueue debugMsgs, ConcurrentQueue verboseMsgs) { - // TODO: Async methods cannot have out ref, so need to handle errorRecords a different way. debugMsgs.Enqueue("In V2ServerAPICalls::HttpRequestCallForContentAsync()"); HttpContent content = null; @@ -1744,13 +1742,7 @@ public override async Task FindVersionGlobbingAsync(string packageN debugMsgs.Enqueue($"Count is '{count}'"); // skip 100 skip += 100; - // TODO: this should be an async method - var tmpResponse = FindVersionGlobbing(packageName, versionRange, includePrerelease, type, skip, getOnlyLatest, out ErrorRecord errRecord); - if (errRecord != null) - { - Utils.EnqueueIfNotNull(errorMsgs, errRecord); - return new FindResults(stringResponse: Utils.EmptyStrArray, hashtableResponse: emptyHashResponses, responseType: v2FindResponseType); - } + var tmpResponse = await FindVersionGlobbingAsync(packageName, versionRange, includePrerelease, type, skip, getOnlyLatest, errorMsgs, warningMsgs, debugMsgs, verboseMsgs); responses.Add(tmpResponse); count--; } diff --git a/src/code/V3ServerAPICalls.cs b/src/code/V3ServerAPICalls.cs index a2beb1545..234fc513a 100644 --- a/src/code/V3ServerAPICalls.cs +++ b/src/code/V3ServerAPICalls.cs @@ -89,14 +89,43 @@ public V3ServerAPICalls(PSRepositoryInfo repository, PSCmdlet cmdletPassedIn, Ne #region Overridden Methods + /// + /// Async find method which allows for searching for single name with specific version. + /// Name: no wildcard support + /// Version: no wildcard support + /// Examples: Search "NuGet.Server.Core" "3.0.0-beta" + /// This is the concurrent (parallel) counterpart of FindVersion(). + /// public override Task FindVersionAsync(string packageName, string version, ResourceType type, ConcurrentQueue errorMsgs, ConcurrentQueue warningMsgs, ConcurrentQueue debugMsgs, ConcurrentQueue verboseMsgs) { - throw new NotImplementedException("FindVersionAsync is not implemented for V3ServerAPICalls."); + debugMsgs.Enqueue("In V3ServerAPICalls::FindVersionAsync()"); + FindResults findResponse = FindVersionHelper(packageName, version, tags: Utils.EmptyStrArray, type, out ErrorRecord errRecord, errorMsgs, debugMsgs, verboseMsgs); + if (errRecord != null) + { + errorMsgs.Enqueue(errRecord); + } + + return Task.FromResult(findResponse); } + /// + /// Async find method which allows for searching for single name with version range. + /// Name: no wildcard support + /// Version: supports wildcards + /// Examples: Search "NuGet.Server.Core" "[1.0.0.0, 5.0.0.0]" + /// Search "NuGet.Server.Core" "3.*" + /// This is the concurrent (parallel) counterpart of FindVersionGlobbing(). + /// public override Task FindVersionGlobbingAsync(string packageName, VersionRange versionRange, bool includePrerelease, ResourceType type, bool getOnlyLatest, ConcurrentQueue errorMsgs, ConcurrentQueue warningMsgs, ConcurrentQueue debugMsgs, ConcurrentQueue verboseMsgs) { - throw new NotImplementedException("FindVersionAsync is not implemented for V3ServerAPICalls."); + debugMsgs.Enqueue("In V3ServerAPICalls::FindVersionGlobbingAsync()"); + FindResults findResponse = FindVersionGlobbingHelper(packageName, versionRange, includePrerelease, type, getOnlyLatest, out ErrorRecord errRecord, errorMsgs, debugMsgs, verboseMsgs); + if (errRecord != null) + { + errorMsgs.Enqueue(errRecord); + } + + return Task.FromResult(findResponse); } /// @@ -122,9 +151,15 @@ public override FindResults FindAll(bool includePrerelease, ResourceType type, o public override FindResults FindTags(string[] tags, bool includePrerelease, ResourceType type, out ErrorRecord errRecord) { _cmdletPassedIn.WriteDebug("In V3ServerAPICalls::FindTags()"); + ConcurrentQueue errorMsgs = new ConcurrentQueue(); + ConcurrentQueue warningMsgs = new ConcurrentQueue(); + ConcurrentQueue debugMsgs = new ConcurrentQueue(); + ConcurrentQueue verboseMsgs = new ConcurrentQueue(); if (_isNuGetRepo || _isJFrogRepo) { - return FindTagsFromNuGetRepo(tags, includePrerelease, out errRecord); + FindResults findResults = FindTagsFromNuGetRepo(tags, includePrerelease, out errRecord, errorMsgs, debugMsgs, verboseMsgs); + Utils.WriteOutConcurrentQueue(_cmdletPassedIn, errorMsgs, warningMsgs, debugMsgs, verboseMsgs); + return findResults; } else { @@ -134,6 +169,7 @@ public override FindResults FindTags(string[] tags, bool includePrerelease, Reso ErrorCategory.InvalidOperation, this); + Utils.WriteOutConcurrentQueue(_cmdletPassedIn, errorMsgs, warningMsgs, debugMsgs, verboseMsgs); return new FindResults(stringResponse: Utils.EmptyStrArray, hashtableResponse: emptyHashResponses, responseType: v3FindResponseType); } } @@ -163,12 +199,31 @@ public override FindResults FindCommandOrDscResource(string[] tags, bool include public override FindResults FindName(string packageName, bool includePrerelease, ResourceType type, out ErrorRecord errRecord) { _cmdletPassedIn.WriteDebug("In V3ServerAPICalls::FindName()"); - return FindNameHelper(packageName, tags: Utils.EmptyStrArray, includePrerelease, type, out errRecord); + ConcurrentQueue errorMsgs = new ConcurrentQueue(); + ConcurrentQueue warningMsgs = new ConcurrentQueue(); + ConcurrentQueue debugMsgs = new ConcurrentQueue(); + ConcurrentQueue verboseMsgs = new ConcurrentQueue(); + FindResults findResults = FindNameHelper(packageName, tags: Utils.EmptyStrArray, includePrerelease, type, out errRecord, errorMsgs, debugMsgs, verboseMsgs); + Utils.WriteOutConcurrentQueue(_cmdletPassedIn, errorMsgs, warningMsgs, debugMsgs, verboseMsgs); + return findResults; } + /// + /// Async find method which allows for searching for single name and returns latest version. + /// Name: no wildcard support + /// Examples: Search "Newtonsoft.Json" + /// This is the concurrent (parallel) counterpart of FindName(). + /// public override Task FindNameAsync(string packageName, bool includePrerelease, ResourceType type, ConcurrentQueue errorMsgs, ConcurrentQueue warningMsgs, ConcurrentQueue debugMsgs, ConcurrentQueue verboseMsgs) { - throw new NotImplementedException("FindVersionAsync is not implemented for V3ServerAPICalls."); + debugMsgs.Enqueue("In V3ServerAPICalls::FindNameAsync()"); + FindResults findResponse = FindNameHelper(packageName, tags: Utils.EmptyStrArray, includePrerelease, type, out ErrorRecord errRecord, errorMsgs, debugMsgs, verboseMsgs); + if (errRecord != null) + { + errorMsgs.Enqueue(errRecord); + } + + return Task.FromResult(findResponse); } /// @@ -180,7 +235,13 @@ public override Task FindNameAsync(string packageName, bool include public override FindResults FindNameWithTag(string packageName, string[] tags, bool includePrerelease, ResourceType type, out ErrorRecord errRecord) { _cmdletPassedIn.WriteDebug("In V3ServerAPICalls::FindNameWithTag()"); - return FindNameHelper(packageName, tags, includePrerelease, type, out errRecord); + ConcurrentQueue errorMsgs = new ConcurrentQueue(); + ConcurrentQueue warningMsgs = new ConcurrentQueue(); + ConcurrentQueue debugMsgs = new ConcurrentQueue(); + ConcurrentQueue verboseMsgs = new ConcurrentQueue(); + FindResults findResults = FindNameHelper(packageName, tags, includePrerelease, type, out errRecord, errorMsgs, debugMsgs, verboseMsgs); + Utils.WriteOutConcurrentQueue(_cmdletPassedIn, errorMsgs, warningMsgs, debugMsgs, verboseMsgs); + return findResults; } /// @@ -190,9 +251,15 @@ public override FindResults FindNameWithTag(string packageName, string[] tags, b public override FindResults FindNameGlobbing(string packageName, bool includePrerelease, ResourceType type, out ErrorRecord errRecord) { _cmdletPassedIn.WriteDebug("In V3ServerAPICalls::FindNameGlobbing()"); + ConcurrentQueue errorMsgs = new ConcurrentQueue(); + ConcurrentQueue warningMsgs = new ConcurrentQueue(); + ConcurrentQueue debugMsgs = new ConcurrentQueue(); + ConcurrentQueue verboseMsgs = new ConcurrentQueue(); if (_isNuGetRepo || _isJFrogRepo || _isGHPkgsRepo || _isMyGetRepo) { - return FindNameGlobbingFromNuGetRepo(packageName, tags: Utils.EmptyStrArray, includePrerelease, out errRecord); + FindResults findResults = FindNameGlobbingFromNuGetRepo(packageName, tags: Utils.EmptyStrArray, includePrerelease, out errRecord, errorMsgs, debugMsgs, verboseMsgs); + Utils.WriteOutConcurrentQueue(_cmdletPassedIn, errorMsgs, warningMsgs, debugMsgs, verboseMsgs); + return findResults; } else { @@ -202,6 +269,7 @@ public override FindResults FindNameGlobbing(string packageName, bool includePre ErrorCategory.InvalidOperation, this); + Utils.WriteOutConcurrentQueue(_cmdletPassedIn, errorMsgs, warningMsgs, debugMsgs, verboseMsgs); return new FindResults(stringResponse: Utils.EmptyStrArray, hashtableResponse: emptyHashResponses, responseType: v3FindResponseType); } } @@ -213,9 +281,15 @@ public override FindResults FindNameGlobbing(string packageName, bool includePre public override FindResults FindNameGlobbingWithTag(string packageName, string[] tags, bool includePrerelease, ResourceType type, out ErrorRecord errRecord) { _cmdletPassedIn.WriteDebug("In V3ServerAPICalls::FindNameGlobbingWithTag()"); + ConcurrentQueue errorMsgs = new ConcurrentQueue(); + ConcurrentQueue warningMsgs = new ConcurrentQueue(); + ConcurrentQueue debugMsgs = new ConcurrentQueue(); + ConcurrentQueue verboseMsgs = new ConcurrentQueue(); if (_isNuGetRepo || _isJFrogRepo || _isGHPkgsRepo || _isMyGetRepo) { - return FindNameGlobbingFromNuGetRepo(packageName, tags, includePrerelease, out errRecord); + FindResults findResults = FindNameGlobbingFromNuGetRepo(packageName, tags, includePrerelease, out errRecord, errorMsgs, debugMsgs, verboseMsgs); + Utils.WriteOutConcurrentQueue(_cmdletPassedIn, errorMsgs, warningMsgs, debugMsgs, verboseMsgs); + return findResults; } else { @@ -225,6 +299,7 @@ public override FindResults FindNameGlobbingWithTag(string packageName, string[] ErrorCategory.InvalidOperation, this); + Utils.WriteOutConcurrentQueue(_cmdletPassedIn, errorMsgs, warningMsgs, debugMsgs, verboseMsgs); return new FindResults(stringResponse: Utils.EmptyStrArray, hashtableResponse: emptyHashResponses, responseType: v3FindResponseType); } } @@ -239,8 +314,19 @@ public override FindResults FindNameGlobbingWithTag(string packageName, string[] /// public override FindResults FindVersionGlobbing(string packageName, VersionRange versionRange, bool includePrerelease, ResourceType type, bool getOnlyLatest, out ErrorRecord errRecord) { - _cmdletPassedIn.WriteDebug("In V3ServerAPICalls::FindVersionGlobbing()"); - string[] versionedResponses = GetVersionedPackageEntriesFromRegistrationsResource(packageName, catalogEntryProperty, isSearch: true, out errRecord); + ConcurrentQueue errorMsgs = new ConcurrentQueue(); + ConcurrentQueue warningMsgs = new ConcurrentQueue(); + ConcurrentQueue debugMsgs = new ConcurrentQueue(); + ConcurrentQueue verboseMsgs = new ConcurrentQueue(); + FindResults findResults = FindVersionGlobbingHelper(packageName, versionRange, includePrerelease, type, getOnlyLatest, out errRecord, errorMsgs, debugMsgs, verboseMsgs); + Utils.WriteOutConcurrentQueue(_cmdletPassedIn, errorMsgs, warningMsgs, debugMsgs, verboseMsgs); + return findResults; + } + + private FindResults FindVersionGlobbingHelper(string packageName, VersionRange versionRange, bool includePrerelease, ResourceType type, bool getOnlyLatest, out ErrorRecord errRecord, ConcurrentQueue errorMsgs, ConcurrentQueue debugMsgs, ConcurrentQueue verboseMsgs) + { + debugMsgs.Enqueue("In V3ServerAPICalls::FindVersionGlobbing()"); + string[] versionedResponses = GetVersionedPackageEntriesFromRegistrationsResource(packageName, catalogEntryProperty, isSearch: true, out errRecord, errorMsgs, debugMsgs, verboseMsgs); if (errRecord != null) { return new FindResults(stringResponse: Utils.EmptyStrArray, hashtableResponse: emptyHashResponses, responseType: v3FindResponseType); @@ -267,7 +353,7 @@ public override FindResults FindVersionGlobbing(string packageName, VersionRange if (NuGetVersion.TryParse(pkgVersionElement.ToString(), out NuGetVersion pkgVersion) && versionRange.Satisfies(pkgVersion)) { - _cmdletPassedIn.WriteDebug($"Package version parsed as '{pkgVersion}' satisfies the version range"); + debugMsgs.Enqueue($"Package version parsed as '{pkgVersion}' satisfies the version range"); if (!pkgVersion.IsPrerelease || includePrerelease) { satisfyingVersions.Add(response); @@ -300,7 +386,13 @@ public override FindResults FindVersionGlobbing(string packageName, VersionRange public override FindResults FindVersion(string packageName, string version, ResourceType type, out ErrorRecord errRecord) { _cmdletPassedIn.WriteDebug("In V3ServerAPICalls::FindVersion()"); - return FindVersionHelper(packageName, version, tags: Utils.EmptyStrArray, type, out errRecord); + ConcurrentQueue errorMsgs = new ConcurrentQueue(); + ConcurrentQueue warningMsgs = new ConcurrentQueue(); + ConcurrentQueue debugMsgs = new ConcurrentQueue(); + ConcurrentQueue verboseMsgs = new ConcurrentQueue(); + FindResults findResults = FindVersionHelper(packageName, version, tags: Utils.EmptyStrArray, type, out errRecord, errorMsgs, debugMsgs, verboseMsgs); + Utils.WriteOutConcurrentQueue(_cmdletPassedIn, errorMsgs, warningMsgs, debugMsgs, verboseMsgs); + return findResults; } /// @@ -313,7 +405,13 @@ public override FindResults FindVersion(string packageName, string version, Reso public override FindResults FindVersionWithTag(string packageName, string version, string[] tags, ResourceType type, out ErrorRecord errRecord) { _cmdletPassedIn.WriteDebug("In V3ServerAPICalls::FindVersionWithTag()"); - return FindVersionHelper(packageName, version, tags: tags, type, out errRecord); + ConcurrentQueue errorMsgs = new ConcurrentQueue(); + ConcurrentQueue warningMsgs = new ConcurrentQueue(); + ConcurrentQueue debugMsgs = new ConcurrentQueue(); + ConcurrentQueue verboseMsgs = new ConcurrentQueue(); + FindResults findResults = FindVersionHelper(packageName, version, tags: tags, type, out errRecord, errorMsgs, debugMsgs, verboseMsgs); + Utils.WriteOutConcurrentQueue(_cmdletPassedIn, errorMsgs, warningMsgs, debugMsgs, verboseMsgs); + return findResults; } /** INSTALL APIS **/ @@ -328,8 +426,11 @@ public override FindResults FindVersionWithTag(string packageName, string versio /// public override Stream InstallPackage(string packageName, string packageVersion, bool includePrerelease, out ErrorRecord errRecord) { - // TODO: pass in ConcurrentQueue to write out debug message. - //_cmdletPassedIn.WriteDebug("In V3ServerAPICalls::InstallPackage()"); + ConcurrentQueue errorMsgs = new ConcurrentQueue(); + ConcurrentQueue warningMsgs = new ConcurrentQueue(); + ConcurrentQueue debugMsgs = new ConcurrentQueue(); + ConcurrentQueue verboseMsgs = new ConcurrentQueue(); + debugMsgs.Enqueue("In V3ServerAPICalls::InstallPackage()"); Stream results = new MemoryStream(); if (string.IsNullOrEmpty(packageVersion)) { @@ -339,10 +440,13 @@ public override Stream InstallPackage(string packageName, string packageVersion, ErrorCategory.InvalidArgument, _cmdletPassedIn); + Utils.WriteOutConcurrentQueue(_cmdletPassedIn, errorMsgs, warningMsgs, debugMsgs, verboseMsgs); return results; } - return InstallVersion(packageName, packageVersion, out errRecord); + Stream installResults = InstallVersion(packageName, packageVersion, out errRecord, errorMsgs, debugMsgs, verboseMsgs); + Utils.WriteOutConcurrentQueue(_cmdletPassedIn, errorMsgs, warningMsgs, debugMsgs, verboseMsgs); + return installResults; } /// @@ -353,9 +457,34 @@ public override Stream InstallPackage(string packageName, string packageVersion, /// Examples: Install "PowerShellGet" -Version "3.5.0-alpha" /// Install "PowerShellGet" -Version "3.0.0" /// - public override Task InstallPackageAsync(string packageName, string packageVersion, bool includePrerelease, ConcurrentQueue errorMsgs, ConcurrentQueue warningMsgs, ConcurrentQueue debugMsgs, ConcurrentQueue verboseMsgs) + public override async Task InstallPackageAsync(string packageName, string packageVersion, bool includePrerelease, ConcurrentQueue errorMsgs, ConcurrentQueue warningMsgs, ConcurrentQueue debugMsgs, ConcurrentQueue verboseMsgs) { - throw new NotImplementedException("InstallPackageAsync is not implemented for NuGetServerAPICalls."); + debugMsgs.Enqueue("In V3ServerAPICalls::InstallPackageAsync()"); + Stream results = new MemoryStream(); + if (string.IsNullOrEmpty(packageVersion)) + { + errorMsgs.Enqueue(new ErrorRecord( + exception: new ArgumentNullException($"Package version could not be found for {packageName}"), + "PackageVersionNullOrEmptyError", + ErrorCategory.InvalidArgument, + this)); + + return results; + } + + if (!NuGetVersion.TryParse(packageVersion, out NuGetVersion requiredVersion)) + { + errorMsgs.Enqueue(new ErrorRecord( + new ArgumentException($"Version {packageVersion} to be installed is not a valid NuGet version."), + "InstallVersionFailure", + ErrorCategory.InvalidArgument, + this)); + + return results; + } + + results = await InstallHelperAsync(packageName, requiredVersion, errorMsgs, warningMsgs, debugMsgs, verboseMsgs); + return results; } #endregion @@ -365,9 +494,9 @@ public override Task InstallPackageAsync(string packageName, string pack /// /// Helper method called by FindNameGlobbing() and FindNameGlobbingWithTag() for special case where repository is NuGet.org repository. /// - private FindResults FindNameGlobbingFromNuGetRepo(string packageName, string[] tags, bool includePrerelease, out ErrorRecord errRecord) + private FindResults FindNameGlobbingFromNuGetRepo(string packageName, string[] tags, bool includePrerelease, out ErrorRecord errRecord, ConcurrentQueue errorMsgs, ConcurrentQueue debugMsgs, ConcurrentQueue verboseMsgs) { - _cmdletPassedIn.WriteDebug("In V3ServerAPICalls::FindNameGlobbingFromNuGetRepo()"); + debugMsgs.Enqueue("In V3ServerAPICalls::FindNameGlobbingFromNuGetRepo()"); var names = packageName.Split(new char[] { '*' }, StringSplitOptions.RemoveEmptyEntries); string querySearchTerm; @@ -403,7 +532,7 @@ private FindResults FindNameGlobbingFromNuGetRepo(string packageName, string[] t return new FindResults(stringResponse: Utils.EmptyStrArray, hashtableResponse: emptyHashResponses, responseType: v3FindResponseType); } - var matchingPkgEntries = GetVersionedPackageEntriesFromSearchQueryResource(querySearchTerm, includePrerelease, out errRecord); + var matchingPkgEntries = GetVersionedPackageEntriesFromSearchQueryResource(querySearchTerm, includePrerelease, out errRecord, errorMsgs, debugMsgs, verboseMsgs); if (errRecord != null) { return new FindResults(stringResponse: Utils.EmptyStrArray, hashtableResponse: emptyHashResponses, responseType: v3FindResponseType); @@ -477,13 +606,13 @@ private FindResults FindNameGlobbingFromNuGetRepo(string packageName, string[] t /// /// Helper method called by FindTags() for special case where repository is NuGet.org repository. /// - private FindResults FindTagsFromNuGetRepo(string[] tags, bool includePrerelease, out ErrorRecord errRecord) + private FindResults FindTagsFromNuGetRepo(string[] tags, bool includePrerelease, out ErrorRecord errRecord, ConcurrentQueue errorMsgs, ConcurrentQueue debugMsgs, ConcurrentQueue verboseMsgs) { - _cmdletPassedIn.WriteDebug("In V3ServerAPICalls::FindTagsFromNuGetRepo()"); + debugMsgs.Enqueue("In V3ServerAPICalls::FindTagsFromNuGetRepo()"); string tagsQueryTerm = $"tags:{String.Join(" ", tags)}"; // Get responses for all packages that contain the required tags // example query: - var tagPkgEntries = GetVersionedPackageEntriesFromSearchQueryResource(tagsQueryTerm, includePrerelease, out errRecord); + var tagPkgEntries = GetVersionedPackageEntriesFromSearchQueryResource(tagsQueryTerm, includePrerelease, out errRecord, errorMsgs, debugMsgs, verboseMsgs); if (errRecord != null) { return new FindResults(stringResponse: Utils.EmptyStrArray, hashtableResponse: emptyHashResponses, responseType: v3FindResponseType); @@ -512,10 +641,10 @@ private FindResults FindTagsFromNuGetRepo(string[] tags, bool includePrerelease, /// /// Helper method called by FindName() and FindNameWithTag() /// - private FindResults FindNameHelper(string packageName, string[] tags, bool includePrerelease, ResourceType type, out ErrorRecord errRecord) + private FindResults FindNameHelper(string packageName, string[] tags, bool includePrerelease, ResourceType type, out ErrorRecord errRecord, ConcurrentQueue errorMsgs, ConcurrentQueue debugMsgs, ConcurrentQueue verboseMsgs) { - _cmdletPassedIn.WriteDebug("In V3ServerAPICalls::FindNameHelper()"); - string[] versionedResponses = GetVersionedPackageEntriesFromRegistrationsResource(packageName, catalogEntryProperty, isSearch: true, out errRecord); + debugMsgs.Enqueue("In V3ServerAPICalls::FindNameHelper()"); + string[] versionedResponses = GetVersionedPackageEntriesFromRegistrationsResource(packageName, catalogEntryProperty, isSearch: true, out errRecord, errorMsgs, debugMsgs, verboseMsgs); if (errRecord != null) { return new FindResults(stringResponse: Utils.EmptyStrArray, hashtableResponse: emptyHashResponses, responseType: v3FindResponseType); @@ -553,7 +682,7 @@ private FindResults FindNameHelper(string packageName, string[] tags, bool inclu if (NuGetVersion.TryParse(pkgVersionElement.ToString(), out NuGetVersion pkgVersion)) { - _cmdletPassedIn.WriteDebug($"'{packageName}' version parsed as '{pkgVersion}'"); + debugMsgs.Enqueue($"'{packageName}' version parsed as '{pkgVersion}'"); if (!pkgVersion.IsPrerelease || includePrerelease) { // Versions are always in descending order i.e 5.0.0, 3.0.0, 1.0.0 so grabbing the first match suffices @@ -608,9 +737,9 @@ private FindResults FindNameHelper(string packageName, string[] tags, bool inclu /// /// Helper method called by FindVersion() and FindVersionWithTag() /// - private FindResults FindVersionHelper(string packageName, string version, string[] tags, ResourceType type, out ErrorRecord errRecord) + private FindResults FindVersionHelper(string packageName, string version, string[] tags, ResourceType type, out ErrorRecord errRecord, ConcurrentQueue errorMsgs, ConcurrentQueue debugMsgs, ConcurrentQueue verboseMsgs) { - _cmdletPassedIn.WriteDebug("In V3ServerAPICalls::FindVersionHelper()"); + debugMsgs.Enqueue("In V3ServerAPICalls::FindVersionHelper()"); if (!NuGetVersion.TryParse(version, out NuGetVersion requiredVersion)) { errRecord = new ErrorRecord( @@ -621,9 +750,9 @@ private FindResults FindVersionHelper(string packageName, string version, string return new FindResults(stringResponse: Utils.EmptyStrArray, hashtableResponse: emptyHashResponses, responseType: v3FindResponseType); } - _cmdletPassedIn.WriteDebug($"'{packageName}' version parsed as '{requiredVersion}'"); + //_cmdletPassedIn.WriteDebug($"'{packageName}' version parsed as '{requiredVersion}'"); - string[] versionedResponses = GetVersionedPackageEntriesFromRegistrationsResource(packageName, catalogEntryProperty, isSearch: true, out errRecord); + string[] versionedResponses = GetVersionedPackageEntriesFromRegistrationsResource(packageName, catalogEntryProperty, isSearch: true, out errRecord, errorMsgs, debugMsgs, verboseMsgs); if (errRecord != null) { return new FindResults(stringResponse: Utils.EmptyStrArray, hashtableResponse: emptyHashResponses, responseType: v3FindResponseType); @@ -716,10 +845,10 @@ private FindResults FindVersionHelper(string packageName, string version, string /// Name: no wildcard support. /// Examples: Install "Newtonsoft.json" /// - private Stream InstallName(string packageName, out ErrorRecord errRecord) + private Stream InstallName(string packageName, out ErrorRecord errRecord, ConcurrentQueue errorMsgs, ConcurrentQueue debugMsgs, ConcurrentQueue verboseMsgs) { - _cmdletPassedIn.WriteDebug("In V3ServerAPICalls::InstallName()"); - return InstallHelper(packageName, version: null, out errRecord); + debugMsgs.Enqueue("In V3ServerAPICalls::InstallName()"); + return InstallHelper(packageName, version: null, out errRecord, errorMsgs, debugMsgs, verboseMsgs); } /// @@ -729,10 +858,9 @@ private Stream InstallName(string packageName, out ErrorRecord errRecord) /// Examples: Install "Newtonsoft.json" -Version "1.0.0.0" /// Install "Newtonsoft.json" -Version "2.5.0-beta" /// - private Stream InstallVersion(string packageName, string version, out ErrorRecord errRecord) + private Stream InstallVersion(string packageName, string version, out ErrorRecord errRecord, ConcurrentQueue errorMsgs, ConcurrentQueue debugMsgs, ConcurrentQueue verboseMsgs) { - // TODO: pass in ConcurrentQueue to write out debug message. - _cmdletPassedIn.WriteDebug("In V3ServerAPICalls::InstallVersion()"); + debugMsgs.Enqueue("In V3ServerAPICalls::InstallVersion()"); if (!NuGetVersion.TryParse(version, out NuGetVersion requiredVersion)) { errRecord = new ErrorRecord( @@ -744,17 +872,16 @@ private Stream InstallVersion(string packageName, string version, out ErrorRecor return null; } - return InstallHelper(packageName, requiredVersion, out errRecord); + return InstallHelper(packageName, requiredVersion, out errRecord, errorMsgs, debugMsgs, verboseMsgs); } /// /// Helper method that is called by InstallName() and InstallVersion() /// For InstallName() we want latest version installed (so version parameter passed in will be null), for InstallVersion() we want specified, non-null version installed. /// - private Stream InstallHelper(string packageName, NuGetVersion version, out ErrorRecord errRecord) + private Stream InstallHelper(string packageName, NuGetVersion version, out ErrorRecord errRecord, ConcurrentQueue errorMsgs, ConcurrentQueue debugMsgs, ConcurrentQueue verboseMsgs) { - // TODO: pass in ConcurrentQueue to write out debug message. - //_cmdletPassedIn.WriteDebug("In V3ServerAPICalls::InstallHelper()"); + debugMsgs.Enqueue("In V3ServerAPICalls::InstallHelper()"); Stream pkgStream = null; bool getLatestVersion = true; if (version != null) @@ -762,7 +889,7 @@ private Stream InstallHelper(string packageName, NuGetVersion version, out Error getLatestVersion = false; } - string[] versionedResponses = GetVersionedPackageEntriesFromRegistrationsResource(packageName, packageContentProperty, isSearch: false, out errRecord); + string[] versionedResponses = GetVersionedPackageEntriesFromRegistrationsResource(packageName, packageContentProperty, isSearch: false, out errRecord, errorMsgs, debugMsgs, verboseMsgs); if (errRecord != null) { return pkgStream; @@ -810,7 +937,7 @@ private Stream InstallHelper(string packageName, NuGetVersion version, out Error return null; } - var content = HttpRequestCallForContent(pkgContentUrl, out errRecord); + var content = HttpRequestCallForContent(pkgContentUrl, out errRecord, errorMsgs, debugMsgs, verboseMsgs); if (errRecord != null) { return null; @@ -830,29 +957,110 @@ private Stream InstallHelper(string packageName, NuGetVersion version, out Error return content.ReadAsStreamAsync().GetAwaiter().GetResult(); } + /// + /// Helper method that is called by InstallPackageAsync() + /// For InstallName() we want latest version installed (so version parameter passed in will be null), for InstallVersion() we want specified, non-null version installed. + /// This is the async counterpart of InstallHelper() used for concurrent (parallel) installation workflows. + /// + private async Task InstallHelperAsync(string packageName, NuGetVersion version, ConcurrentQueue errorMsgs, ConcurrentQueue warningMsgs, ConcurrentQueue debugMsgs, ConcurrentQueue verboseMsgs) + { + debugMsgs.Enqueue("In V3ServerAPICalls::InstallHelperAsync()"); + Stream pkgStream = null; + bool getLatestVersion = true; + if (version != null) + { + getLatestVersion = false; + } + + string[] versionedResponses = GetVersionedPackageEntriesFromRegistrationsResource(packageName, packageContentProperty, isSearch: false, out ErrorRecord errRecord, errorMsgs, debugMsgs, verboseMsgs); + if (errRecord != null) + { + errorMsgs.Enqueue(errRecord); + return pkgStream; + } + + if (versionedResponses.Length == 0) + { + errorMsgs.Enqueue(new ErrorRecord( + new Exception($"Package with name '{packageName}' and version '{version}' could not be found in repository '{Repository.Name}'"), + "InstallFailure", + ErrorCategory.InvalidResult, + this)); + + return null; + } + + string pkgContentUrl = String.Empty; + if (getLatestVersion) + { + pkgContentUrl = versionedResponses[0]; + } + else + { + // loop through responses to find one containing required version + foreach (string response in versionedResponses) + { + // Response will be "packageContent" element value that looks like: "{packageBaseAddress}/{packageName}/{normalizedVersion}/{packageName}.{normalizedVersion}.nupkg" + // Ex: https://api.nuget.org/v3-flatcontainer/test_module/1.0.0/test_module.1.0.0.nupkg + if (response.Contains(version.ToNormalizedString())) + { + pkgContentUrl = response; + break; + } + } + } + + if (String.IsNullOrEmpty(pkgContentUrl)) + { + errorMsgs.Enqueue(new ErrorRecord( + new Exception($"Package with name '{packageName}' and version '{version}' could not be found in repository '{Repository.Name}'"), + "InstallFailure", + ErrorCategory.InvalidResult, + this)); + + return null; + } + + var content = await HttpRequestCallForContentAsync(pkgContentUrl, errorMsgs, warningMsgs, debugMsgs, verboseMsgs); + + if (content is null) + { + errorMsgs.Enqueue(new ErrorRecord( + new Exception($"No content was returned by repository '{Repository.Name}'"), + "InstallFailureContentNullv3Async", + ErrorCategory.InvalidResult, + this)); + + return new MemoryStream(); + } + + pkgStream = await content.ReadAsStreamAsync(); + + return pkgStream; + } + /// /// Gets the versioned package entries from the RegistrationsBaseUrl resource /// i.e when the package Name being searched for does not contain wildcard /// This is called by FindNameHelper(), FindVersionHelper(), FindVersionGlobbing(), InstallHelper() /// - private string[] GetVersionedPackageEntriesFromRegistrationsResource(string packageName, string propertyName, bool isSearch, out ErrorRecord errRecord) + private string[] GetVersionedPackageEntriesFromRegistrationsResource(string packageName, string propertyName, bool isSearch, out ErrorRecord errRecord, ConcurrentQueue errorMsgs, ConcurrentQueue debugMsgs, ConcurrentQueue verboseMsgs) { - // TODO: pass in ConcurrentQueue to write out debug message. - //_cmdletPassedIn.WriteDebug("In V3ServerAPICalls::GetVersionedPackageEntriesFromRegistrationsResource()"); + debugMsgs.Enqueue("In V3ServerAPICalls::GetVersionedPackageEntriesFromRegistrationsResource()"); string[] responses = Utils.EmptyStrArray; - Dictionary resources = GetResourcesFromServiceIndex(out errRecord); + Dictionary resources = GetResourcesFromServiceIndex(out errRecord, errorMsgs, debugMsgs, verboseMsgs); if (errRecord != null) { return responses; } - string registrationsBaseUrl = FindRegistrationsBaseUrl(resources, out errRecord); + string registrationsBaseUrl = FindRegistrationsBaseUrl(resources, out errRecord, errorMsgs, debugMsgs, verboseMsgs); if (errRecord != null) { return responses; } - responses = GetVersionedResponsesFromRegistrationsResource(registrationsBaseUrl, packageName, propertyName, isSearch, out errRecord); + responses = GetVersionedResponsesFromRegistrationsResource(registrationsBaseUrl, packageName, propertyName, isSearch, out errRecord, errorMsgs, debugMsgs, verboseMsgs); if (errRecord != null) { return Utils.EmptyStrArray; @@ -866,11 +1074,11 @@ private string[] GetVersionedPackageEntriesFromRegistrationsResource(string pack /// i.e when the package Name being searched for contains wildcards or a Tag query search is performed /// This is called by FindNameGlobbingFromNuGetRepo() and FindTagsFromNuGetRepo() /// - private List GetVersionedPackageEntriesFromSearchQueryResource(string queryTerm, bool includePrerelease, out ErrorRecord errRecord) + private List GetVersionedPackageEntriesFromSearchQueryResource(string queryTerm, bool includePrerelease, out ErrorRecord errRecord, ConcurrentQueue errorMsgs, ConcurrentQueue debugMsgs, ConcurrentQueue verboseMsgs) { - _cmdletPassedIn.WriteDebug("In V3ServerAPICalls::GetVersionedPackageEntriesFromSearchQueryResource()"); + debugMsgs.Enqueue("In V3ServerAPICalls::GetVersionedPackageEntriesFromSearchQueryResource()"); List pkgEntries = new(); - Dictionary resources = GetResourcesFromServiceIndex(out errRecord); + Dictionary resources = GetResourcesFromServiceIndex(out errRecord, errorMsgs, debugMsgs, verboseMsgs); if (errRecord != null) { return pkgEntries; @@ -887,7 +1095,7 @@ private List GetVersionedPackageEntriesFromSearchQueryResource(stri string query = $"{searchQueryServiceUrl}?q={queryTerm}&prerelease={includePrerelease}&semVerLevel=2.0.0&skip={skip}&take=100"; // Get responses for all packages that contain the required tags - pkgEntries.AddRange(GetJsonElementArr(query, dataName, out int initialCount, out errRecord).ToList()); + pkgEntries.AddRange(GetJsonElementArr(query, dataName, out int initialCount, out errRecord, errorMsgs, debugMsgs, verboseMsgs).ToList()); // check count (ie "totalHits") 425 ==> count/100 ~~> 4 calls ~~> + 1 = 5 calls int count = initialCount / 100 + 1; @@ -896,7 +1104,7 @@ private List GetVersionedPackageEntriesFromSearchQueryResource(stri { skip += 100; query = $"{searchQueryServiceUrl}?q={queryTerm}&prerelease={includePrerelease}&semVerLevel=2.0.0&skip={skip}&take=100"; - pkgEntries.AddRange(GetJsonElementArr(query, dataName, out int unneededCount, out errRecord).ToList()); + pkgEntries.AddRange(GetJsonElementArr(query, dataName, out int unneededCount, out errRecord, errorMsgs, debugMsgs, verboseMsgs).ToList()); count--; } @@ -907,12 +1115,11 @@ private List GetVersionedPackageEntriesFromSearchQueryResource(stri /// Finds all resources present in the repository's service index. /// For example: https://api.nuget.org/v3/index.json /// - private Dictionary GetResourcesFromServiceIndex(out ErrorRecord errRecord) + private Dictionary GetResourcesFromServiceIndex(out ErrorRecord errRecord, ConcurrentQueue errorMsgs, ConcurrentQueue debugMsgs, ConcurrentQueue verboseMsgs) { - // TODO: pass in ConcurrentQueue to write out debug message. - //_cmdletPassedIn.WriteDebug("In V3ServerAPICalls::GetResourcesFromServiceIndex()"); + debugMsgs.Enqueue("In V3ServerAPICalls::GetResourcesFromServiceIndex()"); Dictionary resources = new Dictionary(); - JsonElement[] resourcesArray = GetJsonElementArr($"{Repository.Uri}", resourcesName, out int totalHits, out errRecord); + JsonElement[] resourcesArray = GetJsonElementArr($"{Repository.Uri}", resourcesName, out int totalHits, out errRecord, errorMsgs, debugMsgs, verboseMsgs); if (errRecord != null) { return resources; @@ -968,10 +1175,9 @@ private Dictionary GetResourcesFromServiceIndex(out ErrorRecord /// Gets the resource of type "RegistrationBaseUrl" from the repository's resources. /// A repository can have multiple resources of type "RegistrationsBaseUrl" so it finds the best match according to the guideline comment in the method. /// - private string FindRegistrationsBaseUrl(Dictionary resources, out ErrorRecord errRecord) + private string FindRegistrationsBaseUrl(Dictionary resources, out ErrorRecord errRecord, ConcurrentQueue errorMsgs, ConcurrentQueue debugMsgs, ConcurrentQueue verboseMsgs) { - // TODO: pass in ConcurrentQueue to write out debug message. - //_cmdletPassedIn.WriteDebug("In V3ServerAPICalls::FindRegistrationsBaseUrl()"); + debugMsgs.Enqueue("In V3ServerAPICalls::FindRegistrationsBaseUrl()"); errRecord = null; string registrationsBaseUrl = String.Empty; @@ -1058,15 +1264,15 @@ private string FindSearchQueryService(Dictionary resources, out /// For some packages (that we know of: JFrog repo and some packages on NuGet.org), the metadata is located under outer "items" element > "@id" element > inner "items" element /// This requires a different search. /// - private JsonElement[] GetMetadataElementFromIdLinkElement(JsonElement idLinkElement, string packageName, out string upperVersion, out ErrorRecord errRecord) + private JsonElement[] GetMetadataElementFromIdLinkElement(JsonElement idLinkElement, string packageName, out string upperVersion, out ErrorRecord errRecord, ConcurrentQueue errorMsgs, ConcurrentQueue debugMsgs, ConcurrentQueue verboseMsgs) { - _cmdletPassedIn.WriteDebug("In V3ServerAPICalls::GetMetadataElementFromIdLinkElement()"); + debugMsgs.Enqueue("In V3ServerAPICalls::GetMetadataElementFromIdLinkElement()"); upperVersion = String.Empty; JsonElement[] innerItems = new JsonElement[]{}; List innerItemsList = new List(); string metadataUri = idLinkElement.ToString(); - string response = HttpRequestCall(metadataUri, out errRecord); + string response = HttpRequestCall(metadataUri, out errRecord, errorMsgs, debugMsgs, verboseMsgs); if (errRecord != null) { if (errRecord.Exception is ResourceNotFoundException) { @@ -1102,7 +1308,7 @@ private JsonElement[] GetMetadataElementFromIdLinkElement(JsonElement idLinkElem } else { - _cmdletPassedIn.WriteDebug($"Package with name '{packageName}' did not have 'upper' property so package versions may not be in descending order."); + debugMsgs.Enqueue($"Package with name '{packageName}' did not have 'upper' property so package versions may not be in descending order."); } foreach(JsonElement entry in innerItemsElement.EnumerateArray()) @@ -1127,10 +1333,9 @@ private JsonElement[] GetMetadataElementFromIdLinkElement(JsonElement idLinkElem /// /// For most packages returned from V3 server protocol responses, the metadata is located under outer "items" element > inner "items" element. /// - private JsonElement[] GetMetadataElementFromItemsElement(JsonElement itemsElement, string packageName, out ErrorRecord errRecord) + private JsonElement[] GetMetadataElementFromItemsElement(JsonElement itemsElement, string packageName, out ErrorRecord errRecord, ConcurrentQueue errorMsgs, ConcurrentQueue debugMsgs, ConcurrentQueue verboseMsgs) { - // TODO: pass in ConcurrentQueue to write out debug message. - //_cmdletPassedIn.WriteDebug("In V3ServerAPICalls::GetMetadataElementFromItemsElement()"); + debugMsgs.Enqueue("In V3ServerAPICalls::GetMetadataElementFromItemsElement()"); errRecord = null; List innerItemsList = new List(); @@ -1160,10 +1365,9 @@ private JsonElement[] GetMetadataElementFromItemsElement(JsonElement itemsElemen /// under outer "items" element > inner "items" element (for which we call helper method GetMetadataElementFromItemsElement()), OR /// under outer "items" element > "@Id" element > inner "items" element (for which we call helper method GetMetadataElementFromIdLinkElement) /// - private string[] GetMetadataElementsFromResponse(string response, string property, string packageName, out string upperVersion, out ErrorRecord errRecord) + private string[] GetMetadataElementsFromResponse(string response, string property, string packageName, out string upperVersion, out ErrorRecord errRecord, ConcurrentQueue errorMsgs, ConcurrentQueue debugMsgs, ConcurrentQueue verboseMsgs) { - // TODO: pass in ConcurrentQueue to write out debug message. - //_cmdletPassedIn.WriteDebug("In V3ServerAPICalls::GetMetadataElementsFromResponse()"); + debugMsgs.Enqueue("In V3ServerAPICalls::GetMetadataElementsFromResponse()"); errRecord = null; upperVersion = String.Empty; List versionedPkgResponses = new List(); @@ -1197,7 +1401,7 @@ private string[] GetMetadataElementsFromResponse(string response, string propert if (currentItem.TryGetProperty(itemsName, out JsonElement currentInnerItemsElement)) { // Scenarios: NuGet.org majority responses - JsonElement[] innerItemsFromItemsElement = GetMetadataElementFromItemsElement(currentInnerItemsElement, packageName, out errRecord); + JsonElement[] innerItemsFromItemsElement = GetMetadataElementFromItemsElement(currentInnerItemsElement, packageName, out errRecord, errorMsgs, debugMsgs, verboseMsgs); if (errRecord != null) { continue; @@ -1209,8 +1413,7 @@ private string[] GetMetadataElementsFromResponse(string response, string propert } else { - // TODO: pass in ConcurrentQueue to write out debug message. - // _cmdletPassedIn.WriteDebug($"Package with name '{packageName}' did not have 'upper' property so package versions may not be in descending order."); + debugMsgs.Enqueue($"Package with name '{packageName}' did not have 'upper' property so package versions may not be in descending order."); } innerItemsElements.AddRange(innerItemsFromItemsElement); @@ -1218,7 +1421,7 @@ private string[] GetMetadataElementsFromResponse(string response, string propert else if (currentItem.TryGetProperty(idLinkName, out JsonElement idLinkElement)) { // Scenarios: JFrog responses, some NuGet.org responses - JsonElement[] innerItemsFromIdElement = GetMetadataElementFromIdLinkElement(idLinkElement, packageName, out upperVersion, out errRecord); + JsonElement[] innerItemsFromIdElement = GetMetadataElementFromIdLinkElement(idLinkElement, packageName, out upperVersion, out errRecord, errorMsgs, debugMsgs, verboseMsgs); if (errRecord != null) { continue; @@ -1228,7 +1431,7 @@ private string[] GetMetadataElementsFromResponse(string response, string propert } else { - _cmdletPassedIn.WriteDebug($"Metadata for package with name '{packageName}' did not have inner 'items' or '@Id' properties."); + debugMsgs.Enqueue($"Metadata for package with name '{packageName}' did not have inner 'items' or '@Id' properties."); } } @@ -1267,7 +1470,7 @@ private string[] GetMetadataElementsFromResponse(string response, string propert } else { - _cmdletPassedIn.WriteDebug($"Metadata for package with name '{packageName}' was not of value kind type string or object."); + debugMsgs.Enqueue($"Metadata for package with name '{packageName}' was not of value kind type string or object."); } } } @@ -1294,14 +1497,13 @@ private string[] GetMetadataElementsFromResponse(string response, string propert /// The "packageContent" property is used for download, and the value is a URI for the .nupkg file. /// /// - private string[] GetVersionedResponsesFromRegistrationsResource(string registrationsBaseUrl, string packageName, string property, bool isSearch, out ErrorRecord errRecord) + private string[] GetVersionedResponsesFromRegistrationsResource(string registrationsBaseUrl, string packageName, string property, bool isSearch, out ErrorRecord errRecord, ConcurrentQueue errorMsgs, ConcurrentQueue debugMsgs, ConcurrentQueue verboseMsgs) { - // TODO: pass in ConcurrentQueue to write out debug message. - //_cmdletPassedIn.WriteDebug("In V3ServerAPICalls::GetVersionedResponsesFromRegistrationsResource()"); + debugMsgs.Enqueue("In V3ServerAPICalls::GetVersionedResponsesFromRegistrationsResource()"); List versionedResponses = new List(); var requestPkgMapping = registrationsBaseUrl.EndsWith("/") ? $"{registrationsBaseUrl}{packageName.ToLower()}/index.json" : $"{registrationsBaseUrl}/{packageName.ToLower()}/index.json"; - string pkgMappingResponse = HttpRequestCall(requestPkgMapping, out errRecord); + string pkgMappingResponse = HttpRequestCall(requestPkgMapping, out errRecord, errorMsgs, debugMsgs, verboseMsgs); if (errRecord != null) { if (errRecord.Exception is ResourceNotFoundException) @@ -1317,7 +1519,7 @@ private string[] GetVersionedResponsesFromRegistrationsResource(string registrat } string upperVersion = String.Empty; - string[] versionedResponseArr = GetMetadataElementsFromResponse(pkgMappingResponse, property, packageName, out upperVersion, out errRecord); + string[] versionedResponseArr = GetMetadataElementsFromResponse(pkgMappingResponse, property, packageName, out upperVersion, out errRecord, errorMsgs, debugMsgs, verboseMsgs); if (errRecord != null) { return Utils.EmptyStrArray; @@ -1332,14 +1534,14 @@ private string[] GetVersionedResponsesFromRegistrationsResource(string registrat if (isSearch) { - if (!IsLatestVersionFirstForSearch(versionedResponseArr, out errRecord)) + if (!IsLatestVersionFirstForSearch(versionedResponseArr, out errRecord, errorMsgs, debugMsgs, verboseMsgs)) { Array.Reverse(versionedResponseArr); } } else { - if (!IsLatestVersionFirstForInstall(versionedResponseArr, upperVersion, out errRecord)) + if (!IsLatestVersionFirstForInstall(versionedResponseArr, upperVersion, out errRecord, errorMsgs, debugMsgs, verboseMsgs)) { Array.Reverse(versionedResponseArr); } @@ -1353,9 +1555,9 @@ private string[] GetVersionedResponsesFromRegistrationsResource(string registrat /// ADO feeds usually return version entries in descending order, but Nuget.org repository returns them in ascending order. /// Package versions will reflect prerelease preference, but upper version and lower version would not so we don't use them for comparison. /// - private bool IsLatestVersionFirstForSearch(string[] versionedResponses, out ErrorRecord errRecord) + private bool IsLatestVersionFirstForSearch(string[] versionedResponses, out ErrorRecord errRecord, ConcurrentQueue errorMsgs, ConcurrentQueue debugMsgs, ConcurrentQueue verboseMsgs) { - _cmdletPassedIn.WriteDebug("In V3ServerAPICalls::IsLatestVersionFirstForSearch()"); + debugMsgs.Enqueue("In V3ServerAPICalls::IsLatestVersionFirstForSearch()"); errRecord = null; bool latestVersionFirst = true; int versionResponsesCount = versionedResponses.Length; @@ -1452,10 +1654,9 @@ private bool IsLatestVersionFirstForSearch(string[] versionedResponses, out Erro /// ADO feeds usually return version entries in descending order, but Nuget.org repository returns them in ascending order. /// Entries do not reflect prerelease preference so all versions (including prerelease) are being considered here, so upper version (including prerelease) can be used for comparison. /// - private bool IsLatestVersionFirstForInstall(string[] versionedResponses, string upperVersion, out ErrorRecord errRecord) + private bool IsLatestVersionFirstForInstall(string[] versionedResponses, string upperVersion, out ErrorRecord errRecord, ConcurrentQueue errorMsgs, ConcurrentQueue debugMsgs, ConcurrentQueue verboseMsgs) { - // TODO: pass in ConcurrentQueue to write out debug message. - //_cmdletPassedIn.WriteDebug("In V3ServerAPICalls::IsLatestVersionFirstForInstall()"); + debugMsgs.Enqueue("In V3ServerAPICalls::IsLatestVersionFirstForInstall()"); errRecord = null; bool latestVersionFirst = true; @@ -1531,14 +1732,14 @@ private bool IsRequiredTagSatisfied(JsonElement tagsElement, string[] tags, out /// /// Helper method that parses response for given property and returns result for that property as a JsonElement array. /// - private JsonElement[] GetJsonElementArr(string request, string propertyName, out int totalHits, out ErrorRecord errRecord) + private JsonElement[] GetJsonElementArr(string request, string propertyName, out int totalHits, out ErrorRecord errRecord, ConcurrentQueue errorMsgs, ConcurrentQueue debugMsgs, ConcurrentQueue verboseMsgs) { List responseEntries = new List(); JsonElement[] entries = new JsonElement[0]; totalHits = 0; try { - string response = HttpRequestCall(request, out errRecord); + string response = HttpRequestCall(request, out errRecord, errorMsgs, debugMsgs, verboseMsgs); if (errRecord != null) { return new JsonElement[]{}; @@ -1589,17 +1790,15 @@ private JsonElement[] GetJsonElementArr(string request, string propertyName, out /// /// Helper method that makes the HTTP request for the V3 server protocol url passed in for find APIs. /// - private string HttpRequestCall(string requestUrlV3, out ErrorRecord errRecord) + private string HttpRequestCall(string requestUrlV3, out ErrorRecord errRecord, ConcurrentQueue errorMsgs, ConcurrentQueue debugMsgs, ConcurrentQueue verboseMsgs) { - // TODO: pass in ConcurrentQueue to write out debug message. - //_cmdletPassedIn.WriteDebug("In V3ServerAPICalls::HttpRequestCall()"); + debugMsgs.Enqueue("In V3ServerAPICalls::HttpRequestCall()"); errRecord = null; string response = string.Empty; try { - // TODO: pass in ConcurrentQueue to write out debug message. - //_cmdletPassedIn.WriteDebug($"Request url is '{requestUrlV3}'"); + debugMsgs.Enqueue($"Request url is '{requestUrlV3}'"); HttpRequestMessage request = new HttpRequestMessage(HttpMethod.Get, requestUrlV3); response = SendV3RequestAsync(request, _sessionClient).GetAwaiter().GetResult(); @@ -1643,10 +1842,9 @@ private string HttpRequestCall(string requestUrlV3, out ErrorRecord errRecord) /// /// Helper method that makes the HTTP request for the V3 server protocol url passed in for install APIs. /// - private HttpContent HttpRequestCallForContent(string requestUrlV3, out ErrorRecord errRecord) + private HttpContent HttpRequestCallForContent(string requestUrlV3, out ErrorRecord errRecord, ConcurrentQueue errorMsgs, ConcurrentQueue debugMsgs, ConcurrentQueue verboseMsgs) { - // TODO: pass in ConcurrentQueue to write out debug message. - //_cmdletPassedIn.WriteDebug("In V3ServerAPICalls::HttpRequestCallForContent()"); + debugMsgs.Enqueue("In V3ServerAPICalls::HttpRequestCallForContent()"); errRecord = null; HttpContent content = null; try @@ -1668,8 +1866,41 @@ private HttpContent HttpRequestCallForContent(string requestUrlV3, out ErrorReco if (string.IsNullOrEmpty(content?.ToString())) { - // TODO: pass in ConcurrentQueue to write out debug message. - //_cmdletPassedIn.WriteDebug("Response is empty"); + debugMsgs.Enqueue("Response is empty"); + } + + return content; + } + + /// + /// Helper method that makes the HTTP request for the V3 server protocol url passed in for install APIs asynchronously. + /// This is the async counterpart of HttpRequestCallForContent() used for concurrent (parallel) installation workflows. + /// + private async Task HttpRequestCallForContentAsync(string requestUrlV3, ConcurrentQueue errorMsgs, ConcurrentQueue warningMsgs, ConcurrentQueue debugMsgs, ConcurrentQueue verboseMsgs) + { + debugMsgs.Enqueue("In V3ServerAPICalls::HttpRequestCallForContentAsync()"); + HttpContent content = null; + try + { + debugMsgs.Enqueue($"Request url is '{requestUrlV3}'"); + HttpRequestMessage request = new HttpRequestMessage(HttpMethod.Get, requestUrlV3); + + content = await SendV3RequestForContentAsync(request, _sessionClient); + } + catch (Exception e) + { + errorMsgs.Enqueue(new ErrorRecord( + exception: e, + "HttpRequestCallForContentFailure", + ErrorCategory.InvalidResult, + this)); + + return null; + } + + if (string.IsNullOrEmpty(content?.ToString())) + { + debugMsgs.Enqueue("Response is empty"); } return content;