diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependabotProxy.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependabotProxy.cs index e06970141fed..08b142219a97 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependabotProxy.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependabotProxy.cs @@ -9,7 +9,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching { - public class DependabotProxy : IDisposable + public class DependabotProxy : IDependabotProxy { /// /// Represents configurations for package registries. @@ -21,24 +21,15 @@ public record class RegistryConfig(string Type, string URL); private readonly string host; private readonly string port; - /// - /// The full address of the Dependabot proxy, if available. - /// - internal string Address { get; } - /// - /// The URLs of package registries that are configured for the proxy. - /// - internal HashSet RegistryURLs { get; } - /// - /// The path to the temporary file where the certificate is stored. - /// - internal string? CertificatePath { get; private set; } - /// - /// The certificate used for the Dependabot proxy. - /// - internal X509Certificate2? Certificate { get; private set; } + public string Address { get; } + + public HashSet RegistryURLs { get; } + + public string? CertificatePath { get; private set; } + + public X509Certificate2? Certificate { get; private set; } - internal static DependabotProxy? GetDependabotProxy( + internal static IDependabotProxy? GetDependabotProxy( ILogger logger, IDiagnosticsWriter diagnosticsWriter, TemporaryDirectory tempWorkingDirectory) { // Setting HTTP(S)_PROXY and SSL_CERT_FILE have no effect on Windows or macOS, diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs index 2706d5262931..707fabbf83fb 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs @@ -27,10 +27,10 @@ public sealed partial class DependencyManager : IDisposable, ICompilationInfoCon private readonly ILogger logger; private readonly IDiagnosticsWriter diagnosticsWriter; private readonly NugetPackageRestorer nugetPackageRestorer; - private readonly DependabotProxy? dependabotProxy; + private readonly IDependabotProxy? dependabotProxy; private readonly IDotNet dotnet; private readonly FileContent fileContent; - private readonly FileProvider fileProvider; + private readonly IFileProvider fileProvider; // Only used as a set, but ConcurrentDictionary is the only concurrent set in .NET. private readonly IDictionary usedReferences = new ConcurrentDictionary(); diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs index 9fd3749ae340..9958fbce4e71 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs @@ -31,11 +31,11 @@ private DotNet(IDotNetCliInvoker dotnetCliInvoker, ILogger logger, bool runDotne } } - private DotNet(ILogger logger, string? dotNetPath, TemporaryDirectory tempWorkingDirectory, DependabotProxy? dependabotProxy) : this(new DotNetCliInvoker(logger, Path.Join(dotNetPath ?? string.Empty, "dotnet"), dependabotProxy), logger, dotNetPath is null, tempWorkingDirectory) { } + private DotNet(ILogger logger, string? dotNetPath, TemporaryDirectory tempWorkingDirectory, IDependabotProxy? dependabotProxy) : this(new DotNetCliInvoker(logger, Path.Join(dotNetPath ?? string.Empty, "dotnet"), dependabotProxy), logger, dotNetPath is null, tempWorkingDirectory) { } internal static IDotNet Make(IDotNetCliInvoker dotnetCliInvoker, ILogger logger, bool runDotnetInfo) => new DotNet(dotnetCliInvoker, logger, runDotnetInfo); - public static IDotNet Make(ILogger logger, string? dotNetPath, TemporaryDirectory tempWorkingDirectory, DependabotProxy? dependabotProxy) => new DotNet(logger, dotNetPath, tempWorkingDirectory, dependabotProxy); + public static IDotNet Make(ILogger logger, string? dotNetPath, TemporaryDirectory tempWorkingDirectory, IDependabotProxy? dependabotProxy) => new DotNet(logger, dotNetPath, tempWorkingDirectory, dependabotProxy); private static void HandleRetryExitCode143(string dotnet, int attempt, ILogger logger) { @@ -90,7 +90,8 @@ private List GetRestoreArgs(RestoreSettings restoreSettings) args.Add("/p:EnableWindowsTargeting=true"); } - args.AddRange(restoreSettings.NugetSources); + var nugetSources = restoreSettings.NugetSources.SelectMany(source => ["-s", source]).ToList(); + args.AddRange(nugetSources); return args; } diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNetCliInvoker.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNetCliInvoker.cs index b982cac65477..c6f97c5f8be2 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNetCliInvoker.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNetCliInvoker.cs @@ -12,11 +12,11 @@ namespace Semmle.Extraction.CSharp.DependencyFetching internal sealed class DotNetCliInvoker : IDotNetCliInvoker { private readonly ILogger logger; - private readonly DependabotProxy? proxy; + private readonly IDependabotProxy? proxy; public string Exec { get; } - public DotNetCliInvoker(ILogger logger, string exec, DependabotProxy? dependabotProxy) + public DotNetCliInvoker(ILogger logger, string exec, IDependabotProxy? dependabotProxy) { this.logger = logger; this.proxy = dependabotProxy; diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FeedManager.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FeedManager.cs index 581d4c15e9dd..6c4593f3400c 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FeedManager.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FeedManager.cs @@ -1,15 +1,8 @@ using System; using System.Collections.Generic; using System.Collections.Immutable; -using System.IO; using System.Linq; -using System.Net; -using System.Net.Http; -using System.Security.Cryptography.X509Certificates; -using System.Text; using System.Text.RegularExpressions; -using System.Threading; -using System.Threading.Tasks; using Semmle.Util; using Semmle.Util.Logging; @@ -21,10 +14,10 @@ internal sealed partial class FeedManager : IDisposable private readonly ILogger logger; private readonly IDotNet dotnet; - private readonly FileProvider fileProvider; - private readonly DependabotProxy? dependabotProxy; + private readonly IFileProvider fileProvider; private readonly DependencyDirectory emptyPackageDirectory; private readonly ImmutableHashSet privateRegistryFeeds; + private readonly IFeedManagerIO feedManagerIo; /// /// Gets whether there are private package registries configured for C#. @@ -79,12 +72,12 @@ internal sealed partial class FeedManager : IDisposable /// public ImmutableHashSet ReachableFallbackFeeds => lazyReachableFallbackFeeds.Value; - public FeedManager(ILogger logger, IDotNet dotnet, DependabotProxy? dependabotProxy, FileProvider fileProvider) + public FeedManager(ILogger logger, IDotNet dotnet, IDependabotProxy? dependabotProxy, IFileProvider fileProvider, IFeedManagerIO feedManagerIo) { this.logger = logger; this.dotnet = dotnet; - this.dependabotProxy = dependabotProxy; this.fileProvider = fileProvider; + this.feedManagerIo = feedManagerIo; privateRegistryFeeds = dependabotProxy?.RegistryURLs.ToImmutableHashSet() ?? []; HasPrivateRegistryFeeds = privateRegistryFeeds.Count > 0; emptyPackageDirectory = new DependencyDirectory("empty", "empty package", logger); @@ -105,17 +98,9 @@ public FeedManager(ILogger logger, IDotNet dotnet, DependabotProxy? dependabotPr }); } - private string? GetDirectoryName(string path) + public FeedManager(ILogger logger, IDotNet dotnet, IDependabotProxy? dependabotProxy, IFileProvider fileProvider) + : this(logger, dotnet, dependabotProxy, fileProvider, new FeedManagerIO(logger, dependabotProxy)) { - try - { - return new FileInfo(path).Directory?.FullName; - } - catch (Exception exc) - { - logger.LogWarning($"Failed to get directory of '{path}': {exc}"); - } - return null; } private IEnumerable GetFeeds(Func> getNugetFeeds) @@ -157,19 +142,16 @@ private IEnumerable GetFeedsFromNugetConfig(string nugetConfigPath) => /// If there are no feeds, a dummy source argument is added to override any default feeds that `restore` would use. /// /// The list of feeds to use for the restore command. - /// The prefix to use for each source argument (e.g., "-s"). /// The list of NuGet sources arguments for the restore command. - public List FeedsToRestoreArgument(IEnumerable feeds, string sourceArgumentPrefix) + public List RestoreFeeds(IEnumerable feeds) { // If there are no feeds, we want to override any default feeds that `restore` would use by passing a dummy source argument. if (!feeds.Any()) { - return [sourceArgumentPrefix, emptyPackageDirectory.DirInfo.FullName]; + return [emptyPackageDirectory.DirInfo.FullName]; } - // Add package sources. If any are present, they override all sources specified in - // the configuration file(s). - return feeds.SelectMany(feed => [sourceArgumentPrefix, feed]).ToList(); + return feeds.ToList(); } private IEnumerable FeedsToUseAux(HashSet feedsToConsider) @@ -196,22 +178,12 @@ private IEnumerable FeedsToUseAux(HashSet feedsToConsider) public IEnumerable FeedsToUse(string path) { // Find the path specific feeds. - var folder = GetDirectoryName(path); + var folder = feedManagerIo.GetDirectoryName(path); var feedsToConsider = folder is not null ? GetFeedsFromFolder(folder).ToHashSet() : new HashSet(); return FeedsToUseAux(feedsToConsider); } - /// - /// Constructs the NuGet sources argument for the `dotnet restore` command based on the given feeds. - /// - /// The list of NuGet feeds to use for the restore command. - /// A list representing the NuGet sources arguments for the `dotnet restore` command. - public List FeedsToDotnetRestoreArgument(IEnumerable feeds) - { - return FeedsToRestoreArgument(feeds, "-s"); - } - /// /// Constructs the list of NuGet sources to use for dotnet restore. /// (1) Use the feeds we get from `dotnet nuget list source` @@ -219,7 +191,7 @@ public List FeedsToDotnetRestoreArgument(IEnumerable feeds) /// /// Path to project/solution /// A list representing the NuGet sources arguments for the `dotnet restore` command. - public List MakeDotnetRestoreSourcesArguments(string path) + public List MakeRestoreFeeds(string path) { // Do not construct a set of explicit NuGet sources to use for restore. if (!CheckNugetFeedResponsiveness && !HasPrivateRegistryFeeds) @@ -229,7 +201,7 @@ public List MakeDotnetRestoreSourcesArguments(string path) var feedsToUse = FeedsToUse(path); - return FeedsToDotnetRestoreArgument(feedsToUse); + return RestoreFeeds(feedsToUse); } private (int initialTimeout, int tryCount) GetFeedRequestSettings(bool isFallback) @@ -251,76 +223,6 @@ public List MakeDotnetRestoreSourcesArguments(string path) return (timeoutMilliSeconds, tryCount); } - private static async Task ExecuteGetRequest(string address, HttpClient httpClient, CancellationToken cancellationToken) - { - return await httpClient.GetAsync(address, HttpCompletionOption.ResponseHeadersRead, cancellationToken); - } - - private bool IsFeedReachable(string feed, int timeoutMilliSeconds, int tryCount) - { - logger.LogInfo($"Checking if NuGet feed '{feed}' is reachable..."); - - // Configure the HttpClient to be aware of the Dependabot Proxy, if used. - HttpClientHandler httpClientHandler = new(); - if (dependabotProxy != null) - { - httpClientHandler.Proxy = new WebProxy(dependabotProxy.Address); - - if (dependabotProxy.Certificate != null) - { - httpClientHandler.ServerCertificateCustomValidationCallback = (message, cert, chain, _) => - { - if (chain is null || cert is null) - { - var msg = cert is null && chain is null - ? "certificate and chain" - : chain is null - ? "chain" - : "certificate"; - logger.LogWarning($"Dependabot proxy certificate validation failed due to missing {msg}"); - return false; - } - chain.ChainPolicy.TrustMode = X509ChainTrustMode.CustomRootTrust; - chain.ChainPolicy.CustomTrustStore.Add(dependabotProxy.Certificate); - return chain.Build(cert); - }; - } - } - - using HttpClient client = new(httpClientHandler); - - for (var i = 0; i < tryCount; i++) - { - using var cts = new CancellationTokenSource(); - cts.CancelAfter(timeoutMilliSeconds); - try - { - logger.LogInfo($"Attempt {i + 1}/{tryCount} to reach NuGet feed '{feed}'."); - using var response = ExecuteGetRequest(feed, client, cts.Token).GetAwaiter().GetResult(); - response.EnsureSuccessStatusCode(); - logger.LogInfo($"Querying NuGet feed '{feed}' succeeded."); - return true; - } - catch (Exception exc) - { - if (exc is TaskCanceledException tce && - tce.CancellationToken == cts.Token && - cts.Token.IsCancellationRequested) - { - logger.LogInfo($"Didn't receive answer from NuGet feed '{feed}' in {timeoutMilliSeconds}ms."); - timeoutMilliSeconds *= 2; - continue; - } - - logger.LogInfo($"Querying NuGet feed '{feed}' failed. The reason for the failure: {exc.Message}"); - return false; - } - } - - logger.LogWarning($"Didn't receive answer from NuGet feed '{feed}'. Tried it {tryCount} times."); - return false; - } - /// /// Retrieves a list of excluded NuGet feeds from the corresponding environment variable. /// @@ -374,7 +276,7 @@ public bool IsDefaultFeedReachable() if (CheckNugetFeedResponsiveness) { var (initialTimeout, tryCount) = GetFeedRequestSettings(isFallback: false); - return IsFeedReachable(PublicNugetOrgFeed, initialTimeout, tryCount); + return feedManagerIo.IsFeedReachable(PublicNugetOrgFeed, initialTimeout, tryCount); } return true; @@ -393,7 +295,7 @@ private List GetReachableNuGetFeeds(HashSet feedsToCheck, bool i var (initialTimeout, tryCount) = GetFeedRequestSettings(isFallback); var reachableFeeds = feedsToCheck - .Where(feed => IsFeedReachable(feed, initialTimeout, tryCount)) + .Where(feed => feedManagerIo.IsFeedReachable(feed, initialTimeout, tryCount)) .ToList(); if (reachableFeeds.Count == 0) @@ -477,7 +379,7 @@ private ImmutableHashSet GetAllFeeds() if (nugetConfigs.Count > 0) { var nugetConfigFeeds = nugetConfigs - .Select(GetDirectoryName) + .Select(feedManagerIo.GetDirectoryName) .Where(folder => folder != null) .SelectMany(folder => GetFeedsFromFolder(folder!)) .ToHashSet(); diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FeedManagerIO.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FeedManagerIO.cs new file mode 100644 index 000000000000..8e771f6037a4 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FeedManagerIO.cs @@ -0,0 +1,109 @@ + +using System; +using System.IO; +using Semmle.Util.Logging; +using System.Net.Http; +using System.Net; +using System.Security.Cryptography.X509Certificates; +using System.Threading; +using System.Threading.Tasks; + +namespace Semmle.Extraction.CSharp.DependencyFetching +{ + public class FeedManagerIO : IFeedManagerIO + { + private readonly ILogger logger; + private readonly IDependabotProxy? dependabotProxy; + + public FeedManagerIO(ILogger logger, IDependabotProxy? dependabotProxy) + { + this.logger = logger; + this.dependabotProxy = dependabotProxy; + } + + public string? GetDirectoryName(string path) + { + try + { + return new FileInfo(path).Directory?.FullName; + } + catch (Exception exc) + { + logger.LogWarning($"Failed to get directory of '{path}': {exc}"); + } + return null; + } + + private static async Task ExecuteGetRequest(string address, HttpClient httpClient, CancellationToken cancellationToken) + { + return await httpClient.GetAsync(address, HttpCompletionOption.ResponseHeadersRead, cancellationToken); + } + + public bool IsFeedReachable(string feed, int timeoutMilliSeconds, int tryCount) + { + logger.LogInfo($"Checking if NuGet feed '{feed}' is reachable..."); + + // Configure the HttpClient to be aware of the Dependabot Proxy, if used. + HttpClientHandler httpClientHandler = new(); + if (dependabotProxy != null) + { + httpClientHandler.Proxy = new WebProxy(dependabotProxy.Address); + + if (dependabotProxy.Certificate != null) + { + httpClientHandler.ServerCertificateCustomValidationCallback = (message, cert, chain, _) => + { + if (chain is null || cert is null) + { + var msg = cert is null && chain is null + ? "certificate and chain" + : chain is null + ? "chain" + : "certificate"; + logger.LogWarning($"Dependabot proxy certificate validation failed due to missing {msg}"); + return false; + } + chain.ChainPolicy.TrustMode = X509ChainTrustMode.CustomRootTrust; + chain.ChainPolicy.CustomTrustStore.Add(dependabotProxy.Certificate); + return chain.Build(cert); + }; + } + } + + using HttpClient client = new(httpClientHandler); + + for (var i = 0; i < tryCount; i++) + { + using var cts = new CancellationTokenSource(); + cts.CancelAfter(timeoutMilliSeconds); + try + { + logger.LogInfo($"Attempt {i + 1}/{tryCount} to reach NuGet feed '{feed}'."); + using var response = ExecuteGetRequest(feed, client, cts.Token).GetAwaiter().GetResult(); + response.EnsureSuccessStatusCode(); + logger.LogInfo($"Querying NuGet feed '{feed}' succeeded."); + return true; + } + catch (Exception exc) + { + if (exc is TaskCanceledException tce && + tce.CancellationToken == cts.Token && + cts.Token.IsCancellationRequested) + { + logger.LogInfo($"Didn't receive answer from NuGet feed '{feed}' in {timeoutMilliSeconds}ms."); + timeoutMilliSeconds *= 2; + continue; + } + + logger.LogInfo($"Querying NuGet feed '{feed}' failed. The reason for the failure: {exc.Message}"); + return false; + } + } + + logger.LogWarning($"Didn't receive answer from NuGet feed '{feed}'. Tried it {tryCount} times."); + return false; + } + + + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FileProvider.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FileProvider.cs index 9e6b810b95ec..4f55bbff7f27 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FileProvider.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FileProvider.cs @@ -2,12 +2,11 @@ using System.Collections.Generic; using System.IO; using System.Linq; -using System.Security.Policy; using Semmle.Util.Logging; namespace Semmle.Extraction.CSharp.DependencyFetching { - public class FileProvider + public class FileProvider : IFileProvider { private static readonly HashSet binaryFileExtensions = [".dll", ".exe"]; // TODO: add more binary file extensions. diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IDependabotProxy.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IDependabotProxy.cs new file mode 100644 index 000000000000..37a11900fddf --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IDependabotProxy.cs @@ -0,0 +1,29 @@ +using System; +using System.Collections.Generic; +using System.Security.Cryptography.X509Certificates; + +namespace Semmle.Extraction.CSharp.DependencyFetching +{ + public interface IDependabotProxy : IDisposable + { + /// + /// The full address of the Dependabot proxy, if available. + /// + string Address { get; } + + /// + /// The URLs of package registries that are configured for the proxy. + /// + HashSet RegistryURLs { get; } + + /// + /// The path to the temporary file where the certificate is stored. + /// + string? CertificatePath { get; } + + /// + /// The certificate used for the Dependabot proxy. + /// + X509Certificate2? Certificate { get; } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IFeedManagerIO.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IFeedManagerIO.cs new file mode 100644 index 000000000000..380c5f3b5f28 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IFeedManagerIO.cs @@ -0,0 +1,16 @@ + +namespace Semmle.Extraction.CSharp.DependencyFetching +{ + public interface IFeedManagerIO + { + /// + /// Gets the directory name of the specified path. + /// + string? GetDirectoryName(string path); + + /// + /// Returns true if the feed is reachable within the specified timeout and try count. + /// + bool IsFeedReachable(string feed, int timeoutMilliSeconds, int tryCount); + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IFileProvider.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IFileProvider.cs new file mode 100644 index 000000000000..919be0af61a7 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IFileProvider.cs @@ -0,0 +1,22 @@ +using System.Collections.Generic; +using System.IO; + +namespace Semmle.Extraction.CSharp.DependencyFetching +{ + public interface IFileProvider + { + DirectoryInfo SourceDir { get; } + IEnumerable SmallNonBinary { get; } + IEnumerable Sources { get; } + ICollection Projects { get; } + ICollection Solutions { get; } + IEnumerable Dlls { get; } + ICollection NugetConfigs { get; } + ICollection NugetExes { get; } + string? RootNugetConfig { get; } + IEnumerable GlobalJsons { get; } + ICollection PackagesConfigs { get; } + ICollection RazorViews { get; } + ICollection Resources { get; } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs index 2867a7cc6327..85d6056d7218 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs @@ -15,7 +15,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching { internal sealed partial class NugetPackageRestorer : IDisposable { - private readonly FileProvider fileProvider; + private readonly IFileProvider fileProvider; private readonly FileContent fileContent; private readonly IDotNet dotnet; private readonly IDiagnosticsWriter diagnosticsWriter; @@ -29,10 +29,10 @@ internal sealed partial class NugetPackageRestorer : IDisposable public NugetPackageRestorer( - FileProvider fileProvider, + IFileProvider fileProvider, FileContent fileContent, IDotNet dotnet, - DependabotProxy? dependabotProxy, + IDependabotProxy? dependabotProxy, IDiagnosticsWriter diagnosticsWriter, ILogger logger, ICompilationInfoContainer compilationInfoContainer) @@ -53,7 +53,7 @@ public NugetPackageRestorer( public string? TryRestore(string package) { var feeds = feedManager.CheckNugetFeedResponsiveness ? feedManager.ReachableFeeds : feedManager.AllFeeds; - var nugetSources = feedManager.FeedsToDotnetRestoreArgument(feeds); + var nugetSources = feedManager.RestoreFeeds(feeds); if (TryRestorePackageManually(package, nugetSources)) { var packageDir = DependencyManager.GetPackageDirectory(package, missingPackageDirectory.DirInfo); @@ -216,7 +216,7 @@ private IEnumerable RestoreSolutions(out DependencyContainer dependencie var projects = fileProvider.Solutions.SelectMany(solution => { logger.LogInfo($"Restoring solution {solution}..."); - var nugetSources = feedManager.MakeDotnetRestoreSourcesArguments(solution); + var nugetSources = feedManager.MakeRestoreFeeds(solution); var res = dotnet.Restore(new(solution, PackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: true, NugetSources: nugetSources, TargetWindows: isWindows)); if (res.Success) { @@ -264,7 +264,7 @@ private void RestoreProjects(IEnumerable projects, out ConcurrentBag projects, out ConcurrentBag p.ToLowerInvariant()); var alreadyDownloadedLegacyPackages = GetRestoredLegacyPackageNames(); diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/PackagesConfigRestorer.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/PackagesConfigRestorer.cs index e5a9365a08e3..d4403bb955ef 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/PackagesConfigRestorer.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/PackagesConfigRestorer.cs @@ -1,6 +1,5 @@ using System; using System.Collections.Generic; -using System.Collections.Immutable; using System.Diagnostics; using System.IO; using System.Linq; @@ -34,7 +33,7 @@ internal interface IPackagesConfigRestore /// internal class PackagesConfigRestoreFactory { - public static IPackagesConfigRestore Create(FileProvider fileProvider, DependencyDirectory packageDirectory, Semmle.Util.Logging.ILogger logger, FeedManager feedManager) + public static IPackagesConfigRestore Create(IFileProvider fileProvider, DependencyDirectory packageDirectory, Semmle.Util.Logging.ILogger logger, FeedManager feedManager) { if (SystemBuildActions.Instance.IsWindows() || SystemBuildActions.Instance.IsMonoInstalled()) { @@ -56,7 +55,7 @@ private class NugetExeWrapper : IPackagesConfigRestore public int PackageCount => fileProvider.PackagesConfigs.Count; - private readonly FileProvider fileProvider; + private readonly IFileProvider fileProvider; /// /// The packages directory. @@ -75,7 +74,7 @@ private class NugetExeWrapper : IPackagesConfigRestore /// /// Create the package manager for a specified source tree. /// - public NugetExeWrapper(FileProvider fileProvider, DependencyDirectory packageDirectory, Semmle.Util.Logging.ILogger logger, FeedManager feedManager) + public NugetExeWrapper(IFileProvider fileProvider, DependencyDirectory packageDirectory, Semmle.Util.Logging.ILogger logger, FeedManager feedManager) { this.fileProvider = fileProvider; this.packageDirectory = packageDirectory; @@ -180,7 +179,8 @@ private bool TryRestoreNugetPackage(string packagesConfig) { feedsToUse.Add(FeedManager.PublicNugetOrgFeed); } - sourcesArgument = feedManager.FeedsToRestoreArgument(feedsToUse, "-Source"); + var restoreFeeds = feedManager.RestoreFeeds(feedsToUse); + sourcesArgument = restoreFeeds.SelectMany(feed => ["-Source", feed]).ToList(); } /* Use nuget.exe to install a package. diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/SourceGenerators/DotnetSourceGeneratorBase.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/SourceGenerators/DotnetSourceGeneratorBase.cs index c17981803ddb..6a150e8d542a 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/SourceGenerators/DotnetSourceGeneratorBase.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/SourceGenerators/DotnetSourceGeneratorBase.cs @@ -9,14 +9,14 @@ namespace Semmle.Extraction.CSharp.DependencyFetching { internal abstract class DotnetSourceGeneratorBase : SourceGeneratorBase where T : DotnetSourceGeneratorWrapper { - protected readonly FileProvider fileProvider; + protected readonly IFileProvider fileProvider; protected readonly FileContent fileContent; protected readonly IDotNet dotnet; protected readonly ICompilationInfoContainer compilationInfoContainer; protected readonly IEnumerable references; public DotnetSourceGeneratorBase( - FileProvider fileProvider, + IFileProvider fileProvider, FileContent fileContent, IDotNet dotnet, ICompilationInfoContainer compilationInfoContainer, diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/SourceGenerators/RazorGenerator.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/SourceGenerators/RazorGenerator.cs index 3f9a17dc6b30..8d4536b4503c 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/SourceGenerators/RazorGenerator.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/SourceGenerators/RazorGenerator.cs @@ -8,7 +8,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching internal class RazorGenerator : DotnetSourceGeneratorBase { public RazorGenerator( - FileProvider fileProvider, + IFileProvider fileProvider, FileContent fileContent, IDotNet dotnet, ICompilationInfoContainer compilationInfoContainer, diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/SourceGenerators/ResxGenerator.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/SourceGenerators/ResxGenerator.cs index da66ef275442..e6e27ea27534 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/SourceGenerators/ResxGenerator.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/SourceGenerators/ResxGenerator.cs @@ -10,7 +10,7 @@ internal class ResxGenerator : DotnetSourceGeneratorBase private readonly string? sourceGeneratorFolder = null; public ResxGenerator( - FileProvider fileProvider, + IFileProvider fileProvider, FileContent fileContent, IDotNet dotnet, ICompilationInfoContainer compilationInfoContainer, diff --git a/csharp/extractor/Semmle.Extraction.Tests/DotNet.cs b/csharp/extractor/Semmle.Extraction.Tests/DotNet.cs index 260eae10bd1b..77e88a58443a 100644 --- a/csharp/extractor/Semmle.Extraction.Tests/DotNet.cs +++ b/csharp/extractor/Semmle.Extraction.Tests/DotNet.cs @@ -148,11 +148,11 @@ public void TestDotnetRestoreProjectToDirectory3() var dotnet = MakeDotnet(dotnetCliInvoker); // Execute - var res = dotnet.Restore(new("myproject.csproj", "mypackages", false, [], true)); + var res = dotnet.Restore(new("myproject.csproj", "mypackages", false, ["https://my.nuget.source1"], true)); // Verify var lastArgs = dotnetCliInvoker.GetLastArgs(); - Assert.Equal(["restore", "--no-dependencies", "myproject.csproj", "--packages", "mypackages", "/p:DisableImplicitNuGetFallbackFolder=true", "--verbosity", "normal", "--force"], lastArgs); + Assert.Equal(["restore", "--no-dependencies", "myproject.csproj", "--packages", "mypackages", "/p:DisableImplicitNuGetFallbackFolder=true", "--verbosity", "normal", "--force", "-s", "https://my.nuget.source1"], lastArgs); Assert.Equal(2, res.AssetsFilePaths.Count()); Assert.Contains("/path/to/project.assets.json", res.AssetsFilePaths); Assert.Contains("/path/to/project2.assets.json", res.AssetsFilePaths); @@ -166,11 +166,11 @@ public void TestDotnetRestoreSolutionToDirectory1() var dotnet = MakeDotnet(dotnetCliInvoker); // Execute - var res = dotnet.Restore(new("mysolution.sln", "mypackages", false, [])); + var res = dotnet.Restore(new("mysolution.sln", "mypackages", false, ["https://my.nuget.source1", "https://my.nuget.source2"])); // Verify var lastArgs = dotnetCliInvoker.GetLastArgs(); - Assert.Equal(["restore", "--no-dependencies", "mysolution.sln", "--packages", "mypackages", "/p:DisableImplicitNuGetFallbackFolder=true", "--verbosity", "normal"], lastArgs); + Assert.Equal(["restore", "--no-dependencies", "mysolution.sln", "--packages", "mypackages", "/p:DisableImplicitNuGetFallbackFolder=true", "--verbosity", "normal", "-s", "https://my.nuget.source1", "-s", "https://my.nuget.source2"], lastArgs); Assert.Equal(2, res.RestoredProjects.Count()); Assert.Contains("/path/to/project.csproj", res.RestoredProjects); Assert.Contains("/path/to/project2.csproj", res.RestoredProjects); diff --git a/csharp/extractor/Semmle.Extraction.Tests/DotNetStub.cs b/csharp/extractor/Semmle.Extraction.Tests/DotNetStub.cs new file mode 100644 index 000000000000..119e39fd0974 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.Tests/DotNetStub.cs @@ -0,0 +1,37 @@ +using System; +using System.Collections.Generic; +using Semmle.Extraction.CSharp.DependencyFetching; + +namespace Semmle.Extraction.Tests +{ + internal class DotNetStub : IDotNet + { + private readonly IList runtimes; + private readonly IList sdks; + private readonly IList nugetFeedsFromConfig; + private readonly IList nugetFeedsFromFolder; + + public DotNetStub(IList runtimes, IList sdks, IList nugetFeedsFromConfig, IList nugetFeedsFromFolder) + { + this.runtimes = runtimes; + this.sdks = sdks; + this.nugetFeedsFromConfig = nugetFeedsFromConfig; + this.nugetFeedsFromFolder = nugetFeedsFromFolder; + } + public bool AddPackage(string folder, string package) => true; + + public bool New(string folder) => true; + + public RestoreResult Restore(RestoreSettings restoreSettings) => new(true, Array.Empty()); + + public IList GetListedRuntimes() => runtimes; + + public IList GetListedSdks() => sdks; + + public bool Exec(List execArgs) => true; + + public IList GetNugetFeeds(string nugetConfig) => nugetFeedsFromConfig; + + public IList GetNugetFeedsFromFolder(string folderPath) => nugetFeedsFromFolder; + } +} diff --git a/csharp/extractor/Semmle.Extraction.Tests/FeedManager.cs b/csharp/extractor/Semmle.Extraction.Tests/FeedManager.cs new file mode 100644 index 000000000000..f70efdb4cdcc --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.Tests/FeedManager.cs @@ -0,0 +1,187 @@ +using Xunit; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using Semmle.Extraction.CSharp.DependencyFetching; + +namespace Semmle.Extraction.Tests +{ + public class DependabotProxyStub : IDependabotProxy + { + public string Address { get; } = ""; + public HashSet RegistryURLs { get; } = ["https://example.com/registry1", "https://example.com/registry2"]; + public string? CertificatePath { get; } = null; + public System.Security.Cryptography.X509Certificates.X509Certificate2? Certificate { get; } = null; + + public void Dispose() { } + } + + public class FeedManagerIOStub : IFeedManagerIO + { + private readonly List unreachableFeeds; + + public FeedManagerIOStub(List unreachableFeeds) + { + this.unreachableFeeds = unreachableFeeds; + } + + public string? GetDirectoryName(string path) + { + return "/path/to/folder"; + } + + public bool IsFeedReachable(string feed, int timeoutMilliSeconds, int tryCount) + { + return !unreachableFeeds.Contains(feed); + } + } + + public class FileProviderStub : IFileProvider + { + public DirectoryInfo SourceDir { get; } = new DirectoryInfo("/path/to/source"); + public IEnumerable SmallNonBinary { get; } = Enumerable.Empty(); + public IEnumerable Sources { get; } = Enumerable.Empty(); + public ICollection Projects { get; } = new List(); + public ICollection Solutions { get; } = new List(); + public IEnumerable Dlls { get; } = Enumerable.Empty(); + public ICollection NugetConfigs { get; } = ["/path/to/nuget.config"]; + public ICollection NugetExes { get; } = new List(); + public string? RootNugetConfig { get; } = null; + public IEnumerable GlobalJsons { get; } = Enumerable.Empty(); + public ICollection PackagesConfigs { get; } = new List(); + public ICollection RazorViews { get; } = new List(); + public ICollection Resources { get; } = new List(); + } + + public class FeedManagerTests + { + private static FeedManager MakeFeedManager() + { + var logger = new LoggerStub(); + var dotnet = new DotNetStub([], [], ["E https://feed.from/config"], ["E https://feed.from/folder1", "E https://feed.from/folder2", "D https://feed.from/folder3"]); + var dependabotProxy = new DependabotProxyStub(); + var fileProvider = new FileProviderStub(); + var feedManagerIo = new FeedManagerIOStub(["https://example.com/registry1", "https://feed.from/folder2"]); + return new FeedManager(logger, dotnet, dependabotProxy, fileProvider, feedManagerIo); + } + + [Fact] + public void TestExplicitFeeds() + { + // Setup + var feedManager = MakeFeedManager(); + + // Execute + var actualFeeds = feedManager.ExplicitFeeds; + + // Verify + Assert.Equal([ + "https://example.com/registry1", + "https://example.com/registry2", + "https://feed.from/config" + ], actualFeeds); + } + + [Fact] + public void TestInheritedFeeds() + { + // Setup + var feedManager = MakeFeedManager(); + + // Execute + var inherited = feedManager.InheritedFeeds; + + // Verify + Assert.Equal([ + "https://feed.from/folder1", + "https://feed.from/folder2" + ], inherited); + } + + [Fact] + public void TestAllFeeds() + { + // Setup + var feedManager = MakeFeedManager(); + + // Execute + var all = feedManager.AllFeeds; + + // Verify + Assert.Equal([ + "https://example.com/registry1", + "https://example.com/registry2", + "https://feed.from/config", + "https://feed.from/folder1", + "https://feed.from/folder2" + ], all); + } + + [Fact] + public void TestReachableFeeds() + { + // Setup + var feedManager = MakeFeedManager(); + + // Execute + var reachableFeeds = feedManager.ReachableFeeds; + + // Verify + Assert.Equal([ + "https://example.com/registry2", + "https://feed.from/config", + "https://feed.from/folder1" + ], reachableFeeds); + } + + [Fact] + public void TestReachableExplicitFeeds() + { + // Setup + var feedManager = MakeFeedManager(); + + // Execute + var reachableFeeds = feedManager.ReachableExplicitFeeds; + + // Verify + Assert.Equal([ + "https://example.com/registry2", + "https://feed.from/config" + ], reachableFeeds); + } + + [Fact] + public void TestReachableFallbackFeeds() + { + // Setup + var feedManager = MakeFeedManager(); + + // Execute + var reachableFallback = feedManager.ReachableFallbackFeeds; + + // Verify + Assert.Equal([ + "https://example.com/registry2", + "https://feed.from/config", + "https://api.nuget.org/v3/index.json" + ], reachableFallback); + } + + [Fact] + public void TestFeedsToUse() + { + // Setup + var feedManager = MakeFeedManager(); + + // Execute + var feedsToUse = feedManager.FeedsToUse("/path/to/packages.config").ToHashSet(); + + // Verify + Assert.Equal([ + "https://example.com/registry2", + "https://feed.from/folder1" + ], feedsToUse); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.Tests/Runtime.cs b/csharp/extractor/Semmle.Extraction.Tests/Runtime.cs index 348cadae2fd0..ea2483f3faf3 100644 --- a/csharp/extractor/Semmle.Extraction.Tests/Runtime.cs +++ b/csharp/extractor/Semmle.Extraction.Tests/Runtime.cs @@ -5,37 +5,12 @@ namespace Semmle.Extraction.Tests { - internal class DotNetStub : IDotNet - { - private readonly IList runtimes; - private readonly IList sdks; - - public DotNetStub(IList runtimes, IList sdks) - { - this.runtimes = runtimes; - this.sdks = sdks; - } - public bool AddPackage(string folder, string package) => true; - - public bool New(string folder) => true; - - public RestoreResult Restore(RestoreSettings restoreSettings) => new(true, Array.Empty()); - - public IList GetListedRuntimes() => runtimes; - - public IList GetListedSdks() => sdks; - - public bool Exec(List execArgs) => true; - - public IList GetNugetFeeds(string nugetConfig) => []; - - public IList GetNugetFeedsFromFolder(string folderPath) => []; - } - public class RuntimeTests { private static string FixExpectedPathOnWindows(string path) => path.Replace('\\', '/'); + private static DotNetStub MakeDotNetStub(IList listedRuntimes) => new DotNetStub(listedRuntimes, null!, [], []); + [Fact] public void TestRuntime1() { @@ -50,7 +25,7 @@ public void TestRuntime1() "Microsoft.NETCore.App 7.0.0 [/path/dotnet/shared/Microsoft.NETCore.App]", "Microsoft.NETCore.App 7.0.2 [/path/dotnet/shared/Microsoft.NETCore.App]" }; - var dotnet = new DotNetStub(listedRuntimes, null!); + var dotnet = MakeDotNetStub(listedRuntimes); var runtime = new Runtime(dotnet); // Execute @@ -76,7 +51,7 @@ public void TestRuntime2() "Microsoft.NETCore.App 8.0.0-preview.5.43280.8 [/path/dotnet/shared/Microsoft.NETCore.App]", "Microsoft.NETCore.App 8.0.0-preview.5.23280.8 [/path/dotnet/shared/Microsoft.NETCore.App]" }; - var dotnet = new DotNetStub(listedRuntimes, null!); + var dotnet = new DotNetStub(listedRuntimes, null!, [], []); var runtime = new Runtime(dotnet); // Execute @@ -99,7 +74,7 @@ public void TestRuntime3() "Microsoft.NETCore.App 8.0.0-rc.4.43280.8 [/path/dotnet/shared/Microsoft.NETCore.App]", "Microsoft.NETCore.App 8.0.0-preview.5.23280.8 [/path/dotnet/shared/Microsoft.NETCore.App]" }; - var dotnet = new DotNetStub(listedRuntimes, null!); + var dotnet = MakeDotNetStub(listedRuntimes); var runtime = new Runtime(dotnet); // Execute @@ -128,7 +103,7 @@ public void TestRuntime4() @"Microsoft.WindowsDesktop.App 6.0.20 [C:\Program Files\dotnet\shared\Microsoft.WindowsDesktop.App]", @"Microsoft.WindowsDesktop.App 7.0.4 [C:\Program Files\dotnet\shared\Microsoft.WindowsDesktop.App]" }; - var dotnet = new DotNetStub(listedRuntimes, null!); + var dotnet = MakeDotNetStub(listedRuntimes); var runtime = new Runtime(dotnet); // Execute @@ -149,6 +124,7 @@ public class SdkTests { private static string FixExpectedPathOnWindows(string path) => path.Replace('\\', '/'); + private static DotNetStub MakeDotNetStub(IList listedSdks) => new DotNetStub(null!, listedSdks, [], []); [Fact] public void TestSdk1() { @@ -163,7 +139,7 @@ public void TestSdk1() "6.0.102 [/usr/local/share/dotnet/sdk6]", "6.0.301 [/usr/local/share/dotnet/sdk7]", }; - var dotnet = new DotNetStub(null!, listedSdks); + var dotnet = MakeDotNetStub(listedSdks); var sdk = new Sdk(dotnet, new LoggerStub()); // Execute @@ -185,7 +161,7 @@ public void TestSdk2() "8.0.100-preview.7.23376.3 [/usr/local/share/dotnet/sdk3]", "7.0.400 [/usr/local/share/dotnet/sdk4]", }; - var dotnet = new DotNetStub(null!, listedSdks); + var dotnet = MakeDotNetStub(listedSdks); var sdk = new Sdk(dotnet, new LoggerStub()); // Execute