diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependabotProxy.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependabotProxy.cs
index e06970141fed..08b142219a97 100644
--- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependabotProxy.cs
+++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependabotProxy.cs
@@ -9,7 +9,7 @@
namespace Semmle.Extraction.CSharp.DependencyFetching
{
- public class DependabotProxy : IDisposable
+ public class DependabotProxy : IDependabotProxy
{
///
/// Represents configurations for package registries.
@@ -21,24 +21,15 @@ public record class RegistryConfig(string Type, string URL);
private readonly string host;
private readonly string port;
- ///
- /// The full address of the Dependabot proxy, if available.
- ///
- internal string Address { get; }
- ///
- /// The URLs of package registries that are configured for the proxy.
- ///
- internal HashSet RegistryURLs { get; }
- ///
- /// The path to the temporary file where the certificate is stored.
- ///
- internal string? CertificatePath { get; private set; }
- ///
- /// The certificate used for the Dependabot proxy.
- ///
- internal X509Certificate2? Certificate { get; private set; }
+ public string Address { get; }
+
+ public HashSet RegistryURLs { get; }
+
+ public string? CertificatePath { get; private set; }
+
+ public X509Certificate2? Certificate { get; private set; }
- internal static DependabotProxy? GetDependabotProxy(
+ internal static IDependabotProxy? GetDependabotProxy(
ILogger logger, IDiagnosticsWriter diagnosticsWriter, TemporaryDirectory tempWorkingDirectory)
{
// Setting HTTP(S)_PROXY and SSL_CERT_FILE have no effect on Windows or macOS,
diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs
index 2706d5262931..707fabbf83fb 100644
--- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs
+++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs
@@ -27,10 +27,10 @@ public sealed partial class DependencyManager : IDisposable, ICompilationInfoCon
private readonly ILogger logger;
private readonly IDiagnosticsWriter diagnosticsWriter;
private readonly NugetPackageRestorer nugetPackageRestorer;
- private readonly DependabotProxy? dependabotProxy;
+ private readonly IDependabotProxy? dependabotProxy;
private readonly IDotNet dotnet;
private readonly FileContent fileContent;
- private readonly FileProvider fileProvider;
+ private readonly IFileProvider fileProvider;
// Only used as a set, but ConcurrentDictionary is the only concurrent set in .NET.
private readonly IDictionary usedReferences = new ConcurrentDictionary();
diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs
index 9fd3749ae340..9958fbce4e71 100644
--- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs
+++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs
@@ -31,11 +31,11 @@ private DotNet(IDotNetCliInvoker dotnetCliInvoker, ILogger logger, bool runDotne
}
}
- private DotNet(ILogger logger, string? dotNetPath, TemporaryDirectory tempWorkingDirectory, DependabotProxy? dependabotProxy) : this(new DotNetCliInvoker(logger, Path.Join(dotNetPath ?? string.Empty, "dotnet"), dependabotProxy), logger, dotNetPath is null, tempWorkingDirectory) { }
+ private DotNet(ILogger logger, string? dotNetPath, TemporaryDirectory tempWorkingDirectory, IDependabotProxy? dependabotProxy) : this(new DotNetCliInvoker(logger, Path.Join(dotNetPath ?? string.Empty, "dotnet"), dependabotProxy), logger, dotNetPath is null, tempWorkingDirectory) { }
internal static IDotNet Make(IDotNetCliInvoker dotnetCliInvoker, ILogger logger, bool runDotnetInfo) => new DotNet(dotnetCliInvoker, logger, runDotnetInfo);
- public static IDotNet Make(ILogger logger, string? dotNetPath, TemporaryDirectory tempWorkingDirectory, DependabotProxy? dependabotProxy) => new DotNet(logger, dotNetPath, tempWorkingDirectory, dependabotProxy);
+ public static IDotNet Make(ILogger logger, string? dotNetPath, TemporaryDirectory tempWorkingDirectory, IDependabotProxy? dependabotProxy) => new DotNet(logger, dotNetPath, tempWorkingDirectory, dependabotProxy);
private static void HandleRetryExitCode143(string dotnet, int attempt, ILogger logger)
{
@@ -90,7 +90,8 @@ private List GetRestoreArgs(RestoreSettings restoreSettings)
args.Add("/p:EnableWindowsTargeting=true");
}
- args.AddRange(restoreSettings.NugetSources);
+ var nugetSources = restoreSettings.NugetSources.SelectMany(source => ["-s", source]).ToList();
+ args.AddRange(nugetSources);
return args;
}
diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNetCliInvoker.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNetCliInvoker.cs
index b982cac65477..c6f97c5f8be2 100644
--- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNetCliInvoker.cs
+++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNetCliInvoker.cs
@@ -12,11 +12,11 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
internal sealed class DotNetCliInvoker : IDotNetCliInvoker
{
private readonly ILogger logger;
- private readonly DependabotProxy? proxy;
+ private readonly IDependabotProxy? proxy;
public string Exec { get; }
- public DotNetCliInvoker(ILogger logger, string exec, DependabotProxy? dependabotProxy)
+ public DotNetCliInvoker(ILogger logger, string exec, IDependabotProxy? dependabotProxy)
{
this.logger = logger;
this.proxy = dependabotProxy;
diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FeedManager.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FeedManager.cs
index 581d4c15e9dd..6c4593f3400c 100644
--- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FeedManager.cs
+++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FeedManager.cs
@@ -1,15 +1,8 @@
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
-using System.IO;
using System.Linq;
-using System.Net;
-using System.Net.Http;
-using System.Security.Cryptography.X509Certificates;
-using System.Text;
using System.Text.RegularExpressions;
-using System.Threading;
-using System.Threading.Tasks;
using Semmle.Util;
using Semmle.Util.Logging;
@@ -21,10 +14,10 @@ internal sealed partial class FeedManager : IDisposable
private readonly ILogger logger;
private readonly IDotNet dotnet;
- private readonly FileProvider fileProvider;
- private readonly DependabotProxy? dependabotProxy;
+ private readonly IFileProvider fileProvider;
private readonly DependencyDirectory emptyPackageDirectory;
private readonly ImmutableHashSet privateRegistryFeeds;
+ private readonly IFeedManagerIO feedManagerIo;
///
/// Gets whether there are private package registries configured for C#.
@@ -79,12 +72,12 @@ internal sealed partial class FeedManager : IDisposable
///
public ImmutableHashSet ReachableFallbackFeeds => lazyReachableFallbackFeeds.Value;
- public FeedManager(ILogger logger, IDotNet dotnet, DependabotProxy? dependabotProxy, FileProvider fileProvider)
+ public FeedManager(ILogger logger, IDotNet dotnet, IDependabotProxy? dependabotProxy, IFileProvider fileProvider, IFeedManagerIO feedManagerIo)
{
this.logger = logger;
this.dotnet = dotnet;
- this.dependabotProxy = dependabotProxy;
this.fileProvider = fileProvider;
+ this.feedManagerIo = feedManagerIo;
privateRegistryFeeds = dependabotProxy?.RegistryURLs.ToImmutableHashSet() ?? [];
HasPrivateRegistryFeeds = privateRegistryFeeds.Count > 0;
emptyPackageDirectory = new DependencyDirectory("empty", "empty package", logger);
@@ -105,17 +98,9 @@ public FeedManager(ILogger logger, IDotNet dotnet, DependabotProxy? dependabotPr
});
}
- private string? GetDirectoryName(string path)
+ public FeedManager(ILogger logger, IDotNet dotnet, IDependabotProxy? dependabotProxy, IFileProvider fileProvider)
+ : this(logger, dotnet, dependabotProxy, fileProvider, new FeedManagerIO(logger, dependabotProxy))
{
- try
- {
- return new FileInfo(path).Directory?.FullName;
- }
- catch (Exception exc)
- {
- logger.LogWarning($"Failed to get directory of '{path}': {exc}");
- }
- return null;
}
private IEnumerable GetFeeds(Func> getNugetFeeds)
@@ -157,19 +142,16 @@ private IEnumerable GetFeedsFromNugetConfig(string nugetConfigPath) =>
/// If there are no feeds, a dummy source argument is added to override any default feeds that `restore` would use.
///
/// The list of feeds to use for the restore command.
- /// The prefix to use for each source argument (e.g., "-s").
/// The list of NuGet sources arguments for the restore command.
- public List FeedsToRestoreArgument(IEnumerable feeds, string sourceArgumentPrefix)
+ public List RestoreFeeds(IEnumerable feeds)
{
// If there are no feeds, we want to override any default feeds that `restore` would use by passing a dummy source argument.
if (!feeds.Any())
{
- return [sourceArgumentPrefix, emptyPackageDirectory.DirInfo.FullName];
+ return [emptyPackageDirectory.DirInfo.FullName];
}
- // Add package sources. If any are present, they override all sources specified in
- // the configuration file(s).
- return feeds.SelectMany(feed => [sourceArgumentPrefix, feed]).ToList();
+ return feeds.ToList();
}
private IEnumerable FeedsToUseAux(HashSet feedsToConsider)
@@ -196,22 +178,12 @@ private IEnumerable FeedsToUseAux(HashSet feedsToConsider)
public IEnumerable FeedsToUse(string path)
{
// Find the path specific feeds.
- var folder = GetDirectoryName(path);
+ var folder = feedManagerIo.GetDirectoryName(path);
var feedsToConsider = folder is not null ? GetFeedsFromFolder(folder).ToHashSet() : new HashSet();
return FeedsToUseAux(feedsToConsider);
}
- ///
- /// Constructs the NuGet sources argument for the `dotnet restore` command based on the given feeds.
- ///
- /// The list of NuGet feeds to use for the restore command.
- /// A list representing the NuGet sources arguments for the `dotnet restore` command.
- public List FeedsToDotnetRestoreArgument(IEnumerable feeds)
- {
- return FeedsToRestoreArgument(feeds, "-s");
- }
-
///
/// Constructs the list of NuGet sources to use for dotnet restore.
/// (1) Use the feeds we get from `dotnet nuget list source`
@@ -219,7 +191,7 @@ public List FeedsToDotnetRestoreArgument(IEnumerable feeds)
///
/// Path to project/solution
/// A list representing the NuGet sources arguments for the `dotnet restore` command.
- public List MakeDotnetRestoreSourcesArguments(string path)
+ public List MakeRestoreFeeds(string path)
{
// Do not construct a set of explicit NuGet sources to use for restore.
if (!CheckNugetFeedResponsiveness && !HasPrivateRegistryFeeds)
@@ -229,7 +201,7 @@ public List MakeDotnetRestoreSourcesArguments(string path)
var feedsToUse = FeedsToUse(path);
- return FeedsToDotnetRestoreArgument(feedsToUse);
+ return RestoreFeeds(feedsToUse);
}
private (int initialTimeout, int tryCount) GetFeedRequestSettings(bool isFallback)
@@ -251,76 +223,6 @@ public List MakeDotnetRestoreSourcesArguments(string path)
return (timeoutMilliSeconds, tryCount);
}
- private static async Task ExecuteGetRequest(string address, HttpClient httpClient, CancellationToken cancellationToken)
- {
- return await httpClient.GetAsync(address, HttpCompletionOption.ResponseHeadersRead, cancellationToken);
- }
-
- private bool IsFeedReachable(string feed, int timeoutMilliSeconds, int tryCount)
- {
- logger.LogInfo($"Checking if NuGet feed '{feed}' is reachable...");
-
- // Configure the HttpClient to be aware of the Dependabot Proxy, if used.
- HttpClientHandler httpClientHandler = new();
- if (dependabotProxy != null)
- {
- httpClientHandler.Proxy = new WebProxy(dependabotProxy.Address);
-
- if (dependabotProxy.Certificate != null)
- {
- httpClientHandler.ServerCertificateCustomValidationCallback = (message, cert, chain, _) =>
- {
- if (chain is null || cert is null)
- {
- var msg = cert is null && chain is null
- ? "certificate and chain"
- : chain is null
- ? "chain"
- : "certificate";
- logger.LogWarning($"Dependabot proxy certificate validation failed due to missing {msg}");
- return false;
- }
- chain.ChainPolicy.TrustMode = X509ChainTrustMode.CustomRootTrust;
- chain.ChainPolicy.CustomTrustStore.Add(dependabotProxy.Certificate);
- return chain.Build(cert);
- };
- }
- }
-
- using HttpClient client = new(httpClientHandler);
-
- for (var i = 0; i < tryCount; i++)
- {
- using var cts = new CancellationTokenSource();
- cts.CancelAfter(timeoutMilliSeconds);
- try
- {
- logger.LogInfo($"Attempt {i + 1}/{tryCount} to reach NuGet feed '{feed}'.");
- using var response = ExecuteGetRequest(feed, client, cts.Token).GetAwaiter().GetResult();
- response.EnsureSuccessStatusCode();
- logger.LogInfo($"Querying NuGet feed '{feed}' succeeded.");
- return true;
- }
- catch (Exception exc)
- {
- if (exc is TaskCanceledException tce &&
- tce.CancellationToken == cts.Token &&
- cts.Token.IsCancellationRequested)
- {
- logger.LogInfo($"Didn't receive answer from NuGet feed '{feed}' in {timeoutMilliSeconds}ms.");
- timeoutMilliSeconds *= 2;
- continue;
- }
-
- logger.LogInfo($"Querying NuGet feed '{feed}' failed. The reason for the failure: {exc.Message}");
- return false;
- }
- }
-
- logger.LogWarning($"Didn't receive answer from NuGet feed '{feed}'. Tried it {tryCount} times.");
- return false;
- }
-
///
/// Retrieves a list of excluded NuGet feeds from the corresponding environment variable.
///
@@ -374,7 +276,7 @@ public bool IsDefaultFeedReachable()
if (CheckNugetFeedResponsiveness)
{
var (initialTimeout, tryCount) = GetFeedRequestSettings(isFallback: false);
- return IsFeedReachable(PublicNugetOrgFeed, initialTimeout, tryCount);
+ return feedManagerIo.IsFeedReachable(PublicNugetOrgFeed, initialTimeout, tryCount);
}
return true;
@@ -393,7 +295,7 @@ private List GetReachableNuGetFeeds(HashSet feedsToCheck, bool i
var (initialTimeout, tryCount) = GetFeedRequestSettings(isFallback);
var reachableFeeds = feedsToCheck
- .Where(feed => IsFeedReachable(feed, initialTimeout, tryCount))
+ .Where(feed => feedManagerIo.IsFeedReachable(feed, initialTimeout, tryCount))
.ToList();
if (reachableFeeds.Count == 0)
@@ -477,7 +379,7 @@ private ImmutableHashSet GetAllFeeds()
if (nugetConfigs.Count > 0)
{
var nugetConfigFeeds = nugetConfigs
- .Select(GetDirectoryName)
+ .Select(feedManagerIo.GetDirectoryName)
.Where(folder => folder != null)
.SelectMany(folder => GetFeedsFromFolder(folder!))
.ToHashSet();
diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FeedManagerIO.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FeedManagerIO.cs
new file mode 100644
index 000000000000..8e771f6037a4
--- /dev/null
+++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FeedManagerIO.cs
@@ -0,0 +1,109 @@
+
+using System;
+using System.IO;
+using Semmle.Util.Logging;
+using System.Net.Http;
+using System.Net;
+using System.Security.Cryptography.X509Certificates;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace Semmle.Extraction.CSharp.DependencyFetching
+{
+ public class FeedManagerIO : IFeedManagerIO
+ {
+ private readonly ILogger logger;
+ private readonly IDependabotProxy? dependabotProxy;
+
+ public FeedManagerIO(ILogger logger, IDependabotProxy? dependabotProxy)
+ {
+ this.logger = logger;
+ this.dependabotProxy = dependabotProxy;
+ }
+
+ public string? GetDirectoryName(string path)
+ {
+ try
+ {
+ return new FileInfo(path).Directory?.FullName;
+ }
+ catch (Exception exc)
+ {
+ logger.LogWarning($"Failed to get directory of '{path}': {exc}");
+ }
+ return null;
+ }
+
+ private static async Task ExecuteGetRequest(string address, HttpClient httpClient, CancellationToken cancellationToken)
+ {
+ return await httpClient.GetAsync(address, HttpCompletionOption.ResponseHeadersRead, cancellationToken);
+ }
+
+ public bool IsFeedReachable(string feed, int timeoutMilliSeconds, int tryCount)
+ {
+ logger.LogInfo($"Checking if NuGet feed '{feed}' is reachable...");
+
+ // Configure the HttpClient to be aware of the Dependabot Proxy, if used.
+ HttpClientHandler httpClientHandler = new();
+ if (dependabotProxy != null)
+ {
+ httpClientHandler.Proxy = new WebProxy(dependabotProxy.Address);
+
+ if (dependabotProxy.Certificate != null)
+ {
+ httpClientHandler.ServerCertificateCustomValidationCallback = (message, cert, chain, _) =>
+ {
+ if (chain is null || cert is null)
+ {
+ var msg = cert is null && chain is null
+ ? "certificate and chain"
+ : chain is null
+ ? "chain"
+ : "certificate";
+ logger.LogWarning($"Dependabot proxy certificate validation failed due to missing {msg}");
+ return false;
+ }
+ chain.ChainPolicy.TrustMode = X509ChainTrustMode.CustomRootTrust;
+ chain.ChainPolicy.CustomTrustStore.Add(dependabotProxy.Certificate);
+ return chain.Build(cert);
+ };
+ }
+ }
+
+ using HttpClient client = new(httpClientHandler);
+
+ for (var i = 0; i < tryCount; i++)
+ {
+ using var cts = new CancellationTokenSource();
+ cts.CancelAfter(timeoutMilliSeconds);
+ try
+ {
+ logger.LogInfo($"Attempt {i + 1}/{tryCount} to reach NuGet feed '{feed}'.");
+ using var response = ExecuteGetRequest(feed, client, cts.Token).GetAwaiter().GetResult();
+ response.EnsureSuccessStatusCode();
+ logger.LogInfo($"Querying NuGet feed '{feed}' succeeded.");
+ return true;
+ }
+ catch (Exception exc)
+ {
+ if (exc is TaskCanceledException tce &&
+ tce.CancellationToken == cts.Token &&
+ cts.Token.IsCancellationRequested)
+ {
+ logger.LogInfo($"Didn't receive answer from NuGet feed '{feed}' in {timeoutMilliSeconds}ms.");
+ timeoutMilliSeconds *= 2;
+ continue;
+ }
+
+ logger.LogInfo($"Querying NuGet feed '{feed}' failed. The reason for the failure: {exc.Message}");
+ return false;
+ }
+ }
+
+ logger.LogWarning($"Didn't receive answer from NuGet feed '{feed}'. Tried it {tryCount} times.");
+ return false;
+ }
+
+
+ }
+}
diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FileProvider.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FileProvider.cs
index 9e6b810b95ec..4f55bbff7f27 100644
--- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FileProvider.cs
+++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FileProvider.cs
@@ -2,12 +2,11 @@
using System.Collections.Generic;
using System.IO;
using System.Linq;
-using System.Security.Policy;
using Semmle.Util.Logging;
namespace Semmle.Extraction.CSharp.DependencyFetching
{
- public class FileProvider
+ public class FileProvider : IFileProvider
{
private static readonly HashSet binaryFileExtensions = [".dll", ".exe"]; // TODO: add more binary file extensions.
diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IDependabotProxy.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IDependabotProxy.cs
new file mode 100644
index 000000000000..37a11900fddf
--- /dev/null
+++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IDependabotProxy.cs
@@ -0,0 +1,29 @@
+using System;
+using System.Collections.Generic;
+using System.Security.Cryptography.X509Certificates;
+
+namespace Semmle.Extraction.CSharp.DependencyFetching
+{
+ public interface IDependabotProxy : IDisposable
+ {
+ ///
+ /// The full address of the Dependabot proxy, if available.
+ ///
+ string Address { get; }
+
+ ///
+ /// The URLs of package registries that are configured for the proxy.
+ ///
+ HashSet RegistryURLs { get; }
+
+ ///
+ /// The path to the temporary file where the certificate is stored.
+ ///
+ string? CertificatePath { get; }
+
+ ///
+ /// The certificate used for the Dependabot proxy.
+ ///
+ X509Certificate2? Certificate { get; }
+ }
+}
diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IFeedManagerIO.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IFeedManagerIO.cs
new file mode 100644
index 000000000000..380c5f3b5f28
--- /dev/null
+++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IFeedManagerIO.cs
@@ -0,0 +1,16 @@
+
+namespace Semmle.Extraction.CSharp.DependencyFetching
+{
+ public interface IFeedManagerIO
+ {
+ ///
+ /// Gets the directory name of the specified path.
+ ///
+ string? GetDirectoryName(string path);
+
+ ///
+ /// Returns true if the feed is reachable within the specified timeout and try count.
+ ///
+ bool IsFeedReachable(string feed, int timeoutMilliSeconds, int tryCount);
+ }
+}
diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IFileProvider.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IFileProvider.cs
new file mode 100644
index 000000000000..919be0af61a7
--- /dev/null
+++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IFileProvider.cs
@@ -0,0 +1,22 @@
+using System.Collections.Generic;
+using System.IO;
+
+namespace Semmle.Extraction.CSharp.DependencyFetching
+{
+ public interface IFileProvider
+ {
+ DirectoryInfo SourceDir { get; }
+ IEnumerable SmallNonBinary { get; }
+ IEnumerable Sources { get; }
+ ICollection Projects { get; }
+ ICollection Solutions { get; }
+ IEnumerable Dlls { get; }
+ ICollection NugetConfigs { get; }
+ ICollection NugetExes { get; }
+ string? RootNugetConfig { get; }
+ IEnumerable GlobalJsons { get; }
+ ICollection PackagesConfigs { get; }
+ ICollection RazorViews { get; }
+ ICollection Resources { get; }
+ }
+}
diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs
index 2867a7cc6327..85d6056d7218 100644
--- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs
+++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs
@@ -15,7 +15,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
{
internal sealed partial class NugetPackageRestorer : IDisposable
{
- private readonly FileProvider fileProvider;
+ private readonly IFileProvider fileProvider;
private readonly FileContent fileContent;
private readonly IDotNet dotnet;
private readonly IDiagnosticsWriter diagnosticsWriter;
@@ -29,10 +29,10 @@ internal sealed partial class NugetPackageRestorer : IDisposable
public NugetPackageRestorer(
- FileProvider fileProvider,
+ IFileProvider fileProvider,
FileContent fileContent,
IDotNet dotnet,
- DependabotProxy? dependabotProxy,
+ IDependabotProxy? dependabotProxy,
IDiagnosticsWriter diagnosticsWriter,
ILogger logger,
ICompilationInfoContainer compilationInfoContainer)
@@ -53,7 +53,7 @@ public NugetPackageRestorer(
public string? TryRestore(string package)
{
var feeds = feedManager.CheckNugetFeedResponsiveness ? feedManager.ReachableFeeds : feedManager.AllFeeds;
- var nugetSources = feedManager.FeedsToDotnetRestoreArgument(feeds);
+ var nugetSources = feedManager.RestoreFeeds(feeds);
if (TryRestorePackageManually(package, nugetSources))
{
var packageDir = DependencyManager.GetPackageDirectory(package, missingPackageDirectory.DirInfo);
@@ -216,7 +216,7 @@ private IEnumerable RestoreSolutions(out DependencyContainer dependencie
var projects = fileProvider.Solutions.SelectMany(solution =>
{
logger.LogInfo($"Restoring solution {solution}...");
- var nugetSources = feedManager.MakeDotnetRestoreSourcesArguments(solution);
+ var nugetSources = feedManager.MakeRestoreFeeds(solution);
var res = dotnet.Restore(new(solution, PackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: true, NugetSources: nugetSources, TargetWindows: isWindows));
if (res.Success)
{
@@ -264,7 +264,7 @@ private void RestoreProjects(IEnumerable projects, out ConcurrentBag projects, out ConcurrentBag p.ToLowerInvariant());
var alreadyDownloadedLegacyPackages = GetRestoredLegacyPackageNames();
diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/PackagesConfigRestorer.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/PackagesConfigRestorer.cs
index e5a9365a08e3..d4403bb955ef 100644
--- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/PackagesConfigRestorer.cs
+++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/PackagesConfigRestorer.cs
@@ -1,6 +1,5 @@
using System;
using System.Collections.Generic;
-using System.Collections.Immutable;
using System.Diagnostics;
using System.IO;
using System.Linq;
@@ -34,7 +33,7 @@ internal interface IPackagesConfigRestore
///
internal class PackagesConfigRestoreFactory
{
- public static IPackagesConfigRestore Create(FileProvider fileProvider, DependencyDirectory packageDirectory, Semmle.Util.Logging.ILogger logger, FeedManager feedManager)
+ public static IPackagesConfigRestore Create(IFileProvider fileProvider, DependencyDirectory packageDirectory, Semmle.Util.Logging.ILogger logger, FeedManager feedManager)
{
if (SystemBuildActions.Instance.IsWindows() || SystemBuildActions.Instance.IsMonoInstalled())
{
@@ -56,7 +55,7 @@ private class NugetExeWrapper : IPackagesConfigRestore
public int PackageCount => fileProvider.PackagesConfigs.Count;
- private readonly FileProvider fileProvider;
+ private readonly IFileProvider fileProvider;
///
/// The packages directory.
@@ -75,7 +74,7 @@ private class NugetExeWrapper : IPackagesConfigRestore
///
/// Create the package manager for a specified source tree.
///
- public NugetExeWrapper(FileProvider fileProvider, DependencyDirectory packageDirectory, Semmle.Util.Logging.ILogger logger, FeedManager feedManager)
+ public NugetExeWrapper(IFileProvider fileProvider, DependencyDirectory packageDirectory, Semmle.Util.Logging.ILogger logger, FeedManager feedManager)
{
this.fileProvider = fileProvider;
this.packageDirectory = packageDirectory;
@@ -180,7 +179,8 @@ private bool TryRestoreNugetPackage(string packagesConfig)
{
feedsToUse.Add(FeedManager.PublicNugetOrgFeed);
}
- sourcesArgument = feedManager.FeedsToRestoreArgument(feedsToUse, "-Source");
+ var restoreFeeds = feedManager.RestoreFeeds(feedsToUse);
+ sourcesArgument = restoreFeeds.SelectMany(feed => ["-Source", feed]).ToList();
}
/* Use nuget.exe to install a package.
diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/SourceGenerators/DotnetSourceGeneratorBase.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/SourceGenerators/DotnetSourceGeneratorBase.cs
index c17981803ddb..6a150e8d542a 100644
--- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/SourceGenerators/DotnetSourceGeneratorBase.cs
+++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/SourceGenerators/DotnetSourceGeneratorBase.cs
@@ -9,14 +9,14 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
{
internal abstract class DotnetSourceGeneratorBase : SourceGeneratorBase where T : DotnetSourceGeneratorWrapper
{
- protected readonly FileProvider fileProvider;
+ protected readonly IFileProvider fileProvider;
protected readonly FileContent fileContent;
protected readonly IDotNet dotnet;
protected readonly ICompilationInfoContainer compilationInfoContainer;
protected readonly IEnumerable references;
public DotnetSourceGeneratorBase(
- FileProvider fileProvider,
+ IFileProvider fileProvider,
FileContent fileContent,
IDotNet dotnet,
ICompilationInfoContainer compilationInfoContainer,
diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/SourceGenerators/RazorGenerator.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/SourceGenerators/RazorGenerator.cs
index 3f9a17dc6b30..8d4536b4503c 100644
--- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/SourceGenerators/RazorGenerator.cs
+++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/SourceGenerators/RazorGenerator.cs
@@ -8,7 +8,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
internal class RazorGenerator : DotnetSourceGeneratorBase
{
public RazorGenerator(
- FileProvider fileProvider,
+ IFileProvider fileProvider,
FileContent fileContent,
IDotNet dotnet,
ICompilationInfoContainer compilationInfoContainer,
diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/SourceGenerators/ResxGenerator.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/SourceGenerators/ResxGenerator.cs
index da66ef275442..e6e27ea27534 100644
--- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/SourceGenerators/ResxGenerator.cs
+++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/SourceGenerators/ResxGenerator.cs
@@ -10,7 +10,7 @@ internal class ResxGenerator : DotnetSourceGeneratorBase
private readonly string? sourceGeneratorFolder = null;
public ResxGenerator(
- FileProvider fileProvider,
+ IFileProvider fileProvider,
FileContent fileContent,
IDotNet dotnet,
ICompilationInfoContainer compilationInfoContainer,
diff --git a/csharp/extractor/Semmle.Extraction.Tests/DotNet.cs b/csharp/extractor/Semmle.Extraction.Tests/DotNet.cs
index 260eae10bd1b..77e88a58443a 100644
--- a/csharp/extractor/Semmle.Extraction.Tests/DotNet.cs
+++ b/csharp/extractor/Semmle.Extraction.Tests/DotNet.cs
@@ -148,11 +148,11 @@ public void TestDotnetRestoreProjectToDirectory3()
var dotnet = MakeDotnet(dotnetCliInvoker);
// Execute
- var res = dotnet.Restore(new("myproject.csproj", "mypackages", false, [], true));
+ var res = dotnet.Restore(new("myproject.csproj", "mypackages", false, ["https://my.nuget.source1"], true));
// Verify
var lastArgs = dotnetCliInvoker.GetLastArgs();
- Assert.Equal(["restore", "--no-dependencies", "myproject.csproj", "--packages", "mypackages", "/p:DisableImplicitNuGetFallbackFolder=true", "--verbosity", "normal", "--force"], lastArgs);
+ Assert.Equal(["restore", "--no-dependencies", "myproject.csproj", "--packages", "mypackages", "/p:DisableImplicitNuGetFallbackFolder=true", "--verbosity", "normal", "--force", "-s", "https://my.nuget.source1"], lastArgs);
Assert.Equal(2, res.AssetsFilePaths.Count());
Assert.Contains("/path/to/project.assets.json", res.AssetsFilePaths);
Assert.Contains("/path/to/project2.assets.json", res.AssetsFilePaths);
@@ -166,11 +166,11 @@ public void TestDotnetRestoreSolutionToDirectory1()
var dotnet = MakeDotnet(dotnetCliInvoker);
// Execute
- var res = dotnet.Restore(new("mysolution.sln", "mypackages", false, []));
+ var res = dotnet.Restore(new("mysolution.sln", "mypackages", false, ["https://my.nuget.source1", "https://my.nuget.source2"]));
// Verify
var lastArgs = dotnetCliInvoker.GetLastArgs();
- Assert.Equal(["restore", "--no-dependencies", "mysolution.sln", "--packages", "mypackages", "/p:DisableImplicitNuGetFallbackFolder=true", "--verbosity", "normal"], lastArgs);
+ Assert.Equal(["restore", "--no-dependencies", "mysolution.sln", "--packages", "mypackages", "/p:DisableImplicitNuGetFallbackFolder=true", "--verbosity", "normal", "-s", "https://my.nuget.source1", "-s", "https://my.nuget.source2"], lastArgs);
Assert.Equal(2, res.RestoredProjects.Count());
Assert.Contains("/path/to/project.csproj", res.RestoredProjects);
Assert.Contains("/path/to/project2.csproj", res.RestoredProjects);
diff --git a/csharp/extractor/Semmle.Extraction.Tests/DotNetStub.cs b/csharp/extractor/Semmle.Extraction.Tests/DotNetStub.cs
new file mode 100644
index 000000000000..119e39fd0974
--- /dev/null
+++ b/csharp/extractor/Semmle.Extraction.Tests/DotNetStub.cs
@@ -0,0 +1,37 @@
+using System;
+using System.Collections.Generic;
+using Semmle.Extraction.CSharp.DependencyFetching;
+
+namespace Semmle.Extraction.Tests
+{
+ internal class DotNetStub : IDotNet
+ {
+ private readonly IList runtimes;
+ private readonly IList sdks;
+ private readonly IList nugetFeedsFromConfig;
+ private readonly IList nugetFeedsFromFolder;
+
+ public DotNetStub(IList runtimes, IList sdks, IList nugetFeedsFromConfig, IList nugetFeedsFromFolder)
+ {
+ this.runtimes = runtimes;
+ this.sdks = sdks;
+ this.nugetFeedsFromConfig = nugetFeedsFromConfig;
+ this.nugetFeedsFromFolder = nugetFeedsFromFolder;
+ }
+ public bool AddPackage(string folder, string package) => true;
+
+ public bool New(string folder) => true;
+
+ public RestoreResult Restore(RestoreSettings restoreSettings) => new(true, Array.Empty());
+
+ public IList GetListedRuntimes() => runtimes;
+
+ public IList GetListedSdks() => sdks;
+
+ public bool Exec(List execArgs) => true;
+
+ public IList GetNugetFeeds(string nugetConfig) => nugetFeedsFromConfig;
+
+ public IList GetNugetFeedsFromFolder(string folderPath) => nugetFeedsFromFolder;
+ }
+}
diff --git a/csharp/extractor/Semmle.Extraction.Tests/FeedManager.cs b/csharp/extractor/Semmle.Extraction.Tests/FeedManager.cs
new file mode 100644
index 000000000000..f70efdb4cdcc
--- /dev/null
+++ b/csharp/extractor/Semmle.Extraction.Tests/FeedManager.cs
@@ -0,0 +1,187 @@
+using Xunit;
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using Semmle.Extraction.CSharp.DependencyFetching;
+
+namespace Semmle.Extraction.Tests
+{
+ public class DependabotProxyStub : IDependabotProxy
+ {
+ public string Address { get; } = "";
+ public HashSet RegistryURLs { get; } = ["https://example.com/registry1", "https://example.com/registry2"];
+ public string? CertificatePath { get; } = null;
+ public System.Security.Cryptography.X509Certificates.X509Certificate2? Certificate { get; } = null;
+
+ public void Dispose() { }
+ }
+
+ public class FeedManagerIOStub : IFeedManagerIO
+ {
+ private readonly List unreachableFeeds;
+
+ public FeedManagerIOStub(List unreachableFeeds)
+ {
+ this.unreachableFeeds = unreachableFeeds;
+ }
+
+ public string? GetDirectoryName(string path)
+ {
+ return "/path/to/folder";
+ }
+
+ public bool IsFeedReachable(string feed, int timeoutMilliSeconds, int tryCount)
+ {
+ return !unreachableFeeds.Contains(feed);
+ }
+ }
+
+ public class FileProviderStub : IFileProvider
+ {
+ public DirectoryInfo SourceDir { get; } = new DirectoryInfo("/path/to/source");
+ public IEnumerable SmallNonBinary { get; } = Enumerable.Empty();
+ public IEnumerable Sources { get; } = Enumerable.Empty();
+ public ICollection Projects { get; } = new List();
+ public ICollection Solutions { get; } = new List();
+ public IEnumerable Dlls { get; } = Enumerable.Empty();
+ public ICollection NugetConfigs { get; } = ["/path/to/nuget.config"];
+ public ICollection NugetExes { get; } = new List();
+ public string? RootNugetConfig { get; } = null;
+ public IEnumerable GlobalJsons { get; } = Enumerable.Empty();
+ public ICollection PackagesConfigs { get; } = new List();
+ public ICollection RazorViews { get; } = new List();
+ public ICollection Resources { get; } = new List();
+ }
+
+ public class FeedManagerTests
+ {
+ private static FeedManager MakeFeedManager()
+ {
+ var logger = new LoggerStub();
+ var dotnet = new DotNetStub([], [], ["E https://feed.from/config"], ["E https://feed.from/folder1", "E https://feed.from/folder2", "D https://feed.from/folder3"]);
+ var dependabotProxy = new DependabotProxyStub();
+ var fileProvider = new FileProviderStub();
+ var feedManagerIo = new FeedManagerIOStub(["https://example.com/registry1", "https://feed.from/folder2"]);
+ return new FeedManager(logger, dotnet, dependabotProxy, fileProvider, feedManagerIo);
+ }
+
+ [Fact]
+ public void TestExplicitFeeds()
+ {
+ // Setup
+ var feedManager = MakeFeedManager();
+
+ // Execute
+ var actualFeeds = feedManager.ExplicitFeeds;
+
+ // Verify
+ Assert.Equal([
+ "https://example.com/registry1",
+ "https://example.com/registry2",
+ "https://feed.from/config"
+ ], actualFeeds);
+ }
+
+ [Fact]
+ public void TestInheritedFeeds()
+ {
+ // Setup
+ var feedManager = MakeFeedManager();
+
+ // Execute
+ var inherited = feedManager.InheritedFeeds;
+
+ // Verify
+ Assert.Equal([
+ "https://feed.from/folder1",
+ "https://feed.from/folder2"
+ ], inherited);
+ }
+
+ [Fact]
+ public void TestAllFeeds()
+ {
+ // Setup
+ var feedManager = MakeFeedManager();
+
+ // Execute
+ var all = feedManager.AllFeeds;
+
+ // Verify
+ Assert.Equal([
+ "https://example.com/registry1",
+ "https://example.com/registry2",
+ "https://feed.from/config",
+ "https://feed.from/folder1",
+ "https://feed.from/folder2"
+ ], all);
+ }
+
+ [Fact]
+ public void TestReachableFeeds()
+ {
+ // Setup
+ var feedManager = MakeFeedManager();
+
+ // Execute
+ var reachableFeeds = feedManager.ReachableFeeds;
+
+ // Verify
+ Assert.Equal([
+ "https://example.com/registry2",
+ "https://feed.from/config",
+ "https://feed.from/folder1"
+ ], reachableFeeds);
+ }
+
+ [Fact]
+ public void TestReachableExplicitFeeds()
+ {
+ // Setup
+ var feedManager = MakeFeedManager();
+
+ // Execute
+ var reachableFeeds = feedManager.ReachableExplicitFeeds;
+
+ // Verify
+ Assert.Equal([
+ "https://example.com/registry2",
+ "https://feed.from/config"
+ ], reachableFeeds);
+ }
+
+ [Fact]
+ public void TestReachableFallbackFeeds()
+ {
+ // Setup
+ var feedManager = MakeFeedManager();
+
+ // Execute
+ var reachableFallback = feedManager.ReachableFallbackFeeds;
+
+ // Verify
+ Assert.Equal([
+ "https://example.com/registry2",
+ "https://feed.from/config",
+ "https://api.nuget.org/v3/index.json"
+ ], reachableFallback);
+ }
+
+ [Fact]
+ public void TestFeedsToUse()
+ {
+ // Setup
+ var feedManager = MakeFeedManager();
+
+ // Execute
+ var feedsToUse = feedManager.FeedsToUse("/path/to/packages.config").ToHashSet();
+
+ // Verify
+ Assert.Equal([
+ "https://example.com/registry2",
+ "https://feed.from/folder1"
+ ], feedsToUse);
+ }
+ }
+}
diff --git a/csharp/extractor/Semmle.Extraction.Tests/Runtime.cs b/csharp/extractor/Semmle.Extraction.Tests/Runtime.cs
index 348cadae2fd0..ea2483f3faf3 100644
--- a/csharp/extractor/Semmle.Extraction.Tests/Runtime.cs
+++ b/csharp/extractor/Semmle.Extraction.Tests/Runtime.cs
@@ -5,37 +5,12 @@
namespace Semmle.Extraction.Tests
{
- internal class DotNetStub : IDotNet
- {
- private readonly IList runtimes;
- private readonly IList sdks;
-
- public DotNetStub(IList runtimes, IList sdks)
- {
- this.runtimes = runtimes;
- this.sdks = sdks;
- }
- public bool AddPackage(string folder, string package) => true;
-
- public bool New(string folder) => true;
-
- public RestoreResult Restore(RestoreSettings restoreSettings) => new(true, Array.Empty());
-
- public IList GetListedRuntimes() => runtimes;
-
- public IList GetListedSdks() => sdks;
-
- public bool Exec(List execArgs) => true;
-
- public IList GetNugetFeeds(string nugetConfig) => [];
-
- public IList GetNugetFeedsFromFolder(string folderPath) => [];
- }
-
public class RuntimeTests
{
private static string FixExpectedPathOnWindows(string path) => path.Replace('\\', '/');
+ private static DotNetStub MakeDotNetStub(IList listedRuntimes) => new DotNetStub(listedRuntimes, null!, [], []);
+
[Fact]
public void TestRuntime1()
{
@@ -50,7 +25,7 @@ public void TestRuntime1()
"Microsoft.NETCore.App 7.0.0 [/path/dotnet/shared/Microsoft.NETCore.App]",
"Microsoft.NETCore.App 7.0.2 [/path/dotnet/shared/Microsoft.NETCore.App]"
};
- var dotnet = new DotNetStub(listedRuntimes, null!);
+ var dotnet = MakeDotNetStub(listedRuntimes);
var runtime = new Runtime(dotnet);
// Execute
@@ -76,7 +51,7 @@ public void TestRuntime2()
"Microsoft.NETCore.App 8.0.0-preview.5.43280.8 [/path/dotnet/shared/Microsoft.NETCore.App]",
"Microsoft.NETCore.App 8.0.0-preview.5.23280.8 [/path/dotnet/shared/Microsoft.NETCore.App]"
};
- var dotnet = new DotNetStub(listedRuntimes, null!);
+ var dotnet = new DotNetStub(listedRuntimes, null!, [], []);
var runtime = new Runtime(dotnet);
// Execute
@@ -99,7 +74,7 @@ public void TestRuntime3()
"Microsoft.NETCore.App 8.0.0-rc.4.43280.8 [/path/dotnet/shared/Microsoft.NETCore.App]",
"Microsoft.NETCore.App 8.0.0-preview.5.23280.8 [/path/dotnet/shared/Microsoft.NETCore.App]"
};
- var dotnet = new DotNetStub(listedRuntimes, null!);
+ var dotnet = MakeDotNetStub(listedRuntimes);
var runtime = new Runtime(dotnet);
// Execute
@@ -128,7 +103,7 @@ public void TestRuntime4()
@"Microsoft.WindowsDesktop.App 6.0.20 [C:\Program Files\dotnet\shared\Microsoft.WindowsDesktop.App]",
@"Microsoft.WindowsDesktop.App 7.0.4 [C:\Program Files\dotnet\shared\Microsoft.WindowsDesktop.App]"
};
- var dotnet = new DotNetStub(listedRuntimes, null!);
+ var dotnet = MakeDotNetStub(listedRuntimes);
var runtime = new Runtime(dotnet);
// Execute
@@ -149,6 +124,7 @@ public class SdkTests
{
private static string FixExpectedPathOnWindows(string path) => path.Replace('\\', '/');
+ private static DotNetStub MakeDotNetStub(IList listedSdks) => new DotNetStub(null!, listedSdks, [], []);
[Fact]
public void TestSdk1()
{
@@ -163,7 +139,7 @@ public void TestSdk1()
"6.0.102 [/usr/local/share/dotnet/sdk6]",
"6.0.301 [/usr/local/share/dotnet/sdk7]",
};
- var dotnet = new DotNetStub(null!, listedSdks);
+ var dotnet = MakeDotNetStub(listedSdks);
var sdk = new Sdk(dotnet, new LoggerStub());
// Execute
@@ -185,7 +161,7 @@ public void TestSdk2()
"8.0.100-preview.7.23376.3 [/usr/local/share/dotnet/sdk3]",
"7.0.400 [/usr/local/share/dotnet/sdk4]",
};
- var dotnet = new DotNetStub(null!, listedSdks);
+ var dotnet = MakeDotNetStub(listedSdks);
var sdk = new Sdk(dotnet, new LoggerStub());
// Execute