Merge branch 'main' into 20823-globalVarRef-document-defaultView

Author: Eliav2

csharp/autobuilder/Semmle.Autobuild.CSharp/DotNetRule.cs

@@ -84,11 +84,7 @@ public static BuildScript WithDotNet(IAutobuilder<AutobuildOptionsShared> builde
             var temp = FileUtils.GetTemporaryWorkingDirectory(builder.Actions.GetEnvironmentVariable, builder.Options.Language.UpperCaseName, out var shouldCleanUp);
             return DotNet.WithDotNet(builder.Actions, builder.Logger, builder.Paths.Select(x => x.Item1), temp, shouldCleanUp, ensureDotNetAvailable, builder.Options.DotNetVersion, installDir =>
             {
-                var env = new Dictionary<string, string>
-                {
-                    { "DOTNET_SKIP_FIRST_TIME_EXPERIENCE", "true" },
-                    { "MSBUILDDISABLENODEREUSE", "1" }
-                };
+                var env = DotNet.MinimalEnvironment.ToDictionary();
                 if (installDir is not null)
                 {
                     // The installation succeeded, so use the newly installed .NET

csharp/codeql-extractor.yml

@@ -74,3 +74,8 @@ options:
       [EXPERIMENTAL] The value is a path to the MsBuild binary log file that should be extracted.
       This option only works when `--build-mode none` is also specified.
     type: array
+  buildless_dependency_dir:
+    title: The path where buildless (standalone) extraction should keep dependencies.
+    description: >
+      If set, the buildless (standalone) extractor will store dependencies in this directory.
+    type: string

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyDirectory.cs

@@ -0,0 +1,62 @@
+using System;
+using System.IO;
+using Semmle.Util;
+using Semmle.Util.Logging;
+
+namespace Semmle.Extraction.CSharp.DependencyFetching
+{
+    /// <summary>
+    /// A directory used for storing fetched dependencies.
+    /// When a specific directory is set via the dependency directory extractor option,
+    /// we store dependencies in that directory for caching purposes.
+    /// Otherwise, we create a temporary directory that is deleted upon disposal.
+    /// </summary>
+    public sealed class DependencyDirectory : IDisposable
+    {
+        private readonly string userReportedDirectoryPurpose;
+        private readonly ILogger logger;
+        private readonly bool attemptCleanup;
+
+        public DirectoryInfo DirInfo { get; }
+
+        public DependencyDirectory(string subfolderName, string userReportedDirectoryPurpose, ILogger logger)
+        {
+            this.logger = logger;
+            this.userReportedDirectoryPurpose = userReportedDirectoryPurpose;
+
+            string path;
+            if (EnvironmentVariables.GetBuildlessDependencyDir() is string dir)
+            {
+                path = dir;
+                attemptCleanup = false;
+            }
+            else
+            {
+                path = FileUtils.GetTemporaryWorkingDirectory(out _);
+                attemptCleanup = true;
+            }
+            DirInfo = new DirectoryInfo(Path.Join(path, subfolderName));
+            DirInfo.Create();
+        }
+
+        public void Dispose()
+        {
+            if (!attemptCleanup)
+            {
+                logger.LogInfo($"Keeping {userReportedDirectoryPurpose} directory {DirInfo.FullName} for possible caching purposes.");
+                return;
+            }
+
+            try
+            {
+                DirInfo.Delete(true);
+            }
+            catch (Exception exc)
+            {
+                logger.LogInfo($"Couldn't delete {userReportedDirectoryPurpose} directory {exc.Message}");
+            }
+        }
+
+        public override string ToString() => DirInfo.FullName;
+    }
+}

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.Collections.ObjectModel;
 using System.IO;
 using System.Linq;
 using Newtonsoft.Json.Linq;
@@ -140,6 +141,8 @@ public IList<string> GetNugetFeedsFromFolder(string folderPath)
         // The version number should be kept in sync with the version .NET version used for building the application.
         public const string LatestDotNetSdkVersion = "9.0.300";
 
+        public static ReadOnlyDictionary<string, string> MinimalEnvironment => IDotNetCliInvoker.MinimalEnvironment;
+
         /// <summary>
         /// Returns a script for downloading relevant versions of the
         /// .NET SDK. The SDK(s) will be installed at <code>installDir</code>
@@ -254,7 +257,6 @@ BuildScript GetInstall(string pwsh) =>
                 else
                 {
                     var dotnetInstallPath = actions.PathCombine(tempWorkingDirectory, ".dotnet", "dotnet-install.sh");
-
                     var downloadDotNetInstallSh = BuildScript.DownloadFile(
                         "https://dot.net/v1/dotnet-install.sh",
                         dotnetInstallPath,
@@ -269,17 +271,28 @@ BuildScript GetInstall(string pwsh) =>
                     prelude = downloadDotNetInstallSh & chmod.Script;
                     postlude = shouldCleanUp ? BuildScript.DeleteFile(dotnetInstallPath) : BuildScript.Success;
 
-                    getInstall = version => new CommandBuilder(actions).
-                        RunCommand(dotnetInstallPath).
-                        Argument("--channel").
-                        Argument("release").
-                        Argument("--version").
-                        Argument(version).
-                        Argument("--install-dir").
-                        Argument(path).Script;
+                    getInstall = version =>
+                    {
+                        var cb = new CommandBuilder(actions).
+                            RunCommand(dotnetInstallPath).
+                            Argument("--channel").
+                            Argument("release").
+                            Argument("--version").
+                            Argument(version);
+
+                        // Request ARM64 architecture on Apple Silicon machines
+                        if (actions.IsRunningOnAppleSilicon())
+                        {
+                            cb.Argument("--architecture").
+                                Argument("arm64");
+                        }
+
+                        return cb.Argument("--install-dir").
+                            Argument(path).Script;
+                    };
                 }
 
-                var dotnetInfo = new CommandBuilder(actions).
+                var dotnetInfo = new CommandBuilder(actions, environment: MinimalEnvironment).
                     RunCommand(actions.PathCombine(path, "dotnet")).
                     Argument("--info").Script;
 
@@ -311,7 +324,7 @@ BuildScript GetInstall(string pwsh) =>
 
         private static BuildScript GetInstalledSdksScript(IBuildActions actions)
         {
-            var listSdks = new CommandBuilder(actions, silent: true).
+            var listSdks = new CommandBuilder(actions, silent: true, environment: MinimalEnvironment).
                 RunCommand("dotnet").
                 Argument("--list-sdks");
             return listSdks.Script;

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNetCliInvoker.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.Collections.ObjectModel;
 using System.Diagnostics;
 using Semmle.Util;
 using Semmle.Util.Logging;
@@ -36,10 +37,12 @@ private ProcessStartInfo MakeDotnetStartInfo(string args, string? workingDirecto
             {
                 startInfo.WorkingDirectory = workingDirectory;
             }
-            // Set the .NET CLI language to English to avoid localized output.
-            startInfo.EnvironmentVariables["DOTNET_CLI_UI_LANGUAGE"] = "en";
-            startInfo.EnvironmentVariables["MSBUILDDISABLENODEREUSE"] = "1";
-            startInfo.EnvironmentVariables["DOTNET_SKIP_FIRST_TIME_EXPERIENCE"] = "true";
+
+            // Set minimal environment variables.
+            foreach (var kvp in IDotNetCliInvoker.MinimalEnvironment)
+            {
+                startInfo.EnvironmentVariables[kvp.Key] = kvp.Value;
+            }
 
             // Configure the proxy settings, if applicable.
             if (this.proxy != null)

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IDotNetCliInvoker.cs

@@ -1,4 +1,5 @@
 using System.Collections.Generic;
+using System.Collections.ObjectModel;
 
 namespace Semmle.Extraction.CSharp.DependencyFetching
 {
@@ -9,6 +10,20 @@ internal interface IDotNetCliInvoker
         /// </summary>
         string Exec { get; }
 
+        /// <summary>
+        /// A minimal environment for running the .NET CLI.
+        /// 
+        /// DOTNET_CLI_UI_LANGUAGE: The .NET CLI language is set to English to avoid localized output.
+        /// MSBUILDDISABLENODEREUSE: To ensure clean environment for each build.
+        /// DOTNET_SKIP_FIRST_TIME_EXPERIENCE: To skip first time experience messages.
+        /// </summary>
+        static ReadOnlyDictionary<string, string> MinimalEnvironment { get; } = new(new Dictionary<string, string>
+        {
+            {"DOTNET_CLI_UI_LANGUAGE", "en"},
+            {"MSBUILDDISABLENODEREUSE", "1"},
+            {"DOTNET_SKIP_FIRST_TIME_EXPERIENCE", "true"}
+        });
+
         /// <summary>
         /// Execute `dotnet <paramref name="args"/>` and return true if the command succeeded, otherwise false.
         /// If `silent` is true the output of the command is logged as `debug` otherwise as `info`.

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetExeWrapper.cs

@@ -24,16 +24,16 @@ internal class NugetExeWrapper : IDisposable
         private readonly FileProvider fileProvider;
 
         /// <summary>
-        /// The computed packages directory.
-        /// This will be in the Temp location
+        /// The packages directory.
+        /// This will be in the user-specified or computed Temp location
         /// so as to not trample the source tree.
         /// </summary>
-        private readonly TemporaryDirectory packageDirectory;
+        private readonly DependencyDirectory packageDirectory;
 
         /// <summary>
         /// Create the package manager for a specified source tree.
         /// </summary>
-        public NugetExeWrapper(FileProvider fileProvider, TemporaryDirectory packageDirectory, Semmle.Util.Logging.ILogger logger)
+        public NugetExeWrapper(FileProvider fileProvider, DependencyDirectory packageDirectory, Semmle.Util.Logging.ILogger logger)
         {
             this.fileProvider = fileProvider;
             this.packageDirectory = packageDirectory;

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs

@@ -24,12 +24,12 @@ internal sealed partial class NugetPackageRestorer : IDisposable
         private readonly IDotNet dotnet;
         private readonly DependabotProxy? dependabotProxy;
         private readonly IDiagnosticsWriter diagnosticsWriter;
-        private readonly TemporaryDirectory legacyPackageDirectory;
-        private readonly TemporaryDirectory missingPackageDirectory;
+        private readonly DependencyDirectory legacyPackageDirectory;
+        private readonly DependencyDirectory missingPackageDirectory;
         private readonly ILogger logger;
         private readonly ICompilationInfoContainer compilationInfoContainer;
 
-        public TemporaryDirectory PackageDirectory { get; }
+        public DependencyDirectory PackageDirectory { get; }
 
         public NugetPackageRestorer(
             FileProvider fileProvider,
@@ -48,9 +48,9 @@ public NugetPackageRestorer(
             this.logger = logger;
             this.compilationInfoContainer = compilationInfoContainer;
 
-            PackageDirectory = new TemporaryDirectory(ComputeTempDirectoryPath("packages"), "package", logger);
-            legacyPackageDirectory = new TemporaryDirectory(ComputeTempDirectoryPath("legacypackages"), "legacy package", logger);
-            missingPackageDirectory = new TemporaryDirectory(ComputeTempDirectoryPath("missingpackages"), "missing package", logger);
+            PackageDirectory = new DependencyDirectory("packages", "package", logger);
+            legacyPackageDirectory = new DependencyDirectory("legacypackages", "legacy package", logger);
+            missingPackageDirectory = new DependencyDirectory("missingpackages", "missing package", logger);
         }
 
         public string? TryRestore(string package)

csharp/extractor/Semmle.Util/EnvironmentVariables.cs

@@ -76,5 +76,14 @@ public static IEnumerable<string> GetURLs(string name)
         {
             return Environment.GetEnvironmentVariable("CODEQL_EXTRACTOR_CSHARP_OVERLAY_BASE_METADATA_OUT");
         }
+
+        /// <summary>
+        /// If set, returns the directory where buildless dependencies should be stored.
+        /// This can be used for caching dependencies.
+        /// </summary>
+        public static string? GetBuildlessDependencyDir()
+        {
+            return Environment.GetEnvironmentVariable("CODEQL_EXTRACTOR_CSHARP_OPTION_BUILDLESS_DEPENDENCY_DIR");
+        }
     }
 }