Skip to content

Commit 7a8d7fb

Browse files
devantlerdevantler
authored
Merge branch 'main' into devantler-swift-imported-extension-mangling
2 parents 54a4400 + 27a8a7e commit 7a8d7fb

24 files changed

Lines changed: 466 additions & 113 deletions

File tree

‎.github/workflows/ql-for-ql-build.yml‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -24,9 +24,8 @@ jobs:
2424
fetch-depth: 0
2525
- name: Find codeql
2626
id: find-codeql
27-
uses: github/codeql-action/init@main
27+
uses: github/codeql-action/setup-codeql@main
2828
with:
29-
languages: javascript # does not matter
3029
tools: nightly
3130
- uses: ./.github/actions/os-version
3231
id: os_version
@@ -61,7 +60,7 @@ jobs:
6160
- name: Make database and analyze
6261
run: |
6362
./ql/target/release/buramu | tee deprecated.blame # Add a blame file for the extractor to parse.
64-
${CODEQL} database create -l=ql ${DB} --search-path "${{ github.workspace }}"
63+
${CODEQL} database create --threads=0 -l=ql ${DB} --search-path "${{ github.workspace }}"
6564
${CODEQL} database analyze -j0 --format=sarif-latest --output=ql-for-ql.sarif ${DB} ql/ql/src/codeql-suites/ql-code-scanning.qls --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
6665
env:
6766
CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}

‎.github/workflows/ql-for-ql-dataset_measure.yml‎

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,6 @@ permissions:
1717

1818
jobs:
1919
measure:
20-
env:
21-
CODEQL_THREADS: 4 # TODO: remove this once it's set by the CLI
2220
strategy:
2321
matrix:
2422
repo:
@@ -29,9 +27,7 @@ jobs:
2927

3028
- name: Find codeql
3129
id: find-codeql
32-
uses: github/codeql-action/init@main
33-
with:
34-
languages: javascript # does not matter
30+
uses: github/codeql-action/setup-codeql@main
3531
- uses: ./.github/actions/os-version
3632
id: os_version
3733
- uses: actions/cache@v3

‎.github/workflows/ql-for-ql-tests.yml‎

Lines changed: 5 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -27,9 +27,8 @@ jobs:
2727
- uses: actions/checkout@v5
2828
- name: Find codeql
2929
id: find-codeql
30-
uses: github/codeql-action/init@main
30+
uses: github/codeql-action/setup-codeql@main
3131
with:
32-
languages: javascript # does not matter
3332
tools: nightly
3433
- uses: ./.github/actions/os-version
3534
id: os_version
@@ -54,7 +53,7 @@ jobs:
5453
key: ql-for-ql-tests
5554
- name: Run QL tests
5655
run: |
57-
"${CODEQL}" test run --check-databases --check-unused-labels --check-repeated-labels --check-redefined-labels --check-use-before-definition --search-path "${{ github.workspace }}" --consistency-queries ql/ql/consistency-queries --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}" ql/ql/test
56+
"${CODEQL}" test run --threads=0 --check-databases --check-unused-labels --check-repeated-labels --check-redefined-labels --check-use-before-definition --search-path "${{ github.workspace }}" --consistency-queries ql/ql/consistency-queries --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}" ql/ql/test
5857
env:
5958
CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
6059

@@ -73,9 +72,8 @@ jobs:
7372
echo "/usr/local/opt/gnu-tar/libexec/gnubin" >> $GITHUB_PATH
7473
- name: Find codeql
7574
id: find-codeql
76-
uses: github/codeql-action/init@main
75+
uses: github/codeql-action/setup-codeql@main
7776
with:
78-
languages: javascript # does not matter
7977
tools: nightly
8078
- uses: ./.github/actions/os-version
8179
id: os_version
@@ -102,12 +100,12 @@ jobs:
102100
- name: Run a single QL tests - Unix
103101
if: runner.os != 'Windows'
104102
run: |
105-
"${CODEQL}" test run --check-databases --search-path "${{ github.workspace }}" ql/ql/test/queries/style/DeadCode/DeadCode.qlref
103+
"${CODEQL}" test run --threads=0 --check-databases --search-path "${{ github.workspace }}" ql/ql/test/queries/style/DeadCode/DeadCode.qlref
106104
env:
107105
CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
108106
- name: Run a single QL tests - Windows
109107
if: runner.os == 'Windows'
110108
shell: pwsh
111109
run: |
112110
$Env:PATH += ";$(dirname ${{ steps.find-codeql.outputs.codeql-path }})"
113-
codeql test run --check-databases --search-path "${{ github.workspace }}" ql/ql/test/queries/style/DeadCode/DeadCode.qlref
111+
codeql test run --threads=0 --check-databases --search-path "${{ github.workspace }}" ql/ql/test/queries/style/DeadCode/DeadCode.qlref

‎.github/workflows/rust-analysis.yml‎

Lines changed: 0 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -57,15 +57,6 @@ jobs:
5757
languages: ${{ matrix.language }}
5858
config-file: ./.github/codeql/codeql-config.yml
5959

60-
# TODO: Remove once Rust extractor no longer errors on this file
61-
- name: Remove malformed rust-analyzer parser fixture
62-
shell: bash
63-
run: |
64-
cargo fetch
65-
find "${CARGO_HOME:-$HOME/.cargo}/registry/src" \
66-
-path '*/ra_ap_parser-0.0.352/test_data/lexer/err/incomplete_frontmatter_before_unicode.rs' \
67-
-delete
68-
6960
- name: Autobuild
7061
uses: github/codeql-action/autobuild@main
7162

‎csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNetCliInvoker.cs‎

Lines changed: 1 addition & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -44,14 +44,7 @@ private ProcessStartInfo MakeDotnetStartInfo(List<string> args, string? workingD
4444
}
4545

4646
// Configure the proxy settings, if applicable.
47-
if (this.proxy != null)
48-
{
49-
logger.LogDebug($"Configuring environment variables for the registry proxy at {this.proxy.Address}");
50-
51-
startInfo.EnvironmentVariables["HTTP_PROXY"] = this.proxy.Address;
52-
startInfo.EnvironmentVariables["HTTPS_PROXY"] = this.proxy.Address;
53-
startInfo.EnvironmentVariables["SSL_CERT_FILE"] = this.proxy.CertificatePath;
54-
}
47+
proxy?.SetProcessEnvironment(startInfo);
5548

5649
return startInfo;
5750
}

‎csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IRegistryProxy.cs‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
using System;
22
using System.Collections.Immutable;
3+
using System.Diagnostics;
34
using System.Security.Cryptography.X509Certificates;
45

56
namespace Semmle.Extraction.CSharp.DependencyFetching
@@ -30,5 +31,11 @@ public interface IRegistryProxy : IDisposable
3031
/// The certificate used for the registry proxy.
3132
/// </summary>
3233
X509Certificate2? Certificate { get; }
34+
35+
/// <summary>
36+
/// Configures the environment variables for a process to use the registry proxy.
37+
/// </summary>
38+
/// <param name="pi">The process start info to configure.</param>
39+
void SetProcessEnvironment(ProcessStartInfo pi);
3340
}
3441
}

‎csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ internal sealed partial class NugetPackageRestorer : IDisposable
1818
private readonly IFileProvider fileProvider;
1919
private readonly FileContent fileContent;
2020
private readonly IDotNet dotnet;
21+
private readonly IRegistryProxy? registryProxy;
2122
private readonly IDiagnosticsWriter diagnosticsWriter;
2223
private readonly DependencyDirectory legacyPackageDirectory;
2324
private readonly DependencyDirectory missingPackageDirectory;
@@ -40,6 +41,7 @@ public NugetPackageRestorer(
4041
this.fileProvider = fileProvider;
4142
this.fileContent = fileContent;
4243
this.dotnet = dotnet;
44+
this.registryProxy = registryProxy;
4345
this.diagnosticsWriter = diagnosticsWriter;
4446
this.logger = logger;
4547
this.compilationInfoContainer = compilationInfoContainer;
@@ -133,7 +135,7 @@ public HashSet<AssemblyLookupLocation> Restore()
133135

134136
try
135137
{
136-
var packagesConfigRestore = PackagesConfigRestoreFactory.Create(fileProvider, legacyPackageDirectory, logger, feedManager);
138+
var packagesConfigRestore = PackagesConfigRestoreFactory.Create(fileProvider, legacyPackageDirectory, logger, feedManager, registryProxy);
137139
var count = packagesConfigRestore.InstallPackages();
138140
if (packagesConfigRestore.PackageCount > 0)
139141
{

‎csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/PackagesConfigRestorer.cs‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -33,11 +33,11 @@ internal interface IPackagesConfigRestore
3333
/// </summary>
3434
internal class PackagesConfigRestoreFactory
3535
{
36-
public static IPackagesConfigRestore Create(IFileProvider fileProvider, DependencyDirectory packageDirectory, Semmle.Util.Logging.ILogger logger, FeedManager feedManager)
36+
public static IPackagesConfigRestore Create(IFileProvider fileProvider, DependencyDirectory packageDirectory, Semmle.Util.Logging.ILogger logger, FeedManager feedManager, IRegistryProxy? registryProxy)
3737
{
3838
if (SystemBuildActions.Instance.IsWindows() || SystemBuildActions.Instance.IsMonoInstalled())
3939
{
40-
return new NugetExeWrapper(fileProvider, packageDirectory, logger, feedManager);
40+
return new NugetExeWrapper(fileProvider, packageDirectory, logger, feedManager, registryProxy);
4141
}
4242

4343
return new NoOpPackagesConfig(fileProvider.PackagesConfigs, logger);
@@ -52,6 +52,7 @@ private class NugetExeWrapper : IPackagesConfigRestore
5252
{
5353
private readonly string? nugetExe;
5454
private readonly Semmle.Util.Logging.ILogger logger;
55+
private readonly IRegistryProxy? registryProxy;
5556

5657
public int PackageCount => fileProvider.PackagesConfigs.Count;
5758

@@ -70,12 +71,13 @@ private class NugetExeWrapper : IPackagesConfigRestore
7071
/// <summary>
7172
/// Create the package manager for a specified source tree.
7273
/// </summary>
73-
public NugetExeWrapper(IFileProvider fileProvider, DependencyDirectory packageDirectory, Semmle.Util.Logging.ILogger logger, FeedManager feedManager)
74+
public NugetExeWrapper(IFileProvider fileProvider, DependencyDirectory packageDirectory, Semmle.Util.Logging.ILogger logger, FeedManager feedManager, IRegistryProxy? registryProxy)
7475
{
7576
this.fileProvider = fileProvider;
7677
this.packageDirectory = packageDirectory;
7778
this.logger = logger;
7879
this.feedManager = feedManager;
80+
this.registryProxy = registryProxy;
7981

8082
if (fileProvider.PackagesConfigs.Count > 0)
8183
{
@@ -209,9 +211,13 @@ private bool TryRestoreNugetPackage(string packagesConfig)
209211
UseShellExecute = false
210212
};
211213

214+
// Configure the proxy settings, if applicable.
215+
registryProxy?.SetProcessEnvironment(pi);
216+
212217
var threadId = Environment.CurrentManagedThreadId;
213218
void onOut(string s) => logger.LogDebug(s, threadId);
214219
void onError(string s) => logger.LogError(s, threadId);
220+
logger.LogInfo($"Running '{pi.FileName} {string.Join(" ", pi.ArgumentList)}'");
215221
var exitCode = pi.ReadOutput(out _, onOut, onError);
216222
if (exitCode != 0)
217223
{

‎csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/RegistryProxy.cs‎

Lines changed: 24 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
using System;
22
using System.Collections.Immutable;
33
using System.Collections.Generic;
4+
using System.Diagnostics;
45
using System.IO;
56
using System.Security.Cryptography.X509Certificates;
67
using Semmle.Util;
@@ -36,6 +37,8 @@ public class RegistryConfig
3637

3738
public string Address { get; }
3839

40+
private readonly ILogger logger;
41+
3942
/// <summary>
4043
/// A dictionary mapping registry URLs to a boolean indicating whether they replace the base registry.
4144
/// </summary>
@@ -65,8 +68,9 @@ public class RegistryConfig
6568

6669
public X509Certificate2? Certificate { get; private set; }
6770

68-
private RegistryProxy(IRegistryProxyConfiguration config, ILogger logger, TemporaryDirectory tempWorkingDirectory)
71+
private RegistryProxy(IRegistryProxyConfiguration config, ILogger l, TemporaryDirectory tempWorkingDirectory)
6972
{
73+
logger = l;
7074
Address = $"http://{config.Host}:{config.Port}";
7175

7276
if (!string.IsNullOrWhiteSpace(config.Certificate))
@@ -179,6 +183,25 @@ private RegistryProxy(IRegistryProxyConfiguration config, ILogger logger, Tempor
179183
return result;
180184
}
181185

186+
public void SetProcessEnvironment(ProcessStartInfo pi)
187+
{
188+
logger.LogDebug($"Configuring environment variables for the registry proxy at {Address}");
189+
190+
pi.EnvironmentVariables["HTTP_PROXY"] = Address;
191+
pi.EnvironmentVariables["HTTPS_PROXY"] = Address;
192+
193+
// Also set the lower case variants of the environment variables
194+
// This might be needed on Linux systems.
195+
pi.EnvironmentVariables["http_proxy"] = Address;
196+
pi.EnvironmentVariables["https_proxy"] = Address;
197+
198+
if (CertificatePath != null)
199+
{
200+
logger.LogDebug("Setting the SSL certificate path for the registry proxy.");
201+
pi.EnvironmentVariables["SSL_CERT_FILE"] = CertificatePath;
202+
}
203+
}
204+
182205
public void Dispose()
183206
{
184207
Certificate?.Dispose();

‎csharp/extractor/Semmle.Extraction.Tests/FeedManager.cs‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
using System;
33
using System.Collections.Generic;
44
using System.Collections.Immutable;
5+
using System.Diagnostics;
56
using System.IO;
67
using System.Linq;
78
using System.Security.Cryptography.X509Certificates;
@@ -17,6 +18,8 @@ public class RegistryProxyStub : IRegistryProxy
1718
public string? CertificatePath { get; } = null;
1819
public X509Certificate2? Certificate { get; } = null;
1920

21+
public void SetProcessEnvironment(ProcessStartInfo pi) { }
22+
2023
public void Dispose() { }
2124
}
2225

@@ -28,6 +31,7 @@ public class RegistryProxyStubWithBaseUrls : IRegistryProxy
2831
public string? CertificatePath { get; } = null;
2932
public X509Certificate2? Certificate { get; } = null;
3033

34+
public void SetProcessEnvironment(ProcessStartInfo pi) { }
3135
public void Dispose() { }
3236
}
3337

0 commit comments

Comments
 (0)