From 3b2a9d61b087ae7682fe986f9beaea5ecbc96597 Mon Sep 17 00:00:00 2001 From: viogroza Date: Wed, 19 Aug 2026 16:45:54 +0300 Subject: [PATCH 1/4] Python: Fix venv isolation from user-site and base-install site-packages [STUD-81085] Python Scope's embedded interpreter never goes through CPython's own venv activation path (site.py's venv() function, which only triggers for a normally-launched /Scripts/python.exe), so pointing Path at a venv left both the PEP-370 user-site directory and the base install's own site-packages on sys.path unfiltered. A native package (e.g. pywin32) installed with a different build in either location could resolve its Python module from one and its native DLL dependency from the other, mismatched, copy, surfacing as "DLL load failed while importing win32api: The specified procedure could not be found." Fixes user-site and base-install leakage via PythonEngine.SetNoSiteFlag() (gated on whether the venv requests --system-site-packages), with Controller.ClearUserSiteEnvironmentOverride guarding against an ambient PYTHONNOUSERSITE leaking in from the host's own environment. Also fixes PythonHome being pointed at the venv root instead of its declared base install (breaks stdlib resolution entirely), tightens venv detection against false positives, and adds a version cross-check between a venv and the Python library actually being loaded. Co-Authored-By: Claude Sonnet 5 --- .../Client/Controller.cs | 15 + .../UiPath.Python.Tests/VenvDetectionTests.cs | 134 ++++++++ .../VenvUserSiteIsolationTests.cs | 313 ++++++++++++++++++ .../VenvVersionValidationTests.cs | 79 +++++ .../Python/UiPath.Python/EngineProvider.cs | 40 ++- .../Python/UiPath.Python/Impl/Engine.cs | 104 ++++-- .../UiPath.Python/Impl/OutOfProcessEngine.cs | 22 +- .../UiPath.Python/Impl/VenvDetection.cs | 84 +++++ .../UiPath.Python/Properties/AssemblyInfo.cs | 1 + .../Properties/UiPath.Python.Designer.cs | 9 + .../Properties/UiPath.Python.resx | 3 + 11 files changed, 781 insertions(+), 23 deletions(-) create mode 100644 Activities/Python/UiPath.Python.Tests/VenvDetectionTests.cs create mode 100644 Activities/Python/UiPath.Python.Tests/VenvUserSiteIsolationTests.cs create mode 100644 Activities/Python/UiPath.Python.Tests/VenvVersionValidationTests.cs create mode 100644 Activities/Python/UiPath.Python/Impl/VenvDetection.cs diff --git a/Activities/Python/Shared/UiPath.Shared.Service/Client/Controller.cs b/Activities/Python/Shared/UiPath.Shared.Service/Client/Controller.cs index e7510269e..a1a7d1a91 100644 --- a/Activities/Python/Shared/UiPath.Shared.Service/Client/Controller.cs +++ b/Activities/Python/Shared/UiPath.Shared.Service/Client/Controller.cs @@ -28,6 +28,19 @@ internal class Controller internal TimeSpan StartTimeout { get; set; } = Config.DefaultServiceCreationTimeout; + // Must be set in this (parent) process, before the host is spawned: environment variables + // changed from managed code already running inside the host, right before it initializes + // Python, are not reliably observed by the native interpreter there. Only variables present + // in the process's environment block at OS process-creation time are. + // + // ProcessStartInfo.EnvironmentVariables starts as a copy of this (parent) process's own + // environment, so if PYTHONNOUSERSITE already happens to be set there for unrelated reasons + // (e.g. a customer's own leftover manual workaround), it would otherwise flow straight + // through to the host untouched. When true, explicitly clears it for the host's own + // environment instead, so a venv's declared --system-site-packages behavior is decided by + // that flag alone, not by whatever's ambient on the machine (STUD-81085 follow-up). + internal bool ClearUserSiteEnvironmentOverride { get; set; } + internal HostWrapper Create() { StartHostService(); @@ -97,6 +110,8 @@ private ProcessStartInfo CreateProcessStartInfo(string hostFullPath, string fold // never reach the host until the buffer fills or the interpreter exits — and on // forced shutdown (Process.Kill) any buffered output is lost. psi.EnvironmentVariables["PYTHONUNBUFFERED"] = "1"; + if (ClearUserSiteEnvironmentOverride) + psi.EnvironmentVariables.Remove("PYTHONNOUSERSITE"); if (!isExeMode) psi.ArgumentList.Add(hostFullPath); return psi; diff --git a/Activities/Python/UiPath.Python.Tests/VenvDetectionTests.cs b/Activities/Python/UiPath.Python.Tests/VenvDetectionTests.cs new file mode 100644 index 000000000..63f76f615 --- /dev/null +++ b/Activities/Python/UiPath.Python.Tests/VenvDetectionTests.cs @@ -0,0 +1,134 @@ +using System; +using System.IO; +using UiPath.Python.Impl; +using UiPath.TestUtils; +using Xunit; +using Assert = Xunit.Assert; + +namespace UiPath.Python.Tests +{ + // Direct, fast tests for VenvDetection.GetVenvInfo — no Python engine involved. Covers the + // detection shapes discussed during the STUD-81085 review: venv root, its Scripts/bin + // launcher folder, and the false-positive an unbounded ancestor walk used to allow (an + // unrelated, fully-standalone installation merely sitting near someone else's pyvenv.cfg). + public class VenvDetectionTests : IDisposable + { + private const string Category = "Python"; + + private readonly string _rootDir; + + public VenvDetectionTests() + { + _rootDir = Directory.CreateDirectory(Path.Combine(Path.GetTempPath(), "venv-detection-tests", Guid.NewGuid().ToString("N"))).FullName; + } + + public void Dispose() + { + try { Directory.Delete(_rootDir, true); } catch { /* best effort cleanup */ } + } + + private string WriteVenvCfg(string venvDir, string home = @"C:\FakeBase", string extra = null) + { + Directory.CreateDirectory(venvDir); + var content = $"home = {home}{Environment.NewLine}version = 3.13.0{Environment.NewLine}{extra}"; + File.WriteAllText(Path.Combine(venvDir, "pyvenv.cfg"), content); + return venvDir; + } + + [Fact] + [Trait(TestCategories.Category, Category)] + public void VenvRoot_Is_Detected() + { + var venvDir = WriteVenvCfg(Path.Combine(_rootDir, "myvenv")); + + var venv = VenvDetection.GetVenvInfo(venvDir); + + Assert.NotNull(venv); + Assert.Equal(venvDir, venv.Root); + Assert.Equal(@"C:\FakeBase", venv.Home); + } + + [Theory] + [InlineData("Scripts")] + [InlineData("bin")] + [Trait(TestCategories.Category, Category)] + public void LauncherSubfolder_Is_Detected(string folderName) + { + var venvDir = WriteVenvCfg(Path.Combine(_rootDir, "myvenv")); + var launcherDir = Directory.CreateDirectory(Path.Combine(venvDir, folderName)).FullName; + + var venv = VenvDetection.GetVenvInfo(launcherDir); + + Assert.NotNull(venv); + Assert.Equal(venvDir, venv.Root); + } + + [Fact] + [Trait(TestCategories.Category, Category)] + public void UnrelatedFolder_OneLevelBelow_WrongName_Is_Not_Detected() + { + // pyvenv.cfg one level up, but the intermediate folder isn't a real venv launcher + // name — a fully standalone install could legitimately live here and must not be + // mistaken for being inside someone else's venv. + var venvDir = WriteVenvCfg(Path.Combine(_rootDir, "someones_venv")); + var standaloneDir = Directory.CreateDirectory(Path.Combine(venvDir, "runtime")).FullName; + + var venv = VenvDetection.GetVenvInfo(standaloneDir); + + Assert.Null(venv); + } + + [Fact] + [Trait(TestCategories.Category, Category)] + public void TwoLevelsUp_Is_Not_Detected() + { + // Even with the right launcher name one level further up, detection deliberately + // doesn't walk a second level — no real venv layout ever needs it. + var venvDir = WriteVenvCfg(Path.Combine(_rootDir, "myvenv")); + var scriptsDir = Directory.CreateDirectory(Path.Combine(venvDir, "Scripts")).FullName; + var nestedDir = Directory.CreateDirectory(Path.Combine(scriptsDir, "nested")).FullName; + + var venv = VenvDetection.GetVenvInfo(nestedDir); + + Assert.Null(venv); + } + + [Fact] + [Trait(TestCategories.Category, Category)] + public void StrayFile_Without_HomeOrVersion_Is_Not_Treated_As_Venv() + { + var dir = Directory.CreateDirectory(Path.Combine(_rootDir, "notavenv")).FullName; + File.WriteAllText(Path.Combine(dir, "pyvenv.cfg"), "some-unrelated-key = value" + Environment.NewLine); + + var venv = VenvDetection.GetVenvInfo(dir); + + Assert.Null(venv); + } + + [Fact] + [Trait(TestCategories.Category, Category)] + public void SystemSitePackages_Flag_Is_Captured() + { + var venvDir = WriteVenvCfg(Path.Combine(_rootDir, "myvenv"), extra: "include-system-site-packages = true" + Environment.NewLine); + + var venv = VenvDetection.GetVenvInfo(venvDir); + + Assert.NotNull(venv); + Assert.True(venv.IncludeSystemSitePackages); + Assert.False(venv.ShouldDisableUserSite); + } + + [Fact] + [Trait(TestCategories.Category, Category)] + public void Default_DoesNotIncludeSystemSitePackages_ShouldDisableUserSite() + { + var venvDir = WriteVenvCfg(Path.Combine(_rootDir, "myvenv")); + + var venv = VenvDetection.GetVenvInfo(venvDir); + + Assert.NotNull(venv); + Assert.False(venv.IncludeSystemSitePackages); + Assert.True(venv.ShouldDisableUserSite); + } + } +} diff --git a/Activities/Python/UiPath.Python.Tests/VenvUserSiteIsolationTests.cs b/Activities/Python/UiPath.Python.Tests/VenvUserSiteIsolationTests.cs new file mode 100644 index 000000000..38f05bfa1 --- /dev/null +++ b/Activities/Python/UiPath.Python.Tests/VenvUserSiteIsolationTests.cs @@ -0,0 +1,313 @@ +using System; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using UiPath.TestUtils; +using Xunit; +using Assert = Xunit.Assert; + +namespace UiPath.Python.Tests +{ + // Regression test for STUD-81085: pointing Python Scope's Path at a venv must stop the + // interpreter's PEP-370 user-site directory (%APPDATA%\Roaming\Python\PythonXY\site-packages) + // from being processed at all. pywin32 registers its native DLL search directory + // (os.add_dll_directory) from a .pth file in whichever site directory it's installed into; + // if the user-site copy still gets processed alongside the venv's own copy, a differently + // built pywin32 there collides with the venv's copy and surfaces as + // "DLL load failed while importing win32api: The specified procedure could not be found." + // + // This test doesn't need real pywin32 binaries: the bug is that the user-site .pth runs at + // all when a venv is configured, so a .pth marker is a faithful, hermetic reproduction of the + // exact mechanism pywin32 relies on. Runs out-of-process (the real production default) since + // the fix depends on an environment variable set in the parent process before the host spawns. + public class VenvUserSiteIsolationTests : IDisposable + { + private const string Category = "Python"; + + private static readonly string EmbeddedRuntimePath = EmbeddedPythonRuntimeBootstrap.EnsureRuntimePath(); + private static readonly string EmbeddedLibraryPath = EmbeddedPythonRuntimeBootstrap.GetPythonLibraryPath(EmbeddedRuntimePath); + + // Must match the running embeddable interpreter's sys.version_info (3.14.5) — Windows + // user-site resolves to \Python\site-packages. + private const string UserSiteVersionFolder = "Python314"; + + private readonly string _rootDir; + private readonly string _venvDir; + private readonly string _userBaseDir; + private readonly string _markerFile; + private readonly string _previousUserBase; + private readonly string _previousNoUserSite; + + public VenvUserSiteIsolationTests() + { + _rootDir = Directory.CreateDirectory(Path.Combine(Path.GetTempPath(), "venv-usersite-tests", Guid.NewGuid().ToString("N"))).FullName; + _venvDir = Path.Combine(_rootDir, ".venv"); + _userBaseDir = Path.Combine(_rootDir, "userbase"); + _markerFile = Path.Combine(_rootDir, "marker.txt"); + + _previousUserBase = Environment.GetEnvironmentVariable("PYTHONUSERBASE"); + _previousNoUserSite = Environment.GetEnvironmentVariable("PYTHONNOUSERSITE"); + } + + public void Dispose() + { + Environment.SetEnvironmentVariable("PYTHONUSERBASE", _previousUserBase); + Environment.SetEnvironmentVariable("PYTHONNOUSERSITE", _previousNoUserSite); + try { Directory.Delete(_rootDir, true); } catch { /* best effort cleanup */ } + } + + [Fact] + [Trait(TestCategories.Category, Category)] + public async Task Venv_Does_Not_Process_UserSite_PthFiles() + { + Skip.IfNot(Directory.Exists(EmbeddedRuntimePath)); + + var venvSitePackages = Directory.CreateDirectory(Path.Combine(_venvDir, "Lib", "site-packages")).FullName; + File.WriteAllText(Path.Combine(_venvDir, "pyvenv.cfg"), $"home = {EmbeddedRuntimePath}{Environment.NewLine}"); + WriteMarkerPth(venvSitePackages, "venv"); + + var userSitePackages = Directory.CreateDirectory(Path.Combine(_userBaseDir, UserSiteVersionFolder, "site-packages")).FullName; + WriteMarkerPth(userSitePackages, "usersite"); + Environment.SetEnvironmentVariable("PYTHONUSERBASE", _userBaseDir); + + var engine = EngineProvider.Get(Version.Python_310, _venvDir, EmbeddedLibraryPath, inProcess: false); + try + { + await engine.Initialize(null, CancellationToken.None, 60); + } + finally + { + await engine.Release(); + } + + var markerLines = File.Exists(_markerFile) + ? File.ReadAllLines(_markerFile) + : Array.Empty(); + + // The venv's own .pth must still run — this isn't about disabling site processing, + // only about excluding the unrelated per-user directory. + Assert.Contains("venv", markerLines); + Assert.DoesNotContain("usersite", markerLines); + } + + [Fact] + [Trait(TestCategories.Category, Category)] + public async Task Venv_With_SystemSitePackages_Still_Processes_UserSite_PthFiles() + { + // Mirrors CPython's own site.py venv() function: it only forces + // ENABLE_USER_SITE = False when the venv was created *without* + // --system-site-packages. A real, natively-activated --system-site-packages venv + // leaves user-site enabled (the normal, non-venv computation applies instead) — so + // this fix must not suppress it there either, or it would diverge from native parity + // for a case that already accepts the same DLL-collision exposure as any other + // non-venv interpreter. + Skip.IfNot(Directory.Exists(EmbeddedRuntimePath)); + + var venvSitePackages = Directory.CreateDirectory(Path.Combine(_venvDir, "Lib", "site-packages")).FullName; + File.WriteAllText(Path.Combine(_venvDir, "pyvenv.cfg"), + $"home = {EmbeddedRuntimePath}{Environment.NewLine}include-system-site-packages = true{Environment.NewLine}"); + WriteMarkerPth(venvSitePackages, "venv"); + + var userSitePackages = Directory.CreateDirectory(Path.Combine(_userBaseDir, UserSiteVersionFolder, "site-packages")).FullName; + WriteMarkerPth(userSitePackages, "usersite"); + Environment.SetEnvironmentVariable("PYTHONUSERBASE", _userBaseDir); + + var engine = EngineProvider.Get(Version.Python_310, _venvDir, EmbeddedLibraryPath, inProcess: false); + try + { + await engine.Initialize(null, CancellationToken.None, 60); + } + finally + { + await engine.Release(); + } + + var markerLines = File.Exists(_markerFile) + ? File.ReadAllLines(_markerFile) + : Array.Empty(); + + Assert.Contains("venv", markerLines); + Assert.Contains("usersite", markerLines); + } + + [Fact] + [Trait(TestCategories.Category, Category)] + public async Task Venv_PythonHome_Resolves_To_Declared_Base_Install() + { + // Regression test for a separate bug found while exercising this fix: pointing + // PythonHome directly at the venv root (rather than the base install its own + // pyvenv.cfg declares) makes native init fail outright once a real installer-based + // Python is used, since a venv has no standard library of its own. The embeddable test + // runtime's own ._pth-based bootstrap resolves its stdlib independently of PythonHome, + // so it can't catch that failure directly — but PythonHome still governs what the + // interpreter reports as sys.base_prefix regardless, which is what this asserts. + Skip.IfNot(Directory.Exists(EmbeddedRuntimePath)); + + Directory.CreateDirectory(Path.Combine(_venvDir, "Lib", "site-packages")); + File.WriteAllText(Path.Combine(_venvDir, "pyvenv.cfg"), $"home = {EmbeddedRuntimePath}{Environment.NewLine}"); + + var engine = EngineProvider.Get(Version.Python_310, _venvDir, EmbeddedLibraryPath, inProcess: false); + string basePrefix; + try + { + await engine.Initialize(null, CancellationToken.None, 60); + + var script = await engine.LoadScript( + "import sys\ndef check():\n return sys.base_prefix\n", + CancellationToken.None); + var result = await engine.InvokeMethod(script, "check", null, CancellationToken.None); + basePrefix = (string)engine.Convert(result, typeof(string)); + } + finally + { + await engine.Release(); + } + + Assert.Equal( + Path.GetFullPath(EmbeddedRuntimePath).TrimEnd(Path.DirectorySeparatorChar), + Path.GetFullPath(basePrefix).TrimEnd(Path.DirectorySeparatorChar), + ignoreCase: true); + } + + [Fact] + [Trait(TestCategories.Category, Category)] + public async Task Venv_Does_Not_Leak_BaseInstall_SitePackages() + { + // Regression test for a second, distinct leak path found by inspecting a real venv's + // actual sys.path: CPython's site.main() unconditionally adds the base install's own + // site-packages (and the bare base install prefix itself) unless something narrows + // PREFIXES first — which never happened for the embedded interpreter, since site.py's + // own venv() detection can't trigger for it. PYTHONNOUSERSITE never covered this; only + // SetNoSiteFlag (skipping site.main() entirely for this case) does. + Skip.IfNot(Directory.Exists(EmbeddedRuntimePath)); + + Directory.CreateDirectory(Path.Combine(_venvDir, "Lib", "site-packages")); + File.WriteAllText(Path.Combine(_venvDir, "pyvenv.cfg"), $"home = {EmbeddedRuntimePath}{Environment.NewLine}"); + + var sysPath = await GetSysPath(); + var normalizedBase = Path.GetFullPath(EmbeddedRuntimePath).TrimEnd(Path.DirectorySeparatorChar); + + Assert.DoesNotContain(sysPath, p => Path.GetFullPath(p).TrimEnd(Path.DirectorySeparatorChar) + .Equals(normalizedBase, StringComparison.OrdinalIgnoreCase)); + } + + [Fact] + [Trait(TestCategories.Category, Category)] + public async Task Venv_With_SystemSitePackages_Still_Includes_BaseInstall_SitePackages() + { + // The --system-site-packages case must keep behaving exactly as it did before this + // change: SetNoSiteFlag is only set for the default (ShouldDisableUserSite) case, so + // site.main() still runs normally here and still adds the base install unconditionally + // — which happens to already be correct for this specific flag. + Skip.IfNot(Directory.Exists(EmbeddedRuntimePath)); + + Directory.CreateDirectory(Path.Combine(_venvDir, "Lib", "site-packages")); + File.WriteAllText(Path.Combine(_venvDir, "pyvenv.cfg"), + $"home = {EmbeddedRuntimePath}{Environment.NewLine}include-system-site-packages = true{Environment.NewLine}"); + + var sysPath = await GetSysPath(); + var normalizedBase = Path.GetFullPath(EmbeddedRuntimePath).TrimEnd(Path.DirectorySeparatorChar); + + Assert.Contains(sysPath, p => Path.GetFullPath(p).TrimEnd(Path.DirectorySeparatorChar) + .Equals(normalizedBase, StringComparison.OrdinalIgnoreCase)); + } + + [Fact] + [Trait(TestCategories.Category, Category)] + public async Task Venv_SiteCustomize_Still_Runs() + { + // SetNoSiteFlag skips site.main() entirely for the default venv case, which would also + // silently skip sitecustomize.py auto-import (some environments rely on it for + // corporate setup) unless something restores it — Engine.PostInitializationVenvSetup + // explicitly calls site.execsitecustomize() for exactly this reason. + Skip.IfNot(Directory.Exists(EmbeddedRuntimePath)); + + var venvSitePackages = Directory.CreateDirectory(Path.Combine(_venvDir, "Lib", "site-packages")).FullName; + File.WriteAllText(Path.Combine(_venvDir, "pyvenv.cfg"), $"home = {EmbeddedRuntimePath}{Environment.NewLine}"); + + var markerPath = _markerFile.Replace("\\", "/"); + File.WriteAllText(Path.Combine(venvSitePackages, "sitecustomize.py"), + $"import codecs{Environment.NewLine}codecs.open('{markerPath}', 'a', encoding='utf-8').write('sitecustomize\\n'){Environment.NewLine}"); + + var engine = EngineProvider.Get(Version.Python_310, _venvDir, EmbeddedLibraryPath, inProcess: false); + try + { + await engine.Initialize(null, CancellationToken.None, 60); + } + finally + { + await engine.Release(); + } + + var markerLines = File.Exists(_markerFile) ? File.ReadAllLines(_markerFile) : Array.Empty(); + Assert.Contains("sitecustomize", markerLines); + } + + [Fact] + [Trait(TestCategories.Category, Category)] + public async Task Venv_With_SystemSitePackages_Ignores_Ambient_PYTHONNOUSERSITE() + { + // Regression test: a customer's own leftover PYTHONNOUSERSITE=1 (e.g. a manual + // workaround predating this fix, exactly like the one mentioned in the original + // ticket) sitting in the *ambient* environment used to leak straight into the spawned + // host via ProcessStartInfo.EnvironmentVariables (which starts as a copy of this + // process's own environment), silently defeating a --system-site-packages venv's + // intent to leave user-site enabled — regardless of what our own code did or didn't + // set. Controller.ClearUserSiteEnvironmentOverride exists specifically to guarantee + // this venv flag decides the outcome, not whatever's ambient on the machine. + Skip.IfNot(Directory.Exists(EmbeddedRuntimePath)); + + Environment.SetEnvironmentVariable("PYTHONNOUSERSITE", "1"); + + var venvSitePackages = Directory.CreateDirectory(Path.Combine(_venvDir, "Lib", "site-packages")).FullName; + File.WriteAllText(Path.Combine(_venvDir, "pyvenv.cfg"), + $"home = {EmbeddedRuntimePath}{Environment.NewLine}include-system-site-packages = true{Environment.NewLine}"); + WriteMarkerPth(venvSitePackages, "venv"); + + var userSitePackages = Directory.CreateDirectory(Path.Combine(_userBaseDir, UserSiteVersionFolder, "site-packages")).FullName; + WriteMarkerPth(userSitePackages, "usersite"); + Environment.SetEnvironmentVariable("PYTHONUSERBASE", _userBaseDir); + + var engine = EngineProvider.Get(Version.Python_310, _venvDir, EmbeddedLibraryPath, inProcess: false); + try + { + await engine.Initialize(null, CancellationToken.None, 60); + } + finally + { + await engine.Release(); + } + + var markerLines = File.Exists(_markerFile) ? File.ReadAllLines(_markerFile) : Array.Empty(); + + Assert.Contains("venv", markerLines); + Assert.Contains("usersite", markerLines); + } + + private async Task GetSysPath() + { + var engine = EngineProvider.Get(Version.Python_310, _venvDir, EmbeddedLibraryPath, inProcess: false); + try + { + await engine.Initialize(null, CancellationToken.None, 60); + + var script = await engine.LoadScript( + "import sys\ndef check():\n return list(sys.path)\n", + CancellationToken.None); + var result = await engine.InvokeMethod(script, "check", null, CancellationToken.None); + return (string[])engine.Convert(result, typeof(string[])); + } + finally + { + await engine.Release(); + } + } + + private void WriteMarkerPth(string siteDir, string tag) + { + var markerPath = _markerFile.Replace("\\", "/"); + var pthLine = $"import codecs; codecs.open('{markerPath}', 'a', encoding='utf-8').write('{tag}\\n')"; + File.WriteAllText(Path.Combine(siteDir, $"zzz_{tag}_marker.pth"), pthLine + Environment.NewLine); + } + } +} diff --git a/Activities/Python/UiPath.Python.Tests/VenvVersionValidationTests.cs b/Activities/Python/UiPath.Python.Tests/VenvVersionValidationTests.cs new file mode 100644 index 000000000..d0d9e89be --- /dev/null +++ b/Activities/Python/UiPath.Python.Tests/VenvVersionValidationTests.cs @@ -0,0 +1,79 @@ +using System; +using System.IO; +using UiPath.TestUtils; +using Xunit; +using Assert = Xunit.Assert; + +namespace UiPath.Python.Tests +{ + // Regression tests for EngineProvider.ValidateVenvVersion (called from the public + // ValidateInstallation): a venv's declared Python version (its own pyvenv.cfg) must match the + // version actually loaded from LibraryPath, or initialization would otherwise proceed with a + // mismatched interpreter/site-packages pairing and fail later with a confusing error instead + // of a clear one up front. + public class VenvVersionValidationTests : IDisposable + { + private const string Category = "Python"; + + private static readonly string EmbeddedRuntimePath = EmbeddedPythonRuntimeBootstrap.EnsureRuntimePath(); + private static readonly string EmbeddedLibraryPath = EmbeddedPythonRuntimeBootstrap.GetPythonLibraryPath(EmbeddedRuntimePath); + + private readonly string _rootDir; + private readonly string _venvDir; + + public VenvVersionValidationTests() + { + _rootDir = Directory.CreateDirectory(Path.Combine(Path.GetTempPath(), "venv-version-tests", Guid.NewGuid().ToString("N"))).FullName; + _venvDir = Path.Combine(_rootDir, ".venv"); + } + + public void Dispose() + { + try { Directory.Delete(_rootDir, true); } catch { /* best effort cleanup */ } + } + + private void WriteVenvCfg(string version) + { + Directory.CreateDirectory(_venvDir); + File.WriteAllText(Path.Combine(_venvDir, "pyvenv.cfg"), + $"home = {EmbeddedRuntimePath}{Environment.NewLine}version = {version}{Environment.NewLine}"); + } + + [Fact] + [Trait(TestCategories.Category, Category)] + public void MismatchedVenvVersion_Throws() + { + Skip.IfNot(Directory.Exists(EmbeddedRuntimePath)); + + // The embeddable runtime bootstrap is 3.14.5 — declare something else entirely. + WriteVenvCfg("3.9.0"); + + Assert.Throws(() => EngineProvider.ValidateInstallation(_venvDir, EmbeddedLibraryPath)); + } + + [Fact] + [Trait(TestCategories.Category, Category)] + public void MatchingVenvVersion_DoesNotThrow() + { + Skip.IfNot(Directory.Exists(EmbeddedRuntimePath)); + + WriteVenvCfg("3.14.5"); + + // Must not throw. + EngineProvider.ValidateInstallation(_venvDir, EmbeddedLibraryPath); + } + + [Fact] + [Trait(TestCategories.Category, Category)] + public void NonVenvPath_DoesNotThrow() + { + Skip.IfNot(Directory.Exists(EmbeddedRuntimePath)); + + // _venvDir has no pyvenv.cfg at all here — not a venv, so the version cross-check + // must not even engage. + Directory.CreateDirectory(_venvDir); + + EngineProvider.ValidateInstallation(_venvDir, EmbeddedLibraryPath); + } + } +} diff --git a/Activities/Python/UiPath.Python/EngineProvider.cs b/Activities/Python/UiPath.Python/EngineProvider.cs index 7a0e5e58d..20d47fc49 100644 --- a/Activities/Python/UiPath.Python/EngineProvider.cs +++ b/Activities/Python/UiPath.Python/EngineProvider.cs @@ -142,13 +142,51 @@ private static bool Autodetect(string pythonFullPath, out Version version, out E /// executable) and its dynamic library (, via binary inspection). /// Either input may be absent or invalid independently; each check no-ops when its input is /// missing and throws / - /// when it finds a 32-bit or unsupported runtime. + /// when it finds a 32-bit or unsupported runtime. Also cross-checks a venv at + /// against the version actually loaded from (see + /// ). /// public static void ValidateInstallation(string path, string libraryPath) { // Library first: it is the exact artifact pythonnet loads and the check is cheap (no spawn). ValidatePythonLibrary(libraryPath); ValidatePythonExecutable(path); + ValidateVenvVersion(path, libraryPath); + } + + /// + /// If is a venv, cross-checks the Python version it was created with + /// (from its pyvenv.cfg) against the version actually loaded from . + /// A mismatch means the venv's site-packages — compiled for a different ABI — would end up on + /// sys.path for a differently-versioned interpreter. Surfaced here, before any native + /// initialization, rather than as a confusing failure deep inside the engine (e.g. a missing + /// _sysconfigdata module, or subtly wrong stdlib behavior). No-ops when either version can't be + /// determined — engine initialization will surface its own error in that case. + /// + private static void ValidateVenvVersion(string path, string libraryPath) + { + var venv = VenvDetection.GetVenvInfo(path); + if (venv?.Version == null || !TryParseVenvVersion(venv.Version, out int venvMajor, out int venvMinor)) + return; + + if (!TryGetLibraryVersion(libraryPath, out int libMajor, out int libMinor)) + return; + + if (venvMajor != libMajor || venvMinor != libMinor) + throw new NotSupportedException( + string.Format(Resources.PythonVenvVersionMismatchException, + venv.Root, $"{venvMajor}.{venvMinor}", $"{libMajor}.{libMinor}")); + } + + /// + /// Parses the "major.minor(.patch)" version string pyvenv.cfg's version key always carries. + /// + private static bool TryParseVenvVersion(string version, out int major, out int minor) + { + major = 0; + minor = 0; + var parts = version.Split('.'); + return parts.Length >= 2 && int.TryParse(parts[0], out major) && int.TryParse(parts[1], out minor); } /// diff --git a/Activities/Python/UiPath.Python/Impl/Engine.cs b/Activities/Python/UiPath.Python/Impl/Engine.cs index b87152941..9ab74d9c1 100644 --- a/Activities/Python/UiPath.Python/Impl/Engine.cs +++ b/Activities/Python/UiPath.Python/Impl/Engine.cs @@ -55,20 +55,71 @@ public async Task Initialize(string workingFolder, CancellationToken ct, double Trace.TraceInformation($"Initializing Python runtime using version {_version} and path {_path}"); Stopwatch sw = Stopwatch.StartNew(); + // Detected before Initialize(): a real venv (activated normally, e.g. + // \Scripts\python.exe) always disables the PEP-370 user-site + // directory on its own. Our embedded interpreter never goes through that + // activation path, so nothing does this for us — without it, a native + // package installed in both the venv and the user-site directory (e.g. + // pywin32) can resolve its Python module from one and its native DLL + // dependency from the other, mismatched, copy (STUD-81085). Suppression + // itself happens below, via PythonEngine.SetNoSiteFlag() — see that call + // for why (it also closes a second, related leak path, and is immune to + // whatever PYTHONNOUSERSITE happens to already be set in the ambient + // environment, which an env-var-based approach was not). + var venv = _version == Version.Python_310 ? VenvDetection.GetVenvInfo(_path) : null; + if (_isWindows && !_path.IsNullOrEmpty()) SetDllDirectory(Path.GetFullPath(_path)); if (!_libraryPath.IsNullOrEmpty()) Runtime.PythonDLL = _libraryPath; - if (!_path.IsNullOrEmpty()) - PythonEngine.PythonHome = _path; + // A venv's own folder is not a valid PythonHome: it only has + // Lib\site-packages, not the standard library (Lib\encodings etc.), so + // pointing the native interpreter at it fails at the very first import + // with "Fatal Python error: Failed to import encodings module". Use the + // base install recorded in the venv's own pyvenv.cfg instead — exactly + // what a normally-activated venv resolves to on its own. EngineProvider + // already validated that base install's version matches _libraryPath's. + var pythonHome = venv != null ? ResolvePythonHome(venv.Home) : _path; + if (!pythonHome.IsNullOrEmpty()) + PythonEngine.PythonHome = pythonHome; + + // For a default venv (no --system-site-packages), suppresses both the + // PEP-370 user-site leak (STUD-81085) and a second, distinct leak path + // found by inspecting a real venv's actual sys.path: CPython's own + // site.main() also unconditionally adds the *base install's* own + // site-packages (site.addsitepackages() against sys.prefix/exec_prefix, + // still pointing at the base install at this point). PYTHONNOUSERSITE + // would only ever have covered the first of these — SetNoSiteFlag + // (Py_NoSiteFlag) disables site.main()'s automatic run entirely, so + // *neither* ever gets added in the first place: "prevent, don't clean up + // after" — a cleanup-after-the-fact fix couldn't undo any .pth-triggered + // side effects, e.g. os.add_dll_directory calls, that already ran by the + // time managed code regains control. It's also an in-memory flag on this + // process's loaded Python DLL, never written to os.environ — unlike an + // env-var-based approach, it can't be defeated by (or leak into) whatever + // PYTHONNOUSERSITE the ambient environment happens to already carry, which + // is exactly the failure mode found in Controller.cs's + // ClearUserSiteEnvironmentOverride for the --system-site-packages case. + // `site` itself is still importable — + // this only skips its automatic invocation at startup — so the explicit + // site.addsitedir() call in PostInitializationVenvSetup for the venv's own + // site-packages, and the site.execsitecustomize() call there preserving + // sitecustomize.py support, both keep working. Must come after + // Runtime.PythonDLL/PythonHome are set, not before — calling it earlier + // left Runtime.PythonDLL null by the time the host tried to use it (and, + // per a known pythonnet issue, SetNoSiteFlag itself can be silently + // ignored on Windows unless another PythonEngine call already preceded + // it — PythonHome, set just above, already satisfies that). + if (venv != null && venv.ShouldDisableUserSite) + PythonEngine.SetNoSiteFlag(); PythonEngine.Initialize(); ct.ThrowIfCancellationRequested(); - PostInitializationVenvSetup(); + PostInitializationVenvSetup(venv); PythonEngine.BeginAllowThreads(); sw.Stop(); @@ -266,16 +317,23 @@ private string GetInitializationScript() return reader.ReadToEnd(); } - private static bool IsVenv(string path) => File.Exists(Path.Combine(path, "pyvenv.cfg")); - - private static string GetVenvPath(string venvPath, int maxLevels = 3) + /// + /// Normalizes a venv's pyvenv.cfg "home" value into a PythonHome-compatible prefix. On + /// Windows "home" already is the install root (no adjustment needed). On POSIX it records + /// the base install's bin folder (e.g. "/usr/bin"), one level below the prefix PythonHome + /// actually expects (e.g. "/usr") — strip it when present. + /// + private static string ResolvePythonHome(string venvHome) { - if (string.IsNullOrEmpty(venvPath) || maxLevels == 0) + if (venvHome.IsNullOrEmpty()) return null; - else if (IsVenv(venvPath)) - return venvPath; - else - return GetVenvPath(Path.GetDirectoryName(venvPath), maxLevels - 1); + + var trimmed = venvHome.TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar); + var lastSegment = Path.GetFileName(trimmed); + if (string.Equals(lastSegment, "bin", StringComparison.Ordinal)) + return Path.GetDirectoryName(trimmed); + + return venvHome; } private static string GetEnvSitePackagesPath(string venvPath) @@ -296,29 +354,35 @@ private static string GetEnvSitePackagesPath(string venvPath) return sitePackages; } - private void PostInitializationVenvSetup() + private void PostInitializationVenvSetup(VenvDetection.VenvInfo venv) { - if (_version != Version.Python_310) - return; - - var venvPath = GetVenvPath(_path); - if (!string.IsNullOrWhiteSpace(venvPath)) + if (venv != null) { using (Py.GIL()) { dynamic sys = Py.Import("sys"); dynamic site = Py.Import("site"); - sys.prefix = venvPath; - sys.exec_prefix = venvPath; + sys.prefix = venv.Root; + sys.exec_prefix = venv.Root; - var sitePackagesPath = GetEnvSitePackagesPath(venvPath); + var sitePackagesPath = GetEnvSitePackagesPath(venv.Root); site.addsitedir(sitePackagesPath); if ((bool)sys.path.__contains__(sitePackagesPath)) sys.path.remove(sitePackagesPath); sys.path.insert(0, sitePackagesPath); + + // SetNoSiteFlag (see Initialize()) skips site.main() entirely for a default + // venv, which also skips its sitecustomize.py auto-import — some environments + // rely on that for corporate setup (proxies, logging, etc.), and it did run + // today before this change, so preserve it explicitly. Safe to call even when + // SetNoSiteFlag wasn't set (--system-site-packages venvs): site.main() already + // ran it there, and re-importing an already-imported module is a no-op. + // Deliberately not calling execusercustomize() — its user-site counterpart, + // consistent with suppressing user-site itself. + site.execsitecustomize(); } } } diff --git a/Activities/Python/UiPath.Python/Impl/OutOfProcessEngine.cs b/Activities/Python/UiPath.Python/Impl/OutOfProcessEngine.cs index 06bfe2612..987416847 100644 --- a/Activities/Python/UiPath.Python/Impl/OutOfProcessEngine.cs +++ b/Activities/Python/UiPath.Python/Impl/OutOfProcessEngine.cs @@ -50,11 +50,29 @@ public Task Initialize(string workingFolder, CancellationToken ct, double timeou Stopwatch sw = Stopwatch.StartNew(); - // TODO: expose visible as a property? + var venv = _version == Version.Python_310 ? VenvDetection.GetVenvInfo(_path) : null; + + // Actual user-site suppression for the default (ShouldDisableUserSite) case happens + // inside Engine.Initialize() itself, via PythonEngine.SetNoSiteFlag() — that runs in + // this same host process regardless, so no parent-side plumbing is needed for it + // (an env-var-based attempt used to live here; it turned out to be both unreliable + // when set this late from managed code in an already-spawned process, and defeatable + // by whatever PYTHONNOUSERSITE the ambient environment already carried). + // + // What *does* still need to happen here, in the parent, before the host spawns: for a + // --system-site-packages venv (ShouldDisableUserSite == false), the intent is to leave + // user-site exactly as a normal, non-embedded interpreter would — but ProcessStartInfo + // starts as a copy of this process's own environment, so if PYTHONNOUSERSITE already + // happens to be set there (e.g. a customer's own leftover workaround, unrelated to this + // fix), it would otherwise leak into the host and silently force user-site off anyway, + // regardless of what SetNoSiteFlag does or doesn't do for the other case. Clearing it + // explicitly for the child guarantees the venv's own IncludeSystemSitePackages flag is + // what decides this, not whatever's ambient on the machine. _provider = new Controller() { PythonHostLibFile = ServiceDll_x64, - Visible = _visible + Visible = _visible, + ClearUserSiteEnvironmentOverride = venv != null && venv.IncludeSystemSitePackages }; // Set LogTrace before Create() so the diagnostic file (if enabled) captures diff --git a/Activities/Python/UiPath.Python/Impl/VenvDetection.cs b/Activities/Python/UiPath.Python/Impl/VenvDetection.cs new file mode 100644 index 000000000..9431d0e67 --- /dev/null +++ b/Activities/Python/UiPath.Python/Impl/VenvDetection.cs @@ -0,0 +1,84 @@ +using System; +using System.Collections.Generic; +using System.IO; + +namespace UiPath.Python.Impl +{ + /// + /// Shared by (runs inside the process that calls Py_Initialize) and + /// (runs in the calling process, before the host is spawned). + /// + internal static class VenvDetection + { + internal sealed record VenvInfo(string Root, string Home, bool IncludeSystemSitePackages, string Version) + { + /// + /// Mirrors CPython's own site.py venv() function: it only forces + /// ENABLE_USER_SITE = False for a venv created without --system-site-packages. A + /// --system-site-packages venv leaves it to the normal (non-venv) computation, which is + /// True in the typical case — so a real, natively-activated venv like that does *not* + /// disable user-site. + /// + internal bool ShouldDisableUserSite => !IncludeSystemSitePackages; + } + + // Real venvs put their launcher/executable folder directly under the venv root, named + // exactly this — the same names EngineProvider looks for python.exe/python3 under. + private static readonly string[] VenvBinFolderNames = ["Scripts", "bin"]; + + /// + /// Parses pyvenv.cfg at , if present. Requiring at least one of the + /// keys a real venv config always has (home/version) avoids treating an unrelated file that + /// merely happens to be named pyvenv.cfg as a venv. + /// + private static VenvInfo TryReadVenvConfig(string path) + { + var cfgFile = Path.Combine(path, "pyvenv.cfg"); + if (!File.Exists(cfgFile)) + return null; + + var kv = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var line in File.ReadLines(cfgFile)) + { + var parts = line.Split('=', 2); + if (parts.Length == 2) + kv[parts[0].Trim()] = parts[1].Trim(); + } + + if (!kv.ContainsKey("home") && !kv.ContainsKey("version")) + return null; + + kv.TryGetValue("home", out var home); + kv.TryGetValue("version", out var version); + var includeSystemSitePackages = kv.TryGetValue("include-system-site-packages", out var include) + && string.Equals(include, "true", StringComparison.OrdinalIgnoreCase); + + return new VenvInfo(path, home, includeSystemSitePackages, version); + } + + /// + /// Detects whether is a venv root, or its immediate Scripts/bin + /// launcher folder — the only two layouts a real venv actually produces. Deliberately does + /// not walk further up than that, and only accepts the one-level-up case when the + /// intermediate folder is actually named Scripts/bin: a bare "is there a pyvenv.cfg within + /// N ancestor levels" search (the original implementation) can mistake an unrelated, + /// fully-standalone Python installation that merely happens to sit a level or two beneath + /// someone else's venv for being inside it. + /// + internal static VenvInfo GetVenvInfo(string path) + { + if (string.IsNullOrEmpty(path)) + return null; + + var direct = TryReadVenvConfig(path); + if (direct != null) + return direct; + + var folderName = Path.GetFileName(path.TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar)); + if (!Array.Exists(VenvBinFolderNames, name => string.Equals(name, folderName, StringComparison.OrdinalIgnoreCase))) + return null; + + return TryReadVenvConfig(Path.GetDirectoryName(path)); + } + } +} diff --git a/Activities/Python/UiPath.Python/Properties/AssemblyInfo.cs b/Activities/Python/UiPath.Python/Properties/AssemblyInfo.cs index 8798b2300..353866e2a 100644 --- a/Activities/Python/UiPath.Python/Properties/AssemblyInfo.cs +++ b/Activities/Python/UiPath.Python/Properties/AssemblyInfo.cs @@ -4,6 +4,7 @@ [assembly: XmlnsDefinition("http://schemas.uipath.com/workflow/activities/python", "UiPath.Python")] [assembly: InternalsVisibleTo("UiPath.Python.Activities.API.Tests")] +[assembly: InternalsVisibleTo("UiPath.Python.Tests")] // Setting ComVisible to false makes the types in this assembly not visible // to COM components. If you need to access a type in this assembly from diff --git a/Activities/Python/UiPath.Python/Properties/UiPath.Python.Designer.cs b/Activities/Python/UiPath.Python/Properties/UiPath.Python.Designer.cs index 93e778a99..1216a6083 100644 --- a/Activities/Python/UiPath.Python/Properties/UiPath.Python.Designer.cs +++ b/Activities/Python/UiPath.Python/Properties/UiPath.Python.Designer.cs @@ -212,5 +212,14 @@ public static string PythonVersionNotSupportedException { return ResourceManager.GetString("PythonVersionNotSupportedException", resourceCulture); } } + + /// + /// Looks up a localized string similar to The virtual environment at '{0}' was created with Python {1}, but LibraryPath points to Python {2}. Point LibraryPath to a Python {1} installation that matches the virtual environment. + /// + public static string PythonVenvVersionMismatchException { + get { + return ResourceManager.GetString("PythonVenvVersionMismatchException", resourceCulture); + } + } } } diff --git a/Activities/Python/UiPath.Python/Properties/UiPath.Python.resx b/Activities/Python/UiPath.Python/Properties/UiPath.Python.resx index d460bbd3a..2d2bced01 100644 --- a/Activities/Python/UiPath.Python/Properties/UiPath.Python.resx +++ b/Activities/Python/UiPath.Python/Properties/UiPath.Python.resx @@ -165,6 +165,9 @@ Python {0} is not supported. Supported versions are: {1}. + + The virtual environment at '{0}' was created with Python {1}, but LibraryPath points to Python {2}. Point LibraryPath to a Python {1} installation that matches the virtual environment. + The Python script data size ({0} MB) exceeds the configured limit ({1} MB). Pass large data via a file path instead of as a method argument, or increase the Script Data Size Limit property. From 359f5fc3e2544dd2c179ac2d8ecd64837a0efe0a Mon Sep 17 00:00:00 2001 From: viogroza Date: Tue, 25 Aug 2026 17:59:26 +0300 Subject: [PATCH 2/4] Python: address STUD-81085 venv fix review comments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes findings from PR #596 review: - PythonHome falls back to LibraryPath's directory when the venv's declared home (pyvenv.cfg) doesn't actually carry a stdlib (e.g. a Microsoft Store Python's app-execution-alias folder). - VenvDetection.GetVenvInfo no longer misses the venv root for a Path ending in a separator (Path.GetDirectoryName on a separator-terminated path only strips the separator, it doesn't walk up). - An unreadable pyvenv.cfg (locked ACLs, AV, concurrent pip rewrite) now returns null instead of throwing out of the public ValidateInstallation. - pyvenv.cfg's "version_info" (virtualenv/uv) is read as a fallback for "version" (stdlib venv), so the venv/library version cross-check isn't silently skipped for venvs created by those tools. - sitecustomize.py/usercustomize.py precedence fixed for --system-site-packages venvs: the venv's own copy now actually gets a chance to run instead of being permanently shadowed by whatever site.main() already cached before the venv's site-packages was added. - Removed Controller.ClearUserSiteEnvironmentOverride entirely: an ambient PYTHONNOUSERSITE is now honored for --system-site-packages venvs exactly like a natively-activated interpreter would, instead of being silently stripped. - Extracted Engine.ConfigureRuntime out of Initialize (addresses the Sonar S3776 cognitive-complexity finding, the only genuinely PR-caused issue behind the failing quality gate — the rest is pre-existing debt in touched files, confirmed via diff, left alone). - Made PostInitializationVenvSetup static and added GC.SuppressFinalize to three tests' Dispose() (CA1816/CA1822/S2325 — all genuinely new, cheap, zero-behavior-change). - Added regression tests for all of the above, plus direct tests for the Windows "Lib" vs POSIX "lib" case-sensitive path segments, each verified to actually fail without its corresponding fix. Co-Authored-By: Claude Sonnet 5 --- .../Client/Controller.cs | 15 -- .../EmbeddedPythonRuntimeBootstrap.cs | 16 +- .../EngineStdlibDetectionTests.cs | 107 ++++++++++ .../UiPath.Python.Tests/VenvDetectionTests.cs | 54 ++++- .../VenvUserSiteIsolationTests.cs | 76 +++++-- .../VenvVersionValidationTests.cs | 23 +- .../Python/UiPath.Python/Impl/Engine.cs | 198 ++++++++++++------ .../UiPath.Python/Impl/OutOfProcessEngine.cs | 29 +-- .../UiPath.Python/Impl/VenvDetection.cs | 47 ++++- 9 files changed, 432 insertions(+), 133 deletions(-) create mode 100644 Activities/Python/UiPath.Python.Tests/EngineStdlibDetectionTests.cs diff --git a/Activities/Python/Shared/UiPath.Shared.Service/Client/Controller.cs b/Activities/Python/Shared/UiPath.Shared.Service/Client/Controller.cs index a1a7d1a91..e7510269e 100644 --- a/Activities/Python/Shared/UiPath.Shared.Service/Client/Controller.cs +++ b/Activities/Python/Shared/UiPath.Shared.Service/Client/Controller.cs @@ -28,19 +28,6 @@ internal class Controller internal TimeSpan StartTimeout { get; set; } = Config.DefaultServiceCreationTimeout; - // Must be set in this (parent) process, before the host is spawned: environment variables - // changed from managed code already running inside the host, right before it initializes - // Python, are not reliably observed by the native interpreter there. Only variables present - // in the process's environment block at OS process-creation time are. - // - // ProcessStartInfo.EnvironmentVariables starts as a copy of this (parent) process's own - // environment, so if PYTHONNOUSERSITE already happens to be set there for unrelated reasons - // (e.g. a customer's own leftover manual workaround), it would otherwise flow straight - // through to the host untouched. When true, explicitly clears it for the host's own - // environment instead, so a venv's declared --system-site-packages behavior is decided by - // that flag alone, not by whatever's ambient on the machine (STUD-81085 follow-up). - internal bool ClearUserSiteEnvironmentOverride { get; set; } - internal HostWrapper Create() { StartHostService(); @@ -110,8 +97,6 @@ private ProcessStartInfo CreateProcessStartInfo(string hostFullPath, string fold // never reach the host until the buffer fills or the interpreter exits — and on // forced shutdown (Process.Kill) any buffered output is lost. psi.EnvironmentVariables["PYTHONUNBUFFERED"] = "1"; - if (ClearUserSiteEnvironmentOverride) - psi.EnvironmentVariables.Remove("PYTHONNOUSERSITE"); if (!isExeMode) psi.ArgumentList.Add(hostFullPath); return psi; diff --git a/Activities/Python/UiPath.Python.Tests/EmbeddedPythonRuntimeBootstrap.cs b/Activities/Python/UiPath.Python.Tests/EmbeddedPythonRuntimeBootstrap.cs index 4071c4f21..d2596ecb6 100644 --- a/Activities/Python/UiPath.Python.Tests/EmbeddedPythonRuntimeBootstrap.cs +++ b/Activities/Python/UiPath.Python.Tests/EmbeddedPythonRuntimeBootstrap.cs @@ -10,7 +10,7 @@ namespace UiPath.Python.Tests public static class EmbeddedPythonRuntimeBootstrap { private const string EmbeddedZipFileName = "python-3.14.5-embed-amd64.zip"; - private const string PythonVersion = "3.14.5"; + public const string PythonVersion = "3.14.5"; private static readonly string RuntimeRoot = Path.Combine(Path.GetTempPath(), "pythons", PythonVersion); private static readonly string LockFile = Path.Combine(RuntimeRoot, ".setup.lock"); @@ -62,6 +62,20 @@ public static string EnsureRuntimePath() return RuntimeRoot; } + /// + /// The folder name Python's own per-user site (e.g. %APPDATA%\Roaming\Python\PythonXY on + /// Windows) resolves to for this embedded runtime's version, derived from + /// so it can't drift out of sync when that's bumped. + /// + public static string UserSiteVersionFolder + { + get + { + var parts = PythonVersion.Split('.'); + return $"Python{parts[0]}{parts[1]}"; + } + } + public static string GetPythonLibraryPath(string runtimePath) { ArgumentNullException.ThrowIfNull(runtimePath); diff --git a/Activities/Python/UiPath.Python.Tests/EngineStdlibDetectionTests.cs b/Activities/Python/UiPath.Python.Tests/EngineStdlibDetectionTests.cs new file mode 100644 index 000000000..6105358c0 --- /dev/null +++ b/Activities/Python/UiPath.Python.Tests/EngineStdlibDetectionTests.cs @@ -0,0 +1,107 @@ +using System; +using System.IO; +using UiPath.Python.Impl; +using UiPath.TestUtils; +using Xunit; +using Assert = Xunit.Assert; + +namespace UiPath.Python.Tests +{ + // Direct, fast tests for Engine's stdlib-detection helpers (HasStdlib and friends) — no + // Python engine involved. "Lib" (Windows) vs "lib" (POSIX) is the real, fixed folder name + // each platform's CPython build/installer produces, not a style choice — WindowsStdlibLandmark + // and PosixStdlibDirectory assert on the exact, case-sensitive string each platform's check is + // built from, since a real Directory.Exists call on this repo's Windows-only CI can't tell + // "Lib" apart from "lib" (NTFS resolves both to the same directory by default), so filesystem + // behavior alone can't prove the check asks for the right casing on either platform. + public class EngineStdlibDetectionTests : IDisposable + { + private const string Category = "Python"; + + private readonly string _rootDir; + + public EngineStdlibDetectionTests() + { + _rootDir = Directory.CreateDirectory(Path.Combine(Path.GetTempPath(), "engine-stdlib-tests", Guid.NewGuid().ToString("N"))).FullName; + } + + public void Dispose() + { + try { Directory.Delete(_rootDir, true); } catch { /* best effort cleanup */ } + GC.SuppressFinalize(this); + } + + [Fact] + [Trait(TestCategories.Category, Category)] + public void WindowsStdlibLandmark_Uses_CapitalL_Lib_Segment() + { + var landmark = Engine.WindowsStdlibLandmark(_rootDir); + + Assert.Equal(Path.Combine(_rootDir, "Lib", "encodings"), landmark, StringComparer.Ordinal); + } + + [Fact] + [Trait(TestCategories.Category, Category)] + public void PosixStdlibDirectory_Uses_Lowercase_Lib_Segment() + { + var libDir = Engine.PosixStdlibDirectory(_rootDir); + + Assert.Equal(Path.Combine(_rootDir, "lib"), libDir, StringComparer.Ordinal); + } + + [Fact] + [Trait(TestCategories.Category, Category)] + public void WindowsAndPosix_Segments_Differ_In_Case() + { + // Pins the two platforms' checks to genuinely different, non-interchangeable strings + // — guards against a future edit accidentally making both branches check the same + // (either) casing, which real Directory.Exists calls on this Windows-only CI would + // never catch on their own (NTFS folds the case difference away). + var windowsSegment = Path.GetFileName(Path.GetDirectoryName(Engine.WindowsStdlibLandmark(_rootDir))); + var posixSegment = Path.GetFileName(Engine.PosixStdlibDirectory(_rootDir)); + + Assert.Equal("Lib", windowsSegment, StringComparer.Ordinal); + Assert.Equal("lib", posixSegment, StringComparer.Ordinal); + Assert.NotEqual(windowsSegment, posixSegment, StringComparer.Ordinal); + } + + [Fact] + [Trait(TestCategories.Category, Category)] + public void HasStdlib_True_When_Landmark_Present() + { + // Runs on this repo's actual CI/dev OS (Windows), exercising the real branch HasStdlib + // takes here — confirms the happy path independent of the case-sensitivity question + // above. + Directory.CreateDirectory(Path.Combine(_rootDir, "Lib", "encodings")); + + Assert.True(EngineHasStdlib(_rootDir)); + } + + [Fact] + [Trait(TestCategories.Category, Category)] + public void HasStdlib_False_When_No_Lib_Folder_At_All() + { + // Mirrors a Microsoft Store Python's app-execution-alias folder: exists, but carries + // no Lib folder whatsoever. + Assert.False(EngineHasStdlib(_rootDir)); + } + + [Fact] + [Trait(TestCategories.Category, Category)] + public void HasStdlib_False_For_NullOrEmpty() + { + Assert.False(EngineHasStdlib(null)); + Assert.False(EngineHasStdlib(string.Empty)); + } + + // Engine.HasStdlib itself is private (only ConfigureRuntime should call it directly) — + // reached here via reflection so this suite doesn't need to widen that method's + // visibility just for testing, unlike WindowsStdlibLandmark/PosixStdlibDirectory, whose + // whole purpose is to be asserted on directly. + private static bool EngineHasStdlib(string pythonHome) + { + var method = typeof(Engine).GetMethod("HasStdlib", System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Static); + return (bool)method.Invoke(null, new object[] { pythonHome }); + } + } +} diff --git a/Activities/Python/UiPath.Python.Tests/VenvDetectionTests.cs b/Activities/Python/UiPath.Python.Tests/VenvDetectionTests.cs index 63f76f615..63b9ef617 100644 --- a/Activities/Python/UiPath.Python.Tests/VenvDetectionTests.cs +++ b/Activities/Python/UiPath.Python.Tests/VenvDetectionTests.cs @@ -25,9 +25,10 @@ public VenvDetectionTests() public void Dispose() { try { Directory.Delete(_rootDir, true); } catch { /* best effort cleanup */ } + GC.SuppressFinalize(this); } - private string WriteVenvCfg(string venvDir, string home = @"C:\FakeBase", string extra = null) + private static string WriteVenvCfg(string venvDir, string home = @"C:\FakeBase", string extra = null) { Directory.CreateDirectory(venvDir); var content = $"home = {home}{Environment.NewLine}version = 3.13.0{Environment.NewLine}{extra}"; @@ -78,6 +79,57 @@ public void UnrelatedFolder_OneLevelBelow_WrongName_Is_Not_Detected() Assert.Null(venv); } + [Theory] + [InlineData("Scripts")] + [InlineData("bin")] + [Trait(TestCategories.Category, Category)] + public void LauncherSubfolder_With_TrailingSeparator_Is_Detected(string folderName) + { + // Path.GetDirectoryName on a separator-terminated path only strips the trailing + // separator and returns the launcher folder itself, not its parent — detection must + // derive the parent from the trimmed path instead, or this silently fails to find the + // venv root's pyvenv.cfg. + var venvDir = WriteVenvCfg(Path.Combine(_rootDir, "myvenv")); + var launcherDir = Directory.CreateDirectory(Path.Combine(venvDir, folderName)).FullName; + + var venv = VenvDetection.GetVenvInfo(launcherDir + Path.DirectorySeparatorChar); + + Assert.NotNull(venv); + Assert.Equal(venvDir, venv.Root); + } + + [Fact] + [Trait(TestCategories.Category, Category)] + public void UnreadablePyvenvCfg_Is_Not_Detected_And_Does_Not_Throw() + { + // A pyvenv.cfg that exists but can't be read right now (ACLs, an AV sharing + // violation, a concurrent pip rewrite) must not hard-fail detection — the engine + // should get a chance to surface its own, more specific error instead. + var venvDir = WriteVenvCfg(Path.Combine(_rootDir, "myvenv")); + var cfgFile = Path.Combine(venvDir, "pyvenv.cfg"); + + using (new FileStream(cfgFile, FileMode.Open, FileAccess.Read, FileShare.None)) + { + var venv = VenvDetection.GetVenvInfo(venvDir); + Assert.Null(venv); + } + } + + [Fact] + [Trait(TestCategories.Category, Category)] + public void VersionInfoKey_Is_Read_As_Fallback_For_Version() + { + // virtualenv/uv write "version_info" instead of stdlib venv's "version". + var venvDir = Directory.CreateDirectory(Path.Combine(_rootDir, "myvenv")).FullName; + File.WriteAllText(Path.Combine(venvDir, "pyvenv.cfg"), + $"home = C:\\FakeBase{Environment.NewLine}version_info = 3.10.4.final.0{Environment.NewLine}"); + + var venv = VenvDetection.GetVenvInfo(venvDir); + + Assert.NotNull(venv); + Assert.Equal("3.10.4.final.0", venv.Version); + } + [Fact] [Trait(TestCategories.Category, Category)] public void TwoLevelsUp_Is_Not_Detected() diff --git a/Activities/Python/UiPath.Python.Tests/VenvUserSiteIsolationTests.cs b/Activities/Python/UiPath.Python.Tests/VenvUserSiteIsolationTests.cs index 38f05bfa1..b52b90422 100644 --- a/Activities/Python/UiPath.Python.Tests/VenvUserSiteIsolationTests.cs +++ b/Activities/Python/UiPath.Python.Tests/VenvUserSiteIsolationTests.cs @@ -18,8 +18,8 @@ namespace UiPath.Python.Tests // // This test doesn't need real pywin32 binaries: the bug is that the user-site .pth runs at // all when a venv is configured, so a .pth marker is a faithful, hermetic reproduction of the - // exact mechanism pywin32 relies on. Runs out-of-process (the real production default) since - // the fix depends on an environment variable set in the parent process before the host spawns. + // exact mechanism pywin32 relies on. Runs out-of-process since that's the real production + // default path (PythonScope.Isolated defaults to true). public class VenvUserSiteIsolationTests : IDisposable { private const string Category = "Python"; @@ -27,10 +27,6 @@ public class VenvUserSiteIsolationTests : IDisposable private static readonly string EmbeddedRuntimePath = EmbeddedPythonRuntimeBootstrap.EnsureRuntimePath(); private static readonly string EmbeddedLibraryPath = EmbeddedPythonRuntimeBootstrap.GetPythonLibraryPath(EmbeddedRuntimePath); - // Must match the running embeddable interpreter's sys.version_info (3.14.5) — Windows - // user-site resolves to \Python\site-packages. - private const string UserSiteVersionFolder = "Python314"; - private readonly string _rootDir; private readonly string _venvDir; private readonly string _userBaseDir; @@ -54,6 +50,7 @@ public void Dispose() Environment.SetEnvironmentVariable("PYTHONUSERBASE", _previousUserBase); Environment.SetEnvironmentVariable("PYTHONNOUSERSITE", _previousNoUserSite); try { Directory.Delete(_rootDir, true); } catch { /* best effort cleanup */ } + GC.SuppressFinalize(this); } [Fact] @@ -66,7 +63,7 @@ public async Task Venv_Does_Not_Process_UserSite_PthFiles() File.WriteAllText(Path.Combine(_venvDir, "pyvenv.cfg"), $"home = {EmbeddedRuntimePath}{Environment.NewLine}"); WriteMarkerPth(venvSitePackages, "venv"); - var userSitePackages = Directory.CreateDirectory(Path.Combine(_userBaseDir, UserSiteVersionFolder, "site-packages")).FullName; + var userSitePackages = Directory.CreateDirectory(Path.Combine(_userBaseDir, EmbeddedPythonRuntimeBootstrap.UserSiteVersionFolder, "site-packages")).FullName; WriteMarkerPth(userSitePackages, "usersite"); Environment.SetEnvironmentVariable("PYTHONUSERBASE", _userBaseDir); @@ -108,7 +105,7 @@ public async Task Venv_With_SystemSitePackages_Still_Processes_UserSite_PthFiles $"home = {EmbeddedRuntimePath}{Environment.NewLine}include-system-site-packages = true{Environment.NewLine}"); WriteMarkerPth(venvSitePackages, "venv"); - var userSitePackages = Directory.CreateDirectory(Path.Combine(_userBaseDir, UserSiteVersionFolder, "site-packages")).FullName; + var userSitePackages = Directory.CreateDirectory(Path.Combine(_userBaseDir, EmbeddedPythonRuntimeBootstrap.UserSiteVersionFolder, "site-packages")).FullName; WriteMarkerPth(userSitePackages, "usersite"); Environment.SetEnvironmentVariable("PYTHONUSERBASE", _userBaseDir); @@ -245,16 +242,57 @@ public async Task Venv_SiteCustomize_Still_Runs() [Fact] [Trait(TestCategories.Category, Category)] - public async Task Venv_With_SystemSitePackages_Ignores_Ambient_PYTHONNOUSERSITE() + public async Task Venv_With_SystemSitePackages_OwnSiteCustomize_Still_Runs() + { + // Regression test: for a --system-site-packages venv, SetNoSiteFlag isn't set, so + // site.main() already ran during PythonEngine.Initialize() — before this fix, that + // meant it could already have imported and cached "sitecustomize" from the user site + // (enabled here via PYTHONUSERBASE, same as the ambient-PYTHONNOUSERSITE test below) + // before the venv's own site-packages, added afterwards in + // PostInitializationVenvSetup, ever got a chance to be searched. Without popping that + // cached module first, the venv's own sitecustomize.py would never run at all — only + // the user site's copy would. This asserts the venv's own copy does run. + Skip.IfNot(Directory.Exists(EmbeddedRuntimePath)); + + var venvSitePackages = Directory.CreateDirectory(Path.Combine(_venvDir, "Lib", "site-packages")).FullName; + File.WriteAllText(Path.Combine(_venvDir, "pyvenv.cfg"), + $"home = {EmbeddedRuntimePath}{Environment.NewLine}include-system-site-packages = true{Environment.NewLine}"); + + var markerPath = _markerFile.Replace("\\", "/"); + File.WriteAllText(Path.Combine(venvSitePackages, "sitecustomize.py"), + $"import codecs{Environment.NewLine}codecs.open('{markerPath}', 'a', encoding='utf-8').write('venv-sitecustomize\\n'){Environment.NewLine}"); + + var userSitePackages = Directory.CreateDirectory(Path.Combine(_userBaseDir, EmbeddedPythonRuntimeBootstrap.UserSiteVersionFolder, "site-packages")).FullName; + File.WriteAllText(Path.Combine(userSitePackages, "sitecustomize.py"), + $"import codecs{Environment.NewLine}codecs.open('{markerPath}', 'a', encoding='utf-8').write('usersite-sitecustomize\\n'){Environment.NewLine}"); + Environment.SetEnvironmentVariable("PYTHONUSERBASE", _userBaseDir); + + var engine = EngineProvider.Get(Version.Python_310, _venvDir, EmbeddedLibraryPath, inProcess: false); + try + { + await engine.Initialize(null, CancellationToken.None, 60); + } + finally + { + await engine.Release(); + } + + var markerLines = File.Exists(_markerFile) ? File.ReadAllLines(_markerFile) : Array.Empty(); + Assert.Contains("venv-sitecustomize", markerLines); + } + + [Fact] + [Trait(TestCategories.Category, Category)] + public async Task Venv_With_SystemSitePackages_Honors_Ambient_PYTHONNOUSERSITE() { - // Regression test: a customer's own leftover PYTHONNOUSERSITE=1 (e.g. a manual - // workaround predating this fix, exactly like the one mentioned in the original - // ticket) sitting in the *ambient* environment used to leak straight into the spawned - // host via ProcessStartInfo.EnvironmentVariables (which starts as a copy of this - // process's own environment), silently defeating a --system-site-packages venv's - // intent to leave user-site enabled — regardless of what our own code did or didn't - // set. Controller.ClearUserSiteEnvironmentOverride exists specifically to guarantee - // this venv flag decides the outcome, not whatever's ambient on the machine. + // A --system-site-packages venv's own flag only says "also add the base install's + // site-packages" — it says nothing about user-site, which native CPython computes + // independently from PYTHONNOUSERSITE regardless of --system-site-packages. So an + // ambient PYTHONNOUSERSITE=1 (e.g. an administrator's own workaround, exactly like the + // one mentioned in the original ticket) must still suppress user-site here, matching + // what a normally-activated --system-site-packages venv would do. ProcessStartInfo + // already starts as a copy of this process's own environment, so this just needs + // nothing in our own code to actively defeat it. Skip.IfNot(Directory.Exists(EmbeddedRuntimePath)); Environment.SetEnvironmentVariable("PYTHONNOUSERSITE", "1"); @@ -264,7 +302,7 @@ public async Task Venv_With_SystemSitePackages_Ignores_Ambient_PYTHONNOUSERSITE( $"home = {EmbeddedRuntimePath}{Environment.NewLine}include-system-site-packages = true{Environment.NewLine}"); WriteMarkerPth(venvSitePackages, "venv"); - var userSitePackages = Directory.CreateDirectory(Path.Combine(_userBaseDir, UserSiteVersionFolder, "site-packages")).FullName; + var userSitePackages = Directory.CreateDirectory(Path.Combine(_userBaseDir, EmbeddedPythonRuntimeBootstrap.UserSiteVersionFolder, "site-packages")).FullName; WriteMarkerPth(userSitePackages, "usersite"); Environment.SetEnvironmentVariable("PYTHONUSERBASE", _userBaseDir); @@ -281,7 +319,7 @@ public async Task Venv_With_SystemSitePackages_Ignores_Ambient_PYTHONNOUSERSITE( var markerLines = File.Exists(_markerFile) ? File.ReadAllLines(_markerFile) : Array.Empty(); Assert.Contains("venv", markerLines); - Assert.Contains("usersite", markerLines); + Assert.DoesNotContain("usersite", markerLines); } private async Task GetSysPath() diff --git a/Activities/Python/UiPath.Python.Tests/VenvVersionValidationTests.cs b/Activities/Python/UiPath.Python.Tests/VenvVersionValidationTests.cs index d0d9e89be..60bf8a20d 100644 --- a/Activities/Python/UiPath.Python.Tests/VenvVersionValidationTests.cs +++ b/Activities/Python/UiPath.Python.Tests/VenvVersionValidationTests.cs @@ -30,6 +30,7 @@ public VenvVersionValidationTests() public void Dispose() { try { Directory.Delete(_rootDir, true); } catch { /* best effort cleanup */ } + GC.SuppressFinalize(this); } private void WriteVenvCfg(string version) @@ -57,10 +58,9 @@ public void MatchingVenvVersion_DoesNotThrow() { Skip.IfNot(Directory.Exists(EmbeddedRuntimePath)); - WriteVenvCfg("3.14.5"); + WriteVenvCfg(EmbeddedPythonRuntimeBootstrap.PythonVersion); - // Must not throw. - EngineProvider.ValidateInstallation(_venvDir, EmbeddedLibraryPath); + Assert.Null(Record.Exception(() => EngineProvider.ValidateInstallation(_venvDir, EmbeddedLibraryPath))); } [Fact] @@ -73,7 +73,22 @@ public void NonVenvPath_DoesNotThrow() // must not even engage. Directory.CreateDirectory(_venvDir); - EngineProvider.ValidateInstallation(_venvDir, EmbeddedLibraryPath); + Assert.Null(Record.Exception(() => EngineProvider.ValidateInstallation(_venvDir, EmbeddedLibraryPath))); + } + + [Fact] + [Trait(TestCategories.Category, Category)] + public void VersionInfoOnly_CrossChecksJustLikeVersion() + { + // virtualenv/uv write "version_info" instead of stdlib venv's "version" — the + // cross-check must not silently skip for venvs created by those tools. + Skip.IfNot(Directory.Exists(EmbeddedRuntimePath)); + + Directory.CreateDirectory(_venvDir); + File.WriteAllText(Path.Combine(_venvDir, "pyvenv.cfg"), + $"home = {EmbeddedRuntimePath}{Environment.NewLine}version_info = 3.9.0.final.0{Environment.NewLine}"); + + Assert.Throws(() => EngineProvider.ValidateInstallation(_venvDir, EmbeddedLibraryPath)); } } } diff --git a/Activities/Python/UiPath.Python/Impl/Engine.cs b/Activities/Python/UiPath.Python/Impl/Engine.cs index 9ab74d9c1..65e095753 100644 --- a/Activities/Python/UiPath.Python/Impl/Engine.cs +++ b/Activities/Python/UiPath.Python/Impl/Engine.cs @@ -55,65 +55,17 @@ public async Task Initialize(string workingFolder, CancellationToken ct, double Trace.TraceInformation($"Initializing Python runtime using version {_version} and path {_path}"); Stopwatch sw = Stopwatch.StartNew(); - // Detected before Initialize(): a real venv (activated normally, e.g. - // \Scripts\python.exe) always disables the PEP-370 user-site - // directory on its own. Our embedded interpreter never goes through that - // activation path, so nothing does this for us — without it, a native - // package installed in both the venv and the user-site directory (e.g. - // pywin32) can resolve its Python module from one and its native DLL - // dependency from the other, mismatched, copy (STUD-81085). Suppression - // itself happens below, via PythonEngine.SetNoSiteFlag() — see that call - // for why (it also closes a second, related leak path, and is immune to - // whatever PYTHONNOUSERSITE happens to already be set in the ambient - // environment, which an env-var-based approach was not). + // A real venv (activated normally, e.g. \Scripts\python.exe) + // always disables the PEP-370 user-site directory on its own. Our + // embedded interpreter never goes through that activation path, so + // nothing does this for us — without it, a native package installed in + // both the venv and the user-site directory (e.g. pywin32) can resolve + // its Python module from one and its native DLL dependency from the + // other, mismatched, copy (STUD-81085). See ConfigureRuntime for how + // this is used. var venv = _version == Version.Python_310 ? VenvDetection.GetVenvInfo(_path) : null; - if (_isWindows && !_path.IsNullOrEmpty()) - SetDllDirectory(Path.GetFullPath(_path)); - - if (!_libraryPath.IsNullOrEmpty()) - Runtime.PythonDLL = _libraryPath; - - // A venv's own folder is not a valid PythonHome: it only has - // Lib\site-packages, not the standard library (Lib\encodings etc.), so - // pointing the native interpreter at it fails at the very first import - // with "Fatal Python error: Failed to import encodings module". Use the - // base install recorded in the venv's own pyvenv.cfg instead — exactly - // what a normally-activated venv resolves to on its own. EngineProvider - // already validated that base install's version matches _libraryPath's. - var pythonHome = venv != null ? ResolvePythonHome(venv.Home) : _path; - if (!pythonHome.IsNullOrEmpty()) - PythonEngine.PythonHome = pythonHome; - - // For a default venv (no --system-site-packages), suppresses both the - // PEP-370 user-site leak (STUD-81085) and a second, distinct leak path - // found by inspecting a real venv's actual sys.path: CPython's own - // site.main() also unconditionally adds the *base install's* own - // site-packages (site.addsitepackages() against sys.prefix/exec_prefix, - // still pointing at the base install at this point). PYTHONNOUSERSITE - // would only ever have covered the first of these — SetNoSiteFlag - // (Py_NoSiteFlag) disables site.main()'s automatic run entirely, so - // *neither* ever gets added in the first place: "prevent, don't clean up - // after" — a cleanup-after-the-fact fix couldn't undo any .pth-triggered - // side effects, e.g. os.add_dll_directory calls, that already ran by the - // time managed code regains control. It's also an in-memory flag on this - // process's loaded Python DLL, never written to os.environ — unlike an - // env-var-based approach, it can't be defeated by (or leak into) whatever - // PYTHONNOUSERSITE the ambient environment happens to already carry, which - // is exactly the failure mode found in Controller.cs's - // ClearUserSiteEnvironmentOverride for the --system-site-packages case. - // `site` itself is still importable — - // this only skips its automatic invocation at startup — so the explicit - // site.addsitedir() call in PostInitializationVenvSetup for the venv's own - // site-packages, and the site.execsitecustomize() call there preserving - // sitecustomize.py support, both keep working. Must come after - // Runtime.PythonDLL/PythonHome are set, not before — calling it earlier - // left Runtime.PythonDLL null by the time the host tried to use it (and, - // per a known pythonnet issue, SetNoSiteFlag itself can be silently - // ignored on Windows unless another PythonEngine call already preceded - // it — PythonHome, set just above, already satisfies that). - if (venv != null && venv.ShouldDisableUserSite) - PythonEngine.SetNoSiteFlag(); + ConfigureRuntime(venv); PythonEngine.Initialize(); @@ -147,6 +99,77 @@ public async Task Initialize(string workingFolder, CancellationToken ct, double } } + /// + /// Sets up everything needs before it's called: the + /// native DLL search directory, which pythonnet-loaded library to use, , and whether to suppress site.main()'s automatic + /// run for . + /// + /// + /// A venv's own folder is not a valid PythonHome: it only has Lib\site-packages, not the + /// standard library (Lib\encodings etc.), so pointing the native interpreter at it fails at + /// the very first import with "Fatal Python error: Failed to import encodings module". Use + /// the base install recorded in the venv's own pyvenv.cfg instead — exactly what a + /// normally-activated venv resolves to on its own — falling back to LibraryPath's own + /// directory (validated, mandatory) whenever that recorded base install doesn't actually + /// carry a stdlib either (e.g. a Microsoft Store Python's app-execution-alias folder, or a + /// pyvenv.cfg missing "home" entirely). + /// + /// + /// + /// For a default venv (no --system-site-packages), suppresses both the PEP-370 user-site + /// leak (STUD-81085) and a second, distinct leak path found by inspecting a real venv's + /// actual sys.path: CPython's own site.main() also unconditionally adds the *base + /// install's* own site-packages (site.addsitepackages() against sys.prefix/exec_prefix, + /// still pointing at the base install at this point). PYTHONNOUSERSITE would only ever have + /// covered the first of these — SetNoSiteFlag (Py_NoSiteFlag) disables site.main()'s + /// automatic run entirely, so *neither* ever gets added in the first place: "prevent, don't + /// clean up after" — a cleanup-after-the-fact fix couldn't undo any .pth-triggered side + /// effects, e.g. os.add_dll_directory calls, that already ran by the time managed code + /// regains control. It's also an in-memory flag on this process's loaded Python DLL, never + /// written to os.environ — unlike an env-var-based approach, it can't be defeated by (or + /// leak into) whatever PYTHONNOUSERSITE the ambient environment happens to already carry. + /// `site` itself is still importable — this only skips its automatic invocation at startup + /// — so the explicit site.addsitedir() call in PostInitializationVenvSetup for the venv's + /// own site-packages, and the sitecustomize/usercustomize handling there, both keep + /// working. + /// + /// + /// + /// Py_NoSiteFlag is deprecated since CPython 3.12, with removal planned for 3.15 + /// (VersionExtensions._supportedRuntimeVersions currently tops out at 3.14): before adding + /// 3.15+ support, confirm this flag still applies — its removal would silently re-open the + /// original STUD-81085 leak rather than surface an error. The forward-looking replacement + /// is PyConfig.site_import via Py_InitializeFromConfig. + /// + /// + /// + /// SetNoSiteFlag must be called after Runtime.PythonDLL/PythonEngine.PythonHome are set, + /// not before — calling it earlier left Runtime.PythonDLL null by the time the host tried + /// to use it (and, per a known pythonnet issue, SetNoSiteFlag itself can be silently + /// ignored on Windows unless another PythonEngine call already preceded it — PythonHome, + /// set just above, already satisfies that). + /// + /// + private void ConfigureRuntime(VenvDetection.VenvInfo venv) + { + if (_isWindows && !_path.IsNullOrEmpty()) + SetDllDirectory(Path.GetFullPath(_path)); + + if (!_libraryPath.IsNullOrEmpty()) + Runtime.PythonDLL = _libraryPath; + + var pythonHome = venv != null ? ResolvePythonHome(venv.Home) : _path; + if (!HasStdlib(pythonHome)) + pythonHome = Path.GetDirectoryName(_libraryPath); + + if (!pythonHome.IsNullOrEmpty()) + PythonEngine.PythonHome = pythonHome; + + if (venv != null && venv.ShouldDisableUserSite) + PythonEngine.SetNoSiteFlag(); + } + public Task Release() { lock (this) @@ -336,6 +359,39 @@ private static string ResolvePythonHome(string venvHome) return venvHome; } + /// + /// Whether actually carries a standard library, i.e. is a + /// real, complete Python install root rather than e.g. a Microsoft Store app-execution- + /// alias folder (reparse-point exe stubs only) or a venv root itself. + /// + private static bool HasStdlib(string pythonHome) + { + if (pythonHome.IsNullOrEmpty()) + return false; + + return RuntimeInformation.IsOSPlatform(OSPlatform.Windows) + ? Directory.Exists(WindowsStdlibLandmark(pythonHome)) + : HasPosixStdlib(pythonHome); + } + + // "Lib" (capital L) and "lib" (lowercase) below are each the real, fixed folder name + // CPython's own installer/build produces on that platform — not a style choice. Exposed + // (internal) so a test can assert on the exact, case-sensitive string each platform + // checks for: this repo's CI only runs Windows agents, where NTFS's default + // case-insensitive resolution means a real Directory.Exists call can't distinguish "Lib" + // from "lib" on disk — so the only way to actually pin the casing down here is to assert + // the literal path string being built, rather than relying on filesystem lookup behavior. + internal static string WindowsStdlibLandmark(string pythonHome) => Path.Combine(pythonHome, "Lib", "encodings"); + + internal static string PosixStdlibDirectory(string pythonHome) => Path.Combine(pythonHome, "lib"); + + private static bool HasPosixStdlib(string pythonHome) + { + var libPath = PosixStdlibDirectory(pythonHome); + return Directory.Exists(libPath) + && Directory.GetDirectories(libPath, "python*").Any(dir => Directory.Exists(Path.Combine(dir, "encodings"))); + } + private static string GetEnvSitePackagesPath(string venvPath) { string sitePackages; @@ -354,7 +410,7 @@ private static string GetEnvSitePackagesPath(string venvPath) return sitePackages; } - private void PostInitializationVenvSetup(VenvDetection.VenvInfo venv) + private static void PostInitializationVenvSetup(VenvDetection.VenvInfo venv) { if (venv != null) { @@ -377,12 +433,28 @@ private void PostInitializationVenvSetup(VenvDetection.VenvInfo venv) // SetNoSiteFlag (see Initialize()) skips site.main() entirely for a default // venv, which also skips its sitecustomize.py auto-import — some environments // rely on that for corporate setup (proxies, logging, etc.), and it did run - // today before this change, so preserve it explicitly. Safe to call even when - // SetNoSiteFlag wasn't set (--system-site-packages venvs): site.main() already - // ran it there, and re-importing an already-imported module is a no-op. - // Deliberately not calling execusercustomize() — its user-site counterpart, - // consistent with suppressing user-site itself. + // today before this change, so preserve it explicitly. + // + // For a --system-site-packages venv, SetNoSiteFlag is *not* set, so site.main() + // already ran during PythonEngine.Initialize() above and may already have + // imported sitecustomize/usercustomize from whatever the base/user site + // resolved to sys.path first — before the venv's own site-packages, just + // inserted above, ever got a chance to take precedence. Drop any such cached + // module first so the re-import below resolves against the corrected sys.path + // order, matching a natively-activated venv (whose own site-packages is already + // first on sys.path by the time site.main() runs). This is a no-op for the + // default venv case: SetNoSiteFlag prevented anything from being imported yet. + sys.modules.pop("sitecustomize", null); site.execsitecustomize(); + + // execusercustomize() mirrors site.main()'s own "if ENABLE_USER_SITE:" guard — + // only called when user-site isn't suppressed, consistent with not calling it at + // all for the default (user-site-disabled) case. + if (!venv.ShouldDisableUserSite) + { + sys.modules.pop("usercustomize", null); + site.execusercustomize(); + } } } } diff --git a/Activities/Python/UiPath.Python/Impl/OutOfProcessEngine.cs b/Activities/Python/UiPath.Python/Impl/OutOfProcessEngine.cs index 987416847..9dad2e2a9 100644 --- a/Activities/Python/UiPath.Python/Impl/OutOfProcessEngine.cs +++ b/Activities/Python/UiPath.Python/Impl/OutOfProcessEngine.cs @@ -50,29 +50,18 @@ public Task Initialize(string workingFolder, CancellationToken ct, double timeou Stopwatch sw = Stopwatch.StartNew(); - var venv = _version == Version.Python_310 ? VenvDetection.GetVenvInfo(_path) : null; - - // Actual user-site suppression for the default (ShouldDisableUserSite) case happens - // inside Engine.Initialize() itself, via PythonEngine.SetNoSiteFlag() — that runs in - // this same host process regardless, so no parent-side plumbing is needed for it - // (an env-var-based attempt used to live here; it turned out to be both unreliable - // when set this late from managed code in an already-spawned process, and defeatable - // by whatever PYTHONNOUSERSITE the ambient environment already carried). - // - // What *does* still need to happen here, in the parent, before the host spawns: for a - // --system-site-packages venv (ShouldDisableUserSite == false), the intent is to leave - // user-site exactly as a normal, non-embedded interpreter would — but ProcessStartInfo - // starts as a copy of this process's own environment, so if PYTHONNOUSERSITE already - // happens to be set there (e.g. a customer's own leftover workaround, unrelated to this - // fix), it would otherwise leak into the host and silently force user-site off anyway, - // regardless of what SetNoSiteFlag does or doesn't do for the other case. Clearing it - // explicitly for the child guarantees the venv's own IncludeSystemSitePackages flag is - // what decides this, not whatever's ambient on the machine. + // Venv-driven user-site suppression is handled entirely inside Engine.Initialize() + // (the host process), via PythonEngine.SetNoSiteFlag() for the default case. For a + // --system-site-packages venv, nothing needs to happen here either: ProcessStartInfo + // already starts as a copy of this process's own environment, so whatever + // PYTHONNOUSERSITE is ambient on the machine flows through to the host untouched — + // exactly matching how a normally-activated --system-site-packages venv would behave + // (PYTHONNOUSERSITE governs user-site independently of --system-site-packages in + // native CPython too), so no parent-side plumbing is needed for either case. _provider = new Controller() { PythonHostLibFile = ServiceDll_x64, - Visible = _visible, - ClearUserSiteEnvironmentOverride = venv != null && venv.IncludeSystemSitePackages + Visible = _visible }; // Set LogTrace before Create() so the diagnostic file (if enabled) captures diff --git a/Activities/Python/UiPath.Python/Impl/VenvDetection.cs b/Activities/Python/UiPath.Python/Impl/VenvDetection.cs index 9431d0e67..66e106835 100644 --- a/Activities/Python/UiPath.Python/Impl/VenvDetection.cs +++ b/Activities/Python/UiPath.Python/Impl/VenvDetection.cs @@ -29,7 +29,10 @@ internal sealed record VenvInfo(string Root, string Home, bool IncludeSystemSite /// /// Parses pyvenv.cfg at , if present. Requiring at least one of the /// keys a real venv config always has (home/version) avoids treating an unrelated file that - /// merely happens to be named pyvenv.cfg as a venv. + /// merely happens to be named pyvenv.cfg as a venv. Returns null (rather than throwing) when + /// the file exists but can't be read — e.g. ACLs on a locked-down robot account, an AV + /// sharing violation, or a concurrent pip rewrite — matching the "let engine initialization + /// produce its own, precise error" convention every other validator on this path follows. /// private static VenvInfo TryReadVenvConfig(string path) { @@ -38,18 +41,37 @@ private static VenvInfo TryReadVenvConfig(string path) return null; var kv = new Dictionary(StringComparer.OrdinalIgnoreCase); - foreach (var line in File.ReadLines(cfgFile)) + try { - var parts = line.Split('=', 2); - if (parts.Length == 2) - kv[parts[0].Trim()] = parts[1].Trim(); + foreach (var line in File.ReadLines(cfgFile)) + { + var parts = line.Split('=', 2); + if (parts.Length == 2) + kv[parts[0].Trim()] = parts[1].Trim(); + } } - - if (!kv.ContainsKey("home") && !kv.ContainsKey("version")) + catch (IOException) + { + return null; + } + catch (UnauthorizedAccessException) + { return null; + } kv.TryGetValue("home", out var home); - kv.TryGetValue("version", out var version); + + // CPython's stdlib venv writes "version"; virtualenv and uv write "version_info" + // instead (e.g. "version_info = 3.10.4.final.0") — fall back to it so the venv/library + // version cross-check (EngineProvider.ValidateVenvVersion) isn't silently skipped for + // venvs created by those tools. TryParseVenvVersion only reads the first two dot- + // separated parts, so the extra ".final.0" segments are harmless. + if (!kv.TryGetValue("version", out var version)) + kv.TryGetValue("version_info", out version); + + if (home == null && version == null) + return null; + var includeSystemSitePackages = kv.TryGetValue("include-system-site-packages", out var include) && string.Equals(include, "true", StringComparison.OrdinalIgnoreCase); @@ -74,11 +96,16 @@ internal static VenvInfo GetVenvInfo(string path) if (direct != null) return direct; - var folderName = Path.GetFileName(path.TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar)); + var trimmedPath = path.TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar); + var folderName = Path.GetFileName(trimmedPath); if (!Array.Exists(VenvBinFolderNames, name => string.Equals(name, folderName, StringComparison.OrdinalIgnoreCase))) return null; - return TryReadVenvConfig(Path.GetDirectoryName(path)); + // Must derive the parent from the trimmed path: Path.GetDirectoryName on a + // separator-terminated path (e.g. ".../venv/bin/") only strips the trailing separator + // and returns the launcher folder itself, not its parent — which would silently miss + // the venv root's pyvenv.cfg for any Path ending in a separator. + return TryReadVenvConfig(Path.GetDirectoryName(trimmedPath)); } } } From 49dde878931cf8492315aa13c2e8b0faae1528d2 Mon Sep 17 00:00:00 2001 From: viogroza Date: Tue, 25 Aug 2026 18:05:31 +0300 Subject: [PATCH 3/4] Python: fix double-execution regression in sitecustomize precedence fix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit alexandru-petre caught (PR #596, discussion_r3854255951) that the previous fix — unconditionally popping "sitecustomize" from sys.modules and re-importing for --system-site-packages venvs — was worse than the bug it fixed: when the venv has no sitecustomize.py of its own, the re-import still resolves to the same base/user copy, running its side effects a second time. Gate the pop+reimport on the venv's own site-packages actually containing a sitecustomize.py/usercustomize.py. When it does, the venv's copy (now first on sys.path) wins as intended. When it doesn't, the module site.main() already cached — already the correct, highest-priority one — is left untouched, so it runs exactly once. Added Venv_With_SystemSitePackages_And_No_Own_SiteCustomize_Does_Not_Rerun_UserSite_Copy, verified to fail (2 executions instead of 1) against the previous, unconditional version of the fix. Co-Authored-By: Claude Sonnet 5 --- .../VenvUserSiteIsolationTests.cs | 38 +++++++++++++++++++ .../Python/UiPath.Python/Impl/Engine.cs | 26 ++++++++----- 2 files changed, 55 insertions(+), 9 deletions(-) diff --git a/Activities/Python/UiPath.Python.Tests/VenvUserSiteIsolationTests.cs b/Activities/Python/UiPath.Python.Tests/VenvUserSiteIsolationTests.cs index b52b90422..b34ae2b49 100644 --- a/Activities/Python/UiPath.Python.Tests/VenvUserSiteIsolationTests.cs +++ b/Activities/Python/UiPath.Python.Tests/VenvUserSiteIsolationTests.cs @@ -281,6 +281,44 @@ public async Task Venv_With_SystemSitePackages_OwnSiteCustomize_Still_Runs() Assert.Contains("venv-sitecustomize", markerLines); } + [Fact] + [Trait(TestCategories.Category, Category)] + public async Task Venv_With_SystemSitePackages_And_No_Own_SiteCustomize_Does_Not_Rerun_UserSite_Copy() + { + // Regression test for a real bug in an earlier version of this fix: unconditionally + // popping "sitecustomize" from sys.modules and re-importing, regardless of whether the + // venv itself has its own copy, would still find the *same* user-site copy again when + // the venv has none of its own — re-running its side effects a second time. Gating the + // pop+reimport on the venv's own site-packages actually containing a sitecustomize.py + // (see PostInitializationVenvSetup) avoids this: with no venv-local copy, the module + // site.main() already cached is left untouched, so it runs exactly once. + Skip.IfNot(Directory.Exists(EmbeddedRuntimePath)); + + Directory.CreateDirectory(Path.Combine(_venvDir, "Lib", "site-packages")); + File.WriteAllText(Path.Combine(_venvDir, "pyvenv.cfg"), + $"home = {EmbeddedRuntimePath}{Environment.NewLine}include-system-site-packages = true{Environment.NewLine}"); + + var markerPath = _markerFile.Replace("\\", "/"); + var userSitePackages = Directory.CreateDirectory(Path.Combine(_userBaseDir, EmbeddedPythonRuntimeBootstrap.UserSiteVersionFolder, "site-packages")).FullName; + File.WriteAllText(Path.Combine(userSitePackages, "sitecustomize.py"), + $"import codecs{Environment.NewLine}codecs.open('{markerPath}', 'a', encoding='utf-8').write('usersite-sitecustomize\\n'){Environment.NewLine}"); + Environment.SetEnvironmentVariable("PYTHONUSERBASE", _userBaseDir); + + var engine = EngineProvider.Get(Version.Python_310, _venvDir, EmbeddedLibraryPath, inProcess: false); + try + { + await engine.Initialize(null, CancellationToken.None, 60); + } + finally + { + await engine.Release(); + } + + var markerLines = File.Exists(_markerFile) ? File.ReadAllLines(_markerFile) : Array.Empty(); + var single = Assert.Single(markerLines); + Assert.Equal("usersite-sitecustomize", single); + } + [Fact] [Trait(TestCategories.Category, Category)] public async Task Venv_With_SystemSitePackages_Honors_Ambient_PYTHONNOUSERSITE() diff --git a/Activities/Python/UiPath.Python/Impl/Engine.cs b/Activities/Python/UiPath.Python/Impl/Engine.cs index 65e095753..418d10013 100644 --- a/Activities/Python/UiPath.Python/Impl/Engine.cs +++ b/Activities/Python/UiPath.Python/Impl/Engine.cs @@ -439,18 +439,26 @@ private static void PostInitializationVenvSetup(VenvDetection.VenvInfo venv) // already ran during PythonEngine.Initialize() above and may already have // imported sitecustomize/usercustomize from whatever the base/user site // resolved to sys.path first — before the venv's own site-packages, just - // inserted above, ever got a chance to take precedence. Drop any such cached - // module first so the re-import below resolves against the corrected sys.path - // order, matching a natively-activated venv (whose own site-packages is already - // first on sys.path by the time site.main() runs). This is a no-op for the - // default venv case: SetNoSiteFlag prevented anything from being imported yet. - sys.modules.pop("sitecustomize", null); - site.execsitecustomize(); + // inserted above, ever got a chance to take precedence. Only when the venv + // itself actually carries its own copy do we drop the cached module and + // re-import, so the venv's copy — now first on sys.path — wins, matching a + // natively-activated venv. Gating on the venv actually having its own copy + // matters: unconditionally popping and re-importing would, when the venv has + // no copy of its own, still find the same base/user module again and run its + // side effects a *second* time. When the venv has no copy, the module already + // cached by site.main() (if any) is already the correct, highest-priority one + // — left untouched. No-op either way for the default venv case: SetNoSiteFlag + // prevented anything from being imported yet, so the cache is empty going in. + if (File.Exists(Path.Combine(sitePackagesPath, "sitecustomize.py"))) + { + sys.modules.pop("sitecustomize", null); + site.execsitecustomize(); + } // execusercustomize() mirrors site.main()'s own "if ENABLE_USER_SITE:" guard — // only called when user-site isn't suppressed, consistent with not calling it at - // all for the default (user-site-disabled) case. - if (!venv.ShouldDisableUserSite) + // all for the default (user-site-disabled) case. Same re-import gating as above. + if (!venv.ShouldDisableUserSite && File.Exists(Path.Combine(sitePackagesPath, "usercustomize.py"))) { sys.modules.pop("usercustomize", null); site.execusercustomize(); From 0a1e7ba14ca9df94b0a78295c31f758b62357e80 Mon Sep 17 00:00:00 2001 From: viogroza Date: Tue, 25 Aug 2026 19:20:36 +0300 Subject: [PATCH 4/4] Python: fix POSIX prefix fallback, ._pth homes, and PostInitializationVenvSetup regressions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses further PR #596 review findings from Copilot and alexandru-petre: - ResolvePrefixFromLibraryPath now walks up ancestors testing HasStdlib at each level, instead of trusting the library's immediate parent directory as the prefix. On POSIX the shared library sits one or more levels below the real prefix (plain lib/, or a multiarch triplet like lib/x86_64-linux-gnu/), so the previous one-level fallback could set PythonHome to a directory with no stdlib either. - HasStdlib now also recognizes a *._pth file as valid evidence of a home: CPython's ._pth-based isolation (the official Windows embeddable distribution, but not Windows-exclusive) resolves its stdlib from a bundled zip rather than an unpacked Lib folder, which the folder-based checks alone would wrongly treat as "no stdlib". - Fixed a regression in the previous commit's sitecustomize fix: the File.Exists gate meant to stop double-execution for --system-site-packages venvs was also applied to the default-venv case, where SetNoSiteFlag means nothing was ever cached — so a default venv with no venv-local sitecustomize.py silently stopped picking up a base-install Lib\sitecustomize.py (e.g. corporate proxy/logging setup). Split into two branches: default venv calls execsitecustomize() unconditionally again; --system-site-packages keeps the gated pop+reimport. - Removed the usercustomize block entirely: it checked the venv's own site-packages, but usercustomize.py only ever lives in the per-user site directory, so it was dead code — and on the rare path it could fire, its guard (a static config flag) didn't match the runtime ENABLE_USER_SITE value, so it could fire exactly when user-site customization should be suppressed. Restored the original one-line rationale for not calling it at all here. - SetNoSiteFlag also skips site.main()'s setquit()/setcopyright()/ sethelper(), silently dropping the exit/quit/help/copyright/credits/ license builtins from a default-venv scope. Restored them explicitly (not enablerlcompleter(), which only matters interactively). - VenvUserSiteIsolationTests' constructor now clears PYTHONUSERBASE/ PYTHONNOUSERSITE after saving them, so tests don't inherit whatever the host machine's own ambient environment happens to have. Added regression tests for all of the above (._pth recognition, ancestor-walk success/exhaustion, default-venv sitecustomize from a base install, exit/help builtins), each verified to fail without its corresponding fix and pass with it. Co-Authored-By: Claude Sonnet 5 --- .../EngineStdlibDetectionTests.cs | 55 ++++++++ .../VenvUserSiteIsolationTests.cs | 78 +++++++++++ .../Python/UiPath.Python/Impl/Engine.cs | 123 +++++++++++++----- 3 files changed, 221 insertions(+), 35 deletions(-) diff --git a/Activities/Python/UiPath.Python.Tests/EngineStdlibDetectionTests.cs b/Activities/Python/UiPath.Python.Tests/EngineStdlibDetectionTests.cs index 6105358c0..506e255fb 100644 --- a/Activities/Python/UiPath.Python.Tests/EngineStdlibDetectionTests.cs +++ b/Activities/Python/UiPath.Python.Tests/EngineStdlibDetectionTests.cs @@ -94,6 +94,61 @@ public void HasStdlib_False_For_NullOrEmpty() Assert.False(EngineHasStdlib(string.Empty)); } + [Fact] + [Trait(TestCategories.Category, Category)] + public void HasStdlib_True_When_PthFile_Present_Even_Without_Lib_Folder() + { + // The official Windows embeddable distribution (and any other CPython using ._pth-based + // isolation) resolves its stdlib from a bundled zip referenced by a *._pth file next to + // the interpreter, not an unpacked Lib folder — HasStdlib must not treat that as "no + // stdlib" just because Lib\encodings doesn't literally exist. + File.WriteAllText(Path.Combine(_rootDir, "python314._pth"), "python314.zip" + Environment.NewLine); + + Assert.True(EngineHasStdlib(_rootDir)); + } + + [Fact] + [Trait(TestCategories.Category, Category)] + public void ResolvePrefixFromLibraryPath_WalksUp_To_Ancestor_With_Stdlib() + { + // Models the POSIX case this fix targets: the shared library sits one or more levels + // *below* the real prefix (e.g. lib/x86_64-linux-gnu/libpythonX.Y.so under a prefix + // that only has Lib\encodings at its own root) — the immediate parent directory of the + // library is not itself a valid home, but an ancestor is. Uses the Windows landmark + // (Lib\encodings) since that's the only one this Windows-only CI can exercise via a + // real Directory.Exists call — the walk-up mechanism itself is platform-agnostic; only + // which landmark HasStdlib checks for differs by OS. + Directory.CreateDirectory(Path.Combine(_rootDir, "Lib", "encodings")); + var libDir = Directory.CreateDirectory(Path.Combine(_rootDir, "lib", "x86_64-linux-gnu")).FullName; + var libraryPath = Path.Combine(libDir, "libpython3.12.so"); + File.WriteAllText(libraryPath, string.Empty); + + var prefix = Engine.ResolvePrefixFromLibraryPath(libraryPath); + + Assert.Equal(_rootDir, prefix); + } + + [Fact] + [Trait(TestCategories.Category, Category)] + public void ResolvePrefixFromLibraryPath_Returns_Null_When_No_Ancestor_Has_Stdlib() + { + var libDir = Directory.CreateDirectory(Path.Combine(_rootDir, "lib")).FullName; + var libraryPath = Path.Combine(libDir, "libpython3.12.so"); + File.WriteAllText(libraryPath, string.Empty); + + var prefix = Engine.ResolvePrefixFromLibraryPath(libraryPath); + + Assert.Null(prefix); + } + + [Fact] + [Trait(TestCategories.Category, Category)] + public void ResolvePrefixFromLibraryPath_Null_For_NullOrEmpty() + { + Assert.Null(Engine.ResolvePrefixFromLibraryPath(null)); + Assert.Null(Engine.ResolvePrefixFromLibraryPath(string.Empty)); + } + // Engine.HasStdlib itself is private (only ConfigureRuntime should call it directly) — // reached here via reflection so this suite doesn't need to widen that method's // visibility just for testing, unlike WindowsStdlibLandmark/PosixStdlibDirectory, whose diff --git a/Activities/Python/UiPath.Python.Tests/VenvUserSiteIsolationTests.cs b/Activities/Python/UiPath.Python.Tests/VenvUserSiteIsolationTests.cs index b34ae2b49..260d38dc1 100644 --- a/Activities/Python/UiPath.Python.Tests/VenvUserSiteIsolationTests.cs +++ b/Activities/Python/UiPath.Python.Tests/VenvUserSiteIsolationTests.cs @@ -43,6 +43,15 @@ public VenvUserSiteIsolationTests() _previousUserBase = Environment.GetEnvironmentVariable("PYTHONUSERBASE"); _previousNoUserSite = Environment.GetEnvironmentVariable("PYTHONNOUSERSITE"); + + // Start every test from a known-clean baseline regardless of whatever the host + // machine's own ambient environment happens to have for these two — otherwise a + // machine/CI agent with e.g. PYTHONNOUSERSITE already set would make tests that expect + // user-site processing fail for reasons unrelated to the behavior under test. The one + // test that specifically wants an ambient value sets it itself, after this. Dispose + // restores whatever was actually ambient when this instance was constructed. + Environment.SetEnvironmentVariable("PYTHONUSERBASE", null); + Environment.SetEnvironmentVariable("PYTHONNOUSERSITE", null); } public void Dispose() @@ -166,6 +175,41 @@ public async Task Venv_PythonHome_Resolves_To_Declared_Base_Install() ignoreCase: true); } + [Fact] + [Trait(TestCategories.Category, Category)] + public async Task Venv_ExitAndHelp_Builtins_Still_Resolve() + { + // Regression test: SetNoSiteFlag skips site.main() entirely for a default venv, which + // does more than path setup — it also calls setquit()/setcopyright()/sethelper(), + // installing the exit/quit/help/copyright/credits/license builtins. Without restoring + // them, a script calling exit() (a common, if discouraged, RPA pattern) would die with + // a NameError that gives no hint it's related to venv handling — it works today, works + // natively in a venv, and works in every non-venv scope. + Skip.IfNot(Directory.Exists(EmbeddedRuntimePath)); + + Directory.CreateDirectory(Path.Combine(_venvDir, "Lib", "site-packages")); + File.WriteAllText(Path.Combine(_venvDir, "pyvenv.cfg"), $"home = {EmbeddedRuntimePath}{Environment.NewLine}"); + + var engine = EngineProvider.Get(Version.Python_310, _venvDir, EmbeddedLibraryPath, inProcess: false); + string result; + try + { + await engine.Initialize(null, CancellationToken.None, 60); + + var script = await engine.LoadScript( + "import builtins\ndef check():\n return f\"{hasattr(builtins, 'exit')},{hasattr(builtins, 'help')}\"\n", + CancellationToken.None); + var invoked = await engine.InvokeMethod(script, "check", null, CancellationToken.None); + result = (string)engine.Convert(invoked, typeof(string)); + } + finally + { + await engine.Release(); + } + + Assert.Equal("True,True", result); + } + [Fact] [Trait(TestCategories.Category, Category)] public async Task Venv_Does_Not_Leak_BaseInstall_SitePackages() @@ -240,6 +284,40 @@ public async Task Venv_SiteCustomize_Still_Runs() Assert.Contains("sitecustomize", markerLines); } + [Fact] + [Trait(TestCategories.Category, Category)] + public async Task Venv_SiteCustomize_Still_Runs_From_BaseInstall_When_Venv_Has_None() + { + // Regression test: for a default venv (SetNoSiteFlag set) with no sitecustomize.py of + // its own, the base install's Lib\sitecustomize.py (e.g. corporate proxy/logging setup) + // must still be picked up — the stdlib paths remain on sys.path under Py_NoSiteFlag, and + // this must be an unconditional call for the default-venv case, unlike the + // --system-site-packages case where it's gated on the venv having its own copy. + Skip.IfNot(Directory.Exists(EmbeddedRuntimePath)); + + var fakeBase = Directory.CreateDirectory(Path.Combine(_rootDir, "fakebase")).FullName; + Directory.CreateDirectory(Path.Combine(fakeBase, "Lib", "encodings")); + var markerPath = _markerFile.Replace("\\", "/"); + File.WriteAllText(Path.Combine(fakeBase, "Lib", "sitecustomize.py"), + $"import codecs{Environment.NewLine}codecs.open('{markerPath}', 'a', encoding='utf-8').write('base-sitecustomize\\n'){Environment.NewLine}"); + + Directory.CreateDirectory(Path.Combine(_venvDir, "Lib", "site-packages")); + File.WriteAllText(Path.Combine(_venvDir, "pyvenv.cfg"), $"home = {fakeBase}{Environment.NewLine}"); + + var engine = EngineProvider.Get(Version.Python_310, _venvDir, EmbeddedLibraryPath, inProcess: false); + try + { + await engine.Initialize(null, CancellationToken.None, 60); + } + finally + { + await engine.Release(); + } + + var markerLines = File.Exists(_markerFile) ? File.ReadAllLines(_markerFile) : Array.Empty(); + Assert.Contains("base-sitecustomize", markerLines); + } + [Fact] [Trait(TestCategories.Category, Category)] public async Task Venv_With_SystemSitePackages_OwnSiteCustomize_Still_Runs() diff --git a/Activities/Python/UiPath.Python/Impl/Engine.cs b/Activities/Python/UiPath.Python/Impl/Engine.cs index 418d10013..85234c876 100644 --- a/Activities/Python/UiPath.Python/Impl/Engine.cs +++ b/Activities/Python/UiPath.Python/Impl/Engine.cs @@ -110,10 +110,11 @@ public async Task Initialize(string workingFolder, CancellationToken ct, double /// standard library (Lib\encodings etc.), so pointing the native interpreter at it fails at /// the very first import with "Fatal Python error: Failed to import encodings module". Use /// the base install recorded in the venv's own pyvenv.cfg instead — exactly what a - /// normally-activated venv resolves to on its own — falling back to LibraryPath's own - /// directory (validated, mandatory) whenever that recorded base install doesn't actually - /// carry a stdlib either (e.g. a Microsoft Store Python's app-execution-alias folder, or a - /// pyvenv.cfg missing "home" entirely). + /// normally-activated venv resolves to on its own — falling back to a real prefix derived + /// from LibraryPath itself (validated, mandatory — see ResolvePrefixFromLibraryPath) + /// whenever that recorded base install doesn't actually carry a stdlib either (e.g. a + /// Microsoft Store Python's app-execution-alias folder, or a pyvenv.cfg missing "home" + /// entirely). /// /// /// @@ -161,7 +162,7 @@ private void ConfigureRuntime(VenvDetection.VenvInfo venv) var pythonHome = venv != null ? ResolvePythonHome(venv.Home) : _path; if (!HasStdlib(pythonHome)) - pythonHome = Path.GetDirectoryName(_libraryPath); + pythonHome = ResolvePrefixFromLibraryPath(_libraryPath); if (!pythonHome.IsNullOrEmpty()) PythonEngine.PythonHome = pythonHome; @@ -362,13 +363,21 @@ private static string ResolvePythonHome(string venvHome) /// /// Whether actually carries a standard library, i.e. is a /// real, complete Python install root rather than e.g. a Microsoft Store app-execution- - /// alias folder (reparse-point exe stubs only) or a venv root itself. + /// alias folder (reparse-point exe stubs only) or a venv root itself. Also recognizes a + /// *._pth file directly in this folder as valid evidence of a home: CPython's + /// ._pth-based isolation (used by the official Windows embeddable distribution, but + /// not Windows-exclusive) resolves its stdlib from a bundled zip next to the interpreter + /// rather than an unpacked Lib folder, so the folder-based checks below would otherwise + /// wrongly treat a perfectly valid embeddable-style home as having no stdlib at all. /// private static bool HasStdlib(string pythonHome) { - if (pythonHome.IsNullOrEmpty()) + if (pythonHome.IsNullOrEmpty() || !Directory.Exists(pythonHome)) return false; + if (Directory.EnumerateFiles(pythonHome, "*._pth").Any()) + return true; + return RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? Directory.Exists(WindowsStdlibLandmark(pythonHome)) : HasPosixStdlib(pythonHome); @@ -392,6 +401,39 @@ private static bool HasPosixStdlib(string pythonHome) && Directory.GetDirectories(libPath, "python*").Any(dir => Directory.Exists(Path.Combine(dir, "encodings"))); } + /// + /// Derives a real Python prefix from itself, for when the + /// venv's/Path's declared home doesn't carry a stdlib. On Windows the DLL sits directly at + /// the prefix root, so its own directory normally already qualifies — confirmed by the walk + /// below succeeding on the first iteration there. On POSIX the shared library sits one or + /// more levels *below* the prefix — plain lib/libpythonX.Y.so, or a multiarch triplet + /// like lib/x86_64-linux-gnu/libpythonX.Y.so — so blindly taking the library's own + /// directory (as an earlier version of this fix did) yields something like + /// /opt/python/lib rather than the real /opt/python, which itself has no + /// stdlib directly under it either. Walking up ancestors, testing at + /// each level, finds the real prefix on both platforms without hardcoding how many levels + /// separate the library from it. + /// + internal static string ResolvePrefixFromLibraryPath(string libraryPath) + { + if (libraryPath.IsNullOrEmpty()) + return null; + + var dir = Path.GetDirectoryName(libraryPath); + while (!dir.IsNullOrEmpty()) + { + if (HasStdlib(dir)) + return dir; + + var parent = Path.GetDirectoryName(dir); + if (string.Equals(parent, dir, StringComparison.Ordinal)) + break; + dir = parent; + } + + return null; + } + private static string GetEnvSitePackagesPath(string venvPath) { string sitePackages; @@ -430,39 +472,50 @@ private static void PostInitializationVenvSetup(VenvDetection.VenvInfo venv) sys.path.insert(0, sitePackagesPath); - // SetNoSiteFlag (see Initialize()) skips site.main() entirely for a default - // venv, which also skips its sitecustomize.py auto-import — some environments - // rely on that for corporate setup (proxies, logging, etc.), and it did run - // today before this change, so preserve it explicitly. - // - // For a --system-site-packages venv, SetNoSiteFlag is *not* set, so site.main() - // already ran during PythonEngine.Initialize() above and may already have - // imported sitecustomize/usercustomize from whatever the base/user site - // resolved to sys.path first — before the venv's own site-packages, just - // inserted above, ever got a chance to take precedence. Only when the venv - // itself actually carries its own copy do we drop the cached module and - // re-import, so the venv's copy — now first on sys.path — wins, matching a - // natively-activated venv. Gating on the venv actually having its own copy - // matters: unconditionally popping and re-importing would, when the venv has - // no copy of its own, still find the same base/user module again and run its - // side effects a *second* time. When the venv has no copy, the module already - // cached by site.main() (if any) is already the correct, highest-priority one - // — left untouched. No-op either way for the default venv case: SetNoSiteFlag - // prevented anything from being imported yet, so the cache is empty going in. - if (File.Exists(Path.Combine(sitePackagesPath, "sitecustomize.py"))) + if (venv.ShouldDisableUserSite) { - sys.modules.pop("sitecustomize", null); + // SetNoSiteFlag (see ConfigureRuntime) skipped site.main() entirely for + // this default venv — which means it also skipped the plain, non-path + // parts of site.main() itself: setquit()/setcopyright()/sethelper(), the + // module-level functions that install the quit/exit/help/copyright/ + // credits/license builtins. Without them a script calling exit() (a common + // RPA pattern, however discouraged) dies with a NameError that gives no + // hint it's related to venv handling. enablerlcompleter() is deliberately + // not restored — it only registers an interactive-startup readline hook, + // irrelevant to a non-interactive embedded script. + site.setquit(); + site.setcopyright(); + site.sethelper(); + + // Nothing was ever imported/cached under SetNoSiteFlag — the stdlib paths + // (including the base install's own Lib) are still on sys.path under + // Py_NoSiteFlag, so this unconditionally picks up either the venv's own + // sitecustomize.py or a base-install Lib\sitecustomize.py (corporate + // proxy/logging setup etc.), exactly like it did before this whole fix and + // like a natively-activated default venv does. site.execsitecustomize(); } - - // execusercustomize() mirrors site.main()'s own "if ENABLE_USER_SITE:" guard — - // only called when user-site isn't suppressed, consistent with not calling it at - // all for the default (user-site-disabled) case. Same re-import gating as above. - if (!venv.ShouldDisableUserSite && File.Exists(Path.Combine(sitePackagesPath, "usercustomize.py"))) + else if (File.Exists(Path.Combine(sitePackagesPath, "sitecustomize.py"))) { - sys.modules.pop("usercustomize", null); - site.execusercustomize(); + // --system-site-packages: SetNoSiteFlag was *not* set, so site.main() + // already ran during PythonEngine.Initialize() above and may already have + // cached sitecustomize from whatever the base/user site resolved to + // sys.path first — before the venv's own site-packages, just inserted + // above, ever got a chance to take precedence. Only pop and re-import when + // the venv actually has its own copy, so it wins as intended; when it + // doesn't, the module already cached (already the correct, + // highest-priority one in that case) is left untouched, avoiding running + // its side effects a second time. + sys.modules.pop("sitecustomize", null); + site.execsitecustomize(); } + + // usercustomize.py lives in the *user-site* directory, never in a venv's own + // site-packages, and by the time we get here site.main() (when it ran, i.e. + // the --system-site-packages case) already handled it via its own "if + // ENABLE_USER_SITE:" guard — there is nothing left for this venv-specific setup + // to do for it, in either case. Deliberately not calling execusercustomize() + // ourselves, consistent with not touching user-site handling at all here. } } }