diff --git a/.chronus/changes/python-playground-browser-wheels.md b/.chronus/changes/python-playground-browser-wheels.md new file mode 100644 index 00000000000..daa105caffd --- /dev/null +++ b/.chronus/changes/python-playground-browser-wheels.md @@ -0,0 +1,7 @@ +--- +changeKind: fix +packages: + - "@typespec/http-client-python" +--- + +Fix Python playground generation when package downloads are blocked on corporate networks. Publish browser-compatible dependency wheels with the emitter instead of querying PyPI or CFS at runtime. Prepare these assets only for playground builds. diff --git a/packages/http-client-python/ARCHITECTURE.md b/packages/http-client-python/ARCHITECTURE.md index 69b4122745a..d08e412168e 100644 --- a/packages/http-client-python/ARCHITECTURE.md +++ b/packages/http-client-python/ARCHITECTURE.md @@ -117,6 +117,27 @@ The emitter bridges TypeSpec and Python code generation. 5. Generate intermediate YAML representation 6. Invoke Python generator with YAML input +#### Browser runtime + +The playground loads the generator and its pure-Python dependencies from the +emitter's published `generator/dist/browser/manifest.json` and wheel assets using +Pyodide's `loadPackage`. PyYAML and MarkupSafe come from the existing Pyodide CDN. +Browser generation does not query PyPI or CFS, avoiding blocked package downloads +on corporate networks. The Pyodide CDN and emitter asset host must still be reachable. + +`npm run build:playground` resolves these wheels for Pyodide's Python version and +platform, not the build machine. The playground publication pipeline runs this +command; ordinary `npm run build` skips browser-wheel preparation. +Playground builds also provision a pinned pip in the temporary build environment, +including unseeded environments created by uv. +Resolution honors `PIP_INDEX_URL` or `UV_DEFAULT_INDEX`, defaulting to the public +`azure-sdk-for-python` CFS feed. Native dependencies match the Pyodide lockfile; +the browser-only `platformdirs` pin avoids its unsupported `os.geteuid` API. + +Publish the browser manifest and wheels together with the emitter bundle under +the same package version. Previously published bundles do not pick up this fix +until a new version is uploaded and the playground's latest package index is updated. + ### 2. Generator (Python) **Location**: `packages/http-client-python/generator/pygen/` diff --git a/packages/http-client-python/emitter/src/emitter.ts b/packages/http-client-python/emitter/src/emitter.ts index 9af976be426..f97b7234571 100644 --- a/packages/http-client-python/emitter/src/emitter.ts +++ b/packages/http-client-python/emitter/src/emitter.ts @@ -3,12 +3,7 @@ import type { EmitContext } from "@typespec/compiler"; import { emitFile, joinPaths, listServices, NoTarget } from "@typespec/compiler"; import pkgJson from "../../package.json" with { type: "json" }; import { emitCodeModel } from "./code-model.js"; -import { - BLOB_STORAGE_BASE_URL, - PACKAGE_NAME, - PYGEN_WHEEL_FILENAME, - PYODIDE_VERSION, -} from "./constants.js"; +import { BLOB_STORAGE_BASE_URL, PACKAGE_NAME, PYODIDE_VERSION } from "./constants.js"; import type { PythonEmitterOptions, PythonSdkContext } from "./lib.js"; import { reportDiagnostic } from "./lib.js"; import { runNodeEmit } from "./node-runner.js"; @@ -17,8 +12,21 @@ import { loadPyodide } from "./pyodide-loader.js"; import { getRootNamespace, md2Rst } from "./utils.js"; import { dumpCodeModelToYaml } from "./yaml-utils.js"; -function getBrowserPygenWheelUrl(): string { - return `${BLOB_STORAGE_BASE_URL}/${PACKAGE_NAME}/${pkgJson.version}/generator/dist/${PYGEN_WHEEL_FILENAME}`; +function getBrowserAssetUrl(filename: string): string { + return `${BLOB_STORAGE_BASE_URL}/${PACKAGE_NAME}/${pkgJson.version}/generator/dist/browser/${filename}`; +} + +function isBrowserWheelManifest(manifest: unknown): manifest is { wheels: string[] } { + return ( + typeof manifest === "object" && + manifest !== null && + "wheels" in manifest && + Array.isArray(manifest.wheels) && + manifest.wheels.length > 0 && + manifest.wheels.every( + (name: unknown) => typeof name === "string" && /^[\w.+-]+\.whl$/.test(name), + ) + ); } function addDefaultOptions(sdkContext: PythonSdkContext) { @@ -306,9 +314,25 @@ async function setupPyodideCallBrowser() { // use default MEMFS for browser, since NODEFS is not supported pyodide.FS.mkdirTree("/generator"); - await pyodide.loadPackage("micropip"); - const micropip = pyodide.pyimport("micropip"); - await micropip.install(getBrowserPygenWheelUrl()); + const response = await fetch(getBrowserAssetUrl("manifest.json")); + if (!response.ok) { + throw new Error(`Failed to load Python browser wheel manifest: HTTP ${response.status}`); + } + const manifest: unknown = await response.json(); + if (!isBrowserWheelManifest(manifest)) { + throw new Error("Invalid Python browser wheel manifest."); + } + await loadBrowserPackages(pyodide, manifest.wheels.map(getBrowserAssetUrl)); + await loadBrowserPackages(pyodide, ["pyyaml", "markupsafe"]); return pyodide; } + +async function loadBrowserPackages(pyodide: PyodideInterface, packages: string[]): Promise { + const errors: string[] = []; + // loadPackage can report failed downloads through its callback without rejecting. + await pyodide.loadPackage(packages, { errorCallback: (message) => errors.push(message) }); + if (errors.length > 0) { + throw new Error(`Failed to load Python browser packages: ${errors.join("\n")}`); + } +} diff --git a/packages/http-client-python/emitter/test/browser-pyodide.test.ts b/packages/http-client-python/emitter/test/browser-pyodide.test.ts index 09bad2b8471..fee7f3f065f 100644 --- a/packages/http-client-python/emitter/test/browser-pyodide.test.ts +++ b/packages/http-client-python/emitter/test/browser-pyodide.test.ts @@ -1,5 +1,6 @@ +import { expectDiagnostics } from "@typespec/compiler/testing"; import { strictEqual } from "assert"; -import { afterEach, describe, it, vi } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; const loadPyodide = vi.hoisted(() => vi.fn()); @@ -8,6 +9,7 @@ vi.mock("../src/pyodide-loader.js", () => ({ loadPyodide })); describe("typespec-python: browser pyodide bootstrap", () => { afterEach(() => { delete (globalThis as any).window; + vi.unstubAllGlobals(); loadPyodide.mockReset(); vi.resetModules(); }); @@ -22,4 +24,140 @@ describe("typespec-python: browser pyodide bootstrap", () => { strictEqual(loadPyodide.mock.calls.length, 0); }); + + it("installs published dependency wheels without querying a package index", async () => { + vi.stubGlobal("window", globalThis); + const wheels = ["pygen-0.1.0-py3-none-any.whl", "black-26.3.1-py3-none-any.whl"]; + const fetch = vi.fn().mockResolvedValue({ + ok: true, + json: async () => ({ wheels }), + }); + vi.stubGlobal("fetch", fetch); + const pyodide = { + FS: { + mkdirTree: vi.fn(), + readdir: vi.fn().mockReturnValue([".", ".."]), + writeFile: vi.fn(), + }, + loadPackage: vi.fn(), + toPy: vi.fn(), + runPythonAsync: vi.fn(), + }; + loadPyodide.mockResolvedValue(pyodide); + const { EmitterTester } = await import("./test-host.js"); + const [, diagnostics] = await EmitterTester.compileAndDiagnose(` + import "@typespec/http"; + using TypeSpec.Http; + @service namespace Test; + @get op read(): string; + `); + + expectDiagnostics(diagnostics, []); + expect(fetch).toHaveBeenCalledWith( + expect.stringMatching(/\/generator\/dist\/browser\/manifest\.json$/), + ); + expect(pyodide.loadPackage).toHaveBeenNthCalledWith( + 1, + wheels.map((name) => expect.stringMatching(new RegExp(`/browser/${name}$`))), + { errorCallback: expect.any(Function) }, + ); + expect(pyodide.loadPackage).toHaveBeenNthCalledWith(2, ["pyyaml", "markupsafe"], { + errorCallback: expect.any(Function), + }); + expect(pyodide.runPythonAsync).toHaveBeenCalledOnce(); + }); + + it.each([ + { ok: false, status: 404, body: undefined, message: /HTTP 404/ }, + { ok: true, status: 200, body: null, message: /Invalid Python browser wheel/ }, + { ok: true, status: 200, body: {}, message: /Invalid Python browser wheel/ }, + { + ok: true, + status: 200, + body: { wheels: [] }, + message: /Invalid Python browser wheel/, + }, + { + ok: true, + status: 200, + body: { wheels: "not-an-array" }, + message: /Invalid Python browser wheel/, + }, + { + ok: true, + status: 200, + body: { wheels: [null] }, + message: /Invalid Python browser wheel/, + }, + { + ok: true, + status: 200, + body: { wheels: [42] }, + message: /Invalid Python browser wheel/, + }, + { + ok: true, + status: 200, + body: { wheels: ["../unexpected.whl"] }, + message: /Invalid Python browser wheel/, + }, + ])("reports unusable browser wheel manifests: $body", async ({ ok, status, body, message }) => { + vi.stubGlobal("window", globalThis); + vi.stubGlobal("fetch", vi.fn().mockResolvedValue({ ok, status, json: async () => body })); + const loadPackage = vi.fn(); + loadPyodide.mockResolvedValue({ + FS: { mkdirTree: vi.fn() }, + loadPackage, + }); + const { EmitterTester } = await import("./test-host.js"); + const [, diagnostics] = await EmitterTester.compileAndDiagnose( + "@service namespace Test; op read(): string;", + ); + + expectDiagnostics(diagnostics, { + code: "@typespec/http-client-python/unknown-error", + message, + }); + expect(loadPackage).not.toHaveBeenCalled(); + }); + + it.each(["rejected", "reported"])( + "reports %s wheel loading failures and retries initialization on the next emit", + async (mode) => { + vi.stubGlobal("window", globalThis); + vi.stubGlobal( + "fetch", + vi.fn().mockResolvedValue({ + ok: true, + json: async () => ({ wheels: ["pygen-0.1.0-py3-none-any.whl"] }), + }), + ); + const loadPackage = vi.fn().mockImplementationOnce((_names, options) => { + if (mode === "rejected") return Promise.reject(new Error("Wheel download failed")); + options?.errorCallback("Wheel download failed"); + return Promise.resolve([]); + }); + loadPyodide.mockResolvedValue({ + FS: { + mkdirTree: vi.fn(), + readdir: vi.fn().mockReturnValue([".", ".."]), + writeFile: vi.fn(), + }, + loadPackage, + toPy: vi.fn(), + runPythonAsync: vi.fn(), + }); + const { EmitterTester } = await import("./test-host.js"); + const source = "@service namespace Test; op read(): string;"; + const [, failure] = await EmitterTester.compileAndDiagnose(source); + expectDiagnostics(failure, { + code: "@typespec/http-client-python/unknown-error", + message: /Wheel download failed/, + }); + const [, success] = await EmitterTester.compileAndDiagnose(source); + expectDiagnostics(success, []); + expect(loadPyodide).toHaveBeenCalledTimes(2); + expect(loadPackage).toHaveBeenCalledTimes(3); + }, + ); }); diff --git a/packages/http-client-python/eng/pipeline/publish.yml b/packages/http-client-python/eng/pipeline/publish.yml index 09b3aee24f1..6ea23cd309e 100644 --- a/packages/http-client-python/eng/pipeline/publish.yml +++ b/packages/http-client-python/eng/pipeline/publish.yml @@ -32,3 +32,4 @@ extends: EnableCadlRanchReport: false PythonVersion: "3.15" UploadPlaygroundBundle: true + PlaygroundBundleBuildScript: "build:playground" diff --git a/packages/http-client-python/eng/scripts/setup/build.ts b/packages/http-client-python/eng/scripts/setup/build.ts index 71c258e68c6..ff46e1fc1e1 100644 --- a/packages/http-client-python/eng/scripts/setup/build.ts +++ b/packages/http-client-python/eng/scripts/setup/build.ts @@ -2,7 +2,7 @@ import { exec } from "child_process"; import { runPython3 } from "./run-python3.js"; async function main() { - await runPython3("./eng/scripts/setup/build_pygen_wheel.py"); + await runPython3("./eng/scripts/setup/build_pygen_wheel.py", ...process.argv.slice(2)); // remove the venv_build_wheel directory exec("rimraf ./venv_build_wheel", (error, stdout, _stderr) => { if (error) { diff --git a/packages/http-client-python/eng/scripts/setup/build_pygen_wheel.py b/packages/http-client-python/eng/scripts/setup/build_pygen_wheel.py index 40416a0991d..3efe543956b 100644 --- a/packages/http-client-python/eng/scripts/setup/build_pygen_wheel.py +++ b/packages/http-client-python/eng/scripts/setup/build_pygen_wheel.py @@ -6,6 +6,13 @@ # license information. # -------------------------------------------------------------------------- import sys +import argparse +import json +import os +import shutil +import subprocess +import tempfile +from urllib.request import urlopen if not sys.version_info >= (3, 10, 0): raise Exception("Autorest for Python extension requires Python 3.10 at least") @@ -29,13 +36,76 @@ _ROOT_DIR = Path(__file__).parent.parent.parent.parent -def main(): +def build_browser_wheels(venv_context): + dist = _ROOT_DIR / "generator" / "dist" + wheel = next(dist.glob("pygen-*.whl"), None) + if wheel is None: + raise RuntimeError(f"Generator wheel not found in {dist}. Build the generator before preparing browser wheels.") + package = json.loads((_ROOT_DIR / "package.json").read_text()) + pyodide_url = f"https://cdn.jsdelivr.net/pyodide/v{package['dependencies']['pyodide']}/full/" + with urlopen(f"{pyodide_url}pyodide-lock.json") as response: + lock = json.load(response) + + # These native extensions must match the browser's WebAssembly runtime, not the build host. + native_wheels = [lock["packages"][name]["file_name"] for name in ("pyyaml", "markupsafe")] + _, abi, platform = native_wheels[0].removesuffix(".whl").rsplit("-", 2) + with tempfile.TemporaryDirectory(dir=dist) as directory: + staging = Path(directory) + command = [ + venv_context.env_exe, + "-m", + "pip", + "download", + str(wheel), + # Later platformdirs releases import os.geteuid, which Pyodide does not provide. + "platformdirs==4.3.8", + *[pyodide_url + filename for filename in native_wheels], + f"--dest={staging}", + "--only-binary=:all:", + f"--python-version={lock['info']['python']}", + f"--platform={platform}", + "--implementation=cp", + f"--abi={abi}", + ] + env = os.environ.copy() + env["PIP_INDEX_URL"] = ( + env.get("PIP_INDEX_URL") + or env.get("UV_DEFAULT_INDEX") + or "https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-python/pypi/simple/" + ) + subprocess.run(command, env=env, check=True) + + unexpected = [ + p.name + for p in staging.glob("*.whl") + if not p.name.endswith("-none-any.whl") and p.name not in native_wheels + ] + if unexpected: + raise RuntimeError(f"Unsupported native browser dependencies: {', '.join(unexpected)}") + browser = dist / "browser" + if browser.exists(): + shutil.rmtree(browser) + browser.mkdir() + wheels = sorted(p.name for p in staging.glob("*-none-any.whl")) + for filename in wheels: + shutil.copyfile(staging / filename, browser / filename) + (browser / "manifest.json").write_text(json.dumps({"wheels": wheels}, indent=2) + "\n") + + +def main(playground=False): venv_path = _ROOT_DIR / "venv_build_wheel" venv_context = create_venv_with_package_manager(venv_path) - install_packages(["build"], venv_context) + # uv environments are unseeded; playground wheel downloads also need pip. + install_packages(["build", "pip==26.2.2"] if playground else ["build"], venv_context) python_run(venv_context, "build", ["--wheel"], additional_dir="generator") + if playground: + build_browser_wheels(venv_context) if __name__ == "__main__": - main() + parser = argparse.ArgumentParser() + parser.add_argument( + "--playground", action="store_true", help="Prepare browser dependency wheels for the playground" + ) + main(playground=parser.parse_args().playground) diff --git a/packages/http-client-python/package.json b/packages/http-client-python/package.json index 658262ab694..e6e376bda3a 100644 --- a/packages/http-client-python/package.json +++ b/packages/http-client-python/package.json @@ -35,6 +35,7 @@ "scripts": { "clean": "rimraf ./dist ./temp ./venv ./tests/generated", "build": "tsc -p ./emitter/tsconfig.build.json && tsx ./eng/scripts/setup/build.ts", + "build:playground": "npm run build -- --playground", "watch": "tsc -p ./emitter/tsconfig.build.json --watch", "install": "tsx ./eng/scripts/setup/install.ts", "prepare": "tsx ./eng/scripts/setup/prepare.ts", @@ -86,6 +87,14 @@ { "path": "generator/dist/pygen-*.whl", "contentType": "application/zip" + }, + { + "path": "generator/dist/browser/*.whl", + "contentType": "application/zip" + }, + { + "path": "generator/dist/browser/manifest.json", + "contentType": "application/json" } ], "bundlePeerDependencies": [ diff --git a/packages/http-client-python/tests/unit/test_browser_wheels.py b/packages/http-client-python/tests/unit/test_browser_wheels.py new file mode 100644 index 00000000000..7832f704e91 --- /dev/null +++ b/packages/http-client-python/tests/unit/test_browser_wheels.py @@ -0,0 +1,154 @@ +import importlib.util +import io +import json +from pathlib import Path +from types import SimpleNamespace +import sys +import tempfile +import unittest +from unittest.mock import Mock, call, patch + +SETUP = Path(__file__).resolve().parents[2] / "eng" / "scripts" / "setup" +sys.path.insert(0, str(SETUP)) +spec = importlib.util.spec_from_file_location("build_pygen_wheel", SETUP / "build_pygen_wheel.py") +builder = importlib.util.module_from_spec(spec) +spec.loader.exec_module(builder) +sys.path.remove(str(SETUP)) + + +class BrowserWheelTests(unittest.TestCase): + def test_missing_generator_wheel_reports_build_prerequisite(self): + with tempfile.TemporaryDirectory() as directory: + dist = Path(directory) / "generator" / "dist" + (Path(directory) / "package.json").write_text(json.dumps({"dependencies": {"pyodide": "0.26.2"}})) + lock = {"packages": {"pyyaml": {}, "markupsafe": {}}} + with ( + patch.object(builder, "_ROOT_DIR", Path(directory)), + patch.object(builder, "urlopen", return_value=io.BytesIO(json.dumps(lock).encode())) as fetch, + ): + with self.assertRaisesRegex(RuntimeError, "Generator wheel not found") as error: + builder.build_browser_wheels(SimpleNamespace(env_exe="python")) + + self.assertIn(str(dist), str(error.exception)) + fetch.assert_not_called() + + def test_regular_build_skips_browser_dependencies(self): + context = SimpleNamespace(env_exe="python") + with ( + patch.object(builder, "create_venv_with_package_manager", return_value=context), + patch.object(builder, "install_packages") as install, + patch.object(builder, "python_run") as run, + patch.object(builder, "build_browser_wheels") as browser, + ): + builder.main() + + install.assert_called_once_with(["build"], context) + run.assert_called_once_with(context, "build", ["--wheel"], additional_dir="generator") + browser.assert_not_called() + + def test_playground_build_prepares_browser_dependencies(self): + context = SimpleNamespace(env_exe="python") + with ( + patch.object(builder, "create_venv_with_package_manager", return_value=context), + patch.object(builder, "install_packages") as install, + patch.object(builder, "python_run") as run, + patch.object(builder, "build_browser_wheels") as browser, + ): + builder.main(playground=True) + + install.assert_called_once_with(["build", "pip==26.2.2"], context) + run.assert_called_once_with(context, "build", ["--wheel"], additional_dir="generator") + browser.assert_called_once_with(context) + + def test_uv_playground_build_seeds_pip_before_preparing_browser_wheels(self): + with ( + tempfile.TemporaryDirectory() as directory, + patch.object(builder, "_ROOT_DIR", Path(directory)), + patch("package_manager.detect_package_manager", return_value="uv"), + patch.object(builder.subprocess, "check_call") as create, + patch.object(builder.subprocess, "run", return_value=SimpleNamespace(returncode=0)) as install, + patch.object(builder, "python_run") as build, + patch.object(builder, "build_browser_wheels") as browser, + ): + steps = Mock() + steps.attach_mock(install, "install") + steps.attach_mock(build, "build") + steps.attach_mock(browser, "browser") + + builder.main(playground=True) + + context = browser.call_args.args[0] + create.assert_called_once_with(["uv", "venv", str(Path(directory) / "venv_build_wheel")]) + self.assertEqual( + steps.mock_calls, + [ + call.install( + ["uv", "pip", "install", "--python", context.env_exe, "build", "pip==26.2.2"], + capture_output=True, + text=True, + ), + call.build(context, "build", ["--wheel"], additional_dir="generator"), + call.browser(context), + ], + ) + + def test_resolves_for_pyodide_and_publishes_only_pure_python_wheels(self): + native = [ + {"name": "PyYAML", "version": "6.0.1", "file_name": "PyYAML-6.0.1-cp312-cp312-pyodide_2024_0_wasm32.whl"}, + { + "name": "MarkupSafe", + "version": "2.1.5", + "file_name": "MarkupSafe-2.1.5-cp312-cp312-pyodide_2024_0_wasm32.whl", + }, + ] + lock = {"info": {"python": "3.12.1"}, "packages": dict(zip(("pyyaml", "markupsafe"), native))} + pure = ["pygen-0.1.0-py3-none-any.whl", "black-26.3.1-py3-none-any.whl"] + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / "package.json").write_text(json.dumps({"dependencies": {"pyodide": "0.26.2"}})) + dist = root / "generator" / "dist" + dist.mkdir(parents=True) + (dist / pure[0]).touch() + browser = dist / "browser" + browser.mkdir() + (browser / "stale.whl").touch() + unexpected_native = None + + def download(command, *, env, check): + self.assertTrue(check) + self.assertIn("--python-version=3.12.1", command) + self.assertIn("--platform=pyodide_2024_0_wasm32", command) + self.assertIn("--abi=cp312", command) + self.assertEqual(env["PIP_INDEX_URL"], "https://example.com/authenticated-feed/") + self.assertIn("platformdirs==4.3.8", command) + for package in native: + self.assertIn(f"https://cdn.jsdelivr.net/pyodide/v0.26.2/full/{package['file_name']}", command) + staging = Path(next(arg.removeprefix("--dest=") for arg in command if arg.startswith("--dest="))) + for filename in pure + [p["file_name"] for p in native]: + (staging / filename).touch() + if unexpected_native: + (staging / unexpected_native).touch() + + with ( + patch.object(builder, "_ROOT_DIR", root), + patch.object(builder, "urlopen", side_effect=lambda _url: io.BytesIO(json.dumps(lock).encode())), + patch.object(builder.subprocess, "run", side_effect=download), + patch.dict(builder.os.environ, {"PIP_INDEX_URL": "https://example.com/authenticated-feed/"}), + ): + builder.build_browser_wheels(SimpleNamespace(env_exe="python")) + manifest_before_failure = (browser / "manifest.json").read_text() + unexpected_native = "unsupported-1.0-cp312-cp312-pyodide_2024_0_wasm32.whl" + with self.assertRaisesRegex( + RuntimeError, f"Unsupported native browser dependencies: {unexpected_native}" + ): + builder.build_browser_wheels(SimpleNamespace(env_exe="python")) + self.assertEqual((browser / "manifest.json").read_text(), manifest_before_failure) + + manifest = json.loads((dist / "browser" / "manifest.json").read_text()) + self.assertEqual(manifest["wheels"], sorted(pure)) + self.assertEqual(set(manifest), {"wheels"}) + self.assertEqual(sorted(p.name for p in (dist / "browser").glob("*.whl")), sorted(pure)) + + +if __name__ == "__main__": + unittest.main()