diff --git a/.chronus/changes/python-playground-wheel-rollback.md b/.chronus/changes/python-playground-wheel-rollback.md new file mode 100644 index 00000000000..d77ce9db962 --- /dev/null +++ b/.chronus/changes/python-playground-wheel-rollback.md @@ -0,0 +1,7 @@ +--- +changeKind: internal +packages: + - "@typespec/http-client-python" +--- + +Revert bundled playground dependency wheels from #12484 before retrying the Python emitter 0.40.1 release. diff --git a/packages/http-client-python/ARCHITECTURE.md b/packages/http-client-python/ARCHITECTURE.md index d08e412168e..69b4122745a 100644 --- a/packages/http-client-python/ARCHITECTURE.md +++ b/packages/http-client-python/ARCHITECTURE.md @@ -117,27 +117,6 @@ The emitter bridges TypeSpec and Python code generation. 5. Generate intermediate YAML representation 6. Invoke Python generator with YAML input -#### Browser runtime - -The playground loads the generator and its pure-Python dependencies from the -emitter's published `generator/dist/browser/manifest.json` and wheel assets using -Pyodide's `loadPackage`. PyYAML and MarkupSafe come from the existing Pyodide CDN. -Browser generation does not query PyPI or CFS, avoiding blocked package downloads -on corporate networks. The Pyodide CDN and emitter asset host must still be reachable. - -`npm run build:playground` resolves these wheels for Pyodide's Python version and -platform, not the build machine. The playground publication pipeline runs this -command; ordinary `npm run build` skips browser-wheel preparation. -Playground builds also provision a pinned pip in the temporary build environment, -including unseeded environments created by uv. -Resolution honors `PIP_INDEX_URL` or `UV_DEFAULT_INDEX`, defaulting to the public -`azure-sdk-for-python` CFS feed. Native dependencies match the Pyodide lockfile; -the browser-only `platformdirs` pin avoids its unsupported `os.geteuid` API. - -Publish the browser manifest and wheels together with the emitter bundle under -the same package version. Previously published bundles do not pick up this fix -until a new version is uploaded and the playground's latest package index is updated. - ### 2. Generator (Python) **Location**: `packages/http-client-python/generator/pygen/` diff --git a/packages/http-client-python/emitter/src/emitter.ts b/packages/http-client-python/emitter/src/emitter.ts index f97b7234571..9af976be426 100644 --- a/packages/http-client-python/emitter/src/emitter.ts +++ b/packages/http-client-python/emitter/src/emitter.ts @@ -3,7 +3,12 @@ import type { EmitContext } from "@typespec/compiler"; import { emitFile, joinPaths, listServices, NoTarget } from "@typespec/compiler"; import pkgJson from "../../package.json" with { type: "json" }; import { emitCodeModel } from "./code-model.js"; -import { BLOB_STORAGE_BASE_URL, PACKAGE_NAME, PYODIDE_VERSION } from "./constants.js"; +import { + BLOB_STORAGE_BASE_URL, + PACKAGE_NAME, + PYGEN_WHEEL_FILENAME, + PYODIDE_VERSION, +} from "./constants.js"; import type { PythonEmitterOptions, PythonSdkContext } from "./lib.js"; import { reportDiagnostic } from "./lib.js"; import { runNodeEmit } from "./node-runner.js"; @@ -12,21 +17,8 @@ import { loadPyodide } from "./pyodide-loader.js"; import { getRootNamespace, md2Rst } from "./utils.js"; import { dumpCodeModelToYaml } from "./yaml-utils.js"; -function getBrowserAssetUrl(filename: string): string { - return `${BLOB_STORAGE_BASE_URL}/${PACKAGE_NAME}/${pkgJson.version}/generator/dist/browser/${filename}`; -} - -function isBrowserWheelManifest(manifest: unknown): manifest is { wheels: string[] } { - return ( - typeof manifest === "object" && - manifest !== null && - "wheels" in manifest && - Array.isArray(manifest.wheels) && - manifest.wheels.length > 0 && - manifest.wheels.every( - (name: unknown) => typeof name === "string" && /^[\w.+-]+\.whl$/.test(name), - ) - ); +function getBrowserPygenWheelUrl(): string { + return `${BLOB_STORAGE_BASE_URL}/${PACKAGE_NAME}/${pkgJson.version}/generator/dist/${PYGEN_WHEEL_FILENAME}`; } function addDefaultOptions(sdkContext: PythonSdkContext) { @@ -314,25 +306,9 @@ async function setupPyodideCallBrowser() { // use default MEMFS for browser, since NODEFS is not supported pyodide.FS.mkdirTree("/generator"); - const response = await fetch(getBrowserAssetUrl("manifest.json")); - if (!response.ok) { - throw new Error(`Failed to load Python browser wheel manifest: HTTP ${response.status}`); - } - const manifest: unknown = await response.json(); - if (!isBrowserWheelManifest(manifest)) { - throw new Error("Invalid Python browser wheel manifest."); - } - await loadBrowserPackages(pyodide, manifest.wheels.map(getBrowserAssetUrl)); - await loadBrowserPackages(pyodide, ["pyyaml", "markupsafe"]); + await pyodide.loadPackage("micropip"); + const micropip = pyodide.pyimport("micropip"); + await micropip.install(getBrowserPygenWheelUrl()); return pyodide; } - -async function loadBrowserPackages(pyodide: PyodideInterface, packages: string[]): Promise { - const errors: string[] = []; - // loadPackage can report failed downloads through its callback without rejecting. - await pyodide.loadPackage(packages, { errorCallback: (message) => errors.push(message) }); - if (errors.length > 0) { - throw new Error(`Failed to load Python browser packages: ${errors.join("\n")}`); - } -} diff --git a/packages/http-client-python/emitter/test/browser-pyodide.test.ts b/packages/http-client-python/emitter/test/browser-pyodide.test.ts index fee7f3f065f..09bad2b8471 100644 --- a/packages/http-client-python/emitter/test/browser-pyodide.test.ts +++ b/packages/http-client-python/emitter/test/browser-pyodide.test.ts @@ -1,6 +1,5 @@ -import { expectDiagnostics } from "@typespec/compiler/testing"; import { strictEqual } from "assert"; -import { afterEach, describe, expect, it, vi } from "vitest"; +import { afterEach, describe, it, vi } from "vitest"; const loadPyodide = vi.hoisted(() => vi.fn()); @@ -9,7 +8,6 @@ vi.mock("../src/pyodide-loader.js", () => ({ loadPyodide })); describe("typespec-python: browser pyodide bootstrap", () => { afterEach(() => { delete (globalThis as any).window; - vi.unstubAllGlobals(); loadPyodide.mockReset(); vi.resetModules(); }); @@ -24,140 +22,4 @@ describe("typespec-python: browser pyodide bootstrap", () => { strictEqual(loadPyodide.mock.calls.length, 0); }); - - it("installs published dependency wheels without querying a package index", async () => { - vi.stubGlobal("window", globalThis); - const wheels = ["pygen-0.1.0-py3-none-any.whl", "black-26.3.1-py3-none-any.whl"]; - const fetch = vi.fn().mockResolvedValue({ - ok: true, - json: async () => ({ wheels }), - }); - vi.stubGlobal("fetch", fetch); - const pyodide = { - FS: { - mkdirTree: vi.fn(), - readdir: vi.fn().mockReturnValue([".", ".."]), - writeFile: vi.fn(), - }, - loadPackage: vi.fn(), - toPy: vi.fn(), - runPythonAsync: vi.fn(), - }; - loadPyodide.mockResolvedValue(pyodide); - const { EmitterTester } = await import("./test-host.js"); - const [, diagnostics] = await EmitterTester.compileAndDiagnose(` - import "@typespec/http"; - using TypeSpec.Http; - @service namespace Test; - @get op read(): string; - `); - - expectDiagnostics(diagnostics, []); - expect(fetch).toHaveBeenCalledWith( - expect.stringMatching(/\/generator\/dist\/browser\/manifest\.json$/), - ); - expect(pyodide.loadPackage).toHaveBeenNthCalledWith( - 1, - wheels.map((name) => expect.stringMatching(new RegExp(`/browser/${name}$`))), - { errorCallback: expect.any(Function) }, - ); - expect(pyodide.loadPackage).toHaveBeenNthCalledWith(2, ["pyyaml", "markupsafe"], { - errorCallback: expect.any(Function), - }); - expect(pyodide.runPythonAsync).toHaveBeenCalledOnce(); - }); - - it.each([ - { ok: false, status: 404, body: undefined, message: /HTTP 404/ }, - { ok: true, status: 200, body: null, message: /Invalid Python browser wheel/ }, - { ok: true, status: 200, body: {}, message: /Invalid Python browser wheel/ }, - { - ok: true, - status: 200, - body: { wheels: [] }, - message: /Invalid Python browser wheel/, - }, - { - ok: true, - status: 200, - body: { wheels: "not-an-array" }, - message: /Invalid Python browser wheel/, - }, - { - ok: true, - status: 200, - body: { wheels: [null] }, - message: /Invalid Python browser wheel/, - }, - { - ok: true, - status: 200, - body: { wheels: [42] }, - message: /Invalid Python browser wheel/, - }, - { - ok: true, - status: 200, - body: { wheels: ["../unexpected.whl"] }, - message: /Invalid Python browser wheel/, - }, - ])("reports unusable browser wheel manifests: $body", async ({ ok, status, body, message }) => { - vi.stubGlobal("window", globalThis); - vi.stubGlobal("fetch", vi.fn().mockResolvedValue({ ok, status, json: async () => body })); - const loadPackage = vi.fn(); - loadPyodide.mockResolvedValue({ - FS: { mkdirTree: vi.fn() }, - loadPackage, - }); - const { EmitterTester } = await import("./test-host.js"); - const [, diagnostics] = await EmitterTester.compileAndDiagnose( - "@service namespace Test; op read(): string;", - ); - - expectDiagnostics(diagnostics, { - code: "@typespec/http-client-python/unknown-error", - message, - }); - expect(loadPackage).not.toHaveBeenCalled(); - }); - - it.each(["rejected", "reported"])( - "reports %s wheel loading failures and retries initialization on the next emit", - async (mode) => { - vi.stubGlobal("window", globalThis); - vi.stubGlobal( - "fetch", - vi.fn().mockResolvedValue({ - ok: true, - json: async () => ({ wheels: ["pygen-0.1.0-py3-none-any.whl"] }), - }), - ); - const loadPackage = vi.fn().mockImplementationOnce((_names, options) => { - if (mode === "rejected") return Promise.reject(new Error("Wheel download failed")); - options?.errorCallback("Wheel download failed"); - return Promise.resolve([]); - }); - loadPyodide.mockResolvedValue({ - FS: { - mkdirTree: vi.fn(), - readdir: vi.fn().mockReturnValue([".", ".."]), - writeFile: vi.fn(), - }, - loadPackage, - toPy: vi.fn(), - runPythonAsync: vi.fn(), - }); - const { EmitterTester } = await import("./test-host.js"); - const source = "@service namespace Test; op read(): string;"; - const [, failure] = await EmitterTester.compileAndDiagnose(source); - expectDiagnostics(failure, { - code: "@typespec/http-client-python/unknown-error", - message: /Wheel download failed/, - }); - const [, success] = await EmitterTester.compileAndDiagnose(source); - expectDiagnostics(success, []); - expect(loadPyodide).toHaveBeenCalledTimes(2); - expect(loadPackage).toHaveBeenCalledTimes(3); - }, - ); }); diff --git a/packages/http-client-python/eng/pipeline/publish.yml b/packages/http-client-python/eng/pipeline/publish.yml index 6ea23cd309e..b7a469e9945 100644 --- a/packages/http-client-python/eng/pipeline/publish.yml +++ b/packages/http-client-python/eng/pipeline/publish.yml @@ -1,3 +1,4 @@ +# Publishes the Python emitter whenever changes under packages/http-client-python/ are merged to main. trigger: branches: include: @@ -32,4 +33,3 @@ extends: EnableCadlRanchReport: false PythonVersion: "3.15" UploadPlaygroundBundle: true - PlaygroundBundleBuildScript: "build:playground" diff --git a/packages/http-client-python/eng/scripts/setup/build.ts b/packages/http-client-python/eng/scripts/setup/build.ts index ff46e1fc1e1..71c258e68c6 100644 --- a/packages/http-client-python/eng/scripts/setup/build.ts +++ b/packages/http-client-python/eng/scripts/setup/build.ts @@ -2,7 +2,7 @@ import { exec } from "child_process"; import { runPython3 } from "./run-python3.js"; async function main() { - await runPython3("./eng/scripts/setup/build_pygen_wheel.py", ...process.argv.slice(2)); + await runPython3("./eng/scripts/setup/build_pygen_wheel.py"); // remove the venv_build_wheel directory exec("rimraf ./venv_build_wheel", (error, stdout, _stderr) => { if (error) { diff --git a/packages/http-client-python/eng/scripts/setup/build_pygen_wheel.py b/packages/http-client-python/eng/scripts/setup/build_pygen_wheel.py index 3efe543956b..40416a0991d 100644 --- a/packages/http-client-python/eng/scripts/setup/build_pygen_wheel.py +++ b/packages/http-client-python/eng/scripts/setup/build_pygen_wheel.py @@ -6,13 +6,6 @@ # license information. # -------------------------------------------------------------------------- import sys -import argparse -import json -import os -import shutil -import subprocess -import tempfile -from urllib.request import urlopen if not sys.version_info >= (3, 10, 0): raise Exception("Autorest for Python extension requires Python 3.10 at least") @@ -36,76 +29,13 @@ _ROOT_DIR = Path(__file__).parent.parent.parent.parent -def build_browser_wheels(venv_context): - dist = _ROOT_DIR / "generator" / "dist" - wheel = next(dist.glob("pygen-*.whl"), None) - if wheel is None: - raise RuntimeError(f"Generator wheel not found in {dist}. Build the generator before preparing browser wheels.") - package = json.loads((_ROOT_DIR / "package.json").read_text()) - pyodide_url = f"https://cdn.jsdelivr.net/pyodide/v{package['dependencies']['pyodide']}/full/" - with urlopen(f"{pyodide_url}pyodide-lock.json") as response: - lock = json.load(response) - - # These native extensions must match the browser's WebAssembly runtime, not the build host. - native_wheels = [lock["packages"][name]["file_name"] for name in ("pyyaml", "markupsafe")] - _, abi, platform = native_wheels[0].removesuffix(".whl").rsplit("-", 2) - with tempfile.TemporaryDirectory(dir=dist) as directory: - staging = Path(directory) - command = [ - venv_context.env_exe, - "-m", - "pip", - "download", - str(wheel), - # Later platformdirs releases import os.geteuid, which Pyodide does not provide. - "platformdirs==4.3.8", - *[pyodide_url + filename for filename in native_wheels], - f"--dest={staging}", - "--only-binary=:all:", - f"--python-version={lock['info']['python']}", - f"--platform={platform}", - "--implementation=cp", - f"--abi={abi}", - ] - env = os.environ.copy() - env["PIP_INDEX_URL"] = ( - env.get("PIP_INDEX_URL") - or env.get("UV_DEFAULT_INDEX") - or "https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-python/pypi/simple/" - ) - subprocess.run(command, env=env, check=True) - - unexpected = [ - p.name - for p in staging.glob("*.whl") - if not p.name.endswith("-none-any.whl") and p.name not in native_wheels - ] - if unexpected: - raise RuntimeError(f"Unsupported native browser dependencies: {', '.join(unexpected)}") - browser = dist / "browser" - if browser.exists(): - shutil.rmtree(browser) - browser.mkdir() - wheels = sorted(p.name for p in staging.glob("*-none-any.whl")) - for filename in wheels: - shutil.copyfile(staging / filename, browser / filename) - (browser / "manifest.json").write_text(json.dumps({"wheels": wheels}, indent=2) + "\n") - - -def main(playground=False): +def main(): venv_path = _ROOT_DIR / "venv_build_wheel" venv_context = create_venv_with_package_manager(venv_path) - # uv environments are unseeded; playground wheel downloads also need pip. - install_packages(["build", "pip==26.2.2"] if playground else ["build"], venv_context) + install_packages(["build"], venv_context) python_run(venv_context, "build", ["--wheel"], additional_dir="generator") - if playground: - build_browser_wheels(venv_context) if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument( - "--playground", action="store_true", help="Prepare browser dependency wheels for the playground" - ) - main(playground=parser.parse_args().playground) + main() diff --git a/packages/http-client-python/package.json b/packages/http-client-python/package.json index d4ba83f2902..3fc20d26a49 100644 --- a/packages/http-client-python/package.json +++ b/packages/http-client-python/package.json @@ -35,7 +35,6 @@ "scripts": { "clean": "rimraf ./dist ./temp ./venv ./tests/generated", "build": "tsc -p ./emitter/tsconfig.build.json && tsx ./eng/scripts/setup/build.ts", - "build:playground": "npm run build -- --playground", "watch": "tsc -p ./emitter/tsconfig.build.json --watch", "install": "tsx ./eng/scripts/setup/install.ts", "prepare": "tsx ./eng/scripts/setup/prepare.ts", @@ -87,14 +86,6 @@ { "path": "generator/dist/pygen-*.whl", "contentType": "application/zip" - }, - { - "path": "generator/dist/browser/*.whl", - "contentType": "application/zip" - }, - { - "path": "generator/dist/browser/manifest.json", - "contentType": "application/json" } ], "bundlePeerDependencies": [ diff --git a/packages/http-client-python/tests/unit/test_browser_wheels.py b/packages/http-client-python/tests/unit/test_browser_wheels.py deleted file mode 100644 index 7832f704e91..00000000000 --- a/packages/http-client-python/tests/unit/test_browser_wheels.py +++ /dev/null @@ -1,154 +0,0 @@ -import importlib.util -import io -import json -from pathlib import Path -from types import SimpleNamespace -import sys -import tempfile -import unittest -from unittest.mock import Mock, call, patch - -SETUP = Path(__file__).resolve().parents[2] / "eng" / "scripts" / "setup" -sys.path.insert(0, str(SETUP)) -spec = importlib.util.spec_from_file_location("build_pygen_wheel", SETUP / "build_pygen_wheel.py") -builder = importlib.util.module_from_spec(spec) -spec.loader.exec_module(builder) -sys.path.remove(str(SETUP)) - - -class BrowserWheelTests(unittest.TestCase): - def test_missing_generator_wheel_reports_build_prerequisite(self): - with tempfile.TemporaryDirectory() as directory: - dist = Path(directory) / "generator" / "dist" - (Path(directory) / "package.json").write_text(json.dumps({"dependencies": {"pyodide": "0.26.2"}})) - lock = {"packages": {"pyyaml": {}, "markupsafe": {}}} - with ( - patch.object(builder, "_ROOT_DIR", Path(directory)), - patch.object(builder, "urlopen", return_value=io.BytesIO(json.dumps(lock).encode())) as fetch, - ): - with self.assertRaisesRegex(RuntimeError, "Generator wheel not found") as error: - builder.build_browser_wheels(SimpleNamespace(env_exe="python")) - - self.assertIn(str(dist), str(error.exception)) - fetch.assert_not_called() - - def test_regular_build_skips_browser_dependencies(self): - context = SimpleNamespace(env_exe="python") - with ( - patch.object(builder, "create_venv_with_package_manager", return_value=context), - patch.object(builder, "install_packages") as install, - patch.object(builder, "python_run") as run, - patch.object(builder, "build_browser_wheels") as browser, - ): - builder.main() - - install.assert_called_once_with(["build"], context) - run.assert_called_once_with(context, "build", ["--wheel"], additional_dir="generator") - browser.assert_not_called() - - def test_playground_build_prepares_browser_dependencies(self): - context = SimpleNamespace(env_exe="python") - with ( - patch.object(builder, "create_venv_with_package_manager", return_value=context), - patch.object(builder, "install_packages") as install, - patch.object(builder, "python_run") as run, - patch.object(builder, "build_browser_wheels") as browser, - ): - builder.main(playground=True) - - install.assert_called_once_with(["build", "pip==26.2.2"], context) - run.assert_called_once_with(context, "build", ["--wheel"], additional_dir="generator") - browser.assert_called_once_with(context) - - def test_uv_playground_build_seeds_pip_before_preparing_browser_wheels(self): - with ( - tempfile.TemporaryDirectory() as directory, - patch.object(builder, "_ROOT_DIR", Path(directory)), - patch("package_manager.detect_package_manager", return_value="uv"), - patch.object(builder.subprocess, "check_call") as create, - patch.object(builder.subprocess, "run", return_value=SimpleNamespace(returncode=0)) as install, - patch.object(builder, "python_run") as build, - patch.object(builder, "build_browser_wheels") as browser, - ): - steps = Mock() - steps.attach_mock(install, "install") - steps.attach_mock(build, "build") - steps.attach_mock(browser, "browser") - - builder.main(playground=True) - - context = browser.call_args.args[0] - create.assert_called_once_with(["uv", "venv", str(Path(directory) / "venv_build_wheel")]) - self.assertEqual( - steps.mock_calls, - [ - call.install( - ["uv", "pip", "install", "--python", context.env_exe, "build", "pip==26.2.2"], - capture_output=True, - text=True, - ), - call.build(context, "build", ["--wheel"], additional_dir="generator"), - call.browser(context), - ], - ) - - def test_resolves_for_pyodide_and_publishes_only_pure_python_wheels(self): - native = [ - {"name": "PyYAML", "version": "6.0.1", "file_name": "PyYAML-6.0.1-cp312-cp312-pyodide_2024_0_wasm32.whl"}, - { - "name": "MarkupSafe", - "version": "2.1.5", - "file_name": "MarkupSafe-2.1.5-cp312-cp312-pyodide_2024_0_wasm32.whl", - }, - ] - lock = {"info": {"python": "3.12.1"}, "packages": dict(zip(("pyyaml", "markupsafe"), native))} - pure = ["pygen-0.1.0-py3-none-any.whl", "black-26.3.1-py3-none-any.whl"] - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - (root / "package.json").write_text(json.dumps({"dependencies": {"pyodide": "0.26.2"}})) - dist = root / "generator" / "dist" - dist.mkdir(parents=True) - (dist / pure[0]).touch() - browser = dist / "browser" - browser.mkdir() - (browser / "stale.whl").touch() - unexpected_native = None - - def download(command, *, env, check): - self.assertTrue(check) - self.assertIn("--python-version=3.12.1", command) - self.assertIn("--platform=pyodide_2024_0_wasm32", command) - self.assertIn("--abi=cp312", command) - self.assertEqual(env["PIP_INDEX_URL"], "https://example.com/authenticated-feed/") - self.assertIn("platformdirs==4.3.8", command) - for package in native: - self.assertIn(f"https://cdn.jsdelivr.net/pyodide/v0.26.2/full/{package['file_name']}", command) - staging = Path(next(arg.removeprefix("--dest=") for arg in command if arg.startswith("--dest="))) - for filename in pure + [p["file_name"] for p in native]: - (staging / filename).touch() - if unexpected_native: - (staging / unexpected_native).touch() - - with ( - patch.object(builder, "_ROOT_DIR", root), - patch.object(builder, "urlopen", side_effect=lambda _url: io.BytesIO(json.dumps(lock).encode())), - patch.object(builder.subprocess, "run", side_effect=download), - patch.dict(builder.os.environ, {"PIP_INDEX_URL": "https://example.com/authenticated-feed/"}), - ): - builder.build_browser_wheels(SimpleNamespace(env_exe="python")) - manifest_before_failure = (browser / "manifest.json").read_text() - unexpected_native = "unsupported-1.0-cp312-cp312-pyodide_2024_0_wasm32.whl" - with self.assertRaisesRegex( - RuntimeError, f"Unsupported native browser dependencies: {unexpected_native}" - ): - builder.build_browser_wheels(SimpleNamespace(env_exe="python")) - self.assertEqual((browser / "manifest.json").read_text(), manifest_before_failure) - - manifest = json.loads((dist / "browser" / "manifest.json").read_text()) - self.assertEqual(manifest["wheels"], sorted(pure)) - self.assertEqual(set(manifest), {"wheels"}) - self.assertEqual(sorted(p.name for p in (dist / "browser").glob("*.whl")), sorted(pure)) - - -if __name__ == "__main__": - unittest.main()