Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions .chronus/changes/python-playground-browser-wheels.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
---
changeKind: fix
packages:
- "@typespec/http-client-python"
---

Fix Python playground generation when package downloads are blocked on corporate networks. Publish browser-compatible dependency wheels with the emitter instead of querying PyPI or CFS at runtime. Prepare these assets only for playground builds.
21 changes: 21 additions & 0 deletions packages/http-client-python/ARCHITECTURE.md
Original file line number Diff line number Diff line change
Expand Up @@ -117,6 +117,27 @@ The emitter bridges TypeSpec and Python code generation.
5. Generate intermediate YAML representation
6. Invoke Python generator with YAML input

#### Browser runtime

The playground loads the generator and its pure-Python dependencies from the
emitter's published `generator/dist/browser/manifest.json` and wheel assets using
Pyodide's `loadPackage`. PyYAML and MarkupSafe come from the existing Pyodide CDN.
Browser generation does not query PyPI or CFS, avoiding blocked package downloads
on corporate networks. The Pyodide CDN and emitter asset host must still be reachable.

`npm run build:playground` resolves these wheels for Pyodide's Python version and
platform, not the build machine. The playground publication pipeline runs this
command; ordinary `npm run build` skips browser-wheel preparation.
Playground builds also provision a pinned pip in the temporary build environment,
including unseeded environments created by uv.
Resolution honors `PIP_INDEX_URL` or `UV_DEFAULT_INDEX`, defaulting to the public
`azure-sdk-for-python` CFS feed. Native dependencies match the Pyodide lockfile;
the browser-only `platformdirs` pin avoids its unsupported `os.geteuid` API.

Publish the browser manifest and wheels together with the emitter bundle under
the same package version. Previously published bundles do not pick up this fix
until a new version is uploaded and the playground's latest package index is updated.

### 2. Generator (Python)

**Location**: `packages/http-client-python/generator/pygen/`
Expand Down
46 changes: 35 additions & 11 deletions packages/http-client-python/emitter/src/emitter.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,7 @@ import type { EmitContext } from "@typespec/compiler";
import { emitFile, joinPaths, listServices, NoTarget } from "@typespec/compiler";
import pkgJson from "../../package.json" with { type: "json" };
import { emitCodeModel } from "./code-model.js";
import {
BLOB_STORAGE_BASE_URL,
PACKAGE_NAME,
PYGEN_WHEEL_FILENAME,
PYODIDE_VERSION,
} from "./constants.js";
import { BLOB_STORAGE_BASE_URL, PACKAGE_NAME, PYODIDE_VERSION } from "./constants.js";
import type { PythonEmitterOptions, PythonSdkContext } from "./lib.js";
import { reportDiagnostic } from "./lib.js";
import { runNodeEmit } from "./node-runner.js";
Expand All @@ -17,8 +12,21 @@ import { loadPyodide } from "./pyodide-loader.js";
import { getRootNamespace, md2Rst } from "./utils.js";
import { dumpCodeModelToYaml } from "./yaml-utils.js";

function getBrowserPygenWheelUrl(): string {
return `${BLOB_STORAGE_BASE_URL}/${PACKAGE_NAME}/${pkgJson.version}/generator/dist/${PYGEN_WHEEL_FILENAME}`;
function getBrowserAssetUrl(filename: string): string {
return `${BLOB_STORAGE_BASE_URL}/${PACKAGE_NAME}/${pkgJson.version}/generator/dist/browser/${filename}`;
}

function isBrowserWheelManifest(manifest: unknown): manifest is { wheels: string[] } {
return (
typeof manifest === "object" &&
manifest !== null &&
"wheels" in manifest &&
Array.isArray(manifest.wheels) &&
manifest.wheels.length > 0 &&
manifest.wheels.every(
(name: unknown) => typeof name === "string" && /^[\w.+-]+\.whl$/.test(name),
)
);
}

function addDefaultOptions(sdkContext: PythonSdkContext) {
Expand Down Expand Up @@ -306,9 +314,25 @@ async function setupPyodideCallBrowser() {

// use default MEMFS for browser, since NODEFS is not supported
pyodide.FS.mkdirTree("/generator");
await pyodide.loadPackage("micropip");
const micropip = pyodide.pyimport("micropip");
await micropip.install(getBrowserPygenWheelUrl());
const response = await fetch(getBrowserAssetUrl("manifest.json"));
if (!response.ok) {
throw new Error(`Failed to load Python browser wheel manifest: HTTP ${response.status}`);
}
const manifest: unknown = await response.json();
if (!isBrowserWheelManifest(manifest)) {
throw new Error("Invalid Python browser wheel manifest.");
}
await loadBrowserPackages(pyodide, manifest.wheels.map(getBrowserAssetUrl));
await loadBrowserPackages(pyodide, ["pyyaml", "markupsafe"]);

return pyodide;
}

async function loadBrowserPackages(pyodide: PyodideInterface, packages: string[]): Promise<void> {
const errors: string[] = [];
// loadPackage can report failed downloads through its callback without rejecting.
await pyodide.loadPackage(packages, { errorCallback: (message) => errors.push(message) });
if (errors.length > 0) {
throw new Error(`Failed to load Python browser packages: ${errors.join("\n")}`);
}
}
140 changes: 139 additions & 1 deletion packages/http-client-python/emitter/test/browser-pyodide.test.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import { expectDiagnostics } from "@typespec/compiler/testing";
import { strictEqual } from "assert";
import { afterEach, describe, it, vi } from "vitest";
import { afterEach, describe, expect, it, vi } from "vitest";

const loadPyodide = vi.hoisted(() => vi.fn());

Expand All @@ -8,6 +9,7 @@ vi.mock("../src/pyodide-loader.js", () => ({ loadPyodide }));
describe("typespec-python: browser pyodide bootstrap", () => {
afterEach(() => {
delete (globalThis as any).window;
vi.unstubAllGlobals();
loadPyodide.mockReset();
vi.resetModules();
});
Expand All @@ -22,4 +24,140 @@ describe("typespec-python: browser pyodide bootstrap", () => {

strictEqual(loadPyodide.mock.calls.length, 0);
});

it("installs published dependency wheels without querying a package index", async () => {
vi.stubGlobal("window", globalThis);
const wheels = ["pygen-0.1.0-py3-none-any.whl", "black-26.3.1-py3-none-any.whl"];
const fetch = vi.fn().mockResolvedValue({
ok: true,
json: async () => ({ wheels }),
});
vi.stubGlobal("fetch", fetch);
const pyodide = {
FS: {
mkdirTree: vi.fn(),
readdir: vi.fn().mockReturnValue([".", ".."]),
writeFile: vi.fn(),
},
loadPackage: vi.fn(),
toPy: vi.fn(),
runPythonAsync: vi.fn(),
};
loadPyodide.mockResolvedValue(pyodide);
const { EmitterTester } = await import("./test-host.js");
const [, diagnostics] = await EmitterTester.compileAndDiagnose(`
import "@typespec/http";
using TypeSpec.Http;
@service namespace Test;
@get op read(): string;
`);

expectDiagnostics(diagnostics, []);
expect(fetch).toHaveBeenCalledWith(
expect.stringMatching(/\/generator\/dist\/browser\/manifest\.json$/),
);
expect(pyodide.loadPackage).toHaveBeenNthCalledWith(
1,
wheels.map((name) => expect.stringMatching(new RegExp(`/browser/${name}$`))),
{ errorCallback: expect.any(Function) },
);
expect(pyodide.loadPackage).toHaveBeenNthCalledWith(2, ["pyyaml", "markupsafe"], {
errorCallback: expect.any(Function),
});
expect(pyodide.runPythonAsync).toHaveBeenCalledOnce();
});

it.each([
{ ok: false, status: 404, body: undefined, message: /HTTP 404/ },
{ ok: true, status: 200, body: null, message: /Invalid Python browser wheel/ },
{ ok: true, status: 200, body: {}, message: /Invalid Python browser wheel/ },
{
ok: true,
status: 200,
body: { wheels: [] },
message: /Invalid Python browser wheel/,
},
{
ok: true,
status: 200,
body: { wheels: "not-an-array" },
message: /Invalid Python browser wheel/,
},
{
ok: true,
status: 200,
body: { wheels: [null] },
message: /Invalid Python browser wheel/,
},
{
ok: true,
status: 200,
body: { wheels: [42] },
message: /Invalid Python browser wheel/,
},
{
ok: true,
status: 200,
body: { wheels: ["../unexpected.whl"] },
message: /Invalid Python browser wheel/,
},
])("reports unusable browser wheel manifests: $body", async ({ ok, status, body, message }) => {
vi.stubGlobal("window", globalThis);
vi.stubGlobal("fetch", vi.fn().mockResolvedValue({ ok, status, json: async () => body }));
const loadPackage = vi.fn();
loadPyodide.mockResolvedValue({
FS: { mkdirTree: vi.fn() },
loadPackage,
});
const { EmitterTester } = await import("./test-host.js");
const [, diagnostics] = await EmitterTester.compileAndDiagnose(
"@service namespace Test; op read(): string;",
);

expectDiagnostics(diagnostics, {
code: "@typespec/http-client-python/unknown-error",
message,
});
expect(loadPackage).not.toHaveBeenCalled();
});

it.each(["rejected", "reported"])(
"reports %s wheel loading failures and retries initialization on the next emit",
async (mode) => {
vi.stubGlobal("window", globalThis);
vi.stubGlobal(
"fetch",
vi.fn().mockResolvedValue({
ok: true,
json: async () => ({ wheels: ["pygen-0.1.0-py3-none-any.whl"] }),
}),
);
const loadPackage = vi.fn().mockImplementationOnce((_names, options) => {
if (mode === "rejected") return Promise.reject(new Error("Wheel download failed"));
options?.errorCallback("Wheel download failed");
return Promise.resolve([]);
});
loadPyodide.mockResolvedValue({
FS: {
mkdirTree: vi.fn(),
readdir: vi.fn().mockReturnValue([".", ".."]),
writeFile: vi.fn(),
},
loadPackage,
toPy: vi.fn(),
runPythonAsync: vi.fn(),
});
const { EmitterTester } = await import("./test-host.js");
const source = "@service namespace Test; op read(): string;";
const [, failure] = await EmitterTester.compileAndDiagnose(source);
expectDiagnostics(failure, {
code: "@typespec/http-client-python/unknown-error",
message: /Wheel download failed/,
});
const [, success] = await EmitterTester.compileAndDiagnose(source);
expectDiagnostics(success, []);
expect(loadPyodide).toHaveBeenCalledTimes(2);
expect(loadPackage).toHaveBeenCalledTimes(3);
},
);
});
1 change: 1 addition & 0 deletions packages/http-client-python/eng/pipeline/publish.yml
Original file line number Diff line number Diff line change
Expand Up @@ -32,3 +32,4 @@ extends:
EnableCadlRanchReport: false
PythonVersion: "3.15"
UploadPlaygroundBundle: true
PlaygroundBundleBuildScript: "build:playground"
2 changes: 1 addition & 1 deletion packages/http-client-python/eng/scripts/setup/build.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ import { exec } from "child_process";
import { runPython3 } from "./run-python3.js";

async function main() {
await runPython3("./eng/scripts/setup/build_pygen_wheel.py");
await runPython3("./eng/scripts/setup/build_pygen_wheel.py", ...process.argv.slice(2));
// remove the venv_build_wheel directory
exec("rimraf ./venv_build_wheel", (error, stdout, _stderr) => {
if (error) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,13 @@
# license information.
# --------------------------------------------------------------------------
import sys
import argparse
import json
import os
import shutil
import subprocess
import tempfile
from urllib.request import urlopen

if not sys.version_info >= (3, 10, 0):
raise Exception("Autorest for Python extension requires Python 3.10 at least")
Expand All @@ -29,13 +36,76 @@
_ROOT_DIR = Path(__file__).parent.parent.parent.parent


def main():
def build_browser_wheels(venv_context):
dist = _ROOT_DIR / "generator" / "dist"
wheel = next(dist.glob("pygen-*.whl"), None)
if wheel is None:
raise RuntimeError(f"Generator wheel not found in {dist}. Build the generator before preparing browser wheels.")
package = json.loads((_ROOT_DIR / "package.json").read_text())
pyodide_url = f"https://cdn.jsdelivr.net/pyodide/v{package['dependencies']['pyodide']}/full/"
with urlopen(f"{pyodide_url}pyodide-lock.json") as response:
lock = json.load(response)

# These native extensions must match the browser's WebAssembly runtime, not the build host.
native_wheels = [lock["packages"][name]["file_name"] for name in ("pyyaml", "markupsafe")]
_, abi, platform = native_wheels[0].removesuffix(".whl").rsplit("-", 2)
with tempfile.TemporaryDirectory(dir=dist) as directory:
staging = Path(directory)
command = [
venv_context.env_exe,
"-m",
"pip",
"download",
Comment thread
l0lawrence marked this conversation as resolved.
Resolved
str(wheel),
# Later platformdirs releases import os.geteuid, which Pyodide does not provide.
"platformdirs==4.3.8",
*[pyodide_url + filename for filename in native_wheels],
f"--dest={staging}",
"--only-binary=:all:",
f"--python-version={lock['info']['python']}",
f"--platform={platform}",
"--implementation=cp",
f"--abi={abi}",
]
env = os.environ.copy()
env["PIP_INDEX_URL"] = (
env.get("PIP_INDEX_URL")
or env.get("UV_DEFAULT_INDEX")
or "https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-python/pypi/simple/"
)
subprocess.run(command, env=env, check=True)

unexpected = [
p.name
for p in staging.glob("*.whl")
if not p.name.endswith("-none-any.whl") and p.name not in native_wheels
]
if unexpected:
raise RuntimeError(f"Unsupported native browser dependencies: {', '.join(unexpected)}")
browser = dist / "browser"
if browser.exists():
shutil.rmtree(browser)
browser.mkdir()
wheels = sorted(p.name for p in staging.glob("*-none-any.whl"))
for filename in wheels:
shutil.copyfile(staging / filename, browser / filename)
(browser / "manifest.json").write_text(json.dumps({"wheels": wheels}, indent=2) + "\n")


def main(playground=False):
venv_path = _ROOT_DIR / "venv_build_wheel"
venv_context = create_venv_with_package_manager(venv_path)

install_packages(["build"], venv_context)
# uv environments are unseeded; playground wheel downloads also need pip.
install_packages(["build", "pip==26.2.2"] if playground else ["build"], venv_context)
python_run(venv_context, "build", ["--wheel"], additional_dir="generator")
if playground:
build_browser_wheels(venv_context)


if __name__ == "__main__":
main()
parser = argparse.ArgumentParser()
parser.add_argument(
"--playground", action="store_true", help="Prepare browser dependency wheels for the playground"
)
main(playground=parser.parse_args().playground)
9 changes: 9 additions & 0 deletions packages/http-client-python/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
"scripts": {
"clean": "rimraf ./dist ./temp ./venv ./tests/generated",
"build": "tsc -p ./emitter/tsconfig.build.json && tsx ./eng/scripts/setup/build.ts",
"build:playground": "npm run build -- --playground",
"watch": "tsc -p ./emitter/tsconfig.build.json --watch",
"install": "tsx ./eng/scripts/setup/install.ts",
"prepare": "tsx ./eng/scripts/setup/prepare.ts",
Expand Down Expand Up @@ -86,6 +87,14 @@
{
"path": "generator/dist/pygen-*.whl",
"contentType": "application/zip"
},
{
"path": "generator/dist/browser/*.whl",
"contentType": "application/zip"
},
{
"path": "generator/dist/browser/manifest.json",
"contentType": "application/json"
}
],
"bundlePeerDependencies": [
Expand Down
Loading
Loading