Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
---
changeKind: fix
packages:
- "@typespec/http-client-python"
---

Fix playground generation by bundling the generator's Python dependencies with the emitter instead of resolving them from PyPI in the browser. The playground build obtains hash-pinned dependencies from the configured Python feed and builds pure-Python PyYAML and MarkupSafe wheels without downloading from the Pyodide CDN. Update the generator's PyYAML dependency to 6.0.3.
1 change: 1 addition & 0 deletions cspell.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -245,6 +245,7 @@ words:
- pyright
- pyrightconfig
- pytest
- pytokens
- pytyped
- pyyaml
- rcfile
Expand Down
21 changes: 21 additions & 0 deletions packages/http-client-python/ARCHITECTURE.md
Original file line number Diff line number Diff line change
Expand Up @@ -117,6 +117,27 @@ The emitter bridges TypeSpec and Python code generation.
5. Generate intermediate YAML representation
6. Invoke Python generator with YAML input

#### Browser runtime

The playground loads the generator and all of its Python dependencies, including
pure-Python builds of PyYAML and MarkupSafe, from the emitter's published
`generator/dist/browser/manifest.json` and wheel assets using Pyodide's
`loadPackage`. Browser generation does not query PyPI, CFS, or the Pyodide package
CDN. Only the Pyodide core runtime and the emitter asset host must be reachable.

`npm run build:playground` downloads the exact artifacts and SHA-256 hashes listed
in `eng/playground-requirements.txt` from `PIP_INDEX_URL` or `UV_DEFAULT_INDEX`
(default: the public `azure-sdk-for-python` feed). PyYAML and MarkupSafe are
built from their source distributions without their optional native extensions.
The build rejects native wheels and checks the dependency set with pip before
publishing it. The browser-only `platformdirs` pin avoids its unsupported
`os.geteuid` API. Update the requirements and hashes when generator dependencies
change. Ordinary `npm run build` skips browser-wheel preparation.

Publish the browser manifest and wheels together with the emitter bundle under
the same package version. Previously published bundles do not pick up this fix
until a new version is uploaded and the playground's latest package index is updated.

### 2. Generator (Python)

**Location**: `packages/http-client-python/generator/pygen/`
Expand Down
45 changes: 34 additions & 11 deletions packages/http-client-python/emitter/src/emitter.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,7 @@ import type { EmitContext } from "@typespec/compiler";
import { emitFile, joinPaths, listServices, NoTarget } from "@typespec/compiler";
import pkgJson from "../../package.json" with { type: "json" };
import { emitCodeModel } from "./code-model.js";
import {
BLOB_STORAGE_BASE_URL,
PACKAGE_NAME,
PYGEN_WHEEL_FILENAME,
PYODIDE_VERSION,
} from "./constants.js";
import { BLOB_STORAGE_BASE_URL, PACKAGE_NAME, PYODIDE_VERSION } from "./constants.js";
import type { PythonEmitterOptions, PythonSdkContext } from "./lib.js";
import { reportDiagnostic } from "./lib.js";
import { runNodeEmit } from "./node-runner.js";
Expand All @@ -17,8 +12,21 @@ import { loadPyodide } from "./pyodide-loader.js";
import { getRootNamespace, md2Rst } from "./utils.js";
import { dumpCodeModelToYaml } from "./yaml-utils.js";

function getBrowserPygenWheelUrl(): string {
return `${BLOB_STORAGE_BASE_URL}/${PACKAGE_NAME}/${pkgJson.version}/generator/dist/${PYGEN_WHEEL_FILENAME}`;
function getBrowserAssetUrl(filename: string): string {
return `${BLOB_STORAGE_BASE_URL}/${PACKAGE_NAME}/${pkgJson.version}/generator/dist/browser/${filename}`;
}

function isBrowserWheelManifest(manifest: unknown): manifest is { wheels: string[] } {
return (
typeof manifest === "object" &&
manifest !== null &&
"wheels" in manifest &&
Array.isArray(manifest.wheels) &&
manifest.wheels.length > 0 &&
manifest.wheels.every(
(name: unknown) => typeof name === "string" && /^[\w.+-]+\.whl$/.test(name),
)
);
}

function addDefaultOptions(sdkContext: PythonSdkContext) {
Expand Down Expand Up @@ -306,9 +314,24 @@ async function setupPyodideCallBrowser() {

// use default MEMFS for browser, since NODEFS is not supported
pyodide.FS.mkdirTree("/generator");
await pyodide.loadPackage("micropip");
const micropip = pyodide.pyimport("micropip");
await micropip.install(getBrowserPygenWheelUrl());
const response = await fetch(getBrowserAssetUrl("manifest.json"));
if (!response.ok) {
throw new Error(`Failed to load Python browser wheel manifest: HTTP ${response.status}`);
}
const manifest: unknown = await response.json();
if (!isBrowserWheelManifest(manifest)) {
throw new Error("Invalid Python browser wheel manifest.");
}
await loadBrowserPackages(pyodide, manifest.wheels.map(getBrowserAssetUrl));

return pyodide;
}

async function loadBrowserPackages(pyodide: PyodideInterface, packages: string[]): Promise<void> {
const errors: string[] = [];
// loadPackage can report failed downloads through its callback without rejecting.
await pyodide.loadPackage(packages, { errorCallback: (message) => errors.push(message) });
if (errors.length > 0) {
throw new Error(`Failed to load Python browser packages: ${errors.join("\n")}`);
}
}
140 changes: 139 additions & 1 deletion packages/http-client-python/emitter/test/browser-pyodide.test.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import { expectDiagnostics } from "@typespec/compiler/testing";
import { strictEqual } from "assert";
import { afterEach, describe, it, vi } from "vitest";
import { afterEach, describe, expect, it, vi } from "vitest";

const loadPyodide = vi.hoisted(() => vi.fn());

Expand All @@ -8,6 +9,7 @@ vi.mock("../src/pyodide-loader.js", () => ({ loadPyodide }));
describe("typespec-python: browser pyodide bootstrap", () => {
afterEach(() => {
delete (globalThis as any).window;
vi.unstubAllGlobals();
loadPyodide.mockReset();
vi.resetModules();
});
Expand All @@ -22,4 +24,140 @@ describe("typespec-python: browser pyodide bootstrap", () => {

strictEqual(loadPyodide.mock.calls.length, 0);
});

it("installs all published dependency wheels from emitter assets", async () => {
vi.stubGlobal("window", globalThis);
const wheels = [
"pygen-0.1.0-py3-none-any.whl",
"pyyaml-6.0.3-py3-none-any.whl",
"markupsafe-2.1.5-py3-none-any.whl",
];
const fetch = vi.fn().mockResolvedValue({
ok: true,
json: async () => ({ wheels }),
});
vi.stubGlobal("fetch", fetch);
const pyodide = {
FS: {
mkdirTree: vi.fn(),
readdir: vi.fn().mockReturnValue([".", ".."]),
writeFile: vi.fn(),
},
loadPackage: vi.fn(),
toPy: vi.fn(),
runPythonAsync: vi.fn(),
};
loadPyodide.mockResolvedValue(pyodide);
const { EmitterTester } = await import("./test-host.js");
const [, diagnostics] = await EmitterTester.compileAndDiagnose(`
import "@typespec/http";
using TypeSpec.Http;
@service namespace Test;
@get op read(): string;
`);

expectDiagnostics(diagnostics, []);
expect(fetch).toHaveBeenCalledWith(
expect.stringMatching(/\/generator\/dist\/browser\/manifest\.json$/),
);
expect(pyodide.loadPackage).toHaveBeenCalledExactlyOnceWith(
wheels.map((name) => expect.stringMatching(new RegExp(`/browser/${name}$`))),
{ errorCallback: expect.any(Function) },
);
expect(pyodide.runPythonAsync).toHaveBeenCalledOnce();
});

it.each([
{ ok: false, status: 404, body: undefined, message: /HTTP 404/ },
{ ok: true, status: 200, body: null, message: /Invalid Python browser wheel/ },
{ ok: true, status: 200, body: {}, message: /Invalid Python browser wheel/ },
{
ok: true,
status: 200,
body: { wheels: [] },
message: /Invalid Python browser wheel/,
},
{
ok: true,
status: 200,
body: { wheels: "not-an-array" },
message: /Invalid Python browser wheel/,
},
{
ok: true,
status: 200,
body: { wheels: [null] },
message: /Invalid Python browser wheel/,
},
{
ok: true,
status: 200,
body: { wheels: [42] },
message: /Invalid Python browser wheel/,
},
{
ok: true,
status: 200,
body: { wheels: ["../unexpected.whl"] },
message: /Invalid Python browser wheel/,
},
])("reports unusable browser wheel manifests: $body", async ({ ok, status, body, message }) => {
vi.stubGlobal("window", globalThis);
vi.stubGlobal("fetch", vi.fn().mockResolvedValue({ ok, status, json: async () => body }));
const loadPackage = vi.fn();
loadPyodide.mockResolvedValue({
FS: { mkdirTree: vi.fn() },
loadPackage,
});
const { EmitterTester } = await import("./test-host.js");
const [, diagnostics] = await EmitterTester.compileAndDiagnose(
"@service namespace Test; op read(): string;",
);

expectDiagnostics(diagnostics, {
code: "@typespec/http-client-python/unknown-error",
message,
});
expect(loadPackage).not.toHaveBeenCalled();
});

it.each(["rejected", "reported"])(
"reports %s wheel loading failures and retries initialization on the next emit",
async (mode) => {
vi.stubGlobal("window", globalThis);
vi.stubGlobal(
"fetch",
vi.fn().mockResolvedValue({
ok: true,
json: async () => ({ wheels: ["pygen-0.1.0-py3-none-any.whl"] }),
}),
);
const loadPackage = vi.fn().mockImplementationOnce((_names, options) => {
if (mode === "rejected") return Promise.reject(new Error("Wheel download failed"));
options?.errorCallback("Wheel download failed");
return Promise.resolve([]);
});
loadPyodide.mockResolvedValue({
FS: {
mkdirTree: vi.fn(),
readdir: vi.fn().mockReturnValue([".", ".."]),
writeFile: vi.fn(),
},
loadPackage,
toPy: vi.fn(),
runPythonAsync: vi.fn(),
});
const { EmitterTester } = await import("./test-host.js");
const source = "@service namespace Test; op read(): string;";
const [, failure] = await EmitterTester.compileAndDiagnose(source);
expectDiagnostics(failure, {
code: "@typespec/http-client-python/unknown-error",
message: /Wheel download failed/,
});
const [, success] = await EmitterTester.compileAndDiagnose(source);
expectDiagnostics(success, []);
expect(loadPyodide).toHaveBeenCalledTimes(2);
expect(loadPackage).toHaveBeenCalledTimes(2);
},
);
});
1 change: 1 addition & 0 deletions packages/http-client-python/eng/pipeline/publish.yml
Original file line number Diff line number Diff line change
Expand Up @@ -32,3 +32,4 @@ extends:
EnableCadlRanchReport: false
PythonVersion: "3.15"
UploadPlaygroundBundle: true
PlaygroundBundleBuildScript: "build:playground"
13 changes: 13 additions & 0 deletions packages/http-client-python/eng/playground-requirements.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
black==26.3.1 --hash=sha256:2bd5aa94fc267d38bb21a70d7410a89f1a1d318841855f698746f8e7f51acd1b
click==8.5.1 --hash=sha256:8bb5376d2006950386f8c6427613334a2acadc5f559cb80e638d00b4df957e37
docutils==0.23 --hash=sha256:25d013af9bf23bc1c7b2b093dff4208166c53a94786c9e447808335ef1185fea
Jinja2==3.1.6 --hash=sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67
MarkupSafe==2.1.5 --hash=sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b
mypy-extensions==1.2.0 --hash=sha256:993e7a20aa5671eb41a7132cf555af30062da99729601601b9a8382b11cae768
packaging==26.3 --hash=sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c
pathspec==1.1.1 --hash=sha256:a00ce642f577bf7f473932318056212bc4f8bfdf53128c78bbd5af0b9b20b189
platformdirs==4.3.8 --hash=sha256:ff7059bb7eb1179e2685604f4aaf157cfd9535242bd23742eadc3c13542139b4
pytokens==0.4.1 --hash=sha256:26cef14744a8385f35d0e095dc8b3a7583f6c953c2e3d269c7f82484bf5ad2de
PyYAML==6.0.3 --hash=sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f
setuptools==83.0.0 --hash=sha256:29b23c360f22f414dc7336bb39178cc7bcbf6021ed2733cde173f09dba19abb3
tomli==2.0.1 --hash=sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc
2 changes: 1 addition & 1 deletion packages/http-client-python/eng/scripts/setup/build.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ import { exec } from "child_process";
import { runPython3 } from "./run-python3.js";

async function main() {
await runPython3("./eng/scripts/setup/build_pygen_wheel.py");
await runPython3("./eng/scripts/setup/build_pygen_wheel.py", ...process.argv.slice(2));
// remove the venv_build_wheel directory
exec("rimraf ./venv_build_wheel", (error, stdout, _stderr) => {
if (error) {
Expand Down
Loading
Loading