Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions .chronus/changes/bundler-wheel-assets.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
---
changeKind: feature
packages:
- "@typespec/bundler"
- "@typespec/bundle-uploader"
---

Include imported Python wheels as binary assets alongside browser bundles, including assets imported by bundled dependencies. The Vite plugin and bundle uploader serve and publish these assets with the JavaScript.

```js
import wheel from "./generator.whl";

export const wheelUrl = new URL(wheel, import.meta.url).href;
```
7 changes: 7 additions & 0 deletions .chronus/changes/python-release-browser-wheel.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
---
changeKind: fix
packages:
- "@typespec/http-client-python"
---

Load the browser generator wheel relative to the emitter's JavaScript bundle, so local, PR-preview, and published playgrounds can ship the matching wheel alongside it without version-specific download hosts.
20 changes: 14 additions & 6 deletions packages/bundle-uploader/src/upload-browser-package.ts
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ export class TypeSpecBundledPackageUploader {
});
}

async upload({ manifest, files }: TypeSpecBundle): Promise<UploadBundleResult> {
async upload({ manifest, files, assets }: TypeSpecBundle): Promise<UploadBundleResult> {
const imports = Object.fromEntries(
Object.entries(manifest.imports).map(([key, value]) => {
return [
Expand All @@ -43,13 +43,21 @@ export class TypeSpecBundledPackageUploader {
}),
);
const created = await this.#uploadManifest(manifest);
if (!created) {
return { status: "already-exists", imports };
let uploaded = created;
if (created) {
for (const file of files) {
await this.#uploadJsFile(manifest.name, manifest.version, file);
}
}
for (const file of files) {
await this.#uploadJsFile(manifest.name, manifest.version, file);
for (const asset of assets ?? []) {
const result = await this.uploadBinaryAsset(
join(manifest.name, manifest.version, asset.filename),
Buffer.from(asset.content),
asset.contentType,
);
uploaded ||= result.status === "uploaded";
}
return { status: "uploaded", imports };
return { status: uploaded ? "uploaded" : "already-exists", imports };
}

async getIndex(name: string, version: string): Promise<PackageIndex | undefined> {
Expand Down
57 changes: 43 additions & 14 deletions packages/bundler/src/bundler.ts
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,9 @@ export interface TypeSpecBundle {
*/
files: TypeSpecBundleFile[];

/** Binary assets imported by the browser modules, served alongside {@link files}. */
assets?: TypeSpecBundleAsset[];

/**
* Package identity and relative import paths derived from {@link definition}.
* Not included as a file in {@link files}.
Expand All @@ -91,6 +94,14 @@ export interface TypeSpecBundleFile {
gzipContent?: Buffer;
}

/** A binary dependency emitted alongside a browser bundle. */
export interface TypeSpecBundleAsset {
/** Output path relative to the bundle directory. */
filename: string;
content: Uint8Array;
contentType: string;
}

/** Source package metadata exposed through {@link TypeSpecBundleDefinition.packageJson}. */
interface PackageJson {
/** Package identifier used as the bundle manifest's name. */
Expand Down Expand Up @@ -196,6 +207,11 @@ export async function bundleTypeSpecLibrary(
await writeFile(joinPaths(outputDir, file.filename + ".gz"), file.gzipContent);
}
}
for (const asset of bundle.assets ?? []) {
const path = joinPaths(outputDir, asset.filename);
await mkdir(dirname(path), { recursive: true });
await writeFile(path, asset.content);
}
const manifest = createManifest(bundle.definition);
await writeFile(joinPaths(outputDir, "manifest.json"), JSON.stringify(manifest, null, 2));
if (options?.gzip) {
Expand Down Expand Up @@ -355,6 +371,9 @@ async function createEsBuildContext(
outdir: "out",
platform: "browser",
format: "esm",
// Wheel installers require the original filename; put content hashes in directories.
assetNames: "assets/[hash]/[name]",
loader: { ".whl": "file" },
target: "es2024",
minify,
keepNames: minify,
Expand All @@ -373,27 +392,37 @@ async function resolveTypeSpecBundle(
const shouldGzip = options?.gzip ?? false;

const files: TypeSpecBundleFile[] = await Promise.all(
result.outputFiles!.map(async (file) => {
const entry = definition.exports[basename(file.path)];
const content = file.text;
const bundleFile: TypeSpecBundleFile = {
filename: file.path.replaceAll("\\", "/").split("/out/")[1],
content,
export: entry ? getExportEntryPoint(entry) : undefined,
};

if (shouldGzip) {
bundleFile.gzipContent = await gzipAsync(Buffer.from(content, "utf-8"));
}
result
.outputFiles!.filter((file) => !file.path.endsWith(".whl"))
.map(async (file) => {
const entry = definition.exports[basename(file.path)];
const content = file.text;
const bundleFile: TypeSpecBundleFile = {
filename: file.path.replaceAll("\\", "/").split("/out/")[1],
content,
export: entry ? getExportEntryPoint(entry) : undefined,
};

return bundleFile;
}),
if (shouldGzip) {
bundleFile.gzipContent = await gzipAsync(Buffer.from(content, "utf-8"));
}

return bundleFile;
}),
);
const assets: TypeSpecBundleAsset[] = result
.outputFiles!.filter((file) => file.path.endsWith(".whl"))
.map((file) => ({
filename: file.path.replaceAll("\\", "/").split("/out/")[1],
content: file.contents,
contentType: "application/zip",
}));

return {
definition,
manifest: createManifest(definition),
files,
assets,
};
}

Expand Down
1 change: 1 addition & 0 deletions packages/bundler/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ export {
type BundleManifest,
type CreateTypeSpecBundleOptions,
type TypeSpecBundle,
type TypeSpecBundleAsset,
type TypeSpecBundleDefinition,
type TypeSpecBundleFile,
} from "./bundler.js";
30 changes: 18 additions & 12 deletions packages/bundler/src/vite/vite-plugin.ts
Original file line number Diff line number Diff line change
Expand Up @@ -62,32 +62,25 @@ export function typespecBundlePlugin(options: TypeSpecBundlePluginOptions): Plug
},
async configureServer(server) {
server.middlewares.use((req, res, next) => {
const id = req.url;
const id = req.url?.split("?")[0];
if (id === undefined) {
next();
return;
}
const start = `/${options.folderName}/`;

const resolveFilename = (path: string) => {
if (path === "") {
return "index.js";
} else {
return `${path}.js`;
}
};
const findPkgName = (id: string): [string, string] | undefined => {
const segments = id.slice(start.length, -".js".length).split("/");
const segments = id.slice(start.length).split("/");
if (bundles[segments[0]]) {
return [segments[0], resolveFilename(segments.slice(1).join("/"))];
return [segments[0], segments.slice(1).join("/") || "index.js"];
}
const inFolder = segments[0] + "/" + segments[1];
if (bundles[inFolder]) {
return [inFolder, resolveFilename(segments.slice(2).join("/"))];
return [inFolder, segments.slice(2).join("/") || "index.js"];
}
return undefined;
};
if (id.startsWith(start) && id.endsWith(".js")) {
if (id.startsWith(start)) {
const found = findPkgName(id);
if (found) {
const [pkgId, path] = found;
Expand All @@ -98,6 +91,12 @@ export function typespecBundlePlugin(options: TypeSpecBundlePluginOptions): Plug
res.end();
return;
}
const asset = bundles[pkgId].assets?.find((x) => x.filename === path);
if (asset) {
res.writeHead(200, "Ok", { "Content-Type": asset.contentType });
res.end(asset.content);
return;
}
}
}
next();
Expand Down Expand Up @@ -127,6 +126,13 @@ export function typespecBundlePlugin(options: TypeSpecBundlePluginOptions): Plug
source: file.content,
});
}
for (const asset of bundles[name].assets ?? []) {
this.emitFile({
type: "asset",
fileName: `${options.folderName}/${name}/${asset.filename}`,
source: asset.content,
});
}
}
},

Expand Down
59 changes: 56 additions & 3 deletions packages/bundler/test/test.test.ts
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
import { mkdir, mkdtemp, rm, writeFile } from "fs/promises";
import { mkdir, mkdtemp, readFile, readdir, rm, writeFile } from "fs/promises";
import { tmpdir } from "os";
import { join } from "path";
import { basename, join } from "path";
import { fileURLToPath, pathToFileURL } from "url";
import { expect, it } from "vitest";
import { createTypeSpecBundle } from "../src/bundler.js";
import { bundleTypeSpecLibrary, createTypeSpecBundle } from "../src/bundler.js";

/**
* Regression test: TypeSpec decorator functions are identified by their `.name` property
Expand Down Expand Up @@ -104,3 +105,55 @@ it("includes typespec source files from sub-exports", async () => {
await rm(tmpDir, { recursive: true });
}
});

it("keeps a dependency's binary asset next to its bundle with a relative URL", async () => {
const tmpDir = await mkdtemp(join(tmpdir(), "typespec-bundler-test-"));
try {
const dependency = join(tmpDir, "node_modules", "test-generator");
await mkdir(dependency, { recursive: true });
await writeFile(
join(tmpDir, "package.json"),
JSON.stringify({
name: "test-wrapper",
version: "1.0.0",
type: "module",
main: "index.js",
tspMain: "main.tsp",
peerDependencies: {},
dependencies: { "test-generator": "1.0.0" },
}),
);
await writeFile(join(tmpDir, "main.tsp"), 'import "./index.js"; namespace TestWrapper;');
await writeFile(join(tmpDir, "index.js"), 'export { wheelUrl } from "test-generator";');
await writeFile(
join(dependency, "package.json"),
JSON.stringify({
name: "test-generator",
version: "1.0.0",
type: "module",
main: "index.js",
browser: { "./index.js": "./browser.js" },
}),
);
await writeFile(join(dependency, "index.js"), 'export const wheelUrl = "native";');
await writeFile(
join(dependency, "browser.js"),
'import wheel from "./pygen-0.1.0-py3-none-any.whl"; export const wheelUrl = new URL(wheel, import.meta.url).href;',
);
const bytes = new Uint8Array([0x50, 0x4b, 0x03, 0x04, 0x00, 0xff, 0xfe, 0x80]);
await writeFile(join(dependency, "pygen-0.1.0-py3-none-any.whl"), bytes);

const output = join(tmpDir, "previews", "123", "libs", "test-wrapper");
await bundleTypeSpecLibrary(tmpDir, output);
const wheel = (await readdir(output, { recursive: true })).find((name) =>
name.endsWith(".whl"),
);
expect(wheel).toBeDefined();
expect(basename(wheel!)).toBe("pygen-0.1.0-py3-none-any.whl");
expect(new Uint8Array(await readFile(join(output, wheel!)))).toEqual(bytes);
const emitted = await import(pathToFileURL(join(output, "index.js")).href);
expect(fileURLToPath(emitted.wheelUrl)).toBe(join(output, wheel!));
} finally {
await rm(tmpDir, { recursive: true });
}
});
4 changes: 4 additions & 0 deletions packages/http-client-python/emitter/src/assets.d.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
declare module "*.whl" {
const url: string;
export default url;
}
2 changes: 0 additions & 2 deletions packages/http-client-python/emitter/src/constants.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,5 @@
export const PYODIDE_VERSION = "0.26.2";
export const PYGEN_WHEEL_FILENAME = "pygen-0.1.0-py3-none-any.whl";
export const BLOB_STORAGE_BASE_URL = "https://typespec.blob.core.windows.net/pkgs";
export const PACKAGE_NAME = "@typespec/http-client-python";

export const blackExcludeDirs = [
"__pycache__/",
Expand Down
15 changes: 3 additions & 12 deletions packages/http-client-python/emitter/src/emitter.ts
Original file line number Diff line number Diff line change
@@ -1,14 +1,9 @@
import { createSdkContext } from "@azure-tools/typespec-client-generator-core";
import type { EmitContext } from "@typespec/compiler";
import { emitFile, joinPaths, listServices, NoTarget } from "@typespec/compiler";
import pkgJson from "../../package.json" with { type: "json" };
import { emitCodeModel } from "./code-model.js";
import {
BLOB_STORAGE_BASE_URL,
PACKAGE_NAME,
PYGEN_WHEEL_FILENAME,
PYODIDE_VERSION,
} from "./constants.js";
import { PYODIDE_VERSION } from "./constants.js";
import { generatorWheelUrl } from "./generator-wheel.js";
import type { PythonEmitterOptions, PythonSdkContext } from "./lib.js";
import { reportDiagnostic } from "./lib.js";
import { runNodeEmit } from "./node-runner.js";
Expand All @@ -17,10 +12,6 @@ import { loadPyodide } from "./pyodide-loader.js";
import { getRootNamespace, md2Rst } from "./utils.js";
import { dumpCodeModelToYaml } from "./yaml-utils.js";

function getBrowserPygenWheelUrl(): string {
return `${BLOB_STORAGE_BASE_URL}/${PACKAGE_NAME}/${pkgJson.version}/generator/dist/${PYGEN_WHEEL_FILENAME}`;
}

function addDefaultOptions(sdkContext: PythonSdkContext) {
const defaultOptions = {
"package-version": "1.0.0b1",
Expand Down Expand Up @@ -308,7 +299,7 @@ async function setupPyodideCallBrowser() {
pyodide.FS.mkdirTree("/generator");
await pyodide.loadPackage("micropip");
const micropip = pyodide.pyimport("micropip");
await micropip.install(getBrowserPygenWheelUrl());
await micropip.install(generatorWheelUrl);

return pyodide;
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
import wheel from "../../generator/dist/pygen-0.1.0-py3-none-any.whl";

export const generatorWheelUrl = new URL(wheel, import.meta.url).href;
6 changes: 6 additions & 0 deletions packages/http-client-python/emitter/src/generator-wheel.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
import { PYGEN_WHEEL_FILENAME } from "./constants.js";

export const generatorWheelUrl = new URL(
`../../generator/dist/${PYGEN_WHEEL_FILENAME}`,
import.meta.url,
).href;
7 changes: 1 addition & 6 deletions packages/http-client-python/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
}
},
"browser": {
"./dist/emitter/generator-wheel.js": "./dist/emitter/generator-wheel.browser.js",
"./dist/emitter/node-runner.js": "./dist/emitter/node-runner.browser.js",
"./dist/emitter/pyodide-loader.js": "./dist/emitter/pyodide-loader.browser.js"
},
Expand Down Expand Up @@ -82,12 +83,6 @@
"@typespec/xml": ">=0.88.0 <1.0.0"
},
"playgroundConfig": {
"assets": [
{
"path": "generator/dist/pygen-*.whl",
"contentType": "application/zip"
}
],
"bundlePeerDependencies": [
"@azure-tools/typespec-client-generator-core",
"@azure-tools/typespec-azure-core",
Expand Down
Loading