feat: serve the web UI from sqb server and ship it in the wheels

.github/workflows/build-distributions.yml

@@ -123,6 +123,18 @@ jobs:
         if: inputs.prebuilt-run-id == ''
         with:
           toolchain: stable
+      - uses: actions/setup-node@v4
+        if: inputs.prebuilt-run-id == ''
+        with:
+          node-version: "22"
+          cache: npm
+          cache-dependency-path: ui/package-lock.json
+      - name: Build the web UI that sqb server embeds
+        if: inputs.prebuilt-run-id == ''
+        working-directory: ui
+        run: |
+          npm ci
+          npm run build
       - uses: PyO3/maturin-action@v1
         if: inputs.prebuilt-run-id == ''
         with:
@@ -142,10 +154,12 @@ jobs:
         env:
           RUSTFLAGS: >-
             ${{ runner.os == 'macOS' && '-C link-arg=-undefined -C link-arg=dynamic_lookup' || '' }}
+          # A wheel must never ship a sqb server without its UI.
+          SQLBUILD_UI_REQUIRED: "1"
         with:
           command: build
           args: >-
-            --release --out dist --manifest-path crates/sqlbuild-cli/Cargo.toml
+            --release --out dist --manifest-path crates/sqlbuild-cli/Cargo.toml --features ui
             ${{ matrix.platform.name == 'linux-x86_64' && '--timings' || '' }}
           target: ${{ matrix.platform.target }}
           manylinux: auto
@@ -213,6 +227,19 @@ jobs:
           printf 'MODEL();\nSELECT 1 AS id\n' > "${project}/models/orders.sql"
           "${smoke}/sqb" --project-dir "${project}" scope model:orders --json
           "${smoke}/sqlbuild" --project-dir "${project}" scope model:orders
+          # The packaged sqb server must serve the embedded UI, not only the API.
+          server_dir="${RUNNER_TEMP}/ui-server-smoke"
+          mkdir -p "${server_dir}"
+          (cd "${server_dir}" && env -u SQLBUILD_CONTROL_DATABASE_URL \
+            "${smoke}/sqb" server --host 127.0.0.1 --port 8765 --no-scheduler) &
+          server_pid=$!
+          for _ in $(seq 1 60); do
+            curl -sf -o /dev/null http://127.0.0.1:8765/api/openapi.json && break
+            sleep 1
+          done
+          curl -sf http://127.0.0.1:8765/runs | grep -q '/_app/immutable/'
+          curl -sfI http://127.0.0.1:8765/theme.js | grep -qi '^content-type: text/javascript'
+          kill "${server_pid}"
       - uses: actions/upload-artifact@v4
         if: inputs.upload
         with:

.github/workflows/verify.yml

@@ -148,13 +148,13 @@ jobs:
               *) docs_only=false ;;
             esac
             case "${path}" in
-              crates/*|Cargo.toml|Cargo.lock|pyproject.toml|.github/workflows/verify.yml|.github/workflows/build-distributions.yml|deploy/Dockerfile|deploy/project/*|deploy/compose*|compose.yml|.dockerignore|Makefile|tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_compose.py|tests/integration/src/sqlbuild/compiler/compiled_project/publication/docker/compose/*|benchmarks/duckshop/edge_cases/orchestration/container_deployment.json)
+              crates/*|ui/*|Cargo.toml|Cargo.lock|pyproject.toml|.github/workflows/verify.yml|.github/workflows/build-distributions.yml|deploy/Dockerfile|deploy/project/*|deploy/compose*|compose.yml|.dockerignore|Makefile|tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_compose.py|tests/integration/src/sqlbuild/compiler/compiled_project/publication/docker/compose/*|benchmarks/duckshop/edge_cases/orchestration/container_deployment.json)
                 wheels=true
                 ;;
             esac
             case "${path}" in
               ui/*|clients/typescript/*|.github/workflows/verify.yml)
-                # The web UI is not packaged; its own checks and unit tests cover it. It compiles
+                # The web UI's own checks and unit tests; the wheels embed its build. It compiles
                 # against the generated control client, so client changes run them too.
                 ui=true
                 ;;
@@ -1144,6 +1144,8 @@ jobs:
         run: npm run check
       - name: Unit tests
         run: npm test
+      - name: Build the static bundle sqb server embeds
+        run: npm run build
 
   verify:
     # The required Verify of a Release Please pull request is the commit status that the trusted

crates/sqlbuild-cli/Cargo.toml

@@ -15,6 +15,9 @@ path = "src/bin/sqb.rs"
 name = "sqlbuild"
 path = "src/bin/sqlbuild.rs"
 
+[features]
+ui = ["sqlbuild-orchestration/ui"]
+
 [dependencies]
 blake3.workspace = true
 chrono.workspace = true

crates/sqlbuild-orchestration/Cargo.toml

@@ -7,6 +7,9 @@ license.workspace = true
 publish.workspace = true
 rust-version.workspace = true
 
+[features]
+ui = []
+
 [dependencies]
 argon2.workspace = true
 axum.workspace = true

crates/sqlbuild-orchestration/build.rs

@@ -0,0 +1,68 @@
+//! With the `ui` feature, embed the built web UI (`ui/build`, or `SQLBUILD_UI_DIST`) for `sqb server`.
+
+use std::path::{Path, PathBuf};
+
+fn main() {
+    println!("cargo:rerun-if-env-changed=SQLBUILD_UI_DIST");
+    println!("cargo:rerun-if-env-changed=SQLBUILD_UI_REQUIRED");
+    let output = PathBuf::from(std::env::var_os("OUT_DIR").expect("OUT_DIR")).join("ui_assets.rs");
+    let mut files = Vec::new();
+    if std::env::var_os("CARGO_FEATURE_UI").is_some() {
+        let dist = std::env::var_os("SQLBUILD_UI_DIST").map_or_else(
+            || {
+                Path::new(&std::env::var_os("CARGO_MANIFEST_DIR").expect("manifest"))
+                    .join("../../ui/build")
+            },
+            PathBuf::from,
+        );
+        println!("cargo:rerun-if-changed={}", dist.display());
+        if dist.join("index.html").is_file() {
+            collect(&dist, &dist, &mut files);
+        } else {
+            let message = format!(
+                "sqb server will not serve the UI: {} has no index.html; run `npm ci && npm run build` in ui/ or set SQLBUILD_UI_DIST",
+                dist.display()
+            );
+            assert!(
+                std::env::var_os("SQLBUILD_UI_REQUIRED").is_none(),
+                "{message}"
+            );
+            println!("cargo:warning={message}");
+        }
+    }
+    files.sort();
+    let entries: String = files
+        .iter()
+        .map(|(route, file)| {
+            format!(
+                "    UiAsset {{ path: {route:?}, body: include_bytes!({:?}) }},\n",
+                file.display()
+            )
+        })
+        .collect();
+    std::fs::write(
+        output,
+        format!("pub(crate) static UI_ASSETS: &[UiAsset] = &[\n{entries}];\n"),
+    )
+    .expect("write the embedded UI table");
+}
+
+fn collect(root: &Path, directory: &Path, files: &mut Vec<(String, PathBuf)>) {
+    for entry in std::fs::read_dir(directory).expect("read the UI build") {
+        let path = entry.expect("UI build entry").path();
+        if path.is_dir() {
+            collect(root, &path, files);
+        } else {
+            let relative = path.strip_prefix(root).expect("inside the UI build");
+            let route = relative
+                .components()
+                .map(|part| part.as_os_str().to_string_lossy().into_owned())
+                .collect::<Vec<_>>()
+                .join("/");
+            files.push((
+                format!("/{route}"),
+                path.canonicalize().expect("absolute UI asset path"),
+            ));
+        }
+    }
+}

crates/sqlbuild-orchestration/src/api/_helpers/resources/mod.rs

@@ -16,4 +16,5 @@ pub(crate) mod run_reads;
 pub(crate) mod runs;
 pub(crate) mod snapshots;
 pub(crate) mod trigger_reads;
+pub(crate) mod ui_assets;
 pub(crate) mod warehouse_metadata;

crates/sqlbuild-orchestration/src/api/_helpers/resources/ui_assets.rs

@@ -0,0 +1,107 @@
+//! Serve the embedded web UI for every GET outside the API, falling back to its index for client routes.
+
+use axum::Router;
+use axum::http::{HeaderValue, Method, StatusCode, Uri, header};
+use axum::response::{IntoResponse, Response};
+
+use crate::api::constants::{
+    API_PATH, API_PREFIX, AUTH_PREFIX, ROOT_PATH, UI_DOT_SEGMENTS, UI_DOUBLE_SLASH,
+    UI_FRAME_POLICY, UI_IMMUTABLE_PREFIX, UI_INDEX_PATH, UI_REFUSED_ENCODINGS,
+};
+use crate::api::models::UiAsset;
+
+#[cfg(feature = "ui")]
+include!(concat!(env!("OUT_DIR"), "/ui_assets.rs"));
+#[cfg(not(feature = "ui"))]
+pub(crate) static UI_ASSETS: &[UiAsset] = &[];
+
+pub(crate) fn serve_ui(application: Router, assets: &'static [UiAsset]) -> Router {
+    if assets.is_empty() {
+        return application;
+    }
+    application.fallback(move |method: Method, uri: Uri| async move {
+        respond(assets, &method, uri.path())
+    })
+}
+
+fn respond(assets: &'static [UiAsset], method: &Method, path: &str) -> Response {
+    let outside_ui = path == API_PATH
+        || path.starts_with(API_PREFIX)
+        || path.starts_with(AUTH_PREFIX)
+        || !canonical(path);
+    if outside_ui || (method != Method::GET && method != Method::HEAD) {
+        return StatusCode::NOT_FOUND.into_response();
+    }
+    let wanted = if path == ROOT_PATH {
+        UI_INDEX_PATH
+    } else {
+        path
+    };
+    let file_like = wanted
+        .rsplit('/')
+        .next()
+        .is_some_and(|name| name.contains('.'));
+    let found = assets.iter().find(|asset| asset.path == wanted);
+    let asset = match found {
+        Some(asset) => asset,
+        None if file_like => return StatusCode::NOT_FOUND.into_response(),
+        None => match assets.iter().find(|asset| asset.path == UI_INDEX_PATH) {
+            Some(index) => index,
+            None => return StatusCode::NOT_FOUND.into_response(),
+        },
+    };
+    let caching = if asset.path.starts_with(UI_IMMUTABLE_PREFIX) {
+        "public, max-age=31536000, immutable"
+    } else {
+        "no-cache"
+    };
+    let mut response = asset.body.into_response();
+    let headers = response.headers_mut();
+    headers.insert(
+        header::CONTENT_TYPE,
+        HeaderValue::from_static(content_type(asset.path)),
+    );
+    headers.insert(header::CACHE_CONTROL, HeaderValue::from_static(caching));
+    headers.insert(
+        header::X_CONTENT_TYPE_OPTIONS,
+        HeaderValue::from_static("nosniff"),
+    );
+    headers.insert(
+        header::REFERRER_POLICY,
+        HeaderValue::from_static("no-referrer"),
+    );
+    headers.insert(
+        header::CONTENT_SECURITY_POLICY,
+        HeaderValue::from_static(UI_FRAME_POLICY),
+    );
+    response
+}
+
+/// Only plain, already-normalised paths can name an embedded file or the shell.
+fn canonical(path: &str) -> bool {
+    let lowered = path.to_ascii_lowercase();
+    !path.contains('\\')
+        && !path.contains(UI_DOUBLE_SLASH)
+        && !UI_REFUSED_ENCODINGS
+            .iter()
+            .any(|encoded| lowered.contains(encoded))
+        && path
+            .split('/')
+            .all(|segment| !UI_DOT_SEGMENTS.contains(&segment))
+}
+
+fn content_type(path: &str) -> &'static str {
+    match path.rsplit('.').next().unwrap_or_default() {
+        "html" => "text/html; charset=utf-8",
+        "js" => "text/javascript; charset=utf-8",
+        "css" => "text/css; charset=utf-8",
+        "json" => "application/json",
+        "txt" => "text/plain; charset=utf-8",
+        "png" => "image/png",
+        "svg" => "image/svg+xml",
+        "ico" => "image/x-icon",
+        "woff" => "font/woff",
+        "woff2" => "font/woff2",
+        _ => "application/octet-stream",
+    }
+}

crates/sqlbuild-orchestration/src/api/constants.rs

@@ -13,3 +13,14 @@ pub(crate) const OIDC_RESPONSE_BYTES: usize = 1024 * 1024;
 pub(crate) const DOCKER_IMAGE_HEX_LENGTH: usize = 64;
 pub(crate) const HTTPS_SCHEME: &str = "https";
 pub(crate) const ROOT_PATH: &str = "/";
+pub(crate) const UI_INDEX_PATH: &str = "/index.html";
+pub(crate) const UI_IMMUTABLE_PREFIX: &str = "/_app/immutable/";
+/// The page's own meta policy carries every other directive; a meta tag cannot set frame-ancestors.
+pub(crate) const UI_FRAME_POLICY: &str = "frame-ancestors 'none'";
+pub(crate) const API_PATH: &str = "/api";
+pub(crate) const API_PREFIX: &str = "/api/";
+pub(crate) const AUTH_PREFIX: &str = "/auth/";
+/// Percent-encoded dot, slash and backslash: a UI path carrying one is refused, never decoded.
+pub(crate) const UI_REFUSED_ENCODINGS: [&str; 3] = ["%2e", "%2f", "%5c"];
+pub(crate) const UI_DOT_SEGMENTS: [&str; 2] = [".", ".."];
+pub(crate) const UI_DOUBLE_SLASH: &str = "//";

crates/sqlbuild-orchestration/src/api/main/serve_api.rs

@@ -4,6 +4,7 @@ use axum::routing::get;
 use axum::{Extension, Json, Router, middleware};
 
 use crate::api::_helpers::authentication::{bearer::local_request, configure::configure};
+use crate::api::_helpers::resources::ui_assets::{UI_ASSETS, serve_ui};
 use crate::api::_helpers::resources::{changes, router::router};
 use crate::api::main::openapi_document::openapi_document;
 use crate::api::models::{
@@ -77,6 +78,7 @@ fn development_application(
     scope: Option<DevelopmentScope>,
     address: std::net::SocketAddr,
 ) -> Router {
+    let application = serve_ui(application, UI_ASSETS);
     match scope {
         Some(scope) => application
             .layer(Extension(scope))

crates/sqlbuild-orchestration/src/api/models.rs

@@ -418,3 +418,10 @@ pub(crate) struct ControlClient {
     pub(crate) authorization: reqwest::header::HeaderValue,
     pub(crate) base_url: reqwest::Url,
 }
+
+/// One file of the built web UI, embedded at compile time and served at `path`.
+#[derive(Debug)]
+pub(crate) struct UiAsset {
+    pub(crate) path: &'static str,
+    pub(crate) body: &'static [u8],
+}

crates/sqlbuild-orchestration/tests/api/helpers.rs

@@ -80,3 +80,24 @@ pub(crate) async fn read_json(
     assert_eq!(response.headers()["cache-control"], "no-store");
     response.json().await.expect("typed JSON response")
 }
+
+/// Send `GET <path>` byte for byte, so no client normalises dot segments, slashes or encodings.
+pub(crate) fn raw_get(address: &str, path: &str) -> (u16, String) {
+    use std::io::{Read, Write};
+    let mut stream = std::net::TcpStream::connect(address).expect("raw socket");
+    write!(
+        stream,
+        "GET {path} HTTP/1.1\r\nHost: {address}\r\nConnection: close\r\n\r\n"
+    )
+    .expect("raw request");
+    let mut response = String::new();
+    stream.read_to_string(&mut response).expect("raw response");
+    let (head, body) = response.split_once("\r\n\r\n").expect("HTTP head");
+    let status = head
+        .split(' ')
+        .nth(1)
+        .expect("status")
+        .parse()
+        .expect("numeric status");
+    (status, body.to_owned())
+}

crates/sqlbuild-orchestration/tests/api/test_contracts.rs

@@ -76,6 +76,8 @@ mod test_reads;
 mod test_statistics_recovery;
 #[path = "test_types.rs"]
 mod test_types;
+#[path = "test_ui_assets.rs"]
+mod test_ui_assets;
 #[path = "test_warehouse_metadata.rs"]
 mod test_warehouse_metadata;
 #[path = "test_worker.rs"]

crates/sqlbuild-orchestration/tests/api/test_types.rs

@@ -140,3 +140,19 @@ pub(crate) struct CustomRoleApiTestCase {
     pub(crate) backend: Backend,
     pub(crate) expected_statuses: [u16; 2],
 }
+
+pub(crate) struct UiAssetTestCase {
+    pub(crate) description: &'static str,
+    pub(crate) method: reqwest::Method,
+    pub(crate) path: &'static str,
+    pub(crate) expected_status: u16,
+    pub(crate) expected_body: &'static str,
+    pub(crate) expected_cache: Option<&'static str>,
+}
+
+pub(crate) struct UiPathTestCase {
+    pub(crate) description: &'static str,
+    pub(crate) path: &'static str,
+    pub(crate) expected_status: u16,
+    pub(crate) expected_body: &'static str,
+}