refactor(workflow): one file boundary, one batcher, shared readers, engine types in the designer

apps/e2e/src/tests/workflow-designer-roundtrip.test.ts

@@ -5,8 +5,9 @@
  *
  * Opening a workflow in the designer and saving it keeps what the designer
  * does not edit — a pipe's timeout, the workflow's version, error handler,
- * display name, and the trust its origin records. Saving used to drop all
- * five: an imported workflow came back `authored`, renamed to its id.
+ * display name, the trust its origin records, and a retry policy beyond its
+ * attempts. Saving used to drop each: an imported workflow came back
+ * `authored`, renamed to its id, with the engine's default retry.
  *
  * Needs the web app at E2E_BASE_URL and workflow-server on :8081 with no
  * service token (loopback dev); skips when the engine is not there.
@@ -30,7 +31,17 @@ const doc = (id: string, extra: Record<string, unknown> = {}) => ({
     {
       id: 'main',
       name: 'Main',
-      pipes: [{ id: 'start', type: 'source.triggerPayload', role: 'source', config: {}, timeoutMs: 1234 }],
+      pipes: [
+        {
+          id: 'start',
+          type: 'source.triggerPayload',
+          role: 'source',
+          config: {},
+          timeoutMs: 1234,
+          // Beyond the attempts the canvas edits: a save used to drop the rest.
+          retry: { attempts: 5, backoff: 'fixed', maxDelayMs: 900, jitter: false, on: 'all' },
+        },
+      ],
       edges: [],
     },
   ],
@@ -99,6 +110,7 @@ describe('workflow designer round trip', () => {
       errorHandler: { workflowId: HANDLER },
     });
     expect(after.pipelines[0].pipes[0].timeoutMs).toBe(1234);
+    expect(after.pipelines[0].pipes[0].retry).toEqual({ attempts: 5, backoff: 'fixed', maxDelayMs: 900, jitter: false, on: 'all' });
   });
 
   it('shows only the trigger sections that have a form, and no dead buttons', async (ctx) => {
@@ -117,4 +129,17 @@ describe('workflow designer round trip', () => {
     ]);
     await driver.screenshot({ path: process.env.E2E_SHOT ?? '/tmp/workflow-trigger-dialog.png' });
   });
+
+  it('keeps the section you are in while you type in a trigger', async (ctx) => {
+    // The dialog used to re-sync its selection from props in an effect; with
+    // the triggers among its dependencies, every keystroke snapped it back to
+    // General. Mounted only while open, it starts once and stays put.
+    if (!engineUp) ctx.skip();
+    const section = driver.locator('[data-testid="workflow-trigger-config-section-input parameters"]');
+    await section.click();
+    await driver.locator(`${byTestId('workflow-trigger-config')} .monaco-editor .view-lines`).click();
+    await driver.keyboard.type('{"region": "west"}');
+    expect(await section.getAttribute('class')).toContain('active');
+    expect(await driver.locator(`${byTestId('workflow-trigger-config')} .monaco-editor`).isVisible()).toBe(true);
+  });
 });

apps/web/src/frontend/features/workflow/components/CredentialsPanel.tsx

@@ -25,8 +25,8 @@ const _kindsMatch: SameMembers<(typeof KINDS)[number], CredentialKind> = true;
 const _sourcesMatch: SameMembers<(typeof SOURCES)[number], Exclude<CredentialSource, 'foxschema'>> = true;
 void [_kindsMatch, _sourcesMatch];
 
-type Kind = (typeof KINDS)[number];
-type Source = (typeof SOURCES)[number];
+type Kind = CredentialKind;
+type Source = Exclude<CredentialSource, 'foxschema'>;
 
 const KIND_FIELDS: Record<Kind, Array<{ key: string; label: string; secret?: boolean }>> = {
   database: [

apps/web/src/frontend/features/workflow/components/DelimitedSourceEditor.tsx

@@ -12,10 +12,11 @@ import { compileZodSchema } from '../lib/zodSchema';
 import type {
   ColumnCheck as EngineColumnCheck,
   ColumnCheckKind,
+  ColumnRule,
   ColumnType,
   FixedWidthColumn,
 } from '@foxschema/workflow-engine/definitions';
-import type { KnowsEveryField, SameMembers } from '../lib/engineShapes';
+import type { SameMembers } from '../lib/engineShapes';
 
 /**
  * Config editor for the delimited file sources (`source.file.csv`,
@@ -59,33 +60,14 @@ const COLUMN_TYPES = ['string', 'integer', 'number', 'boolean', 'date'] as const
 const CHECK_KINDS = ['null', 'length', 'format'] as const;
 const _columnTypesMatch: SameMembers<(typeof COLUMN_TYPES)[number], ColumnType> = true;
 const _checkKindsMatch: SameMembers<(typeof CHECK_KINDS)[number], ColumnCheckKind> = true;
-const _checksKnown: KnowsEveryField<ColumnCheck, EngineColumnCheck> = true;
-const _fixedKnown: KnowsEveryField<FixedColumn, FixedWidthColumn> = true;
-void [_columnTypesMatch, _checkKindsMatch, _checksKnown, _fixedKnown];
-
-interface ColumnCheck {
-  pattern: string;
-  kind?: string;
-  message?: string;
-}
-
-interface ColumnRuleFields {
-  type?: string;
-  /** Legacy single regex — migrated into `checks` in the editor when present. */
-  pattern?: string;
-  checks?: ColumnCheck[];
-  required?: boolean;
-}
+void [_columnTypesMatch, _checkKindsMatch];
 
-interface FixedColumn extends ColumnRuleFields {
-  name: string;
-  start: number;
-  length: number;
-}
-
-interface FieldRule extends ColumnRuleFields {
-  name: string;
-}
+// The engine's own column shapes, type-only. Hand-kept copies here typed a
+// column's `type` as any string, so a value the engine rejects compiled.
+type ColumnCheck = EngineColumnCheck;
+type ColumnRuleFields = Omit<ColumnRule, 'name'>;
+type FixedColumn = FixedWidthColumn;
+type FieldRule = ColumnRule;
 
 const parseChecks = (value: unknown): ColumnCheck[] => {
   if (!Array.isArray(value)) return [];
@@ -103,6 +85,10 @@ const parseChecks = (value: unknown): ColumnCheck[] => {
   });
 };
 
+/** A column type the engine knows, or undefined — the engine refuses any other. */
+const asColumnType = (value: unknown): ColumnType | undefined =>
+  (COLUMN_TYPES as readonly unknown[]).includes(value) ? (value as ColumnType) : undefined;
+
 /** Normalize rule fields: fold legacy `pattern` into `checks` for editing. */
 const ruleFields = (raw: ColumnRuleFields): ColumnRuleFields => {
   const checks = parseChecks(raw.checks);
@@ -111,7 +97,7 @@ const ruleFields = (raw: ColumnRuleFields): ColumnRuleFields => {
     checks.unshift({ pattern: legacy });
   }
   return {
-    ...(raw.type ? { type: String(raw.type) } : {}),
+    ...(asColumnType(raw.type) ? { type: asColumnType(raw.type) } : {}),
     ...(checks.length > 0 ? { checks } : {}),
     ...(raw.required ? { required: true } : {}),
   };
@@ -209,7 +195,7 @@ function ColumnRuleEditor({
         <select data-testid={`workflow-source-rule-type-${name}`}
           title="Value type — validated and converted"
           value={fields.type ?? 'string'}
-          onChange={(ev) => onFieldsChange({ ...fields, type: ev.target.value })}
+          onChange={(ev) => onFieldsChange({ ...fields, type: asColumnType(ev.target.value) })}
         >
           {COLUMN_TYPES.map((type) => (
             <option key={type} value={type}>

apps/web/src/frontend/features/workflow/components/HttpRequestEditor.tsx

@@ -16,16 +16,17 @@ import type {
   HttpRequestDef as EngineRequest,
   HttpSession as EngineSession,
   HttpTokenRefresh as EngineTokenRefresh,
+  HttpAuth as EngineAuth,
+  HttpBody as EngineBody,
 } from '@foxschema/workflow-engine/definitions';
-import type { KnowsEveryField, SameMembers } from '../lib/engineShapes';
+import type { SameMembers } from '../lib/engineShapes';
 import type { CredentialMeta } from '../api/engineClient';
 import { compileZodSchema } from '../lib/zodSchema';
 
 /**
- * The editor's own shapes for the engine's HttpRequestDef. Values stay local
- * (importing them pulls zod into the bundle); each list and shape is checked
- * against the engine's types below, so a field the engine adds fails the build
- * instead of being dropped on the next edit.
+ * The request model is the engine's own (`HttpRequestDef` and its parts),
+ * imported type-only. A hand-kept copy here had already drifted: it lacked a
+ * token refresh's headers and body, so every edit dropped them.
  */
 export const HTTP_KV_OPS = [
   'eq',
@@ -37,7 +38,7 @@ export const HTTP_KV_OPS = [
   'or',
   'range',
 ] as const;
-export type HttpKvOp = (typeof HTTP_KV_OPS)[number];
+export type HttpKvOp = EngineKvOp;
 
 /** UI labels + value hints per operator; serialized as `key[op]=value` (eq = plain). */
 const OP_META: Record<HttpKvOp, { label: string; placeholder: string }> = {
@@ -51,37 +52,19 @@ const OP_META: Record<HttpKvOp, { label: string; placeholder: string }> = {
   range: { label: 'range', placeholder: 'min,max' },
 };
 
-export type HttpKv = {
-  key: string;
-  value: string;
-  enabled: boolean;
-  op?: HttpKvOp;
-  /** Legacy path mapping — no longer editable in the UI, preserved on load. */
-  valueFrom?: string;
-  keyFrom?: string;
-};
+export type HttpKv = EngineKv;
 
-export type HttpAuth =
-  | { type: 'none' }
-  | { type: 'bearer'; token: string }
-  | { type: 'basic'; username: string; password: string }
-  | { type: 'apiKey'; key: string; value: string; in: 'header' | 'query' }
-  | { type: 'credential'; credentialId: string };
+export type HttpAuth = EngineAuth;
 
 export const HTTP_BODY_FIELD_TYPES = [
   'string',
   'number',
   'boolean',
   'json',
 ] as const;
-export type HttpBodyFieldType = (typeof HTTP_BODY_FIELD_TYPES)[number];
+export type HttpBodyFieldType = EngineBodyFieldType;
 
-export type HttpBodyField = {
-  key: string;
-  type: HttpBodyFieldType;
-  value: string;
-  enabled: boolean;
-};
+export type HttpBodyField = EngineBodyField;
 
 /** Optional JSON Schema — the runtime validates the body against it before sending. */
 type BodySchema = Record<string, unknown> | undefined;
@@ -93,47 +76,13 @@ type BodySchema = Record<string, unknown> | undefined;
  */
 type BodyValidation = { schema?: BodySchema; schemaSource?: string };
 
-export type HttpBody =
-  | { mode: 'none' }
-  | ({ mode: 'json'; json: unknown } & BodyValidation)
-  | ({ mode: 'raw'; raw: string; contentType: string } & BodyValidation)
-  | ({ mode: 'form'; form: HttpKv[] } & BodyValidation)
-  | ({ mode: 'fields'; fields: HttpBodyField[] } & BodyValidation);
+export type HttpBody = EngineBody;
 
-export type HttpSession = {
-  enabled: boolean;
-  persistToCredential: boolean;
-};
+export type HttpSession = EngineSession;
 
-export type HttpTokenRefresh = {
-  enabled: boolean;
-  url: string;
-  method: 'POST' | 'PUT';
-  /**
-   * Not edited here, but carried through: dropping them on every edit sent
-   * the engine's default refresh body instead of the one written by hand.
-   */
-  headers?: EngineTokenRefresh['headers'];
-  body?: EngineTokenRefresh['body'];
-  accessTokenPath: string;
-  refreshTokenPath?: string;
-  expiresInPath: string;
-  onStatus: number[];
-  skewSeconds: number;
-};
+export type HttpTokenRefresh = EngineTokenRefresh;
 
-export type HttpRequestValue = {
-  url: string;
-  method: 'GET' | 'POST' | 'PUT' | 'PATCH' | 'DELETE' | 'HEAD';
-  query: HttpKv[];
-  headers: HttpKv[];
-  auth: HttpAuth;
-  body: HttpBody;
-  timeoutMs: number;
-  variables: Record<string, unknown>;
-  session: HttpSession;
-  tokenRefresh?: HttpTokenRefresh;
-};
+export type HttpRequestValue = EngineRequest;
 
 export const EMPTY_HTTP_REQUEST: HttpRequestValue = {
   url: 'https://',
@@ -152,6 +101,9 @@ const DEFAULT_TOKEN_REFRESH: HttpTokenRefresh = {
   enabled: false,
   url: 'https://',
   method: 'POST',
+  // The engine's own defaults: no extra headers, the standard refresh_token body.
+  headers: [],
+  body: { mode: 'json', json: { refresh_token: '{{secrets.refreshToken}}', grant_type: 'refresh_token' }, form: [] },
   accessTokenPath: 'access_token',
   expiresInPath: 'expires_in',
   onStatus: [401],
@@ -1232,14 +1184,11 @@ function BodySchemaEditor({
 }
 
 /** Normalize pipe/trigger JSON into the editor value (legacy Record headers ok). */
-const _kvOpsMatch: SameMembers<HttpKvOp, EngineKvOp> = true;
-const _bodyFieldTypesMatch: SameMembers<HttpBodyFieldType, EngineBodyFieldType> = true;
-const _kvKnown: KnowsEveryField<HttpKv, EngineKv> = true;
-const _bodyFieldKnown: KnowsEveryField<HttpBodyField, EngineBodyField> = true;
-const _sessionKnown: KnowsEveryField<HttpSession, EngineSession> = true;
-const _tokenRefreshKnown: KnowsEveryField<HttpTokenRefresh, EngineTokenRefresh> = true;
-const _requestKnown: KnowsEveryField<HttpRequestValue, EngineRequest> = true;
-void [_kvOpsMatch, _bodyFieldTypesMatch, _kvKnown, _bodyFieldKnown, _sessionKnown, _tokenRefreshKnown, _requestKnown];
+// The two value lists stay local (importing them pulls zod into the bundle);
+// a value the engine adds or drops fails the build here.
+const _kvOpsMatch: SameMembers<(typeof HTTP_KV_OPS)[number], EngineKvOp> = true;
+const _bodyFieldTypesMatch: SameMembers<(typeof HTTP_BODY_FIELD_TYPES)[number], EngineBodyFieldType> = true;
+void [_kvOpsMatch, _bodyFieldTypesMatch];
 
 export function toHttpRequestValue(input: unknown): HttpRequestValue {
   if (!input || typeof input !== 'object') return { ...EMPTY_HTTP_REQUEST };
@@ -1390,8 +1339,10 @@ function coerceTokenRefresh(value: unknown): HttpTokenRefresh | undefined {
     enabled: raw.enabled === true,
     url: typeof raw.url === 'string' ? raw.url : DEFAULT_TOKEN_REFRESH.url,
     method: raw.method === 'PUT' ? 'PUT' : 'POST',
-    ...(Array.isArray(raw.headers) ? { headers: raw.headers as EngineTokenRefresh['headers'] } : {}),
-    ...(raw.body && typeof raw.body === 'object' ? { body: raw.body as EngineTokenRefresh['body'] } : {}),
+    // Not edited here, but carried through: dropping them on every edit sent
+    // the engine's default refresh body instead of the one written by hand.
+    headers: Array.isArray(raw.headers) ? (raw.headers as HttpTokenRefresh['headers']) : DEFAULT_TOKEN_REFRESH.headers,
+    body: raw.body && typeof raw.body === 'object' ? (raw.body as HttpTokenRefresh['body']) : DEFAULT_TOKEN_REFRESH.body,
     accessTokenPath:
       typeof raw.accessTokenPath === 'string'
         ? raw.accessTokenPath

apps/web/src/frontend/features/workflow/components/PipeNode.tsx

@@ -8,6 +8,8 @@
 import { Handle, Position, type Node, type NodeProps } from '@xyflow/react';
 import { ArrowDownToLine, ArrowUpFromLine, Shuffle, Zap } from 'lucide-react';
 import { portLabel } from '../lib/ports';
+import type { z } from 'zod';
+import type { retrySchema } from '@foxschema/workflow-engine/definitions';
 
 export type PortInfo = { name: string; type: string };
 
@@ -24,6 +26,8 @@ export type PipeData = {
   credentialId: string;
   /** Not edited on the canvas; kept so a save does not drop it. */
   timeoutMs?: number;
+  /** The pipe's whole retry policy as loaded; the canvas edits only `retryAttempts`. */
+  retry?: z.input<typeof retrySchema>;
   inputs: PortInfo[];
   outputs: PortInfo[];
 };