E2E testing

.eslint.config.mjs

@@ -1,17 +1,17 @@
 import globals from "globals";
 import pluginJs from "@eslint/js";
 import tseslint from "typescript-eslint";
-import stylisticJs from '@stylistic/eslint-plugin-js'
+import stylisticJs from '@stylistic/eslint-plugin'
 
 
 /** @type {import('eslint').Linter.Config[]} */
 export default [
   pluginJs.configs.recommended,
   ...tseslint.configs.recommended,
   { plugins: {
-      '@stylistic/js': stylisticJs
+      '@stylistic': stylisticJs
     }, rules: {
-      '@stylistic/js/indent': ['error', 4, { "SwitchCase": 1 }],
+      '@stylistic/indent': ['error', 4, { "SwitchCase": 1 }],
       '@typescript-eslint/no-wrapper-object-types': 'off',
       '@typescript-eslint/no-unused-vars': ['error', { argsIgnorePattern: '^_', varsIgnorePattern: '^_', caughtErrorsIgnorePattern: '^_' }]
     }

.github/actions/build-warduino/action.yml

@@ -0,0 +1,62 @@
+name: Build WARDuino CLI
+description: Check out and build a WARDuino CLI version
+
+inputs:
+  version:
+    description: WARDuino version tag without the optional v prefix, latest, or dirty
+    required: false
+    default: latest
+
+runs:
+  using: composite
+  steps:
+    - name: Resolve WARDuino version
+      id: version
+      shell: bash
+      env:
+        WARDUINO_VERSION: ${{ inputs.version }}
+      run: |
+        if [ "$WARDUINO_VERSION" = "latest" ]; then
+          VERSION=$(git ls-remote --refs --tags https://github.com/TOPLLab/WARDuino.git 'v*' \
+            | sed 's#.*refs/tags/##' \
+            | sort --version-sort \
+            | tail --lines=1)
+          test -n "$VERSION"
+        elif [ "$WARDUINO_VERSION" = "dirty" ]; then
+          VERSION=main
+        else
+          VERSION="${WARDUINO_VERSION#v}"
+          VERSION="v$VERSION"
+        fi
+        echo "ref=$VERSION" >> "$GITHUB_OUTPUT"
+
+    - name: Check out WARDuino
+      uses: actions/checkout@v4
+      with:
+        repository: TOPLLab/WARDuino
+        ref: ${{ steps.version.outputs.ref }}
+        path: tests/artifacts/warduino
+        submodules: recursive
+
+    - name: Get WARDuino commit ID
+      id: commit
+      shell: bash
+      working-directory: tests/artifacts/warduino
+      run: echo "value=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"
+
+    - name: Cache WARDuino CLI
+      uses: actions/cache@v4
+      id: cache-warduino
+      with:
+        key: ${{ runner.os }}-warduino-${{ steps.commit.outputs.value }}
+        path: tests/artifacts/warduino/build
+        restore-keys: |
+          ${{ runner.os }}-warduino-
+
+    - name: Build WARDuino CLI
+      if: steps.cache-warduino.outputs.cache-hit != 'true'
+      shell: bash
+      working-directory: tests/artifacts/warduino
+      run: |
+        cmake -S . -B build -D BUILD_EMULATOR=ON
+        cmake --build build

.github/workflows/test.yml

@@ -42,29 +42,11 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-        with:
-          submodules: recursive
-
-      - name: Get WDCLI commit ID
-        working-directory: tests/artifacts/warduino
-        run: echo "WDCLI_VERSION=$(git rev-parse HEAD)" >> $GITHUB_ENV
 
-      - name: Cache WARDuino CLI
-        uses: actions/cache@v4
-        id: cache-warduino
+      - name: Build WARDuino CLI
+        uses: ./.github/actions/build-warduino
         with:
-          key: ${{ runner.os }}-warduino-${{ env.WDCLI_VERSION }}
-          path: tests/artifacts/warduino
-          restore-keys: |
-            ${{ runner.os }}-warduino-
-
-      - name: Build WARDuino CLI  # Build latest version
-        if: steps.cache-warduino.outputs.cache-hit == false
-        working-directory: tests/artifacts/warduino
-        run: |
-          mkdir build; cd build
-          cmake .. -D BUILD_EMULATOR=ON
-          cmake --build .
+          version: 0.8.1
 
       - name: Upload built tools
         uses: actions/upload-artifact@v4
@@ -111,8 +93,81 @@ jobs:
         env:
           WABT: ${{ github.workspace }}/.tools
 
-      - name: Run example tests
-        run: npm run test:example
+  integration:
+    name: Integration tests
+    runs-on: ubuntu-latest
+    needs: [build-wabt, build-wdcli]
+    if: github.event.pull_request.draft == false
+    steps:
+      - uses: actions/checkout@v4
+
+      - run: npm i
+
+      - name: Prebuild files
+        run: npm run test:prebuild
+
+      - name: Download WAT tools
+        uses: actions/download-artifact@v4
+        with:
+          name: wabt-build-${{ github.run_id }}
+          path: .tools
+
+      - name: Download WARDuino CLI
+        uses: actions/download-artifact@v4
+        with:
+          name: warduino-build-${{ github.run_id }}
+          path: .warduino
+
+      - name: Configure test tools
+        run: |
+          chmod u+x "$GITHUB_WORKSPACE"/.tools/*
+          "$GITHUB_WORKSPACE"/.tools/wat2wasm --version
+          EMULATOR_PATH="$(find "$GITHUB_WORKSPACE"/.warduino -type f -name wdcli -print -quit)"
+          test -n "$EMULATOR_PATH"
+          chmod u+x "$EMULATOR_PATH"
+          echo "EMULATOR=$EMULATOR_PATH" >> "$GITHUB_ENV"
+
+      - name: Run integration tests
+        run: npm run test:integration
+        env:
+          WABT: ${{ github.workspace }}/.tools
+
+  end-to-end:
+    name: End-to-end tests
+    runs-on: ubuntu-latest
+    needs: [build-wabt, build-wdcli]
+    if: github.event.pull_request.draft == false
+    steps:
+      - uses: actions/checkout@v4
+
+      - run: npm i
+
+      - name: Prebuild files
+        run: npm run test:prebuild
+
+      - name: Download WAT tools
+        uses: actions/download-artifact@v4
+        with:
+          name: wabt-build-${{ github.run_id }}
+          path: .tools
+
+      - name: Download WARDuino CLI
+        uses: actions/download-artifact@v4
+        with:
+          name: warduino-build-${{ github.run_id }}
+          path: .warduino
+
+      - name: Configure emulator
+        run: |
+          chmod u+x "$GITHUB_WORKSPACE"/.tools/*
+          "$GITHUB_WORKSPACE"/.tools/wasm-objdump --version
+          EMULATOR_PATH="$(find "$GITHUB_WORKSPACE"/.warduino -type f -name wdcli -print -quit)"
+          test -n "$EMULATOR_PATH"
+          chmod u+x "$EMULATOR_PATH"
+          echo "EMULATOR=$EMULATOR_PATH" >> "$GITHUB_ENV"
+
+      - name: Run end-to-end tests
+        run: npm run test:end-to-end
         env:
           WABT: ${{ github.workspace }}/.tools
 

package.json

@@ -42,7 +42,7 @@
     "test:all": "npm run test:prebuild && npm run test:ava && npm run test:integration && npm run test:end-to-end",
     "test:ava": "ava",
     "test:integration": "npx ts-node ./tests/integration/precision.ts",
-    "test:end-to-end": "npx ts-node ./tests/end-to-end/r3/r3.test.ts",
+    "test:end-to-end": "npx ts-node ./tests/end-to-end/end2end.test.ts",
     "test:example": "npx ts-node ./tests/examples/example.ts",
     "coverage:test:ava": "c8 --src src/ --all ava"
   },
@@ -60,7 +60,7 @@
   "devDependencies": {
     "@ava/typescript": "^7.0.0",
     "@eslint/js": "^10.0.1",
-    "@stylistic/eslint-plugin-js": "^4.4.1",
+    "@stylistic/eslint-plugin": "^5.10.0",
     "@types/chai": "^5.2.3",
     "@types/mocha": "^10.0.10",
     "@types/node": "^25.9.1",

src/debug/WARDuino.ts

@@ -1,3 +1,4 @@
+/* eslint-disable @typescript-eslint/no-namespace */
 import {WASM} from '../sourcemap/Wasm';
 
 export namespace WARDuino {

src/framework/Archiver.ts

@@ -1,28 +1,28 @@
 import {writeFileSync} from 'fs';
 
 export class Archiver {
-    private readonly information: any;
+    private readonly information: Record<string, unknown> = {};
     public readonly archive: string;
 
     constructor(file: string) {
-        this.information = new Map<string, string[]>();
         this.archive = file;
     }
 
-    public set(key: string, value: string | number) {
+    public set(key: string, value: unknown) {
         this.information[key] = value;
     }
 
     public extend(key: string, value: string | number) {
-        if (!Object.prototype.hasOwnProperty.call(this.information, key)) {
+        const values = this.information[key];
+        if (!Array.isArray(values)) {
             this.information[key] = [];
         }
-        this.information[key].push(value);
+        (this.information[key] as (string | number)[]).push(value);
     }
 
     public write() {
         writeFileSync(this.archive, `${JSON.stringify(this.information, null, 2)}\n`, {flag: 'w'});
     }
 
     // TODO also add access functions to compare with previous runs
-}
+}

src/framework/Framework.ts

@@ -65,7 +65,9 @@ export class Framework {
         return this.scheduled;
     }
 
-    public async sequential(suites: Suite[]) {
+    public async sequential(suites: Suite[]): Promise<boolean> {
+        let success: boolean = true;
+
         this.scheduled = this.scheduled.concat(suites);
         this.reporter.start();
         const t0 = performance.now();
@@ -74,14 +76,17 @@ export class Framework {
             for (const suite of suites) {
                 for (const testee of suite.testees) {
                     const order: TestScenario[] = suite.scheduler.sequential(suite);
-                    await this.executeSuite(suite, testee, order, ++executionIndex);
+                    const result = await this.executeSuite(suite, testee, order, ++executionIndex);
+                    success = success && result.outcome === Outcome.succeeded;
                 }
             }
 
             await this.shutdown(suites);
+
+            return success;
         } finally {
             const t1 = performance.now();
-            this.reporter.finish(t1 - t0);
+            await this.reporter.finish(t1 - t0);
             await this.reporter.close();
         }
     }
@@ -107,7 +112,7 @@ export class Framework {
             return success;
         } finally {
             const t1 = performance.now();
-            this.reporter.finish(t1 - t0);
+            await this.reporter.finish(t1 - t0);
             await this.reporter.close();
         }
     }
@@ -137,7 +142,8 @@ export class Framework {
 
                     try {
                         const first: TestScenario = order[i][0];
-                        await timeout<Object | void>('Initialize testbed', testee.connector.timeout, testee.initialize(first.program, first.args ?? []).catch((e: Error) => result.error(e.message)));
+                        await timeout<Object | void>('Initialize testbed', testee.connector.timeout, testee.initialize(first.program, first.args ?? []).catch((e: unknown) => result.error(errorMessage(e))));
+                        this.reportMetadata(runId, testee);
 
                         for (let j = i; j < order.length; j += suite.testees.length) {
                             await this.runSuite(result, testee, order[j], runId);
@@ -155,7 +161,7 @@ export class Framework {
             }))
         } finally {
             const t1 = performance.now();
-            this.reporter.finish(t1 - t0);
+            await this.reporter.finish(t1 - t0);
             await this.reporter.close();
         }
     }
@@ -188,7 +194,8 @@ export class Framework {
 
         try {
             const first: TestScenario = order[0];
-            await timeout<Object | void>('Initialize testbed', testee.connector.timeout, testee.initialize(first.program, first.args ?? []).catch((e: Error) => result.error(e.message)));
+            await timeout<Object | void>('Initialize testbed', testee.connector.timeout, testee.initialize(first.program, first.args ?? []).catch((e: unknown) => result.error(errorMessage(e))));
+            this.reportMetadata(runId, testee);
             await this.runSuite(result, testee, order, runId);
         } catch (e) {
             result.error(e instanceof Error ? e.message : `${e}`);
@@ -209,6 +216,13 @@ export class Framework {
         return `${suite.title}:${testee.name}:${executionIndex}`;
     }
 
+    private reportMetadata(runId: string, testee: Testee): void {
+        const testbed = testee.bed();
+        if (testbed !== undefined) {
+            this.reporter.metadata?.(runId, testbed.meta());
+        }
+    }
+
     public static getImplementation() {
         if (!Framework.implementation) {
             Framework.implementation = new Framework();
@@ -217,3 +231,7 @@ export class Framework {
         return Framework.implementation;
     }
 }
+
+function errorMessage(error: unknown): string {
+    return error instanceof Error ? error.message : String(error);
+}