Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 7 additions & 3 deletions .github/workflows/BuildJobs.yml
Original file line number Diff line number Diff line change
Expand Up @@ -66,10 +66,10 @@ jobs:
- name: Verify changelog entries
run: pnpm run -w check-changelogs
if: >-
${{ github.event_name == 'pull_request' && github.event.pull_request.user.login != 'uicontent' }}
${{ github.event_name == 'pull_request' && github.event.pull_request.user.login != 'uicontent' }}
env:
BASE_SHA: ${{ github.event.pull_request.base.sha }}
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
BASE_SHA: ${{ github.event.pull_request.base.sha }}
HEAD_SHA: ${{ github.event.pull_request.head.sha }}

mxversion:
name: Read versions file
Expand Down Expand Up @@ -134,6 +134,10 @@ jobs:
turbo-cache-${{ runner.os }}-${{ matrix.target }}
- name: Install dependencies
run: pnpm install
# Exercises the release scripts' file and zip helpers with the runner's own tooling
# (zip/unzip on Linux, PowerShell on Windows), so a break shows up before a release does.
- name: Test release scripts
run: pnpm --filter @mendix/automation-utils run test
- name: Run ${{ matrix.target }} task
run: pnpm run ${{ matrix.target }} --concurrency=1 ${{ env.since_flag }}
env:
Expand Down
1 change: 0 additions & 1 deletion .npmrc
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@ public-hoist-pattern[]=make-dir
# public-hoist-pattern rule.
public-hoist-pattern[]=*babel*
public-hoist-pattern[]=big.js
public-hoist-pattern[]=*shelljs*
public-hoist-pattern[]=*rollup*
public-hoist-pattern[]=@types/node
public-hoist-pattern[]=@types/big.js
8 changes: 3 additions & 5 deletions automation/run-e2e/lib/ci.mjs
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
import c from "ansi-colors";
import findFreePort from "find-free-port";
import { execSync } from "node:child_process";
import { globSync } from "node:fs";
import { createServer } from "node:net";
import { networkInterfaces } from "node:os";
import { fileURLToPath } from "node:url";
import p from "node:path";
import sh from "shelljs";
import parseArgs from "yargs-parser";
import {
createDeploymentBundle,
Expand All @@ -20,8 +20,6 @@ import { updateTestProject } from "./update-test-project.mjs";
import { fetchWithReport } from "./utils.mjs";
import * as config from "./config.mjs";

const { ls } = sh;

// Path to the docker directory (contains docker-compose.yml + Dockerfiles)
const DOCKER_DIR = fileURLToPath(new URL("../docker", import.meta.url));
const COMPOSE_FILE = p.join(DOCKER_DIR, "docker-compose.yml");
Expand Down Expand Up @@ -114,7 +112,7 @@ async function runWithCompose({ mendixVersion, ip, freePort }) {
await prepareImage("mxbuild", mendixVersion);
await prepareImage("mxruntime", mendixVersion);

const projectFile = ls(config.mprFileGlob).toString();
const [projectFile] = globSync(config.mprFileGlob);
if (!projectFile) {
throw new Error(`No .mpr file found matching ${config.mprFileGlob}`);
}
Expand Down Expand Up @@ -171,7 +169,7 @@ async function runWithDockerRaw({ mendixVersion, ip, freePort }) {
const mxbuildImage = await prepareImage("mxbuild", mendixVersion);
const mxruntimeImage = await prepareImage("mxruntime", mendixVersion);

const projectFile = ls(config.mprFileGlob).toString();
const [projectFile] = globSync(config.mprFileGlob);
createDeploymentBundle(mxbuildImage, projectFile);

runtimeContainerId = await startRuntime(mxruntimeImage, mendixVersion, ip, freePort, {
Expand Down
10 changes: 4 additions & 6 deletions automation/run-e2e/lib/dev.mjs
Original file line number Diff line number Diff line change
@@ -1,17 +1,15 @@
import { spawnSync, execSync } from "node:child_process";
import { globSync } from "node:fs";
import { delimiter } from "node:path";
import { fileURLToPath } from "node:url";
import parseArgs from "yargs-parser";
import c from "ansi-colors";
import enquirer from "enquirer";
import sh from "shelljs";
import { setupTestProject } from "./setup-test-project.mjs";
import { updateTestProject } from "./update-test-project.mjs";
import { await200 } from "./utils.mjs";
import * as config from "./config.mjs";

const { ls, exec } = sh;

export async function dev() {
console.log(c.cyan("Run e2e tests in development environment"));

Expand All @@ -35,9 +33,9 @@ export async function dev() {
if (!process.env.GITHUB_TOKEN) {
console.log("GITHUB_TOKEN not found. Fetching from GitHub CLI...");

const result = exec("gh auth token", { silent: true });
const result = spawnSync("gh", ["auth", "token"], { encoding: "utf-8" });

if (result.code === 0) {
if (result.status === 0) {
process.env.GITHUB_TOKEN = result.stdout.trim();
console.log("Successfully set GITHUB_TOKEN from gh CLI.");
} else {
Expand Down Expand Up @@ -72,7 +70,7 @@ export async function dev() {

// Print out Mendix version from MPR file
try {
const mprFiles = ls(config.mprFileGlob);
const mprFiles = globSync(config.mprFileGlob);
if (mprFiles.length > 0) {
const mprFile = mprFiles[0];
try {
Expand Down
8 changes: 3 additions & 5 deletions automation/run-e2e/lib/docker-utils.mjs
Original file line number Diff line number Diff line change
@@ -1,12 +1,9 @@
import { execSync, spawn, spawnSync } from "node:child_process";
import { existsSync } from "node:fs";
import { existsSync, readFileSync } from "node:fs";
import p from "node:path";
import { fileURLToPath } from "node:url";
import fetch from "node-fetch";
import c from "ansi-colors";
import sh from "shelljs";

const { cat } = sh;

export function getFullImageName(name, mendixVersion) {
return `${name}:${mendixVersion}`;
Expand Down Expand Up @@ -160,7 +157,7 @@
if (response.ok) {
attempts = 0;
}
} catch (e) {

Check warning on line 160 in automation/run-e2e/lib/docker-utils.mjs

View workflow job for this annotation

GitHub Actions / Run code quality check

'e' is defined but never used
console.log(`Could not reach http://${ip}:${freePort}, trying again...`);
}
await new Promise(resolve => setTimeout(resolve, 3000));
Expand All @@ -169,7 +166,8 @@
if (attempts === 0) {
console.log("Runtime didn't start");
console.log("Print runtime.log...");
console.log(cat("results/runtime.log").toString());
const runtimeLog = "results/runtime.log";
console.log(existsSync(runtimeLog) ? readFileSync(runtimeLog, "utf-8") : `${runtimeLog} not found`);
throw new Error("Runtime didn't start in time, exiting now...");
}

Expand Down
27 changes: 12 additions & 15 deletions automation/run-e2e/lib/setup-test-project.mjs
Original file line number Diff line number Diff line change
@@ -1,23 +1,16 @@
#!/usr/bin/env node
import { createWriteStream } from "node:fs";
import { createWriteStream, existsSync, globSync, mkdirSync, renameSync, rmSync } from "node:fs";
import { join } from "node:path";
import sh from "shelljs";
import crossZip from "cross-zip";
import { packageMeta, fetchWithReport, usetmp, streamPipe } from "./utils.mjs";
import * as config from "./config.mjs";

const { ls, mkdir, rm, mv } = sh;

export async function setupTestProject() {
console.log("Copying test project from GitHub repository");

sh.config.silent = true;
const testsFiles = ls(config.testsDir);
sh.config.silent = false;

if (testsFiles.length !== 0) {
if (existsSync(config.testsDir)) {
console.log("Test project directory already exists, cleaning up...");
rm("-rf", config.testsDir);
rmSync(config.testsDir, { recursive: true, force: true });
}

const archivePath = await downloadTestProject(
Expand All @@ -26,12 +19,16 @@ export async function setupTestProject() {
);

try {
mkdir("-p", config.testsDir);
mkdirSync(config.testsDir, { recursive: true });
crossZip.unzipSync(archivePath, config.testsDir);
mv(config.postUnzipProjectDirGlob, config.testProjectDir);
rm("-f", archivePath);
const [unzippedProjectDir] = globSync(config.postUnzipProjectDirGlob);
if (!unzippedProjectDir) {
throw new Error(`No directory matching ${config.postUnzipProjectDirGlob} after unzipping.`);
}
renameSync(unzippedProjectDir, config.testProjectDir);
rmSync(archivePath, { force: true });

if (ls(config.mprFileGlob).length === 0) {
if (globSync(config.mprFileGlob).length === 0) {
throw new Error('Invalid test project retrieved from GitHub: "mpr" file is missing.');
}
} catch (e) {
Expand All @@ -55,7 +52,7 @@ async function downloadTestProject(repository, branch) {
);
return downloadedArchivePath;
} catch (e) {
rm("-f", downloadedArchivePath);
rmSync(downloadedArchivePath, { force: true });
throw new Error("Cannot find test project in GitHub repository. Try again later.", { cause: e });
}
}
46 changes: 24 additions & 22 deletions automation/run-e2e/lib/update-test-project.mjs
Original file line number Diff line number Diff line change
@@ -1,15 +1,12 @@
import crossZip from "cross-zip";
import assert from "node:assert/strict";
import { spawnSync } from "node:child_process";
import { createWriteStream } from "node:fs";
import { join, resolve } from "node:path";
import sh from "shelljs";
import { cpSync, createWriteStream, existsSync, globSync, mkdirSync, rmSync } from "node:fs";
import { basename, join, resolve } from "node:path";
import * as config from "./config.mjs";
import { fetchGithubRestAPI, fetchWithReport, packageMeta, streamPipe, usetmp } from "./utils.mjs";
import { atlasCoreReleaseUrl } from "./config.mjs";

const { cp, rm, mkdir, test } = sh;

async function getReleaseByTag(tag) {
const url = `${atlasCoreReleaseUrl}/tags/${tag}`;
const response = await fetchGithubRestAPI(url);
Expand All @@ -26,7 +23,7 @@ async function downloadAndExtract(url, downloadPath, extractPath) {
} catch (e) {
throw new Error(`Unable to download and extract from ${url}`, { cause: e });
} finally {
rm("-f", downloadPath);
rmSync(downloadPath, { force: true });
}
}

Expand All @@ -35,7 +32,9 @@ async function updateAtlasThemeSource() {
const tag = `atlas-core-v${version}`;
console.log(`Copying Atlas themesource files from Atlas Core release ${tag}`);

rm("-rf", config.atlasDirsToRemove);
for (const dir of config.atlasDirsToRemove) {
rmSync(dir, { recursive: true, force: true });
}

const release = await getReleaseByTag(tag);
if (!release.assets || release.assets.length === 0) {
Expand All @@ -48,24 +47,24 @@ async function updateAtlasThemeSource() {
await downloadAndExtract(browser_download_url, downloadedPath, outPath);

const themeSourcePath = join(outPath, "themesource");
if (!test("-d", themeSourcePath)) {
if (!existsSync(themeSourcePath)) {
console.log(`Directory not found: ${themeSourcePath}. Creating it.`);
mkdir("-p", themeSourcePath);
mkdirSync(themeSourcePath, { recursive: true });
}

cp("-r", themeSourcePath, config.testProjectDir);
cpSync(themeSourcePath, join(config.testProjectDir, "themesource"), { recursive: true });

// Fix file permissions to ensure Docker can write to theme files
// The Atlas theme files are copied with read-only permissions
// but mxbuild needs to write to some generated files during build
sh.exec(`chmod -R +w "${config.testProjectDir}/themesource"`, { silent: true });
spawnSync("chmod", ["-R", "+w", join(config.testProjectDir, "themesource")]);
}

async function updateAtlasTheme() {
const tag = packageMeta.testProject?.atlasThemeTag ?? config.defaultAtlasThemeTag;
console.log(`Copying Atlas theme files from Atlas UI theme release ${tag}`);

rm("-rf", "tests/testProject/theme");
rmSync("tests/testProject/theme", { recursive: true, force: true });

// Fetch the specific release by tag from GitHub API
const release = await getReleaseByTag(tag);
Expand All @@ -79,16 +78,14 @@ async function updateAtlasTheme() {
await downloadAndExtract(browser_download_url, downloadedPath, outPath);

const themePath = join(outPath, "theme");
if (!test("-d", themePath)) {
if (!existsSync(themePath)) {
console.log(`Directory not found: ${themePath}. Creating it.`);
mkdir("-p", themePath);
mkdirSync(themePath, { recursive: true });
}
const webPath = join(outPath, "web");
const nativePath = join(outPath, "native");
cp("-r", webPath, themePath);
cp("-r", nativePath, themePath);
cpSync(join(outPath, "web"), join(themePath, "web"), { recursive: true });
cpSync(join(outPath, "native"), join(themePath, "native"), { recursive: true });

cp("-r", themePath, config.testProjectDir);
cpSync(themePath, join(config.testProjectDir, "theme"), { recursive: true });
}

async function runReleaseScript() {
Expand All @@ -113,10 +110,15 @@ async function runReleaseScript() {
spawnSync(command, args, { stdio: "inherit", shell: true });

console.log("Copying widget mpk.");
const mpkPath = `dist/${version}/*.mpk`;
const mpkPaths = globSync(`dist/${version}/*.mpk`);
if (mpkPaths.length === 0) {
throw new Error(`No mpk found in dist/${version}`);
}
const outDir = join(config.testProjectDir, "widgets");
mkdir("-p", outDir);
cp("-f", mpkPath, outDir);
mkdirSync(outDir, { recursive: true });
for (const mpkPath of mpkPaths) {
cpSync(mpkPath, join(outDir, basename(mpkPath)));
}
}

async function runUpdateProjectScript() {
Expand Down
1 change: 0 additions & 1 deletion automation/run-e2e/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,6 @@
"enquirer": "^2.4.1",
"find-free-port": "^2.0.0",
"node-fetch": "^3.3.2",
"shelljs": "^0.10.0",
"yargs-parser": "^22.0.0"
},
"devDependencies": {
Expand Down
7 changes: 2 additions & 5 deletions automation/utils/bin/rui-create-gh-release.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import { updateChangelogsAndCreatePR } from "../src/changelog.ts";
import { gh } from "../src/github.ts";
import { addRemoteWithAuthentication } from "../src/index.ts";
import { getPublishedInfo } from "../src/package-info.ts";
import { exec, find } from "../src/shell.ts";
import { exec, findFiles } from "../src/shell.ts";

async function resolveChangelogBranch(releaseTag: string, remoteName: string): Promise<string> {
const { stdout } = await exec(`git rev-parse --abbrev-ref HEAD`, { stdio: "pipe" });
Expand Down Expand Up @@ -75,10 +75,7 @@ async function main(): Promise<void> {
// Create release
console.log("Creating Github release...");

const mpk = find(join(path, "dist", version, "*.mpk")).toString();
if (!mpk) {
throw new Error("MPK file not found");
}
const [mpk] = findFiles(join(path, "dist", version, "*.mpk"));

await gh.createGithubReleaseFrom({
title: `${appName} v${version}`,
Expand Down
5 changes: 2 additions & 3 deletions automation/utils/bin/rui-include-oss-in-artifact.ts
Original file line number Diff line number Diff line change
@@ -1,12 +1,11 @@
#!/usr/bin/env node

import { mkdtemp, stat } from "node:fs/promises";
import { mkdtemp, rm, stat } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import chalk from "chalk";
import { gh } from "../src/github.ts";
import { includeReadmeOssIntoMpk } from "../src/oss-clearance.ts";
import { rm } from "../src/shell.ts";

/**
* This script processes GitHub release artifacts to include READMEOSS HTML files into MPK files.
Expand Down Expand Up @@ -107,7 +106,7 @@ async function main(): Promise<void> {
} finally {
// Cleanup temp files
console.log(chalk.gray("🧹 Cleaning up temporary files..."));
await rm("-rf", tmpFolder);
await rm(tmpFolder, { recursive: true, force: true });
}
}

Expand Down
10 changes: 3 additions & 7 deletions automation/utils/bin/rui-oss-clearance.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,8 @@
// Enable quiet mode for fetch calls to reduce logging noise
process.env.FETCH_QUIET = "true";

import { homedir } from "node:os";
import { cp, mkdtemp, rm, stat } from "node:fs/promises";
import { homedir, tmpdir } from "node:os";
import { basename, join } from "path";
import chalk from "chalk";
import enquirer from "enquirer";
Expand Down Expand Up @@ -331,8 +332,6 @@ async function handleIncludeAsEmbeddedAction(release: GitHubDraftRelease, readme
printInfo(`Found MPK: ${mpkAsset.name} (${mpkAsset.size} bytes)`);

// Create temp folder
const { mkdtemp } = await import("node:fs/promises");
const { tmpdir } = await import("node:os");
const tmpFolder = await mkdtemp(join(tmpdir(), "mpk-oss-embed-"));
const mpkPath = join(tmpFolder, mpkAsset.name);
const htmlPath = join(tmpFolder, basename(readmeToInclude));
Expand All @@ -344,7 +343,6 @@ async function handleIncludeAsEmbeddedAction(release: GitHubDraftRelease, readme
printProgressCheck("Download completed");

// Copy HTML to temp folder
const { cp } = await import("../src/shell.ts");
await cp(readmeToInclude, htmlPath);

// Embed HTML into MPK
Expand All @@ -353,7 +351,6 @@ async function handleIncludeAsEmbeddedAction(release: GitHubDraftRelease, readme
printProgressCheck("Merge completed");

// Get modified MPK size
const { stat } = await import("node:fs/promises");
const modifiedMpkStats = await stat(mpkPath);
const sizeDiff = modifiedMpkStats.size - mpkAsset.size;
printInfo(`Modified MPK size: ${modifiedMpkStats.size} bytes (+${sizeDiff} bytes)`);
Expand Down Expand Up @@ -400,8 +397,7 @@ async function handleIncludeAsEmbeddedAction(release: GitHubDraftRelease, readme
} finally {
// Cleanup temp files
printProgress("Cleaning up temporary files...");
const { rm } = await import("../src/shell.ts");
await rm("-rf", tmpFolder);
await rm(tmpFolder, { recursive: true, force: true });
}
}

Expand Down
Loading
Loading