From 473a29a7e514279594dcc2d64a61c33d905e4456 Mon Sep 17 00:00:00 2001 From: andypalmi Date: Wed, 21 Oct 2026 12:18:21 +0200 Subject: [PATCH 1/3] test: Make the Cypress specs independent of run order Specs left state behind (device groups, pipelines, devices, a rolled back instance, a deactivated stack, a removed team member) that later specs relied on not being there, and the EE environment did not await the RBAC fixtures. Each spec now cleans up after itself or targets its own data, instance state no longer depends on which spec loads it first, and the developer mode and admin navigation races are waited on. --- .../tests-ee/applications/applications.spec.js | 4 ++-- .../tests-ee/applications/pipelines.spec.js | 10 ++++++++++ .../tests-ee/devices/developer-mode.spec.js | 10 ++++++++++ test/e2e/frontend/cypress/tests/admin.spec.js | 8 ++++++++ .../frontend/cypress/tests/admin/stacks.spec.js | 10 +++++----- .../cypress/tests/devices/assignment.spec.js | 13 +++++++++++++ .../instances/version-history/snapshots.spec.js | 3 ++- .../frontend/cypress/tests/team/instances.spec.js | 3 +++ .../cypress/tests/team/team-membership.spec.js | 4 ++-- test/e2e/frontend/environments/standard.js | 14 +++++++++++--- test/e2e/frontend/lib/projectStackFactory.js | 4 +++- test/e2e/frontend/test_environment_ee.js | 2 +- 12 files changed, 70 insertions(+), 15 deletions(-) diff --git a/test/e2e/frontend/cypress/tests-ee/applications/applications.spec.js b/test/e2e/frontend/cypress/tests-ee/applications/applications.spec.js index 4caec1ad07..df7b76a7a7 100644 --- a/test/e2e/frontend/cypress/tests-ee/applications/applications.spec.js +++ b/test/e2e/frontend/cypress/tests-ee/applications/applications.spec.js @@ -178,13 +178,13 @@ describe('FlowFuse - Applications - With Billing', () => { cy.get('[data-el="application-instances"]').should('exist') cy.get('[data-el="application-instances"] [data-state="running"]').should('exist') - cy.get('[data-el="application-instances"] [data-state="running"]').contains('2') + cy.get('[data-el="application-instances"] [data-state="running"]').contains('1') cy.get('[data-el="application-instances"] [data-state="error"]').should('exist') cy.get('[data-el="application-instances"] [data-state="error"]').contains('0') cy.get('[data-el="application-instances"] [data-state="stopped"]').should('exist') - cy.get('[data-el="application-instances"] [data-state="stopped"]').contains('0') + cy.get('[data-el="application-instances"] [data-state="stopped"]').contains('1') cy.get('[data-el="application-devices"] [data-state="running"]').should('exist') cy.get('[data-el="application-devices"] [data-state="running"]').contains('0') diff --git a/test/e2e/frontend/cypress/tests-ee/applications/pipelines.spec.js b/test/e2e/frontend/cypress/tests-ee/applications/pipelines.spec.js index d16b471478..4bf0e20ae9 100644 --- a/test/e2e/frontend/cypress/tests-ee/applications/pipelines.spec.js +++ b/test/e2e/frontend/cypress/tests-ee/applications/pipelines.spec.js @@ -756,6 +756,16 @@ describe('FlowFuse - Application - DevOps Pipelines', () => { cy.get('[data-el="application-device-group-name"] input').should('have.value', GROUP_NAME) cy.get('[data-el="application-device-group-description"] input').should('have.value', `Description for group ${GROUP_NAME}`) + + // Tidy Up, so later specs see only the fixture device group + cy.request('GET', `/api/v1/applications/${application.id}/pipelines`).then((response) => { + const pipeline = response.body.pipelines.find((pipeline) => pipeline.name === PIPELINE_NAME) + return cy.request('DELETE', `/api/v1/pipelines/${pipeline.id}`) + }) + cy.request('GET', `/api/v1/applications/${application.id}/device-groups`).then((response) => { + const group = response.body.groups.find((group) => group.name === GROUP_NAME) + return cy.request('DELETE', `/api/v1/applications/${application.id}/device-groups/${group.id}`) + }) }) it('can not push to a protected instance', () => { diff --git a/test/e2e/frontend/cypress/tests-ee/devices/developer-mode.spec.js b/test/e2e/frontend/cypress/tests-ee/devices/developer-mode.spec.js index 4a9b5a5b16..24a23979e1 100644 --- a/test/e2e/frontend/cypress/tests-ee/devices/developer-mode.spec.js +++ b/test/e2e/frontend/cypress/tests-ee/devices/developer-mode.spec.js @@ -13,6 +13,16 @@ describe('FlowFuse - Devices - With Billing', () => { cy.login('bob', 'bbPassword') + // Start in fleet mode, even if an earlier test's reset had not landed yet + cy.request('GET', '/api/v1/user/teams').then((response) => { + const team = response.body.teams.find((team) => team.name === 'BTeam') + return cy.request('GET', `/api/v1/teams/${team.id}/devices?limit=100`) + }).then((response) => { + response.body.devices + .filter((device) => device.mode === 'developer') + .forEach((device) => cy.request('PUT', `/api/v1/devices/${device.id}/mode`, { mode: 'autonomous' })) + }) + cy.visit('/team/bteam/devices') }) diff --git a/test/e2e/frontend/cypress/tests/admin.spec.js b/test/e2e/frontend/cypress/tests/admin.spec.js index 23b0cbc0e2..ce56dc9eee 100644 --- a/test/e2e/frontend/cypress/tests/admin.spec.js +++ b/test/e2e/frontend/cypress/tests/admin.spec.js @@ -68,6 +68,8 @@ describe('FlowFuse platform admin users', () => { // check that we're on the admin/teams page cy.url().should('match', /\/admin\/teams(\/[^/]+)?/) + cy.intercept('GET', '/api/*/teams/*/projects*').as('getBTeamHomeInstances') + // Not a member of BTeam cy.get('[data-el="teams-table"]').contains('BTeam').click() @@ -75,6 +77,8 @@ describe('FlowFuse platform admin users', () => { cy.url().should('match', /\/overview(\/[^/]+)?/) cy.get('[data-el="banner-team-as-admin"]').should('exist') + // A nav click made before the overview has loaded can be lost + cy.wait('@getBTeamHomeInstances') // Unique alias to avoid colliding with cy.home()'s getTeamApplications cy.intercept('GET', '/api/*/teams/*/applications*').as('getBTeamApplications') @@ -105,12 +109,16 @@ describe('FlowFuse platform admin users', () => { cy.intercept('GET', '/api/*/teams/*/devices*').as('getDevices') cy.intercept('GET', '/api/*/devices/*').as('getDevice') + cy.intercept('GET', '/api/*/teams/*/projects*').as('getBTeamHomeInstances') + cy.visit('/admin/overview') cy.get('[data-nav="admin-teams"]').click() // Not a member of BTeam cy.get('[data-el="teams-table"]').contains('BTeam').click() + // A nav click made before the overview has loaded can be lost + cy.wait('@getBTeamHomeInstances') cy.get('[data-nav="team-applications"]').click() diff --git a/test/e2e/frontend/cypress/tests/admin/stacks.spec.js b/test/e2e/frontend/cypress/tests/admin/stacks.spec.js index 670a18d6f7..76a7cf75e0 100644 --- a/test/e2e/frontend/cypress/tests/admin/stacks.spec.js +++ b/test/e2e/frontend/cypress/tests/admin/stacks.spec.js @@ -13,7 +13,7 @@ describe('FlowFuse - Stacks', () => { }) it('loads stacks into relevant tables', () => { - cy.get('[data-el="active-stacks"] tbody').find('tr').should('have.length', 3) + cy.get('[data-el="active-stacks"] tbody').find('tr').should('have.length', 4) cy.get('[data-el="active-stacks"] tbody').contains('td', 'stack1') cy.get('[data-el="inactive-stacks"] tbody').find('tr').should('have.length', 1) @@ -23,7 +23,7 @@ describe('FlowFuse - Stacks', () => { it('can create a new version of a stack, and deactivate the replaced stack', () => { cy.intercept('POST', '/api/*/stacks').as('createStack') - cy.get('[data-el="active-stacks"] tbody tr:first .ff-kebab-menu').click() + cy.get('[data-el="active-stacks"] tbody').contains('tr', 'stack-to-replace').find('.ff-kebab-menu').click() cy.get('[data-el="kebab-options"].ff-kebab-options').find('.ff-kebab-item').eq(0).click() cy.get('.ff-dialog-box').should('be.visible') @@ -35,12 +35,12 @@ describe('FlowFuse - Stacks', () => { cy.wait('@createStack') // check it has been created - cy.get('[data-el="active-stacks"] tbody').find('tr').should('have.length', 3) - cy.get('[data-el="active-stacks"] tbody').contains('td', 'stack1-copy') + cy.get('[data-el="active-stacks"] tbody').find('tr').should('have.length', 4) + cy.get('[data-el="active-stacks"] tbody').contains('td', 'stack-to-replace-copy') // and that old stack has been moved cy.get('[data-el="inactive-stacks"] tbody').find('tr').should('have.length', 1) - cy.get('[data-el="inactive-stacks"] tbody').contains('td', 'stack1') + cy.get('[data-el="inactive-stacks"] tbody').contains('td', 'stack-to-replace') }) it('can load multiple pages of stacks when the API paginates', function () { diff --git a/test/e2e/frontend/cypress/tests/devices/assignment.spec.js b/test/e2e/frontend/cypress/tests/devices/assignment.spec.js index 2a137d6794..c3f7f5787d 100644 --- a/test/e2e/frontend/cypress/tests/devices/assignment.spec.js +++ b/test/e2e/frontend/cypress/tests/devices/assignment.spec.js @@ -145,6 +145,19 @@ describe('FlowFuse - Application - Devices - Create', () => { cy.login('bob', 'bbPassword') }) + after(() => { + // Remove the devices created in this spec, so later specs see only the fixture devices + cy.login('bob', 'bbPassword') + cy.request('GET', '/api/v1/user/teams').then((response) => { + const team = response.body.teams.find((team) => team.name === 'BTeam') + return cy.request('GET', `/api/v1/teams/${team.id}/devices?limit=100`) + }).then((response) => { + response.body.devices + .filter((device) => device.name.startsWith('new-device-')) + .forEach((device) => cy.request('DELETE', `/api/v1/devices/${device.id}`)) + }) + }) + it('creates the Device and assigned it to the current application', () => { navigateToApplicationDevices('BTeam', 'application-2') const deviceName = 'new-device-auto-assigned--' + Date.now() diff --git a/test/e2e/frontend/cypress/tests/instances/version-history/snapshots.spec.js b/test/e2e/frontend/cypress/tests/instances/version-history/snapshots.spec.js index d1498f050f..e2ffd9f776 100644 --- a/test/e2e/frontend/cypress/tests/instances/version-history/snapshots.spec.js +++ b/test/e2e/frontend/cypress/tests/instances/version-history/snapshots.spec.js @@ -682,7 +682,8 @@ describe('FlowFuse - Instance Snapshots', () => { it('Can rollback a snapshot', () => { // Premise: Ensure the rollback endpoint is available and callable // (NOTE: this is not testing the full mechanics of the rollback feature, only to prevent repeat regression. See #2032) - cy.intercept('POST', '/api/*/projects/*/actions/rollback').as('rollbackSnapshot') + // The response is stubbed so the instance keeps its settings for later specs + cy.intercept('POST', '/api/*/projects/*/actions/rollback', { statusCode: 200, body: {} }).as('rollbackSnapshot') cy.get('[data-el="snapshots"] tbody').find('tr').first().click() diff --git a/test/e2e/frontend/cypress/tests/team/instances.spec.js b/test/e2e/frontend/cypress/tests/team/instances.spec.js index a6cfb0355c..dcd18acc76 100644 --- a/test/e2e/frontend/cypress/tests/team/instances.spec.js +++ b/test/e2e/frontend/cypress/tests/team/instances.spec.js @@ -76,10 +76,13 @@ describe('Team - Instances', () => { }) it('are shown a list of dashboards and can open one in the viewer', () => { + cy.intercept('GET', '/api/*/teams/*/dashboard-instances').as('getDashboardInstances') + cy.login('dashboard-dave', 'ddPassword') cy.visit('/team/ateam') cy.wait('@getTeam') + cy.wait('@getDashboardInstances') cy.get('[data-el="dashboards-table"]').within(() => { cy.contains('instance-1-1') diff --git a/test/e2e/frontend/cypress/tests/team/team-membership.spec.js b/test/e2e/frontend/cypress/tests/team/team-membership.spec.js index 413a975aba..4c5f186a42 100644 --- a/test/e2e/frontend/cypress/tests/team/team-membership.spec.js +++ b/test/e2e/frontend/cypress/tests/team/team-membership.spec.js @@ -90,8 +90,8 @@ describe('FlowFuse - Team Membership', () => { // check we now have 4 members (charlie was added in previous test) cy.get('[data-el="members-table"] tbody tr:not(.collapsible)').should('have.length', 4) - // open click kebab menu of 3rd member (charlie) - cy.get('[data-el="members-table"] tbody').find('.ff-kebab-menu').eq(2).click() + // open kebab menu of charlie (added in previous test) + cy.get('[data-el="members-table"] tbody tr:not(.collapsible)').contains('tr', 'charlie').find('.ff-kebab-menu').click() // click "member-remove-from-team" cy.get('[data-el="kebab-options"].ff-kebab-options').find('[data-action="member-remove-from-team"]').click() diff --git a/test/e2e/frontend/environments/standard.js b/test/e2e/frontend/environments/standard.js index d524926250..f363dd0c68 100644 --- a/test/e2e/frontend/environments/standard.js +++ b/test/e2e/frontend/environments/standard.js @@ -61,6 +61,15 @@ module.exports = async function (settings = {}, config = {}) { const factory = new TestModelFactory(forge) + // The stub driver reports an instance that was never started as 'unknown', and the first + // live status request stores that. Store it up front so the state does not depend on + // whether an earlier spec has loaded the instance. + async function createUnstartedInstance (details, application, stack, template, projectType, options = {}) { + const instance = await factory.createInstance(details, application, stack, template, projectType, { ...options, start: false }) + await instance.update({ state: 'unknown' }) + return instance + } + // Create users // full platform & team1 admin const userAlice = await factory.createUser({ admin: true, username: 'alice', name: 'Alice Skywalker', email: 'alice@example.com', email_verified: true, password: 'aaPassword' }) @@ -161,8 +170,7 @@ module.exports = async function (settings = {}, config = {}) { } } }) - await factory.createInstance({ name: 'instance-1-2' }, application1, stack2, template, projectType, { - start: false, + await createUnstartedInstance({ name: 'instance-1-2' }, application1, stack2, template, projectType, { settings: { palette: { modules: [ @@ -199,7 +207,7 @@ module.exports = async function (settings = {}, config = {}) { const application2 = await factory.createApplication({ name: 'application-2' }, team2, stack, template, projectType) await factory.createInstance({ name: 'instance-2-1' }, application2, stack, template, projectType) - const instanceWithDevices = await factory.createInstance({ name: 'instance-2-with-devices' }, application2, stack, template, projectType, { start: false }) + const instanceWithDevices = await createUnstartedInstance({ name: 'instance-2-with-devices' }, application2, stack, template, projectType) await factory.createDevice({ name: 'assigned-device-a', type: 'type2', lastSeenAt: Date.now() }, team2, instanceWithDevices) await factory.createDevice({ name: 'assigned-device-b', type: 'type2', lastSeenAt: Date.now() }, team2, instanceWithDevices) await factory.createDevice({ name: 'assigned-device-c', type: 'type2', lastSeenAt: Date.now() }, team2, instanceWithDevices) diff --git a/test/e2e/frontend/lib/projectStackFactory.js b/test/e2e/frontend/lib/projectStackFactory.js index 28628e0158..ce0a54ce4c 100644 --- a/test/e2e/frontend/lib/projectStackFactory.js +++ b/test/e2e/frontend/lib/projectStackFactory.js @@ -16,8 +16,10 @@ async function init (forge) { factory.createStack({ name: 'stack2', label: 'stack 2' }, projectType), factory.createStack({ name: 'stack1-for-type2', label: 'stack 1 for type2' }, spareProjectType) ]) + // Not used by any instance, so admin/stacks.spec.js can replace it without affecting other specs + const stackToReplace = await factory.createStack({ name: 'stack-to-replace', label: 'stack to replace' }, projectType) _projectStacks = { - stack1, stack2, stack3 + stack1, stack2, stack3, stackToReplace } return _projectStacks })() diff --git a/test/e2e/frontend/test_environment_ee.js b/test/e2e/frontend/test_environment_ee.js index 13958808e8..54169f3143 100644 --- a/test/e2e/frontend/test_environment_ee.js +++ b/test/e2e/frontend/test_environment_ee.js @@ -160,7 +160,7 @@ if (fs.existsSync(configPath)) { const deviceB = flowforge.applicationDevices.find((device) => device.name === 'application-device-b') await factory.createDeviceSnapshot({ name: 'application-device-b snapshot 1', lastSeenAt: Date.now() }, deviceB, userTerry) - rbacTeamFactory(flowforge) + await rbacTeamFactory(flowforge) flowforge.listen({ port: PORT }, function (err, address) { console.info(`EE Environment running at http://localhost:${PORT}`) From eedf6242334f2b336c70dd0effa00de49c792277 Mon Sep 17 00:00:00 2001 From: andypalmi Date: Wed, 21 Oct 2026 12:28:37 +0200 Subject: [PATCH 2/3] test: Compare application devices against the API count devices/bulk.spec.js deletes one of the application-2 devices, so the fixed count failed whenever it ran first. --- .../frontend/cypress/tests/applications/devices.spec.js | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/test/e2e/frontend/cypress/tests/applications/devices.spec.js b/test/e2e/frontend/cypress/tests/applications/devices.spec.js index 2360129cc7..ba99f8231e 100644 --- a/test/e2e/frontend/cypress/tests/applications/devices.spec.js +++ b/test/e2e/frontend/cypress/tests/applications/devices.spec.js @@ -30,7 +30,12 @@ describe('FlowFuse - Application - Devices', () => { cy.visit(`/team/${team.slug}/applications/${application.id}/devices`) cy.url().should('include', `/applications/${application.id}/devices`) - cy.get('[data-el="devices-browser"] tbody').find('tr').should('have.length', 3) + // devices/bulk.spec.js deletes application-device-c-bulk-test-3, so compare against the API + cy.request('GET', `/api/v1/applications/${application.id}/devices`).then((response) => { + cy.get('[data-el="devices-browser"] tbody').find('tr').should('have.length', response.body.devices.length) + }) + cy.get('[data-el="devices-browser"] tbody').contains('application-device-a') + cy.get('[data-el="devices-browser"] tbody').contains('application-device-b') }) it('is presented with a dialog confirmation when choosing "Remove from Application"', () => { From 192fb40ca30451f9233da82330926adbbf3914fe Mon Sep 17 00:00:00 2001 From: andypalmi Date: Wed, 21 Oct 2026 13:13:06 +0200 Subject: [PATCH 3/3] test: Wait for the application options to close before picking an instance The closing application options stay in the DOM during their leave transition, so the instance name could be read from them, failing the test after the device was assigned. Each test now also starts with the device unassigned, so one failure no longer fails its retries and devices/assignment.spec.js. --- test/e2e/frontend/cypress/tests/devices.spec.js | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/test/e2e/frontend/cypress/tests/devices.spec.js b/test/e2e/frontend/cypress/tests/devices.spec.js index a8ed4c3c42..e2e79a5757 100644 --- a/test/e2e/frontend/cypress/tests/devices.spec.js +++ b/test/e2e/frontend/cypress/tests/devices.spec.js @@ -242,6 +242,11 @@ describe('FlowFuse - Team Devices', () => { cy.request('GET', '/api/v1/user/teams') .then((response) => { const team = response.body.teams.find((team) => team.name === 'BTeam') + // A failed assign test would otherwise leave the device assigned for its retries and later specs + cy.request('GET', `/api/v1/teams/${team.id}/devices?limit=100`).then((response) => { + const device = response.body.devices.find((device) => device.name === 'team2-unassigned-device') + cy.request('PUT', `/api/v1/devices/${device.id}`, { instance: null, application: null }) + }) cy.visit(`/team/${team.slug}/devices`) cy.wait('@getDevices') }) @@ -524,6 +529,8 @@ describe('FlowFuse - Team Devices', () => { // Click first Application cy.get('[data-el="listbox-options"] > .ff-option:first').click() + // The closing application options stay in the DOM during their leave transition + cy.get('[data-el="listbox-options"]').should('not.exist') cy.get('[data-el="assign-device-to-instance-dialog"]') .within(() => {