test: Make the Cypress specs independent of run order

test/e2e/frontend/cypress/tests-ee/applications/applications.spec.js

@@ -178,13 +178,13 @@ describe('FlowFuse - Applications - With Billing', () => {
 
                     cy.get('[data-el="application-instances"]').should('exist')
                     cy.get('[data-el="application-instances"] [data-state="running"]').should('exist')
-                    cy.get('[data-el="application-instances"] [data-state="running"]').contains('2')
+                    cy.get('[data-el="application-instances"] [data-state="running"]').contains('1')
 
                     cy.get('[data-el="application-instances"] [data-state="error"]').should('exist')
                     cy.get('[data-el="application-instances"] [data-state="error"]').contains('0')
 
                     cy.get('[data-el="application-instances"] [data-state="stopped"]').should('exist')
-                    cy.get('[data-el="application-instances"] [data-state="stopped"]').contains('0')
+                    cy.get('[data-el="application-instances"] [data-state="stopped"]').contains('1')
 
                     cy.get('[data-el="application-devices"] [data-state="running"]').should('exist')
                     cy.get('[data-el="application-devices"] [data-state="running"]').contains('0')

test/e2e/frontend/cypress/tests-ee/applications/pipelines.spec.js

@@ -756,6 +756,16 @@ describe('FlowFuse - Application - DevOps Pipelines', () => {
 
         cy.get('[data-el="application-device-group-name"] input').should('have.value', GROUP_NAME)
         cy.get('[data-el="application-device-group-description"] input').should('have.value', `Description for group ${GROUP_NAME}`)
+
+        // Tidy Up, so later specs see only the fixture device group
+        cy.request('GET', `/api/v1/applications/${application.id}/pipelines`).then((response) => {
+            const pipeline = response.body.pipelines.find((pipeline) => pipeline.name === PIPELINE_NAME)
+            return cy.request('DELETE', `/api/v1/pipelines/${pipeline.id}`)
+        })
+        cy.request('GET', `/api/v1/applications/${application.id}/device-groups`).then((response) => {
+            const group = response.body.groups.find((group) => group.name === GROUP_NAME)
+            return cy.request('DELETE', `/api/v1/applications/${application.id}/device-groups/${group.id}`)
+        })
     })
 
     it('can not push to a protected instance', () => {

test/e2e/frontend/cypress/tests-ee/devices/developer-mode.spec.js

@@ -13,6 +13,16 @@ describe('FlowFuse - Devices - With Billing', () => {
 
         cy.login('bob', 'bbPassword')
 
+        // Start in fleet mode, even if an earlier test's reset had not landed yet
+        cy.request('GET', '/api/v1/user/teams').then((response) => {
+            const team = response.body.teams.find((team) => team.name === 'BTeam')
+            return cy.request('GET', `/api/v1/teams/${team.id}/devices?limit=100`)
+        }).then((response) => {
+            response.body.devices
+                .filter((device) => device.mode === 'developer')
+                .forEach((device) => cy.request('PUT', `/api/v1/devices/${device.id}/mode`, { mode: 'autonomous' }))
+        })
+
         cy.visit('/team/bteam/devices')
     })
 

test/e2e/frontend/cypress/tests/admin.spec.js

@@ -68,13 +68,17 @@ describe('FlowFuse platform admin users', () => {
         // check that we're on the admin/teams page
         cy.url().should('match', /\/admin\/teams(\/[^/]+)?/)
 
+        cy.intercept('GET', '/api/*/teams/*/projects*').as('getBTeamHomeInstances')
+
         // Not a member of BTeam
         cy.get('[data-el="teams-table"]').contains('BTeam').click()
 
         // check that we're on the selected team's overview page
         cy.url().should('match', /\/overview(\/[^/]+)?/)
 
         cy.get('[data-el="banner-team-as-admin"]').should('exist')
+        // A nav click made before the overview has loaded can be lost
+        cy.wait('@getBTeamHomeInstances')
 
         // Unique alias to avoid colliding with cy.home()'s getTeamApplications
         cy.intercept('GET', '/api/*/teams/*/applications*').as('getBTeamApplications')
@@ -105,12 +109,16 @@ describe('FlowFuse platform admin users', () => {
         cy.intercept('GET', '/api/*/teams/*/devices*').as('getDevices')
         cy.intercept('GET', '/api/*/devices/*').as('getDevice')
 
+        cy.intercept('GET', '/api/*/teams/*/projects*').as('getBTeamHomeInstances')
+
         cy.visit('/admin/overview')
 
         cy.get('[data-nav="admin-teams"]').click()
 
         // Not a member of BTeam
         cy.get('[data-el="teams-table"]').contains('BTeam').click()
+        // A nav click made before the overview has loaded can be lost
+        cy.wait('@getBTeamHomeInstances')
 
         cy.get('[data-nav="team-applications"]').click()
 

test/e2e/frontend/cypress/tests/admin/stacks.spec.js

@@ -13,7 +13,7 @@ describe('FlowFuse - Stacks', () => {
     })
 
     it('loads stacks into relevant tables', () => {
-        cy.get('[data-el="active-stacks"] tbody').find('tr').should('have.length', 3)
+        cy.get('[data-el="active-stacks"] tbody').find('tr').should('have.length', 4)
         cy.get('[data-el="active-stacks"] tbody').contains('td', 'stack1')
 
         cy.get('[data-el="inactive-stacks"] tbody').find('tr').should('have.length', 1)
@@ -23,7 +23,7 @@ describe('FlowFuse - Stacks', () => {
     it('can create a new version of a stack, and deactivate the replaced stack', () => {
         cy.intercept('POST', '/api/*/stacks').as('createStack')
 
-        cy.get('[data-el="active-stacks"] tbody tr:first .ff-kebab-menu').click()
+        cy.get('[data-el="active-stacks"] tbody').contains('tr', 'stack-to-replace').find('.ff-kebab-menu').click()
         cy.get('[data-el="kebab-options"].ff-kebab-options').find('.ff-kebab-item').eq(0).click()
 
         cy.get('.ff-dialog-box').should('be.visible')
@@ -35,12 +35,12 @@ describe('FlowFuse - Stacks', () => {
         cy.wait('@createStack')
 
         // check it has been created
-        cy.get('[data-el="active-stacks"] tbody').find('tr').should('have.length', 3)
-        cy.get('[data-el="active-stacks"] tbody').contains('td', 'stack1-copy')
+        cy.get('[data-el="active-stacks"] tbody').find('tr').should('have.length', 4)
+        cy.get('[data-el="active-stacks"] tbody').contains('td', 'stack-to-replace-copy')
 
         // and that old stack has been moved
         cy.get('[data-el="inactive-stacks"] tbody').find('tr').should('have.length', 1)
-        cy.get('[data-el="inactive-stacks"] tbody').contains('td', 'stack1')
+        cy.get('[data-el="inactive-stacks"] tbody').contains('td', 'stack-to-replace')
     })
 
     it('can load multiple pages of stacks when the API paginates', function () {

test/e2e/frontend/cypress/tests/applications/devices.spec.js

@@ -30,7 +30,12 @@ describe('FlowFuse - Application - Devices', () => {
         cy.visit(`/team/${team.slug}/applications/${application.id}/devices`)
         cy.url().should('include', `/applications/${application.id}/devices`)
 
-        cy.get('[data-el="devices-browser"] tbody').find('tr').should('have.length', 3)
+        // devices/bulk.spec.js deletes application-device-c-bulk-test-3, so compare against the API
+        cy.request('GET', `/api/v1/applications/${application.id}/devices`).then((response) => {
+            cy.get('[data-el="devices-browser"] tbody').find('tr').should('have.length', response.body.devices.length)
+        })
+        cy.get('[data-el="devices-browser"] tbody').contains('application-device-a')
+        cy.get('[data-el="devices-browser"] tbody').contains('application-device-b')
     })
 
     it('is presented with a dialog confirmation when choosing "Remove from Application"', () => {

test/e2e/frontend/cypress/tests/devices.spec.js

@@ -242,6 +242,11 @@ describe('FlowFuse - Team Devices', () => {
             cy.request('GET', '/api/v1/user/teams')
                 .then((response) => {
                     const team = response.body.teams.find((team) => team.name === 'BTeam')
+                    // A failed assign test would otherwise leave the device assigned for its retries and later specs
+                    cy.request('GET', `/api/v1/teams/${team.id}/devices?limit=100`).then((response) => {
+                        const device = response.body.devices.find((device) => device.name === 'team2-unassigned-device')
+                        cy.request('PUT', `/api/v1/devices/${device.id}`, { instance: null, application: null })
+                    })
                     cy.visit(`/team/${team.slug}/devices`)
                     cy.wait('@getDevices')
                 })
@@ -524,6 +529,8 @@ describe('FlowFuse - Team Devices', () => {
 
             // Click first Application
             cy.get('[data-el="listbox-options"] > .ff-option:first').click()
+            // The closing application options stay in the DOM during their leave transition
+            cy.get('[data-el="listbox-options"]').should('not.exist')
 
             cy.get('[data-el="assign-device-to-instance-dialog"]')
                 .within(() => {

test/e2e/frontend/cypress/tests/devices/assignment.spec.js

@@ -145,6 +145,19 @@ describe('FlowFuse - Application - Devices - Create', () => {
         cy.login('bob', 'bbPassword')
     })
 
+    after(() => {
+        // Remove the devices created in this spec, so later specs see only the fixture devices
+        cy.login('bob', 'bbPassword')
+        cy.request('GET', '/api/v1/user/teams').then((response) => {
+            const team = response.body.teams.find((team) => team.name === 'BTeam')
+            return cy.request('GET', `/api/v1/teams/${team.id}/devices?limit=100`)
+        }).then((response) => {
+            response.body.devices
+                .filter((device) => device.name.startsWith('new-device-'))
+                .forEach((device) => cy.request('DELETE', `/api/v1/devices/${device.id}`))
+        })
+    })
+
     it('creates the Device and assigned it to the current application', () => {
         navigateToApplicationDevices('BTeam', 'application-2')
         const deviceName = 'new-device-auto-assigned--' + Date.now()

test/e2e/frontend/cypress/tests/instances/version-history/snapshots.spec.js

@@ -682,7 +682,8 @@ describe('FlowFuse - Instance Snapshots', () => {
     it('Can rollback a snapshot', () => {
         // Premise: Ensure the rollback endpoint is available and callable
         // (NOTE: this is not testing the full mechanics of the rollback feature, only to prevent repeat regression. See #2032)
-        cy.intercept('POST', '/api/*/projects/*/actions/rollback').as('rollbackSnapshot')
+        // The response is stubbed so the instance keeps its settings for later specs
+        cy.intercept('POST', '/api/*/projects/*/actions/rollback', { statusCode: 200, body: {} }).as('rollbackSnapshot')
 
         cy.get('[data-el="snapshots"] tbody').find('tr').first().click()
 

test/e2e/frontend/cypress/tests/team/instances.spec.js

@@ -76,10 +76,13 @@ describe('Team - Instances', () => {
         })
 
         it('are shown a list of dashboards and can open one in the viewer', () => {
+            cy.intercept('GET', '/api/*/teams/*/dashboard-instances').as('getDashboardInstances')
+
             cy.login('dashboard-dave', 'ddPassword')
             cy.visit('/team/ateam')
 
             cy.wait('@getTeam')
+            cy.wait('@getDashboardInstances')
 
             cy.get('[data-el="dashboards-table"]').within(() => {
                 cy.contains('instance-1-1')

test/e2e/frontend/cypress/tests/team/team-membership.spec.js

@@ -90,8 +90,8 @@ describe('FlowFuse - Team Membership', () => {
         // check we now have 4 members (charlie was added in previous test)
         cy.get('[data-el="members-table"] tbody tr:not(.collapsible)').should('have.length', 4)
 
-        // open click kebab menu of 3rd member (charlie)
-        cy.get('[data-el="members-table"] tbody').find('.ff-kebab-menu').eq(2).click()
+        // open kebab menu of charlie (added in previous test)
+        cy.get('[data-el="members-table"] tbody tr:not(.collapsible)').contains('tr', 'charlie').find('.ff-kebab-menu').click()
 
         // click "member-remove-from-team"
         cy.get('[data-el="kebab-options"].ff-kebab-options').find('[data-action="member-remove-from-team"]').click()

test/e2e/frontend/environments/standard.js

@@ -61,6 +61,15 @@ module.exports = async function (settings = {}, config = {}) {
 
     const factory = new TestModelFactory(forge)
 
+    // The stub driver reports an instance that was never started as 'unknown', and the first
+    // live status request stores that. Store it up front so the state does not depend on
+    // whether an earlier spec has loaded the instance.
+    async function createUnstartedInstance (details, application, stack, template, projectType, options = {}) {
+        const instance = await factory.createInstance(details, application, stack, template, projectType, { ...options, start: false })
+        await instance.update({ state: 'unknown' })
+        return instance
+    }
+
     // Create users
     // full platform & team1 admin
     const userAlice = await factory.createUser({ admin: true, username: 'alice', name: 'Alice Skywalker', email: 'alice@example.com', email_verified: true, password: 'aaPassword' })
@@ -161,8 +170,7 @@ module.exports = async function (settings = {}, config = {}) {
             }
         }
     })
-    await factory.createInstance({ name: 'instance-1-2' }, application1, stack2, template, projectType, {
-        start: false,
+    await createUnstartedInstance({ name: 'instance-1-2' }, application1, stack2, template, projectType, {
         settings: {
             palette: {
                 modules: [
@@ -199,7 +207,7 @@ module.exports = async function (settings = {}, config = {}) {
     const application2 = await factory.createApplication({ name: 'application-2' }, team2, stack, template, projectType)
     await factory.createInstance({ name: 'instance-2-1' }, application2, stack, template, projectType)
 
-    const instanceWithDevices = await factory.createInstance({ name: 'instance-2-with-devices' }, application2, stack, template, projectType, { start: false })
+    const instanceWithDevices = await createUnstartedInstance({ name: 'instance-2-with-devices' }, application2, stack, template, projectType)
     await factory.createDevice({ name: 'assigned-device-a', type: 'type2', lastSeenAt: Date.now() }, team2, instanceWithDevices)
     await factory.createDevice({ name: 'assigned-device-b', type: 'type2', lastSeenAt: Date.now() }, team2, instanceWithDevices)
     await factory.createDevice({ name: 'assigned-device-c', type: 'type2', lastSeenAt: Date.now() }, team2, instanceWithDevices)

test/e2e/frontend/lib/projectStackFactory.js

@@ -16,8 +16,10 @@ async function init (forge) {
             factory.createStack({ name: 'stack2', label: 'stack 2' }, projectType),
             factory.createStack({ name: 'stack1-for-type2', label: 'stack 1 for type2' }, spareProjectType)
         ])
+        // Not used by any instance, so admin/stacks.spec.js can replace it without affecting other specs
+        const stackToReplace = await factory.createStack({ name: 'stack-to-replace', label: 'stack to replace' }, projectType)
         _projectStacks = {
-            stack1, stack2, stack3
+            stack1, stack2, stack3, stackToReplace
         }
         return _projectStacks
     })()

test/e2e/frontend/test_environment_ee.js

@@ -160,7 +160,7 @@ if (fs.existsSync(configPath)) {
     const deviceB = flowforge.applicationDevices.find((device) => device.name === 'application-device-b')
     await factory.createDeviceSnapshot({ name: 'application-device-b snapshot 1', lastSeenAt: Date.now() }, deviceB, userTerry)
 
-    rbacTeamFactory(flowforge)
+    await rbacTeamFactory(flowforge)
 
     flowforge.listen({ port: PORT }, function (err, address) {
         console.info(`EE Environment running at http://localhost:${PORT}`)