diff --git a/README.md b/README.md index f8c1da32c..293cd67ea 100644 --- a/README.md +++ b/README.md @@ -121,6 +121,11 @@ Allure.step("Create order", () -> { For HTTP clients, browser tools, assertion libraries, and other integrations, add the matching module from the catalog above. Each module README shows the registration snippet for that tool. +## Muted Test Execution + +See [Skipping muted tests](docs/muted-tests.md) for test-plan configuration, supported runners, and +skipped-result versus discovery-exclusion behavior. + ## Resources - [Allure Report documentation](https://allurereport.org/docs/) diff --git a/allure-citrus/README.md b/allure-citrus/README.md index 5764240f6..9615fde4f 100644 --- a/allure-citrus/README.md +++ b/allure-citrus/README.md @@ -35,6 +35,11 @@ Maven, with `allure-bom` imported in dependency management: Register `io.qameta.allure.citrus.AllureCitrus` with Citrus as a test, test suite, and test action listener. The listener translates Citrus suite, test case, and action events into Allure tests and steps. +## Muted Tests + +Muted-test execution skipping is not supported through this listener alone; its callbacks occur too late for a safe execution veto. +See [Skipping muted tests](../docs/muted-tests.md) for plan configuration, matching, and lifecycle limits. + ## Report Output - Citrus test cases and suite lifecycle. diff --git a/allure-cucumber7-jvm/README.md b/allure-cucumber7-jvm/README.md index 936bf4831..9517105a2 100644 --- a/allure-cucumber7-jvm/README.md +++ b/allure-cucumber7-jvm/README.md @@ -42,6 +42,11 @@ Register the plugin with Cucumber: You can also pass the plugin through your Cucumber runner or build tool configuration. +## Muted Tests + +An automatically discovered backend skips muted scenarios and retains skipped-and-muted results. +See [Skipping muted tests](../docs/muted-tests.md) for plan configuration, matching, and lifecycle limits. + ## Report Output - Features, scenarios, scenario outlines, examples, hooks, and steps. diff --git a/allure-cucumber7-jvm/build.gradle.kts b/allure-cucumber7-jvm/build.gradle.kts index 756df3c10..9fba00671 100644 --- a/allure-cucumber7-jvm/build.gradle.kts +++ b/allure-cucumber7-jvm/build.gradle.kts @@ -8,6 +8,8 @@ dependencies { api(project(":allure-java-commons")) compileOnly(platform("io.cucumber:cucumber-bom:$cucumberVersion")) compileOnly("io.cucumber:cucumber-plugin") + compileOnly("io.cucumber:cucumber-core") + implementation("org.opentest4j:opentest4j:1.3.0") compileOnly("io.cucumber:gherkin") testImplementation("commons-io:commons-io") testImplementation(platform("io.cucumber:cucumber-bom:$cucumberVersion")) @@ -26,9 +28,16 @@ dependencies { testRuntimeOnly("org.junit.platform:junit-platform-launcher") } +tasks.processResources { + from("src/main/services") { + into("META-INF/services") + } +} + tasks.test { useJUnitPlatform() systemProperty("allure.test.cucumber.version", cucumberVersion) + systemProperty("allure.test.runtime.classpath", sourceSets.test.get().runtimeClasspath.asPath) } val minimumCucumberTestRuntimeClasspath = configurations.testRuntimeClasspath.get().copyRecursive().apply { @@ -55,9 +64,14 @@ val cucumber7MinimumVersionTest = tasks.register("cucumber7MinimumVersionT ) } + filter { + includeTestsMatching("io.qameta.allure.cucumber7jvm.AllureMutedExecutionTest") + } + val standardTest = tasks.test.get() systemProperties(standardTest.systemProperties) systemProperty("allure.test.cucumber.version", minimumCucumberVersion) + systemProperty("allure.test.runtime.classpath", classpath.asPath) jvmArgs = standardTest.jvmArgs maxHeapSize = standardTest.maxHeapSize maxParallelForks = standardTest.maxParallelForks diff --git a/allure-cucumber7-jvm/src/main/java/io/qameta/allure/cucumber7jvm/AllureBackendProvider.java b/allure-cucumber7-jvm/src/main/java/io/qameta/allure/cucumber7jvm/AllureBackendProvider.java new file mode 100644 index 000000000..d35bb13ea --- /dev/null +++ b/allure-cucumber7-jvm/src/main/java/io/qameta/allure/cucumber7jvm/AllureBackendProvider.java @@ -0,0 +1,98 @@ +/* + * Copyright 2016-2026 Qameta Software Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.qameta.allure.cucumber7jvm; + +import io.cucumber.core.backend.Backend; +import io.cucumber.core.backend.BackendProviderService; +import io.cucumber.core.backend.Container; +import io.cucumber.core.backend.Glue; +import io.cucumber.core.backend.HookDefinition; +import io.cucumber.core.backend.Lookup; +import io.cucumber.core.backend.Snippet; +import io.cucumber.core.backend.TestCaseState; +import org.opentest4j.TestAbortedException; + +import java.net.URI; +import java.util.List; +import java.util.function.Supplier; + +/** + * Automatically installs the native pre-scenario muted-test guard. + */ +public class AllureBackendProvider implements BackendProviderService { + + @Override + public Backend create(final Lookup lookup, final Container container, + final Supplier classLoader) { + return new MutedBackend(); + } + + private static final class MutedBackend implements Backend { + + private final AllureTestPlan plan = new AllureTestPlan(); + + @Override + public void loadGlue(final Glue glue, final List gluePaths) { + if (!plan.hasExcludedTests()) { + return; + } + glue.addBeforeHook(new HookDefinition() { + @Override + public void execute(final TestCaseState state) { + if (plan.isMuted(state.getSourceTagNames(), state.getUri(), state.getLine())) { + throw new TestAbortedException("Skipped by Allure test plan"); + } + } + + @Override + public String getTagExpression() { + return ""; + } + + @Override + public int getOrder() { + return Integer.MIN_VALUE; + } + + @Override + public boolean isDefinedAt(final StackTraceElement element) { + return false; + } + + @Override + public String getLocation() { + return AllureBackendProvider.class.getName(); + } + }); + } + + @Override + public void buildWorld() { + // The hook requires no scenario-scoped world. + } + + @Override + public void disposeWorld() { + // The hook owns no scenario resources. + } + + @Override + public Snippet getSnippet() { + // Reporting backends do not supply step-definition snippets. + return null; + } + } +} diff --git a/allure-cucumber7-jvm/src/main/java/io/qameta/allure/cucumber7jvm/AllureCucumber7Jvm.java b/allure-cucumber7-jvm/src/main/java/io/qameta/allure/cucumber7jvm/AllureCucumber7Jvm.java index d7699cf68..a1884f06b 100644 --- a/allure-cucumber7-jvm/src/main/java/io/qameta/allure/cucumber7jvm/AllureCucumber7Jvm.java +++ b/allure-cucumber7-jvm/src/main/java/io/qameta/allure/cucumber7jvm/AllureCucumber7Jvm.java @@ -54,7 +54,6 @@ import java.io.ByteArrayInputStream; import java.nio.charset.StandardCharsets; -import java.nio.file.Paths; import java.util.Collections; import java.util.Deque; import java.util.LinkedList; @@ -119,7 +118,7 @@ public class AllureCucumber7Jvm implements ConcurrentEventListener { private final Map hookStepContainerUuid = new ConcurrentHashMap<>(); private static final String TEXT_PLAIN = "text/plain"; - private static final String CUCUMBER_WORKING_DIR = Paths.get("").toUri().getSchemeSpecificPart(); + private final AllureTestPlan testPlan = new AllureTestPlan(); /** * Creates an Allure cucumber7 jvm with default configuration. @@ -260,7 +259,11 @@ private void handleTestCaseFinished(final TestCaseFinished event) { final TagParser tagParser = new TagParser(feature, testCase); statusDetails .setFlaky(tagParser.isFlaky()) - .setMuted(tagParser.isMuted()) + .setMuted( + tagParser.isMuted() || testPlan.isMuted( + testCase.getTags(), testCase.getUri(), testCase.getLocation().getLine() + ) + ) .setKnown(tagParser.isKnown()); final AllureExternalKey testKey = testKey(uuid); @@ -398,11 +401,7 @@ private String getTestCaseId(final TestCase testCase) { } private String getTestCaseUri(final TestCase testCase) { - final String testCaseUri = testCase.getUri().getSchemeSpecificPart(); - if (testCaseUri.startsWith(CUCUMBER_WORKING_DIR)) { - return testCaseUri.substring(CUCUMBER_WORKING_DIR.length()); - } - return testCaseUri; + return ScenarioUtils.scenarioUri(testCase.getUri()); } private Status translateTestCaseStatus(final Result testCaseResult) { diff --git a/allure-cucumber7-jvm/src/main/java/io/qameta/allure/cucumber7jvm/AllureTestPlan.java b/allure-cucumber7-jvm/src/main/java/io/qameta/allure/cucumber7jvm/AllureTestPlan.java new file mode 100644 index 000000000..f88c03ba5 --- /dev/null +++ b/allure-cucumber7-jvm/src/main/java/io/qameta/allure/cucumber7jvm/AllureTestPlan.java @@ -0,0 +1,53 @@ +/* + * Copyright 2016-2026 Qameta Software Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.qameta.allure.cucumber7jvm; + +import io.qameta.allure.testfilter.FileTestPlanSupplier; +import io.qameta.allure.testfilter.TestPlan; +import io.qameta.allure.testfilter.TestPlanV1_0; + +import java.net.URI; +import java.util.Collection; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Test-plan muting decisions using the scenario's reporting identity. + */ +final class AllureTestPlan { + + private static final Pattern ALLURE_ID = Pattern.compile("^@allure\\.id[:=](.+)$"); + + private final TestPlan plan = new FileTestPlanSupplier().supply().orElse(null); + + boolean hasExcludedTests() { + return plan instanceof TestPlanV1_0 + && ((TestPlanV1_0) plan).getExcludedTests() != null + && !((TestPlanV1_0) plan).getExcludedTests().isEmpty(); + } + + boolean isMuted(final Collection tags, final URI uri, final int line) { + if (!(plan instanceof TestPlanV1_0)) { + return false; + } + final TestPlanV1_0 knownPlan = (TestPlanV1_0) plan; + final String selector = ScenarioUtils.scenarioUri(uri) + ":" + line; + return knownPlan.isMuted(null, selector) || tags.stream() + .map(ALLURE_ID::matcher) + .filter(Matcher::matches) + .anyMatch(matcher -> knownPlan.isMuted(matcher.group(1), selector)); + } +} diff --git a/allure-cucumber7-jvm/src/main/java/io/qameta/allure/cucumber7jvm/ScenarioUtils.java b/allure-cucumber7-jvm/src/main/java/io/qameta/allure/cucumber7jvm/ScenarioUtils.java new file mode 100644 index 000000000..276476916 --- /dev/null +++ b/allure-cucumber7-jvm/src/main/java/io/qameta/allure/cucumber7jvm/ScenarioUtils.java @@ -0,0 +1,35 @@ +/* + * Copyright 2016-2026 Qameta Software Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.qameta.allure.cucumber7jvm; + +import java.net.URI; +import java.nio.file.Paths; + +/** + * Shared scenario identities for execution and reporting. + */ +final class ScenarioUtils { + + private static final String WORKING_DIRECTORY = Paths.get("").toUri().getSchemeSpecificPart(); + + private ScenarioUtils() { + } + + static String scenarioUri(final URI uri) { + final String path = uri.getSchemeSpecificPart(); + return path.startsWith(WORKING_DIRECTORY) ? path.substring(WORKING_DIRECTORY.length()) : path; + } +} diff --git a/allure-cucumber7-jvm/src/main/java/module-info.java b/allure-cucumber7-jvm/src/main/java/module-info.java index d7f072e0f..c8327f0f8 100644 --- a/allure-cucumber7-jvm/src/main/java/module-info.java +++ b/allure-cucumber7-jvm/src/main/java/module-info.java @@ -16,9 +16,13 @@ module io.qameta.allure.cucumber7jvm { requires transitive io.qameta.allure.commons; requires io.cucumber.plugin; + requires io.cucumber.core; + requires org.opentest4j; requires io.cucumber.gherkin; requires io.cucumber.messages; requires org.slf4j; exports io.qameta.allure.cucumber7jvm; + + provides io.cucumber.core.backend.BackendProviderService with io.qameta.allure.cucumber7jvm.AllureBackendProvider; } diff --git a/allure-cucumber7-jvm/src/main/services/io.cucumber.core.backend.BackendProviderService b/allure-cucumber7-jvm/src/main/services/io.cucumber.core.backend.BackendProviderService new file mode 100644 index 000000000..cefd2ca69 --- /dev/null +++ b/allure-cucumber7-jvm/src/main/services/io.cucumber.core.backend.BackendProviderService @@ -0,0 +1 @@ +io.qameta.allure.cucumber7jvm.AllureBackendProvider diff --git a/allure-cucumber7-jvm/src/test/java/io/qameta/allure/cucumber7jvm/AllureMutedExecutionTest.java b/allure-cucumber7-jvm/src/test/java/io/qameta/allure/cucumber7jvm/AllureMutedExecutionTest.java new file mode 100644 index 000000000..197afc5e2 --- /dev/null +++ b/allure-cucumber7-jvm/src/test/java/io/qameta/allure/cucumber7jvm/AllureMutedExecutionTest.java @@ -0,0 +1,103 @@ +/* + * Copyright 2016-2026 Qameta Software Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.qameta.allure.cucumber7jvm; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.assertj.core.api.Assertions.assertThat; + +class AllureMutedExecutionTest { + + @TempDir + private Path directory; + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void shouldSkipMutedBodiesUsingAutomaticBackend(final boolean allMuted) throws Exception { + final Path feature = directory.resolve("muted.feature"); + Files.writeString( + feature, "Feature: muted execution\n" + + " @allure.id:123\n" + + " Scenario Outline: muted row\n" + + " Given a muted body must not execute\n" + + " Examples:\n" + + " | value |\n" + + " | one |\n" + + " | two |\n" + + (allMuted ? "" : " Scenario: control\n Given an active body executes\n") + ); + final Path plan = directory.resolve("testplan.json"); + Files.writeString( + plan, allMuted + ? "{\"version\":\"1.0\",\"excludedTests\":[{\"id\":\"unmatched\",\"selector\":\"" + + feature.toUri().getSchemeSpecificPart() + ":7\"},{\"selector\":\"" + + feature.toUri().getSchemeSpecificPart() + ":8\"}]}" + : "{\"version\":\"1.0\",\"excludedTests\":[{\"id\":123}]}" + ); + final Path results = directory.resolve("results"); + final Path output = directory.resolve("output.txt"); + final ProcessBuilder builder = new ProcessBuilder( + Path.of(System.getProperty("java.home"), "bin", "java").toString(), + "-Dallure.results.directory=" + results, + "-cp", System.getProperty("allure.test.runtime.classpath"), + "io.cucumber.core.cli.Main", + "--glue", "io.qameta.allure.cucumber7jvm.muted", + "--plugin", "io.qameta.allure.cucumber7jvm.AllureCucumber7Jvm", feature.toString() + ).redirectErrorStream(true).redirectOutput(output.toFile()); + builder.environment().put("ALLURE_TESTPLAN_PATH", plan.toString()); + builder.environment().put("ALLURE_RESULTS_DIRECTORY", results.toString()); + + final Process process = builder.start(); + final boolean finished = process.waitFor(60, TimeUnit.SECONDS); + if (!finished) { + process.destroyForcibly(); + } + + assertThat(finished).as(Files.readString(output)).isTrue(); + assertThat(process.exitValue()).as(Files.readString(output)).isZero(); + final ObjectMapper mapper = new ObjectMapper(); + final List emitted; + try (Stream paths = Files.list(results)) { + emitted = paths.filter(path -> path.toString().endsWith("-result.json")) + .map(path -> { + try { + return mapper.readTree(path.toFile()); + } catch (java.io.IOException error) { + throw new java.io.UncheckedIOException(error); + } + }).collect(Collectors.toList()); + } + assertThat(emitted).hasSize(allMuted ? 2 : 3); + assertThat(emitted).filteredOn(result -> result.path("name").asText().startsWith("muted row")) + .hasSize(2).allSatisfy(result -> { + assertThat(result.path("status").asText()).isEqualTo("skipped"); + assertThat(result.path("statusDetails").path("muted").asBoolean()).isTrue(); + }); + assertThat(emitted).filteredOn(result -> result.path("name").asText().equals("control")) + .allSatisfy(result -> assertThat(result.path("status").asText()).isEqualTo("passed")); + } +} diff --git a/allure-cucumber7-jvm/src/test/java/io/qameta/allure/cucumber7jvm/muted/MutedSteps.java b/allure-cucumber7-jvm/src/test/java/io/qameta/allure/cucumber7jvm/muted/MutedSteps.java new file mode 100644 index 000000000..743b23ef8 --- /dev/null +++ b/allure-cucumber7-jvm/src/test/java/io/qameta/allure/cucumber7jvm/muted/MutedSteps.java @@ -0,0 +1,31 @@ +/* + * Copyright 2016-2026 Qameta Software Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.qameta.allure.cucumber7jvm.muted; + +import io.cucumber.java.en.Given; + +public class MutedSteps { + + @Given("a muted body must not execute") + public void mutedBody() { + throw new AssertionError("Muted body executed"); + } + + @Given("an active body executes") + public void control() { + // Successful control scenario. + } +} diff --git a/allure-cucumber8-jvm/README.md b/allure-cucumber8-jvm/README.md index 1eefabd14..d460cde31 100644 --- a/allure-cucumber8-jvm/README.md +++ b/allure-cucumber8-jvm/README.md @@ -43,6 +43,11 @@ Register the plugin with Cucumber: You can also pass the plugin through your Cucumber runner or build tool configuration. +## Muted Tests + +An automatically discovered backend skips muted scenarios and retains skipped-and-muted results. +See [Skipping muted tests](../docs/muted-tests.md) for plan configuration, matching, and lifecycle limits. + ## Report Output - Features, scenarios, scenario outlines, examples, hooks, and steps. diff --git a/allure-cucumber8-jvm/build.gradle.kts b/allure-cucumber8-jvm/build.gradle.kts index 607b71d47..da2478576 100644 --- a/allure-cucumber8-jvm/build.gradle.kts +++ b/allure-cucumber8-jvm/build.gradle.kts @@ -7,6 +7,8 @@ dependencies { api(project(":allure-java-commons")) compileOnly(platform("io.cucumber:cucumber-bom:$cucumberVersion")) compileOnly("io.cucumber:cucumber-plugin") + compileOnly("io.cucumber:cucumber-core") + implementation("org.opentest4j:opentest4j:1.3.0") compileOnly("io.cucumber:gherkin") testImplementation("commons-io:commons-io") testImplementation(platform("io.cucumber:cucumber-bom:$cucumberVersion")) @@ -16,6 +18,7 @@ dependencies { testImplementation("io.github.glytching:junit-extensions") testImplementation("org.assertj:assertj-core") testImplementation("org.junit.jupiter:junit-jupiter-api") + testImplementation("org.junit.jupiter:junit-jupiter-params") testImplementation("org.slf4j:slf4j-simple") testImplementation(project(":allure-assertj")) testImplementation(project(":allure-java-commons-test")) @@ -24,7 +27,14 @@ dependencies { testRuntimeOnly("org.junit.platform:junit-platform-launcher") } +tasks.processResources { + from("src/main/services") { + into("META-INF/services") + } +} + tasks.test { useJUnitPlatform() systemProperty("allure.test.cucumber.version", cucumberVersion) + systemProperty("allure.test.runtime.classpath", sourceSets.test.get().runtimeClasspath.asPath) } diff --git a/allure-cucumber8-jvm/src/main/java/io/qameta/allure/cucumber8jvm/AllureBackendProvider.java b/allure-cucumber8-jvm/src/main/java/io/qameta/allure/cucumber8jvm/AllureBackendProvider.java new file mode 100644 index 000000000..294044acf --- /dev/null +++ b/allure-cucumber8-jvm/src/main/java/io/qameta/allure/cucumber8jvm/AllureBackendProvider.java @@ -0,0 +1,98 @@ +/* + * Copyright 2016-2026 Qameta Software Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.qameta.allure.cucumber8jvm; + +import io.cucumber.core.backend.Backend; +import io.cucumber.core.backend.BackendProviderService; +import io.cucumber.core.backend.Container; +import io.cucumber.core.backend.Glue; +import io.cucumber.core.backend.HookDefinition; +import io.cucumber.core.backend.Lookup; +import io.cucumber.core.backend.Snippet; +import io.cucumber.core.backend.TestCaseState; +import org.opentest4j.TestAbortedException; + +import java.net.URI; +import java.util.List; +import java.util.function.Supplier; + +/** + * Automatically installs the native pre-scenario muted-test guard. + */ +public class AllureBackendProvider implements BackendProviderService { + + @Override + public Backend create(final Lookup lookup, final Container container, + final Supplier classLoader) { + return new MutedBackend(); + } + + private static final class MutedBackend implements Backend { + + private final AllureTestPlan plan = new AllureTestPlan(); + + @Override + public void loadGlue(final Glue glue, final List gluePaths) { + if (!plan.hasExcludedTests()) { + return; + } + glue.addBeforeHook(new HookDefinition() { + @Override + public void execute(final TestCaseState state) { + if (plan.isMuted(state.getSourceTagNames(), state.getUri(), state.getLine())) { + throw new TestAbortedException("Skipped by Allure test plan"); + } + } + + @Override + public String getTagExpression() { + return ""; + } + + @Override + public int getOrder() { + return Integer.MIN_VALUE; + } + + @Override + public boolean isDefinedAt(final StackTraceElement element) { + return false; + } + + @Override + public String getLocation() { + return AllureBackendProvider.class.getName(); + } + }); + } + + @Override + public void buildWorld() { + // The hook requires no scenario-scoped world. + } + + @Override + public void disposeWorld() { + // The hook owns no scenario resources. + } + + @Override + public Snippet getSnippet() { + // Reporting backends do not supply step-definition snippets. + return null; + } + } +} diff --git a/allure-cucumber8-jvm/src/main/java/io/qameta/allure/cucumber8jvm/AllureCucumber8Jvm.java b/allure-cucumber8-jvm/src/main/java/io/qameta/allure/cucumber8jvm/AllureCucumber8Jvm.java index a45c6e4ee..26722be8d 100644 --- a/allure-cucumber8-jvm/src/main/java/io/qameta/allure/cucumber8jvm/AllureCucumber8Jvm.java +++ b/allure-cucumber8-jvm/src/main/java/io/qameta/allure/cucumber8jvm/AllureCucumber8Jvm.java @@ -58,7 +58,6 @@ import java.io.ByteArrayInputStream; import java.io.PrintWriter; import java.nio.charset.StandardCharsets; -import java.nio.file.Paths; import java.util.Collections; import java.util.Deque; import java.util.LinkedList; @@ -124,7 +123,7 @@ public class AllureCucumber8Jvm implements ConcurrentEventListener { private final Map hookStepContainerUuid = new ConcurrentHashMap<>(); private static final String TEXT_PLAIN = "text/plain"; - private static final String CUCUMBER_WORKING_DIR = Paths.get("").toUri().getSchemeSpecificPart(); + private final AllureTestPlan testPlan = new AllureTestPlan(); /** * Creates an Allure cucumber8 jvm with default configuration. @@ -348,7 +347,11 @@ private void handleTestCaseFinished(final TestCaseFinished event) { final TagParser tagParser = new TagParser(feature, testCase); statusDetails .setFlaky(tagParser.isFlaky()) - .setMuted(tagParser.isMuted()) + .setMuted( + tagParser.isMuted() || testPlan.isMuted( + testCase.getTags(), testCase.getUri(), testCase.getLocation().getLine() + ) + ) .setKnown(tagParser.isKnown()); final AllureExternalKey testKey = testKey(uuid); @@ -486,11 +489,7 @@ private String getTestCaseId(final TestCase testCase) { } private String getTestCaseUri(final TestCase testCase) { - final String testCaseUri = testCase.getUri().getSchemeSpecificPart(); - if (testCaseUri.startsWith(CUCUMBER_WORKING_DIR)) { - return testCaseUri.substring(CUCUMBER_WORKING_DIR.length()); - } - return testCaseUri; + return ScenarioUtils.scenarioUri(testCase.getUri()); } private Status translateTestCaseStatus(final Result testCaseResult) { diff --git a/allure-cucumber8-jvm/src/main/java/io/qameta/allure/cucumber8jvm/AllureTestPlan.java b/allure-cucumber8-jvm/src/main/java/io/qameta/allure/cucumber8jvm/AllureTestPlan.java new file mode 100644 index 000000000..d95ced55c --- /dev/null +++ b/allure-cucumber8-jvm/src/main/java/io/qameta/allure/cucumber8jvm/AllureTestPlan.java @@ -0,0 +1,53 @@ +/* + * Copyright 2016-2026 Qameta Software Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.qameta.allure.cucumber8jvm; + +import io.qameta.allure.testfilter.FileTestPlanSupplier; +import io.qameta.allure.testfilter.TestPlan; +import io.qameta.allure.testfilter.TestPlanV1_0; + +import java.net.URI; +import java.util.Collection; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Test-plan muting decisions using the scenario's reporting identity. + */ +final class AllureTestPlan { + + private static final Pattern ALLURE_ID = Pattern.compile("^@allure\\.id[:=](.+)$"); + + private final TestPlan plan = new FileTestPlanSupplier().supply().orElse(null); + + boolean hasExcludedTests() { + return plan instanceof TestPlanV1_0 + && ((TestPlanV1_0) plan).getExcludedTests() != null + && !((TestPlanV1_0) plan).getExcludedTests().isEmpty(); + } + + boolean isMuted(final Collection tags, final URI uri, final int line) { + if (!(plan instanceof TestPlanV1_0)) { + return false; + } + final TestPlanV1_0 knownPlan = (TestPlanV1_0) plan; + final String selector = ScenarioUtils.scenarioUri(uri) + ":" + line; + return knownPlan.isMuted(null, selector) || tags.stream() + .map(ALLURE_ID::matcher) + .filter(Matcher::matches) + .anyMatch(matcher -> knownPlan.isMuted(matcher.group(1), selector)); + } +} diff --git a/allure-cucumber8-jvm/src/main/java/io/qameta/allure/cucumber8jvm/ScenarioUtils.java b/allure-cucumber8-jvm/src/main/java/io/qameta/allure/cucumber8jvm/ScenarioUtils.java new file mode 100644 index 000000000..6238c77cf --- /dev/null +++ b/allure-cucumber8-jvm/src/main/java/io/qameta/allure/cucumber8jvm/ScenarioUtils.java @@ -0,0 +1,35 @@ +/* + * Copyright 2016-2026 Qameta Software Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.qameta.allure.cucumber8jvm; + +import java.net.URI; +import java.nio.file.Paths; + +/** + * Shared scenario identities for execution and reporting. + */ +final class ScenarioUtils { + + private static final String WORKING_DIRECTORY = Paths.get("").toUri().getSchemeSpecificPart(); + + private ScenarioUtils() { + } + + static String scenarioUri(final URI uri) { + final String path = uri.getSchemeSpecificPart(); + return path.startsWith(WORKING_DIRECTORY) ? path.substring(WORKING_DIRECTORY.length()) : path; + } +} diff --git a/allure-cucumber8-jvm/src/main/java/module-info.java b/allure-cucumber8-jvm/src/main/java/module-info.java index 7d1e38527..733bc45f1 100644 --- a/allure-cucumber8-jvm/src/main/java/module-info.java +++ b/allure-cucumber8-jvm/src/main/java/module-info.java @@ -16,9 +16,13 @@ module io.qameta.allure.cucumber8jvm { requires transitive io.qameta.allure.commons; requires io.cucumber.plugin; + requires io.cucumber.core; + requires org.opentest4j; requires io.cucumber.gherkin; requires io.cucumber.messages; requires org.slf4j; exports io.qameta.allure.cucumber8jvm; + + provides io.cucumber.core.backend.BackendProviderService with io.qameta.allure.cucumber8jvm.AllureBackendProvider; } diff --git a/allure-cucumber8-jvm/src/main/services/io.cucumber.core.backend.BackendProviderService b/allure-cucumber8-jvm/src/main/services/io.cucumber.core.backend.BackendProviderService new file mode 100644 index 000000000..d49cc41f5 --- /dev/null +++ b/allure-cucumber8-jvm/src/main/services/io.cucumber.core.backend.BackendProviderService @@ -0,0 +1 @@ +io.qameta.allure.cucumber8jvm.AllureBackendProvider diff --git a/allure-cucumber8-jvm/src/test/java/io/qameta/allure/cucumber8jvm/AllureMutedExecutionTest.java b/allure-cucumber8-jvm/src/test/java/io/qameta/allure/cucumber8jvm/AllureMutedExecutionTest.java new file mode 100644 index 000000000..6b361d95d --- /dev/null +++ b/allure-cucumber8-jvm/src/test/java/io/qameta/allure/cucumber8jvm/AllureMutedExecutionTest.java @@ -0,0 +1,103 @@ +/* + * Copyright 2016-2026 Qameta Software Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.qameta.allure.cucumber8jvm; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.assertj.core.api.Assertions.assertThat; + +class AllureMutedExecutionTest { + + @TempDir + private Path directory; + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void shouldSkipMutedBodiesUsingAutomaticBackend(final boolean allMuted) throws Exception { + final Path feature = directory.resolve("muted.feature"); + Files.writeString( + feature, "Feature: muted execution\n" + + " @allure.id:123\n" + + " Scenario Outline: muted row\n" + + " Given a muted body must not execute\n" + + " Examples:\n" + + " | value |\n" + + " | one |\n" + + " | two |\n" + + (allMuted ? "" : " Scenario: control\n Given an active body executes\n") + ); + final Path plan = directory.resolve("testplan.json"); + Files.writeString( + plan, allMuted + ? "{\"version\":\"1.0\",\"excludedTests\":[{\"id\":\"unmatched\",\"selector\":\"" + + feature.toUri().getSchemeSpecificPart() + ":7\"},{\"selector\":\"" + + feature.toUri().getSchemeSpecificPart() + ":8\"}]}" + : "{\"version\":\"1.0\",\"excludedTests\":[{\"id\":123}]}" + ); + final Path results = directory.resolve("results"); + final Path output = directory.resolve("output.txt"); + final ProcessBuilder builder = new ProcessBuilder( + Path.of(System.getProperty("java.home"), "bin", "java").toString(), + "-Dallure.results.directory=" + results, + "-cp", System.getProperty("allure.test.runtime.classpath"), + "io.cucumber.core.cli.Main", + "--glue", "io.qameta.allure.cucumber8jvm.muted", + "--plugin", "io.qameta.allure.cucumber8jvm.AllureCucumber8Jvm", feature.toString() + ).redirectErrorStream(true).redirectOutput(output.toFile()); + builder.environment().put("ALLURE_TESTPLAN_PATH", plan.toString()); + builder.environment().put("ALLURE_RESULTS_DIRECTORY", results.toString()); + + final Process process = builder.start(); + final boolean finished = process.waitFor(60, TimeUnit.SECONDS); + if (!finished) { + process.destroyForcibly(); + } + + assertThat(finished).as(Files.readString(output)).isTrue(); + assertThat(process.exitValue()).as(Files.readString(output)).isZero(); + final ObjectMapper mapper = new ObjectMapper(); + final List emitted; + try (Stream paths = Files.list(results)) { + emitted = paths.filter(path -> path.toString().endsWith("-result.json")) + .map(path -> { + try { + return mapper.readTree(path.toFile()); + } catch (java.io.IOException error) { + throw new java.io.UncheckedIOException(error); + } + }).collect(Collectors.toList()); + } + assertThat(emitted).hasSize(allMuted ? 2 : 3); + assertThat(emitted).filteredOn(result -> result.path("name").asText().startsWith("muted row")) + .hasSize(2).allSatisfy(result -> { + assertThat(result.path("status").asText()).isEqualTo("skipped"); + assertThat(result.path("statusDetails").path("muted").asBoolean()).isTrue(); + }); + assertThat(emitted).filteredOn(result -> result.path("name").asText().equals("control")) + .allSatisfy(result -> assertThat(result.path("status").asText()).isEqualTo("passed")); + } +} diff --git a/allure-cucumber8-jvm/src/test/java/io/qameta/allure/cucumber8jvm/muted/MutedSteps.java b/allure-cucumber8-jvm/src/test/java/io/qameta/allure/cucumber8jvm/muted/MutedSteps.java new file mode 100644 index 000000000..9db714a01 --- /dev/null +++ b/allure-cucumber8-jvm/src/test/java/io/qameta/allure/cucumber8jvm/muted/MutedSteps.java @@ -0,0 +1,31 @@ +/* + * Copyright 2016-2026 Qameta Software Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.qameta.allure.cucumber8jvm.muted; + +import io.cucumber.java.en.Given; + +public class MutedSteps { + + @Given("a muted body must not execute") + public void mutedBody() { + throw new AssertionError("Muted body executed"); + } + + @Given("an active body executes") + public void control() { + // Successful control scenario. + } +} diff --git a/allure-java-commons/src/main/java/io/qameta/allure/testfilter/TestPlanV1_0.java b/allure-java-commons/src/main/java/io/qameta/allure/testfilter/TestPlanV1_0.java index 46796cc70..a4b78c393 100644 --- a/allure-java-commons/src/main/java/io/qameta/allure/testfilter/TestPlanV1_0.java +++ b/allure-java-commons/src/main/java/io/qameta/allure/testfilter/TestPlanV1_0.java @@ -16,7 +16,6 @@ package io.qameta.allure.testfilter; import java.io.Serializable; -import java.util.Arrays; import java.util.List; import java.util.Objects; @@ -36,6 +35,27 @@ public class TestPlanV1_0 implements TestPlan, Serializable { */ private List tests; + /** + * Tests whose execution should be skipped independently of selection. + */ + private List excludedTests; + + /** + * @return tests whose execution should be skipped, or null if unspecified. + */ + public List getExcludedTests() { + return excludedTests; + } + + /** + * @param excludedTests tests whose execution should be skipped. + * @return current instance. + */ + public TestPlanV1_0 setExcludedTests(final List excludedTests) { + this.excludedTests = excludedTests; + return this; + } + /** * Gets the value of the tests property. * @@ -62,9 +82,28 @@ public TestPlanV1_0 setTests(final List tests) { * @return true if test case with given allureId or selection is selected, false otherwise. */ public boolean isSelected(final String allureId, final String selector) { - return getTests().stream().map(test -> new String[]{test.getId(), test.getSelector()}) - .flatMap(array -> Arrays.stream(array).filter(Objects::nonNull)) - .anyMatch(item -> item.equals(selector) || item.equals(allureId)); + return tests == null || tests.isEmpty() || matches(tests, allureId, selector); + } + + /** + * @param allureId value of the test's Allure ID, or null if absent. + * @param selector exact framework selector, or null if absent. + * @return true if the test matches a muted test entry. + */ + public boolean isMuted(final String allureId, final String selector) { + return matches(excludedTests, allureId, selector); + } + + private static boolean matches(final List entries, final String allureId, final String selector) { + return entries != null && entries.stream().filter(Objects::nonNull) + .anyMatch( + test -> matchesIdentity(test.getId(), allureId) + || matchesIdentity(test.getSelector(), selector) + ); + } + + private static boolean matchesIdentity(final String expected, final String actual) { + return expected != null && !expected.isEmpty() && expected.equals(actual); } /** diff --git a/allure-java-commons/src/main/module/module-info.java b/allure-java-commons/src/main/module/module-info.java index 7aa8abc41..a0dba3d50 100644 --- a/allure-java-commons/src/main/module/module-info.java +++ b/allure-java-commons/src/main/module/module-info.java @@ -30,8 +30,11 @@ exports io.qameta.allure.http; exports io.qameta.allure.listener; exports io.qameta.allure.testfilter to + io.qameta.allure.cucumber7jvm, + io.qameta.allure.cucumber8jvm, io.qameta.allure.junitplatform, io.qameta.allure.junit4, + io.qameta.allure.karate, io.qameta.allure.spock2, io.qameta.allure.testng; exports io.qameta.allure.internal.json to diff --git a/allure-java-commons/src/test/java/io/qameta/allure/testfilter/TestPlanV1_0Test.java b/allure-java-commons/src/test/java/io/qameta/allure/testfilter/TestPlanV1_0Test.java index 6a06eb87b..d7521f714 100644 --- a/allure-java-commons/src/test/java/io/qameta/allure/testfilter/TestPlanV1_0Test.java +++ b/allure-java-commons/src/test/java/io/qameta/allure/testfilter/TestPlanV1_0Test.java @@ -20,6 +20,8 @@ import java.io.ByteArrayInputStream; import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Collections; import java.util.List; import static org.assertj.core.api.Assertions.assertThat; @@ -68,6 +70,104 @@ void shouldFallbackToUnknownPlanForUnknownVersion() throws Exception { ); } + @Test + void shouldDeserializeExcludedOnlyPlan() throws Exception { + final TestPlanV1_0 plan = (TestPlanV1_0) readPlan( + "{\"version\":\"1.0\",\"excludedTests\":[{\"selector\":\"pkg.Test.case\"}]}" + ); + + assertThat(plan.isSelected(null, "pkg.Test.case")).isTrue(); + assertThat(plan.isMuted(null, "pkg.Test.case")).isTrue(); + assertThat(plan.isMuted(null, "pkg.Test.other")).isFalse(); + } + + @Test + void shouldNormalizeNumericExcludedIds() throws Exception { + final TestPlanV1_0 plan = (TestPlanV1_0) readPlan( + "{\"version\":\"1.0\",\"excludedTests\":[{\"id\":123}]}" + ); + + assertThat(plan.isMuted("123", null)).isTrue(); + assertThat(plan.getExcludedTests()).extracting(TestPlanV1_0.TestCase::getId).containsExactly("123"); + } + + @Test + void shouldMatchMutedIdentityAlternatives() { + final TestPlanV1_0 plan = new TestPlanV1_0().setExcludedTests( + List.of( + new TestPlanV1_0.TestCase().setId("123").setSelector("pkg.Test.case") + ) + ); + + assertThat(plan.isMuted("123", "other")).isTrue(); + assertThat(plan.isMuted("unmatched", "pkg.Test.case")).isTrue(); + assertThat(plan.isMuted("unmatched", "other")).isFalse(); + } + + @Test + void shouldIgnoreMissingMutedIdentities() { + final TestPlanV1_0 plan = new TestPlanV1_0().setExcludedTests( + Arrays.asList( + null, new TestPlanV1_0.TestCase(), new TestPlanV1_0.TestCase().setId("").setSelector("") + ) + ); + + assertThat(plan.isMuted(null, null)).isFalse(); + assertThat(plan.isMuted("", "")).isFalse(); + assertThat(plan.isMuted(null, "unrelated")).isFalse(); + } + + @Test + void shouldKeepSelectionIndependentOfMuting() { + final TestPlanV1_0 plan = new TestPlanV1_0() + .setTests(List.of(new TestPlanV1_0.TestCase().setId("selected"))) + .setExcludedTests(List.of(new TestPlanV1_0.TestCase().setId("muted"))); + + assertThat(plan.isSelected("muted", null)).isFalse(); + assertThat(plan.isMuted("muted", null)).isTrue(); + assertThat(plan.isSelected("selected", null)).isTrue(); + assertThat(plan.isMuted("selected", null)).isFalse(); + } + + @Test + void shouldLeaveSelectionUnrestrictedWithoutEntries() { + final TestPlanV1_0 plan = new TestPlanV1_0(); + + assertThat(plan.isSelected(null, "any")).isTrue(); + assertThat(plan.isMuted(null, "any")).isFalse(); + plan.setTests(Collections.emptyList()).setExcludedTests(Collections.emptyList()); + assertThat(plan.isSelected(null, "any")).isTrue(); + assertThat(plan.isMuted(null, "any")).isFalse(); + plan.setTests(null).setExcludedTests(null); + assertThat(plan.isSelected(null, "any")).isTrue(); + assertThat(plan.isMuted(null, "any")).isFalse(); + } + + @Test + void shouldNotCrossMatchIdsAndSelectors() { + final List entries = Arrays.asList( + null, new TestPlanV1_0.TestCase().setId("id").setSelector("selector") + ); + final TestPlanV1_0 plan = new TestPlanV1_0().setTests(entries).setExcludedTests(entries); + + assertThat(plan.isSelected("selector", "id")).isFalse(); + assertThat(plan.isMuted("selector", "id")).isFalse(); + assertThat(plan.isSelected(null, null)).isFalse(); + } + + @Test + void shouldMatchLiteralSelectors() { + final TestPlanV1_0 plan = new TestPlanV1_0().setExcludedTests( + List.of( + new TestPlanV1_0.TestCase().setSelector("pkg.Test.case[1]") + ) + ); + + assertThat(plan.isMuted(null, "pkg.Test.case[1]")).isTrue(); + assertThat(plan.isMuted(null, "pkg.Test.case1")).isFalse(); + assertThat(plan.isMuted(null, "prefix.pkg.Test.case[1]")).isFalse(); + } + private TestPlan readPlan(final String json) throws Exception { return Allure.step("Deserialize test plan JSON", step -> { step.parameter("json", json); diff --git a/allure-jbehave5/README.md b/allure-jbehave5/README.md index 6fc915c88..327650c7a 100644 --- a/allure-jbehave5/README.md +++ b/allure-jbehave5/README.md @@ -35,6 +35,11 @@ Maven, with `allure-bom` imported in dependency management: Register `io.qameta.allure.jbehave5.AllureJbehave5` as a JBehave story reporter in your JBehave configuration. +## Muted Tests + +Muted-test execution skipping is not supported through this story reporter alone; reporting callbacks cannot safely prevent scenario bodies. +See [Skipping muted tests](../docs/muted-tests.md) for plan configuration, matching, and lifecycle limits. + ## Report Output - Stories, scenarios, examples, and steps. diff --git a/allure-jpms-tests/build.gradle.kts b/allure-jpms-tests/build.gradle.kts index 6b529b117..fd5ed51be 100644 --- a/allure-jpms-tests/build.gradle.kts +++ b/allure-jpms-tests/build.gradle.kts @@ -9,6 +9,21 @@ val consumerJars = configurations.create("consumerJars") { } } +// Keep incompatible Cucumber major versions in separate consumer workspaces. +val cucumberConsumers = listOf(7 to "7.34.8", 8 to "8.0.4").associate { (major, version) -> + major to configurations.create("cucumber${major}ConsumerJars") { + isCanBeConsumed = false + attributes { + attribute(Usage.USAGE_ATTRIBUTE, objects.named(Usage.JAVA_RUNTIME)) + attribute(LibraryElements.LIBRARY_ELEMENTS_ATTRIBUTE, objects.named(LibraryElements.JAR)) + } + dependencies.add(project.dependencies.project(mapOf("path" to ":allure-cucumber${major}-jvm"))) + dependencies.add(project.dependencies.create("io.cucumber:cucumber-core:$version")) + dependencies.add(project.dependencies.create("io.cucumber:cucumber-java:$version")) + dependencies.add(project.dependencies.create("org.slf4j:slf4j-simple:2.0.18")) + } +} + // Verify the packaged artifacts, including metadata contributed by Jar and SPI-off tasks. val publishedJars = configurations.create("publishedJars") { isCanBeConsumed = false @@ -59,6 +74,7 @@ dependencies { consumerJars(project(":allure-testng")) consumerJars(project(":allure-java-httpclient")) consumerJars(project(":allure-junit4")) + consumerJars(project(":allure-junit4-aspect")) consumerJars(project(":allure-jsonunit")) consumerJars(project(":allure-kotlin-coroutines")) consumerJars(project(":allure-spock2")) @@ -93,6 +109,11 @@ dependencies { } tasks.processTestResources { + cucumberConsumers.forEach { (major, jars) -> + from(jars) { + into("cucumber$major-jars") + } + } from(publishedJars) { into("published-jars") } diff --git a/allure-jpms-tests/src/test/java/io/qameta/allure/jpms/JavaConsumer.java b/allure-jpms-tests/src/test/java/io/qameta/allure/jpms/JavaConsumer.java index 0b7df0bd7..42a338324 100644 --- a/allure-jpms-tests/src/test/java/io/qameta/allure/jpms/JavaConsumer.java +++ b/allure-jpms-tests/src/test/java/io/qameta/allure/jpms/JavaConsumer.java @@ -42,6 +42,11 @@ final class JavaConsumer { private final Path directory; private int commandIndex; + private final java.util.Map environment = new java.util.HashMap<>(); + + void environment(final String name, final String value) { + environment.put(name, value); + } JavaConsumer(final Path directory, final String fixture) throws Exception { this.directory = directory; @@ -111,6 +116,19 @@ String jars(final String... artifacts) throws Exception { return String.join(File.pathSeparator, paths); } + String allJarsFrom(final String resourceDirectory) throws Exception { + final List paths = new ArrayList<>(); + try (Stream jars = Files.list(resource(resourceDirectory))) { + for (Path jar : jars.filter(path -> path.toString().endsWith(".jar")).sorted().toList()) { + final Path target = file("lib").resolve(jar.getFileName()); + Files.copy(jar, target, StandardCopyOption.REPLACE_EXISTING); + paths.add(directory.relativize(target).toString()); + } + } + assertThat(paths).isNotEmpty(); + return String.join(File.pathSeparator, paths); + } + void javac(final String... arguments) throws Exception { execute("javac", Path.of(System.getProperty("java.home")), 0, arguments); } @@ -157,6 +175,7 @@ private String execute(final String tool, final Path runtime, final int expected builder.environment().remove("JAVA_TOOL_OPTIONS"); builder.environment().remove("_JAVA_OPTIONS"); builder.environment().remove("CLASSPATH"); + builder.environment().putAll(environment); final Process process = builder.start(); final boolean finished; try { diff --git a/allure-jpms-tests/src/test/java/io/qameta/allure/jpms/JpmsIntegrationTest.java b/allure-jpms-tests/src/test/java/io/qameta/allure/jpms/JpmsIntegrationTest.java index d87ed3d61..62a39d9f2 100644 --- a/allure-jpms-tests/src/test/java/io/qameta/allure/jpms/JpmsIntegrationTest.java +++ b/allure-jpms-tests/src/test/java/io/qameta/allure/jpms/JpmsIntegrationTest.java @@ -557,6 +557,167 @@ void shouldRunJupiterWithoutOptionalParameterModule() throws Exception { assertThat(beforeSteps).contains("prepare operation"); } + /** + * Packaged adapters consume a muted-only plan through the real environment, prevent throwing + * bodies, and finish an all-muted scope successfully on both supported Java paths. + */ + @ParameterizedTest + @CsvSource({"junit4,true", "junit4,false", "jupiter,true", "jupiter,false", "testng,true", "testng,false"}) + @Description + void shouldReadMutedPlanFromEnvironment(final String framework, final boolean modular) throws Exception { + final JavaConsumer consumer = new JavaConsumer(directory, "muted-" + framework); + Files.writeString( + consumer.file("plan.json"), + "{\"version\":\"1.0\",\"excludedTests\":[{\"selector\":\"fixture.tests.SampleTest.muted\"}]}" + ); + consumer.environment("ALLURE_TESTPLAN_PATH", consumer.file("plan.json").toString()); + final String dependencies; + if ("jupiter".equals(framework)) { + dependencies = jupiterDependencies(consumer, false); + } else if ("testng".equals(framework)) { + dependencies = testNgDependencies(consumer, false); + } else { + dependencies = coreDependencies(consumer) + File.pathSeparator + + consumer.jars("allure-junit4", "junit", "hamcrest"); + } + + runFixture(consumer, dependencies, modular); + + if ("testng".equals(framework)) { + final JsonNode result = readOnlyResult(consumer.results(), "-result.json"); + assertThat(result.path("status").asText()).isEqualTo("skipped"); + assertThat(result.path("statusDetails").path("muted").asBoolean()).isTrue(); + } else { + assertThat(resultFiles(consumer.results(), "-result.json")).isEmpty(); + } + } + + /** + * Cucumber discovers the execution guard from built JAR service metadata and JPMS providers, + * without registering anything besides the existing reporting plugin. + */ + @ParameterizedTest + @CsvSource({"7,true", "7,false", "8,true", "8,false"}) + @Description + void shouldDiscoverPackagedCucumberMutedGuard(final int major, final boolean modular) throws Exception { + final JavaConsumer consumer = new JavaConsumer(directory, "muted-cucumber"); + final Path descriptor = consumer.file("src/module-info.java"); + Files.writeString(descriptor, Files.readString(descriptor).replace("cucumber7", "cucumber" + major)); + Files.writeString( + consumer.file("muted.feature"), + "@allure.id:123\nFeature: muted\nScenario: muted\n Given a throwing muted body\n" + ); + Files.writeString( + consumer.file("plan.json"), + "{\"version\":\"1.0\",\"excludedTests\":[{\"id\":123}]}" + ); + consumer.environment("ALLURE_TESTPLAN_PATH", consumer.file("plan.json").toString()); + final String dependencies = consumer.allJarsFrom("cucumber" + major + "-jars"); + compileFixture(consumer, dependencies, modular); + + final List arguments = new ArrayList<>( + List.of( + "-Dallure.results.directory=results", + modular ? "--module-path" : "--class-path", "classes" + File.pathSeparator + dependencies + ) + ); + arguments.addAll( + modular + ? List.of("--add-modules", "ALL-MODULE-PATH", "--module", "allure.fixture/fixture.Main") + : List.of("fixture.Main") + ); + arguments.add("io.qameta.allure.cucumber" + major + "jvm.AllureCucumber" + major + "Jvm"); + consumer.java(arguments.toArray(String[]::new)); + + final JsonNode result = readOnlyResult(consumer.results(), "-result.json"); + assertThat(result.path("status").asText()).isEqualTo("skipped"); + assertThat(result.path("statusDetails").path("muted").asBoolean()).isTrue(); + } + + /** + * The packaged Spock global extension aborts an all-muted feature through the default + * environment supplier, retaining a skipped-and-muted result on either Java path. + */ + @ParameterizedTest + @ValueSource(booleans = {true, false}) + @Description + void shouldReadSpockMutedPlanFromEnvironment(final boolean modular) throws Exception { + final JavaConsumer consumer = new JavaConsumer(directory, "muted-spock"); + Files.writeString( + consumer.file("plan.json"), + "{\"version\":\"1.0\",\"excludedTests\":[{\"id\":123}]}" + ); + consumer.environment("ALLURE_TESTPLAN_PATH", consumer.file("plan.json").toString()); + final String dependencies = String.join( + File.pathSeparator, coreDependencies(consumer), + consumer.jars( + "allure-spock2", "spock-core", "groovy", "junit-platform-engine", + "junit-platform-commons", "junit-platform-launcher", "apiguardian-api", + "jspecify", "opentest4j", "hamcrest" + ) + ); + consumer.java( + "--class-path", dependencies, "org.codehaus.groovy.tools.FileSystemCompiler", + "--classpath", dependencies, "-d", "classes", "src/fixture/tests/SampleTest.groovy" + ); + final List compile = new ArrayList<>( + List.of( + "--release", "17", "-d", "classes", + modular ? "--module-path" : "--class-path", dependencies + ) + ); + if (modular) { + compile.addAll(List.of("--patch-module", "allure.fixture=classes")); + } else { + compile.set(compile.size() - 1, "classes" + File.pathSeparator + dependencies); + } + compile.addAll(consumer.sources(modular)); + consumer.javac(compile.toArray(String[]::new)); + + final List run = new ArrayList<>( + List.of( + "-Dallure.results.directory=results", + modular ? "--module-path" : "--class-path", "classes" + File.pathSeparator + dependencies + ) + ); + run.addAll(modular ? List.of("--module", "allure.fixture/fixture.Main") : List.of("fixture.Main")); + consumer.java(run.toArray(String[]::new)); + + final JsonNode result = readOnlyResult(consumer.results(), "-result.json"); + assertThat(result.path("status").asText()).isEqualTo("skipped"); + assertThat(result.path("statusDetails").path("muted").asBoolean()).isTrue(); + } + + /** + * Existing AspectJ registration excludes an all-muted JUnit 4 scope without introducing a + * synthetic no-tests failure or requiring an explicit filter in the consumer. + */ + @Test + @Description + void shouldApplyMutedPlanThroughJunit4Aspect() throws Exception { + final JavaConsumer consumer = new JavaConsumer(directory, "muted-junit4"); + final Path main = consumer.file("src/fixture/Main.java"); + Files.writeString( + main, Files.readString(main) + .replace(".filterWith(new io.qameta.allure.junit4.AllureJunit4Filter())", "") + ); + Files.writeString( + consumer.file("plan.json"), + "{\"version\":\"1.0\",\"excludedTests\":[{\"selector\":\"fixture.tests.SampleTest.muted\"}]}" + ); + consumer.environment("ALLURE_TESTPLAN_PATH", consumer.file("plan.json").toString()); + final String dependencies = coreDependencies(consumer) + File.pathSeparator + + consumer.jars("allure-junit4", "allure-junit4-aspect", "junit", "hamcrest", "aspectjrt"); + compileFixture(consumer, dependencies, false); + + consumer.java( + "-Dallure.results.directory=results", "-javaagent:" + consumer.jars("aspectjweaver"), + "--class-path", "classes" + File.pathSeparator + dependencies, "fixture.Main" + ); + + assertThat(resultFiles(consumer.results(), "-result.json")).isEmpty(); + } + private static String runFixture(final JavaConsumer consumer, final String dependencies, final boolean modular) throws Exception { diff --git a/allure-jpms-tests/src/test/resources/fixtures/muted-cucumber/fixture/Main.java b/allure-jpms-tests/src/test/resources/fixtures/muted-cucumber/fixture/Main.java new file mode 100644 index 000000000..721fa891a --- /dev/null +++ b/allure-jpms-tests/src/test/resources/fixtures/muted-cucumber/fixture/Main.java @@ -0,0 +1,29 @@ +/* + * Copyright 2016-2026 Qameta Software Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package fixture; + +public final class Main { + public static void main(final String[] args) { + final byte status = io.cucumber.core.cli.Main.run( + new String[]{ + "--glue", "fixture", "--plugin", args[0], "muted.feature" + }, Main.class.getClassLoader() + ); + if (status != 0) { + throw new AssertionError("Cucumber exit status: " + status); + } + } +} diff --git a/allure-jpms-tests/src/test/resources/fixtures/muted-cucumber/fixture/Steps.java b/allure-jpms-tests/src/test/resources/fixtures/muted-cucumber/fixture/Steps.java new file mode 100644 index 000000000..dc13ba9bc --- /dev/null +++ b/allure-jpms-tests/src/test/resources/fixtures/muted-cucumber/fixture/Steps.java @@ -0,0 +1,25 @@ +/* + * Copyright 2016-2026 Qameta Software Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package fixture; + +import io.cucumber.java.en.Given; + +public class Steps { + @Given("a throwing muted body") + public void muted() { + throw new AssertionError("Muted body executed"); + } +} diff --git a/allure-jpms-tests/src/test/resources/fixtures/muted-cucumber/module-info.java b/allure-jpms-tests/src/test/resources/fixtures/muted-cucumber/module-info.java new file mode 100644 index 000000000..b48597af9 --- /dev/null +++ b/allure-jpms-tests/src/test/resources/fixtures/muted-cucumber/module-info.java @@ -0,0 +1,6 @@ +module allure.fixture { + requires io.cucumber.core; + requires io.cucumber.java; + requires io.qameta.allure.cucumber7jvm; + opens fixture to io.cucumber.java; +} diff --git a/allure-jpms-tests/src/test/resources/fixtures/muted-junit4/fixture/Main.java b/allure-jpms-tests/src/test/resources/fixtures/muted-junit4/fixture/Main.java new file mode 100644 index 000000000..ec06c7452 --- /dev/null +++ b/allure-jpms-tests/src/test/resources/fixtures/muted-junit4/fixture/Main.java @@ -0,0 +1,30 @@ +/* + * Copyright 2016-2026 Qameta Software Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package fixture; + +public final class Main { + public static void main(final String[] args) { + final var runner = new org.junit.runner.JUnitCore(); + runner.addListener(new io.qameta.allure.junit4.AllureJunit4()); + final var result = runner.run( + org.junit.runner.Request.aClass(fixture.tests.SampleTest.class) + .filterWith(new io.qameta.allure.junit4.AllureJunit4Filter()) + ); + if (!result.wasSuccessful() || result.getRunCount() != 0) { + throw new AssertionError(result.getFailures()); + } + } +} diff --git a/allure-jpms-tests/src/test/resources/fixtures/muted-junit4/fixture/tests/SampleTest.java b/allure-jpms-tests/src/test/resources/fixtures/muted-junit4/fixture/tests/SampleTest.java new file mode 100644 index 000000000..26850f5fa --- /dev/null +++ b/allure-jpms-tests/src/test/resources/fixtures/muted-junit4/fixture/tests/SampleTest.java @@ -0,0 +1,25 @@ +/* + * Copyright 2016-2026 Qameta Software Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package fixture.tests; + +import org.junit.Test; + +public class SampleTest { + @Test + public void muted() { + throw new AssertionError("Muted body executed"); + } +} diff --git a/allure-jpms-tests/src/test/resources/fixtures/muted-junit4/module-info.java b/allure-jpms-tests/src/test/resources/fixtures/muted-junit4/module-info.java new file mode 100644 index 000000000..a2d8a5f5b --- /dev/null +++ b/allure-jpms-tests/src/test/resources/fixtures/muted-junit4/module-info.java @@ -0,0 +1,20 @@ +/* + * Copyright 2016-2026 Qameta Software Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +module allure.fixture { + requires io.qameta.allure.junit4; + requires junit; + exports fixture.tests; +} diff --git a/allure-jpms-tests/src/test/resources/fixtures/muted-jupiter/fixture/Main.java b/allure-jpms-tests/src/test/resources/fixtures/muted-jupiter/fixture/Main.java new file mode 100644 index 000000000..3f394701c --- /dev/null +++ b/allure-jpms-tests/src/test/resources/fixtures/muted-jupiter/fixture/Main.java @@ -0,0 +1,32 @@ +/* + * Copyright 2016-2026 Qameta Software Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package fixture; + +public final class Main { + public static void main(final String[] args) { + final var summary = new org.junit.platform.launcher.listeners.SummaryGeneratingListener(); + final var runner = org.junit.platform.launcher.core.LauncherFactory.create(); + runner.registerTestExecutionListeners(summary); + runner.execute( + org.junit.platform.launcher.core.LauncherDiscoveryRequestBuilder.request() + .selectors(org.junit.platform.engine.discovery.DiscoverySelectors.selectClass(fixture.tests.SampleTest.class)) + .build() + ); + if (summary.getSummary().getTestsFoundCount() != 0 || summary.getSummary().getTotalFailureCount() != 0) { + throw new AssertionError(summary.getSummary().getFailures()); + } + } +} diff --git a/allure-jpms-tests/src/test/resources/fixtures/muted-jupiter/fixture/tests/SampleTest.java b/allure-jpms-tests/src/test/resources/fixtures/muted-jupiter/fixture/tests/SampleTest.java new file mode 100644 index 000000000..a5c0f52b6 --- /dev/null +++ b/allure-jpms-tests/src/test/resources/fixtures/muted-jupiter/fixture/tests/SampleTest.java @@ -0,0 +1,25 @@ +/* + * Copyright 2016-2026 Qameta Software Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package fixture.tests; + +import org.junit.jupiter.api.Test; + +public class SampleTest { + @Test + public void muted() { + throw new AssertionError("Muted body executed"); + } +} diff --git a/allure-jpms-tests/src/test/resources/fixtures/muted-jupiter/module-info.java b/allure-jpms-tests/src/test/resources/fixtures/muted-jupiter/module-info.java new file mode 100644 index 000000000..a6f80f660 --- /dev/null +++ b/allure-jpms-tests/src/test/resources/fixtures/muted-jupiter/module-info.java @@ -0,0 +1,23 @@ +/* + * Copyright 2016-2026 Qameta Software Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +module allure.fixture { + requires io.qameta.allure.commons; + requires org.junit.jupiter.api; + requires org.junit.platform.launcher; + requires org.junit.jupiter.params; + + opens fixture.tests to org.junit.platform.commons, io.qameta.allure.commons, io.qameta.allure.jupiter; +} diff --git a/allure-jpms-tests/src/test/resources/fixtures/muted-spock/fixture/Main.java b/allure-jpms-tests/src/test/resources/fixtures/muted-spock/fixture/Main.java new file mode 100644 index 000000000..82c772a53 --- /dev/null +++ b/allure-jpms-tests/src/test/resources/fixtures/muted-spock/fixture/Main.java @@ -0,0 +1,32 @@ +/* + * Copyright 2016-2026 Qameta Software Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package fixture; + +public final class Main { + public static void main(final String[] arguments) { + final var summary = new org.junit.platform.launcher.listeners.SummaryGeneratingListener(); + final var runner = org.junit.platform.launcher.core.LauncherFactory.create(); + runner.registerTestExecutionListeners(summary); + runner.execute( + org.junit.platform.launcher.core.LauncherDiscoveryRequestBuilder.request() + .selectors(org.junit.platform.engine.discovery.DiscoverySelectors.selectClass("fixture.tests.SampleTest")) + .build() + ); + if (summary.getSummary().getTestsAbortedCount() != 1 || summary.getSummary().getTotalFailureCount() != 0) { + throw new AssertionError(summary.getSummary().getFailures()); + } + } +} diff --git a/allure-jpms-tests/src/test/resources/fixtures/muted-spock/fixture/tests/SampleTest.groovy b/allure-jpms-tests/src/test/resources/fixtures/muted-spock/fixture/tests/SampleTest.groovy new file mode 100644 index 000000000..40de17a3b --- /dev/null +++ b/allure-jpms-tests/src/test/resources/fixtures/muted-spock/fixture/tests/SampleTest.groovy @@ -0,0 +1,31 @@ +/* + * Copyright 2016-2026 Qameta Software Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package fixture.tests + +import io.qameta.allure.AllureId +import spock.lang.Specification + +class SampleTest extends Specification { + @AllureId("123") + def "muted feature"() { + expect: + body() + } + + boolean body() { + throw new AssertionError("Muted body executed") + } +} diff --git a/allure-jpms-tests/src/test/resources/fixtures/muted-spock/module-info.java b/allure-jpms-tests/src/test/resources/fixtures/muted-spock/module-info.java new file mode 100644 index 000000000..c45febe9f --- /dev/null +++ b/allure-jpms-tests/src/test/resources/fixtures/muted-spock/module-info.java @@ -0,0 +1,24 @@ +/* + * Copyright 2016-2026 Qameta Software Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +module allure.fixture { + requires io.qameta.allure.spock2; + requires org.spockframework.core; + requires org.apache.groovy; + requires org.junit.platform.launcher; + requires java.desktop; + opens fixture.tests to org.spockframework.core, org.apache.groovy, + io.qameta.allure.commons, io.qameta.allure.spock2; +} diff --git a/allure-jpms-tests/src/test/resources/fixtures/muted-testng/fixture/Main.java b/allure-jpms-tests/src/test/resources/fixtures/muted-testng/fixture/Main.java new file mode 100644 index 000000000..978cc8237 --- /dev/null +++ b/allure-jpms-tests/src/test/resources/fixtures/muted-testng/fixture/Main.java @@ -0,0 +1,30 @@ +/* + * Copyright 2016-2026 Qameta Software Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package fixture; + +public final class Main { + public static void main(final String[] args) { + final var runner = new org.testng.TestNG(); + final var summary = new org.testng.TestListenerAdapter(); + runner.addListener(summary); + runner.setUseDefaultListeners(false); + runner.setTestClasses(new Class[]{fixture.tests.SampleTest.class}); + runner.run(); + if (runner.hasFailure() || summary.getSkippedTests().size() != 1) { + throw new AssertionError("Expected one skipped test: " + summary.getFailedTests()); + } + } +} diff --git a/allure-jpms-tests/src/test/resources/fixtures/muted-testng/fixture/tests/SampleTest.java b/allure-jpms-tests/src/test/resources/fixtures/muted-testng/fixture/tests/SampleTest.java new file mode 100644 index 000000000..a75f50ab2 --- /dev/null +++ b/allure-jpms-tests/src/test/resources/fixtures/muted-testng/fixture/tests/SampleTest.java @@ -0,0 +1,25 @@ +/* + * Copyright 2016-2026 Qameta Software Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package fixture.tests; + +import org.testng.annotations.Test; + +public class SampleTest { + @Test + public void muted() { + throw new AssertionError("Muted body executed"); + } +} diff --git a/allure-jpms-tests/src/test/resources/fixtures/muted-testng/module-info.java b/allure-jpms-tests/src/test/resources/fixtures/muted-testng/module-info.java new file mode 100644 index 000000000..5199d3a7e --- /dev/null +++ b/allure-jpms-tests/src/test/resources/fixtures/muted-testng/module-info.java @@ -0,0 +1,21 @@ +/* + * Copyright 2016-2026 Qameta Software Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +module allure.fixture { + requires io.qameta.allure.testng; + requires org.testng; + + opens fixture.tests to org.testng, io.qameta.allure.testng; +} diff --git a/allure-junit-platform/README.md b/allure-junit-platform/README.md index 83a729226..04d3d6d7e 100644 --- a/allure-junit-platform/README.md +++ b/allure-junit-platform/README.md @@ -39,6 +39,11 @@ The module registers these JUnit Platform services: Custom engines and launchers can reuse those services to write Allure results from JUnit Platform execution events. +## Muted Tests + +Muted test-plan matches are excluded during discovery, including Jupiter tests; no Allure result is written. +See [Skipping muted tests](../docs/muted-tests.md) for plan configuration, matching, and lifecycle limits. + ## Report Output - JUnit Platform test descriptors and execution events. diff --git a/allure-junit-platform/src/main/java/io/qameta/allure/junitplatform/AllurePostDiscoveryFilter.java b/allure-junit-platform/src/main/java/io/qameta/allure/junitplatform/AllurePostDiscoveryFilter.java index 7d6a8d0b7..acce7c4bc 100644 --- a/allure-junit-platform/src/main/java/io/qameta/allure/junitplatform/AllurePostDiscoveryFilter.java +++ b/allure-junit-platform/src/main/java/io/qameta/allure/junitplatform/AllurePostDiscoveryFilter.java @@ -90,24 +90,12 @@ private boolean isIncluded(final TestPlan testPlan, final String fullName) { if (testPlan instanceof TestPlanV1_0) { final TestPlanV1_0 tp = (TestPlanV1_0) testPlan; - return Objects.isNull(tp.getTests()) || tp.getTests() - .stream() - .filter(Objects::nonNull) - .anyMatch(tc -> match(tc, allureId, uniqueId, fullName)); + return (tp.isSelected(allureId, uniqueId) || tp.isSelected(allureId, fullName)) + && !tp.isMuted(allureId, uniqueId) && !tp.isMuted(allureId, fullName); } return true; } - @SuppressWarnings("BooleanExpressionComplexity") - private boolean match(final TestPlanV1_0.TestCase tc, - final String allureId, - final String uniqueId, - final String fullName) { - return Objects.nonNull(tc.getId()) && tc.getId().equals(allureId) - || Objects.nonNull(tc.getSelector()) && tc.getSelector().equals(uniqueId) - || Objects.nonNull(tc.getSelector()) && tc.getSelector().equals(fullName); - } - private String findAllureId(final TestDescriptor object) { return object.getSource() .flatMap(AllureJunitPlatformUtils::getTestMethod) diff --git a/allure-junit-platform/src/test/java/io/qameta/allure/junitplatform/AllureMutedDiscoveryTest.java b/allure-junit-platform/src/test/java/io/qameta/allure/junitplatform/AllureMutedDiscoveryTest.java new file mode 100644 index 000000000..ff1723c05 --- /dev/null +++ b/allure-junit-platform/src/test/java/io/qameta/allure/junitplatform/AllureMutedDiscoveryTest.java @@ -0,0 +1,93 @@ +/* + * Copyright 2016-2026 Qameta Software Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.qameta.allure.junitplatform; + +import io.qameta.allure.junitplatform.features.MutedPlanTests; +import io.qameta.allure.test.AllureResults; +import io.qameta.allure.test.IsolatedLifecycle; +import io.qameta.allure.test.RunUtils; +import io.qameta.allure.testfilter.TestPlanV1_0; +import org.junit.jupiter.api.Test; +import org.junit.platform.engine.discovery.DiscoverySelectors; +import org.junit.platform.launcher.core.LauncherConfig; +import org.junit.platform.launcher.core.LauncherDiscoveryRequestBuilder; +import org.junit.platform.launcher.core.LauncherFactory; +import org.junit.platform.launcher.listeners.SummaryGeneratingListener; + +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; + +@IsolatedLifecycle +class AllureMutedDiscoveryTest { + + @Test + void shouldExcludeMutedBodiesAndResults() { + final TestPlanV1_0 plan = new TestPlanV1_0().setTests(List.of()).setExcludedTests( + List.of( + new TestPlanV1_0.TestCase().setId("muted") + ) + ); + final AllureResults results = run(plan, 1); + + assertThat(results.getTestResults()).singleElement() + .satisfies(result -> assertThat(result.getName()).isEqualTo("control()")); + } + + @Test + void shouldCompleteAllMutedRunSuccessfully() { + final TestPlanV1_0 plan = new TestPlanV1_0().setExcludedTests( + List.of( + new TestPlanV1_0.TestCase().setId("muted"), + new TestPlanV1_0.TestCase().setSelector(MutedPlanTests.class.getName() + ".control") + ) + ); + final AllureResults results = run(plan, 0); + + assertThat(results.getTestResults()).isEmpty(); + } + + @Test + void shouldMatchUniqueIdWithUnmatchedExplicitId() { + final String uniqueId = "[engine:junit-jupiter]/[class:" + MutedPlanTests.class.getName() + + "]/[method:muted()]"; + final TestPlanV1_0 plan = new TestPlanV1_0().setExcludedTests( + List.of( + new TestPlanV1_0.TestCase().setId("other").setSelector(uniqueId) + ) + ); + final AllureResults results = run(plan, 1); + + assertThat(results.getTestResults()).hasSize(1); + } + + private AllureResults run(final TestPlanV1_0 plan, final int expectedCount) { + return RunUtils.runTests(lifecycle -> { + final SummaryGeneratingListener summary = new SummaryGeneratingListener(); + final LauncherConfig config = LauncherConfig.builder() + .enableTestExecutionListenerAutoRegistration(false) + .enablePostDiscoveryFilterAutoRegistration(false) + .addTestExecutionListeners(new AllureJunitPlatform(lifecycle), summary).build(); + LauncherFactory.create(config).execute( + LauncherDiscoveryRequestBuilder.request() + .selectors(DiscoverySelectors.selectClass(MutedPlanTests.class)) + .filters(new AllurePostDiscoveryFilter(plan)).build() + ); + assertThat(summary.getSummary().getFailures()).isEmpty(); + assertThat(summary.getSummary().getTestsStartedCount()).isEqualTo(expectedCount); + }); + } +} diff --git a/allure-junit-platform/src/test/java/io/qameta/allure/junitplatform/AllurePostDiscoveryFilterTest.java b/allure-junit-platform/src/test/java/io/qameta/allure/junitplatform/AllurePostDiscoveryFilterTest.java index 1690b65e3..0ae5ac252 100644 --- a/allure-junit-platform/src/test/java/io/qameta/allure/junitplatform/AllurePostDiscoveryFilterTest.java +++ b/allure-junit-platform/src/test/java/io/qameta/allure/junitplatform/AllurePostDiscoveryFilterTest.java @@ -140,6 +140,31 @@ void shouldFilterParameterizedTestCasesByAllureId() { .hasSize(4); } + @Test + void shouldExcludeMutedParameterizedMethod() { + final TestPlanV1_0 plan = new TestPlanV1_0().setExcludedTests( + Arrays.asList( + new TestPlanV1_0.TestCase().setSelector(FilterParameterizedTests.class.getName() + ".first"), + new TestPlanV1_0.TestCase().setId("20") + ) + ); + + final AllureResults results = runClasses(plan, FilterParameterizedTests.class); + + assertThat(results.getTestResults()).hasSize(2); + } + + @Test + void shouldNotExpandSelectionForMutedTests() { + final TestPlanV1_0 plan = new TestPlanV1_0() + .setTests(Arrays.asList(new TestPlanV1_0.TestCase().setId("10"))) + .setExcludedTests(Arrays.asList(new TestPlanV1_0.TestCase().setId("10"))); + + final AllureResults results = runClasses(plan, FilterSimpleTests.class); + + assertThat(results.getTestResults()).isEmpty(); + } + private String testId(final Class testClass, String method) { return String.format( "[engine:%s]/[class:%s]/[method:%s()]", diff --git a/allure-junit-platform/src/test/java/io/qameta/allure/junitplatform/features/MutedPlanTests.java b/allure-junit-platform/src/test/java/io/qameta/allure/junitplatform/features/MutedPlanTests.java new file mode 100644 index 000000000..238504f53 --- /dev/null +++ b/allure-junit-platform/src/test/java/io/qameta/allure/junitplatform/features/MutedPlanTests.java @@ -0,0 +1,33 @@ +/* + * Copyright 2016-2026 Qameta Software Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.qameta.allure.junitplatform.features; + +import io.qameta.allure.AllureId; +import org.junit.jupiter.api.Test; + +public class MutedPlanTests { + + @Test + @AllureId("muted") + public void muted() { + throw new AssertionError("Muted body executed"); + } + + @Test + public void control() { + // A passing control proves muting does not exclude unrelated tests. + } +} diff --git a/allure-junit4/README.md b/allure-junit4/README.md index 221ddbfa8..eb80d7bc3 100644 --- a/allure-junit4/README.md +++ b/allure-junit4/README.md @@ -41,6 +41,11 @@ Register `io.qameta.allure.junit4.AllureJunit4` as a JUnit 4 `RunListener` when For Gradle's built-in JUnit 4 execution, use `allure-junit4-aspect` instead. Gradle does not expose a supported listener registration hook for JUnit 4, so the AspectJ artifact performs automatic listener registration. +## Muted Tests + +Muted test-plan matches are excluded from execution and Allure results by the existing filter. +See [Skipping muted tests](../docs/muted-tests.md) for plan configuration, matching, and lifecycle limits. + ## Report Output - JUnit 4 test cases, ignored tests, failures, and assumptions. diff --git a/allure-junit4/src/main/java/io/qameta/allure/junit4/AllureJunit4Filter.java b/allure-junit4/src/main/java/io/qameta/allure/junit4/AllureJunit4Filter.java index 03c9f79fc..2bcfec23f 100644 --- a/allure-junit4/src/main/java/io/qameta/allure/junit4/AllureJunit4Filter.java +++ b/allure-junit4/src/main/java/io/qameta/allure/junit4/AllureJunit4Filter.java @@ -22,6 +22,8 @@ import io.qameta.allure.util.AnnotationUtils; import org.junit.runner.Description; import org.junit.runner.manipulation.Filter; +import org.junit.runner.manipulation.NoTestsRemainException; +import org.junit.runners.ParentRunner; import java.util.Objects; import java.util.Optional; @@ -72,6 +74,24 @@ public boolean shouldRun(final Description description) { return isIncluded(testPlan, allureId, fullName); } + /** + * Applies the filter without turning an empty standard runner into an initialization failure. + * + * @param child runner or test to filter. + * @throws NoTestsRemainException if filtering fails without leaving an empty standard runner. + */ + @Override + public void apply(final Object child) throws NoTestsRemainException { + try { + super.apply(child); + } catch (NoTestsRemainException exception) { + if (!(child instanceof ParentRunner) + || !((ParentRunner) child).getDescription().getChildren().isEmpty()) { + throw exception; + } + } + } + /** * {@inheritDoc} */ @@ -85,22 +105,11 @@ private boolean isIncluded(final TestPlan testPlan, final String fullName) { if (testPlan instanceof TestPlanV1_0) { final TestPlanV1_0 tp = (TestPlanV1_0) testPlan; - return Objects.isNull(tp.getTests()) || tp.getTests() - .stream() - .filter(Objects::nonNull) - .anyMatch(tc -> match(tc, allureId, fullName)); + return tp.isSelected(allureId, fullName) && !tp.isMuted(allureId, fullName); } return true; } - @SuppressWarnings("BooleanExpressionComplexity") - private boolean match(final TestPlanV1_0.TestCase tc, - final String allureId, - final String fullName) { - return Objects.nonNull(tc.getId()) && tc.getId().equals(allureId) - || Objects.nonNull(tc.getSelector()) && tc.getSelector().equals(fullName); - } - private static Optional findAllureId(final Description description) { return AnnotationUtils.getLabels(description.getAnnotations()) .stream() diff --git a/allure-junit4/src/test/java/io/qameta/allure/junit4/AllureJunit4MutedTest.java b/allure-junit4/src/test/java/io/qameta/allure/junit4/AllureJunit4MutedTest.java new file mode 100644 index 000000000..4b8b3c45c --- /dev/null +++ b/allure-junit4/src/test/java/io/qameta/allure/junit4/AllureJunit4MutedTest.java @@ -0,0 +1,73 @@ +/* + * Copyright 2016-2026 Qameta Software Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.qameta.allure.junit4; + +import io.qameta.allure.junit4.samples.MutedPlanTests; +import io.qameta.allure.test.AllureResults; +import io.qameta.allure.test.IsolatedLifecycle; +import io.qameta.allure.test.RunUtils; +import io.qameta.allure.testfilter.TestPlanV1_0; +import org.junit.jupiter.api.Test; +import org.junit.runner.JUnitCore; +import org.junit.runner.Request; +import org.junit.runner.Result; + +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; + +@IsolatedLifecycle +class AllureJunit4MutedTest { + + @Test + void shouldExcludeMutedBodiesAndResults() { + final TestPlanV1_0 plan = new TestPlanV1_0().setTests(List.of()).setExcludedTests( + List.of( + new TestPlanV1_0.TestCase().setId("muted") + ) + ); + final AllureResults results = run(plan, 1); + + assertThat(results.getTestResults()).singleElement() + .satisfies(result -> assertThat(result.getName()).isEqualTo("control")); + } + + @Test + void shouldCompleteAllMutedRunSuccessfully() { + final TestPlanV1_0 plan = new TestPlanV1_0().setExcludedTests( + List.of( + new TestPlanV1_0.TestCase().setId("muted"), + new TestPlanV1_0.TestCase().setSelector(MutedPlanTests.class.getName() + ".control") + ) + ); + final AllureResults results = run(plan, 0); + + assertThat(results.getTestResults()).isEmpty(); + } + + private AllureResults run(final TestPlanV1_0 plan, final int expectedCount) { + return RunUtils.runTests(lifecycle -> { + final JUnitCore core = new JUnitCore(); + core.addListener(new AllureJunit4(lifecycle)); + final Result result = core.run( + Request.aClass(MutedPlanTests.class) + .filterWith(new AllureJunit4Filter(plan)) + ); + assertThat(result.getFailures()).isEmpty(); + assertThat(result.getRunCount()).isEqualTo(expectedCount); + }); + } +} diff --git a/allure-junit4/src/test/java/io/qameta/allure/junit4/samples/MutedPlanTests.java b/allure-junit4/src/test/java/io/qameta/allure/junit4/samples/MutedPlanTests.java new file mode 100644 index 000000000..062444238 --- /dev/null +++ b/allure-junit4/src/test/java/io/qameta/allure/junit4/samples/MutedPlanTests.java @@ -0,0 +1,33 @@ +/* + * Copyright 2016-2026 Qameta Software Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.qameta.allure.junit4.samples; + +import io.qameta.allure.AllureId; +import org.junit.Test; + +public class MutedPlanTests { + + @Test + @AllureId("muted") + public void muted() { + throw new AssertionError("Muted body executed"); + } + + @Test + public void control() { + // A passing control proves muting does not exclude unrelated tests. + } +} diff --git a/allure-jupiter/README.md b/allure-jupiter/README.md index 4b07d608b..b3f19e5d4 100644 --- a/allure-jupiter/README.md +++ b/allure-jupiter/README.md @@ -42,6 +42,11 @@ No explicit listener registration is needed in most projects. The module contrib Run your JUnit Jupiter tests as usual. Allure result files are written to the configured Allure results directory, usually `build/allure-results` for Gradle or `target/allure-results` for Maven. +## Muted Tests + +Muted-test execution control comes from the Allure JUnit Platform filter, not this invocation extension alone. +See [Skipping muted tests](../docs/muted-tests.md) for plan configuration, matching, and lifecycle limits. + ## Report Output - Test cases for JUnit Jupiter tests, dynamic tests, repeated tests, and parameterized tests. diff --git a/allure-karate/README.md b/allure-karate/README.md index 814fbb91d..b6c275d33 100644 --- a/allure-karate/README.md +++ b/allure-karate/README.md @@ -42,6 +42,11 @@ Runner.builder() .parallel(4); ``` +## Muted Tests + +Muted top-level scenarios are vetoed before execution and retain skipped-and-muted Allure results. +See [Skipping muted tests](../docs/muted-tests.md) for plan configuration, matching, and lifecycle limits. + ## Report Output - One Allure test result for each top-level Karate scenario. diff --git a/allure-karate/build.gradle.kts b/allure-karate/build.gradle.kts index 9d8fb2cce..148918dcb 100644 --- a/allure-karate/build.gradle.kts +++ b/allure-karate/build.gradle.kts @@ -16,6 +16,7 @@ dependencies { testImplementation("io.karatelabs:karate-core:${karateVersion}") testImplementation("io.karatelabs:karate-junit6:${karateVersion}") testImplementation("io.github.glytching:junit-extensions") + testImplementation("com.fasterxml.jackson.core:jackson-databind") testImplementation("org.assertj:assertj-core") testImplementation(project(":allure-assertj")) testImplementation("org.junit.jupiter:junit-jupiter-api") @@ -39,6 +40,7 @@ tasks.jar { } tasks.test { + systemProperty("allure.test.runtime.classpath", sourceSets.test.get().runtimeClasspath.asPath) systemProperty("junit.jupiter.execution.parallel.enabled", "false") useJUnitPlatform() exclude("**/features/*") diff --git a/allure-karate/src/main/java/io/qameta/allure/karate/AllureKarate.java b/allure-karate/src/main/java/io/qameta/allure/karate/AllureKarate.java index 990de7407..4b51e7e2b 100644 --- a/allure-karate/src/main/java/io/qameta/allure/karate/AllureKarate.java +++ b/allure-karate/src/main/java/io/qameta/allure/karate/AllureKarate.java @@ -49,6 +49,9 @@ import io.qameta.allure.model.Status; import io.qameta.allure.model.StatusDetails; import io.qameta.allure.model.TestResult; +import io.qameta.allure.testfilter.FileTestPlanSupplier; +import io.qameta.allure.testfilter.TestPlan; +import io.qameta.allure.testfilter.TestPlanV1_0; import io.qameta.allure.util.ResultsUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -96,6 +99,7 @@ public class AllureKarate implements RunListener { private static final String BINARY_CONTENT_TYPE = "application/octet-stream"; private final AllureLifecycle lifecycle; + private final TestPlan testPlan; private final Map> reportedScenarios = new ConcurrentHashMap<>(); private final Map scenarioContexts = new ConcurrentHashMap<>(); @@ -116,7 +120,18 @@ public AllureKarate() { * @param lifecycle the Allure lifecycle to use */ public AllureKarate(final AllureLifecycle lifecycle) { + this(lifecycle, new FileTestPlanSupplier().supply().orElse(null)); + } + + /** + * Creates a listener with an explicit test plan. + * + * @param lifecycle the Allure lifecycle to use + * @param testPlan the execution plan, or null + */ + public AllureKarate(final AllureLifecycle lifecycle, final TestPlan testPlan) { this.lifecycle = lifecycle; + this.testPlan = testPlan; } /** @@ -179,15 +194,20 @@ private boolean beforeScenario(final ScenarioRuntime sr) { result.setLinks(links); } + final String allureId = result.getLabels().stream() + .filter(label -> ResultsUtils.ALLURE_ID_LABEL_NAME.equals(label.getName())) + .map(Label::getValue).findFirst().orElse(null); + final boolean muted = testPlan instanceof TestPlanV1_0 + && ((TestPlanV1_0) testPlan).isMuted(allureId, fullName); final AllureExternalKey testKey = testKey(uuid); lifecycle.scheduleTest(testKey, result); lifecycle.addDefaultLabels(testKey, List.of(ResultsUtils.createFeatureLabel(featureName))); lifecycle.startTest(testKey); scenarioContexts.put( sr, - new ScenarioContext(uuid, testKey, null, null, null, true, sr.isReportDisabled()) + new ScenarioContext(uuid, testKey, null, null, null, true, sr.isReportDisabled(), muted) ); - return true; + return !muted; } private boolean beforeCalledScenario(final ScenarioRuntime sr) { @@ -212,7 +232,8 @@ private boolean beforeCalledScenario(final ScenarioRuntime sr) { parentStepKey, caller, false, - true + true, + false ) ); return true; @@ -233,6 +254,7 @@ private boolean beforeCalledScenario(final ScenarioRuntime sr) { parentStepKey, caller, false, + false, false ) ); @@ -293,8 +315,11 @@ private void afterScenario(final ScenarioRunEvent event) { final boolean failed = maybeResult.map(ScenarioResult::isFailed).orElse(true); final Throwable error = maybeResult.map(ScenarioResult::getError).orElse(null); final boolean redactFailure = context.reportDisabled() || redactedScenarioFailures.remove(sr); - final Status status = getStatus(failed, error); + final Status status = context.muted() && !failed ? Status.SKIPPED : getStatus(failed, error); final StatusDetails statusDetails = getStatusDetails(failed, error, redactFailure); + final StatusDetails finalStatusDetails = context.muted() + ? Objects.requireNonNullElseGet(statusDetails, StatusDetails::new).setMuted(true) + : statusDetails; if (!context.topLevel()) { finishCalledScenario(context, failed, redactFailure, status, statusDetails); @@ -321,7 +346,7 @@ private void afterScenario(final ScenarioRunEvent event) { final AllureExternalKey testKey = testKey(context.testUuid()); lifecycle.updateTest(testKey, tr -> { tr.setStatus(status); - tr.setStatusDetails(statusDetails); + tr.setStatusDetails(finalStatusDetails); tr.getParameters().addAll(list); }); @@ -757,6 +782,7 @@ private record ScenarioContext( AllureExternalKey parentStepKey, ScenarioRuntime callerRuntime, boolean topLevel, - boolean reportDisabled) { + boolean reportDisabled, + boolean muted) { } } diff --git a/allure-karate/src/test/java/io/qameta/allure/karate/AllureKarateMutedTest.java b/allure-karate/src/test/java/io/qameta/allure/karate/AllureKarateMutedTest.java new file mode 100644 index 000000000..d2ba20c3f --- /dev/null +++ b/allure-karate/src/test/java/io/qameta/allure/karate/AllureKarateMutedTest.java @@ -0,0 +1,136 @@ +/* + * Copyright 2016-2026 Qameta Software Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.qameta.allure.karate; + +import com.fasterxml.jackson.databind.ObjectMapper; +import io.karatelabs.core.Runner; +import io.qameta.allure.model.Status; +import io.qameta.allure.test.AllureResults; +import io.qameta.allure.test.IsolatedLifecycle; +import io.qameta.allure.test.RunUtils; +import io.qameta.allure.testfilter.TestPlanV1_0; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.concurrent.TimeUnit; + +import static org.assertj.core.api.Assertions.assertThat; + +@IsolatedLifecycle +class AllureKarateMutedTest { + + @TempDir + private Path directory; + + @Test + void shouldReadMutedPlanFromEnvironment() throws Exception { + final Path plan = directory.resolve("plan.json"); + final Path results = directory.resolve("results"); + final Path log = directory.resolve("child.log"); + Files.writeString( + plan, "{\"version\":\"1.0\",\"excludedTests\":[{\"id\":123}," + + "{\"selector\":\"testdata/muted-plan.feature:12\"}]}" + ); + final ProcessBuilder builder = new ProcessBuilder( + Path.of(System.getProperty("java.home"), "bin", "java").toString(), + "-Dallure.results.directory=" + results, "-cp", + System.getProperty("allure.test.runtime.classpath"), MutedMain.class.getName(), + directory.resolve("reports").toString() + ).redirectErrorStream(true).redirectOutput(log.toFile()); + builder.environment().put("ALLURE_TESTPLAN_PATH", plan.toString()); + + final Process process = builder.start(); + final boolean finished = process.waitFor(60, TimeUnit.SECONDS); + if (!finished) { + process.destroyForcibly(); + } + + assertThat(finished).as(Files.readString(log)).isTrue(); + assertThat(process.exitValue()).as(Files.readString(log)).isZero(); + final List emitted; + try (var files = Files.list(results)) { + emitted = files.filter(path -> path.toString().endsWith("-result.json")).toList(); + } + final ObjectMapper mapper = new ObjectMapper(); + for (Path result : emitted) { + final var json = mapper.readTree(result.toFile()); + assertThat(json.path("status").asText()).isEqualTo("skipped"); + assertThat(json.path("statusDetails").path("muted").asBoolean()).isTrue(); + } + assertThat(emitted).hasSize(3); + } + + public static final class MutedMain { + private MutedMain() { + } + + public static void main(final String[] arguments) { + final var results = Runner.builder().path("classpath:testdata/muted-plan.feature") + .listener(new AllureKarate()).outputDir(Path.of(arguments[0])) + .outputHtmlReport(false).outputJunitXml(false).outputCucumberJson(false).parallel(2); + if (results.isFailed()) { + throw new AssertionError(results.getErrors()); + } + } + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void shouldSkipMutedScenarioBodies(final boolean allMuted) { + final TestPlanV1_0 plan = new TestPlanV1_0().setExcludedTests( + allMuted + ? List.of( + new TestPlanV1_0.TestCase().setId("123"), + new TestPlanV1_0.TestCase().setId("unmatched") + .setSelector("testdata/muted-plan.feature:12") + ) + : List.of(new TestPlanV1_0.TestCase().setId("123")) + ); + + final AllureResults results = RunUtils.runTests(lifecycle -> { + final var nativeResults = Runner.builder() + .path("classpath:testdata/muted-plan.feature") + .listener(new AllureKarate(lifecycle, plan)) + .outputDir(directory) + .outputHtmlReport(false) + .outputJunitXml(false) + .outputCucumberJson(false) + .parallel(2); + assertThat(nativeResults.getScenarioFailedCount()).isZero(); + }); + + assertThat(results.getGlobals()).isEmpty(); + assertThat(results.getTestResults()).hasSize(3); + assertThat(results.getTestResults()).filteredOn(result -> result.getName().equals("muted row")) + .hasSize(2).allSatisfy(result -> { + assertThat(result.getStatus()).isEqualTo(Status.SKIPPED); + assertThat(result.getStatusDetails().isMuted()).isTrue(); + assertThat(result.getParameters()).hasSize(1); + }); + assertThat(results.getTestResults()).filteredOn(result -> result.getName().equals("control")) + .singleElement().satisfies(result -> { + assertThat(result.getStatus()).isEqualTo(allMuted ? Status.SKIPPED : Status.PASSED); + if (allMuted) { + assertThat(result.getStatusDetails().isMuted()).isTrue(); + } + }); + } +} diff --git a/allure-karate/src/test/resources/testdata/muted-plan.feature b/allure-karate/src/test/resources/testdata/muted-plan.feature new file mode 100644 index 000000000..f6da00d01 --- /dev/null +++ b/allure-karate/src/test/resources/testdata/muted-plan.feature @@ -0,0 +1,13 @@ +Feature: muted execution + +@allure.id:123 +Scenario Outline: muted row + * assert false + +Examples: + | value | + | one | + | two | + +Scenario: control + * match 1 == 1 diff --git a/allure-scalatest/README.md b/allure-scalatest/README.md index ae5cdbe31..8175cbb86 100644 --- a/allure-scalatest/README.md +++ b/allure-scalatest/README.md @@ -42,6 +42,11 @@ Example sbt test option: Test / testOptions += Tests.Argument(TestFrameworks.ScalaTest, "-C", "io.qameta.allure.scalatest.AllureScalatest") ``` +## Muted Tests + +Muted-test execution skipping is not supported through this reporter alone; reporting callbacks cannot safely prevent test bodies. +See [Skipping muted tests](../docs/muted-tests.md) for plan configuration, matching, and lifecycle limits. + ## Report Output - ScalaTest suites and tests. diff --git a/allure-spock2/README.md b/allure-spock2/README.md index 0fd04a270..b3f9ad0b5 100644 --- a/allure-spock2/README.md +++ b/allure-spock2/README.md @@ -37,6 +37,11 @@ Add the dependency to a Spock 2 project. The module registers `io.qameta.allure. Block reporting uses Spock runtime events and does not install a Groovy AST transformation, so the order of the Spock and Allure dependencies does not affect compilation. +## Muted Tests + +Muted test-plan matches abort each iteration before its feature body and retain skipped-and-muted results. +See [Skipping muted tests](../docs/muted-tests.md) for plan configuration, matching, and lifecycle limits. + ## Report Output - Specifications, features, iterations, fixture methods, and errors. diff --git a/allure-spock2/src/main/java/io/qameta/allure/spock2/AllureSpock2.java b/allure-spock2/src/main/java/io/qameta/allure/spock2/AllureSpock2.java index bca3c1bfa..c62caed73 100644 --- a/allure-spock2/src/main/java/io/qameta/allure/spock2/AllureSpock2.java +++ b/allure-spock2/src/main/java/io/qameta/allure/spock2/AllureSpock2.java @@ -33,6 +33,7 @@ import io.qameta.allure.util.AnnotationUtils; import io.qameta.allure.util.ExceptionUtils; import io.qameta.allure.util.ResultsUtils; +import org.opentest4j.TestAbortedException; import org.spockframework.runtime.AbstractRunListener; import org.spockframework.runtime.extension.IBlockListener; import org.spockframework.runtime.extension.IGlobalExtension; @@ -139,7 +140,7 @@ public AllureSpock2(final AllureLifecycle lifecycle, final TestPlan plan) { @Override public void visitSpec(final SpecInfo spec) { spec.getAllFeatures().forEach(featureInfo -> { - featureInfo.setSkipped(this.isSkipped(featureInfo)); + featureInfo.setSkipped(isSkipped(featureInfo)); featureInfo.addBlockListener(this); }); @@ -165,6 +166,13 @@ public void visitSpec(final SpecInfo spec) { getLifecycle().getCurrentExecutableKey().ifPresent( key -> getLifecycle().addTestToScope(scopeKey(specContainerUuid), key) ); + if (isMuted(i.getFeature())) { + getLifecycle().updateTest(testKey(testResults.get()), result -> { + result.setStatus(Status.SKIPPED); + result.getStatusDetails().setMuted(true); + }); + throw new TestAbortedException("Skipped by Allure test plan"); + } i.proceed(); })); } @@ -416,21 +424,15 @@ private String getQualifiedName(final String specName, final String testName) { } private boolean isSkipped(final FeatureInfo featureInfo) { + if (featureInfo.isSkipped()) { + return true; + } if (Objects.isNull(this.testPlan)) { return false; } if (this.testPlan instanceof TestPlanV1_0) { final TestPlanV1_0 tp = (TestPlanV1_0) testPlan; - return !Objects.isNull(tp.getTests()) && tp.getTests() - .stream() - .filter(Objects::nonNull) - .noneMatch( - tc -> this.match( - tc, - this.getAllureId(featureInfo), - this.getQualifiedName(featureInfo) - ) - ); + return !tp.isSelected(getAllureId(featureInfo), getQualifiedName(featureInfo)); } return false; } @@ -443,8 +445,9 @@ private String getAllureId(final FeatureInfo featureInfo) { return null; } - private boolean match(final TestPlanV1_0.TestCase tc, final String allureId, final String qualifiedName) { - return Objects.equals(allureId, tc.getId()) || Objects.equals(qualifiedName, tc.getSelector()); + private boolean isMuted(final FeatureInfo feature) { + return testPlan instanceof TestPlanV1_0 + && ((TestPlanV1_0) testPlan).isMuted(getAllureId(feature), getQualifiedName(feature)); } /** @@ -459,7 +462,11 @@ public void error(final ErrorInfo error) { final Throwable exception = error.getException(); updateBlockStep(error); getLifecycle().updateTest(testKey(uuid), testResult -> { - testResult.setStatus(getStatus(exception).orElse(null)); + testResult.setStatus( + exception instanceof TestAbortedException + ? Status.SKIPPED + : getStatus(exception).orElse(null) + ); final StatusDetails details = getStatusDetails(exception).orElse(null); if (Objects.isNull(details)) { diff --git a/allure-spock2/src/main/java/module-info.java b/allure-spock2/src/main/java/module-info.java index 014b647d2..2d1f01aa9 100644 --- a/allure-spock2/src/main/java/module-info.java +++ b/allure-spock2/src/main/java/module-info.java @@ -16,6 +16,7 @@ module io.qameta.allure.spock2 { requires transitive io.qameta.allure.commons; requires org.spockframework.core; + requires org.opentest4j; exports io.qameta.allure.spock2; diff --git a/allure-spock2/src/spi-off/java/module-info.java b/allure-spock2/src/spi-off/java/module-info.java index 2c111a172..e36e15eae 100644 --- a/allure-spock2/src/spi-off/java/module-info.java +++ b/allure-spock2/src/spi-off/java/module-info.java @@ -16,6 +16,7 @@ module io.qameta.allure.spock2 { requires transitive io.qameta.allure.commons; requires org.spockframework.core; + requires org.opentest4j; exports io.qameta.allure.spock2; } diff --git a/allure-spock2/src/test/groovy/io/qameta/allure/spock2/AllureSpock2Test.java b/allure-spock2/src/test/groovy/io/qameta/allure/spock2/AllureSpock2Test.java index 13c814a3d..0df8d81d8 100644 --- a/allure-spock2/src/test/groovy/io/qameta/allure/spock2/AllureSpock2Test.java +++ b/allure-spock2/src/test/groovy/io/qameta/allure/spock2/AllureSpock2Test.java @@ -1127,7 +1127,10 @@ public static AllureResults runClasses(final TestPlan testPlan, final Class.. .andThenTry(GlobalExtensionRegistry.class::cast) .toOptional() .orElseThrow(() -> new AssertionError("could not access globalExtensionRegistry field of RunContext")); - extensionRegistry.getGlobalExtensions().add(new AllureSpock2(lifecycle, testPlan)); + final List existing = new java.util.ArrayList<>(extensionRegistry.getGlobalExtensions()); + extensionRegistry.getGlobalExtensions().removeIf(AllureSpock2.class::isInstance); + final AllureSpock2 extension = new AllureSpock2(lifecycle, testPlan); + extensionRegistry.getGlobalExtensions().add(extension); final LauncherConfig config = LauncherConfig.builder() .enableTestEngineAutoRegistration(false) @@ -1137,7 +1140,51 @@ public static AllureResults runClasses(final TestPlan testPlan, final Class.. .build(); final Launcher launcher = LauncherFactory.create(config); - launcher.execute(request); + try { + launcher.execute(request); + } finally { + extensionRegistry.getGlobalExtensions().clear(); + extensionRegistry.getGlobalExtensions().addAll(existing); + } + }); + } + + @Test + void shouldSkipMutedIterationsWithoutExecutingBodies() { + final TestPlanV1_0 plan = new TestPlanV1_0().setTests(Collections.emptyList()).setExcludedTests( + List.of( + new TestPlanV1_0.TestCase().setId("muted") + ) + ); + final AllureResults results = runClasses(plan, io.qameta.allure.spock2.samples.MutedPlanSpec.class); + + assertThat(results.getTestResults()).filteredOn(r -> r.getStatusDetails().isMuted()).hasSize(2) + .allSatisfy(r -> { + assertThat(r.getStatus()).isEqualTo(Status.SKIPPED); + assertThat(r.getParameters()).hasSize(1); + }); + assertThat(results.getTestResults()).filteredOn("name", "control").singleElement() + .satisfies(r -> assertThat(r.getStatus()).isEqualTo(Status.PASSED)); + assertThat(results.getTestResults()).extracting(TestResult::getName).doesNotContain("user ignored"); + } + + @Test + void shouldCompleteAllMutedIterations() { + final Class spec = io.qameta.allure.spock2.samples.MutedPlanSpec.class; + final TestPlanV1_0 plan = new TestPlanV1_0() + .setTests(List.of(new TestPlanV1_0.TestCase().setId("muted"))) + .setExcludedTests( + List.of( + new TestPlanV1_0.TestCase().setId("unmatched") + .setSelector(spec.getName() + ".muted row #value") + ) + ); + + final AllureResults results = runClasses(plan, spec); + + assertThat(results.getTestResults()).hasSize(2).allSatisfy(r -> { + assertThat(r.getStatus()).isEqualTo(Status.SKIPPED); + assertThat(r.getStatusDetails().isMuted()).isTrue(); }); } diff --git a/allure-spock2/src/test/groovy/io/qameta/allure/spock2/samples/MutedPlanSpec.groovy b/allure-spock2/src/test/groovy/io/qameta/allure/spock2/samples/MutedPlanSpec.groovy new file mode 100644 index 000000000..bdfbc9146 --- /dev/null +++ b/allure-spock2/src/test/groovy/io/qameta/allure/spock2/samples/MutedPlanSpec.groovy @@ -0,0 +1,43 @@ +/* + * Copyright 2016-2026 Qameta Software Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.qameta.allure.spock2.samples + +import io.qameta.allure.AllureId +import spock.lang.Ignore +import spock.lang.Specification +import spock.lang.Unroll + +class MutedPlanSpec extends Specification { + @AllureId("muted") + @Unroll + def "muted row #value"() { + expect: + throw new AssertionError("Muted body executed: " + value) + where: + value << ["first", "second"] + } + + def "control"() { + expect: + true + } + + @Ignore + def "user ignored"() { + expect: + throw new AssertionError("Ignored body executed") + } +} diff --git a/allure-testng/README.md b/allure-testng/README.md index 0744b8429..1b9be3694 100644 --- a/allure-testng/README.md +++ b/allure-testng/README.md @@ -44,6 +44,11 @@ class MyTest { } ``` +## Muted Tests + +Muted test-plan matches use native skips and retain skipped-and-muted Allure results. +See [Skipping muted tests](../docs/muted-tests.md) for plan configuration, matching, and lifecycle limits. + ## Report Output - TestNG suites, tests, classes, methods, configuration methods, and data-provider invocations. diff --git a/allure-testng/src/main/java/io/qameta/allure/testng/AllureTestNg.java b/allure-testng/src/main/java/io/qameta/allure/testng/AllureTestNg.java index c00b1ac4e..8ee702ab7 100644 --- a/allure-testng/src/main/java/io/qameta/allure/testng/AllureTestNg.java +++ b/allure-testng/src/main/java/io/qameta/allure/testng/AllureTestNg.java @@ -130,6 +130,8 @@ public class AllureTestNg private static final String ALLURE_FIXTURE_UUID = "ALLURE_FIXTURE_UUID"; + private static final String ALLURE_MUTED = "ALLURE_MUTED"; + private static final String UNSUPPORTED_TEST_CONTEXT = "the test context is not org.testng.TestRunner"; /** * How far the cause chain of a failure is walked when looking for the intent behind it. @@ -642,6 +644,9 @@ public void onTestSkipped(final ITestResult result) { } final String uuid = ensureTestStarted(result); + if (Boolean.TRUE.equals(result.getAttribute(ALLURE_MUTED))) { + getLifecycle().updateTest(testKey(uuid), test -> test.getStatusDetails().setMuted(true)); + } stopTest(uuid, result.getThrowable(), Status.SKIPPED); } @@ -745,6 +750,12 @@ public void onTestFailedButWithinSuccessPercentage(final ITestResult result) { public void beforeInvocation(final IInvokedMethod method, final ITestResult testResult) { final ITestNGMethod testMethod = method.getTestMethod(); final ITestContext context = testResult.getTestContext(); + if (method.isTestMethod() && testResult.getStatus() != ITestResult.SKIP + && testResult.getThrowable() == null && !shouldSkipReportingFor(testResult) + && testFilter.isMuted(testMethod)) { + testResult.setAttribute(ALLURE_MUTED, true); + throw new SkipException("Skipped by Allure test plan"); + } if (isSupportedConfigurationFixture(testMethod)) { orderConfigurationListeners(context); testResult.setAttribute(ALLURE_FIXTURE_UUID, currentExecutable.get()); diff --git a/allure-testng/src/main/java/io/qameta/allure/testng/AllureTestNgTestFilter.java b/allure-testng/src/main/java/io/qameta/allure/testng/AllureTestNgTestFilter.java index 91511914a..45dd20426 100644 --- a/allure-testng/src/main/java/io/qameta/allure/testng/AllureTestNgTestFilter.java +++ b/allure-testng/src/main/java/io/qameta/allure/testng/AllureTestNgTestFilter.java @@ -125,6 +125,25 @@ public boolean isSelected(final ITestNGMethod testNGMethod, final TestPlanV1_0 t return false; } + /** + * Returns whether the method matches an independent muted test entry. + * + * @param testNGMethod the TestNG method to evaluate + * @return true when the configured plan mutes this method + */ + public boolean isMuted(final ITestNGMethod testNGMethod) { + return testPlan instanceof TestPlanV1_0 && Optional.ofNullable(testNGMethod) + .map(ITestNGMethod::getConstructorOrMethod) + .map(ConstructorOrMethod::getMethod) + .map( + method -> ((TestPlanV1_0) testPlan).isMuted( + Optional.ofNullable(method.getAnnotation(AllureId.class)).map(AllureId::value).orElse(null), + getSelector(method) + ) + ) + .orElse(false); + } + private String getSelector(final Method method) { return String.format( "%s.%s", diff --git a/allure-testng/src/test/java/io/qameta/allure/testng/AllureTestNgMutedTest.java b/allure-testng/src/test/java/io/qameta/allure/testng/AllureTestNgMutedTest.java new file mode 100644 index 000000000..9b5ba6749 --- /dev/null +++ b/allure-testng/src/test/java/io/qameta/allure/testng/AllureTestNgMutedTest.java @@ -0,0 +1,110 @@ +/* + * Copyright 2016-2026 Qameta Software Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.qameta.allure.testng; + +import io.qameta.allure.model.Status; +import io.qameta.allure.test.AllureResults; +import io.qameta.allure.test.IsolatedLifecycle; +import io.qameta.allure.test.RunUtils; +import io.qameta.allure.testfilter.TestPlanV1_0; +import io.qameta.allure.testng.samples.MutedPlanAfterFailure; +import io.qameta.allure.testng.samples.MutedPlanDependency; +import io.qameta.allure.testng.samples.MutedPlanTests; +import org.junit.jupiter.api.Test; +import org.testng.TestNG; + +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; + +@IsolatedLifecycle +class AllureTestNgMutedTest { + private TestPlanV1_0 plan() { + return new TestPlanV1_0().setExcludedTests( + List.of( + new TestPlanV1_0.TestCase().setId("muted"), + new TestPlanV1_0.TestCase().setId("unmatched") + .setSelector(MutedPlanTests.class.getCanonicalName() + ".repeated") + ) + ); + } + + @Test + void shouldSkipRealInvocationsWithoutExecutingBodies() { + final AllureResults results = run(plan().setTests(List.of())); + + assertThat(results.getTestResults()).hasSize(6); + assertThat(results.getTestResults()).filteredOn(r -> r.getStatusDetails().isMuted()).hasSize(4) + .allSatisfy(r -> assertThat(r.getStatus()).isEqualTo(Status.SKIPPED)); + assertThat(results.getTestResults()).filteredOn("name", "parameterized").hasSize(2) + .allSatisfy(r -> assertThat(r.getParameters()).hasSize(1)); + assertThat(results.getTestResults()).filteredOn("name", "ordinarySkip").singleElement() + .satisfies(r -> assertThat(r.getStatusDetails().isMuted()).isFalse()); + assertThat(results.getTestResults()).filteredOn("name", "control").singleElement() + .satisfies(r -> assertThat(r.getStatus()).isEqualTo(Status.PASSED)); + } + + @Test + void shouldCompleteAllMutedScopeWithoutFailures() { + final TestPlanV1_0 plan = plan().setTests( + List.of( + new TestPlanV1_0.TestCase().setId("muted"), + new TestPlanV1_0.TestCase().setId("repeated") + ) + ); + + final AllureResults results = run(plan); + + assertThat(results.getTestResults()).hasSize(4) + .allSatisfy(r -> { + assertThat(r.getStatus()).isEqualTo(Status.SKIPPED); + assertThat(r.getStatusDetails().isMuted()).isTrue(); + }); + } + + @Test + void shouldPreserveDependencySkips() { + final AllureResults results = run(plan(), false, MutedPlanDependency.class); + + assertThat(results.getTestResults()).hasSize(2) + .allSatisfy(r -> assertThat(r.getStatus()).isEqualTo(Status.SKIPPED)); + assertThat(results.getTestResults()).filteredOn("name", "dependent").singleElement() + .satisfies(r -> assertThat(r.getStatusDetails().isMuted()).isFalse()); + } + + @Test + void shouldNotHideRealCleanupFailure() { + final AllureResults results = run(plan(), true, MutedPlanAfterFailure.class); + + assertThat(results.getTestResultContainers()).flatExtracting(c -> c.getAfters()) + .anySatisfy(f -> assertThat(f.getStatus()).isEqualTo(Status.FAILED)); + } + + private AllureResults run(final TestPlanV1_0 plan) { + return run(plan, false, MutedPlanTests.class); + } + + private AllureResults run(final TestPlanV1_0 plan, final boolean hasFailure, final Class... classes) { + return RunUtils.runTests(lifecycle -> { + final TestNG runner = new TestNG(false); + runner.addListener(new AllureTestNg(lifecycle, new AllureTestNgTestFilter(plan))); + runner.setTestClasses(classes); + runner.setOutputDirectory("build/test-output"); + runner.run(); + assertThat(runner.hasFailure()).isEqualTo(hasFailure); + }); + } +} diff --git a/allure-testng/src/test/java/io/qameta/allure/testng/samples/MutedPlanAfterFailure.java b/allure-testng/src/test/java/io/qameta/allure/testng/samples/MutedPlanAfterFailure.java new file mode 100644 index 000000000..3e0ba060c --- /dev/null +++ b/allure-testng/src/test/java/io/qameta/allure/testng/samples/MutedPlanAfterFailure.java @@ -0,0 +1,32 @@ +/* + * Copyright 2016-2026 Qameta Software Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.qameta.allure.testng.samples; + +import io.qameta.allure.AllureId; +import org.testng.annotations.Test; + +public class MutedPlanAfterFailure { + @Test + @AllureId("muted") + public void muted() { + throw new AssertionError("Muted body executed"); + } + + @org.testng.annotations.AfterMethod(alwaysRun = true) + public void cleanup() { + throw new AssertionError("Real cleanup failure"); + } +} diff --git a/allure-testng/src/test/java/io/qameta/allure/testng/samples/MutedPlanDependency.java b/allure-testng/src/test/java/io/qameta/allure/testng/samples/MutedPlanDependency.java new file mode 100644 index 000000000..1311aa393 --- /dev/null +++ b/allure-testng/src/test/java/io/qameta/allure/testng/samples/MutedPlanDependency.java @@ -0,0 +1,32 @@ +/* + * Copyright 2016-2026 Qameta Software Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.qameta.allure.testng.samples; + +import io.qameta.allure.AllureId; +import org.testng.annotations.Test; + +public class MutedPlanDependency { + @Test + @AllureId("muted") + public void muted() { + throw new AssertionError("Muted body executed"); + } + + @Test(dependsOnMethods = "muted") + public void dependent() { + throw new AssertionError("Dependent body executed"); + } +} diff --git a/allure-testng/src/test/java/io/qameta/allure/testng/samples/MutedPlanTests.java b/allure-testng/src/test/java/io/qameta/allure/testng/samples/MutedPlanTests.java new file mode 100644 index 000000000..4558d3d4f --- /dev/null +++ b/allure-testng/src/test/java/io/qameta/allure/testng/samples/MutedPlanTests.java @@ -0,0 +1,49 @@ +/* + * Copyright 2016-2026 Qameta Software Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.qameta.allure.testng.samples; + +import io.qameta.allure.AllureId; +import org.testng.SkipException; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +public class MutedPlanTests { + @DataProvider + public Object[][] rows() { + return new Object[][]{{"first"}, {"second"}}; + } + + @Test(dataProvider = "rows") + @AllureId("muted") + public void parameterized(final String value) { + throw new AssertionError("Muted body executed: " + value); + } + + @Test(invocationCount = 2) + @AllureId("repeated") + public void repeated() { + throw new AssertionError("Muted body executed"); + } + + @Test + public void control() { + } + + @Test + public void ordinarySkip() { + throw new SkipException("User skip"); + } +} diff --git a/docs/muted-tests.md b/docs/muted-tests.md new file mode 100644 index 000000000..19a1ff863 --- /dev/null +++ b/docs/muted-tests.md @@ -0,0 +1,85 @@ +# Skipping muted tests + +Supported test-runner adapters read `excludedTests` from the version 1.0 test plan referenced by +`ALLURE_TESTPLAN_PATH`. A match prevents the test body from executing; it is not just a reporting flag. +No TestOps connection or additional adapter registration is required. + +```json +{ + "version": "1.0", + "excludedTests": [ + { "id": 123 }, + { "selector": "com.example.PaymentTest.refund" } + ] +} +``` + +```shell +ALLURE_TESTPLAN_PATH=/absolute/path/to/testplan.json ./gradlew test +``` + +## Adapter behavior + +| Adapter | Matched test behavior | +| --- | --- | +| TestNG | Native skip; Allure result has `status: skipped` and `statusDetails.muted: true` | +| Spock 2 | Native abort per iteration; skipped-and-muted results retain iteration parameters | +| Cucumber JVM 7 and 8 | Automatic backend before hook aborts the scenario; skipped-and-muted result | +| Karate | Top-level scenario execution veto; skipped-and-muted result | +| JUnit 4 | Excluded by the existing Allure test-plan filter; no Allure result | +| JUnit Platform, including Jupiter | Excluded during discovery; no Allure result | +| ScalaTest, JBehave 5, Citrus | Not supported through the existing reporter/listener registration | + +JUnit 4 still needs its normal test-plan filter registration, either explicit filtering or the +`allure-junit4-aspect` integration. JUnit Platform still needs its normal Allure post-discovery filter. +The Jupiter invocation extension alone does not provide muted-test execution control. + +ScalaTest, JBehave, and Citrus reporting callbacks do not offer a safe execution veto. Their results +are not silently suppressed as a substitute for preventing execution. HTTP, browser, assertion, and +other non-runner integrations rely on their hosting test runner. + +## Matching and selection + +- `excludedTests` and the existing `tests` selection list are independent. A test must be selected before + muting can affect its execution. +- An omitted, null, or empty `tests` list imposes no selection restriction, so muted-only plans work + with `tests: []` too. Muting still applies independently. +- Cucumber and Karate use `excludedTests` without introducing ordinary `tests` selection in those adapters. +- IDs match IDs; selectors match selectors. Either field can match. An unmatched explicit ID does not + prevent a matching selector from taking effect. +- Numeric IDs in JSON are normalized to strings. Null entries and missing or empty identities do not + match. Selectors are exact strings, not patterns or prefixes. +- Existing missing/unreadable/invalid-plan and unknown-version handling is unchanged. +- The `@Muted` annotation and existing muted tags remain reporting metadata; they do not independently + enable execution skipping without a matching test-plan entry. + +Use the adapter's existing identity, normally the Allure result's `fullName`, as the selector: + +| Adapter | Selector | +| --- | --- | +| JUnit 4, TestNG | Declaring class canonical name + `.` + method name | +| JUnit Platform | Existing source-derived full name or the platform unique ID | +| Spock 2 | Specification class name + `.` + feature name (not the expanded iteration title) | +| Cucumber JVM | Existing normalized scenario URI + `:` + scenario location line | +| Karate | Feature qualified name + `:` + scenario line | + +Cucumber URI normalization is shared with reporting, including working-directory stripping. Outline +rows retain their existing scenario location identities. Karate matching applies to top-level +scenarios; called features continue to be reported and executed as nested steps. + +## Lifecycle and exit behavior + +Native skipped results preserve normal names, history identity, labels, and available parameters. +Framework setup, cleanup, dependency, and user-skip behavior still follows the runner's lifecycle; +muting is not a guarantee that every fixture is bypassed. Genuine fixture failures remain failures. +Cucumber's automatically discovered before hook is registered only when a known plan contains muted +entries. It runs at the earliest hook order and may appear as a fixture in Allure results. + +Successful muted-only runs are supported without converting skips into failures. Native CLI exit +conventions are not normalized: for example, TestNG's standalone CLI can return its skipped-tests +status even when there are no failures. Gradle/Maven retain their normal handling of native skips; +any separate build-tool policy that rejects an empty discovered suite still applies to exclusion. + +Classpath and JPMS consumers use the same matching rules. Cucumber's backend is registered through +both `META-INF/services` and its module descriptor. Cucumber 8 module-path launches may need +`--add-modules ALL-MODULE-PATH` to resolve upstream runtime modules such as tag expressions.