Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 44 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,50 @@ pytest plugin and distributed via
Status](https://img.shields.io/pypi/v/allure-nose2)](https://pypi.python.org/pypi/allure-nose2)
[![Downloads](https://img.shields.io/pypi/dm/allure-nose2)](https://pypi.python.org/pypi/allure-nose2)

## Skipping muted tests

All adapters support `excludedTests` in the JSON test plan supplied through
`ALLURE_TESTPLAN_PATH`:

```json
{
"version": "1.0",
"excludedTests": [
{ "id": 101 },
{ "selector": "test_checkout#test_expired_card" }
]
}
```

A matching test is skipped before its body runs and, when reporting is enabled,
retained with `status: "skipped"` and `statusDetails.muted: true`. Existing
runner filters and test-plan selection still apply: muting never adds tests to
the run. Omitting `tests`, or using `"tests": []`, leaves selection unrestricted
under the adapter’s existing rules. pytest-bdd and nose2 do not add test-plan
selection; they apply muting to the tests selected by their runners.

IDs match static Allure ID annotations/tags, with numeric and string IDs treated
alike. IDs assigned dynamically inside a test body cannot be used for skipping.
Selectors match the adapter’s existing `fullName` exactly (not as a regex):

| Adapter | Selector |
| --- | --- |
| pytest | `module#test`, or `module.Class#test` |
| Behave | `Feature name: Scenario name` |
| Robot Framework | `Suite.Test name` |
| pytest-bdd | `<feature.rel_filename>:Scenario name` |
| nose2 | `module.test`, or `module.Class.test` |

An entry matches by ID **or** selector; an unmatched ID does not prevent a
selector match. Parameterized tests and scenario outlines keep their existing
identities, so one selector may skip multiple cases. Use the reported `fullName`
for the exact selector, including framework-specific paths and rendered names.

Skipping uses each runner’s lifecycle. Discovery, suite setup and other hooks
may still run. Robot Framework uses its native `robot:skip` tag (Robot Framework
4+), and works with the listener alone or the existing test-plan prerun modifier.
Behave’s existing skipped-result visibility options still apply.

## Allure python commons

Common engine for all modules. It is useful for make integration with
Expand Down
9 changes: 8 additions & 1 deletion allure-behave/src/formatter.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,9 @@
from allure_commons.logger import AllureFileLogger
from allure_behave.listener import AllureListener
from allure_commons.utils import get_testplan
from allure_behave.utils import is_planned_scenario
from allure_behave.utils import is_planned_scenario, get_fullname, scenario_labels, TEST_PLAN_SKIP_REASON
from allure_commons.utils import matches_testplan, MUTED_TEST_REASON
from allure_commons.types import LabelType


class AllureFormatter(Formatter):
Expand All @@ -18,6 +20,7 @@ def __init__(self, stream_opener, config):
allure_commons.plugin_manager.register(self.file_logger)

self.testplan = get_testplan()
self.muted_tests = get_testplan("excludedTests")

def _wrap_scenario(self, scenarios):
for scenario in scenarios:
Expand All @@ -26,6 +29,10 @@ def _wrap_scenario(self, scenarios):
else:
scenario.run = allure_commons.test(scenario.run, context={"scenario": scenario})
is_planned_scenario(scenario, self.testplan)
if not isinstance(scenario, ScenarioOutline) and scenario.skip_reason != TEST_PLAN_SKIP_REASON:
ids = [label.value for label in scenario_labels(scenario) if label.name == LabelType.ID]
if matches_testplan(self.muted_tests, allure_ids=ids, selector=get_fullname(scenario)):
scenario.skip(reason=MUTED_TEST_REASON)

def uri(self, uri):
self.listener.start_file()
Expand Down
6 changes: 5 additions & 1 deletion allure-behave/src/listener.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,8 @@
from allure_commons.utils import now
from allure_commons.utils import platform_label
from allure_commons.types import LabelType, AttachmentType, LinkType
from allure_commons.model2 import TestResult
from allure_commons.model2 import TestResult, Status, StatusDetails
from allure_commons.utils import MUTED_TEST_REASON
from allure_commons.model2 import TestStepResult
from allure_commons.model2 import TestBeforeResult, TestAfterResult
from allure_commons.model2 import TestResultContainer
Expand Down Expand Up @@ -112,6 +113,9 @@ def stop_scenario(self, scenario):
else:
status = scenario_status(scenario)
status_details = scenario_status_details(scenario)
if scenario.skip_reason == MUTED_TEST_REASON:
status = Status.SKIPPED
status_details = StatusDetails(message=MUTED_TEST_REASON, muted=True)

self.flush_steps()
test_result = self.logger.get_test(self.current_scenario_uuid)
Expand Down
29 changes: 26 additions & 3 deletions allure-nose2/src/plugin.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
from unittest import skip

from nose2.events import Plugin
from allure_commons import plugin_manager
from allure_commons.logger import AllureFileLogger
Expand All @@ -8,7 +10,8 @@
from allure_commons.model2 import StatusDetails
from allure_commons.model2 import Label
from allure_commons.types import LabelType
from allure_commons.utils import host_tag, thread_tag
from allure_commons.utils import host_tag, thread_tag, get_testplan
from allure_commons.utils import matches_testplan, MUTED_TEST_REASON

from allure_commons.utils import platform_label, md5

Expand Down Expand Up @@ -54,6 +57,7 @@ def __init__(self, *args, **kwargs):
self.lifecycle = AllureLifecycle()
self.logger = AllureFileLogger("allure-result")
self.listener = AllureListener(self.lifecycle)
self._muted_tests = {}

def registerInSubprocess(self, event):
self.unregister_allure_plugins()
Expand All @@ -70,8 +74,9 @@ def register_allure_plugins(self):
plugin_manager.register(self.logger)

def unregister_allure_plugins(self):
plugin_manager.unregister(plugin=self.listener)
plugin_manager.unregister(plugin=self.logger)
for plugin in (self.listener, self.logger):
if plugin_manager.is_registered(plugin):
plugin_manager.unregister(plugin=plugin)

def is_registered(self):
return all([plugin_manager.is_registered(self.listener),
Expand Down Expand Up @@ -99,11 +104,26 @@ def startTest(self, event):
test_result.labels.append(Label(name=LabelType.LANGUAGE, value=platform_label()))
test_result.parameters = params(event)

if matches_testplan(
get_testplan("excludedTests"),
allure_ids=(label.value for label in labels(event.test) if label.name == LabelType.ID),
selector=fullname(event),
):
method_name = event.test._testMethodName
self._muted_tests[event.test] = (method_name, event.test.__dict__.get(method_name))
setattr(event.test, method_name, skip(MUTED_TEST_REASON)(lambda: None))

def stopTest(self, event):
if self.is_registered():
with self.lifecycle.update_test_case() as test_result:
test_result.stop = timestamp_millis(event.stopTime)
self.lifecycle.write_test_case()
if event.test in self._muted_tests:
method_name, original = self._muted_tests.pop(event.test)
if original is None:
delattr(event.test, method_name)
else:
setattr(event.test, method_name, original)

def testOutcome(self, event):
if self.is_registered():
Expand All @@ -122,6 +142,9 @@ def testOutcome(self, event):
elif event.outcome == result.SKIP:
test_result.status = Status.SKIPPED
test_result.statusDetails = status_details(event)
if event.test in self._muted_tests:
test_result.statusDetails = test_result.statusDetails or StatusDetails()
test_result.statusDetails.muted = True
# Todo default status and other cases
# elif event.outcome == result.FAIL and event.expected:
# pass
Expand Down
1 change: 1 addition & 0 deletions allure-nose2/src/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
"epic",
"feature",
"story",
"as_id",
]


Expand Down
18 changes: 18 additions & 0 deletions allure-pytest-bdd/src/plugin.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,11 @@
import argparse
import os
import pytest

from allure_commons.utils import get_testplan
from allure_commons.utils import matches_testplan, MUTED_TEST_REASON
from allure_commons.types import LabelType
from .utils import iter_label_values, get_full_name

import allure_commons
from allure_commons.logger import AllureFileLogger
Expand Down Expand Up @@ -71,6 +77,18 @@ def register_marks(config):
config.addinivalue_line("markers", f"{ALLURE_EXPECTED_RESULTS_HTML_MARK}: allure expected results html")


@pytest.hookimpl(trylast=True)
def pytest_bdd_before_scenario(request, feature, scenario):
item = request.node
item._allure_muted = matches_testplan(
get_testplan("excludedTests"),
allure_ids=(value for _, value in iter_label_values(item, LabelType.ID)),
selector=get_full_name(feature, scenario),
)
if item._allure_muted:
pytest.skip(MUTED_TEST_REASON)


def pytest_configure(config):
register_marks(config)

Expand Down
6 changes: 5 additions & 1 deletion allure-pytest-bdd/src/pytest_bdd_listener.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@

from allure_commons.utils import now
from allure_commons.model2 import Label
from allure_commons.model2 import Status
from allure_commons.model2 import Status, StatusDetails
from allure_commons.types import LabelType, AttachmentType
from allure_commons.utils import platform_label
from allure_commons.utils import host_tag, thread_tag
Expand Down Expand Up @@ -153,6 +153,10 @@ def pytest_runtest_makereport(self, item, call):
attach_data(self.lifecycle, report.capstdout, "stdout", AttachmentType.TEXT, None)
if report.capstderr:
attach_data(self.lifecycle, report.capstderr, "stderr", AttachmentType.TEXT, None)
if getattr(item, "_allure_muted", False):
test_result.stop = test_result.stop or now()
test_result.statusDetails = test_result.statusDetails or StatusDetails()
test_result.statusDetails.muted = True
post_process_test_result(item, test_result)

if report.when == "teardown":
Expand Down
4 changes: 4 additions & 0 deletions allure-pytest/src/listener.py
Original file line number Diff line number Diff line change
Expand Up @@ -228,6 +228,10 @@ def pytest_runtest_makereport(self, item, call):
message = f"XPASS {reason}" if reason else "XPASS"
status_details = StatusDetails(message=message)

if getattr(item, "_allure_muted", False):
status_details = status_details or StatusDetails()
status_details.muted = True

if report.when == "setup":
test_result.status = status
test_result.statusDetails = status_details
Expand Down
10 changes: 10 additions & 0 deletions allure-pytest/src/plugin.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import argparse
import pytest

import allure
import allure_commons
Expand All @@ -7,6 +8,7 @@
from allure_commons.types import LabelType, Severity
from allure_commons.logger import AllureFileLogger
from allure_commons.utils import get_testplan
from allure_commons.utils import matches_testplan, MUTED_TEST_REASON

from allure_pytest.utils import allure_label, allure_labels, allure_full_name
from allure_pytest.helper import AllureTestHelper, AllureTitleHelper
Expand Down Expand Up @@ -240,6 +242,14 @@ def pytest_collection_modifyitems(items, config):
selected, deselected_by_testcase = select_by_testcase(items, config)
selected, deselected_by_labels = select_by_labels(selected, config)

muted_tests = get_testplan("excludedTests")
for item in selected:
item._allure_muted = matches_testplan(
muted_tests, allure_ids=allure_label(item, LabelType.ID), selector=allure_full_name(item)
)
if item._allure_muted:
item.add_marker(pytest.mark.skip(reason=MUTED_TEST_REASON))

items[:] = selected

if deselected_by_testcase or deselected_by_labels:
Expand Down
1 change: 1 addition & 0 deletions allure-python-commons/src/allure_commons/model2.py
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,7 @@ class StatusDetails:
flaky = attrib(default=None)
message = attrib(default=None)
trace = attrib(default=None)
muted = attrib(default=None)


@attrs
Expand Down
26 changes: 20 additions & 6 deletions allure-python-commons/src/allure_commons/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,13 @@
import traceback
import collections

from functools import lru_cache
from traceback import format_exception_only


MUTED_TEST_REASON = "Skipped by Allure test plan"


def md5(*args):
m = hashlib.md5()
for arg in args:
Expand Down Expand Up @@ -323,16 +327,26 @@ def format_exception(etype, value):
return "\n".join(format_exception_only(etype, value)) if etype or value else None


def get_testplan():
planned_tests = []
@lru_cache(maxsize=1)
def _load_testplan():
file_path = os.environ.get("ALLURE_TESTPLAN_PATH")

if file_path and os.path.exists(file_path):
with open(file_path, "r") as plan_file:
plan = json.load(plan_file)
planned_tests = plan.get("tests", [])
return json.load(plan_file)
return {}


return planned_tests
def get_testplan(field="tests"):
return _load_testplan().get(field, [])


def matches_testplan(entries, allure_ids=(), selector=None):
ids = {str(value) for value in allure_ids if value is not None}
return any(
(entry.get("id") is not None and str(entry["id"]) in ids)
or (selector is not None and entry.get("selector") == selector)
for entry in entries or []
)


class SafeFormatter(string.Formatter):
Expand Down
4 changes: 4 additions & 0 deletions allure-robotframework/src/listener/allure_listener.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
from allure_commons.utils import format_exception, format_traceback
from allure_commons.model2 import Label, Link
from allure_commons.model2 import Status, StatusDetails
from allure_commons.utils import MUTED_TEST_REASON
from allure_commons.model2 import Parameter
from allure_commons.types import LabelType, AttachmentType, Severity, LinkType
from allure_robotframework.utils import get_allure_status
Expand Down Expand Up @@ -158,6 +159,9 @@ def stop_test(self, _, attributes, messages):
test_result.labels.extend(allure_labels(tags))
test_result.statusDetails = StatusDetails(message=attributes.get("message"),
trace=self._current_tb)
if attributes.get("allure_muted"):
test_result.status = Status.SKIPPED
test_result.statusDetails = StatusDetails(message=MUTED_TEST_REASON, muted=True)

if attributes.get("critical") == "yes":
test_result.labels.append(Label(name=LabelType.SEVERITY, value=Severity.CRITICAL))
Expand Down
13 changes: 12 additions & 1 deletion allure-robotframework/src/listener/robot_listener.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,10 @@
import os
import allure_commons
from robot.libraries.BuiltIn import BuiltIn
from allure_commons.utils import get_testplan
from allure_commons.utils import matches_testplan
from allure_commons.types import LabelType
from allure_robotframework.utils import allure_labels

from allure_commons.lifecycle import AllureLifecycle
from allure_commons.logger import AllureFileLogger
Expand All @@ -17,6 +22,8 @@ class allure_robotframework:
def __init__(self, logger_path=DEFAULT_OUTPUT_PATH):
self.messages = Messages()
self.title_path = []
self.muted_tests = get_testplan("excludedTests")
self._muted = False

self.logger = AllureFileLogger(logger_path)
self.lifecycle = AllureLifecycle()
Expand All @@ -37,12 +44,16 @@ def end_suite(self, name, attributes):

def start_test(self, name, attributes):
self.messages.start_context()
ids = [label.value for label in allure_labels(attributes.get("tags", ())) if label.name == LabelType.ID]
self._muted = matches_testplan(self.muted_tests, allure_ids=ids, selector=attributes.get("longname"))
if self._muted:
BuiltIn().set_tags("robot:skip")
self.listener.start_test_container(name, attributes)
self.listener.start_test(name, {**attributes, "titlepath": self.title_path})

def end_test(self, name, attributes):
messages = self.messages.stop_context()
self.listener.stop_test(name, attributes, messages)
self.listener.stop_test(name, {**attributes, "allure_muted": self._muted}, messages)
self.listener.stop_test_container(name, attributes)

def start_keyword(self, name, attributes):
Expand Down
1 change: 1 addition & 0 deletions requirements/testing/allure-pytest-bdd.txt
Original file line number Diff line number Diff line change
@@ -1 +1,2 @@
# allure_pytest_bdd testing deps
pytest-xdist
Loading