Tests: Preserve default excluded groups when using --exclude-group

phpunit.xml.dist

@@ -32,6 +32,9 @@
 			<group>html-api-html5lib-tests</group>
 		</exclude>
 	</groups>
+	<listeners>
+		<listener class="WP_PHPUnit_Default_Group_Filter" />
+	</listeners>
 	<logging>
 		<log type="junit" target="tests/phpunit/build/logs/junit.xml" />
 	</logging>

tests/phpunit/includes/abstract-testcase.php

@@ -143,6 +143,31 @@ public function set_up() {
 		add_filter( 'wp_hash_password_options', array( $this, 'wp_hash_password_options' ), 1, 2 );
 	}
 
+	/**
+	 * Returns the default-excluded groups that apply to a test and were not explicitly requested.
+	 *
+	 * @param string[] $groups Test groups.
+	 * @return string[] Matching excluded groups.
+	 */
+	public static function get_default_excluded_groups_for_test( array $groups ) {
+		$default_excluded_groups = WP_PHPUnit_Util_Getopt::get_skipped_groups();
+
+		foreach ( $groups as $group ) {
+			if ( WP_PHPUnit_Util_Getopt::is_group_requested( $group ) ) {
+				return array();
+			}
+		}
+
+		return array_values(
+			array_filter(
+				$groups,
+				static function ( $group ) use ( $default_excluded_groups ) {
+					return in_array( $group, $default_excluded_groups, true );
+				}
+			)
+		);
+	}
+
 	/**
 	 * Sets the bcrypt cost option for password hashing during tests.
 	 *

tests/phpunit/includes/bootstrap.php

@@ -345,42 +345,88 @@ function wp_tests_options( $value ) {
  * how you call phpunit has no effect.
  */
 class WP_PHPUnit_Util_Getopt {
-
-	public function __construct( $argv ) {
+	/**
+	 * Groups that are skipped by default for the current test run.
+	 *
+	 * @var array<string, bool>
+	 */
+	protected static $skipped_groups = array();
+
+	/**
+	 * Groups that were explicitly requested via the command line.
+	 *
+	 * @var array<string, bool>
+	 */
+	protected static $requested_groups = array();
+
+	/**
+	 * Initializes the default skipped groups for the current test mode.
+	 *
+	 * @return array<string, bool> The groups skipped by default.
+	 */
+	protected static function get_default_skipped_groups() {
 		$skipped_groups = array(
-			'ajax'          => true,
-			'ms-files'      => true,
-			'external-http' => true,
+			'ajax'                    => true,
+			'ms-files'                => true,
+			'external-http'           => true,
+			'html-api-html5lib-tests' => true,
 		);
 
-		while ( current( $argv ) ) {
-			$option = current( $argv );
-			$value  = next( $argv );
+		if ( is_multisite() ) {
+			$skipped_groups['ms-excluded']    = true;
+			$skipped_groups['oembed-headers'] = true;
+		} else {
+			$skipped_groups['ms-required'] = true;
+		}
+
+		return $skipped_groups;
+	}
+
+	/**
+	 * Initializes the parsed command line state.
+	 */
+	protected static function reset() {
+		self::$skipped_groups   = self::get_default_skipped_groups();
+		self::$requested_groups = array();
+	}
+
+	public function __construct( $argv ) {
+		self::reset();
+
+		foreach ( $argv as $index => $argument ) {
+			$option = $argument;
+			$value  = isset( $argv[ $index + 1 ] ) ? $argv[ $index + 1 ] : '';
+
+			if ( 0 === strpos( $option, '--group=' ) ) {
+				$value  = substr( $option, strlen( '--group=' ) );
+				$option = '--group';
+			}
 
 			switch ( $option ) {
 				case '--exclude-group':
-					foreach ( $skipped_groups as $group_name => $skipped ) {
-						$skipped_groups[ $group_name ] = false;
-					}
-					continue 2;
+					// PHPUnit replaces XML exclusions when this option is provided.
+					// Keep our default skipped groups active unless they are requested explicitly.
+					break;
 				case '--group':
 					$groups = explode( ',', $value );
 					foreach ( $groups as $group ) {
+						self::$requested_groups[ $group ] = true;
+
 						if ( is_numeric( $group ) || preg_match( '/^(UT|Plugin)\d+$/', $group ) ) {
 							WP_UnitTestCase::forceTicket( $group );
 						}
 					}
 
-					foreach ( $skipped_groups as $group_name => $skipped ) {
+					foreach ( self::$skipped_groups as $group_name => $skipped ) {
 						if ( in_array( $group_name, $groups, true ) ) {
-							$skipped_groups[ $group_name ] = false;
+							self::$skipped_groups[ $group_name ] = false;
 						}
 					}
-					continue 2;
+					break;
 			}
 		}
 
-		$skipped_groups = array_filter( $skipped_groups );
+		$skipped_groups = array_filter( self::$skipped_groups );
 		foreach ( $skipped_groups as $group_name => $skipped ) {
 			echo sprintf( 'Not running %1$s tests. To execute these, use --group %1$s.', $group_name ) . PHP_EOL;
 		}
@@ -392,5 +438,76 @@ public function __construct( $argv ) {
 			echo PHP_EOL;
 		}
 	}
+
+	/**
+	 * Returns the groups that are still skipped by default for this run.
+	 *
+	 * @return string[]
+	 */
+	public static function get_skipped_groups() {
+		return array_keys( array_filter( self::$skipped_groups ) );
+	}
+
+	/**
+	 * Determines whether a group was explicitly requested via the command line.
+	 *
+	 * @param string $group Group name.
+	 * @return bool Whether the group was explicitly requested.
+	 */
+	public static function is_group_requested( $group ) {
+		return isset( self::$requested_groups[ $group ] );
+	}
 }
 new WP_PHPUnit_Util_Getopt( $_SERVER['argv'] );
+
+/**
+ * Removes default-excluded tests from PHPUnit suites before class fixtures run.
+ */
+trait WP_PHPUnit_Default_Group_Filter_Methods {
+
+	/**
+	 * Filters default-excluded tests from a suite and all nested suites.
+	 *
+	 * @param PHPUnit\Framework\TestSuite $suite Test suite.
+	 */
+	public function start_test_suite( $suite ) {
+		$this->filter_suite( $suite );
+	}
+
+	/**
+	 * Filters a test suite recursively.
+	 *
+	 * @param PHPUnit\Framework\TestSuite $suite Test suite.
+	 */
+	protected function filter_suite( $suite ) {
+		$tests = array();
+
+		foreach ( $suite->tests() as $test ) {
+			if ( $test instanceof PHPUnit\Framework\TestSuite || $test instanceof PHPUnit_Framework_TestSuite ) {
+				$this->filter_suite( $test );
+
+				if ( count( $test ) ) {
+					$tests[] = $test;
+				}
+			} elseif ( ! method_exists( $test, 'getGroups' ) || ! WP_UnitTestCase_Base::get_default_excluded_groups_for_test( $test->getGroups() ) ) {
+				$tests[] = $test;
+			}
+		}
+
+		$suite->setTests( $tests );
+	}
+}
+
+if ( version_compare( tests_get_phpunit_version(), '6.0', '<' ) ) {
+	class WP_PHPUnit_Default_Group_Filter implements PHPUnit_Framework_TestListener {
+		use Yoast\PHPUnitPolyfills\TestListeners\TestListenerDefaultImplementation, WP_PHPUnit_Default_Group_Filter_Methods {
+			WP_PHPUnit_Default_Group_Filter_Methods::start_test_suite insteadof Yoast\PHPUnitPolyfills\TestListeners\TestListenerDefaultImplementation;
+		}
+	}
+} else {
+	class WP_PHPUnit_Default_Group_Filter implements PHPUnit\Framework\TestListener {
+		use Yoast\PHPUnitPolyfills\TestListeners\TestListenerDefaultImplementation, WP_PHPUnit_Default_Group_Filter_Methods {
+			WP_PHPUnit_Default_Group_Filter_Methods::start_test_suite insteadof Yoast\PHPUnitPolyfills\TestListeners\TestListenerDefaultImplementation;
+		}
+	}
+}

tests/phpunit/multisite.xml

@@ -37,6 +37,9 @@
 			<group>html-api-html5lib-tests</group>
 		</exclude>
 	</groups>
+	<listeners>
+		<listener class="WP_PHPUnit_Default_Group_Filter" />
+	</listeners>
 	<filter>
 		<whitelist addUncoveredFilesFromWhitelist="true">
 			<directory suffix=".php">../../src</directory>

tests/phpunit/tests/phpunit/groupFilters.php

@@ -0,0 +1,102 @@
+<?php
+/**
+ * @group phpunit
+ */
+class Tests_PHPUnit_GroupFilters extends WP_UnitTestCase {
+
+	public function set_up() {
+		parent::set_up();
+		$this->parse_arguments( array( 'phpunit' ) );
+	}
+
+	public function tear_down() {
+		$this->parse_arguments( $_SERVER['argv'] );
+		parent::tear_down();
+	}
+
+	public function test_single_site_default_excluded_groups_are_reported() {
+		$this->assertSame(
+			array( 'ajax', 'external-http' ),
+			WP_UnitTestCase_Base::get_default_excluded_groups_for_test( array( 'ajax', 'external-http', 'query' ) )
+		);
+	}
+
+	/**
+	 * @dataProvider data_group_argument_syntax
+	 *
+	 * @param string[] $arguments Command line arguments.
+	 */
+	public function test_requested_group_is_not_reported_as_default_excluded( $arguments ) {
+		$this->parse_arguments( $arguments );
+
+		$this->assertSame(
+			array(),
+			WP_UnitTestCase_Base::get_default_excluded_groups_for_test( array( 'ajax' ) )
+		);
+	}
+
+	/**
+	 * Data provider.
+	 *
+	 * @return array<string, array<int, array<int, string>>>
+	 */
+	public function data_group_argument_syntax() {
+		return array(
+			'spaced' => array( array( 'phpunit', '--group', 'ajax' ) ),
+			'equals' => array( array( 'phpunit', '--group=ajax' ) ),
+		);
+	}
+
+	public function test_exclude_group_does_not_clear_default_excluded_groups() {
+		$this->parse_arguments(
+			array(
+				'phpunit',
+				'--exclude-group',
+				'import',
+			)
+		);
+
+		$multisite_excluded_group = is_multisite() ? 'ms-excluded' : 'ms-required';
+
+		$this->assertSame(
+			array( 'ajax', $multisite_excluded_group ),
+			WP_UnitTestCase_Base::get_default_excluded_groups_for_test( array( 'ajax', $multisite_excluded_group ) )
+		);
+	}
+
+	public function test_requested_group_allows_a_test_in_a_default_excluded_group() {
+		$this->parse_arguments( array( 'phpunit', '--group', 'xdebug' ) );
+
+		$this->assertSame(
+			array(),
+			WP_UnitTestCase_Base::get_default_excluded_groups_for_test( array( 'ajax', 'xdebug' ) )
+		);
+	}
+
+	public function test_default_group_filter_removes_tests_from_nested_suites() {
+		$class_suite = class_exists( 'PHPUnit_Framework_TestSuite' ) ? new PHPUnit_Framework_TestSuite() : new PHPUnit\Framework\TestSuite();
+		$class_suite->addTest( clone $this, array( 'phpunit' ) );
+		$class_suite->addTest( clone $this, array( 'ajax' ) );
+		$class_suite->addTest( clone $this, array( 'phpunit' ) );
+
+		$root_suite = new PHPUnit\Framework\TestSuite();
+		$root_suite->addTest( $class_suite );
+
+		$filter = new WP_PHPUnit_Default_Group_Filter();
+		$filter->startTestSuite( $root_suite );
+
+		$this->assertCount( 2, $root_suite );
+		$this->assertCount( 2, $class_suite );
+	}
+
+	/**
+	 * Parses command line arguments without printing group notices.
+	 *
+	 * @param string[] $arguments Command line arguments.
+	 */
+	private function parse_arguments( $arguments ) {
+		ob_start();
+		new WP_PHPUnit_Util_Getopt( $arguments );
+		ob_end_clean();
+	}
+}