diff --git a/src/wp-includes/css-api/class-wp-css-builder.php b/src/wp-includes/css-api/class-wp-css-builder.php new file mode 100644 index 0000000000000..d778780c91462 --- /dev/null +++ b/src/wp-includes/css-api/class-wp-css-builder.php @@ -0,0 +1,110 @@ + & , ; + * { }`, which may be problematic in HTML or for rudimentary CSS + * processors. CSS cannot represent NULL, so it reads back as U+FFFD. + * Invalid UTF-8 reads back as U+FFFD. + * + * Example: + * + * ```php interactive + * echo 'content: ' . WP_CSS_Builder::string( 'Say "hi"' ); + * // Outputs: content: "Say \22 hi\22 " + * ``` + * + * @since 7.3.0 + * + * @param string $value Value the string should read back as. + * @return string Quoted CSS string. + */ + public static function string( string $value ): string { + return WP_CSS_Token_Serializer::escape_string( $value ); + } + + /** + * Writes a URL as a CSS `url()` function holding a quoted string. + * + * The URL is written as string() writes it. It is not validated or + * sanitized; pass it through esc_url() or sanitize_url() first when it + * comes from input. + * + * Example: + * + * ```php interactive + * echo WP_CSS_Builder::url( 'https://example.com/a(1).png' ); + * // Outputs: url("https://example.com/a(1).png") + * ``` + * + * @since 7.3.0 + * + * @link https://www.w3.org/TR/css-values-4/#urls + * + * @param string $url URL the function should read back as. + * @return string CSS `url()` function. + */ + public static function url( string $url ): string { + return 'url(' . self::string( $url ) . ')'; + } + + /** + * Writes a value as a CSS identifier. + * + * Code points that cannot appear in an identifier at their position are + * hex-escaped, as are the characters string() escapes. Use it for a + * property name, a custom property name in `var()`, or a keyword or + * family name written without quotes. + * + * An empty value returns an empty string, which is not an identifier. + * + * Example: + * + * ```php interactive + * echo 'font-family: ' . WP_CSS_Builder::ident( '1st Font' ); + * // Outputs: font-family: \31 st\20 Font + * ``` + * + * @since 7.3.0 + * + * @link https://www.w3.org/TR/css-syntax-3/#ident-token-diagram + * + * @param string $value Value the identifier should read back as. + * @return string CSS identifier, or an empty string for an empty value. + */ + public static function ident( string $value ): string { + return WP_CSS_Token_Serializer::escape_ident( $value ); + } +} diff --git a/src/wp-includes/css-api/class-wp-css-token-processor.php b/src/wp-includes/css-api/class-wp-css-token-processor.php new file mode 100644 index 0000000000000..e9b0e27e4bd7e --- /dev/null +++ b/src/wp-includes/css-api/class-wp-css-token-processor.php @@ -0,0 +1,1791 @@ + Replace any U+000D CARRIAGE RETURN (CR) code points, U+000C FORM FEED (FF) + * > code points, or pairs of U+000D CARRIAGE RETURN (CR) followed by U+000A LINE + * > FEED (LF) in input by a single U+000A LINE FEED (LF) code point. + * > Replace any U+0000 NULL or surrogate code points in input with U+FFFD REPLACEMENT + * > CHARACTER (�). + * + * This processor delays normalization as much as possible. That keeps the raw byte + * positions intact for accurate rewrites; WP_CSS_Token::get_value() normalizes and + * decodes a token's value when it is asked for. + * For example, `url(ab)` is a URL token whose value is `a�b`, where `` + * means one zero byte in the source. That byte stays in the raw token, so later + * edits still use byte offsets from the original input. + * + * ### No EOF token + * + * The EOF token is a CSS parsing concept, not CSS tokenization concept. Therefore, + * this processor does not produce it. + * + * ### UTF-8 handling + * + * Only UTF-8 strings are supported. Invalid sequences are replaced with U+FFFD (�) + * by the maximal subpart rule of the Unicode Standard, section 3.9.6, U+FFFD + * Substitution of Maximal Subparts. + * + * ## Usage + * + * Basic iteration: + * + * ```php interactive + * $processor = new WP_CSS_Token_Processor( 'width: 10px;' ); + * while ( $processor->next_token() ) { + * echo $processor->get_token_type() . "\n"; + * } + * // Outputs: + * // ident-token + * // colon-token + * // whitespace-token + * // dimension-token + * // semicolon-token + * ``` + * + * Gathering diagnostics with byte offsets: + * + * ```php interactive + * $css = "color: red;\ncolor: \"re\nd\";"; + * $processor = new WP_CSS_Token_Processor( $css ); + * while ( $processor->next_token() ) { + * if ( WP_CSS_Token_Processor::TOKEN_BAD_STRING === $processor->get_token_type() ) { + * echo "Bad string at byte {$processor->get_token_start()}, {$processor->get_token_length()} bytes long.\n"; + * } + * } + * // Outputs: "Bad string at byte 19, 3 bytes long." + * ``` + * + * @since 7.3.0 + * + * @link https://www.w3.org/TR/css-syntax-3/#tokenization + * @link https://www.unicode.org/versions/Unicode18.0.0/core-spec/chapter-3/#G66453 + */ +class WP_CSS_Token_Processor { + /** + * Token type constants matching the CSS Syntax Level 3 specification. + * + * @link https://www.w3.org/TR/css-syntax-3/#tokenization + */ + public const TOKEN_WHITESPACE = 'whitespace-token'; + + /** + * Comment tokens, which open with `/` `*` and close with `*` `/`. + * + * The specification's tokenizer produces no comment tokens. This processor + * does, so callers can find comments by their byte offsets. CSS Syntax + * Level 3 §9 allows this: + * + * > Implementations may preserve the contents of comments and their location in the token stream. + * > If they do, this preserved information must have no effect on the parsing step. + * + * @link https://www.w3.org/TR/css-syntax-3/#serialization + */ + public const TOKEN_COMMENT = 'comment'; + + /** + * String tokens, such as `"hello"` and `'world'`. + * + * @see WP_CSS_Token_Processor::TOKEN_BAD_STRING + * @link https://www.w3.org/TR/css-syntax-3/#typedef-string-token + */ + public const TOKEN_STRING = 'string-token'; + + /** + * BAD-STRING tokens occur when a string contains an unescaped newline. + * + * These are valid strings; the last contains an escaped newline: + * + * "hello" 'world' "line1\Aline2" + * + * This string produces a bad-string token, because a literal newline ends it: + * + * "hello + * world" + * + * The processor stops at the newline and produces a bad-string token for error recovery. + * + * @see WP_CSS_Token_Processor::TOKEN_STRING + * @link https://www.w3.org/TR/css-syntax-3/#typedef-bad-string-token + */ + public const TOKEN_BAD_STRING = 'bad-string-token'; + + /** + * Hash tokens, such as `#fff` and `#header`. + * + * A hash token has an "id" or "unrestricted" type flag. + * + * @see WP_CSS_Token_Processor::HASH_TOKEN_ID + * @see WP_CSS_Token_Processor::HASH_TOKEN_UNRESTRICTED + * @link https://www.w3.org/TR/css-syntax-3/#typedef-hash-token + */ + public const TOKEN_HASH = 'hash-token'; + + /** + * Type flag of a hash token whose name would start an identifier, such as `#header`. + * + * @see WP_CSS_Token_Processor::TOKEN_HASH + */ + public const HASH_TOKEN_ID = 'id'; + + /** + * Type flag of a hash token whose name would not start an identifier, such as `#123`. + * + * @see WP_CSS_Token_Processor::TOKEN_HASH + */ + public const HASH_TOKEN_UNRESTRICTED = 'unrestricted'; + + public const TOKEN_DELIM = 'delim-token'; + public const TOKEN_NUMBER = 'number-token'; + public const TOKEN_PERCENTAGE = 'percentage-token'; + public const TOKEN_DIMENSION = 'dimension-token'; + public const TOKEN_AT_KEYWORD = 'at-keyword-token'; + public const TOKEN_COLON = 'colon-token'; + public const TOKEN_SEMICOLON = 'semicolon-token'; + public const TOKEN_COMMA = 'comma-token'; + public const TOKEN_LEFT_PAREN = '(-token'; + public const TOKEN_RIGHT_PAREN = ')-token'; + public const TOKEN_LEFT_BRACKET = '[-token'; + public const TOKEN_RIGHT_BRACKET = ']-token'; + public const TOKEN_LEFT_BRACE = '{-token'; + public const TOKEN_RIGHT_BRACE = '}-token'; + public const TOKEN_FUNCTION = 'function-token'; + + /** + * URL tokens represent unquoted URLs in url() notation. + * + * For example, `url(image.jpg)` is a URL token. + * + * Quoted URLs like `url( "https://example.com" )` are handled as a function + * token, _not_ a URL token. + * + * Bad URL tokens are created when invalid characters are encountered in + * a URL token. + * + * @see WP_CSS_Token_Processor::TOKEN_BAD_URL + * @see WP_CSS_Token_Processor::TOKEN_FUNCTION + * @link https://www.w3.org/TR/css-syntax-3/#typedef-url-token + */ + public const TOKEN_URL = 'url-token'; + + /** + * BAD-URL tokens occur when a URL contains invalid characters. + * + * Invalid characters include quotes (`"`), apostrophes (`'`) and opening + * parentheses (`(`), as in `url(image(.jpg)` or `url(image".jpg)`. + * + * When detected, the processor consumes everything up to `)` or EOF. + * This prevents the bad URL from breaking subsequent tokens. + * + * @see WP_CSS_Token_Processor::TOKEN_URL + * @link https://www.w3.org/TR/css-syntax-3/#typedef-bad-url-token + */ + public const TOKEN_BAD_URL = 'bad-url-token'; + + /** + * Identifier tokens, such as `color`, `margin-top`, `red`, + * `inherit`, `--my-var`, `\x-escaped`, `über` (Unicode), etc. + * + * There are restrictions on the codepoints that start or are contained in + * an identifier, and identifiers may contain escape sequences. + * + * @link https://www.w3.org/TR/css-syntax-3/#typedef-ident-token + */ + public const TOKEN_IDENT = 'ident-token'; + + /** + * CDC (Comment Delimiter Close) token: `-->` + * + * Legacy token from when CSS was embedded in HTML ` + * + * Modern CSS no longer needs these, but they're preserved for compatibility. + * In stylesheets, they're typically treated like whitespace. + * + * @see WP_CSS_Token_Processor::TOKEN_CDO + * @link https://www.w3.org/TR/css-syntax-3/#typedef-cdc-token + */ + public const TOKEN_CDC = 'CDC-token'; + + /** + * CDO (Comment Delimiter Open) token: ``). + * + * Comment Delimiter Close - legacy HTML comment syntax in CSS. + * + * @link https://www.w3.org/TR/css-syntax-3/#CDC-token-diagram + */ + if ( + $this->at + 2 < $this->length && + '-' === $this->css[ $this->at + 1 ] && + '>' === $this->css[ $this->at + 2 ] + ) { + // Consume them and return a . + $this->at += 3; + $this->token_type = self::TOKEN_CDC; + $this->token_length = 3; + return true; + } + + // Otherwise, if the input stream starts with an ident sequence, + // reconsume the current input code point, consume an ident-like + // token, and return it. + if ( $this->check_if_3_code_points_start_an_ident_sequence( $this->at ) ) { + return $this->consume_ident_like(); + } + + // Otherwise, return a with its value set to the current input code point. + ++$this->at; + $this->token_type = self::TOKEN_DELIM; + $this->token_length = 1; + return true; + } + + /* + * U+003C LESS-THAN SIGN (<) + * If followed by `!--`, this is a CDO token (`', + WP_CSS_Token_Processor::TOKEN_BAD_STRING => "\"\n", + WP_CSS_Token_Processor::TOKEN_BAD_URL => 'url(()', + ); + + /** + * Closing token type of each token type that opens a block. + * + * @since 7.3.0 + * @ignore + * + * @var array + */ + private const BLOCK_CLOSERS = array( + WP_CSS_Token_Processor::TOKEN_FUNCTION => WP_CSS_Token_Processor::TOKEN_RIGHT_PAREN, + WP_CSS_Token_Processor::TOKEN_LEFT_PAREN => WP_CSS_Token_Processor::TOKEN_RIGHT_PAREN, + WP_CSS_Token_Processor::TOKEN_LEFT_BRACKET => WP_CSS_Token_Processor::TOKEN_RIGHT_BRACKET, + WP_CSS_Token_Processor::TOKEN_LEFT_BRACE => WP_CSS_Token_Processor::TOKEN_RIGHT_BRACE, + ); + + /** + * Serializes tokens as CSS text that parses to the same CSS. + * + * This follows CSS Syntax Level 3 §9 Serialization: + * + * > The only requirement for serialization is that it must "round-trip" with parsing, + * > that is, parsing the stylesheet must produce the same data structures as + * > parsing, serializing, and parsing again, + * > except for consecutive s, + * > which may be collapsed into a single token. + * + * Comment tokens are dropped. Each run of whitespace, including a run + * broken by comments, is written as one space; a run is never removed. + * + * Tokens are written from their decoded values. Identifiers, function + * names, at-keywords, hash names, dimension units, strings and url values + * are escaped again, so `\66 oo` becomes `foo` and `"a&b"` becomes `"a\26 b"`. + * + * > A containing U+005C REVERSE SOLIDUS (\) + * > must be serialized as U+005C REVERSE SOLIDUS + * > followed by a newline. + * > (The tokenizer only ever emits such a token followed by a + * > that starts with a newline.) + * + * > A with the "unrestricted" type flag may not need + * > as much escaping as the same token with the "id" type flag. + * + * A hash with the "id" flag is escaped as an identifier, so `#\31 a` stays + * `#\31 a`. A hash with the "unrestricted" flag may start with a digit or + * `-`, so `#1a` stays `#1a`. + * + * > The unit of a may need escaping + * > to disambiguate with scientific notation. + * + * A unit that starts with `e`, an optional `-`, and a digit has the `e` + * escaped, so the unit `e3` is written `1\65 3`, not `1e3`. + * + * > For any consecutive pair of tokens, + * > if the first token shows up in the row headings of the following table, + * > and the second token shows up in the column headings, + * > and there’s a ✗ in the cell denoted by the intersection of the chosen row and column, + * > the pair of tokens must be serialized with a comment between them. + * > + * > Single characters in the row and column headings represent a with that value, + * > except for "(", + * > which represents a (-token. + * + * ┌────────────┬───────┬──────────┬─────┬─────────┬───┬────────┬────────────┬───────────┬─────┬───┬───┬───┐ + * │ │ ident │ function │ url │ bad url │ - │ number │ percentage │ dimension │ CDC │ ( │ * │ % │ + * ├────────────┼───────┼──────────┼─────┼─────────┼───┼────────┼────────────┼───────────┼─────┼───┼───┼───┤ + * │ ident │ ✗ │ ✗ │ ✗ │ ✗ │ ✗ │ ✗ │ ✗ │ ✗ │ ✗ │ ✗ │ │ │ + * │ at-keyword │ ✗ │ ✗ │ ✗ │ ✗ │ ✗ │ ✗ │ ✗ │ ✗ │ ✗ │ │ │ │ + * │ hash │ ✗ │ ✗ │ ✗ │ ✗ │ ✗ │ ✗ │ ✗ │ ✗ │ ✗ │ │ │ │ + * │ dimension │ ✗ │ ✗ │ ✗ │ ✗ │ ✗ │ ✗ │ ✗ │ ✗ │ ✗ │ │ │ │ + * │ # │ ✗ │ ✗ │ ✗ │ ✗ │ ✗ │ ✗ │ ✗ │ ✗ │ ✗ │ │ │ │ + * │ - │ ✗ │ ✗ │ ✗ │ ✗ │ ✗ │ ✗ │ ✗ │ ✗ │ ✗ │ │ │ │ + * │ number │ ✗ │ ✗ │ ✗ │ ✗ │ │ ✗ │ ✗ │ ✗ │ ✗ │ │ │ ✗ │ + * │ @ │ ✗ │ ✗ │ ✗ │ ✗ │ ✗ │ │ │ │ ✗ │ │ │ │ + * │ . │ │ │ │ │ │ ✗ │ ✗ │ ✗ │ │ │ │ │ + * │ + │ │ │ │ │ │ ✗ │ ✗ │ ✗ │ │ │ │ │ + * │ / │ │ │ │ │ │ │ │ │ │ │ ✗ │ │ + * └────────────┴───────┴──────────┴─────┴─────────┴───┴────────┴────────────┴───────────┴─────┴───┴───┴───┘ + * + * Comments are not preserved, so an empty CSS comment is written between + * the tokens of a marked pair when no whitespace is between them. + * + * The table compares token types in pairs, so it misses two sequences that + * the tokenizer reads as a CDO or CDC token: `<`, `!` and a token that + * starts with `--`, which spell ``. An empty CSS comment is written before the last token + * of each. + * + * Bad-string and bad-url tokens have no value. A bad string is written as + * a quote followed by a newline, where the tokenizer ends one; a bad url is + * written as `url(()`. Both re-tokenize as the same bad token. If a later + * step replaces the newline after a bad string with a space, the quote + * starts a string that includes the text after it. + * + * Strings and urls cut off by the end of the input are written closed. + * Blocks still open at the end are closed in reverse order, so `a(b[c` is + * written `a(b[c])`. + * + * Example: + * + * ```php interactive + * $processor = new WP_CSS_Token_Processor( 'url( a&b ) calc(1px + 2px' ); + * $tokens = array(); + * while ( $processor->next_token() ) { + * $tokens[] = $processor->get_token(); + * } + * echo WP_CSS_Token_Serializer::serialize( $tokens ); + * // Outputs: "url(a\26 b) calc(1px + 2px)" + * ``` + * + * @link https://www.w3.org/TR/css-syntax-3/#serialization + * + * @since 7.3.0 + * + * @param iterable $tokens Tokens read by WP_CSS_Token_Processor::get_token(). + * @return string CSS text. + */ + public static function serialize( iterable $tokens ): string { + $output = ''; + $closers = array(); + + // Pair key of the last token written, or the whitespace type after a space. + $last = null; + + // Text of the last two tokens written, or a space after whitespace. + $last_text = ''; + $before_last_text = ''; + + foreach ( $tokens as $token ) { + $type = $token->get_type(); + + if ( WP_CSS_Token_Processor::TOKEN_COMMENT === $type ) { + continue; + } + + if ( WP_CSS_Token_Processor::TOKEN_WHITESPACE === $type ) { + if ( WP_CSS_Token_Processor::TOKEN_WHITESPACE !== $last ) { + $output .= ' '; + $last = WP_CSS_Token_Processor::TOKEN_WHITESPACE; + $before_last_text = $last_text; + $last_text = ' '; + } + continue; + } + + $key = WP_CSS_Token_Processor::TOKEN_DELIM === $type ? $token->get_value() : $type; + $text = self::serialize_token( $token ); + + if ( + ( null !== $last && isset( self::MARKED_PAIRS[ $last ] ) && in_array( $key, self::MARKED_PAIRS[ $last ], true ) ) || + // The table misses `<`, `!`, `--x`, which reads as ``. + ( '<' === $before_last_text && '!' === $last_text && 0 === strncmp( $text, '--', 2 ) ) || + ( '--' === $last_text && '>' === $text ) + ) { + $output .= '/**/'; + } + + $output .= $text; + $last = $key; + $before_last_text = $last_text; + $last_text = $text; + + if ( isset( self::BLOCK_CLOSERS[ $type ] ) ) { + $closers[] = self::BLOCK_CLOSERS[ $type ]; + } elseif ( end( $closers ) === $type ) { + array_pop( $closers ); + } + } + + while ( array() !== $closers ) { + $output .= self::FIXED_TEXT[ array_pop( $closers ) ]; + } + + return $output; + } + + /** + * Serializes one token other than whitespace or a comment. + * + * @since 7.3.0 + * @ignore + * + * @param WP_CSS_Token $token Token. + * @return string CSS text. + */ + private static function serialize_token( WP_CSS_Token $token ): string { + $value = $token->get_value(); + + switch ( $token->get_type() ) { + case WP_CSS_Token_Processor::TOKEN_IDENT: + return self::escape_ident( $value ); + + case WP_CSS_Token_Processor::TOKEN_FUNCTION: + return self::escape_ident( $value ) . '('; + + case WP_CSS_Token_Processor::TOKEN_AT_KEYWORD: + return '@' . self::escape_ident( $value ); + + case WP_CSS_Token_Processor::TOKEN_HASH: + return WP_CSS_Token_Processor::HASH_TOKEN_ID === $token->get_flag() + ? '#' . self::escape_ident( $value ) + : '#' . self::escape_name( $value ); + + case WP_CSS_Token_Processor::TOKEN_STRING: + return self::escape_string( $value ); + + case WP_CSS_Token_Processor::TOKEN_URL: + return self::escape_url( $value ); + + case WP_CSS_Token_Processor::TOKEN_NUMBER: + return $value; + + case WP_CSS_Token_Processor::TOKEN_PERCENTAGE: + return $value . '%'; + + case WP_CSS_Token_Processor::TOKEN_DIMENSION: + $unit = self::escape_ident( $token->get_unit() ); + // A unit that starts with `e`, an optional `-`, and a digit would read as an exponent. + if ( 1 === preg_match( '/^[eE]-?[0-9]/', $unit ) ) { + $unit = sprintf( '\\%x ', ord( $unit[0] ) ) . substr( $unit, 1 ); + } + return $value . $unit; + + case WP_CSS_Token_Processor::TOKEN_DELIM: + return '\\' === $value ? "\\\n" : $value; + + default: + return self::FIXED_TEXT[ $token->get_type() ]; + } + } + + /** + * Escapes a decoded value as identifier text. + * + * Characters not valid in CSS identifiers are hex-escaped, as are the + * characters escape_string() escapes for HTML and CSS. + * + * An empty value returns an empty string. Callers that require an ident + * token must reject an empty value. + * + * @link https://www.w3.org/TR/css-syntax-3/#escaping + * @link https://www.w3.org/TR/css-syntax-3/#would-start-an-identifier + * + * @since 7.3.0 + * @access private + * @ignore + * + * @param string $value Decoded value. + * @return string Identifier text. + */ + public static function escape_ident( string $value ): string { + // A lone `-` cannot start an identifier. + if ( '-' === $value ) { + return '\\2d '; + } + + // Neither can a digit or a `-` followed by a digit, so the digit is escaped. + $digit_at = ( isset( $value[0] ) && '-' === $value[0] ) ? 1 : 0; + if ( isset( $value[ $digit_at ] ) && $value[ $digit_at ] >= '0' && $value[ $digit_at ] <= '9' ) { + return substr( $value, 0, $digit_at ) . sprintf( '\\%x ', ord( $value[ $digit_at ] ) ) . self::escape_name( substr( $value, $digit_at + 1 ) ); + } + + return self::escape_name( $value ); + } + + /** + * Escapes a decoded value as a quoted string. + * + * Quotes, `\` and the control characters U+0001 to U+001F and U+007F are + * hex-escaped. So are `< > & , ; { }`, which may be problematic in HTML or + * for rudimentary CSS processors. The string reads back as the same value, + * except that NULL becomes U+FFFD. + * + * @link https://www.w3.org/TR/css-syntax-3/#escaping + * @link https://drafts.csswg.org/cssom/#serialize-a-string + * + * @since 7.3.0 + * @access private + * @ignore + * + * @see WP_CSS_Builder::string() + * + * @param string $value Decoded value. + * @return string Quoted string text. + */ + public static function escape_string( string $value ): string { + $value = wp_scrub_utf8( $value ); + $result = ''; + $length = strlen( $value ); + + for ( $i = 0; $i < $length; $i++ ) { + $byte = ord( $value[ $i ] ); + + // CSS cannot represent NULL; preprocessing replaces it with U+FFFD. + if ( 0x00 === $byte ) { + $result .= "\u{FFFD}"; + continue; + } + + if ( + $byte < 0x20 || + 0x7F === $byte || + false !== strpos( '"\'\\<>&,;{}', $value[ $i ] ) + ) { + $result .= sprintf( '\\%x ', $byte ); + continue; + } + + $result .= $value[ $i ]; + } + + return "\"{$result}\""; + } + + /** + * Escapes every code point that cannot appear in an identifier. + * + * Letters, digits, `-`, `_` and the non-ASCII ident code points are + * written as they are; every other code point is hex-escaped. Unlike + * escape_ident(), a leading digit or `-` is not escaped, as the name of + * a hash token with the "unrestricted" flag requires. + * + * The tokenizer reads every code point above U+007F as an identifier code + * point, as browsers do. CSS Syntax allows fewer, so the others are escaped + * and the output reads the same under both rules. + * + * @link https://drafts.csswg.org/css-syntax-3/#non-ascii-ident-code-point + * + * @since 7.3.0 + * @ignore + * + * @param string $value Decoded name. + * @return string Escaped name. + */ + private static function escape_name( string $value ): string { + $value = wp_scrub_utf8( $value ); + $result = ''; + $length = strlen( $value ); + + for ( $i = 0; $i < $length; $i++ ) { + $byte = ord( $value[ $i ] ); + + if ( 0x00 === $byte ) { + $result .= "\u{FFFD}"; + continue; + } + + if ( $byte >= 0x80 ) { + // The value is valid UTF-8, so the leading byte gives the sequence length. + $sequence_length = $byte >= 0xF0 ? 4 : ( $byte >= 0xE0 ? 3 : 2 ); + $code_point = $byte & ( 0xFF >> ( $sequence_length + 1 ) ); + for ( $j = 1; $j < $sequence_length; $j++ ) { + $code_point = ( $code_point << 6 ) | ( ord( $value[ $i + $j ] ) & 0x3F ); + } + + $result .= self::is_non_ascii_ident_code_point( $code_point ) + ? substr( $value, $i, $sequence_length ) + : sprintf( '\\%x ', $code_point ); + $i += $sequence_length - 1; + continue; + } + + if ( + ( $byte >= 0x30 && $byte <= 0x39 ) || // 0-9 + ( $byte >= 0x41 && $byte <= 0x5A ) || // A-Z + ( $byte >= 0x61 && $byte <= 0x7A ) || // a-z + 0x5F === $byte || // _ + 0x2D === $byte // - + ) { + $result .= $value[ $i ]; + continue; + } + + $result .= sprintf( '\\%x ', $byte ); + } + + return $result; + } + + /** + * Reports whether a code point above U+007F is a non-ASCII ident code point. + * + * @link https://drafts.csswg.org/css-syntax-3/#non-ascii-ident-code-point + * + * @since 7.3.0 + * @ignore + * + * @param int $code_point Code point above U+007F. + * @return bool Whether the code point may appear unescaped in an identifier. + */ + private static function is_non_ascii_ident_code_point( int $code_point ): bool { + return ( + 0xB7 === $code_point || + ( $code_point >= 0xC0 && $code_point <= 0xD6 ) || + ( $code_point >= 0xD8 && $code_point <= 0xF6 ) || + ( $code_point >= 0xF8 && $code_point <= 0x37D ) || + ( $code_point >= 0x37F && $code_point <= 0x1FFF ) || + 0x200C === $code_point || + 0x200D === $code_point || + 0x203F === $code_point || + 0x2040 === $code_point || + ( $code_point >= 0x2070 && $code_point <= 0x218F ) || + ( $code_point >= 0x2C00 && $code_point <= 0x2FEF ) || + ( $code_point >= 0x3001 && $code_point <= 0xD7FF ) || + ( $code_point >= 0xF900 && $code_point <= 0xFDCF ) || + ( $code_point >= 0xFDF0 && $code_point <= 0xFFFD ) || + $code_point >= 0x10000 + ); + } + + /** + * Escapes a decoded value as an unquoted url. + * + * The code points an unquoted url cannot contain are hex-escaped: quotes, + * parentheses, `\`, whitespace and non-printable code points. So are the + * code points escape_string() escapes for HTML and CSS: `< > & , ; { }`. + * + * @link https://www.w3.org/TR/css-syntax-3/#consume-url-token + * + * @since 7.3.0 + * @ignore + * + * @param string $value Decoded value. + * @return string Url text, including `url(` and `)`. + */ + private static function escape_url( string $value ): string { + $value = wp_scrub_utf8( $value ); + $result = ''; + $length = strlen( $value ); + + for ( $i = 0; $i < $length; $i++ ) { + $byte = ord( $value[ $i ] ); + + if ( 0x00 === $byte ) { + $result .= "\u{FFFD}"; + continue; + } + + if ( + $byte <= 0x20 || + 0x7F === $byte || + false !== strpos( '"\'()\\<>&,;{}', $value[ $i ] ) + ) { + $result .= sprintf( '\\%x ', $byte ); + continue; + } + + $result .= $value[ $i ]; + } + + return "url({$result})"; + } +} diff --git a/src/wp-includes/css-api/class-wp-css-token.php b/src/wp-includes/css-api/class-wp-css-token.php new file mode 100644 index 0000000000000..55af708415637 --- /dev/null +++ b/src/wp-includes/css-api/class-wp-css-token.php @@ -0,0 +1,364 @@ +next_token() ) { + * $tokens[] = $processor->get_token(); + * } + * echo $tokens[3]->get_type() . "\n"; + * echo $tokens[3]->get_value(); + * // Outputs: + * // string-token + * // Exo 2 + * ``` + * + * @since 7.3.0 + * + * @see WP_CSS_Token_Processor::get_token() + */ +final class WP_CSS_Token { + /** + * CSS source the offsets index. + * + * @since 7.3.0 + * @ignore + * + * @var string + */ + private $css; + + /** + * Token type. + * + * @since 7.3.0 + * @ignore + * + * @var string + * @phpstan-var WP_CSS_Token_Processor::TOKEN_* + */ + private $type; + + /** + * Byte offset at which the token starts. + * + * @since 7.3.0 + * @ignore + * + * @var int + */ + private $start; + + /** + * Byte length of the token. + * + * @since 7.3.0 + * @ignore + * + * @var int + */ + private $length; + + /** + * Type flag of a hash, number or dimension token. + * + * @since 7.3.0 + * @ignore + * + * @var string|null + * @phpstan-var 'id'|'unrestricted'|'integer'|'number'|null + */ + private $flag; + + /** + * Decoded unit of a dimension token. + * + * @since 7.3.0 + * @ignore + * + * @var string|null + */ + private $unit; + + /** + * Byte offset at which the value of a string, url or comment token starts. + * + * @since 7.3.0 + * @ignore + * + * @var int|null + */ + private $value_start; + + /** + * Byte length of the value of a string, url or comment token. + * + * @since 7.3.0 + * @ignore + * + * @var int|null + */ + private $value_length; + + /** + * Whether the token ended at its closing delimiter. + * + * @since 7.3.0 + * @ignore + * + * @var bool + */ + private $is_terminated; + + /** + * Decoded value, or false before the first call to get_value(). + * + * @since 7.3.0 + * @ignore + * + * @var string|null|false + */ + private $value = false; + + /** + * Constructor. + * + * Use WP_CSS_Token_Processor::get_token() to create tokens. + * + * @since 7.3.0 + * @access private + * @ignore + * + * @param string $css CSS source the offsets index. + * @param string $type Token type, one of the WP_CSS_Token_Processor::TOKEN_* constants. + * @param int $start Byte offset at which the token starts. + * @param int $length Byte length of the token. + * @param string|null $flag Type flag of a hash, number or dimension token. + * @param string|null $unit Decoded unit of a dimension token. + * @param int|null $value_start Byte offset at which the value of a string, url or comment token starts. + * @param int|null $value_length Byte length of the value of a string, url or comment token. + * @param bool $is_terminated Whether the token ended at its closing delimiter. + */ + public function __construct( + string $css, + string $type, + int $start, + int $length, + ?string $flag, + ?string $unit, + ?int $value_start, + ?int $value_length, + bool $is_terminated + ) { + $this->css = $css; + $this->type = $type; + $this->start = $start; + $this->length = $length; + $this->flag = $flag; + $this->unit = $unit; + $this->value_start = $value_start; + $this->value_length = $value_length; + $this->is_terminated = $is_terminated; + } + + /** + * Gets the token type. + * + * @since 7.3.0 + * + * @return string One of the WP_CSS_Token_Processor::TOKEN_* constants. + * @phpstan-return WP_CSS_Token_Processor::TOKEN_* + */ + public function get_type(): string { + return $this->type; + } + + /** + * Gets the type flag. + * + * Hash tokens have an "id" or "unrestricted" flag. Number and dimension + * tokens have an "integer" or "number" flag. Other tokens have none. + * + * @since 7.3.0 + * + * @return string|null The type flag, or null. + * @phpstan-return 'id'|'unrestricted'|'integer'|'number'|null + */ + public function get_flag(): ?string { + return $this->flag; + } + + /** + * Gets the decoded value. + * + * The value is the token's text after input preprocessing and escape + * decoding, without its delimiters: + * + * - ident, delim: the whole token. + * - function: the name, without `(`. + * - at-keyword, hash: the name, without `@` or `#`. + * - string, url: the contents, without quotes or `url(` and `)`. + * - comment: the text between `/*` and `*` `/`. Comments contain no + * escapes, so only preprocessing applies. + * - number, percentage, dimension: the number as written in the source, + * without `%` or the unit. CSS Syntax 3 parses it as a number; this + * method keeps the text. + * - Every other type, including bad-string and bad-url: null. + * + * The value is decoded on the first call and kept for later calls. + * + * Example: + * + * ```php interactive + * $processor = new WP_CSS_Token_Processor( '"a\\62 c"' ); + * $processor->next_token(); + * echo $processor->get_token()->get_value(); + * // Outputs: abc + * ``` + * + * @since 7.3.0 + * + * @link https://www.w3.org/TR/css-syntax-3/#tokenization + * + * @return string|null The decoded value, or null for token types without a value. + */ + public function get_value(): ?string { + if ( false === $this->value ) { + $this->value = $this->decode_value(); + } + + return $this->value; + } + + /** + * Gets the decoded unit of a dimension token. + * + * @since 7.3.0 + * + * @return string|null The unit, or null for other token types. + */ + public function get_unit(): ?string { + return $this->unit; + } + + /** + * Gets the byte offset at which the token starts in the CSS source. + * + * @since 7.3.0 + * + * @return int Byte offset. + */ + public function get_start(): int { + return $this->start; + } + + /** + * Gets the byte length of the token in the CSS source. + * + * @since 7.3.0 + * + * @return int Byte length. + */ + public function get_length(): int { + return $this->length; + } + + /** + * Reports whether the token ended at its closing delimiter. + * + * Comment, string, url and bad-url tokens have a closing delimiter: `*` `/`, + * the matching quote, or `)`. When the input ends before the delimiter, + * the tokenizer still returns the token, and this method returns false. + * It returns true for every other token type, which have no closing + * delimiter. + * + * Example: + * + * ```php interactive + * $processor = new WP_CSS_Token_Processor( '"abc' ); + * $processor->next_token(); + * $token = $processor->get_token(); + * echo $token->get_type() . "\n"; + * var_dump( $token->is_terminated() ); + * // Outputs: + * // string-token + * // bool(false) + * ``` + * + * @since 7.3.0 + * + * @link https://www.w3.org/TR/css-syntax-3/#consume-comment + * @link https://www.w3.org/TR/css-syntax-3/#consume-string-token + * @link https://www.w3.org/TR/css-syntax-3/#consume-url-token + * + * @return bool Whether the token ended at its closing delimiter. + */ + public function is_terminated(): bool { + return $this->is_terminated; + } + + /** + * Decodes the value from the CSS source. + * + * @since 7.3.0 + * @ignore + * + * @return string|null The decoded value, or null for token types without a value. + */ + private function decode_value(): ?string { + switch ( $this->type ) { + case WP_CSS_Token_Processor::TOKEN_HASH: + case WP_CSS_Token_Processor::TOKEN_AT_KEYWORD: + // The value starts after the `#` or `@`. + return WP_CSS_Token_Processor::decode_range( $this->css, $this->start + 1, $this->length - 1 ); + + case WP_CSS_Token_Processor::TOKEN_FUNCTION: + // The value ends before the `(`. + return WP_CSS_Token_Processor::decode_range( $this->css, $this->start, $this->length - 1 ); + + case WP_CSS_Token_Processor::TOKEN_IDENT: + case WP_CSS_Token_Processor::TOKEN_DELIM: + return WP_CSS_Token_Processor::decode_range( $this->css, $this->start, $this->length ); + + case WP_CSS_Token_Processor::TOKEN_STRING: + return WP_CSS_Token_Processor::decode_range( $this->css, $this->value_start, $this->value_length, true ); + + case WP_CSS_Token_Processor::TOKEN_URL: + return WP_CSS_Token_Processor::decode_range( $this->css, $this->value_start, $this->value_length ); + + case WP_CSS_Token_Processor::TOKEN_COMMENT: + return WP_CSS_Token_Processor::preprocess_range( $this->css, $this->value_start, $this->value_length ); + + case WP_CSS_Token_Processor::TOKEN_NUMBER: + // Numbers are ASCII and have no escapes. + return substr( $this->css, $this->start, $this->length ); + + case WP_CSS_Token_Processor::TOKEN_PERCENTAGE: + // The value ends before the `%`. + return substr( $this->css, $this->start, $this->length - 1 ); + + case WP_CSS_Token_Processor::TOKEN_DIMENSION: + // The value ends before the unit. + return substr( WP_CSS_Token_Processor::decode_range( $this->css, $this->start, $this->length ), 0, -strlen( $this->unit ) ); + + default: + return null; + } + } +} diff --git a/src/wp-settings.php b/src/wp-settings.php index 92f9456aa02ca..366a7a469b8dd 100644 --- a/src/wp-settings.php +++ b/src/wp-settings.php @@ -289,6 +289,10 @@ require ABSPATH . WPINC . '/html-api/class-wp-html-stack-event.php'; require ABSPATH . WPINC . '/html-api/class-wp-html-processor-state.php'; require ABSPATH . WPINC . '/html-api/class-wp-html-processor.php'; +require ABSPATH . WPINC . '/css-api/class-wp-css-token.php'; +require ABSPATH . WPINC . '/css-api/class-wp-css-token-processor.php'; +require ABSPATH . WPINC . '/css-api/class-wp-css-token-serializer.php'; +require ABSPATH . WPINC . '/css-api/class-wp-css-builder.php'; require ABSPATH . WPINC . '/class-wp-block-processor.php'; require ABSPATH . WPINC . '/class-wp-http.php'; require ABSPATH . WPINC . '/class-wp-http-streams.php'; diff --git a/tests/phpunit/data/css-api/LICENSE b/tests/phpunit/data/css-api/LICENSE new file mode 100644 index 0000000000000..13b1ea364b27e --- /dev/null +++ b/tests/phpunit/data/css-api/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) Romain Menke + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/tests/phpunit/data/css-api/README.md b/tests/phpunit/data/css-api/README.md new file mode 100644 index 0000000000000..93da314e4ef01 --- /dev/null +++ b/tests/phpunit/data/css-api/README.md @@ -0,0 +1,42 @@ +# CSS tokenizer test corpus + +This directory contains a third-party test corpus used for testing the WordPress CSS API tokenizer. + +`css-test-cases.json` derives from the npm package +[`@rmenke/css-tokenizer-tests`](https://www.npmjs.com/package/@rmenke/css-tokenizer-tests) +by Romain Menke, MIT licensed (see `LICENSE`). The package source is on GitHub at +[romainmenke/css-tokenizer-tests](https://github.com/romainmenke/css-tokenizer-tests). + +The 185 cases match version 1.3.0 of the package, whose test content is identical to +version 1.2.0. Version 1.4.0 adds 102 cases that are not in this file. + +## Transform + +The upstream corpus maps a case name to `{ css, tokens }`. Each upstream token has +`type`, `raw`, `startIndex`, `endIndex` and `structured`. This file keeps the case names, +`css`, `type` and `raw` unchanged and transforms the rest: + +- `startIndex` and `endIndex` are byte offsets into the UTF-8 source. Upstream uses + JavaScript string indexes (UTF-16 code units). +- `value` is `structured.value` flattened onto the token. Numeric values are the + number's source text as a string (`"10e2"`, not `1000`); for dimension and + percentage tokens the unit or `%` is removed. A comment token, which has no + structured data upstream, has the text between its delimiters after input + preprocessing. Other tokens without structured data have `value: null`. +- `unit` is `structured.unit`, present on dimension tokens only. +- `numberType` is `structured.type` on number and dimension tokens. The upstream + `structured.type` of hash tokens (`id` or `unrestricted`) and `signCharacter` are + not kept. + +Three cases carry WordPress's token expectations where the upstream package contradicts +CSS Syntax Level 3: `tests/ident/0007` (`-§`), `tests/ident/0008` (`-×`) and +`tests/fuzz/b69ece36-057f-4450-9423-a1661787bce6`. In each, a non-ASCII code point or +U+0000 (which preprocessing replaces with U+FFFD) is an ident code point, so the +hyphen or identifier before it continues as one ident token. Upstream splits them into +delim tokens. + +## Updating + +There is no conversion script. To update, take `testCorpus` from the package's +`index.cjs`, apply the transform above, set `normalized` for each token per the +specification, and update the version noted in this README. diff --git a/tests/phpunit/data/css-api/css-test-cases.json b/tests/phpunit/data/css-api/css-test-cases.json new file mode 100644 index 0000000000000..30e0dc8cdcccd --- /dev/null +++ b/tests/phpunit/data/css-api/css-test-cases.json @@ -0,0 +1,4491 @@ +{ + "tests/at-keyword/0001": { + "css": "@foo\n", + "tokens": [ + { + "type": "at-keyword-token", + "raw": "@foo", + "startIndex": 0, + "endIndex": 4, + "value": "foo" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 4, + "endIndex": 5, + "value": null + } + ] + }, + "tests/at-keyword/0002": { + "css": "@--\n", + "tokens": [ + { + "type": "at-keyword-token", + "raw": "@--", + "startIndex": 0, + "endIndex": 3, + "value": "--" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 3, + "endIndex": 4, + "value": null + } + ] + }, + "tests/at-keyword/0003": { + "css": "@-1\n", + "tokens": [ + { + "type": "delim-token", + "raw": "@", + "startIndex": 0, + "endIndex": 1, + "value": "@" + }, + { + "type": "number-token", + "raw": "-1", + "startIndex": 1, + "endIndex": 3, + "value": "-1", + "numberType": "integer" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 3, + "endIndex": 4, + "value": null + } + ] + }, + "tests/at-keyword/0004": { + "css": "@--1\n", + "tokens": [ + { + "type": "at-keyword-token", + "raw": "@--1", + "startIndex": 0, + "endIndex": 4, + "value": "--1" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 4, + "endIndex": 5, + "value": null + } + ] + }, + "tests/at-keyword/0005": { + "css": "@\\@\n", + "tokens": [ + { + "type": "at-keyword-token", + "raw": "@\\@", + "startIndex": 0, + "endIndex": 3, + "value": "@" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 3, + "endIndex": 4, + "value": null + } + ] + }, + "tests/at-keyword/0006": { + "css": "@_\n", + "tokens": [ + { + "type": "at-keyword-token", + "raw": "@_", + "startIndex": 0, + "endIndex": 2, + "value": "_" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 2, + "endIndex": 3, + "value": null + } + ] + }, + "tests/at-keyword/0007": { + "css": "@\n", + "tokens": [ + { + "type": "delim-token", + "raw": "@", + "startIndex": 0, + "endIndex": 1, + "value": "@" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 1, + "endIndex": 2, + "value": null + } + ] + }, + "tests/at-keyword/0008": { + "css": "pvA3@\\\neBnP\n", + "tokens": [ + { + "type": "ident-token", + "raw": "pvA3", + "startIndex": 0, + "endIndex": 4, + "value": "pvA3" + }, + { + "type": "delim-token", + "raw": "@", + "startIndex": 4, + "endIndex": 5, + "value": "@" + }, + { + "type": "delim-token", + "raw": "\\", + "startIndex": 5, + "endIndex": 6, + "value": "\\" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 6, + "endIndex": 7, + "value": null + }, + { + "type": "ident-token", + "raw": "eBnP", + "startIndex": 7, + "endIndex": 11, + "value": "eBnP" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 11, + "endIndex": 12, + "value": null + } + ] + }, + "tests/at-keyword/0009": { + "css": "@aa𐀀\n", + "tokens": [ + { + "type": "at-keyword-token", + "raw": "@aa𐀀", + "startIndex": 0, + "endIndex": 7, + "value": "aa𐀀" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 7, + "endIndex": 8, + "value": null + } + ] + }, + "tests/bad-string/0001": { + "css": "\"foo\n\"\n", + "tokens": [ + { + "type": "bad-string-token", + "raw": "\"foo", + "startIndex": 0, + "endIndex": 4, + "value": null + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 4, + "endIndex": 5, + "value": null + }, + { + "type": "bad-string-token", + "raw": "\"", + "startIndex": 5, + "endIndex": 6, + "value": null + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 6, + "endIndex": 7, + "value": null + } + ] + }, + "tests/bad-string/0002": { + "css": "\"foo\\\n\"\n", + "tokens": [ + { + "type": "string-token", + "raw": "\"foo\\\n\"", + "startIndex": 0, + "endIndex": 7, + "value": "foo" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 7, + "endIndex": 8, + "value": null + } + ] + }, + "tests/bad-string/0003": { + "css": "\"foo\r\n\"\n", + "tokens": [ + { + "type": "bad-string-token", + "raw": "\"foo", + "startIndex": 0, + "endIndex": 4, + "value": null + }, + { + "type": "whitespace-token", + "raw": "\r\n", + "startIndex": 4, + "endIndex": 6, + "value": null + }, + { + "type": "bad-string-token", + "raw": "\"", + "startIndex": 6, + "endIndex": 7, + "value": null + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 7, + "endIndex": 8, + "value": null + } + ] + }, + "tests/bad-string/0004": { + "css": "\"foo\\\r\n\"\n", + "tokens": [ + { + "type": "string-token", + "raw": "\"foo\\\r\n\"", + "startIndex": 0, + "endIndex": 8, + "value": "foo" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 8, + "endIndex": 9, + "value": null + } + ] + }, + "tests/bad-string/0005": { + "css": "\"aa𐀀\n", + "tokens": [ + { + "type": "bad-string-token", + "raw": "\"aa𐀀", + "startIndex": 0, + "endIndex": 7, + "value": null + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 7, + "endIndex": 8, + "value": null + } + ] + }, + "tests/bad-url/0001": { + "css": "url(\n", + "tokens": [ + { + "type": "url-token", + "raw": "url(\n", + "startIndex": 0, + "endIndex": 5, + "value": "" + } + ] + }, + "tests/bad-url/0002": { + "css": "url( a\n", + "tokens": [ + { + "type": "url-token", + "raw": "url( a\n", + "startIndex": 0, + "endIndex": 7, + "value": "a" + } + ] + }, + "tests/bad-url/0003": { + "css": "url( a a\n", + "tokens": [ + { + "type": "bad-url-token", + "raw": "url( a a\n", + "startIndex": 0, + "endIndex": 9, + "value": null + } + ] + }, + "tests/bad-url/0004": { + "css": "url( a a)\n", + "tokens": [ + { + "type": "bad-url-token", + "raw": "url( a a)", + "startIndex": 0, + "endIndex": 9, + "value": null + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 9, + "endIndex": 10, + "value": null + } + ] + }, + "tests/bad-url/0005": { + "css": "url( a a\\)\n", + "tokens": [ + { + "type": "bad-url-token", + "raw": "url( a a\\)\n", + "startIndex": 0, + "endIndex": 11, + "value": null + } + ] + }, + "tests/bad-url/0006": { + "css": "url( \\\n", + "tokens": [ + { + "type": "bad-url-token", + "raw": "url( \\\n", + "startIndex": 0, + "endIndex": 7, + "value": null + } + ] + }, + "tests/bad-url/0007": { + "css": "url(a'')\n", + "tokens": [ + { + "type": "bad-url-token", + "raw": "url(a'')", + "startIndex": 0, + "endIndex": 8, + "value": null + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 8, + "endIndex": 9, + "value": null + } + ] + }, + "tests/bad-url/0008": { + "css": "url(a\")\n", + "tokens": [ + { + "type": "bad-url-token", + "raw": "url(a\")", + "startIndex": 0, + "endIndex": 7, + "value": null + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 7, + "endIndex": 8, + "value": null + } + ] + }, + "tests/colon/0001": { + "css": ":\n", + "tokens": [ + { + "type": "colon-token", + "raw": ":", + "startIndex": 0, + "endIndex": 1, + "value": null + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 1, + "endIndex": 2, + "value": null + } + ] + }, + "tests/comma/0001": { + "css": ",\n", + "tokens": [ + { + "type": "comma-token", + "raw": ",", + "startIndex": 0, + "endIndex": 1, + "value": null + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 1, + "endIndex": 2, + "value": null + } + ] + }, + "tests/comment/0001": { + "css": "/* a comment */\n", + "tokens": [ + { + "type": "comment", + "raw": "/* a comment */", + "startIndex": 0, + "endIndex": 15, + "value": " a comment " + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 15, + "endIndex": 16, + "value": null + } + ] + }, + "tests/comment/0002": { + "css": "/* a comment ", + "tokens": [ + { + "type": "comment", + "raw": "/* a comment ", + "startIndex": 0, + "endIndex": 13, + "value": " a comment " + } + ] + }, + "tests/comment/0003": { + "css": "a/**/b\n", + "tokens": [ + { + "type": "ident-token", + "raw": "a", + "startIndex": 0, + "endIndex": 1, + "value": "a" + }, + { + "type": "comment", + "raw": "/**/", + "startIndex": 1, + "endIndex": 5, + "value": "" + }, + { + "type": "ident-token", + "raw": "b", + "startIndex": 5, + "endIndex": 6, + "value": "b" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 6, + "endIndex": 7, + "value": null + } + ] + }, + "tests/comment/0004": { + "css": "/*\\*/*/\n", + "tokens": [ + { + "type": "comment", + "raw": "/*\\*/", + "startIndex": 0, + "endIndex": 5, + "value": "\\" + }, + { + "type": "delim-token", + "raw": "*", + "startIndex": 5, + "endIndex": 6, + "value": "*" + }, + { + "type": "delim-token", + "raw": "/", + "startIndex": 6, + "endIndex": 7, + "value": "/" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 7, + "endIndex": 8, + "value": null + } + ] + }, + "tests/comment/0005": { + "css": "/* a comment *", + "tokens": [ + { + "type": "comment", + "raw": "/* a comment *", + "startIndex": 0, + "endIndex": 14, + "value": " a comment *" + } + ] + }, + "tests/comment/0006": { + "css": "/*a𐀀*/\n", + "tokens": [ + { + "type": "comment", + "raw": "/*a𐀀*/", + "startIndex": 0, + "endIndex": 9, + "value": "a𐀀" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 9, + "endIndex": 10, + "value": null + } + ] + }, + "tests/digit/0001": { + "css": "0\n1\n2\n3\n4\n5\n6\n7\n8\n9\n", + "tokens": [ + { + "type": "number-token", + "raw": "0", + "startIndex": 0, + "endIndex": 1, + "value": "0", + "numberType": "integer" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 1, + "endIndex": 2, + "value": null + }, + { + "type": "number-token", + "raw": "1", + "startIndex": 2, + "endIndex": 3, + "value": "1", + "numberType": "integer" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 3, + "endIndex": 4, + "value": null + }, + { + "type": "number-token", + "raw": "2", + "startIndex": 4, + "endIndex": 5, + "value": "2", + "numberType": "integer" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 5, + "endIndex": 6, + "value": null + }, + { + "type": "number-token", + "raw": "3", + "startIndex": 6, + "endIndex": 7, + "value": "3", + "numberType": "integer" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 7, + "endIndex": 8, + "value": null + }, + { + "type": "number-token", + "raw": "4", + "startIndex": 8, + "endIndex": 9, + "value": "4", + "numberType": "integer" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 9, + "endIndex": 10, + "value": null + }, + { + "type": "number-token", + "raw": "5", + "startIndex": 10, + "endIndex": 11, + "value": "5", + "numberType": "integer" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 11, + "endIndex": 12, + "value": null + }, + { + "type": "number-token", + "raw": "6", + "startIndex": 12, + "endIndex": 13, + "value": "6", + "numberType": "integer" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 13, + "endIndex": 14, + "value": null + }, + { + "type": "number-token", + "raw": "7", + "startIndex": 14, + "endIndex": 15, + "value": "7", + "numberType": "integer" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 15, + "endIndex": 16, + "value": null + }, + { + "type": "number-token", + "raw": "8", + "startIndex": 16, + "endIndex": 17, + "value": "8", + "numberType": "integer" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 17, + "endIndex": 18, + "value": null + }, + { + "type": "number-token", + "raw": "9", + "startIndex": 18, + "endIndex": 19, + "value": "9", + "numberType": "integer" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 19, + "endIndex": 20, + "value": null + } + ] + }, + "tests/dimension/0001": { + "css": "10px\n", + "tokens": [ + { + "type": "dimension-token", + "raw": "10px", + "startIndex": 0, + "endIndex": 4, + "value": "10", + "unit": "px", + "numberType": "integer" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 4, + "endIndex": 5, + "value": null + } + ] + }, + "tests/dimension/0002": { + "css": "10\\70 x\n", + "tokens": [ + { + "type": "dimension-token", + "raw": "10\\70 x", + "startIndex": 0, + "endIndex": 7, + "value": "10", + "unit": "px", + "numberType": "integer" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 7, + "endIndex": 8, + "value": null + } + ] + }, + "tests/dimension/0003": { + "css": "10--custom-px\n", + "tokens": [ + { + "type": "dimension-token", + "raw": "10--custom-px", + "startIndex": 0, + "endIndex": 13, + "value": "10", + "unit": "--custom-px", + "numberType": "integer" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 13, + "endIndex": 14, + "value": null + } + ] + }, + "tests/dimension/0004": { + "css": "10e2px\n", + "tokens": [ + { + "type": "dimension-token", + "raw": "10e2px", + "startIndex": 0, + "endIndex": 6, + "value": "10e2", + "unit": "px", + "numberType": "number" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 6, + "endIndex": 7, + "value": null + } + ] + }, + "tests/dimension/0005": { + "css": "10E2PX\n", + "tokens": [ + { + "type": "dimension-token", + "raw": "10E2PX", + "startIndex": 0, + "endIndex": 6, + "value": "10E2", + "unit": "PX", + "numberType": "number" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 6, + "endIndex": 7, + "value": null + } + ] + }, + "tests/dimension/0006": { + "css": "10\\0\n", + "tokens": [ + { + "type": "dimension-token", + "raw": "10\\0\n", + "startIndex": 0, + "endIndex": 5, + "value": "10", + "unit": "�", + "numberType": "integer" + } + ] + }, + "tests/dimension/0007": { + "css": "10a𐀀\n", + "tokens": [ + { + "type": "dimension-token", + "raw": "10a𐀀", + "startIndex": 0, + "endIndex": 7, + "value": "10", + "unit": "a𐀀", + "numberType": "integer" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 7, + "endIndex": 8, + "value": null + } + ] + }, + "tests/dimension/0008": { + "css": "10a\u0000", + "tokens": [ + { + "type": "dimension-token", + "raw": "10a\u0000", + "startIndex": 0, + "endIndex": 4, + "value": "10", + "unit": "a�", + "numberType": "integer" + } + ] + }, + "tests/escaped-code-point/0001": { + "css": "\\", + "tokens": [ + { + "type": "ident-token", + "raw": "\\", + "startIndex": 0, + "endIndex": 1, + "value": "�" + } + ] + }, + "tests/escaped-code-point/0002": { + "css": "\\0", + "tokens": [ + { + "type": "ident-token", + "raw": "\\0", + "startIndex": 0, + "endIndex": 2, + "value": "�" + } + ] + }, + "tests/escaped-code-point/0003": { + "css": "\\\\", + "tokens": [ + { + "type": "ident-token", + "raw": "\\\\", + "startIndex": 0, + "endIndex": 2, + "value": "\\" + } + ] + }, + "tests/escaped-code-point/0004": { + "css": "\\0a b\n", + "tokens": [ + { + "type": "ident-token", + "raw": "\\0a b", + "startIndex": 0, + "endIndex": 5, + "value": "\nb" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 5, + "endIndex": 6, + "value": null + } + ] + }, + "tests/escaped-code-point/0005": { + "css": "\\0ab \n", + "tokens": [ + { + "type": "ident-token", + "raw": "\\0ab ", + "startIndex": 0, + "endIndex": 5, + "value": "«" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 5, + "endIndex": 6, + "value": null + } + ] + }, + "tests/escaped-code-point/0006": { + "css": "\\0ab (foo)\n", + "tokens": [ + { + "type": "function-token", + "raw": "\\0ab (", + "startIndex": 0, + "endIndex": 6, + "value": "«" + }, + { + "type": "ident-token", + "raw": "foo", + "startIndex": 6, + "endIndex": 9, + "value": "foo" + }, + { + "type": ")-token", + "raw": ")", + "startIndex": 9, + "endIndex": 10, + "value": null + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 10, + "endIndex": 11, + "value": null + } + ] + }, + "tests/escaped-code-point/0007": { + "css": "\\0ab (foo)\n", + "tokens": [ + { + "type": "ident-token", + "raw": "\\0ab ", + "startIndex": 0, + "endIndex": 5, + "value": "«" + }, + { + "type": "whitespace-token", + "raw": " ", + "startIndex": 5, + "endIndex": 6, + "value": null + }, + { + "type": "(-token", + "raw": "(", + "startIndex": 6, + "endIndex": 7, + "value": null + }, + { + "type": "ident-token", + "raw": "foo", + "startIndex": 7, + "endIndex": 10, + "value": "foo" + }, + { + "type": ")-token", + "raw": ")", + "startIndex": 10, + "endIndex": 11, + "value": null + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 11, + "endIndex": 12, + "value": null + } + ] + }, + "tests/escaped-code-point/0008": { + "css": "\\0000ab\n", + "tokens": [ + { + "type": "ident-token", + "raw": "\\0000ab\n", + "startIndex": 0, + "endIndex": 8, + "value": "«" + } + ] + }, + "tests/escaped-code-point/0009": { + "css": "\\00000ab\n", + "tokens": [ + { + "type": "ident-token", + "raw": "\\00000ab", + "startIndex": 0, + "endIndex": 8, + "value": "\nb" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 8, + "endIndex": 9, + "value": null + } + ] + }, + "tests/escaped-code-point/0010": { + "css": "\\110000\n", + "tokens": [ + { + "type": "ident-token", + "raw": "\\110000\n", + "startIndex": 0, + "endIndex": 8, + "value": "�" + } + ] + }, + "tests/escaped-code-point/0011": { + "css": "\\00D800\n", + "tokens": [ + { + "type": "ident-token", + "raw": "\\00D800\n", + "startIndex": 0, + "endIndex": 8, + "value": "�" + } + ] + }, + "tests/escaped-code-point/0012": { + "css": "\\00DFFF\n", + "tokens": [ + { + "type": "ident-token", + "raw": "\\00DFFF\n", + "startIndex": 0, + "endIndex": 8, + "value": "�" + } + ] + }, + "tests/escaped-code-point/0013": { + "css": "\\\n", + "tokens": [ + { + "type": "delim-token", + "raw": "\\", + "startIndex": 0, + "endIndex": 1, + "value": "\\" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 1, + "endIndex": 2, + "value": null + } + ] + }, + "tests/escaped-code-point/0014": { + "css": "\\\u0000\n", + "tokens": [ + { + "type": "ident-token", + "raw": "\\\u0000", + "startIndex": 0, + "endIndex": 2, + "value": "�" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 2, + "endIndex": 3, + "value": null + } + ] + }, + "tests/escaped-code-point/0015": { + "css": "\\\u0000\b\n", + "tokens": [ + { + "type": "ident-token", + "raw": "\\\u0000", + "startIndex": 0, + "endIndex": 2, + "value": "�" + }, + { + "type": "delim-token", + "raw": "\b", + "startIndex": 2, + "endIndex": 3, + "value": "\b" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 3, + "endIndex": 4, + "value": null + } + ] + }, + "tests/escaped-code-point/0016": { + "css": "\"a\\12\r\nb\"", + "tokens": [ + { + "type": "string-token", + "raw": "\"a\\12\r\nb\"", + "startIndex": 0, + "endIndex": 9, + "value": "a\u0012b" + } + ] + }, + "tests/full-stop/0001": { + "css": ".\n", + "tokens": [ + { + "type": "delim-token", + "raw": ".", + "startIndex": 0, + "endIndex": 1, + "value": "." + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 1, + "endIndex": 2, + "value": null + } + ] + }, + "tests/full-stop/0002": { + "css": ".a\n", + "tokens": [ + { + "type": "delim-token", + "raw": ".", + "startIndex": 0, + "endIndex": 1, + "value": "." + }, + { + "type": "ident-token", + "raw": "a", + "startIndex": 1, + "endIndex": 2, + "value": "a" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 2, + "endIndex": 3, + "value": null + } + ] + }, + "tests/full-stop/0003": { + "css": ".1\n", + "tokens": [ + { + "type": "number-token", + "raw": ".1", + "startIndex": 0, + "endIndex": 2, + "value": ".1", + "numberType": "number" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 2, + "endIndex": 3, + "value": null + } + ] + }, + "tests/fuzz/01a166c0-ca20-43a5-9ab0-0984e4a5362b": { + "css": "4waPtwEEGH\\\u0000jV3zM6hh6w30N0PC 7m8KM0HcWGOPw28Gt(r19", + "tokens": [ + { + "type": "dimension-token", + "raw": "4waPtwEEGH\\\u0000jV3zM6hh6w30N0PC", + "startIndex": 0, + "endIndex": 28, + "value": "4", + "unit": "waPtwEEGH�jV3zM6hh6w30N0PC", + "numberType": "integer" + }, + { + "type": "whitespace-token", + "raw": " ", + "startIndex": 28, + "endIndex": 29, + "value": null + }, + { + "type": "dimension-token", + "raw": "7m8KM0HcWGOPw28Gt", + "startIndex": 29, + "endIndex": 46, + "value": "7", + "unit": "m8KM0HcWGOPw28Gt", + "numberType": "integer" + }, + { + "type": "(-token", + "raw": "(", + "startIndex": 46, + "endIndex": 47, + "value": null + }, + { + "type": "ident-token", + "raw": "r19", + "startIndex": 47, + "endIndex": 50, + "value": "r19" + } + ] + }, + "tests/fuzz/2abe9406-c063-4e9a-85ac-b13660671553": { + "css": "ak]P0A}808G\"lQh{R5M!QyOWE}oC2{2K TIa9}zb2oXWREY]0aj5J\\\r\nBJ5CO-16W5H7noF 19䀹41H3e8Z9%tg[O5AHEY24xh'9\"\"c34Q\"iiC0e45Da5f\"F5X3\"o(", + "tokens": [ + { + "type": "ident-token", + "raw": "ak", + "startIndex": 0, + "endIndex": 2, + "value": "ak" + }, + { + "type": "]-token", + "raw": "]", + "startIndex": 2, + "endIndex": 3, + "value": null + }, + { + "type": "ident-token", + "raw": "P0A", + "startIndex": 3, + "endIndex": 6, + "value": "P0A" + }, + { + "type": "}-token", + "raw": "}", + "startIndex": 6, + "endIndex": 7, + "value": null + }, + { + "type": "dimension-token", + "raw": "808G", + "startIndex": 7, + "endIndex": 11, + "value": "808", + "unit": "G", + "numberType": "integer" + }, + { + "type": "string-token", + "raw": "\"lQh{R5M!QyOWE}oC2{2K TIa9}zb2oXWREY]0aj5J\\\r\nBJ5CO-16W5H7noF 19䀹41H3e8Z9%tg[O5AHEY24xh'9\"", + "startIndex": 11, + "endIndex": 102, + "value": "lQh{R5M!QyOWE}oC2{2K TIa9}zb2oXWREY]0aj5JBJ5CO-16W5H7noF 19䀹41H3e8Z9%tg[O5AHEY24xh'9" + }, + { + "type": "string-token", + "raw": "\"c34Q\"", + "startIndex": 102, + "endIndex": 108, + "value": "c34Q" + }, + { + "type": "ident-token", + "raw": "iiC0e45Da5f", + "startIndex": 108, + "endIndex": 119, + "value": "iiC0e45Da5f" + }, + { + "type": "string-token", + "raw": "\"F5X3\"", + "startIndex": 119, + "endIndex": 125, + "value": "F5X3" + }, + { + "type": "function-token", + "raw": "o(", + "startIndex": 125, + "endIndex": 127, + "value": "o" + } + ] + }, + "tests/fuzz/4e630a47-507b-4b79-b00f-57f7dc1cc79d": { + "css": "\u000e7rSD6I5L1lglVRlL2X7BbEk\\3HCd\r94 \\\u0000skoW25d4%l64UUskN\"pHun\"!", + "tokens": [ + { + "type": "delim-token", + "raw": "\u000e", + "startIndex": 0, + "endIndex": 1, + "value": "\u000e" + }, + { + "type": "dimension-token", + "raw": "7rSD6I5L1lglVRlL2X7BbEk\\3HCd", + "startIndex": 1, + "endIndex": 29, + "value": "7", + "unit": "rSD6I5L1lglVRlL2X7BbEk\u0003HCd", + "numberType": "integer" + }, + { + "type": "whitespace-token", + "raw": "\r", + "startIndex": 29, + "endIndex": 30, + "value": null + }, + { + "type": "number-token", + "raw": "94", + "startIndex": 30, + "endIndex": 32, + "value": "94", + "numberType": "integer" + }, + { + "type": "whitespace-token", + "raw": " ", + "startIndex": 32, + "endIndex": 33, + "value": null + }, + { + "type": "ident-token", + "raw": "\\\u0000skoW25d4", + "startIndex": 33, + "endIndex": 43, + "value": "�skoW25d4" + }, + { + "type": "delim-token", + "raw": "%", + "startIndex": 43, + "endIndex": 44, + "value": "%" + }, + { + "type": "ident-token", + "raw": "l64UUskN", + "startIndex": 44, + "endIndex": 52, + "value": "l64UUskN" + }, + { + "type": "string-token", + "raw": "\"pHun\"", + "startIndex": 52, + "endIndex": 58, + "value": "pHun" + }, + { + "type": "delim-token", + "raw": "!", + "startIndex": 58, + "endIndex": 59, + "value": "!" + } + ] + }, + "tests/fuzz/4f865903-e4dd-4a0b-83ed-e630cfa9dcca": { + "css": "gzO0{(p{DzQ7\u0000(a1;r1iN7w)", + "tokens": [ + { + "type": "ident-token", + "raw": "gzO0", + "startIndex": 0, + "endIndex": 4, + "value": "gzO0" + }, + { + "type": "{-token", + "raw": "{", + "startIndex": 4, + "endIndex": 5, + "value": null + }, + { + "type": "(-token", + "raw": "(", + "startIndex": 5, + "endIndex": 6, + "value": null + }, + { + "type": "ident-token", + "raw": "p", + "startIndex": 6, + "endIndex": 7, + "value": "p" + }, + { + "type": "{-token", + "raw": "{", + "startIndex": 7, + "endIndex": 8, + "value": null + }, + { + "type": "function-token", + "raw": "DzQ7\u0000(", + "startIndex": 8, + "endIndex": 14, + "value": "DzQ7�" + }, + { + "type": "ident-token", + "raw": "a1", + "startIndex": 14, + "endIndex": 16, + "value": "a1" + }, + { + "type": "semicolon-token", + "raw": ";", + "startIndex": 16, + "endIndex": 17, + "value": null + }, + { + "type": "ident-token", + "raw": "r1iN7w", + "startIndex": 17, + "endIndex": 23, + "value": "r1iN7w" + }, + { + "type": ")-token", + "raw": ")", + "startIndex": 23, + "endIndex": 24, + "value": null + } + ] + }, + "tests/fuzz/5181013c-60ab-483b-9c06-fb32c7e1e7e8": { + "css": "565'E{z\u0000U\u001fEG2}2Verb>nj3TVk3mu7wX1J\b.H\u000bi1Ga8f5 dserqydJ3\"xj398xy.W\" uHQbv7Bw1NtF;N3PwNY7Vx00BF o\"4CXzvP\"{594 6r}8QQKNQw135i1\\\r\nrey\thg7[5%rBK8RUC64Lu␌17O{E\\90873u}1O3vx4gHTC55Q9i4\"V3Vx4\"7r(34L]F\"ns2pPf\"V7b)EOBGH8rdC7\"\u000eVJ4OQ[ 9jtoMdINgS7o�206vo72kTcKkZR9wl30G'vK\ndhCEs3tValX ", + "tokens": [ + { + "type": "number-token", + "raw": "565", + "startIndex": 0, + "endIndex": 3, + "value": "565", + "numberType": "integer" + }, + { + "type": "string-token", + "raw": "'E{z\u0000U\u001fEG2}2Verb>nj3TVk3mu7wX1J\b.H\u000bi1Ga8f5 dserqydJ3\"xj398xy.W\" uHQbv7Bw1NtF;N3PwNY7Vx00BF o\"4CXzvP\"{594 6r}8QQKNQw135i1\\\r\nrey\thg7[5%rBK8RUC64Lu␌17O{E\\90873u}1O3vx4gHTC55Q9i4\"V3Vx4\"7r(34L]F\"ns2pPf\"V7b)EOBGH8rdC7\"\u000eVJ4OQ[ 9jtoMdINgS7o�206vo72kTcKkZR9wl30G'", + "startIndex": 3, + "endIndex": 263, + "value": "E{z�U\u001fEG2}2Verb>nj3TVk3mu7wX1J\b.H\u000bi1Ga8f5 dserqydJ3\"xj398xy.W\" uHQbv7Bw1NtF;N3PwNY7Vx00BF o\"4CXzvP\"{594 6r}8QQKNQw135i1rey\thg7[5%rBK8RUC64Lu␌17O{E򐡳u}1O3vx4gHTC55Q9i4\"V3Vx4\"7r(34L]F\"ns2pPf\"V7b)EOBGH8rdC7\"\u000eVJ4OQ[ 9jtoMdINgS7o�206vo72kTcKkZR9wl30G" + }, + { + "type": "ident-token", + "raw": "vK", + "startIndex": 263, + "endIndex": 265, + "value": "vK" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 265, + "endIndex": 266, + "value": null + }, + { + "type": "ident-token", + "raw": "dhCEs3tValX", + "startIndex": 266, + "endIndex": 277, + "value": "dhCEs3tValX" + }, + { + "type": "whitespace-token", + "raw": " ", + "startIndex": 277, + "endIndex": 278, + "value": null + } + ] + }, + "tests/fuzz/6d07fc79-586f-4efa-a0a2-37d4dd3beb09": { + "css": "FWUNqr7uv8300nz\b,8lU0j6B186kh \u00009 \u000eGZafxf2GIhL9%", + "tokens": [ + { + "type": "ident-token", + "raw": "FWUNqr7uv8300nz", + "startIndex": 0, + "endIndex": 15, + "value": "FWUNqr7uv8300nz" + }, + { + "type": "delim-token", + "raw": "\b", + "startIndex": 15, + "endIndex": 16, + "value": "\b" + }, + { + "type": "comma-token", + "raw": ",", + "startIndex": 16, + "endIndex": 17, + "value": null + }, + { + "type": "dimension-token", + "raw": "8lU0j6B186kh", + "startIndex": 17, + "endIndex": 29, + "value": "8", + "unit": "lU0j6B186kh", + "numberType": "integer" + }, + { + "type": "whitespace-token", + "raw": " ", + "startIndex": 29, + "endIndex": 30, + "value": null + }, + { + "type": "ident-token", + "raw": "\u00009", + "startIndex": 30, + "endIndex": 32, + "value": "�9" + }, + { + "type": "whitespace-token", + "raw": " ", + "startIndex": 32, + "endIndex": 33, + "value": null + }, + { + "type": "delim-token", + "raw": "\u000e", + "startIndex": 33, + "endIndex": 34, + "value": "\u000e" + }, + { + "type": "ident-token", + "raw": "GZafxf2GIhL9", + "startIndex": 34, + "endIndex": 46, + "value": "GZafxf2GIhL9" + }, + { + "type": "delim-token", + "raw": "%", + "startIndex": 46, + "endIndex": 47, + "value": "%" + } + ] + }, + "tests/fuzz/7f49c8fc-8292-4a3e-828b-b5d028a80d5f": { + "css": "FZ 0B120h5QUbNbmTD2K8mAD傿i+Yv9V0KS14Ng18ag'\\\r\n{Xu)k2a76}y4\\6fb9ONI\\", + "tokens": [ + { + "type": "delim-token", + "raw": ">", + "startIndex": 0, + "endIndex": 1, + "value": ">" + }, + { + "type": "ident-token", + "raw": "u", + "startIndex": 1, + "endIndex": 2, + "value": "u" + }, + { + "type": ")-token", + "raw": ")", + "startIndex": 2, + "endIndex": 3, + "value": null + }, + { + "type": "ident-token", + "raw": "k2a76", + "startIndex": 3, + "endIndex": 8, + "value": "k2a76" + }, + { + "type": "}-token", + "raw": "}", + "startIndex": 8, + "endIndex": 9, + "value": null + }, + { + "type": "ident-token", + "raw": "y4\\6fb9ONI\\", + "startIndex": 9, + "endIndex": 20, + "value": "y4澹ONI�" + } + ] + }, + "tests/hash/0001": { + "css": "#1\n", + "tokens": [ + { + "type": "hash-token", + "raw": "#1", + "startIndex": 0, + "endIndex": 2, + "value": "1" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 2, + "endIndex": 3, + "value": null + } + ] + }, + "tests/hash/0002": { + "css": "#-2\n", + "tokens": [ + { + "type": "hash-token", + "raw": "#-2", + "startIndex": 0, + "endIndex": 3, + "value": "-2" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 3, + "endIndex": 4, + "value": null + } + ] + }, + "tests/hash/0003": { + "css": "#--3\n", + "tokens": [ + { + "type": "hash-token", + "raw": "#--3", + "startIndex": 0, + "endIndex": 4, + "value": "--3" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 4, + "endIndex": 5, + "value": null + } + ] + }, + "tests/hash/0004": { + "css": "#---4\n", + "tokens": [ + { + "type": "hash-token", + "raw": "#---4", + "startIndex": 0, + "endIndex": 5, + "value": "---4" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 5, + "endIndex": 6, + "value": null + } + ] + }, + "tests/hash/0005": { + "css": "#a\n", + "tokens": [ + { + "type": "hash-token", + "raw": "#a", + "startIndex": 0, + "endIndex": 2, + "value": "a" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 2, + "endIndex": 3, + "value": null + } + ] + }, + "tests/hash/0006": { + "css": "#-b\n", + "tokens": [ + { + "type": "hash-token", + "raw": "#-b", + "startIndex": 0, + "endIndex": 3, + "value": "-b" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 3, + "endIndex": 4, + "value": null + } + ] + }, + "tests/hash/0007": { + "css": "#--c\n", + "tokens": [ + { + "type": "hash-token", + "raw": "#--c", + "startIndex": 0, + "endIndex": 4, + "value": "--c" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 4, + "endIndex": 5, + "value": null + } + ] + }, + "tests/hash/0008": { + "css": "#---d\n", + "tokens": [ + { + "type": "hash-token", + "raw": "#---d", + "startIndex": 0, + "endIndex": 5, + "value": "---d" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 5, + "endIndex": 6, + "value": null + } + ] + }, + "tests/hash/0009": { + "css": "#_\n", + "tokens": [ + { + "type": "hash-token", + "raw": "#_", + "startIndex": 0, + "endIndex": 2, + "value": "_" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 2, + "endIndex": 3, + "value": null + } + ] + }, + "tests/hash/0010": { + "css": "#_1\n", + "tokens": [ + { + "type": "hash-token", + "raw": "#_1", + "startIndex": 0, + "endIndex": 3, + "value": "_1" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 3, + "endIndex": 4, + "value": null + } + ] + }, + "tests/hash/0011": { + "css": "#-\n", + "tokens": [ + { + "type": "hash-token", + "raw": "#-", + "startIndex": 0, + "endIndex": 2, + "value": "-" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 2, + "endIndex": 3, + "value": null + } + ] + }, + "tests/hash/0012": { + "css": "#+\n", + "tokens": [ + { + "type": "delim-token", + "raw": "#", + "startIndex": 0, + "endIndex": 1, + "value": "#" + }, + { + "type": "delim-token", + "raw": "+", + "startIndex": 1, + "endIndex": 2, + "value": "+" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 2, + "endIndex": 3, + "value": null + } + ] + }, + "tests/hash/0013": { + "css": "##\n", + "tokens": [ + { + "type": "delim-token", + "raw": "#", + "startIndex": 0, + "endIndex": 1, + "value": "#" + }, + { + "type": "delim-token", + "raw": "#", + "startIndex": 1, + "endIndex": 2, + "value": "#" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 2, + "endIndex": 3, + "value": null + } + ] + }, + "tests/hash/0014": { + "css": "#", + "tokens": [ + { + "type": "delim-token", + "raw": "#", + "startIndex": 0, + "endIndex": 1, + "value": "#" + } + ] + }, + "tests/hash/0015": { + "css": "#aa𐀀\n", + "tokens": [ + { + "type": "hash-token", + "raw": "#aa𐀀", + "startIndex": 0, + "endIndex": 7, + "value": "aa𐀀" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 7, + "endIndex": 8, + "value": null + } + ] + }, + "tests/hyphen-minus/0001": { + "css": "-\n", + "tokens": [ + { + "type": "delim-token", + "raw": "-", + "startIndex": 0, + "endIndex": 1, + "value": "-" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 1, + "endIndex": 2, + "value": null + } + ] + }, + "tests/hyphen-minus/0002": { + "css": "-1\n", + "tokens": [ + { + "type": "number-token", + "raw": "-1", + "startIndex": 0, + "endIndex": 2, + "value": "-1", + "numberType": "integer" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 2, + "endIndex": 3, + "value": null + } + ] + }, + "tests/hyphen-minus/0003": { + "css": "-.1\n", + "tokens": [ + { + "type": "number-token", + "raw": "-.1", + "startIndex": 0, + "endIndex": 3, + "value": "-.1", + "numberType": "number" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 3, + "endIndex": 4, + "value": null + } + ] + }, + "tests/hyphen-minus/0004": { + "css": "--1\n", + "tokens": [ + { + "type": "ident-token", + "raw": "--1", + "startIndex": 0, + "endIndex": 3, + "value": "--1" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 3, + "endIndex": 4, + "value": null + } + ] + }, + "tests/hyphen-minus/0005": { + "css": "-0\n", + "tokens": [ + { + "type": "number-token", + "raw": "-0", + "startIndex": 0, + "endIndex": 2, + "value": "-0", + "numberType": "integer" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 2, + "endIndex": 3, + "value": null + } + ] + }, + "tests/hyphen-minus/0006": { + "css": "-->\n", + "tokens": [ + { + "type": "CDC-token", + "raw": "-->", + "startIndex": 0, + "endIndex": 3, + "value": null + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 3, + "endIndex": 4, + "value": null + } + ] + }, + "tests/ident/0001": { + "css": "foo\n", + "tokens": [ + { + "type": "ident-token", + "raw": "foo", + "startIndex": 0, + "endIndex": 3, + "value": "foo" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 3, + "endIndex": 4, + "value": null + } + ] + }, + "tests/ident/0002": { + "css": "--\n", + "tokens": [ + { + "type": "ident-token", + "raw": "--", + "startIndex": 0, + "endIndex": 2, + "value": "--" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 2, + "endIndex": 3, + "value": null + } + ] + }, + "tests/ident/0003": { + "css": "--0\n", + "tokens": [ + { + "type": "ident-token", + "raw": "--0", + "startIndex": 0, + "endIndex": 3, + "value": "--0" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 3, + "endIndex": 4, + "value": null + } + ] + }, + "tests/ident/0004": { + "css": "-\\\n", + "tokens": [ + { + "type": "delim-token", + "raw": "-", + "startIndex": 0, + "endIndex": 1, + "value": "-" + }, + { + "type": "delim-token", + "raw": "\\", + "startIndex": 1, + "endIndex": 2, + "value": "\\" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 2, + "endIndex": 3, + "value": null + } + ] + }, + "tests/ident/0005": { + "css": "-\\ \n", + "tokens": [ + { + "type": "ident-token", + "raw": "-\\ ", + "startIndex": 0, + "endIndex": 3, + "value": "- " + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 3, + "endIndex": 4, + "value": null + } + ] + }, + "tests/ident/0006": { + "css": "--💅\n", + "tokens": [ + { + "type": "ident-token", + "raw": "--💅", + "startIndex": 0, + "endIndex": 6, + "value": "--💅" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 6, + "endIndex": 7, + "value": null + } + ] + }, + "tests/ident/0007": { + "css": "-§\n", + "tokens": [ + { + "type": "ident-token", + "raw": "-§", + "startIndex": 0, + "endIndex": 3, + "value": "-§" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 3, + "endIndex": 4, + "value": null + } + ] + }, + "tests/ident/0008": { + "css": "-×\n", + "tokens": [ + { + "type": "ident-token", + "raw": "-×", + "startIndex": 0, + "endIndex": 3, + "value": "-×" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 3, + "endIndex": 4, + "value": null + } + ] + }, + "tests/ident/0009": { + "css": "--a𐀀\n", + "tokens": [ + { + "type": "ident-token", + "raw": "--a𐀀", + "startIndex": 0, + "endIndex": 7, + "value": "--a𐀀" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 7, + "endIndex": 8, + "value": null + } + ] + }, + "tests/ident-like/0001": { + "css": "url(foo)\n", + "tokens": [ + { + "type": "url-token", + "raw": "url(foo)", + "startIndex": 0, + "endIndex": 8, + "value": "foo" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 8, + "endIndex": 9, + "value": null + } + ] + }, + "tests/ident-like/0002": { + "css": "\\75 Rl(foo)\n", + "tokens": [ + { + "type": "url-token", + "raw": "\\75 Rl(foo)", + "startIndex": 0, + "endIndex": 11, + "value": "foo" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 11, + "endIndex": 12, + "value": null + } + ] + }, + "tests/ident-like/0003": { + "css": "uR\\6c (foo)\n", + "tokens": [ + { + "type": "url-token", + "raw": "uR\\6c (foo)", + "startIndex": 0, + "endIndex": 11, + "value": "foo" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 11, + "endIndex": 12, + "value": null + } + ] + }, + "tests/ident-like/0004": { + "css": "url('foo')\n", + "tokens": [ + { + "type": "function-token", + "raw": "url(", + "startIndex": 0, + "endIndex": 4, + "value": "url" + }, + { + "type": "string-token", + "raw": "'foo'", + "startIndex": 4, + "endIndex": 9, + "value": "foo" + }, + { + "type": ")-token", + "raw": ")", + "startIndex": 9, + "endIndex": 10, + "value": null + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 10, + "endIndex": 11, + "value": null + } + ] + }, + "tests/ident-like/0005": { + "css": "url( 'foo')\n", + "tokens": [ + { + "type": "function-token", + "raw": "url(", + "startIndex": 0, + "endIndex": 4, + "value": "url" + }, + { + "type": "whitespace-token", + "raw": " ", + "startIndex": 4, + "endIndex": 5, + "value": null + }, + { + "type": "string-token", + "raw": "'foo'", + "startIndex": 5, + "endIndex": 10, + "value": "foo" + }, + { + "type": ")-token", + "raw": ")", + "startIndex": 10, + "endIndex": 11, + "value": null + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 11, + "endIndex": 12, + "value": null + } + ] + }, + "tests/ident-like/0006": { + "css": "url( 'foo')\n", + "tokens": [ + { + "type": "function-token", + "raw": "url(", + "startIndex": 0, + "endIndex": 4, + "value": "url" + }, + { + "type": "whitespace-token", + "raw": " ", + "startIndex": 4, + "endIndex": 6, + "value": null + }, + { + "type": "string-token", + "raw": "'foo'", + "startIndex": 6, + "endIndex": 11, + "value": "foo" + }, + { + "type": ")-token", + "raw": ")", + "startIndex": 11, + "endIndex": 12, + "value": null + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 12, + "endIndex": 13, + "value": null + } + ] + }, + "tests/ident-like/0007": { + "css": "url( 'foo')\n", + "tokens": [ + { + "type": "function-token", + "raw": "url(", + "startIndex": 0, + "endIndex": 4, + "value": "url" + }, + { + "type": "whitespace-token", + "raw": " ", + "startIndex": 4, + "endIndex": 7, + "value": null + }, + { + "type": "string-token", + "raw": "'foo'", + "startIndex": 7, + "endIndex": 12, + "value": "foo" + }, + { + "type": ")-token", + "raw": ")", + "startIndex": 12, + "endIndex": 13, + "value": null + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 13, + "endIndex": 14, + "value": null + } + ] + }, + "tests/ident-like/0008": { + "css": "not-url( 'foo')\n", + "tokens": [ + { + "type": "function-token", + "raw": "not-url(", + "startIndex": 0, + "endIndex": 8, + "value": "not-url" + }, + { + "type": "whitespace-token", + "raw": " ", + "startIndex": 8, + "endIndex": 11, + "value": null + }, + { + "type": "string-token", + "raw": "'foo'", + "startIndex": 11, + "endIndex": 16, + "value": "foo" + }, + { + "type": ")-token", + "raw": ")", + "startIndex": 16, + "endIndex": 17, + "value": null + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 17, + "endIndex": 18, + "value": null + } + ] + }, + "tests/ident-like/0009": { + "css": "url( foo)\n", + "tokens": [ + { + "type": "url-token", + "raw": "url( foo)", + "startIndex": 0, + "endIndex": 11, + "value": "foo" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 11, + "endIndex": 12, + "value": null + } + ] + }, + "tests/left-curly-bracket/0001": { + "css": "{\n", + "tokens": [ + { + "type": "{-token", + "raw": "{", + "startIndex": 0, + "endIndex": 1, + "value": null + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 1, + "endIndex": 2, + "value": null + } + ] + }, + "tests/left-parenthesis/0001": { + "css": "(\n", + "tokens": [ + { + "type": "(-token", + "raw": "(", + "startIndex": 0, + "endIndex": 1, + "value": null + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 1, + "endIndex": 2, + "value": null + } + ] + }, + "tests/left-square-bracket/0001": { + "css": "[\n", + "tokens": [ + { + "type": "[-token", + "raw": "[", + "startIndex": 0, + "endIndex": 1, + "value": null + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 1, + "endIndex": 2, + "value": null + } + ] + }, + "tests/less-than/0001": { + "css": "<\n", + "tokens": [ + { + "type": "delim-token", + "raw": "<", + "startIndex": 0, + "endIndex": 1, + "value": "<" + }, + { + "type": "whitespace-token", + "raw": "\n", + "startIndex": 1, + "endIndex": 2, + "value": null + } + ] + }, + "tests/less-than/0002": { + "css": "', '-->' ), + '(' => array( '()', '()' ), + '*' => array( '*', '*' ), + '%' => array( '%', '%' ), + ); + + $marked = array( + 'ident' => array( 'ident', 'function', 'url', 'bad url', '-', 'number', 'percentage', 'dimension', 'CDC', '(' ), + 'at-keyword' => array( 'ident', 'function', 'url', 'bad url', '-', 'number', 'percentage', 'dimension', 'CDC' ), + 'hash' => array( 'ident', 'function', 'url', 'bad url', '-', 'number', 'percentage', 'dimension', 'CDC' ), + 'dimension' => array( 'ident', 'function', 'url', 'bad url', '-', 'number', 'percentage', 'dimension', 'CDC' ), + '#' => array( 'ident', 'function', 'url', 'bad url', '-', 'number', 'percentage', 'dimension', 'CDC' ), + '-' => array( 'ident', 'function', 'url', 'bad url', '-', 'number', 'percentage', 'dimension', 'CDC' ), + 'number' => array( 'ident', 'function', 'url', 'bad url', 'number', 'percentage', 'dimension', 'CDC', '%' ), + '@' => array( 'ident', 'function', 'url', 'bad url', '-', 'CDC' ), + '.' => array( 'number', 'percentage', 'dimension' ), + '+' => array( 'number', 'percentage', 'dimension' ), + '/' => array( '*' ), + ); + + foreach ( $marked as $row => $row_columns ) { + foreach ( $row_columns as $column ) { + list( $column_css, $column_serialized ) = $columns[ $column ]; + yield "{$row} followed by {$column}" => array( + "{$rows[ $row ]}/**/{$column_css}", + "{$rows[ $row ]}/**/{$column_serialized}", + ); + } + } + } + + /** + * Tests that serialize() writes no comment between the tokens of an unmarked pair. + * + * @ticket 65738 + * @dataProvider data_unmarked_pairs + * + * @covers ::serialize + */ + public function test_unmarked_pair_is_not_separated( string $css, string $expected ): void { + $this->assertSame( $expected, WP_CSS_Token_Serializer::serialize( self::tokenize( $css ) ) ); + $this->assertSame( self::significant_tokens( $css ), self::significant_tokens( $expected ), 'The output must read as the same tokens.' ); + } + + /** + * Data provider for test_unmarked_pair_is_not_separated(). + * + * @return array[] + */ + public static function data_unmarked_pairs(): array { + return array( + 'ident followed by *' => array( 'a/**/*', 'a*' ), + 'ident followed by %' => array( 'a/**/%', 'a%' ), + 'ident followed by colon' => array( 'a/**/:', 'a:' ), + 'at-keyword followed by (' => array( '@a/**/()', '@a()' ), + 'hash followed by *' => array( '#a/**/*', '#a*' ), + 'dimension followed by %' => array( '1px/**/%', '1px%' ), + '# followed by %' => array( '#/**/%', '#%' ), + '- followed by *' => array( '-/**/*', '-*' ), + 'number followed by -' => array( '1/**/-', '1-' ), + 'number followed by (' => array( '1/**/()', '1()' ), + '@ followed by number' => array( '@/**/1', '@1' ), + '. followed by ident' => array( './**/a', '.a' ), + '+ followed by ident' => array( '+/**/a', '+a' ), + '/ followed by ident' => array( '//**/a', '/a' ), + 'string followed by ident' => array( '"a"/**/b', '"a"b' ), + 'url followed by ident' => array( 'url(a)/**/b', 'url(a)b' ), + 'function followed by ident' => array( 'a(/**/b)', 'a(b)' ), + ') followed by ident' => array( '(a)/**/b', '(a)b' ), + 'CDC followed by ident' => array( '-->/**/a', '-->a' ), + 'percentage followed by ident' => array( '1%/**/a', '1%a' ), + ); + } + + /** + * Tests that serialize() drops comments and writes each whitespace run as one space. + * + * @ticket 65738 + * @dataProvider data_whitespace_and_comments + * + * @covers ::serialize + */ + public function test_whitespace_and_comments( string $css, string $expected ): void { + $this->assertSame( $expected, WP_CSS_Token_Serializer::serialize( self::tokenize( $css ) ) ); + } + + /** + * Data provider for test_whitespace_and_comments(). + * + * @return array[] + */ + public static function data_whitespace_and_comments(): array { + return array( + 'empty input' => array( '', '' ), + 'one space' => array( 'a b', 'a b' ), + 'mixed whitespace run' => array( "a \t\r\n\f b", 'a b' ), + 'run broken by a comment' => array( 'a /* x */ b', 'a b' ), + 'whitespace then a comment' => array( 'a /**/b', 'a b' ), + 'a comment then whitespace' => array( 'a/**/ b', 'a b' ), + 'leading whitespace' => array( "\n a", ' a' ), + 'trailing whitespace' => array( "a \n", 'a ' ), + 'only whitespace' => array( "\t\n", ' ' ), + 'only a comment' => array( '/* x */', '' ), + 'comment in unmarked pair' => array( 'a/* x */,b', 'a,b' ), + 'unterminated comment' => array( 'a /* x', 'a ' ), + ); + } + + /** + * Tests how serialize() writes each token type. + * + * @ticket 65738 + * @dataProvider data_token_serialization + * + * @covers ::serialize + */ + public function test_token_serialization( string $css, string $expected ): void { + $this->assertSame( $expected, WP_CSS_Token_Serializer::serialize( self::tokenize( $css ) ) ); + $this->assertSame( self::significant_tokens( $css ), self::significant_tokens( $expected ), 'The output must read as the same tokens.' ); + } + + /** + * Data provider for test_token_serialization(). + * + * @return array[] + */ + public static function data_token_serialization(): array { + return array( + 'ident escapes decoded' => array( '\\66 oo', 'foo' ), + 'ident escapes added' => array( 'a\\:b', 'a\\3a b' ), + 'function' => array( '\\66 oo(1)', 'foo(1)' ), + 'at-keyword' => array( '@\\66 oo', '@foo' ), + 'hash id' => array( '#a\\:', '#a\\3a ' ), + 'hash id escaped digit' => array( '#\\31 a', '#\\31 a' ), + 'hash unrestricted' => array( '#1a', '#1a' ), + 'hash unrestricted escape' => array( '#1\\:', '#1\\3a ' ), + 'hash unrestricted hyphen' => array( '#-', '#-' ), + 'hash id non-ident code point' => array( '#a\\d7 b', '#a\\d7 b' ), + 'hash unrestricted non-ident code point' => array( '#1\\d7 b', '#1\\d7 b' ), + 'dimension unit non-ident code point' => array( '1\\a0 b', '1\\a0 b' ), + 'string' => array( "'a\"b'", '"a\\22 b"' ), + 'string unterminated' => array( '"abc', '"abc"' ), + 'bad string' => array( "'a\nb", "\"\n b" ), + 'url' => array( 'url( a )', 'url(a)' ), + 'url empty' => array( 'url()', 'url()' ), + 'url escaped parenthesis' => array( 'url(a\\29 b)', 'url(a\\29 b)' ), + 'url escaped space' => array( 'url(a\\20 b)', 'url(a\\20 b)' ), + 'url escaped quote' => array( 'url(a\\"b)', 'url(a\\22 b)' ), + 'url escaped backslash' => array( 'url(a\\\\b)', 'url(a\\5c b)' ), + 'url escaped control' => array( 'url(a\\1 b)', 'url(a\\1 b)' ), + 'url escaped newline' => array( 'url(a\\A b)', 'url(a\\a b)' ), + 'url ampersand' => array( 'url(a&b)', 'url(a\\26 b)' ), + 'url NUL' => array( "url(a\x00b)", "url(a\u{FFFD}b)" ), + 'url unterminated' => array( 'url(a', 'url(a)' ), + 'url function with a string' => array( "url( 'a')", 'url( "a")' ), + 'bad url' => array( 'url(a b)', 'url(()' ), + 'number' => array( '+.5e-3', '+.5e-3' ), + 'percentage' => array( '-1.5%', '-1.5%' ), + 'dimension' => array( '1.5e3PX', '1.5e3PX' ), + 'dimension unit escape' => array( '1\\2D ', '1\\2d ' ), + 'dimension unit read as exponent' => array( '1\\65 3', '1\\65 3' ), + 'dimension unit read as signed exponent' => array( '1\\45 -3', '1\\45 -3' ), + 'dimension unit e' => array( '1e', '1e' ), + 'backslash delim keeps newline' => array( "a\\\nb", "a\\\n b" ), + 'delim' => array( '!', '!' ), + 'DEL delim' => array( "a\x7Fb", "a\x7Fb" ), + 'CDO and CDC' => array( '', '' ), + 'punctuation' => array( 'a:b;c,d', 'a:b;c,d' ), + ); + } + + /** + * Tests that serialize() does not spell a CDO or CDC token from other tokens. + * + * The table in CSS Syntax Level 3 §9 compares token pairs by type. It does + * not cover `<`, `!` and a token that starts with `--`, which spell ``. + * + * @ticket 65738 + * @dataProvider data_cdo_and_cdc_sequences + * + * @covers ::serialize + */ + public function test_cdo_and_cdc_are_not_spelled( string $css, string $expected ): void { + $this->assertSame( $expected, WP_CSS_Token_Serializer::serialize( self::tokenize( $css ) ) ); + $this->assertSame( self::significant_tokens( $css ), self::significant_tokens( $expected ), 'The output must read as the same tokens.' ); + } + + /** + * Data provider for test_cdo_and_cdc_are_not_spelled(). + * + * @return array[] + */ + public static function data_cdo_and_cdc_sequences(): array { + return array( + '< ! ident' => array( ' array( ' array( ' array( '', '' ), + 'ident -- >' => array( '--/**/>', '--/**/>' ), + '< space ! ident' => array( '< !/**/--x', '< !--x' ), + '! ident without <' => array( 'a!/**/--x', 'a!--x' ), + '< ! ident without --' => array( '' => array( 'a--/**/>', 'a-->' ), + 'ident -- space >' => array( '-- >', '-- >' ), + ); + } + + /** + * Tests that serialize() closes blocks still open at the end, in reverse order. + * + * @ticket 65738 + * @dataProvider data_open_blocks + * + * @covers ::serialize + */ + public function test_open_blocks_are_closed( string $css, string $expected ): void { + $this->assertSame( $expected, WP_CSS_Token_Serializer::serialize( self::tokenize( $css ) ) ); + } + + /** + * Data provider for test_open_blocks_are_closed(). + * + * @return array[] + */ + public static function data_open_blocks(): array { + return array( + 'function' => array( 'calc(1px', 'calc(1px)' ), + 'nested blocks' => array( 'a(b[c{d', 'a(b[c{d}])' ), + 'closed block' => array( '(a)', '(a)' ), + 'mismatched closer' => array( '(]', '(])' ), + 'closer without a block' => array( ')', ')' ), + 'brace' => array( '{', '{}' ), + 'function with a string' => array( 'url("a"', 'url("a")' ), + 'inner block closed' => array( 'a((b)', 'a((b))' ), + ); + } + + /** + * Tests that the hash type flag is preserved. + * + * @ticket 65738 + * @dataProvider data_hash_flag_round_trips + * + * @covers ::serialize + */ + public function test_hash_flag_round_trips( string $css, string $expected_flag ): void { + $tokens = self::tokenize( $css ); + $serialized = WP_CSS_Token_Serializer::serialize( $tokens ); + $reparsed = self::tokenize( $serialized ); + + $this->assertSame( $expected_flag, $tokens[0]->get_flag(), 'The input has the wrong type flag.' ); + $this->assertCount( 1, $reparsed, 'The hash must serialize as one token.' ); + $this->assertSame( $tokens[0]->get_value(), $reparsed[0]->get_value(), 'The value changed.' ); + $this->assertSame( $expected_flag, $reparsed[0]->get_flag(), 'The type flag changed.' ); + } + + /** + * Data provider for test_hash_flag_round_trips(). + * + * @return array[] + */ + public static function data_hash_flag_round_trips(): array { + return array( + 'id with an escaped digit' => array( '#\\31 a', WP_CSS_Token_Processor::HASH_TOKEN_ID ), + 'id' => array( '#a1', WP_CSS_Token_Processor::HASH_TOKEN_ID ), + 'id starting with --' => array( '#--', WP_CSS_Token_Processor::HASH_TOKEN_ID ), + 'unrestricted digit' => array( '#1a', WP_CSS_Token_Processor::HASH_TOKEN_UNRESTRICTED ), + 'unrestricted - and digit' => array( '#-1', WP_CSS_Token_Processor::HASH_TOKEN_UNRESTRICTED ), + 'unrestricted lone -' => array( '#-', WP_CSS_Token_Processor::HASH_TOKEN_UNRESTRICTED ), + ); + } + + /** + * Tests that serialize() accepts any iterable, such as a generator. + * + * @ticket 65738 + * + * @covers ::serialize + */ + public function test_serialize_accepts_a_generator(): void { + $generator = ( static function ( string $css ): Generator { + $processor = new WP_CSS_Token_Processor( $css ); + while ( $processor->next_token() ) { + yield $processor->get_token(); + } + } )( 'a /* x */ b(' ); + + $this->assertSame( 'a b()', WP_CSS_Token_Serializer::serialize( $generator ) ); + } + + /** + * Tests that serializing the corpus round-trips and is idempotent. + * + * The output must read as the input tokens, except that comments are dropped, + * whitespace runs are collapsed, and blocks still open are closed. + * + * @ticket 65738 + * @dataProvider data_corpus + * + * @covers ::serialize + */ + public function test_corpus_round_trips( string $css ): void { + $serialized = WP_CSS_Token_Serializer::serialize( self::tokenize( $css ) ); + + $expected = self::significant_tokens( $css ); + $actual = self::significant_tokens( $serialized ); + $closers = array_splice( $actual, count( $expected ) ); + + $this->assertSame( $expected, $actual, 'The output must read as the same tokens.' ); + foreach ( $closers as $closer ) { + $this->assertContains( + $closer[0], + array( WP_CSS_Token_Processor::TOKEN_RIGHT_PAREN, WP_CSS_Token_Processor::TOKEN_RIGHT_BRACKET, WP_CSS_Token_Processor::TOKEN_RIGHT_BRACE ), + 'Only block closers may follow the input tokens.' + ); + } + $this->assertSame( $serialized, WP_CSS_Token_Serializer::serialize( self::tokenize( $serialized ) ), 'Serialization must be idempotent.' ); + } + + /** + * Data provider for test_corpus_round_trips(). + * + * @return array[] + */ + public static function data_corpus(): array { + $cases = json_decode( file_get_contents( DIR_TESTDATA . '/css-api/css-test-cases.json' ), true ); + return array_map( + static function ( array $test_case ): array { + return array( $test_case['css'] ); + }, + $cases + ); + } + + /** + * Tokenizes CSS. + * + * @param string $css CSS text. + * @return WP_CSS_Token[] Tokens. + */ + private static function tokenize( string $css ): array { + $processor = new WP_CSS_Token_Processor( $css ); + $tokens = array(); + while ( $processor->next_token() ) { + $tokens[] = $processor->get_token(); + } + return $tokens; + } + + /** + * Tokenizes CSS for comparison: comments are dropped, and each run of + * whitespace and comments that includes whitespace is one whitespace token. + * + * @param string $css CSS text. + * @return array[] Type, flag, value and unit of each token. + */ + private static function significant_tokens( string $css ): array { + $tokens = array(); + foreach ( self::tokenize( $css ) as $token ) { + $type = $token->get_type(); + if ( WP_CSS_Token_Processor::TOKEN_COMMENT === $type ) { + continue; + } + if ( WP_CSS_Token_Processor::TOKEN_WHITESPACE === $type ) { + if ( array( $type ) !== end( $tokens ) ) { + $tokens[] = array( $type ); + } + continue; + } + $tokens[] = array( $type, $token->get_flag(), $token->get_value(), $token->get_unit() ); + } + return $tokens; + } +}