Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
## 2.10.0 - Upcoming

* Added PHP 8.6 support.
* Added support for JEP-18 lexical scoping with let expressions.
* Fixed the compiled runtime to evaluate sub-expressions when the left side is null.
* Fixed the ordering operators to return null instead of false for non-numbers.
* Fixed aggregate and sort functions to error on an invalid first or only element.
Expand Down
26 changes: 24 additions & 2 deletions src/Lexer.php
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ class Lexer
const T_STAR = 'star';
const T_COMMA = 'comma';
const T_COLON = 'colon';
const T_ASSIGN = 'assign';
const T_CURRENT = 'current';
const T_EXPREF = 'expref';
const T_LPAREN = 'lparen';
Expand All @@ -20,6 +21,7 @@ class Lexer
const T_RBRACKET = 'rbracket';
const T_FLATTEN = 'flatten';
const T_IDENTIFIER = 'identifier';
const T_VARIABLE = 'variable';
const T_NUMBER = 'number';
const T_QUOTED_IDENTIFIER = 'quoted_identifier';
const T_UNKNOWN = 'unknown';
Expand All @@ -46,6 +48,7 @@ class Lexer
const STATE_EQ = 11;
const STATE_NOT = 12;
const STATE_AND = 13;
const STATE_VARIABLE = 14;

/** @var array We know what token we are consuming based on each char */
private static $transitionTable = [
Expand All @@ -56,6 +59,7 @@ class Lexer
'[' => self::STATE_LBRACKET,
'|' => self::STATE_PIPE,
'&' => self::STATE_AND,
'$' => self::STATE_VARIABLE,
'`' => self::STATE_JSON_LITERAL,
'"' => self::STATE_QUOTED_STRING,
"'" => self::STATE_STRING_LITERAL,
Expand Down Expand Up @@ -235,6 +239,24 @@ public function tokenize($input)
'pos' => $start
];

} elseif ($state === self::STATE_VARIABLE) {

// Consume variable references
$start = key($chars);
$buffer = '';
$current = next($chars);
if ($current !== false && !isset($this->numbers[$current])) {
while ($current !== false && isset($this->validIdentifier[$current])) {
$buffer .= $current;
$current = next($chars);
}
}
$tokens[] = [
'type' => $buffer === '' ? self::T_UNKNOWN : self::T_VARIABLE,
'value' => $buffer === '' ? '$' : $buffer,
'pos' => $start
];

} elseif ($state === self::STATE_WHITESPACE) {

// Skip whitespace
Expand Down Expand Up @@ -317,8 +339,8 @@ public function tokenize($input)

} elseif ($state === self::STATE_EQ) {

// Consume equals
$tokens[] = $this->matchOr($chars, '=', '=', self::T_COMPARATOR, self::T_UNKNOWN);
// Consume "==" and "="
$tokens[] = $this->matchOr($chars, '=', '=', self::T_COMPARATOR, self::T_ASSIGN);

} elseif ($state == self::STATE_AND) {

Expand Down
56 changes: 56 additions & 0 deletions src/Parser.php
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ class Parser
T::T_EOF => 0,
T::T_QUOTED_IDENTIFIER => 0,
T::T_IDENTIFIER => 0,
T::T_VARIABLE => 0,
T::T_UNKNOWN => 0,
T::T_LITERAL => 0,
T::T_RBRACKET => 0,
Expand All @@ -32,6 +33,7 @@ class Parser
T::T_CURRENT => 0,
T::T_EXPREF => 0,
T::T_COLON => 0,
T::T_ASSIGN => 0,
T::T_PIPE => 1,
T::T_OR => 2,
T::T_AND => 3,
Expand Down Expand Up @@ -106,6 +108,10 @@ private function expr($rbp = 0)

private function nud_identifier()
{
if ($this->isLet()) {
return $this->parseLet();
}

$token = $this->token;
$this->next();
return ['type' => 'field', 'value' => $token['value']];
Expand All @@ -119,6 +125,13 @@ private function nud_quoted_identifier()
return ['type' => 'field', 'value' => $token['value']];
}

private function nud_variable()
{
$token = $this->token;
$this->next();
return ['type' => 'variable', 'value' => $token['value']];
}

private function nud_current()
{
$this->next();
Expand Down Expand Up @@ -357,6 +370,8 @@ private function parseDot($bp)
// Like the multi-select list above, a multi-select hash ends any
// projection: tokens that follow apply to the projected list.
return $this->nud_lbrace();
} elseif ($this->isLet()) {
throw $this->syntax('A let expression is not allowed after a dot');
}

return $this->expr($bp);
Expand All @@ -376,6 +391,39 @@ private function parseKeyValuePair()
];
}

private function parseLet()
{
static $validVariable = [T::T_VARIABLE => true];
static $validAssign = [T::T_ASSIGN => true];
$bindings = [];

do {
$this->next($validVariable);
$name = $this->token['value'];
$this->next($validAssign);
$this->next();
$bindings[] = [
'type' => 'variable_binding',
'value' => $name,
'children' => [$this->expr()]
];
} while ($this->token['type'] == T::T_COMMA);

if ($this->token['type'] !== T::T_IDENTIFIER || $this->token['value'] !== 'in') {
throw $this->syntax('Expected "in" after the let bindings');
}

$this->next();

return [
'type' => 'let',
'children' => [
['type' => 'bindings', 'children' => $bindings],
$this->expr()
]
];
}

private function parseWildcardObject(?array $left = null)
{
$this->next();
Expand Down Expand Up @@ -476,6 +524,14 @@ private function syntax($msg)
return new SyntaxErrorException($msg, $this->token, $this->expression);
}

private function isLet()
{
// "let" is only a keyword before a variable; elsewhere it is a field.
return $this->token['type'] === T::T_IDENTIFIER
&& $this->token['value'] === 'let'
&& $this->lookahead() === T::T_VARIABLE;
}

private function lookahead()
{
return (!isset($this->tokens[$this->tpos + 1]))
Expand Down
54 changes: 51 additions & 3 deletions src/TreeCompiler.php
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ class TreeCompiler
private $indentation;
private $source;
private $vars;
private $scope;

/**
* @param array $ast AST to compile.
Expand All @@ -19,7 +20,7 @@ class TreeCompiler
*/
public function visit(array $ast, $fnName, $expr)
{
$this->vars = [];
$this->vars = $this->scope = [];
$this->source = $this->indentation = '';
$this->write("<?php\n")
->write('use JmesPath\\TreeInterpreter as Ti;')
Expand Down Expand Up @@ -203,6 +204,29 @@ private function visit_pipe(array $node)
->dispatch($node['children'][1]);
}

private function visit_let(array $node)
{
$value = $this->makeVar('beforeLet');
$scope = $this->scope;
$bound = [];
$this->write('%s = $value;', $value);

foreach ($node['children'][0]['children'] as $binding) {
$var = $this->makeVar('let');
$this->dispatch($binding['children'][0])
->write('%s = $value;', $var)
->write('$value = %s;', $value);
$bound[$binding['value']] = $var;
}

// Bindings are only visible to the body, not to each other.
$this->scope = $bound + $scope;
$this->dispatch($node['children'][1]);
$this->scope = $scope;

return $this;
}

private function visit_multi_select_list(array $node)
{
return $this->visit_multi_select_hash($node);
Expand Down Expand Up @@ -274,12 +298,36 @@ private function visit_current(array $node)
return $this->write('// Visiting current node (no-op)');
}

private function visit_variable(array $node)
{
if (isset($this->scope[$node['value']])) {
return $this->write('$value = %s;', $this->scope[$node['value']]);
}

return $this->write(
'throw new \\RuntimeException(%s);',
var_export('Undefined variable: $' . $node['value'], true)
);
}

private function visit_expref(array $node)
{
$child = var_export($node['children'][0], true);
return $this->write('$value = function ($value) use ($interpreter) {')
$interpreter = '$interpreter';

if ($this->scope) {
$bindings = [];
foreach ($this->scope as $name => $var) {
$bindings[] = var_export($name, true) . ' => ' . $var;
}
$scoped = $this->makeVar('scoped');
$this->write('%s = $interpreter->withScope([%s]);', $scoped, implode(', ', $bindings));
$interpreter = $scoped;
}

return $this->write('$value = function ($value) use (%s) {', $interpreter)
->indent()
->write('return $interpreter->visit(%s, $value);', $child)
->write('return %s->visit(%s, $value);', $interpreter, $child)
->outdent()
->write('};');
}
Expand Down
36 changes: 36 additions & 0 deletions src/TreeInterpreter.php
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,9 @@ class TreeInterpreter
/** @var callable */
private $fnDispatcher;

/** @var array Variables in scope, keyed by name */
private $scope = [];

/**
* @param callable|null $fnDispatcher Function dispatching function that accepts
* a function name argument and an array of
Expand All @@ -32,6 +35,22 @@ public function visit(array $node, $data)
return $this->dispatch($node, $data);
}

/**
* Returns a copy of the interpreter that has the given variables in scope.
*
* @param array $scope Variables to bind, keyed by name
*
* @return static
*/
public function withScope(array $scope)
{
// Bind on a copy so that expression references keep their variables.
$interpreter = clone $this;
$interpreter->scope = $scope + $this->scope;

return $interpreter;
}

/**
* Recursively traverses an AST using depth-first, pre-order traversal.
* The evaluation logic for each node type is embedded into a large switch
Expand Down Expand Up @@ -123,6 +142,12 @@ private function dispatch(array $node, $value)
case 'current':
return $value;

case 'variable':
if (!array_key_exists($node['value'], $this->scope)) {
throw new \RuntimeException('Undefined variable: $' . $node['value']);
}
return $this->scope[$node['value']];

case 'or':
$result = $this->dispatch($node['children'][0], $value);
return Utils::isTruthy($result)
Expand All @@ -146,6 +171,17 @@ private function dispatch(array $node, $value)
$this->dispatch($node['children'][0], $value)
);

case 'let':
$bound = [];
foreach ($node['children'][0]['children'] as $binding) {
$bound[$binding['value']] = $this->dispatch(
$binding['children'][0],
$value
);
}

return $this->withScope($bound)->dispatch($node['children'][1], $value);

case 'multi_select_list':
if ($value === null) {
return null;
Expand Down
21 changes: 21 additions & 0 deletions tests/CompilerRuntimeTest.php
Original file line number Diff line number Diff line change
Expand Up @@ -304,6 +304,27 @@ public function testRuntimesEvaluateSubexpressionsOfNull(): void
}
}

public function testRuntimesReportUndefinedVariables(): void
{
$dir = $this->createTempDir();

try {
foreach ([new AstRuntime(), new CompilerRuntime($dir)] as $runtime) {
$message = null;

try {
$runtime('[let $foo = @ in $foo, $foo]', []);
} catch (\RuntimeException $e) {
$message = $e->getMessage();
}

$this->assertSame('Undefined variable: $foo', $message);
}
} finally {
$this->removeTempDir($dir);
}
}

public function testCompiledRuntimeSortsNumerically(): void
{
$dir = $this->createTempDir();
Expand Down
21 changes: 21 additions & 0 deletions tests/LexerTest.php
Original file line number Diff line number Diff line change
Expand Up @@ -33,9 +33,16 @@ public static function inputProvider(): array
[']', 'rbracket'],
[':', 'colon'],
[',', 'comma'],
['=', 'assign'],
['==', 'comparator'],
['||', 'or'],
['*', 'star'],
['foo', 'identifier'],
['$foo', 'variable'],
['$_', 'variable'],
['$', 'unknown'],
['$1', 'unknown'],
['$ foo', 'unknown'],
['"foo"', 'quoted_identifier'],
['`true`', 'literal'],
['`false`', 'literal'],
Expand Down Expand Up @@ -79,6 +86,20 @@ public function testTokenizesJsonNumbers(): void
$this->assertEquals(120, $tokens[6]['value']);
}

public function testTokenizesVariables(): void
{
$l = new Lexer();
$tokens = $l->tokenize('let $foo_1 = $Bar');
$this->assertSame('identifier', $tokens[0]['type']);
$this->assertSame('variable', $tokens[1]['type']);
$this->assertSame('foo_1', $tokens[1]['value']);
$this->assertSame(4, $tokens[1]['pos']);
$this->assertSame('assign', $tokens[2]['type']);
$this->assertSame('variable', $tokens[3]['type']);
$this->assertSame('Bar', $tokens[3]['value']);
$this->assertSame(13, $tokens[3]['pos']);
}

public function testCanWorkWithElidedJsonLiterals(): void
{
$l = new Lexer();
Expand Down
Loading