From 5ce3f0744fbe7620074d7ffb750c3a0dbb57b21d Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Fri, 4 Aug 2023 12:18:26 +0200 Subject: [PATCH 1/5] Update .gitignore --- .gitignore | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index dd8b26f..cfd3027 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,9 @@ build -composer.lock +profiling vendor -phpcs.xml -phpunit.xml +.DS_Store .phpunit.cache .phpunit.result.cache +composer.lock +phpcs.xml +phpunit.xml From ec84c4b703594e1d5d3fe7838cdcf152e6a038ce Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Sun, 6 Aug 2023 01:43:40 +0200 Subject: [PATCH 2/5] Add option to wrap the parser --- README.md | 19 +++++++++++- src/JsonParser.php | 13 +++++++++ src/Tokens/Parser.php | 3 +- src/ValueObjects/Config.php | 9 ++++++ tests/Feature/ParsingTest.php | 8 ++++++ tests/Pest.php | 54 +++++++++++++++++++++++++++++++++++ 6 files changed, 104 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 3cf790e..a56da8c 100644 --- a/README.md +++ b/README.md @@ -270,7 +270,7 @@ $array = JsonParser::parse($source)->pointers(['/results/0/gender', '/results/0/ ### 🐼 Lazy pointers -JSON Parser only keeps one key and one value in memory at a time. However, if the value is a large array or object, it may be inefficient to keep it all in memory. +JSON Parser only keeps one key and one value in memory at a time. However, if the value is a large array or object, it may be inefficient or even impossible to keep it all in memory. To solve this problem, we can use lazy pointers. These pointers recursively keep in memory only one key and one value at a time for any nested array or object. @@ -323,6 +323,23 @@ foreach (JsonParser::parse($source)->lazy() as $key => $value) { } ``` +We can recursively wrap any instance of `Cerbero\JsonParser\Tokens\Parser` by chaining `wrap()`. This lets us wrap lazy loaded JSON arrays and objects into classes with advanced functionalities, like mapping or filtering: + +```php +$json = JsonParser::parse($source) + ->wrap(fn (Parser $parser) => new MyWrapper(fn () => yield from $parser)) + ->lazy(); + +foreach ($json as $key => $value) { + // 1st iteration: $key === 'results', $value instanceof MyWrapper + foreach ($value as $nestedKey => $nestedValue) { + // 1st iteration: $nestedKey === 0, $nestedValue instanceof MyWrapper + // 2nd iteration: $nestedKey === 1, $nestedValue instanceof MyWrapper + // ... + } +} +``` + Lazy pointers also have all the other functionalities of normal pointers: they accept callbacks, can be set one by one or all together, can be eager loaded into an array and can be mixed with normal pointers as well: ```php diff --git a/src/JsonParser.php b/src/JsonParser.php index d74ae19..f3cef0c 100644 --- a/src/JsonParser.php +++ b/src/JsonParser.php @@ -246,4 +246,17 @@ public function onSyntaxError(Closure $callback): self return $this; } + + /** + * Set the logic to run for wrapping the parser + * + * @param Closure $callback + * @return self + */ + public function wrap(Closure $callback): self + { + $this->config->wrapper = $callback; + + return $this; + } } diff --git a/src/Tokens/Parser.php b/src/Tokens/Parser.php index b21a60c..3e0aa7c 100644 --- a/src/Tokens/Parser.php +++ b/src/Tokens/Parser.php @@ -71,8 +71,9 @@ public function getIterator(): Traversable /** @var string|int $key */ $key = $this->decoder->decode($state->tree->currentKey()); $value = $this->decoder->decode($state->value()); + $wrapper = $value instanceof self ? ($this->config->wrapper)($value) : $value; - yield $key => $state->callPointer($value, $key); + yield $key => $state->callPointer($wrapper, $key); $value instanceof self && $value->fastForward(); } diff --git a/src/ValueObjects/Config.php b/src/ValueObjects/Config.php index f4fedfc..e7bd5cb 100644 --- a/src/ValueObjects/Config.php +++ b/src/ValueObjects/Config.php @@ -10,6 +10,7 @@ use Cerbero\JsonParser\Exceptions\SyntaxException; use Cerbero\JsonParser\Pointers\Pointer; use Cerbero\JsonParser\Pointers\Pointers; +use Cerbero\JsonParser\Tokens\Parser; use Closure; /** @@ -53,6 +54,13 @@ final class Config */ public Closure $onSyntaxError; + /** + * The callback to run for wrapping the parser. + * + * @var Closure + */ + public Closure $wrapper; + /** * Instantiate the class * @@ -63,6 +71,7 @@ public function __construct() $this->pointers = new Pointers(); $this->onDecodingError = fn (DecodedValue $decoded) => throw new DecodingException($decoded); $this->onSyntaxError = fn (SyntaxException $e) => throw $e; + $this->wrapper = fn (Parser $parser) => $parser; } /** diff --git a/tests/Feature/ParsingTest.php b/tests/Feature/ParsingTest.php index 63d9e96..22c65d9 100644 --- a/tests/Feature/ParsingTest.php +++ b/tests/Feature/ParsingTest.php @@ -3,6 +3,8 @@ use Cerbero\JsonParser\Dataset; use Cerbero\JsonParser\Decoders\SimdjsonDecoder; use Cerbero\JsonParser\JsonParser; +use Cerbero\JsonParser\Tokens\Parser; +use Pest\Expectation; use function Cerbero\JsonParser\parseJson; @@ -42,3 +44,9 @@ expect($parser->progress()->percentage())->toBe(100.0); }); + +it('wraps the parser recursively', function (string $source) { + $json = JsonParser::parse($source)->wrap(fn (Parser $parser) => yield from $parser)->lazy(); + + expect($json)->traverse(fn (Expectation $value) => $value->toBeWrappedInto(Generator::class)); +})->with([fixture('json/complex_array.json'), fixture('json/complex_array.json')]); diff --git a/tests/Pest.php b/tests/Pest.php index 41ffd16..05f591f 100644 --- a/tests/Pest.php +++ b/tests/Pest.php @@ -1,6 +1,7 @@ extend('traverse', function (mixed ...$callbacks) { + if (! is_iterable($this->value)) { + throw new BadMethodCallException('Expectation value is not iterable.'); + } + + if (empty($callbacks)) { + throw new InvalidArgumentException('No sequence expectations defined.'); + } + + $index = $valuesCount = 0; + + foreach ($this->value as $key => $value) { + $valuesCount++; + + if ($callbacks[$index] instanceof Closure) { + $callbacks[$index](new self($value), new self($key)); + } else { + (new self($value))->toEqual($callbacks[$index]); + } + + $index = isset($callbacks[$index + 1]) ? $index + 1 : 0; + } + + if (count($callbacks) > $valuesCount) { + throw new OutOfRangeException('Sequence expectations are more than the iterable items'); + } + + return $this; +}); + /** * Expect that keys and values are parsed correctly * @@ -74,4 +112,20 @@ function fixture(string $fixture): string expect($key)->toBe($expectedKey)->and($value)->toLazyLoadRecursively($keys, $expected); } } + + return $this; +}); + +/** + * Expect that all Parser instances are wrapped recursively + * + * @param string $wrapper + * @return Expectation + */ +expect()->extend('toBeWrappedInto', function (string $wrapper) { + return $this->when(is_object($this->value), fn (Expectation $value) => $value + ->toBeInstanceOf($wrapper) + ->not->toBeInstanceOf(Parser::class) + ->traverse(fn (Expectation $value) => $value->toBeWrappedInto($wrapper)) + ); }); From a752b191af7cd1e752c2e54b4c7964c81e23ae27 Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Sun, 6 Aug 2023 17:15:00 +0200 Subject: [PATCH 3/5] Remove unused method --- src/ValueObjects/State.php | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/src/ValueObjects/State.php b/src/ValueObjects/State.php index b4bd7b7..14259ac 100644 --- a/src/ValueObjects/State.php +++ b/src/ValueObjects/State.php @@ -54,16 +54,6 @@ public function __construct(private readonly Pointers $pointers, private readonl $this->tree = new Tree($pointers); } - /** - * Retrieve the JSON tree - * - * @return Tree - */ - public function tree(): Tree - { - return $this->tree; - } - /** * Determine whether the parser can stop parsing * From d755ff1f7368fbcc250b0e5a9e426f1e64dfc0c9 Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Sun, 6 Aug 2023 17:29:27 +0200 Subject: [PATCH 4/5] Support wildcards when turning into array --- README.md | 2 + src/Tokens/Parser.php | 12 +- .../pointers/multiple_pointers_to_array.php | 166 ++++++++++++++--- .../pointers/single_pointer_to_array.php | 173 +++++++++++++++--- 4 files changed, 307 insertions(+), 46 deletions(-) diff --git a/README.md b/README.md index a56da8c..088f12f 100644 --- a/README.md +++ b/README.md @@ -340,6 +340,8 @@ foreach ($json as $key => $value) { } ``` +> ℹ️ If your wrapper class implements the method `toArray()`, such method will be called when eager loading sub-trees into an array. + Lazy pointers also have all the other functionalities of normal pointers: they accept callbacks, can be set one by one or all together, can be eager loaded into an array and can be mixed with normal pointers as well: ```php diff --git a/src/Tokens/Parser.php b/src/Tokens/Parser.php index 3e0aa7c..5676e2a 100644 --- a/src/Tokens/Parser.php +++ b/src/Tokens/Parser.php @@ -113,13 +113,21 @@ public function lazyLoad(): Generator */ public function toArray(): array { + $index = 0; $array = []; + $hasWildcards = false; foreach ($this as $key => $value) { - $array[$key] = $value instanceof self ? $value->toArray() : $value; + if (isset($array[$index][$key])) { + $index++; + $hasWildcards = true; + } + + $turnsIntoArray = is_object($value) && method_exists($value, 'toArray'); + $array[$index][$key] = $turnsIntoArray ? $value->toArray() : $value; } - return $array; + return $hasWildcards || empty($array) ? $array : $array[0]; } /** diff --git a/tests/fixtures/pointers/multiple_pointers_to_array.php b/tests/fixtures/pointers/multiple_pointers_to_array.php index a57adf3..229e138 100644 --- a/tests/fixtures/pointers/multiple_pointers_to_array.php +++ b/tests/fixtures/pointers/multiple_pointers_to_array.php @@ -4,44 +4,168 @@ 'complex_array' => [ '/-1,/-2' => [], '/-/id,/-/batters/batter/-/type' => [ - 'id' => '0003', - 'type' => 'Chocolate', + [ + 'id' => '0001', + 'type' => 'Regular', + ], + [ + 'type' => 'Chocolate', + ], + [ + 'type' => 'Blueberry', + ], + [ + 'type' => 'Devil\'s Food', + 'id' => '0002', + ], + [ + 'type' => 'Regular', + 'id' => '0003', + ], + [ + 'type' => 'Regular', + ], + [ + 'type' => 'Chocolate', + ], ], '/-/name,/-/topping/-/type,/-/id' => [ - 'id' => '0003', - 'name' => 'Old Fashioned', - 'type' => 'Maple', - ], - '/-/batters/batter/-,/-/name' => [ - 'name' => 'Old Fashioned', [ - "id" => "1001", - "type" => "Regular", + 'id' => '0001', + 'name' => 'Cake', + 'type' => 'None', ], [ - "id" => "1002", - "type" => "Chocolate", + 'type' => 'Glazed', ], [ - "id" => "1003", - "type" => "Blueberry", + 'type' => 'Sugar', ], [ - "id" => "1004", - "type" => "Devil's Food", + 'type' => 'Powdered Sugar', + ], + [ + 'type' => 'Chocolate with Sprinkles', + ], + [ + 'type' => 'Chocolate', + ], + [ + 'type' => 'Maple', + 'id' => '0002', + 'name' => 'Raised', + ], + [ + 'type' => 'None', + ], + [ + 'type' => 'Glazed', + ], + [ + 'type' => 'Sugar', + ], + [ + 'type' => 'Chocolate', + ], + [ + 'type' => 'Maple', + 'id' => '0003', + 'name' => 'Old Fashioned', + ], + [ + 'type' => 'None', + ], + [ + 'type' => 'Glazed', + ], + [ + 'type' => 'Chocolate', + ], + [ + 'type' => 'Maple', + ], + ], + '/-/batters/batter/-,/-/name' => [ + [ + 'name' => 'Cake', + [ + 'id' => '1001', + 'type' => 'Regular', + ], + [ + 'id' => '1002', + 'type' => 'Chocolate', + ], + [ + 'id' => '1003', + 'type' => 'Blueberry', + ], + [ + 'id' => '1004', + 'type' => 'Devil\'s Food', + ], + ], + [ + 'name' => 'Raised', + [ + 'id' => '1001', + 'type' => 'Regular', + ], + ], + [ + 'name' => 'Old Fashioned', + [ + 'id' => '1001', + 'type' => 'Regular', + ], + [ + 'id' => '1002', + 'type' => 'Chocolate', + ], ], ], ], 'complex_object' => [ '/-1,/-2' => [], '/id,/batters/batter/-/type' => [ - 'id' => '0001', - 'type' => "Devil's Food", + [ + 'id' => '0001', + 'type' => 'Regular', + ], + [ + 'type' => 'Chocolate', + ], + [ + 'type' => 'Blueberry', + ], + [ + 'type' => 'Devil\'s Food', + ], ], '/name,/topping/-/type,/id' => [ - 'id' => '0001', - 'name' => 'Cake', - 'type' => 'Maple', + [ + 'id' => '0001', + 'name' => 'Cake', + 'type' => 'None', + ], + [ + 'type' => 'Glazed', + ], + [ + 'type' => 'Sugar', + ], + [ + 'type' => 'Powdered Sugar', + ], + [ + 'type' => 'Chocolate with Sprinkles', + ], + [ + 'type' => 'Chocolate', + ], + [ + 'type' => 'Maple', + ], ], '/batters/batter/-,/type' => [ 'type' => 'donut', diff --git a/tests/fixtures/pointers/single_pointer_to_array.php b/tests/fixtures/pointers/single_pointer_to_array.php index fd116f3..8ef2e15 100644 --- a/tests/fixtures/pointers/single_pointer_to_array.php +++ b/tests/fixtures/pointers/single_pointer_to_array.php @@ -4,52 +4,166 @@ 'complex_array' => [ '' => $complexArray = require __DIR__ . '/../parsing/complex_array.php', '/-' => $complexArray, - '/-/id' => ['id' => '0003'], + '/-/id' => [ + [ + 'id' => '0001', + ], + [ + 'id' => '0002', + ], + [ + 'id' => '0003', + ], + ], '/-/batters' => [ - 'batters' => [ + [ + 'batters' => [ + 'batter' => [ + [ + 'id' => '1001', + 'type' => 'Regular', + ], + [ + 'id' => '1002', + 'type' => 'Chocolate', + ], + [ + 'id' => '1003', + 'type' => 'Blueberry', + ], + [ + 'id' => '1004', + 'type' => 'Devil\'s Food', + ], + ], + ], + ], + [ + 'batters' => [ + 'batter' => [ + [ + 'id' => '1001', + 'type' => 'Regular', + ], + ], + ], + ], + [ + 'batters' => [ + 'batter' => [ + [ + 'id' => '1001', + 'type' => 'Regular', + ], + [ + 'id' => '1002', + 'type' => 'Chocolate', + ], + ], + ], + ], + ], + '/-/batters/batter' => [ + [ 'batter' => [ [ - "id" => "1001", - "type" => "Regular", + 'id' => '1001', + 'type' => 'Regular', ], [ - "id" => "1002", - "type" => "Chocolate", + 'id' => '1002', + 'type' => 'Chocolate', + ], + [ + 'id' => '1003', + 'type' => 'Blueberry', + ], + [ + 'id' => '1004', + 'type' => 'Devil\'s Food', + ], + ], + ], + [ + 'batter' => [ + [ + 'id' => '1001', + 'type' => 'Regular', + ], + ], + ], + [ + 'batter' => [ + [ + 'id' => '1001', + 'type' => 'Regular', + ], + [ + 'id' => '1002', + 'type' => 'Chocolate', ], ], ], ], - '/-/batters/batter' => [ - 'batter' => [ + '/-/batters/batter/-' => [ + [ [ - "id" => "1001", - "type" => "Regular", + 'id' => '1001', + 'type' => 'Regular', ], [ - "id" => "1002", - "type" => "Chocolate", + 'id' => '1002', + 'type' => 'Chocolate', + ], + [ + 'id' => '1003', + 'type' => 'Blueberry', + ], + [ + 'id' => '1004', + 'type' => 'Devil\'s Food', + ], + ], + [ + [ + 'id' => '1001', + 'type' => 'Regular', + ], + ], + [ + [ + 'id' => '1001', + 'type' => 'Regular', + ], + [ + 'id' => '1002', + 'type' => 'Chocolate', ], ], ], - '/-/batters/batter/-' => [ + '/-/batters/batter/-/id' => [ [ - "id" => "1001", - "type" => "Regular", + 'id' => '1001', ], [ - "id" => "1002", - "type" => "Chocolate", + 'id' => '1002', ], [ - "id" => "1003", - "type" => "Blueberry", + 'id' => '1003', ], [ - "id" => "1004", - "type" => "Devil's Food", + 'id' => '1004', + ], + [ + 'id' => '1001', + ], + [ + 'id' => '1001', + ], + [ + 'id' => '1002', ], ], - '/-/batters/batter/-/id' => ['id' => "1002"], ], 'complex_object' => [ '' => require __DIR__ . '/../parsing/complex_object.php', @@ -115,7 +229,20 @@ "type" => "Devil's Food", ], ], - '/batters/batter/-/id' => ['id' => "1004"], + '/batters/batter/-/id' => [ + [ + 'id' => '1001', + ], + [ + 'id' => '1002', + ], + [ + 'id' => '1003', + ], + [ + 'id' => '1004', + ], + ], ], 'empty_array' => [ '' => [], From f3898b834bf97701df81c90df5af7e2e8c44f4c5 Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Sun, 6 Aug 2023 17:38:43 +0200 Subject: [PATCH 5/5] Update changelog --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5e7f4c1..0708157 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,14 @@ Updates should follow the [Keep a CHANGELOG](https://keepachangelog.com/) princi - Nothing +## 1.1.0 - 2023-08-06 + +### Added +- Ability to wrap Parser instances recursively when lazy loading +- Support for turning Parser wrappers into array +- Support for turning sub-trees using wildcards into array + + ## 1.0.0 - 2023-06-16 ### Added