From b804b3736db59e03e6a1b136e911f3dfeab9ded1 Mon Sep 17 00:00:00 2001 From: zds <49744633+zds-s@users.noreply.github.com> Date: Mon, 3 Jun 2024 10:13:15 +0800 Subject: [PATCH] Added methods `lazyById` and `lazyByIdDesc` for lazy queries (#6822) --- src/Concerns/BuildsQueries.php | 60 ++++++++++++++++++++++ src/Model/Builder.php | 8 +++ src/Query/Builder.php | 8 +++ tests/ModelRealBuilderTest.php | 91 ++++++++++++++++++++++++++++++++++ 4 files changed, 167 insertions(+) diff --git a/src/Concerns/BuildsQueries.php b/src/Concerns/BuildsQueries.php index 3a3153e..d1a6d52 100644 --- a/src/Concerns/BuildsQueries.php +++ b/src/Concerns/BuildsQueries.php @@ -14,6 +14,7 @@ use Closure; use Hyperf\Collection\Collection as BaseCollection; +use Hyperf\Collection\LazyCollection; use Hyperf\Context\ApplicationContext; use Hyperf\Contract\LengthAwarePaginatorInterface; use Hyperf\Contract\PaginatorInterface; @@ -26,6 +27,7 @@ use Hyperf\Paginator\CursorPaginator; use Hyperf\Paginator\Paginator; use Hyperf\Stringable\Str; +use InvalidArgumentException; use RuntimeException; trait BuildsQueries @@ -102,6 +104,22 @@ public function each(callable $callback, $count = 1000) }); } + /** + * Query lazily, by chunking the results of a query by comparing IDs. + */ + public function lazyById(int $chunkSize = 1000, ?string $column = null, ?string $alias = null): LazyCollection + { + return $this->orderedLazyById($chunkSize, $column, $alias); + } + + /** + * Query lazily, by chunking the results of a query by comparing IDs in descending order. + */ + public function lazyByIdDesc(int $chunkSize = 1000, ?string $column = null, ?string $alias = null): LazyCollection + { + return $this->orderedLazyById($chunkSize, $column, $alias, true); + } + /** * Execute the query and get the first result. * @@ -165,6 +183,48 @@ public function unless(mixed $value, callable $callback, ?callable $default = nu return $this; } + /** + * Query lazily, by chunking the results of a query by comparing IDs in a given order. + */ + protected function orderedLazyById(int $chunkSize = 1000, ?string $column = null, ?string $alias = null, bool $descending = false): LazyCollection + { + if ($chunkSize < 1) { + throw new InvalidArgumentException('The chunk size should be at least 1'); + } + + $column ??= $this->defaultKeyName(); + + $alias ??= $column; + + return LazyCollection::make(function () use ($chunkSize, $column, $alias, $descending) { + $lastId = null; + + while (true) { + $clone = clone $this; + + if ($descending) { + $results = $clone->forPageBeforeId($chunkSize, $lastId, $column)->get(); + } else { + $results = $clone->forPageAfterId($chunkSize, $lastId, $column)->get(); + } + + foreach ($results as $result) { + yield $result; + } + + if ($results->count() < $chunkSize) { + return; + } + + $lastId = $results->last()->{$alias}; + + if ($lastId === null) { + throw new RuntimeException("The lazyById operation was aborted because the [{$alias}] column is not present in the query result."); + } + } + }); + } + /** * Create a new length-aware paginator instance. */ diff --git a/src/Model/Builder.php b/src/Model/Builder.php index e695666..79bbec4 100755 --- a/src/Model/Builder.php +++ b/src/Model/Builder.php @@ -1206,6 +1206,14 @@ public static function hasGlobalMacro($name) return isset(static::$macros[$name]); } + /** + * Get the default key name of the table. + */ + protected function defaultKeyName(): string + { + return $this->getModel()->getKeyName(); + } + /** * Ensure the proper order by required for cursor pagination. */ diff --git a/src/Query/Builder.php b/src/Query/Builder.php index beeafce..6dc1946 100755 --- a/src/Query/Builder.php +++ b/src/Query/Builder.php @@ -2919,6 +2919,14 @@ public function cloneWithoutBindings(array $except) }); } + /** + * Get the default key name of the table. + */ + protected function defaultKeyName(): string + { + return 'id'; + } + /** * Ensure the proper order by required for cursor pagination. */ diff --git a/tests/ModelRealBuilderTest.php b/tests/ModelRealBuilderTest.php index 3e9702c..a1cfc67 100644 --- a/tests/ModelRealBuilderTest.php +++ b/tests/ModelRealBuilderTest.php @@ -26,6 +26,7 @@ use Hyperf\Database\Events\QueryExecuted; use Hyperf\Database\Model\EnumCollector; use Hyperf\Database\Model\Events\Saved; +use Hyperf\Database\Model\Model; use Hyperf\Database\MySqlBitConnection; use Hyperf\Database\Query\Builder as QueryBuilder; use Hyperf\Database\Query\Expression; @@ -1184,6 +1185,91 @@ public function testDecrementEach() Schema::drop('accounting_test'); } + public function testOrderedLazyById(): void + { + $container = $this->getContainer(); + $container->shouldReceive('get')->with(Db::class)->andReturn(new Db($container)); + Schema::create('lazy_users', function (Blueprint $table) { + $table->id(); + $table->string('name'); + $table->timestamps(); + }); + $now = Carbon::now(); + Db::table('lazy_users')->insert([ + ['name' => 'Hyperf1', 'created_at' => $now, 'updated_at' => $now], + ['name' => 'Hyperf2', 'created_at' => $now->addMinutes(), 'updated_at' => $now->addMinutes()], + ['name' => 'Hyperf3', 'created_at' => $now->addMinutes(2), 'updated_at' => $now->addMinutes(2)], + ['name' => 'Hyperf4', 'created_at' => $now->addMinutes(3), 'updated_at' => $now->addMinutes(3)], + ['name' => 'Hyperf5', 'created_at' => $now->addMinutes(4), 'updated_at' => $now->addMinutes(4)], + ['name' => 'Hyperf6', 'created_at' => $now->addMinutes(5), 'updated_at' => $now->addMinutes(5)], + ['name' => 'Hyperf7', 'created_at' => $now->addMinutes(6), 'updated_at' => $now->addMinutes(6)], + ['name' => 'Hyperf8', 'created_at' => $now->addMinutes(7), 'updated_at' => $now->addMinutes(7)], + ['name' => 'Hyperf9', 'created_at' => $now->addMinutes(8), 'updated_at' => $now->addMinutes(8)], + ['name' => 'Hyperf10', 'created_at' => $now->addMinutes(9), 'updated_at' => $now->addMinutes(9)], + ]); + $results = LazyUserModel::query()->lazyById(10); + $this->assertCount(10, $results); + foreach ($results as $index => $value) { + $this->assertSame('Hyperf' . ($index + 1), $value->name); + } + $dbResults = Db::table('lazy_users')->lazyById(10); + $this->assertCount(10, $dbResults); + foreach ($dbResults as $index => $value) { + $this->assertSame('Hyperf' . ($index + 1), $value->name); + } + $results = LazyUserModel::query()->lazyById(5); + $dbResults = Db::table('lazy_users')->lazyById(5); + $this->assertCount(10, $results); + foreach ($results as $index => $value) { + $this->assertSame('Hyperf' . ($index + 1), $value->name); + } + $this->assertCount(10, $dbResults); + foreach ($dbResults as $index => $value) { + $this->assertSame('Hyperf' . ($index + 1), $value->name); + } + $results = LazyUserModel::query()->lazyByIdDesc(10); + $this->assertCount(10, $results); + foreach ($results as $index => $value) { + $this->assertSame('Hyperf' . (10 - $index), $value->name); + } + $dbResults = Db::table('lazy_users')->lazyByIdDesc(10); + $this->assertCount(10, $dbResults); + foreach ($dbResults as $index => $value) { + $this->assertSame('Hyperf' . (10 - $index), $value->name); + } + $results = LazyUserModel::query()->lazyByIdDesc(5); + $dbResults = Db::table('lazy_users')->lazyByIdDesc(5); + $this->assertCount(10, $dbResults); + foreach ($dbResults as $index => $value) { + $this->assertSame('Hyperf' . (10 - $index), $value->name); + } + $this->assertCount(10, $results); + foreach ($results as $index => $value) { + $this->assertSame('Hyperf' . (10 - $index), $value->name); + } + $results = LazyUserModel::query()->select(['id', 'name', 'created_at as create_date', 'updated_at'])->lazyByIdDesc(10, 'created_at', 'create_date'); + $dbResults = Db::table('lazy_users')->select(['id', 'name', 'created_at as create_date', 'updated_at'])->lazyByIdDesc(10, 'created_at', 'create_date'); + $this->assertCount(10, $results); + foreach ($results as $index => $value) { + $this->assertSame('Hyperf' . ($index + 1), $value->name); + } + $this->assertCount(10, $dbResults); + foreach ($dbResults as $index => $value) { + $this->assertSame('Hyperf' . ($index + 1), $value->name); + } + $results = LazyUserModel::query()->select(['id', 'name', 'created_at as create_date', 'updated_at'])->lazyById(10, 'created_at', 'create_date'); + $dbResults = Db::table('lazy_users')->select(['id', 'name', 'created_at as create_date', 'updated_at'])->lazyById(10, 'created_at', 'create_date'); + $this->assertCount(10, $results); + foreach ($results as $index => $value) { + $this->assertSame('Hyperf' . ($index + 1), $value->name); + } + $this->assertCount(10, $dbResults); + foreach ($dbResults as $index => $value) { + $this->assertSame('Hyperf' . ($index + 1), $value->name); + } + Schema::dropIfExists('lazy_users'); + } + protected function getContainer() { $dispatcher = Mockery::mock(EventDispatcherInterface::class); @@ -1198,3 +1284,8 @@ protected function getContainer() return $container; } } + +class LazyUserModel extends Model +{ + protected ?string $table = 'lazy_users'; +}