From 20ad5e6ba90593ae2669ffd5154f24a838887d75 Mon Sep 17 00:00:00 2001 From: geni_jaho Date: Sun, 12 May 2024 16:16:41 +0200 Subject: [PATCH] Fix exporting large amounts of data --- app/Actions/Photos/ExportPhotosAction.php | 11 +++++------ app/Console/Commands/GenerateRandomPhotos.php | 3 ++- app/DTO/PhotoExport.php | 5 ++--- .../Photos/ExportPhotosController.php | 7 ++++--- tests/Feature/Photos/ExportPhotosTest.php | 16 +++++++++++++++- tests/Unit/Actions/ExportPhotosActionTest.php | 18 ++++++++++++------ 6 files changed, 40 insertions(+), 20 deletions(-) diff --git a/app/Actions/Photos/ExportPhotosAction.php b/app/Actions/Photos/ExportPhotosAction.php index 25cea8ab..e236c4ed 100644 --- a/app/Actions/Photos/ExportPhotosAction.php +++ b/app/Actions/Photos/ExportPhotosAction.php @@ -4,15 +4,12 @@ use App\DTO\PhotoExport; use App\Models\User; +use Generator; use Illuminate\Contracts\Database\Eloquent\Builder; -use Illuminate\Support\LazyCollection; class ExportPhotosAction { - /** - * @return LazyCollection - */ - public function run(User $user): LazyCollection + public function run(User $user): Generator { $photos = $user ->photos() @@ -23,6 +20,8 @@ public function run(User $user): LazyCollection ]) ->lazyById(); - return PhotoExport::collect($photos); + foreach ($photos as $photo) { + yield PhotoExport::fromModel($photo); + } } } diff --git a/app/Console/Commands/GenerateRandomPhotos.php b/app/Console/Commands/GenerateRandomPhotos.php index 3d146c18..2f692e30 100644 --- a/app/Console/Commands/GenerateRandomPhotos.php +++ b/app/Console/Commands/GenerateRandomPhotos.php @@ -16,7 +16,7 @@ public function handle(): void { /** @var User $user */ $user = User::query() - ->where('email', 'trashkiller@litterhero.com') + ->where('email', 'trashkiller@litterapp.com') ->first(); $bar = progress('Generating 1M photos with tags...', 1_000_000); @@ -34,6 +34,7 @@ public function handle(): void $photos[] = [ 'user_id' => $user->id, 'path' => 'photos/default.jpg', + 'original_file_name' => fake()->unique()->sentence().'default.jpg', 'created_at' => $now, 'updated_at' => $now, ]; diff --git a/app/DTO/PhotoExport.php b/app/DTO/PhotoExport.php index 86a26c7e..52c6b312 100644 --- a/app/DTO/PhotoExport.php +++ b/app/DTO/PhotoExport.php @@ -4,7 +4,6 @@ use App\Models\Photo; use App\Models\PhotoItem; -use Illuminate\Support\Carbon; use Illuminate\Support\Collection; use Spatie\LaravelData\Data; @@ -20,7 +19,7 @@ public function __construct( public ?float $latitude, public ?float $longitude, public ?string $taken_at_local, - public Carbon $created_at, + public string $created_at, public Collection $items, ) { } @@ -33,7 +32,7 @@ public static function fromModel(Photo $photo): static latitude: $photo->latitude, longitude: $photo->longitude, taken_at_local: $photo->taken_at_local, - created_at: $photo->created_at, + created_at: $photo->created_at?->toIso8601String(), items: $photo->photoItems->map(fn (PhotoItem $photoItem): array => [ 'name' => $photoItem->item?->name, 'picked_up' => $photoItem->picked_up, diff --git a/app/Http/Controllers/Photos/ExportPhotosController.php b/app/Http/Controllers/Photos/ExportPhotosController.php index 4f7a8277..b2b712d2 100644 --- a/app/Http/Controllers/Photos/ExportPhotosController.php +++ b/app/Http/Controllers/Photos/ExportPhotosController.php @@ -16,8 +16,9 @@ public function __invoke(ExportPhotosAction $action): StreamedResponse $photos = $action->run($user); - return response()->streamDownload(function () use ($photos): void { - echo $photos->toJson(); - }, 'photos.json'); + return response()->streamJson(['photos' => $photos], 200, [ + 'Content-Type' => 'application/json', + 'Content-Disposition' => 'attachment; filename="photos.json"', + ]); } } diff --git a/tests/Feature/Photos/ExportPhotosTest.php b/tests/Feature/Photos/ExportPhotosTest.php index 50042916..55cf4ddf 100644 --- a/tests/Feature/Photos/ExportPhotosTest.php +++ b/tests/Feature/Photos/ExportPhotosTest.php @@ -6,12 +6,23 @@ test('a user can export their photos', function (): void { $this->actingAs($user = User::factory()->create()); - Photo::factory()->for($user)->create(['created_at' => now()]); + $photo = Photo::factory()->for($user)->create(); $response = $this->get('/photos/export'); $response->assertOk(); $response->assertDownload('photos.json'); + $response->assertStreamedJsonContent([ + 'photos' => [[ + 'id' => $photo->id, + 'original_file_name' => $photo->original_file_name, + 'latitude' => $photo->latitude, + 'longitude' => $photo->longitude, + 'taken_at_local' => $photo->taken_at_local, + 'created_at' => $photo->created_at->toIso8601String(), + 'items' => [], + ]], + ]); }); test('if there are no photos the export still works', function (): void { @@ -21,4 +32,7 @@ $response->assertOk(); $response->assertDownload('photos.json'); + $response->assertStreamedJsonContent([ + 'photos' => [], + ]); }); diff --git a/tests/Unit/Actions/ExportPhotosActionTest.php b/tests/Unit/Actions/ExportPhotosActionTest.php index a01e33b7..ff2ce93b 100644 --- a/tests/Unit/Actions/ExportPhotosActionTest.php +++ b/tests/Unit/Actions/ExportPhotosActionTest.php @@ -9,7 +9,6 @@ use App\Models\Tag; use App\Models\User; use Illuminate\Support\Collection; -use Illuminate\Support\LazyCollection; use function Pest\Laravel\freezeTime; @@ -30,15 +29,18 @@ $exportPhotosAction = new ExportPhotosAction(); $result = $exportPhotosAction->run($user); - expect($result)->toBeInstanceOf(LazyCollection::class) - ->and($result->count())->toBe(1) - ->and($result->first())->toBeInstanceOf(PhotoExport::class) + expect($result)->toBeInstanceOf(Generator::class); + + $result = iterator_to_array($result); + + expect($result)->toHaveCount(1) + ->and($result[0])->toBeInstanceOf(PhotoExport::class) ->id->toBe($photo->id) ->original_file_name->toBe($photo->original_file_name) ->latitude->toBe($photo->latitude) ->longitude->toBe($photo->longitude) ->taken_at_local->toBe($photo->taken_at_local) - ->created_at->toEqual($photo->created_at) + ->created_at->toEqual($photo->created_at->toIso8601String()) ->items->toBeInstanceOf(Collection::class) ->items->toHaveCount(1) ->items->first()->toBe([ @@ -66,6 +68,10 @@ $exportPhotosAction = new ExportPhotosAction(); $result = $exportPhotosAction->run($user); + expect($result)->toBeInstanceOf(Generator::class); + + $result = iterator_to_array($result); + expect($result)->toHaveCount(1) - ->and($result->first()->id)->toBe($pickedUpPhoto->id); + ->and($result[0]->id)->toBe($pickedUpPhoto->id); });