From cd79554cfbf7c1603b57936d4e333fae9ac9ec6b Mon Sep 17 00:00:00 2001 From: Paulo Castellano Date: Thu, 23 Jul 2026 11:04:52 -0300 Subject: [PATCH] fix: multipart chunked video uploads directly to R2 The last chunk was reassembling the full file locally and pushing ~185MB to R2 in one request (53s+). Videos/PDFs now upload each chunk as an S3 multipart part so finalize only completes the upload. Co-authored-by: Cursor --- app/Http/Controllers/App/AssetController.php | 49 ++++- app/Models/Traits/HasMedia.php | 32 +++ app/Services/Media/ChunkedCloudUploader.php | 208 +++++++++++++++++++ tests/Feature/ChunkedCloudUploadTest.php | 135 ++++++++++++ 4 files changed, 419 insertions(+), 5 deletions(-) create mode 100644 app/Services/Media/ChunkedCloudUploader.php create mode 100644 tests/Feature/ChunkedCloudUploadTest.php diff --git a/app/Http/Controllers/App/AssetController.php b/app/Http/Controllers/App/AssetController.php index 10eff45b..0cab100d 100644 --- a/app/Http/Controllers/App/AssetController.php +++ b/app/Http/Controllers/App/AssetController.php @@ -10,6 +10,7 @@ use App\Http\Resources\App\MediaResource; use App\Models\Media; use App\Services\Brand\SafeHttpFetcher; +use App\Services\Media\ChunkedCloudUploader; use App\Services\UnsplashService; use Illuminate\Http\JsonResponse; use Illuminate\Http\RedirectResponse; @@ -68,7 +69,7 @@ public function store(StoreAssetRequest $request): MediaResource return new MediaResource($media); } - public function storeChunked(StoreChunkedAssetRequest $request): JsonResponse + public function storeChunked(StoreChunkedAssetRequest $request, ChunkedCloudUploader $cloudUploader): JsonResponse { $workspace = $request->user()->currentWorkspace; @@ -78,8 +79,49 @@ public function storeChunked(StoreChunkedAssetRequest $request): JsonResponse $rangeEnd = (int) $request->validated('range_end'); $totalSize = (int) $request->validated('total_size'); $fileName = (string) $request->validated('file_name'); + $chunk = $request->getContent(); $identifier = md5($request->user()->id.$fileName.$totalSize); + + if ($cloudUploader->supports() && $cloudUploader->shouldUseMultipart($fileName)) { + $result = $cloudUploader->receiveChunk( + $identifier, + $fileName, + $chunk, + $rangeStart, + $rangeEnd, + $totalSize, + ); + + if (! data_get($result, 'done')) { + return response()->json([ + 'done' => false, + 'progress' => data_get($result, 'progress'), + ]); + } + + $media = $workspace->addMediaFromStoredPath( + (string) data_get($result, 'path'), + $fileName, + (string) data_get($result, 'mime_type'), + (int) data_get($result, 'size'), + 'assets', + ); + + return response()->json([ + 'done' => true, + 'id' => $media->id, + 'path' => $media->path, + 'url' => $media->url, + 'type' => $media->type->value, + 'mime_type' => $media->mime_type, + 'original_filename' => $media->original_filename, + 'size' => $media->size, + 'meta' => $media->meta, + 'created_at' => $media->created_at->toISOString(), + ]); + } + $tempFile = storage_path("app/private/chunks/{$identifier}"); $directory = dirname($tempFile); @@ -87,7 +129,7 @@ public function storeChunked(StoreChunkedAssetRequest $request): JsonResponse mkdir($directory, 0755, true); } - file_put_contents($tempFile, $request->getContent(), $rangeStart === 0 ? 0 : FILE_APPEND); + file_put_contents($tempFile, $chunk, $rangeStart === 0 ? 0 : FILE_APPEND); $isLastChunk = ($rangeEnd + 1) >= $totalSize; @@ -98,9 +140,6 @@ public function storeChunked(StoreChunkedAssetRequest $request): JsonResponse ]); } - // Finalize may stream hundreds of MB to object storage (R2/S3). - set_time_limit(0); - $media = $workspace->addMediaFromPath($tempFile, $fileName, 'assets'); @unlink($tempFile); diff --git a/app/Models/Traits/HasMedia.php b/app/Models/Traits/HasMedia.php index e86ab6e0..f8ee4a48 100644 --- a/app/Models/Traits/HasMedia.php +++ b/app/Models/Traits/HasMedia.php @@ -164,6 +164,38 @@ public function addMediaFromPath(string $filePath, string $originalFilename, str ]); } + /** + * Register media that is already stored on the default disk (e.g. after a + * multipart cloud upload). + */ + public function addMediaFromStoredPath( + string $storagePath, + string $originalFilename, + string $mimeType, + int $size, + string $collection = 'default', + array $meta = [], + ?string $groupId = null, + ): Media { + if ($this->isSingleMediaCollection($collection)) { + $this->clearMediaCollection($collection); + } + + $type = $this->getMediaType($mimeType); + + return $this->media()->create([ + 'group_id' => $groupId ?? Str::uuid()->toString(), + 'collection' => $collection, + 'type' => $type, + 'path' => $storagePath, + 'original_filename' => $originalFilename, + 'mime_type' => $mimeType, + 'size' => $size, + 'order' => 0, + 'meta' => $meta, + ]); + } + public function clearMediaCollection(string $collection = 'default'): void { $this->getMedia($collection)->each(fn (Media $media) => $media->delete()); diff --git a/app/Services/Media/ChunkedCloudUploader.php b/app/Services/Media/ChunkedCloudUploader.php new file mode 100644 index 00000000..40e2004b --- /dev/null +++ b/app/Services/Media/ChunkedCloudUploader.php @@ -0,0 +1,208 @@ +diskName(); + + return config("filesystems.disks.{$disk}.driver") === 's3'; + } + + /** + * Images still need local assembly for format normalization. Videos and + * PDFs are safe to multipart directly to object storage. + */ + public function shouldUseMultipart(string $fileName): bool + { + $type = MediaType::fromExtension(pathinfo($fileName, PATHINFO_EXTENSION)); + + return in_array($type, [MediaType::Video, MediaType::Document], true); + } + + /** + * @return array{done: bool, progress: int, path?: string, size?: int, mime_type?: string} + */ + public function receiveChunk( + string $identifier, + string $fileName, + string $chunk, + int $rangeStart, + int $rangeEnd, + int $totalSize, + ): array { + $cacheKey = self::CACHE_PREFIX.$identifier; + $client = $this->s3(); + $bucket = $this->bucket(); + + if ($rangeStart === 0) { + $this->abortIfPresent($cacheKey); + $state = $this->startUpload($client, $bucket, $fileName, $chunk); + } else { + $state = $this->cache->get($cacheKey); + + if (! is_array($state)) { + throw new RuntimeException('Chunked cloud upload session expired or missing.'); + } + } + + $partNumber = count(data_get($state, 'parts', [])) + 1; + + $result = $client->uploadPart([ + 'Bucket' => $bucket, + 'Key' => data_get($state, 'key'), + 'UploadId' => data_get($state, 'upload_id'), + 'PartNumber' => $partNumber, + 'Body' => $chunk, + ]); + + $state['parts'][] = [ + 'ETag' => data_get($result, 'ETag'), + 'PartNumber' => $partNumber, + ]; + + $isLastChunk = ($rangeEnd + 1) >= $totalSize; + + if (! $isLastChunk) { + $this->cache->put($cacheKey, $state, now()->addHours(self::CACHE_TTL_HOURS)); + + return [ + 'done' => false, + 'progress' => (int) round(($rangeEnd + 1) / $totalSize * 100), + ]; + } + + $client->completeMultipartUpload([ + 'Bucket' => $bucket, + 'Key' => data_get($state, 'key'), + 'UploadId' => data_get($state, 'upload_id'), + 'MultipartUpload' => [ + 'Parts' => data_get($state, 'parts', []), + ], + ]); + + $this->cache->forget($cacheKey); + + return [ + 'done' => true, + 'progress' => 100, + 'path' => (string) data_get($state, 'key'), + 'size' => $totalSize, + 'mime_type' => (string) data_get($state, 'mime_type'), + ]; + } + + /** + * @return array{upload_id: string, key: string, mime_type: string, parts: array} + */ + private function startUpload(S3Client $client, string $bucket, string $fileName, string $firstChunk): array + { + $extension = strtolower((string) pathinfo($fileName, PATHINFO_EXTENSION)); + $key = 'medias/'.Str::uuid().'.'.$extension; + $mimeType = $this->detectMimeType($firstChunk, $extension); + + $created = $client->createMultipartUpload([ + 'Bucket' => $bucket, + 'Key' => $key, + 'ContentType' => $mimeType, + ]); + + return [ + 'upload_id' => (string) data_get($created, 'UploadId'), + 'key' => $key, + 'mime_type' => $mimeType, + 'parts' => [], + ]; + } + + private function abortIfPresent(string $cacheKey): void + { + $existing = $this->cache->get($cacheKey); + + if (! is_array($existing)) { + return; + } + + try { + $this->s3()->abortMultipartUpload([ + 'Bucket' => $this->bucket(), + 'Key' => data_get($existing, 'key'), + 'UploadId' => data_get($existing, 'upload_id'), + ]); + } catch (Throwable) { + // Best-effort cleanup of a previous incomplete upload. + } + + $this->cache->forget($cacheKey); + } + + private function detectMimeType(string $chunk, string $extension): string + { + $detected = (new finfo(FILEINFO_MIME_TYPE))->buffer($chunk) ?: null; + + if (is_string($detected) && $detected !== 'application/octet-stream') { + return $detected; + } + + return MediaType::fromExtension($extension)?->allowedMimeTypes()[0] + ?? 'application/octet-stream'; + } + + private function s3(): S3Client + { + if ($this->client instanceof S3Client) { + return $this->client; + } + + $adapter = Storage::disk($this->diskName()); + + if (! $adapter instanceof AwsS3V3Adapter) { + throw new RuntimeException('Chunked cloud uploads require an S3-compatible disk.'); + } + + return $adapter->getClient(); + } + + private function bucket(): string + { + if (filled($this->bucket)) { + return $this->bucket; + } + + return (string) config("filesystems.disks.{$this->diskName()}.bucket"); + } + + private function diskName(): string + { + return $this->disk ?? (string) config('filesystems.default'); + } +} diff --git a/tests/Feature/ChunkedCloudUploadTest.php b/tests/Feature/ChunkedCloudUploadTest.php new file mode 100644 index 00000000..59d4b04c --- /dev/null +++ b/tests/Feature/ChunkedCloudUploadTest.php @@ -0,0 +1,135 @@ +supports('local'))->toBeFalse(); + + config(['filesystems.disks.r2.driver' => 's3']); + expect($uploader->supports('r2'))->toBeTrue(); +}); + +test('chunked cloud uploader uses multipart for video and pdf only', function () { + $uploader = new ChunkedCloudUploader(Cache::store()); + + expect($uploader->shouldUseMultipart('clip.mp4'))->toBeTrue(); + expect($uploader->shouldUseMultipart('deck.pdf'))->toBeTrue(); + expect($uploader->shouldUseMultipart('photo.png'))->toBeFalse(); +}); + +test('chunked cloud uploader uploads parts and completes multipart', function () { + $client = Mockery::mock(S3Client::class); + + $client->shouldReceive('createMultipartUpload') + ->once() + ->andReturn(new Result(['UploadId' => 'upload-1'])); + + $client->shouldReceive('uploadPart') + ->twice() + ->andReturn(new Result(['ETag' => '"etag-a"']), new Result(['ETag' => '"etag-b"'])); + + $client->shouldReceive('completeMultipartUpload') + ->once() + ->withArgs(function (array $args) { + expect(data_get($args, 'UploadId'))->toBe('upload-1'); + expect(data_get($args, 'MultipartUpload.Parts'))->toHaveCount(2); + + return true; + }) + ->andReturn(new Result([])); + + $uploader = new ChunkedCloudUploader(Cache::store(), $client, 'test-bucket', 'r2'); + $chunk1 = str_repeat('a', 100); + $chunk2 = str_repeat('b', 50); + $total = strlen($chunk1) + strlen($chunk2); + + $mid = $uploader->receiveChunk('id-1', 'video.mp4', $chunk1, 0, strlen($chunk1) - 1, $total); + expect($mid)->toMatchArray(['done' => false]); + + $done = $uploader->receiveChunk( + 'id-1', + 'video.mp4', + $chunk2, + strlen($chunk1), + $total - 1, + $total, + ); + + expect($done['done'])->toBeTrue(); + expect($done['size'])->toBe($total); + expect($done['path'])->toStartWith('medias/'); + expect($done['path'])->toEndWith('.mp4'); + expect(Cache::get('chunked-cloud-upload:id-1'))->toBeNull(); +}); + +test('chunked asset upload uses cloud multipart for videos when disk is s3', function () { + $this->account = Account::factory()->create(); + $this->user = User::factory()->create(['account_id' => $this->account->id]); + $this->account->update(['owner_id' => $this->user->id]); + $this->workspace = Workspace::factory()->create([ + 'account_id' => $this->account->id, + 'user_id' => $this->user->id, + ]); + $this->workspace->members()->attach($this->user->id, ['role' => Role::Member->value]); + $this->user->update(['current_workspace_id' => $this->workspace->id]); + $this->account->subscriptions()->create([ + 'type' => Account::SUBSCRIPTION_NAME, + 'stripe_id' => 'sub_test_'.fake()->uuid(), + 'stripe_status' => 'active', + 'stripe_price' => 'price_123', + ]); + + $fake = Mockery::mock(ChunkedCloudUploader::class); + $fake->shouldReceive('supports')->andReturn(true); + $fake->shouldReceive('shouldUseMultipart')->with('clip.mp4')->andReturn(true); + $fake->shouldReceive('receiveChunk') + ->once() + ->andReturn([ + 'done' => true, + 'progress' => 100, + 'path' => 'medias/cloud-clip.mp4', + 'size' => 12, + 'mime_type' => 'video/mp4', + ]); + $this->app->instance(ChunkedCloudUploader::class, $fake); + + $content = 'fake-video!!'; + $size = strlen($content); + + $response = $this->actingAs($this->user)->call( + 'POST', + route('app.assets.store-chunked'), + [], [], [], + [ + 'HTTP_CONTENT_RANGE' => 'bytes 0-'.($size - 1).'/'.$size, + 'HTTP_X_FILE_NAME' => rawurlencode('clip.mp4'), + 'HTTP_ACCEPT' => 'application/json', + 'CONTENT_TYPE' => 'application/octet-stream', + ], + $content, + ); + + $response->assertSuccessful(); + $response->assertJson([ + 'done' => true, + 'path' => 'medias/cloud-clip.mp4', + 'type' => 'video', + ]); + expect($this->workspace->getMedia('assets')->first()->path)->toBe('medias/cloud-clip.mp4'); +});