Merge pull request #116 from trypostit/fix/x-video-upload-oom
fix(social): stop chunked video uploads from exhausting memory
This commit is contained in:
commit
05faa49c30
5 changed files with 137 additions and 1 deletions
|
|
@ -370,6 +370,9 @@ private function doUploadVideo(string $tempFile, $mediaItem): ?string
|
|||
if ($etag) {
|
||||
$uploadedPartIds[] = $etag;
|
||||
}
|
||||
|
||||
unset($chunkData, $chunkResponse);
|
||||
$this->freeChunkMemory();
|
||||
}
|
||||
} finally {
|
||||
fclose($handle);
|
||||
|
|
|
|||
|
|
@ -32,7 +32,7 @@ protected function socialHttp(): PendingRequest
|
|||
return Http::retry(
|
||||
times: 3,
|
||||
sleepMilliseconds: 5000,
|
||||
when: fn ($exception, $request) => $exception->response?->status() === 429,
|
||||
when: fn ($exception, $request) => ($exception->response ?? null)?->status() === 429,
|
||||
throw: false,
|
||||
)->timeout(120);
|
||||
}
|
||||
|
|
@ -41,4 +41,16 @@ protected function redactResponseBody(string $body): string
|
|||
{
|
||||
return TokenRedactor::redact($body);
|
||||
}
|
||||
|
||||
/**
|
||||
* Reclaim memory between chunks of a streaming upload. Each request's body
|
||||
* and the HTTP client's request/response objects are held alive by reference
|
||||
* cycles the PHP runtime only frees when the cycle collector runs, so a
|
||||
* long chunked upload accumulates the whole file in memory without this.
|
||||
* Callers must `unset()` the chunk and response first.
|
||||
*/
|
||||
protected function freeChunkMemory(): void
|
||||
{
|
||||
gc_collect_cycles();
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -236,6 +236,9 @@ private function chunkedUpload(string $tempFile, int $totalBytes, string $mimeTy
|
|||
}
|
||||
|
||||
$index++;
|
||||
|
||||
unset($chunk, $appendResponse);
|
||||
$this->freeChunkMemory();
|
||||
}
|
||||
} finally {
|
||||
fclose($handle);
|
||||
|
|
|
|||
|
|
@ -431,6 +431,75 @@
|
|||
Http::assertSent(fn ($request) => str_contains($request->url(), '/rest/posts'));
|
||||
});
|
||||
|
||||
test('linkedin publisher uploads a video across multiple chunks', function () {
|
||||
$this->post->update([
|
||||
'media' => [
|
||||
[
|
||||
'id' => 'test-media-video',
|
||||
'path' => 'media/2026-01/test-video.mp4',
|
||||
'url' => 'https://example.com/media/2026-01/test-video.mp4',
|
||||
'mime_type' => 'video/mp4',
|
||||
'original_filename' => 'test-video.mp4',
|
||||
],
|
||||
],
|
||||
]);
|
||||
|
||||
$chunkBase = 'https://www.linkedin.com/dms/upload/v2/chunk/video/';
|
||||
$chunkPuts = 0;
|
||||
|
||||
Http::fake(function ($request) use ($chunkBase, &$chunkPuts) {
|
||||
$url = $request->url();
|
||||
|
||||
if (str_contains($url, 'initializeUpload') && str_contains($url, '/rest/videos')) {
|
||||
return Http::response([
|
||||
'value' => [
|
||||
'video' => 'urn:li:video:FakeVideoUrn',
|
||||
'uploadToken' => 'upload-token-abc',
|
||||
'uploadInstructions' => [
|
||||
['uploadUrl' => $chunkBase.'0', 'firstByte' => 0, 'lastByte' => 1023],
|
||||
['uploadUrl' => $chunkBase.'1', 'firstByte' => 1024, 'lastByte' => 2047],
|
||||
['uploadUrl' => $chunkBase.'2', 'firstByte' => 2048, 'lastByte' => 3071],
|
||||
],
|
||||
],
|
||||
], 200);
|
||||
}
|
||||
|
||||
if (str_starts_with($url, $chunkBase)) {
|
||||
$chunkPuts++;
|
||||
|
||||
return Http::response(null, 200, ['etag' => '"etag-'.$chunkPuts.'"']);
|
||||
}
|
||||
|
||||
if (str_contains($url, 'finalizeUpload') && str_contains($url, '/rest/videos')) {
|
||||
return Http::response(null, 200);
|
||||
}
|
||||
|
||||
if (str_contains($url, '/rest/videos/')) {
|
||||
return Http::response(['status' => 'AVAILABLE'], 200);
|
||||
}
|
||||
|
||||
if (str_contains($url, '/rest/posts')) {
|
||||
return Http::response(null, 201, ['x-restli-id' => 'urn:li:share:multichunk']);
|
||||
}
|
||||
|
||||
return Http::response(str_repeat('x', 4096), 200);
|
||||
});
|
||||
|
||||
$result = $this->publisher->publish($this->postPlatform);
|
||||
|
||||
expect($result['id'])->toBe('urn:li:share:multichunk');
|
||||
expect($chunkPuts)->toBe(3);
|
||||
|
||||
// All three chunk etags are collected and sent to finalize.
|
||||
Http::assertSent(function ($request) {
|
||||
if (! str_contains($request->url(), 'finalizeUpload')) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return count(data_get($request->data(), 'finalizeUploadRequest.uploadedPartIds', [])) === 3;
|
||||
});
|
||||
});
|
||||
|
||||
test('linkedin publisher can publish a document (pdf carousel) with a title', function () {
|
||||
$this->postPlatform->update([
|
||||
'content_type' => ContentType::LinkedInPost,
|
||||
|
|
|
|||
|
|
@ -304,3 +304,52 @@
|
|||
Http::assertSent(fn ($request) => str_contains($request->url(), '/finalize'));
|
||||
Http::assertSent(fn ($request) => str_contains($request->url(), '/2/tweets'));
|
||||
});
|
||||
|
||||
test('x publisher uploads a large video in sequential 1MB segments', function () {
|
||||
$this->post->update([
|
||||
'media' => [
|
||||
[
|
||||
'id' => 'test-media-video',
|
||||
'path' => 'media/2026-01/big-video.mp4',
|
||||
'url' => 'https://example.com/media/2026-01/big-video.mp4',
|
||||
'mime_type' => 'video/mp4',
|
||||
'original_filename' => 'big-video.mp4',
|
||||
],
|
||||
],
|
||||
]);
|
||||
|
||||
$appendCount = 0;
|
||||
|
||||
Http::fake(function ($request) use (&$appendCount) {
|
||||
$url = $request->url();
|
||||
|
||||
if (str_contains($url, '/2/media/upload/initialize')) {
|
||||
return Http::response(['data' => ['id' => 'media_id_999']], 200);
|
||||
}
|
||||
|
||||
if (str_contains($url, '/append')) {
|
||||
$appendCount++;
|
||||
|
||||
return Http::response(null, 204);
|
||||
}
|
||||
|
||||
if (str_contains($url, '/finalize')) {
|
||||
return Http::response(['data' => ['id' => 'media_id_999']], 200);
|
||||
}
|
||||
|
||||
if (str_contains($url, '/2/media/')) {
|
||||
return Http::response(['processing_info' => ['state' => 'succeeded']], 200);
|
||||
}
|
||||
|
||||
if (str_contains($url, '/2/tweets')) {
|
||||
return Http::response(['data' => ['id' => '9876543210987654321', 'text' => 'Hello from X!']], 200);
|
||||
}
|
||||
|
||||
// ~2.5MB download -> at least three 1MB segments.
|
||||
return Http::response(str_repeat('x', (int) (2.5 * 1024 * 1024)), 200);
|
||||
});
|
||||
|
||||
$this->publisher->publish($this->postPlatform);
|
||||
|
||||
expect($appendCount)->toBeGreaterThan(1);
|
||||
});
|
||||
|
|
|
|||
Loading…
Reference in a new issue