From fb5d2e4f022a5ac8d517f9cbb158019b3fa58f99 Mon Sep 17 00:00:00 2001 From: Paulo Castellano Date: Thu, 16 Apr 2026 09:56:25 -0300 Subject: [PATCH] feat: add GenerateImage, GenerateVideo, GenerateAudio SDK tools MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Each tool implements Laravel\Ai\Contracts\Tool with description(), handle(Request), and schema(JsonSchema). They share a common pattern: - Constructor receives Workspace, optional Post, userId, and an optional AttachmentCollector (resolves from container if not injected). - handle() does the work (Image::of(), Audio::of(), or calls our custom VideoGenerationService for Veo), persists media + usage log, pushes the full attachment shape into the collector, and returns a short text summary to the LLM. - schema() exposes typed parameters with enum constraints and doc strings so the LLM selects valid inputs. Also fix the Ai::extend closure signature — MultipleInstanceManager passes (app, config) not just (config) to custom creators. --- app/Ai/Tools/GenerateAudio.php | 92 +++++++++++++++ app/Ai/Tools/GenerateImage.php | 112 +++++++++++++++++++ app/Ai/Tools/GenerateVideo.php | 77 +++++++++++++ app/Providers/AppServiceProvider.php | 6 +- tests/Feature/Ai/Tools/GenerateAudioTest.php | 65 +++++++++++ tests/Feature/Ai/Tools/GenerateImageTest.php | 88 +++++++++++++++ tests/Feature/Ai/Tools/GenerateVideoTest.php | 65 +++++++++++ 7 files changed, 502 insertions(+), 3 deletions(-) create mode 100644 app/Ai/Tools/GenerateAudio.php create mode 100644 app/Ai/Tools/GenerateImage.php create mode 100644 app/Ai/Tools/GenerateVideo.php create mode 100644 tests/Feature/Ai/Tools/GenerateAudioTest.php create mode 100644 tests/Feature/Ai/Tools/GenerateImageTest.php create mode 100644 tests/Feature/Ai/Tools/GenerateVideoTest.php diff --git a/app/Ai/Tools/GenerateAudio.php b/app/Ai/Tools/GenerateAudio.php new file mode 100644 index 00000000..e807f8bb --- /dev/null +++ b/app/Ai/Tools/GenerateAudio.php @@ -0,0 +1,92 @@ +collector ??= app(AttachmentCollector::class); + } + + public function description(): Stringable|string + { + return <<<'TXT' +Generate an AI voiceover or narration audio from text. + +Call this when the user asks for a voiceover, narration, TTS, podcast clip, +or any voice content. The output language will match the language of the +input text — so pass the text in the target language. + +Audio is generated via ElevenLabs with the configured default voice. +TXT; + } + + public function handle(Request $request): Stringable|string + { + $text = (string) $request['text']; + $voice = config('services.elevenlabs.default_voice', 'EXAVITQu4vr4xnSDxMaL'); + + $response = Audio::of($text)->voice($voice)->generate(); + + $storedPath = $response->store('medias', 'public'); + + $media = $this->workspace->media()->create([ + 'group_id' => Str::uuid()->toString(), + 'collection' => 'assets', + 'type' => 'video', + 'path' => $storedPath, + 'original_filename' => 'ai-generated.mp3', + 'mime_type' => 'audio/mpeg', + 'size' => Storage::disk('public')->size($storedPath), + 'order' => 0, + 'meta' => ['ai_generated' => true, 'text' => Str::limit($text, 200)], + ]); + + AiUsageLog::create([ + 'account_id' => $this->workspace->account_id, + 'workspace_id' => $this->workspace->id, + 'user_id' => $this->userId, + 'post_id' => $this->post?->id, + 'type' => UsageType::Audio, + 'provider' => 'elevenlabs', + ]); + + $this->collector->push([ + 'id' => $media->id, + 'path' => $media->path, + 'url' => $media->url, + 'mime_type' => 'audio/mpeg', + 'type' => 'audio', + ]); + + return "Generated audio (id: {$media->id}) and attached it to the post."; + } + + public function schema(JsonSchema $schema): array + { + return [ + 'text' => $schema->string() + ->description('The text to convert into speech. Write it in the target language — the output voice language will match.') + ->required(), + ]; + } +} diff --git a/app/Ai/Tools/GenerateImage.php b/app/Ai/Tools/GenerateImage.php new file mode 100644 index 00000000..03198e9b --- /dev/null +++ b/app/Ai/Tools/GenerateImage.php @@ -0,0 +1,112 @@ +collector ??= app(AttachmentCollector::class); + } + + public function description(): Stringable|string + { + return <<<'TXT' +Generate an AI image and attach it to the current post. + +Call this when the user asks for an image, photo, carousel slide, or any visual content. +Pass a detailed visual prompt describing what to generate, and an orientation: +- "vertical" (9:16) for Instagram Reel/Story, Pinterest, TikTok, YouTube Shorts +- "horizontal" (16:9) for X/Twitter, LinkedIn, Facebook + +The image is generated, stored on the public disk, registered in the workspace's +media library, logged in monthly usage tracking, and attached to the assistant's +response message. +TXT; + } + + public function handle(Request $request): Stringable|string + { + $prompt = (string) $request['prompt']; + $orientationString = (string) ($request['orientation'] ?? 'vertical'); + $orientationEnum = Orientation::tryFrom($orientationString) ?? Orientation::Vertical; + $aspectRatio = $orientationEnum->aspectRatio(); + + $renderedPrompt = view('prompts.assistant.image', [ + 'prompt' => $prompt, + 'brand_name' => $this->workspace->name ?? '', + 'tone' => $this->workspace->brand_tone ?? 'professional', + 'aspect_ratio' => $aspectRatio, + ])->render(); + + $response = Image::of($renderedPrompt) + ->size($aspectRatio) + ->quality('high') + ->generate(); + + $storedPath = $response->store('medias', 'public'); + + $media = $this->workspace->media()->create([ + 'group_id' => Str::uuid()->toString(), + 'collection' => 'assets', + 'type' => 'image', + 'path' => $storedPath, + 'original_filename' => 'ai-generated.png', + 'mime_type' => 'image/png', + 'size' => Storage::disk('public')->size($storedPath), + 'order' => 0, + 'meta' => ['ai_generated' => true, 'prompt' => Str::limit($prompt, 200)], + ]); + + AiUsageLog::create([ + 'account_id' => $this->workspace->account_id, + 'workspace_id' => $this->workspace->id, + 'user_id' => $this->userId, + 'post_id' => $this->post?->id, + 'type' => UsageType::Image, + 'provider' => 'gemini', + ]); + + $this->collector->push([ + 'id' => $media->id, + 'path' => $media->path, + 'url' => $media->url, + 'mime_type' => 'image/png', + 'type' => 'image', + ]); + + return "Generated a {$orientationString} image (id: {$media->id}) and attached it to the post."; + } + + public function schema(JsonSchema $schema): array + { + return [ + 'prompt' => $schema->string() + ->description('A detailed visual description of the image to generate. Include subject, style, composition, mood, and any text that should appear in the image.') + ->required(), + 'orientation' => $schema->string() + ->enum(['vertical', 'horizontal']) + ->description('"vertical" for 9:16 (Instagram Reel/Story, TikTok, YouTube Shorts, Pinterest). "horizontal" for 16:9 (X/Twitter, LinkedIn, Facebook).') + ->required(), + ]; + } +} diff --git a/app/Ai/Tools/GenerateVideo.php b/app/Ai/Tools/GenerateVideo.php new file mode 100644 index 00000000..601745a8 --- /dev/null +++ b/app/Ai/Tools/GenerateVideo.php @@ -0,0 +1,77 @@ +collector ??= app(AttachmentCollector::class); + } + + public function description(): Stringable|string + { + return <<<'TXT' +Generate a short AI video and attach it to the current post. + +Call this when the user asks for a video, Reel, TikTok, YouTube Short, +Facebook Reel, or any motion content. Pass a detailed visual description +and an orientation: +- "vertical" (9:16) for TikTok, Instagram Reel, YouTube Shorts, Facebook Reel, Pinterest Video Pin +- "horizontal" (16:9) for X/Twitter video, LinkedIn video + +Videos are generated via Veo 3.1 (not yet supported by the SDK's Image/Audio +entry points), which this tool wraps internally. +TXT; + } + + public function handle(Request $request): Stringable|string + { + $prompt = (string) $request['prompt']; + $orientationString = (string) ($request['orientation'] ?? 'vertical'); + $orientationEnum = Orientation::tryFrom($orientationString) ?? Orientation::Vertical; + + /** @var VideoGenerationService $service */ + $service = app(VideoGenerationService::class); + + $attachment = $service->generate( + prompt: $prompt, + workspace: $this->workspace, + userId: $this->userId, + postId: $this->post?->id, + orientation: $orientationEnum, + ); + + $this->collector->push($attachment); + + return "Generated a {$orientationString} video (id: {$attachment['id']}) and attached it to the post."; + } + + public function schema(JsonSchema $schema): array + { + return [ + 'prompt' => $schema->string() + ->description('A detailed visual description of the video to generate. Include subject, motion, style, mood, and any key moments.') + ->required(), + 'orientation' => $schema->string() + ->enum(['vertical', 'horizontal']) + ->description('"vertical" for 9:16 (TikTok, Reels, Shorts, Stories). "horizontal" for 16:9 (X, LinkedIn, Facebook).') + ->required(), + ]; + } +} diff --git a/app/Providers/AppServiceProvider.php b/app/Providers/AppServiceProvider.php index 794a1939..3d8f4cde 100644 --- a/app/Providers/AppServiceProvider.php +++ b/app/Providers/AppServiceProvider.php @@ -109,11 +109,11 @@ public function boot(): void protected function configureAi(): void { - Ai::extend('gemini', function (array $config) { + Ai::extend('gemini', function ($app, array $config) { return new ExtendedGeminiProvider( - new GeminiGateway($this->app['events']), + new GeminiGateway($app['events']), $config, - $this->app->make(Dispatcher::class), + $app->make(Dispatcher::class), ); }); } diff --git a/tests/Feature/Ai/Tools/GenerateAudioTest.php b/tests/Feature/Ai/Tools/GenerateAudioTest.php new file mode 100644 index 00000000..e52cc305 --- /dev/null +++ b/tests/Feature/Ai/Tools/GenerateAudioTest.php @@ -0,0 +1,65 @@ +user = User::factory()->create(['setup' => Setup::Completed]); + $this->workspace = Workspace::factory()->create(['user_id' => $this->user->id]); + $this->workspace->members()->attach($this->user->id, ['role' => Role::Member->value]); + $this->post = Post::factory()->create([ + 'workspace_id' => $this->workspace->id, + 'user_id' => $this->user->id, + ]); + app(AttachmentCollector::class)->clear(); +}); + +test('tool generates audio, pushes attachment to collector, creates usage log', function () { + Audio::fake(); + + $tool = new GenerateAudio( + workspace: $this->workspace, + post: $this->post, + userId: $this->user->id, + ); + + $summary = $tool->handle(new ToolRequest([ + 'text' => 'Welcome to my channel', + ])); + + expect((string) $summary)->toContain('audio'); + expect(app(AttachmentCollector::class)->all())->toHaveCount(1); + expect(app(AttachmentCollector::class)->all()[0]['type'])->toBe('audio'); + expect(app(AttachmentCollector::class)->all()[0]['mime_type'])->toBe('audio/mpeg'); + + Audio::assertGenerated(fn ($prompt) => true); + + $this->assertDatabaseHas('workspace_ai_usages', [ + 'workspace_id' => $this->workspace->id, + 'type' => UsageType::Audio->value, + 'provider' => 'elevenlabs', + ]); +}); + +test('tool exposes schema with text parameter', function () { + $tool = new GenerateAudio( + workspace: $this->workspace, + post: $this->post, + userId: $this->user->id, + ); + + $schema = $tool->schema(new JsonSchemaTypeFactory); + + expect($schema)->toHaveKey('text'); +}); diff --git a/tests/Feature/Ai/Tools/GenerateImageTest.php b/tests/Feature/Ai/Tools/GenerateImageTest.php new file mode 100644 index 00000000..9139d566 --- /dev/null +++ b/tests/Feature/Ai/Tools/GenerateImageTest.php @@ -0,0 +1,88 @@ +user = User::factory()->create(['setup' => Setup::Completed]); + $this->workspace = Workspace::factory()->create(['user_id' => $this->user->id]); + $this->workspace->members()->attach($this->user->id, ['role' => Role::Member->value]); + $this->post = Post::factory()->create([ + 'workspace_id' => $this->workspace->id, + 'user_id' => $this->user->id, + ]); + app(AttachmentCollector::class)->clear(); +}); + +test('tool generates image, pushes attachment to collector, creates usage log', function () { + Image::fake(); + + $tool = new GenerateImage( + workspace: $this->workspace, + post: $this->post, + userId: $this->user->id, + ); + + $summary = $tool->handle(new ToolRequest([ + 'prompt' => 'A sunset beach', + 'orientation' => 'vertical', + ])); + + expect((string) $summary)->toContain('image'); + + $collector = app(AttachmentCollector::class); + expect($collector->all())->toHaveCount(1); + + $attachment = $collector->all()[0]; + expect($attachment)->toHaveKeys(['id', 'path', 'url', 'mime_type', 'type']); + expect($attachment['type'])->toBe('image'); + expect($attachment['mime_type'])->toBe('image/png'); + + Image::assertGenerated(fn ($prompt) => true); + + $this->assertDatabaseHas('workspace_ai_usages', [ + 'workspace_id' => $this->workspace->id, + 'type' => UsageType::Image->value, + 'provider' => 'gemini', + ]); +}); + +test('tool defaults to vertical when orientation is unknown', function () { + Image::fake(); + + $tool = new GenerateImage( + workspace: $this->workspace, + post: $this->post, + userId: $this->user->id, + ); + + $tool->handle(new ToolRequest([ + 'prompt' => 'A forest', + 'orientation' => 'gibberish', + ])); + + expect(app(AttachmentCollector::class)->all()[0]['type'])->toBe('image'); +}); + +test('tool exposes schema with prompt and orientation parameters', function () { + $tool = new GenerateImage( + workspace: $this->workspace, + post: $this->post, + userId: $this->user->id, + ); + + $schema = $tool->schema(new JsonSchemaTypeFactory); + + expect($schema)->toHaveKeys(['prompt', 'orientation']); +}); diff --git a/tests/Feature/Ai/Tools/GenerateVideoTest.php b/tests/Feature/Ai/Tools/GenerateVideoTest.php new file mode 100644 index 00000000..2f801fb9 --- /dev/null +++ b/tests/Feature/Ai/Tools/GenerateVideoTest.php @@ -0,0 +1,65 @@ +user = User::factory()->create(['setup' => Setup::Completed]); + $this->workspace = Workspace::factory()->create(['user_id' => $this->user->id]); + $this->workspace->members()->attach($this->user->id, ['role' => Role::Member->value]); + $this->post = Post::factory()->create([ + 'workspace_id' => $this->workspace->id, + 'user_id' => $this->user->id, + ]); + app(AttachmentCollector::class)->clear(); +}); + +test('tool delegates to VideoGenerationService and pushes attachment to collector', function () { + $expected = [ + 'id' => 'video-uuid', + 'path' => 'medias/video.mp4', + 'url' => 'https://example.com/medias/video.mp4', + 'mime_type' => 'video/mp4', + 'type' => 'video', + ]; + + $mock = $this->mock(VideoGenerationService::class); + $mock->shouldReceive('generate')->once()->andReturn($expected); + + $tool = new GenerateVideo( + workspace: $this->workspace, + post: $this->post, + userId: $this->user->id, + ); + + $summary = $tool->handle(new ToolRequest([ + 'prompt' => 'A cat dancing', + 'orientation' => 'vertical', + ])); + + expect((string) $summary)->toContain('video'); + expect(app(AttachmentCollector::class)->all())->toHaveCount(1); + expect(app(AttachmentCollector::class)->all()[0])->toBe($expected); +}); + +test('tool exposes schema with prompt and orientation parameters', function () { + $tool = new GenerateVideo( + workspace: $this->workspace, + post: $this->post, + userId: $this->user->id, + ); + + $schema = $tool->schema(new JsonSchemaTypeFactory); + + expect($schema)->toHaveKeys(['prompt', 'orientation']); +});