diff --git a/app/Actions/Automation/Node/RunFetchRssNode.php b/app/Actions/Automation/Node/RunFetchRssNode.php index cc0b02b3..a8394ee7 100644 --- a/app/Actions/Automation/Node/RunFetchRssNode.php +++ b/app/Actions/Automation/Node/RunFetchRssNode.php @@ -10,11 +10,11 @@ use App\Models\AutomationNodeState; use App\Models\AutomationRun; use App\Services\Automation\ExpressionResolver; +use App\Services\Automation\FeedParser; use App\Services\Brand\SafeHttpFetcher; use Carbon\CarbonImmutable; use Illuminate\Support\Facades\Http; use RuntimeException; -use SimpleXMLElement; use Throwable; /** @@ -40,6 +40,7 @@ public function __construct( private ExpressionResolver $resolver, private SafeHttpFetcher $safeHttp, private AdvanceAutomationRun $advance, + private FeedParser $parser, ) {} public function __invoke(AutomationRun $run, array $config): NodeRunResult @@ -65,9 +66,9 @@ public function __invoke(AutomationRun $run, array $config): NodeRunResult return NodeRunResult::failed(__('automations.errors.fetch_rss_request_failed'), ['status' => $response->status()]); } - try { - $xml = new SimpleXMLElement($response->body()); - } catch (Throwable) { + $items = $this->parser->parse($response->body()); + + if ($items === null) { return NodeRunResult::failed(__('automations.errors.fetch_rss_malformed')); } @@ -83,7 +84,7 @@ public function __invoke(AutomationRun $run, array $config): NodeRunResult ? CarbonImmutable::createFromTimestamp(0) : $this->parseWatermark(data_get($state->data, 'last_item_date')); - [$newItems, $newestSeen] = $this->collectNewItems($xml, $watermark); + [$newItems, $newestSeen] = $this->collectNewItems($items, $watermark); if ($state !== null && $newestSeen !== null) { $state->update(['data' => array_merge($state->data ?? [], [ @@ -107,46 +108,44 @@ public function __invoke(AutomationRun $run, array $config): NodeRunResult ]); } - private function collectNewItems(SimpleXMLElement $xml, CarbonImmutable $watermark): array + /** + * @param list> $parsed Normalized items from FeedParser. + * @return array{0: list>, 1: ?CarbonImmutable} + */ + private function collectNewItems(array $parsed, CarbonImmutable $watermark): array { $items = []; $newestSeen = null; - foreach ($xml->channel->item ?? [] as $item) { - $key = (string) ($item->guid ?? $item->link); + foreach ($parsed as $item) { + $key = (string) data_get($item, 'key', ''); if ($key === '') { continue; } - $pubDate = $this->parsePubDate((string) $item->pubDate); - if ($pubDate === null) { + $date = $this->parsePubDate((string) data_get($item, 'date', '')); + if ($date === null) { continue; } - if ($newestSeen === null || $pubDate->greaterThan($newestSeen)) { - $newestSeen = $pubDate; + if ($newestSeen === null || $date->greaterThan($newestSeen)) { + $newestSeen = $date; } - if (! $pubDate->greaterThan($watermark)) { + if (! $date->greaterThan($watermark)) { continue; } - $items[] = [ - '_pubDate' => $pubDate, - 'key' => $key, - 'title' => (string) $item->title, - 'link' => (string) $item->link, - 'description' => (string) $item->description, - 'pubDate' => (string) $item->pubDate, - ]; + $item['_sort'] = $date->getTimestamp(); + $items[] = $item; } // Process oldest-first so siblings inherit a stable order matching feed chronology. - usort($items, fn ($a, $b) => $a['_pubDate']->getTimestamp() <=> $b['_pubDate']->getTimestamp()); + usort($items, fn ($a, $b) => $a['_sort'] <=> $b['_sort']); // Drop the internal sort key — downstream nodes shouldn't see it. $items = array_map(function (array $item): array { - unset($item['_pubDate']); + unset($item['_sort']); return $item; }, $items); diff --git a/app/Http/Controllers/App/AutomationController.php b/app/Http/Controllers/App/AutomationController.php index cfd6b898..b31c74c4 100644 --- a/app/Http/Controllers/App/AutomationController.php +++ b/app/Http/Controllers/App/AutomationController.php @@ -17,11 +17,13 @@ use App\Actions\Automation\Run\TestAutomation; use App\Http\Controllers\Controller; use App\Http\Requests\App\Automations\ActivateAutomationRequest; +use App\Http\Requests\App\Automations\InspectFeedRequest; use App\Http\Requests\App\Automations\PauseAutomationRequest; use App\Http\Requests\App\Automations\RetryRunRequest; use App\Http\Requests\App\Automations\StoreAutomationRequest; use App\Http\Requests\App\Automations\TestAutomationRequest; use App\Http\Requests\App\Automations\UpdateAutomationRequest; +use App\Http\Resources\App\Automation\FeedInspectionResource; use App\Http\Resources\App\PlatformConfigResource; use App\Http\Resources\App\SocialAccountResource; use App\Http\Resources\AutomationInvocationResource; @@ -31,11 +33,17 @@ use App\Models\Automation; use App\Models\AutomationNodeRun; use App\Models\AutomationRun; +use App\Services\Automation\ExpressionResolver; +use App\Services\Automation\FeedParser; +use App\Services\Brand\SafeHttpFetcher; use Illuminate\Http\JsonResponse; use Illuminate\Http\RedirectResponse; use Illuminate\Http\Response as HttpResponse; +use Illuminate\Support\Facades\Http; use Inertia\Inertia; use Inertia\Response; +use RuntimeException; +use Symfony\Component\HttpFoundation\Response as SymfonyResponse; class AutomationController extends Controller { @@ -203,6 +211,41 @@ public function test(TestAutomationRequest $request, Automation $automation, Tes return response()->json(['run_id' => $run->id]); } + public function inspectFeed( + InspectFeedRequest $request, + Automation $automation, + ExpressionResolver $resolver, + SafeHttpFetcher $safeHttp, + FeedParser $parser, + ): JsonResponse|FeedInspectionResource { + $this->authorize('update', $automation); + + $feedUrl = $resolver->resolve( + (string) $request->validated('feed_url'), + ['variables' => $automation->resolvedVariables()], + ); + + try { + $safeHttp->guardAgainstSsrf($feedUrl); + } catch (RuntimeException) { + return response()->json(['message' => __('automations.errors.url_not_allowed')], SymfonyResponse::HTTP_UNPROCESSABLE_ENTITY); + } + + $response = Http::get($feedUrl); + + if (! $response->successful()) { + return response()->json(['message' => __('automations.errors.fetch_rss_request_failed')], SymfonyResponse::HTTP_UNPROCESSABLE_ENTITY); + } + + $items = $parser->parse($response->body()); + + if ($items === null) { + return response()->json(['message' => __('automations.errors.fetch_rss_malformed')], SymfonyResponse::HTTP_UNPROCESSABLE_ENTITY); + } + + return new FeedInspectionResource($items[0] ?? []); + } + public function showRun(Automation $automation, AutomationRun $run): JsonResponse { $this->authorize('view', $automation); diff --git a/app/Http/Requests/App/Automations/InspectFeedRequest.php b/app/Http/Requests/App/Automations/InspectFeedRequest.php new file mode 100644 index 00000000..e3d878e5 --- /dev/null +++ b/app/Http/Requests/App/Automations/InspectFeedRequest.php @@ -0,0 +1,25 @@ + + */ + public function rules(): array + { + return [ + 'feed_url' => ['required', 'string', 'max:2048'], + ]; + } +} diff --git a/app/Http/Requests/App/Automations/UpdateAutomationRequest.php b/app/Http/Requests/App/Automations/UpdateAutomationRequest.php index bae5eb20..ed94fa4b 100644 --- a/app/Http/Requests/App/Automations/UpdateAutomationRequest.php +++ b/app/Http/Requests/App/Automations/UpdateAutomationRequest.php @@ -12,6 +12,7 @@ use App\Enums\Automation\Publish\Mode as PublishMode; use App\Enums\Automation\ScheduleField; use App\Enums\Automation\Trigger\Type as TriggerType; +use App\Rules\ResolvableUrl; use App\Services\Automation\AutomationConfigValidator; use App\Services\Automation\GenerateNodeValidator; use Illuminate\Contracts\Validation\Validator; @@ -129,7 +130,10 @@ private function dataRulesForNodeType(?string $type, int $i): array 'schedule_timezone' => ['sometimes', 'string', 'timezone'], ], NodeType::FetchRss->value => [ - 'feed_url' => ['required', 'url'], + 'feed_url' => ['required', new ResolvableUrl], + 'discovered_fields' => ['sometimes', 'array'], + 'discovered_fields.*.path' => ['required', 'string'], + 'discovered_fields.*.sample' => ['nullable', 'string'], ], NodeType::HttpRequest->value => [ 'url' => ['required', 'url'], diff --git a/app/Http/Resources/App/Automation/FeedInspectionResource.php b/app/Http/Resources/App/Automation/FeedInspectionResource.php new file mode 100644 index 00000000..7aa2aed0 --- /dev/null +++ b/app/Http/Resources/App/Automation/FeedInspectionResource.php @@ -0,0 +1,69 @@ + $resource The first normalized feed item. + */ +class FeedInspectionResource extends JsonResource +{ + private const SAMPLE_MAX_LENGTH = 120; + + /** + * @return array{fields: list} + */ + public function toArray(Request $request): array + { + return [ + 'fields' => $this->flatten((array) $this->resource), + ]; + } + + /** + * @param array $item + * @return list + */ + private function flatten(array $item, string $prefix = 'fetched'): array + { + $fields = []; + + foreach ($item as $key => $value) { + $path = "{$prefix}.{$key}"; + + if (is_array($value) && $this->isAssoc($value)) { + $fields = array_merge($fields, $this->flatten($value, $path)); + + continue; + } + + $fields[] = ['path' => $path, 'sample' => $this->sample($value)]; + } + + return $fields; + } + + /** + * @param array $value + */ + private function isAssoc(array $value): bool + { + return $value !== [] && ! array_is_list($value); + } + + private function sample(mixed $value): string + { + $text = is_array($value) + ? (json_encode($value, JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE) ?: '') + : (string) $value; + + return str($text)->squish()->limit(self::SAMPLE_MAX_LENGTH)->value(); + } +} diff --git a/app/Rules/ResolvableUrl.php b/app/Rules/ResolvableUrl.php new file mode 100644 index 00000000..af5db97d --- /dev/null +++ b/app/Rules/ResolvableUrl.php @@ -0,0 +1,36 @@ +translate(); + + return; + } + + $candidate = preg_replace('/\{\{\s*[\w.]+\s*\}\}/', 'placeholder', $value); + + if (filter_var($candidate, FILTER_VALIDATE_URL) === false) { + $fail('validation.url')->translate(); + } + } +} diff --git a/app/Services/Automation/FeedParser.php b/app/Services/Automation/FeedParser.php new file mode 100644 index 00000000..52bc9ae6 --- /dev/null +++ b/app/Services/Automation/FeedParser.php @@ -0,0 +1,179 @@ + }}`. + * + * Aliases win on key collisions. The HTTP fetch and SSRF guard stay in the caller; + * the body is handed to SimplePie via `set_raw_data()` so the SSRF guard isn't bypassed. + */ +class FeedParser +{ + /** + * Namespace URI → short prefix for the raw layer. + */ + private const NS_PREFIX = [ + 'http://search.yahoo.com/mrss/' => 'media', + 'http://www.youtube.com/xml/schemas/2015' => 'yt', + 'http://purl.org/dc/elements/1.1/' => 'dc', + 'http://purl.org/dc/terms/' => 'dc', + 'http://purl.org/rss/1.0/modules/content/' => 'content', + 'http://www.itunes.com/dtds/podcast-1.0.dtd' => 'itunes', + 'https://podcastindex.org/namespace/1.0' => 'podcast', + 'http://purl.org/rss/1.0/modules/slash/' => 'slash', + 'http://wellformedweb.org/CommentAPI/' => 'wfw', + 'http://www.georss.org/georss' => 'georss', + ]; + + /** + * Namespaces treated as the feed "core" — their tags get no prefix, so RSS and + * Atom land on the same raw key names (and are then overridden by aliases). + */ + private const NS_CORE = [ + '', + 'http://www.w3.org/2005/Atom', + 'http://purl.org/rss/1.0/', + 'http://backend.userland.com/rss2', + 'http://my.netscape.com/rdf/simple/0.9/', + ]; + + /** + * @return list>|null Null when the body is not a valid feed. + */ + public function parse(string $body): ?array + { + $feed = new SimplePie; + $feed->enable_cache(false); + $feed->set_raw_data($body); + + if (! @$feed->init()) { + return null; + } + + return array_map(fn (Item $item): array => $this->normalize($item), $feed->get_items()); + } + + /** + * @return array + */ + private function normalize(Item $item): array + { + $enclosure = $item->get_enclosure(); + + $aliases = [ + 'key' => $item->get_id(false, false) ?: $item->get_permalink(), + 'title' => $item->get_title(), + 'link' => $item->get_permalink(), + 'date' => $item->get_date('c'), + 'pubDate' => $item->get_date('c'), + 'content' => $item->get_content(), + 'description' => $item->get_description(), + 'author' => $item->get_author()?->get_name(), + 'id' => $item->get_id(false, false), + 'image' => $enclosure?->get_thumbnail() ?: null, + 'categories' => array_values(array_filter(array_map( + fn ($category) => $category->get_label(), + $item->get_categories() ?? [], + ))), + 'enclosure' => $enclosure === null ? null : array_filter([ + 'url' => $enclosure->get_link(), + 'type' => $enclosure->get_type(), + 'length' => $enclosure->get_length(), + ], fn ($value) => $value !== null), + ]; + + // Aliases win on collision, so merge them over the raw layer. + return array_merge($this->rawFields($item), $aliases); + } + + /** + * @return array + */ + private function rawFields(Item $item): array + { + return $this->flattenChildren((array) data_get($item->data, 'child', [])); + } + + /** + * @param array $children Namespace-URI keyed tag map. + * @return array + */ + private function flattenChildren(array $children): array + { + $out = []; + + foreach ($children as $namespace => $tags) { + $prefix = $this->prefixFor((string) $namespace); + + foreach ($tags as $tag => $nodes) { + $key = $prefix === '' ? (string) $tag : "{$prefix}_{$tag}"; + $values = array_map(fn ($node) => $this->flattenNode((array) $node), $nodes); + $out[$key] = count($values) === 1 ? $values[0] : $values; + } + } + + return $out; + } + + /** + * @param array $node + */ + private function flattenNode(array $node): mixed + { + $attribs = $this->attributes($node); + $children = (array) data_get($node, 'child', []); + + if ($children !== []) { + return array_merge($this->flattenChildren($children), $attribs); + } + + $text = trim((string) data_get($node, 'data', '')); + + return match (true) { + $text !== '' && $attribs === [] => $text, + $text === '' && $attribs !== [] => $attribs, + $text !== '' && $attribs !== [] => array_merge(['_text' => $text], $attribs), + default => $text, + }; + } + + /** + * @param array $node + * @return array + */ + private function attributes(array $node): array + { + $out = []; + + foreach ((array) data_get($node, 'attribs', []) as $attrs) { + foreach ((array) $attrs as $name => $value) { + $out[(string) $name] = (string) $value; + } + } + + return $out; + } + + private function prefixFor(string $namespace): string + { + if (in_array($namespace, self::NS_CORE, true)) { + return ''; + } + + return self::NS_PREFIX[$namespace] ?? ''; + } +} diff --git a/composer.json b/composer.json index 831fcd1f..644ed3ef 100644 --- a/composer.json +++ b/composer.json @@ -56,6 +56,7 @@ "posthog/posthog-php": "^4.1", "predis/predis": "^3.3", "sendkit/sendkit-laravel": "^1.1", + "simplepie/simplepie": "^1.9", "socialiteproviders/facebook": "^4.1", "socialiteproviders/instagram": "^5.1", "socialiteproviders/linkedin": "^5.0", diff --git a/composer.lock b/composer.lock index 4f28b729..4b5bb064 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "4d71722b650e74b41e1943d3ae9c814f", + "content-hash": "19849eb01cb4d8f93733666b53444c5a", "packages": [ { "name": "aws/aws-crt-php", @@ -7402,6 +7402,87 @@ }, "time": "2026-03-20T00:53:21+00:00" }, + { + "name": "simplepie/simplepie", + "version": "1.9.0", + "source": { + "type": "git", + "url": "https://github.com/simplepie/simplepie.git", + "reference": "76cccb1b2c5dcaf44f304c925ab30c0f48643992" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/simplepie/simplepie/zipball/76cccb1b2c5dcaf44f304c925ab30c0f48643992", + "reference": "76cccb1b2c5dcaf44f304c925ab30c0f48643992", + "shasum": "" + }, + "require": { + "ext-pcre": "*", + "ext-xml": "*", + "ext-xmlreader": "*", + "php": ">=7.2.0" + }, + "require-dev": { + "donatj/mock-webserver": "^2.7", + "friendsofphp/php-cs-fixer": "^2.19 || ^3.8", + "mf2/mf2": "^0.5.0", + "phpstan/phpstan": "~1.12.2", + "phpunit/phpunit": "^8 || ^9 || ^10", + "psr/http-client": "^1.0", + "psr/http-factory": "^1.0", + "psr/simple-cache": "^1 || ^2 || ^3" + }, + "suggest": { + "ext-curl": "", + "ext-iconv": "", + "ext-intl": "", + "ext-mbstring": "", + "mf2/mf2": "Microformat module that allows for parsing HTML for microformats" + }, + "type": "library", + "autoload": { + "psr-0": { + "SimplePie": "library" + }, + "psr-4": { + "SimplePie\\": "src" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Ryan Parman", + "homepage": "http://ryanparman.com/", + "role": "Creator, alumnus developer" + }, + { + "name": "Sam Sneddon", + "homepage": "https://gsnedders.com/", + "role": "Alumnus developer" + }, + { + "name": "Ryan McCue", + "email": "me@ryanmccue.info", + "homepage": "http://ryanmccue.info/", + "role": "Developer" + } + ], + "description": "A simple Atom/RSS parsing library for PHP", + "homepage": "http://simplepie.org/", + "keywords": [ + "atom", + "feeds", + "rss" + ], + "support": { + "issues": "https://github.com/simplepie/simplepie/issues", + "source": "https://github.com/simplepie/simplepie/tree/1.9.0" + }, + "time": "2025-09-12T06:34:27+00:00" + }, { "name": "socialiteproviders/facebook", "version": "4.1.0", diff --git a/lang/en/automations.php b/lang/en/automations.php index 92089863..308f748e 100644 --- a/lang/en/automations.php +++ b/lang/en/automations.php @@ -144,7 +144,13 @@ 'trigger_post_published_at' => 'When the post was published', 'fetched_title' => 'Fetched item title', 'fetched_link' => 'Fetched item link', - 'fetched_description' => 'Fetched item description', + 'fetched_date' => 'Fetched item publish date', + 'fetched_content' => 'Fetched item full content', + 'fetched_description' => 'Fetched item summary', + 'fetched_author' => 'Fetched item author', + 'fetched_image' => 'Fetched item image URL', + 'fetched_categories' => 'Fetched item categories', + 'fetched_enclosure' => 'Fetched item media (audio/video/file)', 'fetched_pubdate' => 'Fetched item publish date', 'fetched_http' => 'Fetched HTTP item (append a field)', 'generated_content' => 'AI-generated post content', @@ -330,6 +336,12 @@ 'fetch_rss' => [ 'feed_url' => 'Feed URL', 'feed_url_hint' => 'On first run, the watermark is set to "now" so historical items don\'t flood downstream nodes. Subsequent runs only see items newer than the previous poll.', + 'inspect' => 'Inspect feed', + 'inspecting' => 'Inspecting…', + 'inspect_hint' => 'Fetch a sample to discover the available fields for use in downstream nodes.', + 'inspect_error' => 'Could not read this feed. Check the URL and try again.', + 'discovered_fields' => 'Available fields', + 'discovered_empty' => 'No fields found in the latest item.', ], 'http_request' => [ 'url' => 'URL', diff --git a/lang/es/automations.php b/lang/es/automations.php index a4284c34..0371e776 100644 --- a/lang/es/automations.php +++ b/lang/es/automations.php @@ -144,7 +144,13 @@ 'trigger_post_published_at' => 'Cuándo se publicó el post', 'fetched_title' => 'Título del ítem obtenido', 'fetched_link' => 'Enlace del ítem obtenido', - 'fetched_description' => 'Descripción del ítem obtenido', + 'fetched_date' => 'Fecha de publicación del ítem obtenido', + 'fetched_content' => 'Contenido completo del ítem obtenido', + 'fetched_description' => 'Resumen del ítem obtenido', + 'fetched_author' => 'Autor del ítem obtenido', + 'fetched_image' => 'URL de la imagen del ítem obtenido', + 'fetched_categories' => 'Categorías del ítem obtenido', + 'fetched_enclosure' => 'Multimedia del ítem (audio/vídeo/archivo)', 'fetched_pubdate' => 'Fecha de publicación del ítem obtenido', 'fetched_http' => 'Ítem HTTP obtenido (añade un campo)', 'generated_content' => 'Contenido del post generado por IA', @@ -330,6 +336,12 @@ 'fetch_rss' => [ 'feed_url' => 'URL del feed', 'feed_url_hint' => 'En la primera ejecución, el watermark se fija en "ahora" para no inundar los siguientes nodos con ítems históricos. Ejecuciones siguientes solo ven ítems nuevos.', + 'inspect' => 'Inspeccionar feed', + 'inspecting' => 'Inspeccionando…', + 'inspect_hint' => 'Obtén una muestra para descubrir los campos disponibles para usar en los siguientes nodos.', + 'inspect_error' => 'No se pudo leer este feed. Revisa la URL e inténtalo de nuevo.', + 'discovered_fields' => 'Campos disponibles', + 'discovered_empty' => 'No se encontraron campos en el último ítem.', ], 'http_request' => [ 'url' => 'URL', diff --git a/lang/pt-BR/automations.php b/lang/pt-BR/automations.php index 605cba9d..d937d8f0 100644 --- a/lang/pt-BR/automations.php +++ b/lang/pt-BR/automations.php @@ -144,7 +144,13 @@ 'trigger_post_published_at' => 'Quando o post foi publicado', 'fetched_title' => 'Título do item buscado', 'fetched_link' => 'Link do item buscado', - 'fetched_description' => 'Descrição do item buscado', + 'fetched_date' => 'Data de publicação do item buscado', + 'fetched_content' => 'Conteúdo completo do item buscado', + 'fetched_description' => 'Resumo do item buscado', + 'fetched_author' => 'Autor do item buscado', + 'fetched_image' => 'URL da imagem do item buscado', + 'fetched_categories' => 'Categorias do item buscado', + 'fetched_enclosure' => 'Mídia do item (áudio/vídeo/arquivo)', 'fetched_pubdate' => 'Data de publicação do item buscado', 'fetched_http' => 'Item HTTP buscado (adicione um campo)', 'generated_content' => 'Conteúdo do post gerado por IA', @@ -330,6 +336,12 @@ 'fetch_rss' => [ 'feed_url' => 'URL do feed', 'feed_url_hint' => 'Na primeira execução, o watermark é setado pra "agora" pra não inundar os próximos nós com items históricos. Execuções seguintes só veem items novos.', + 'inspect' => 'Inspecionar feed', + 'inspecting' => 'Inspecionando…', + 'inspect_hint' => 'Busca uma amostra pra descobrir os campos disponíveis pra usar nos próximos nós.', + 'inspect_error' => 'Não foi possível ler esse feed. Confira a URL e tente de novo.', + 'discovered_fields' => 'Campos disponíveis', + 'discovered_empty' => 'Nenhum campo encontrado no último item.', ], 'http_request' => [ 'url' => 'URL', diff --git a/resources/js/components/automations/config/FetchRssNodeConfig.vue b/resources/js/components/automations/config/FetchRssNodeConfig.vue index e38e79b5..74249ae5 100644 --- a/resources/js/components/automations/config/FetchRssNodeConfig.vue +++ b/resources/js/components/automations/config/FetchRssNodeConfig.vue @@ -1,12 +1,23 @@ diff --git a/resources/js/composables/useExpressionCompletions.ts b/resources/js/composables/useExpressionCompletions.ts index 88c5e28e..1fac1b8a 100644 --- a/resources/js/composables/useExpressionCompletions.ts +++ b/resources/js/composables/useExpressionCompletions.ts @@ -43,13 +43,29 @@ const providedBy = (node: GraphNode): ExpressionSuggestion[] => { } return [suggestion('trigger.event', 'trigger_event'), suggestion('trigger.fired_at', 'trigger_fired_at')]; } - case 'fetch_rss': - return [ + case 'fetch_rss': { + const aliases = [ suggestion('fetched.title', 'fetched_title'), suggestion('fetched.link', 'fetched_link'), + suggestion('fetched.date', 'fetched_date'), + suggestion('fetched.content', 'fetched_content'), suggestion('fetched.description', 'fetched_description'), - suggestion('fetched.pubDate', 'fetched_pubdate'), + suggestion('fetched.author', 'fetched_author'), + suggestion('fetched.image', 'fetched_image'), + suggestion('fetched.categories', 'fetched_categories'), + suggestion('fetched.enclosure', 'fetched_enclosure'), ]; + // Fields discovered by inspecting the real feed — the dynamic raw layer + // (yt_videoId, media_group.*, dc_creator, …) that aliases can't enumerate. + const discovered = Array.isArray(node.data?.discovered_fields) + ? (node.data.discovered_fields as Array<{ path: string; sample?: string }>).map( + (field): ExpressionSuggestion => ({ label: field.path, info: field.sample ?? '' }), + ) + : []; + const known = new Set(aliases.map((alias) => alias.label)); + + return [...aliases, ...discovered.filter((field) => !known.has(field.label))]; + } case 'http_request': return [suggestion('fetched', 'fetched_http')]; case 'generate': diff --git a/resources/js/pages/automations/Form.vue b/resources/js/pages/automations/Form.vue index 891ec93f..b3afe367 100644 --- a/resources/js/pages/automations/Form.vue +++ b/resources/js/pages/automations/Form.vue @@ -137,6 +137,7 @@ provide( 'automationExpressionCompletions', computed(() => buildExpressionCatalog(selectedNodeId.value, nodes.value, edges.value, variables.value)), ); +provide('automationId', props.automation.id); // Toggled by an expandable CodeEditor (see CodeEditor.vue) to slide out the // side-by-side editing panel, pushing the sidebar left to make room. diff --git a/routes/app.php b/routes/app.php index 403d9a60..39affce2 100644 --- a/routes/app.php +++ b/routes/app.php @@ -224,6 +224,7 @@ Route::post('automations/{automation}/pause', [AutomationController::class, 'pause'])->name('app.automations.pause'); Route::post('automations/{automation}/runs/{run}/retry', [AutomationController::class, 'retryRun'])->name('app.automations.runs.retry'); Route::post('automations/{automation}/test', [AutomationController::class, 'test'])->name('app.automations.test'); + Route::post('automations/{automation}/feed/inspect', [AutomationController::class, 'inspectFeed'])->name('app.automations.feed.inspect'); Route::get('automations/{automation}/runs/{run}', [AutomationController::class, 'showRun'])->name('app.automations.runs.show'); // API Keys diff --git a/tests/Feature/Automation/FeedInspectionTest.php b/tests/Feature/Automation/FeedInspectionTest.php new file mode 100644 index 00000000..7d31baed --- /dev/null +++ b/tests/Feature/Automation/FeedInspectionTest.php @@ -0,0 +1,138 @@ +workspace = Workspace::factory()->create(); + $this->user = User::factory()->create(['current_workspace_id' => $this->workspace->id]); + $this->workspace->members()->attach($this->user->id, ['role' => Role::Admin->value]); + $this->user->refresh(); +}); + +it('returns the discovered fields for a feed', function () { + Http::fake(['1.1.1.1/*' => Http::response(file_get_contents(base_path('tests/fixtures/feeds/youtube_atom.xml')), 200)]); + + $automation = Automation::factory()->for($this->workspace)->create(); + + $response = $this->actingAs($this->user) + ->postJson(route('app.automations.feed.inspect', $automation->id), [ + 'feed_url' => 'https://1.1.1.1/feed.xml', + ]); + + $response->assertOk(); + + $paths = collect($response->json('fields'))->pluck('path'); + expect($paths)->toContain('fetched.title') + ->toContain('fetched.link') + ->toContain('fetched.yt_videoId') + ->toContain('fetched.media_group.media_thumbnail.url'); +}); + +it('resolves workspace variables in the feed url before fetching', function () { + Http::fake(['1.1.1.1/*' => Http::response(file_get_contents(base_path('tests/fixtures/feeds/youtube_atom.xml')), 200)]); + + $automation = Automation::factory()->for($this->workspace)->create([ + 'variables' => [['key' => 'HOST', 'value' => '1.1.1.1']], + ]); + + $this->actingAs($this->user) + ->postJson(route('app.automations.feed.inspect', $automation->id), [ + 'feed_url' => 'https://{{ variables.HOST }}/feed.xml', + ]) + ->assertOk(); + + Http::assertSent(fn ($request) => str_contains($request->url(), '1.1.1.1/feed.xml')); +}); + +it('returns 422 for a malformed feed', function () { + Http::fake(['1.1.1.1/*' => Http::response('not a feed', 200)]); + + $automation = Automation::factory()->for($this->workspace)->create(); + + $this->actingAs($this->user) + ->postJson(route('app.automations.feed.inspect', $automation->id), [ + 'feed_url' => 'https://1.1.1.1/feed.xml', + ]) + ->assertStatus(422); +}); + +it('validates that feed_url is required', function () { + $automation = Automation::factory()->for($this->workspace)->create(); + + $this->actingAs($this->user) + ->postJson(route('app.automations.feed.inspect', $automation->id), []) + ->assertStatus(422) + ->assertJsonValidationErrorFor('feed_url'); +}); + +it('persists discovered_fields stored on a fetch_rss node through an update', function () { + $automation = Automation::factory()->for($this->workspace)->create(); + + $this->actingAs($this->user) + ->put(route('app.automations.update', $automation->id), [ + 'nodes' => [[ + 'id' => 'fetch_1', + 'type' => 'fetch_rss', + 'position' => ['x' => 0, 'y' => 0], + 'data' => [ + 'feed_url' => 'https://example.com/feed.xml', + 'discovered_fields' => [ + ['path' => 'fetched.yt_videoId', 'sample' => 'abc123'], + ], + ], + ]], + ]) + ->assertRedirect(); + + $node = collect($automation->fresh()->nodes)->firstWhere('id', 'fetch_1'); + expect(data_get($node, 'data.discovered_fields.0.path'))->toBe('fetched.yt_videoId'); +}); + +it('accepts a feed_url containing a workflow-variable expression', function () { + $automation = Automation::factory()->for($this->workspace)->create(); + + $this->actingAs($this->user) + ->put(route('app.automations.update', $automation->id), [ + 'nodes' => [[ + 'id' => 'fetch_1', + 'type' => 'fetch_rss', + 'position' => ['x' => 0, 'y' => 0], + 'data' => ['feed_url' => 'https://www.youtube.com/feeds/videos.xml?channel_id={{ variables.CHANNEL_ID }}'], + ]], + ]) + ->assertRedirect() + ->assertSessionHasNoErrors(); +}); + +it('rejects a feed_url that is not a URL even after stripping expressions', function () { + $automation = Automation::factory()->for($this->workspace)->create(); + + $this->actingAs($this->user) + ->put(route('app.automations.update', $automation->id), [ + 'nodes' => [[ + 'id' => 'fetch_1', + 'type' => 'fetch_rss', + 'position' => ['x' => 0, 'y' => 0], + 'data' => ['feed_url' => 'not a url {{ variables.X }}'], + ]], + ]) + ->assertSessionHasErrors('nodes.0.data.feed_url'); +}); + +it('forbids inspecting a feed for an automation in another workspace', function () { + Http::fake(['1.1.1.1/*' => Http::response('', 200)]); + + $automation = Automation::factory()->for(Workspace::factory()->create())->create(); + + $this->actingAs($this->user) + ->postJson(route('app.automations.feed.inspect', $automation->id), [ + 'feed_url' => 'https://1.1.1.1/feed.xml', + ]) + ->assertForbidden(); +}); diff --git a/tests/Feature/Automation/Node/FetchRssNodeTest.php b/tests/Feature/Automation/Node/FetchRssNodeTest.php index 48ae91f2..f915098e 100644 --- a/tests/Feature/Automation/Node/FetchRssNodeTest.php +++ b/tests/Feature/Automation/Node/FetchRssNodeTest.php @@ -28,7 +28,7 @@ Olderhttps://1.1.1.1/aaSat, 01 Feb 2025 12:00:00 +0000 Newer1https://1.1.1.1/bbSun, 01 Jun 2025 12:00:00 +0000 - Newer2https://1.1.1.1/ccMon, 15 Jun 2025 12:00:00 +0000 + Newer2https://1.1.1.1/ccSun, 15 Jun 2025 12:00:00 +0000 XML; @@ -256,7 +256,31 @@ $state = AutomationNodeState::for($automation->id, 'fetch_1'); expect(CarbonImmutable::parse($state->data['last_item_date'])->toIso8601String()) - ->toBe(CarbonImmutable::parse('Mon, 15 Jun 2025 12:00:00 +0000')->toIso8601String()); + ->toBe(CarbonImmutable::parse('Sun, 15 Jun 2025 12:00:00 +0000')->toIso8601String()); +}); + +it('processes an Atom feed (YouTube) that the old RSS-only parser dropped', function () { + Carbon::setTestNow('2026-06-16 10:00:00'); + Http::fake(['1.1.1.1/*' => Http::response(file_get_contents(base_path('tests/fixtures/feeds/youtube_atom.xml')), 200)]); + + $automation = Automation::factory()->active()->create(); + + // Watermark predates the single entry, so it counts as new. + AutomationNodeState::create([ + 'automation_id' => $automation->id, + 'node_id' => 'fetch_1', + 'data' => ['last_item_date' => '2026-01-01T00:00:00+00:00'], + ]); + + $run = AutomationRun::factory()->for($automation)->create(['current_node_id' => 'fetch_1']); + + $result = app(RunFetchRssNode::class)($run, ['feed_url' => 'https://1.1.1.1/feed']); + + expect($result->status)->toBe(NodeRunStatus::Completed); + expect($result->output['fetch']['count'])->toBe(1); + expect($result->output['fetched']['title'])->toBe('Ninguém aguenta o TikTok Shop'); + expect($result->output['fetched']['link'])->toBe('https://www.youtube.com/watch?v=bIQVeW4sTcE'); + expect($result->output['fetched']['yt_videoId'])->toBe('bIQVeW4sTcE'); }); it('fails when feed_url is missing', function () { diff --git a/tests/Unit/Automation/FeedParserTest.php b/tests/Unit/Automation/FeedParserTest.php new file mode 100644 index 00000000..d12e948d --- /dev/null +++ b/tests/Unit/Automation/FeedParserTest.php @@ -0,0 +1,81 @@ +parser = new FeedParser; +}); + +it('parses an Atom (YouTube) feed with aliases and raw namespaced fields', function () { + $item = $this->parser->parse(feedFixture('youtube_atom'))[0]; + + // Aliases resolve cross-format: link comes from the Atom href attribute, + // date from , author from . + expect($item['title'])->toBe('Ninguém aguenta o TikTok Shop'); + expect($item['link'])->toBe('https://www.youtube.com/watch?v=bIQVeW4sTcE'); + expect($item['date'])->toBe('2026-06-15T20:24:32+00:00'); + expect($item['author'])->toBe('Beta Boechat'); + expect($item['key'])->toBe('yt:video:bIQVeW4sTcE'); + + // Raw namespaced fields stay reachable. + expect($item['yt_videoId'])->toBe('bIQVeW4sTcE'); + expect(data_get($item, 'media_group.media_thumbnail.url')) + ->toBe('https://i3.ytimg.com/vi/bIQVeW4sTcE/hqdefault.jpg'); + expect(data_get($item, 'media_group.media_description')) + ->toBe('Descrição completa do vídeo aqui.'); +}); + +it('parses an RSS 2.0 feed with dc and content namespaces', function () { + $item = $this->parser->parse(feedFixture('rss_namespaces'))[0]; + + expect($item['title'])->toBe('Polish is here'); + expect($item['link'])->toBe('https://feedback.changelogfy.com/en/polish-is-here'); + expect($item['author'])->toBe('Paulo Castellano'); + expect($item['key'])->toBe('16092'); + + // Extension fields the old parser threw away. + expect($item['dc_creator'])->toBe('Paulo Castellano'); + expect($item['content_encoded'])->toBe('

You can now serve your changelogs fully translated in Polish.

'); + expect($item['categories'])->toBe(['Feature', 'i18n']); +}); + +it('parses a podcast feed exposing the enclosure and itunes fields', function () { + $item = $this->parser->parse(feedFixture('podcast'))[0]; + + expect(data_get($item, 'enclosure.url'))->toBe('https://cdn.changelog.com/the-changelog-682.mp3'); + expect(data_get($item, 'enclosure.type'))->toBe('audio/mpeg'); + expect($item['itunes_duration'])->toBe('1:46:28'); + expect($item['key'])->toBe('changelog.com/1/2829'); +}); + +it('preserves backward-compatible RSS 2.0 alias keys', function () { + $item = $this->parser->parse(feedFixture('rss_namespaces'))[0]; + + // Automations saved before multi-format support reference these exact keys. + expect($item)->toHaveKeys(['title', 'link', 'description', 'pubDate', 'key']); + expect($item['pubDate'])->toBe($item['date']); +}); + +it('keeps a falsy element from a tag that only carries attributes', function () { + $item = $this->parser->parse(feedFixture('podcast'))[0]; + + // has no text — the attribute object is exposed. + expect(data_get($item, 'itunes_image.href'))->toBe('https://cdn.changelog.com/cover.png'); +}); + +it('returns null for a body that is not a valid feed', function () { + expect($this->parser->parse('this is not xml at all'))->toBeNull(); +}); + +it('returns an empty list for a valid but empty feed', function () { + $empty = ''; + + expect($this->parser->parse($empty))->toBe([]); +}); diff --git a/tests/fixtures/feeds/podcast.xml b/tests/fixtures/feeds/podcast.xml new file mode 100644 index 00000000..fa8c2068 --- /dev/null +++ b/tests/fixtures/feeds/podcast.xml @@ -0,0 +1,20 @@ + + + + The Changelog + https://changelog.com/podcast + Software news + + From open source hits to OpenAI + changelog.com/1/2829 + https://changelog.com/podcast/682 + Fri, 05 Jun 2026 19:00:00 +0000 + + This week I am talking with Max Stoiber. + full + + 1:46:28 + false + + + diff --git a/tests/fixtures/feeds/rss_namespaces.xml b/tests/fixtures/feeds/rss_namespaces.xml new file mode 100644 index 00000000..6a2157ec --- /dev/null +++ b/tests/fixtures/feeds/rss_namespaces.xml @@ -0,0 +1,20 @@ + + + + Changelogfy + Changelogfy + https://feedback.changelogfy.com + + + <![CDATA[Polish is here]]> + + https://feedback.changelogfy.com/en/polish-is-here + 16092 + Paulo Castellano + Thu, 05 Mar 2026 16:20:09 +0000 + + + You can now serve your changelogs fully translated in Polish.

]]>
+
+
+
diff --git a/tests/fixtures/feeds/youtube_atom.xml b/tests/fixtures/feeds/youtube_atom.xml new file mode 100644 index 00000000..b3370058 --- /dev/null +++ b/tests/fixtures/feeds/youtube_atom.xml @@ -0,0 +1,36 @@ + + + + yt:channel:123 + 123 + Beta Boechat + + + Beta Boechat + https://www.youtube.com/channel/UC123 + + 2015-08-11T18:07:31+00:00 + + yt:video:bIQVeW4sTcE + bIQVeW4sTcE + 123 + Ninguém aguenta o TikTok Shop + + + Beta Boechat + https://www.youtube.com/channel/UC123 + + 2026-06-15T20:24:32+00:00 + 2026-06-15T21:15:11+00:00 + + Ninguém aguenta o TikTok Shop + + + Descrição completa do vídeo aqui. + + + + + + +