feat(automations): multi-format RSS/Atom feeds with dynamic variables
Replace the RSS-2.0-only SimpleXML parser with SimplePie so the Fetch RSS
node reads Atom 1.0 (YouTube, GitHub, The Verge…) and RSS 2.0 + namespace
extensions (dc:, content:, media:, yt:, itunes:). Each item exposes stable
cross-format aliases (title, link, date, content, author, …) plus every
namespaced field flattened for use as {{ fetched.* }}.
Add a feed-inspection endpoint that discovers a feed's real fields and feeds
them into the editor's expression autocomplete. Allow {{ }} expressions in the
feed URL via a ResolvableUrl rule. Parsing moves to a dedicated FeedParser
service; the fetch keeps the SSRF guard and gains XXE-safe parsing.
This commit is contained in:
Paulo Castellano
parent
b18b9b2303
commit
246a159f34
22 changed files with 904 additions and 35 deletions
|
|
@ -10,11 +10,11 @@
|
|||
use App\Models\AutomationNodeState;
|
||||
use App\Models\AutomationRun;
|
||||
use App\Services\Automation\ExpressionResolver;
|
||||
use App\Services\Automation\FeedParser;
|
||||
use App\Services\Brand\SafeHttpFetcher;
|
||||
use Carbon\CarbonImmutable;
|
||||
use Illuminate\Support\Facades\Http;
|
||||
use RuntimeException;
|
||||
use SimpleXMLElement;
|
||||
use Throwable;
|
||||
|
||||
/**
|
||||
|
|
@ -40,6 +40,7 @@ public function __construct(
|
|||
private ExpressionResolver $resolver,
|
||||
private SafeHttpFetcher $safeHttp,
|
||||
private AdvanceAutomationRun $advance,
|
||||
private FeedParser $parser,
|
||||
) {}
|
||||
|
||||
public function __invoke(AutomationRun $run, array $config): NodeRunResult
|
||||
|
|
@ -65,9 +66,9 @@ public function __invoke(AutomationRun $run, array $config): NodeRunResult
|
|||
return NodeRunResult::failed(__('automations.errors.fetch_rss_request_failed'), ['status' => $response->status()]);
|
||||
}
|
||||
|
||||
try {
|
||||
$xml = new SimpleXMLElement($response->body());
|
||||
} catch (Throwable) {
|
||||
$items = $this->parser->parse($response->body());
|
||||
|
||||
if ($items === null) {
|
||||
return NodeRunResult::failed(__('automations.errors.fetch_rss_malformed'));
|
||||
}
|
||||
|
||||
|
|
@ -83,7 +84,7 @@ public function __invoke(AutomationRun $run, array $config): NodeRunResult
|
|||
? CarbonImmutable::createFromTimestamp(0)
|
||||
: $this->parseWatermark(data_get($state->data, 'last_item_date'));
|
||||
|
||||
[$newItems, $newestSeen] = $this->collectNewItems($xml, $watermark);
|
||||
[$newItems, $newestSeen] = $this->collectNewItems($items, $watermark);
|
||||
|
||||
if ($state !== null && $newestSeen !== null) {
|
||||
$state->update(['data' => array_merge($state->data ?? [], [
|
||||
|
|
@ -107,46 +108,44 @@ public function __invoke(AutomationRun $run, array $config): NodeRunResult
|
|||
]);
|
||||
}
|
||||
|
||||
private function collectNewItems(SimpleXMLElement $xml, CarbonImmutable $watermark): array
|
||||
/**
|
||||
* @param list<array<string, mixed>> $parsed Normalized items from FeedParser.
|
||||
* @return array{0: list<array<string, mixed>>, 1: ?CarbonImmutable}
|
||||
*/
|
||||
private function collectNewItems(array $parsed, CarbonImmutable $watermark): array
|
||||
{
|
||||
$items = [];
|
||||
$newestSeen = null;
|
||||
|
||||
foreach ($xml->channel->item ?? [] as $item) {
|
||||
$key = (string) ($item->guid ?? $item->link);
|
||||
foreach ($parsed as $item) {
|
||||
$key = (string) data_get($item, 'key', '');
|
||||
if ($key === '') {
|
||||
continue;
|
||||
}
|
||||
|
||||
$pubDate = $this->parsePubDate((string) $item->pubDate);
|
||||
if ($pubDate === null) {
|
||||
$date = $this->parsePubDate((string) data_get($item, 'date', ''));
|
||||
if ($date === null) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if ($newestSeen === null || $pubDate->greaterThan($newestSeen)) {
|
||||
$newestSeen = $pubDate;
|
||||
if ($newestSeen === null || $date->greaterThan($newestSeen)) {
|
||||
$newestSeen = $date;
|
||||
}
|
||||
|
||||
if (! $pubDate->greaterThan($watermark)) {
|
||||
if (! $date->greaterThan($watermark)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$items[] = [
|
||||
'_pubDate' => $pubDate,
|
||||
'key' => $key,
|
||||
'title' => (string) $item->title,
|
||||
'link' => (string) $item->link,
|
||||
'description' => (string) $item->description,
|
||||
'pubDate' => (string) $item->pubDate,
|
||||
];
|
||||
$item['_sort'] = $date->getTimestamp();
|
||||
$items[] = $item;
|
||||
}
|
||||
|
||||
// Process oldest-first so siblings inherit a stable order matching feed chronology.
|
||||
usort($items, fn ($a, $b) => $a['_pubDate']->getTimestamp() <=> $b['_pubDate']->getTimestamp());
|
||||
usort($items, fn ($a, $b) => $a['_sort'] <=> $b['_sort']);
|
||||
|
||||
// Drop the internal sort key — downstream nodes shouldn't see it.
|
||||
$items = array_map(function (array $item): array {
|
||||
unset($item['_pubDate']);
|
||||
unset($item['_sort']);
|
||||
|
||||
return $item;
|
||||
}, $items);
|
||||
|
|
|
|||
|
|
@ -17,11 +17,13 @@
|
|||
use App\Actions\Automation\Run\TestAutomation;
|
||||
use App\Http\Controllers\Controller;
|
||||
use App\Http\Requests\App\Automations\ActivateAutomationRequest;
|
||||
use App\Http\Requests\App\Automations\InspectFeedRequest;
|
||||
use App\Http\Requests\App\Automations\PauseAutomationRequest;
|
||||
use App\Http\Requests\App\Automations\RetryRunRequest;
|
||||
use App\Http\Requests\App\Automations\StoreAutomationRequest;
|
||||
use App\Http\Requests\App\Automations\TestAutomationRequest;
|
||||
use App\Http\Requests\App\Automations\UpdateAutomationRequest;
|
||||
use App\Http\Resources\App\Automation\FeedInspectionResource;
|
||||
use App\Http\Resources\App\PlatformConfigResource;
|
||||
use App\Http\Resources\App\SocialAccountResource;
|
||||
use App\Http\Resources\AutomationInvocationResource;
|
||||
|
|
@ -31,11 +33,17 @@
|
|||
use App\Models\Automation;
|
||||
use App\Models\AutomationNodeRun;
|
||||
use App\Models\AutomationRun;
|
||||
use App\Services\Automation\ExpressionResolver;
|
||||
use App\Services\Automation\FeedParser;
|
||||
use App\Services\Brand\SafeHttpFetcher;
|
||||
use Illuminate\Http\JsonResponse;
|
||||
use Illuminate\Http\RedirectResponse;
|
||||
use Illuminate\Http\Response as HttpResponse;
|
||||
use Illuminate\Support\Facades\Http;
|
||||
use Inertia\Inertia;
|
||||
use Inertia\Response;
|
||||
use RuntimeException;
|
||||
use Symfony\Component\HttpFoundation\Response as SymfonyResponse;
|
||||
|
||||
class AutomationController extends Controller
|
||||
{
|
||||
|
|
@ -203,6 +211,41 @@ public function test(TestAutomationRequest $request, Automation $automation, Tes
|
|||
return response()->json(['run_id' => $run->id]);
|
||||
}
|
||||
|
||||
public function inspectFeed(
|
||||
InspectFeedRequest $request,
|
||||
Automation $automation,
|
||||
ExpressionResolver $resolver,
|
||||
SafeHttpFetcher $safeHttp,
|
||||
FeedParser $parser,
|
||||
): JsonResponse|FeedInspectionResource {
|
||||
$this->authorize('update', $automation);
|
||||
|
||||
$feedUrl = $resolver->resolve(
|
||||
(string) $request->validated('feed_url'),
|
||||
['variables' => $automation->resolvedVariables()],
|
||||
);
|
||||
|
||||
try {
|
||||
$safeHttp->guardAgainstSsrf($feedUrl);
|
||||
} catch (RuntimeException) {
|
||||
return response()->json(['message' => __('automations.errors.url_not_allowed')], SymfonyResponse::HTTP_UNPROCESSABLE_ENTITY);
|
||||
}
|
||||
|
||||
$response = Http::get($feedUrl);
|
||||
|
||||
if (! $response->successful()) {
|
||||
return response()->json(['message' => __('automations.errors.fetch_rss_request_failed')], SymfonyResponse::HTTP_UNPROCESSABLE_ENTITY);
|
||||
}
|
||||
|
||||
$items = $parser->parse($response->body());
|
||||
|
||||
if ($items === null) {
|
||||
return response()->json(['message' => __('automations.errors.fetch_rss_malformed')], SymfonyResponse::HTTP_UNPROCESSABLE_ENTITY);
|
||||
}
|
||||
|
||||
return new FeedInspectionResource($items[0] ?? []);
|
||||
}
|
||||
|
||||
public function showRun(Automation $automation, AutomationRun $run): JsonResponse
|
||||
{
|
||||
$this->authorize('view', $automation);
|
||||
|
|
|
|||
25
app/Http/Requests/App/Automations/InspectFeedRequest.php
Normal file
25
app/Http/Requests/App/Automations/InspectFeedRequest.php
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace App\Http\Requests\App\Automations;
|
||||
|
||||
use Illuminate\Foundation\Http\FormRequest;
|
||||
|
||||
class InspectFeedRequest extends FormRequest
|
||||
{
|
||||
public function authorize(): bool
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array<string, mixed>
|
||||
*/
|
||||
public function rules(): array
|
||||
{
|
||||
return [
|
||||
'feed_url' => ['required', 'string', 'max:2048'],
|
||||
];
|
||||
}
|
||||
}
|
||||
|
|
@ -12,6 +12,7 @@
|
|||
use App\Enums\Automation\Publish\Mode as PublishMode;
|
||||
use App\Enums\Automation\ScheduleField;
|
||||
use App\Enums\Automation\Trigger\Type as TriggerType;
|
||||
use App\Rules\ResolvableUrl;
|
||||
use App\Services\Automation\AutomationConfigValidator;
|
||||
use App\Services\Automation\GenerateNodeValidator;
|
||||
use Illuminate\Contracts\Validation\Validator;
|
||||
|
|
@ -129,7 +130,10 @@ private function dataRulesForNodeType(?string $type, int $i): array
|
|||
'schedule_timezone' => ['sometimes', 'string', 'timezone'],
|
||||
],
|
||||
NodeType::FetchRss->value => [
|
||||
'feed_url' => ['required', 'url'],
|
||||
'feed_url' => ['required', new ResolvableUrl],
|
||||
'discovered_fields' => ['sometimes', 'array'],
|
||||
'discovered_fields.*.path' => ['required', 'string'],
|
||||
'discovered_fields.*.sample' => ['nullable', 'string'],
|
||||
],
|
||||
NodeType::HttpRequest->value => [
|
||||
'url' => ['required', 'url'],
|
||||
|
|
|
|||
69
app/Http/Resources/App/Automation/FeedInspectionResource.php
Normal file
69
app/Http/Resources/App/Automation/FeedInspectionResource.php
Normal file
|
|
@ -0,0 +1,69 @@
|
|||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace App\Http\Resources\App\Automation;
|
||||
|
||||
use Illuminate\Http\Request;
|
||||
use Illuminate\Http\Resources\Json\JsonResource;
|
||||
|
||||
/**
|
||||
* Flattens the first parsed feed item into a catalog of `{{ fetched.* }}` paths
|
||||
* with sample values, so the editor can offer them as expression completions.
|
||||
*
|
||||
* @property array<string, mixed> $resource The first normalized feed item.
|
||||
*/
|
||||
class FeedInspectionResource extends JsonResource
|
||||
{
|
||||
private const SAMPLE_MAX_LENGTH = 120;
|
||||
|
||||
/**
|
||||
* @return array{fields: list<array{path: string, sample: string}>}
|
||||
*/
|
||||
public function toArray(Request $request): array
|
||||
{
|
||||
return [
|
||||
'fields' => $this->flatten((array) $this->resource),
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array<string, mixed> $item
|
||||
* @return list<array{path: string, sample: string}>
|
||||
*/
|
||||
private function flatten(array $item, string $prefix = 'fetched'): array
|
||||
{
|
||||
$fields = [];
|
||||
|
||||
foreach ($item as $key => $value) {
|
||||
$path = "{$prefix}.{$key}";
|
||||
|
||||
if (is_array($value) && $this->isAssoc($value)) {
|
||||
$fields = array_merge($fields, $this->flatten($value, $path));
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
$fields[] = ['path' => $path, 'sample' => $this->sample($value)];
|
||||
}
|
||||
|
||||
return $fields;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array<int|string, mixed> $value
|
||||
*/
|
||||
private function isAssoc(array $value): bool
|
||||
{
|
||||
return $value !== [] && ! array_is_list($value);
|
||||
}
|
||||
|
||||
private function sample(mixed $value): string
|
||||
{
|
||||
$text = is_array($value)
|
||||
? (json_encode($value, JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE) ?: '')
|
||||
: (string) $value;
|
||||
|
||||
return str($text)->squish()->limit(self::SAMPLE_MAX_LENGTH)->value();
|
||||
}
|
||||
}
|
||||
36
app/Rules/ResolvableUrl.php
Normal file
36
app/Rules/ResolvableUrl.php
Normal file
|
|
@ -0,0 +1,36 @@
|
|||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace App\Rules;
|
||||
|
||||
use Closure;
|
||||
use Illuminate\Contracts\Validation\ValidationRule;
|
||||
use Illuminate\Translation\PotentiallyTranslatedString;
|
||||
|
||||
/**
|
||||
* Validates a URL that may contain `{{ ... }}` workflow-variable expressions.
|
||||
* The expressions are resolved at runtime, so they're replaced with a placeholder
|
||||
* before checking the result is a valid URL — letting users template the host or
|
||||
* query (e.g. `https://host/feed.xml?channel_id={{ variables.CHANNEL_ID }}`).
|
||||
*/
|
||||
class ResolvableUrl implements ValidationRule
|
||||
{
|
||||
/**
|
||||
* @param Closure(string, ?string=): PotentiallyTranslatedString $fail
|
||||
*/
|
||||
public function validate(string $attribute, mixed $value, Closure $fail): void
|
||||
{
|
||||
if (! is_string($value)) {
|
||||
$fail('validation.url')->translate();
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
$candidate = preg_replace('/\{\{\s*[\w.]+\s*\}\}/', 'placeholder', $value);
|
||||
|
||||
if (filter_var($candidate, FILTER_VALIDATE_URL) === false) {
|
||||
$fail('validation.url')->translate();
|
||||
}
|
||||
}
|
||||
}
|
||||
179
app/Services/Automation/FeedParser.php
Normal file
179
app/Services/Automation/FeedParser.php
Normal file
|
|
@ -0,0 +1,179 @@
|
|||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace App\Services\Automation;
|
||||
|
||||
use SimplePie\Item;
|
||||
use SimplePie\SimplePie;
|
||||
|
||||
/**
|
||||
* Parses an RSS 2.0 / Atom 1.0 / RDF feed body into normalized item arrays.
|
||||
*
|
||||
* Each item carries two merged layers:
|
||||
* - **Aliases**: stable cross-format keys (title, link, date, content, author, …)
|
||||
* resolved through SimplePie's normalized getters, so a downstream node works the
|
||||
* same whether the feed is RSS or Atom.
|
||||
* - **Raw**: every namespaced extension tag (yt:videoId, media:thumbnail,
|
||||
* dc:creator, content:encoded, …) flattened with a `prefix_localname` convention,
|
||||
* so feed-specific data stays reachable via `{{ fetched.<field> }}`.
|
||||
*
|
||||
* Aliases win on key collisions. The HTTP fetch and SSRF guard stay in the caller;
|
||||
* the body is handed to SimplePie via `set_raw_data()` so the SSRF guard isn't bypassed.
|
||||
*/
|
||||
class FeedParser
|
||||
{
|
||||
/**
|
||||
* Namespace URI → short prefix for the raw layer.
|
||||
*/
|
||||
private const NS_PREFIX = [
|
||||
'http://search.yahoo.com/mrss/' => 'media',
|
||||
'http://www.youtube.com/xml/schemas/2015' => 'yt',
|
||||
'http://purl.org/dc/elements/1.1/' => 'dc',
|
||||
'http://purl.org/dc/terms/' => 'dc',
|
||||
'http://purl.org/rss/1.0/modules/content/' => 'content',
|
||||
'http://www.itunes.com/dtds/podcast-1.0.dtd' => 'itunes',
|
||||
'https://podcastindex.org/namespace/1.0' => 'podcast',
|
||||
'http://purl.org/rss/1.0/modules/slash/' => 'slash',
|
||||
'http://wellformedweb.org/CommentAPI/' => 'wfw',
|
||||
'http://www.georss.org/georss' => 'georss',
|
||||
];
|
||||
|
||||
/**
|
||||
* Namespaces treated as the feed "core" — their tags get no prefix, so RSS and
|
||||
* Atom land on the same raw key names (and are then overridden by aliases).
|
||||
*/
|
||||
private const NS_CORE = [
|
||||
'',
|
||||
'http://www.w3.org/2005/Atom',
|
||||
'http://purl.org/rss/1.0/',
|
||||
'http://backend.userland.com/rss2',
|
||||
'http://my.netscape.com/rdf/simple/0.9/',
|
||||
];
|
||||
|
||||
/**
|
||||
* @return list<array<string, mixed>>|null Null when the body is not a valid feed.
|
||||
*/
|
||||
public function parse(string $body): ?array
|
||||
{
|
||||
$feed = new SimplePie;
|
||||
$feed->enable_cache(false);
|
||||
$feed->set_raw_data($body);
|
||||
|
||||
if (! @$feed->init()) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return array_map(fn (Item $item): array => $this->normalize($item), $feed->get_items());
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array<string, mixed>
|
||||
*/
|
||||
private function normalize(Item $item): array
|
||||
{
|
||||
$enclosure = $item->get_enclosure();
|
||||
|
||||
$aliases = [
|
||||
'key' => $item->get_id(false, false) ?: $item->get_permalink(),
|
||||
'title' => $item->get_title(),
|
||||
'link' => $item->get_permalink(),
|
||||
'date' => $item->get_date('c'),
|
||||
'pubDate' => $item->get_date('c'),
|
||||
'content' => $item->get_content(),
|
||||
'description' => $item->get_description(),
|
||||
'author' => $item->get_author()?->get_name(),
|
||||
'id' => $item->get_id(false, false),
|
||||
'image' => $enclosure?->get_thumbnail() ?: null,
|
||||
'categories' => array_values(array_filter(array_map(
|
||||
fn ($category) => $category->get_label(),
|
||||
$item->get_categories() ?? [],
|
||||
))),
|
||||
'enclosure' => $enclosure === null ? null : array_filter([
|
||||
'url' => $enclosure->get_link(),
|
||||
'type' => $enclosure->get_type(),
|
||||
'length' => $enclosure->get_length(),
|
||||
], fn ($value) => $value !== null),
|
||||
];
|
||||
|
||||
// Aliases win on collision, so merge them over the raw layer.
|
||||
return array_merge($this->rawFields($item), $aliases);
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array<string, mixed>
|
||||
*/
|
||||
private function rawFields(Item $item): array
|
||||
{
|
||||
return $this->flattenChildren((array) data_get($item->data, 'child', []));
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array<string, mixed> $children Namespace-URI keyed tag map.
|
||||
* @return array<string, mixed>
|
||||
*/
|
||||
private function flattenChildren(array $children): array
|
||||
{
|
||||
$out = [];
|
||||
|
||||
foreach ($children as $namespace => $tags) {
|
||||
$prefix = $this->prefixFor((string) $namespace);
|
||||
|
||||
foreach ($tags as $tag => $nodes) {
|
||||
$key = $prefix === '' ? (string) $tag : "{$prefix}_{$tag}";
|
||||
$values = array_map(fn ($node) => $this->flattenNode((array) $node), $nodes);
|
||||
$out[$key] = count($values) === 1 ? $values[0] : $values;
|
||||
}
|
||||
}
|
||||
|
||||
return $out;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array<string, mixed> $node
|
||||
*/
|
||||
private function flattenNode(array $node): mixed
|
||||
{
|
||||
$attribs = $this->attributes($node);
|
||||
$children = (array) data_get($node, 'child', []);
|
||||
|
||||
if ($children !== []) {
|
||||
return array_merge($this->flattenChildren($children), $attribs);
|
||||
}
|
||||
|
||||
$text = trim((string) data_get($node, 'data', ''));
|
||||
|
||||
return match (true) {
|
||||
$text !== '' && $attribs === [] => $text,
|
||||
$text === '' && $attribs !== [] => $attribs,
|
||||
$text !== '' && $attribs !== [] => array_merge(['_text' => $text], $attribs),
|
||||
default => $text,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array<string, mixed> $node
|
||||
* @return array<string, string>
|
||||
*/
|
||||
private function attributes(array $node): array
|
||||
{
|
||||
$out = [];
|
||||
|
||||
foreach ((array) data_get($node, 'attribs', []) as $attrs) {
|
||||
foreach ((array) $attrs as $name => $value) {
|
||||
$out[(string) $name] = (string) $value;
|
||||
}
|
||||
}
|
||||
|
||||
return $out;
|
||||
}
|
||||
|
||||
private function prefixFor(string $namespace): string
|
||||
{
|
||||
if (in_array($namespace, self::NS_CORE, true)) {
|
||||
return '';
|
||||
}
|
||||
|
||||
return self::NS_PREFIX[$namespace] ?? '';
|
||||
}
|
||||
}
|
||||
|
|
@ -56,6 +56,7 @@
|
|||
"posthog/posthog-php": "^4.1",
|
||||
"predis/predis": "^3.3",
|
||||
"sendkit/sendkit-laravel": "^1.1",
|
||||
"simplepie/simplepie": "^1.9",
|
||||
"socialiteproviders/facebook": "^4.1",
|
||||
"socialiteproviders/instagram": "^5.1",
|
||||
"socialiteproviders/linkedin": "^5.0",
|
||||
|
|
|
|||
83
composer.lock
generated
83
composer.lock
generated
|
|
@ -4,7 +4,7 @@
|
|||
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
|
||||
"This file is @generated automatically"
|
||||
],
|
||||
"content-hash": "4d71722b650e74b41e1943d3ae9c814f",
|
||||
"content-hash": "19849eb01cb4d8f93733666b53444c5a",
|
||||
"packages": [
|
||||
{
|
||||
"name": "aws/aws-crt-php",
|
||||
|
|
@ -7402,6 +7402,87 @@
|
|||
},
|
||||
"time": "2026-03-20T00:53:21+00:00"
|
||||
},
|
||||
{
|
||||
"name": "simplepie/simplepie",
|
||||
"version": "1.9.0",
|
||||
"source": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/simplepie/simplepie.git",
|
||||
"reference": "76cccb1b2c5dcaf44f304c925ab30c0f48643992"
|
||||
},
|
||||
"dist": {
|
||||
"type": "zip",
|
||||
"url": "https://api.github.com/repos/simplepie/simplepie/zipball/76cccb1b2c5dcaf44f304c925ab30c0f48643992",
|
||||
"reference": "76cccb1b2c5dcaf44f304c925ab30c0f48643992",
|
||||
"shasum": ""
|
||||
},
|
||||
"require": {
|
||||
"ext-pcre": "*",
|
||||
"ext-xml": "*",
|
||||
"ext-xmlreader": "*",
|
||||
"php": ">=7.2.0"
|
||||
},
|
||||
"require-dev": {
|
||||
"donatj/mock-webserver": "^2.7",
|
||||
"friendsofphp/php-cs-fixer": "^2.19 || ^3.8",
|
||||
"mf2/mf2": "^0.5.0",
|
||||
"phpstan/phpstan": "~1.12.2",
|
||||
"phpunit/phpunit": "^8 || ^9 || ^10",
|
||||
"psr/http-client": "^1.0",
|
||||
"psr/http-factory": "^1.0",
|
||||
"psr/simple-cache": "^1 || ^2 || ^3"
|
||||
},
|
||||
"suggest": {
|
||||
"ext-curl": "",
|
||||
"ext-iconv": "",
|
||||
"ext-intl": "",
|
||||
"ext-mbstring": "",
|
||||
"mf2/mf2": "Microformat module that allows for parsing HTML for microformats"
|
||||
},
|
||||
"type": "library",
|
||||
"autoload": {
|
||||
"psr-0": {
|
||||
"SimplePie": "library"
|
||||
},
|
||||
"psr-4": {
|
||||
"SimplePie\\": "src"
|
||||
}
|
||||
},
|
||||
"notification-url": "https://packagist.org/downloads/",
|
||||
"license": [
|
||||
"BSD-3-Clause"
|
||||
],
|
||||
"authors": [
|
||||
{
|
||||
"name": "Ryan Parman",
|
||||
"homepage": "http://ryanparman.com/",
|
||||
"role": "Creator, alumnus developer"
|
||||
},
|
||||
{
|
||||
"name": "Sam Sneddon",
|
||||
"homepage": "https://gsnedders.com/",
|
||||
"role": "Alumnus developer"
|
||||
},
|
||||
{
|
||||
"name": "Ryan McCue",
|
||||
"email": "me@ryanmccue.info",
|
||||
"homepage": "http://ryanmccue.info/",
|
||||
"role": "Developer"
|
||||
}
|
||||
],
|
||||
"description": "A simple Atom/RSS parsing library for PHP",
|
||||
"homepage": "http://simplepie.org/",
|
||||
"keywords": [
|
||||
"atom",
|
||||
"feeds",
|
||||
"rss"
|
||||
],
|
||||
"support": {
|
||||
"issues": "https://github.com/simplepie/simplepie/issues",
|
||||
"source": "https://github.com/simplepie/simplepie/tree/1.9.0"
|
||||
},
|
||||
"time": "2025-09-12T06:34:27+00:00"
|
||||
},
|
||||
{
|
||||
"name": "socialiteproviders/facebook",
|
||||
"version": "4.1.0",
|
||||
|
|
|
|||
|
|
@ -144,7 +144,13 @@
|
|||
'trigger_post_published_at' => 'When the post was published',
|
||||
'fetched_title' => 'Fetched item title',
|
||||
'fetched_link' => 'Fetched item link',
|
||||
'fetched_description' => 'Fetched item description',
|
||||
'fetched_date' => 'Fetched item publish date',
|
||||
'fetched_content' => 'Fetched item full content',
|
||||
'fetched_description' => 'Fetched item summary',
|
||||
'fetched_author' => 'Fetched item author',
|
||||
'fetched_image' => 'Fetched item image URL',
|
||||
'fetched_categories' => 'Fetched item categories',
|
||||
'fetched_enclosure' => 'Fetched item media (audio/video/file)',
|
||||
'fetched_pubdate' => 'Fetched item publish date',
|
||||
'fetched_http' => 'Fetched HTTP item (append a field)',
|
||||
'generated_content' => 'AI-generated post content',
|
||||
|
|
@ -330,6 +336,12 @@
|
|||
'fetch_rss' => [
|
||||
'feed_url' => 'Feed URL',
|
||||
'feed_url_hint' => 'On first run, the watermark is set to "now" so historical items don\'t flood downstream nodes. Subsequent runs only see items newer than the previous poll.',
|
||||
'inspect' => 'Inspect feed',
|
||||
'inspecting' => 'Inspecting…',
|
||||
'inspect_hint' => 'Fetch a sample to discover the available fields for use in downstream nodes.',
|
||||
'inspect_error' => 'Could not read this feed. Check the URL and try again.',
|
||||
'discovered_fields' => 'Available fields',
|
||||
'discovered_empty' => 'No fields found in the latest item.',
|
||||
],
|
||||
'http_request' => [
|
||||
'url' => 'URL',
|
||||
|
|
|
|||
|
|
@ -144,7 +144,13 @@
|
|||
'trigger_post_published_at' => 'Cuándo se publicó el post',
|
||||
'fetched_title' => 'Título del ítem obtenido',
|
||||
'fetched_link' => 'Enlace del ítem obtenido',
|
||||
'fetched_description' => 'Descripción del ítem obtenido',
|
||||
'fetched_date' => 'Fecha de publicación del ítem obtenido',
|
||||
'fetched_content' => 'Contenido completo del ítem obtenido',
|
||||
'fetched_description' => 'Resumen del ítem obtenido',
|
||||
'fetched_author' => 'Autor del ítem obtenido',
|
||||
'fetched_image' => 'URL de la imagen del ítem obtenido',
|
||||
'fetched_categories' => 'Categorías del ítem obtenido',
|
||||
'fetched_enclosure' => 'Multimedia del ítem (audio/vídeo/archivo)',
|
||||
'fetched_pubdate' => 'Fecha de publicación del ítem obtenido',
|
||||
'fetched_http' => 'Ítem HTTP obtenido (añade un campo)',
|
||||
'generated_content' => 'Contenido del post generado por IA',
|
||||
|
|
@ -330,6 +336,12 @@
|
|||
'fetch_rss' => [
|
||||
'feed_url' => 'URL del feed',
|
||||
'feed_url_hint' => 'En la primera ejecución, el watermark se fija en "ahora" para no inundar los siguientes nodos con ítems históricos. Ejecuciones siguientes solo ven ítems nuevos.',
|
||||
'inspect' => 'Inspeccionar feed',
|
||||
'inspecting' => 'Inspeccionando…',
|
||||
'inspect_hint' => 'Obtén una muestra para descubrir los campos disponibles para usar en los siguientes nodos.',
|
||||
'inspect_error' => 'No se pudo leer este feed. Revisa la URL e inténtalo de nuevo.',
|
||||
'discovered_fields' => 'Campos disponibles',
|
||||
'discovered_empty' => 'No se encontraron campos en el último ítem.',
|
||||
],
|
||||
'http_request' => [
|
||||
'url' => 'URL',
|
||||
|
|
|
|||
|
|
@ -144,7 +144,13 @@
|
|||
'trigger_post_published_at' => 'Quando o post foi publicado',
|
||||
'fetched_title' => 'Título do item buscado',
|
||||
'fetched_link' => 'Link do item buscado',
|
||||
'fetched_description' => 'Descrição do item buscado',
|
||||
'fetched_date' => 'Data de publicação do item buscado',
|
||||
'fetched_content' => 'Conteúdo completo do item buscado',
|
||||
'fetched_description' => 'Resumo do item buscado',
|
||||
'fetched_author' => 'Autor do item buscado',
|
||||
'fetched_image' => 'URL da imagem do item buscado',
|
||||
'fetched_categories' => 'Categorias do item buscado',
|
||||
'fetched_enclosure' => 'Mídia do item (áudio/vídeo/arquivo)',
|
||||
'fetched_pubdate' => 'Data de publicação do item buscado',
|
||||
'fetched_http' => 'Item HTTP buscado (adicione um campo)',
|
||||
'generated_content' => 'Conteúdo do post gerado por IA',
|
||||
|
|
@ -330,6 +336,12 @@
|
|||
'fetch_rss' => [
|
||||
'feed_url' => 'URL do feed',
|
||||
'feed_url_hint' => 'Na primeira execução, o watermark é setado pra "agora" pra não inundar os próximos nós com items históricos. Execuções seguintes só veem items novos.',
|
||||
'inspect' => 'Inspecionar feed',
|
||||
'inspecting' => 'Inspecionando…',
|
||||
'inspect_hint' => 'Busca uma amostra pra descobrir os campos disponíveis pra usar nos próximos nós.',
|
||||
'inspect_error' => 'Não foi possível ler esse feed. Confira a URL e tente de novo.',
|
||||
'discovered_fields' => 'Campos disponíveis',
|
||||
'discovered_empty' => 'Nenhum campo encontrado no último item.',
|
||||
],
|
||||
'http_request' => [
|
||||
'url' => 'URL',
|
||||
|
|
|
|||
|
|
@ -1,12 +1,23 @@
|
|||
<script setup lang="ts">
|
||||
import { ref, watch } from 'vue';
|
||||
import { useHttp } from '@inertiajs/vue3';
|
||||
import { IconLoader2, IconRefresh } from '@tabler/icons-vue';
|
||||
import { trans } from 'laravel-vue-i18n';
|
||||
import { inject, ref, watch } from 'vue';
|
||||
|
||||
import { inspectFeed } from '@/actions/App/Http/Controllers/App/AutomationController';
|
||||
import InputError from '@/components/InputError.vue';
|
||||
import { Button } from '@/components/ui/button';
|
||||
import { Input } from '@/components/ui/input';
|
||||
import { Label } from '@/components/ui/label';
|
||||
|
||||
interface DiscoveredField {
|
||||
path: string;
|
||||
sample: string;
|
||||
}
|
||||
|
||||
interface FetchRssConfig {
|
||||
feed_url: string;
|
||||
discovered_fields: DiscoveredField[];
|
||||
}
|
||||
|
||||
const props = defineProps<{
|
||||
|
|
@ -15,20 +26,68 @@ const props = defineProps<{
|
|||
}>();
|
||||
const emit = defineEmits<{ update: [Record<string, unknown>] }>();
|
||||
|
||||
const automationId = inject<string>('automationId', '');
|
||||
|
||||
const local = ref<FetchRssConfig>({
|
||||
feed_url: (props.data.feed_url as string) ?? '',
|
||||
discovered_fields: (props.data.discovered_fields as DiscoveredField[]) ?? [],
|
||||
});
|
||||
|
||||
watch(local, (val) => emit('update', val), { deep: true });
|
||||
|
||||
const inspectHttp = useHttp<{ feed_url: string }, { fields: DiscoveredField[] }>({ feed_url: '' });
|
||||
const isInspecting = ref(false);
|
||||
const inspectError = ref<string | null>(null);
|
||||
|
||||
const expression = (path: string): string => `{{ ${path} }}`;
|
||||
|
||||
const inspect = async () => {
|
||||
if (isInspecting.value || local.value.feed_url.trim() === '') {
|
||||
return;
|
||||
}
|
||||
|
||||
isInspecting.value = true;
|
||||
inspectError.value = null;
|
||||
|
||||
try {
|
||||
inspectHttp.feed_url = local.value.feed_url;
|
||||
const { fields } = await inspectHttp.post(inspectFeed.url(automationId));
|
||||
local.value.discovered_fields = fields;
|
||||
} catch {
|
||||
inspectError.value = trans('automations.config.fetch_rss.inspect_error');
|
||||
} finally {
|
||||
isInspecting.value = false;
|
||||
}
|
||||
};
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<div class="space-y-3">
|
||||
<div class="space-y-4">
|
||||
<div>
|
||||
<Label class="mb-1 block">{{ $t('automations.config.fetch_rss.feed_url') }}</Label>
|
||||
<Input v-model="local.feed_url" placeholder="https://example.com/feed.xml" />
|
||||
<InputError :message="errors?.feed_url" class="mt-1" />
|
||||
<p class="mt-1 text-xs text-foreground/50">{{ $t('automations.config.fetch_rss.feed_url_hint') }}</p>
|
||||
</div>
|
||||
|
||||
<div>
|
||||
<Button type="button" variant="outline" size="sm" :disabled="isInspecting || local.feed_url.trim() === ''" @click="inspect">
|
||||
<IconLoader2 v-if="isInspecting" class="size-4 animate-spin" />
|
||||
<IconRefresh v-else class="size-4" />
|
||||
{{ isInspecting ? $t('automations.config.fetch_rss.inspecting') : $t('automations.config.fetch_rss.inspect') }}
|
||||
</Button>
|
||||
<p class="mt-1 text-xs text-foreground/50">{{ $t('automations.config.fetch_rss.inspect_hint') }}</p>
|
||||
<InputError v-if="inspectError" :message="inspectError" class="mt-1" />
|
||||
</div>
|
||||
|
||||
<div v-if="local.discovered_fields.length > 0">
|
||||
<Label class="mb-1 block">{{ $t('automations.config.fetch_rss.discovered_fields') }}</Label>
|
||||
<ul class="max-h-48 space-y-1.5 overflow-y-auto rounded-md border border-border p-2">
|
||||
<li v-for="field in local.discovered_fields" :key="field.path" class="flex flex-col gap-0.5">
|
||||
<code class="text-xs text-foreground">{{ expression(field.path) }}</code>
|
||||
<span v-if="field.sample" class="truncate text-xs text-foreground/50">{{ field.sample }}</span>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</template>
|
||||
|
|
|
|||
|
|
@ -43,13 +43,29 @@ const providedBy = (node: GraphNode): ExpressionSuggestion[] => {
|
|||
}
|
||||
return [suggestion('trigger.event', 'trigger_event'), suggestion('trigger.fired_at', 'trigger_fired_at')];
|
||||
}
|
||||
case 'fetch_rss':
|
||||
return [
|
||||
case 'fetch_rss': {
|
||||
const aliases = [
|
||||
suggestion('fetched.title', 'fetched_title'),
|
||||
suggestion('fetched.link', 'fetched_link'),
|
||||
suggestion('fetched.date', 'fetched_date'),
|
||||
suggestion('fetched.content', 'fetched_content'),
|
||||
suggestion('fetched.description', 'fetched_description'),
|
||||
suggestion('fetched.pubDate', 'fetched_pubdate'),
|
||||
suggestion('fetched.author', 'fetched_author'),
|
||||
suggestion('fetched.image', 'fetched_image'),
|
||||
suggestion('fetched.categories', 'fetched_categories'),
|
||||
suggestion('fetched.enclosure', 'fetched_enclosure'),
|
||||
];
|
||||
// Fields discovered by inspecting the real feed — the dynamic raw layer
|
||||
// (yt_videoId, media_group.*, dc_creator, …) that aliases can't enumerate.
|
||||
const discovered = Array.isArray(node.data?.discovered_fields)
|
||||
? (node.data.discovered_fields as Array<{ path: string; sample?: string }>).map(
|
||||
(field): ExpressionSuggestion => ({ label: field.path, info: field.sample ?? '' }),
|
||||
)
|
||||
: [];
|
||||
const known = new Set(aliases.map((alias) => alias.label));
|
||||
|
||||
return [...aliases, ...discovered.filter((field) => !known.has(field.label))];
|
||||
}
|
||||
case 'http_request':
|
||||
return [suggestion('fetched', 'fetched_http')];
|
||||
case 'generate':
|
||||
|
|
|
|||
|
|
@ -137,6 +137,7 @@ provide(
|
|||
'automationExpressionCompletions',
|
||||
computed(() => buildExpressionCatalog(selectedNodeId.value, nodes.value, edges.value, variables.value)),
|
||||
);
|
||||
provide('automationId', props.automation.id);
|
||||
|
||||
// Toggled by an expandable CodeEditor (see CodeEditor.vue) to slide out the
|
||||
// side-by-side editing panel, pushing the sidebar left to make room.
|
||||
|
|
|
|||
|
|
@ -224,6 +224,7 @@
|
|||
Route::post('automations/{automation}/pause', [AutomationController::class, 'pause'])->name('app.automations.pause');
|
||||
Route::post('automations/{automation}/runs/{run}/retry', [AutomationController::class, 'retryRun'])->name('app.automations.runs.retry');
|
||||
Route::post('automations/{automation}/test', [AutomationController::class, 'test'])->name('app.automations.test');
|
||||
Route::post('automations/{automation}/feed/inspect', [AutomationController::class, 'inspectFeed'])->name('app.automations.feed.inspect');
|
||||
Route::get('automations/{automation}/runs/{run}', [AutomationController::class, 'showRun'])->name('app.automations.runs.show');
|
||||
|
||||
// API Keys
|
||||
|
|
|
|||
138
tests/Feature/Automation/FeedInspectionTest.php
Normal file
138
tests/Feature/Automation/FeedInspectionTest.php
Normal file
|
|
@ -0,0 +1,138 @@
|
|||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
use App\Enums\UserWorkspace\Role;
|
||||
use App\Models\Automation;
|
||||
use App\Models\User;
|
||||
use App\Models\Workspace;
|
||||
use Illuminate\Support\Facades\Http;
|
||||
|
||||
beforeEach(function () {
|
||||
$this->workspace = Workspace::factory()->create();
|
||||
$this->user = User::factory()->create(['current_workspace_id' => $this->workspace->id]);
|
||||
$this->workspace->members()->attach($this->user->id, ['role' => Role::Admin->value]);
|
||||
$this->user->refresh();
|
||||
});
|
||||
|
||||
it('returns the discovered fields for a feed', function () {
|
||||
Http::fake(['1.1.1.1/*' => Http::response(file_get_contents(base_path('tests/fixtures/feeds/youtube_atom.xml')), 200)]);
|
||||
|
||||
$automation = Automation::factory()->for($this->workspace)->create();
|
||||
|
||||
$response = $this->actingAs($this->user)
|
||||
->postJson(route('app.automations.feed.inspect', $automation->id), [
|
||||
'feed_url' => 'https://1.1.1.1/feed.xml',
|
||||
]);
|
||||
|
||||
$response->assertOk();
|
||||
|
||||
$paths = collect($response->json('fields'))->pluck('path');
|
||||
expect($paths)->toContain('fetched.title')
|
||||
->toContain('fetched.link')
|
||||
->toContain('fetched.yt_videoId')
|
||||
->toContain('fetched.media_group.media_thumbnail.url');
|
||||
});
|
||||
|
||||
it('resolves workspace variables in the feed url before fetching', function () {
|
||||
Http::fake(['1.1.1.1/*' => Http::response(file_get_contents(base_path('tests/fixtures/feeds/youtube_atom.xml')), 200)]);
|
||||
|
||||
$automation = Automation::factory()->for($this->workspace)->create([
|
||||
'variables' => [['key' => 'HOST', 'value' => '1.1.1.1']],
|
||||
]);
|
||||
|
||||
$this->actingAs($this->user)
|
||||
->postJson(route('app.automations.feed.inspect', $automation->id), [
|
||||
'feed_url' => 'https://{{ variables.HOST }}/feed.xml',
|
||||
])
|
||||
->assertOk();
|
||||
|
||||
Http::assertSent(fn ($request) => str_contains($request->url(), '1.1.1.1/feed.xml'));
|
||||
});
|
||||
|
||||
it('returns 422 for a malformed feed', function () {
|
||||
Http::fake(['1.1.1.1/*' => Http::response('not a feed', 200)]);
|
||||
|
||||
$automation = Automation::factory()->for($this->workspace)->create();
|
||||
|
||||
$this->actingAs($this->user)
|
||||
->postJson(route('app.automations.feed.inspect', $automation->id), [
|
||||
'feed_url' => 'https://1.1.1.1/feed.xml',
|
||||
])
|
||||
->assertStatus(422);
|
||||
});
|
||||
|
||||
it('validates that feed_url is required', function () {
|
||||
$automation = Automation::factory()->for($this->workspace)->create();
|
||||
|
||||
$this->actingAs($this->user)
|
||||
->postJson(route('app.automations.feed.inspect', $automation->id), [])
|
||||
->assertStatus(422)
|
||||
->assertJsonValidationErrorFor('feed_url');
|
||||
});
|
||||
|
||||
it('persists discovered_fields stored on a fetch_rss node through an update', function () {
|
||||
$automation = Automation::factory()->for($this->workspace)->create();
|
||||
|
||||
$this->actingAs($this->user)
|
||||
->put(route('app.automations.update', $automation->id), [
|
||||
'nodes' => [[
|
||||
'id' => 'fetch_1',
|
||||
'type' => 'fetch_rss',
|
||||
'position' => ['x' => 0, 'y' => 0],
|
||||
'data' => [
|
||||
'feed_url' => 'https://example.com/feed.xml',
|
||||
'discovered_fields' => [
|
||||
['path' => 'fetched.yt_videoId', 'sample' => 'abc123'],
|
||||
],
|
||||
],
|
||||
]],
|
||||
])
|
||||
->assertRedirect();
|
||||
|
||||
$node = collect($automation->fresh()->nodes)->firstWhere('id', 'fetch_1');
|
||||
expect(data_get($node, 'data.discovered_fields.0.path'))->toBe('fetched.yt_videoId');
|
||||
});
|
||||
|
||||
it('accepts a feed_url containing a workflow-variable expression', function () {
|
||||
$automation = Automation::factory()->for($this->workspace)->create();
|
||||
|
||||
$this->actingAs($this->user)
|
||||
->put(route('app.automations.update', $automation->id), [
|
||||
'nodes' => [[
|
||||
'id' => 'fetch_1',
|
||||
'type' => 'fetch_rss',
|
||||
'position' => ['x' => 0, 'y' => 0],
|
||||
'data' => ['feed_url' => 'https://www.youtube.com/feeds/videos.xml?channel_id={{ variables.CHANNEL_ID }}'],
|
||||
]],
|
||||
])
|
||||
->assertRedirect()
|
||||
->assertSessionHasNoErrors();
|
||||
});
|
||||
|
||||
it('rejects a feed_url that is not a URL even after stripping expressions', function () {
|
||||
$automation = Automation::factory()->for($this->workspace)->create();
|
||||
|
||||
$this->actingAs($this->user)
|
||||
->put(route('app.automations.update', $automation->id), [
|
||||
'nodes' => [[
|
||||
'id' => 'fetch_1',
|
||||
'type' => 'fetch_rss',
|
||||
'position' => ['x' => 0, 'y' => 0],
|
||||
'data' => ['feed_url' => 'not a url {{ variables.X }}'],
|
||||
]],
|
||||
])
|
||||
->assertSessionHasErrors('nodes.0.data.feed_url');
|
||||
});
|
||||
|
||||
it('forbids inspecting a feed for an automation in another workspace', function () {
|
||||
Http::fake(['1.1.1.1/*' => Http::response('<rss version="2.0"><channel></channel></rss>', 200)]);
|
||||
|
||||
$automation = Automation::factory()->for(Workspace::factory()->create())->create();
|
||||
|
||||
$this->actingAs($this->user)
|
||||
->postJson(route('app.automations.feed.inspect', $automation->id), [
|
||||
'feed_url' => 'https://1.1.1.1/feed.xml',
|
||||
])
|
||||
->assertForbidden();
|
||||
});
|
||||
|
|
@ -28,7 +28,7 @@
|
|||
<rss version="2.0"><channel>
|
||||
<item><title>Older</title><link>https://1.1.1.1/a</link><guid>a</guid><pubDate>Sat, 01 Feb 2025 12:00:00 +0000</pubDate></item>
|
||||
<item><title>Newer1</title><link>https://1.1.1.1/b</link><guid>b</guid><pubDate>Sun, 01 Jun 2025 12:00:00 +0000</pubDate></item>
|
||||
<item><title>Newer2</title><link>https://1.1.1.1/c</link><guid>c</guid><pubDate>Mon, 15 Jun 2025 12:00:00 +0000</pubDate></item>
|
||||
<item><title>Newer2</title><link>https://1.1.1.1/c</link><guid>c</guid><pubDate>Sun, 15 Jun 2025 12:00:00 +0000</pubDate></item>
|
||||
</channel></rss>
|
||||
XML;
|
||||
|
||||
|
|
@ -256,7 +256,31 @@
|
|||
|
||||
$state = AutomationNodeState::for($automation->id, 'fetch_1');
|
||||
expect(CarbonImmutable::parse($state->data['last_item_date'])->toIso8601String())
|
||||
->toBe(CarbonImmutable::parse('Mon, 15 Jun 2025 12:00:00 +0000')->toIso8601String());
|
||||
->toBe(CarbonImmutable::parse('Sun, 15 Jun 2025 12:00:00 +0000')->toIso8601String());
|
||||
});
|
||||
|
||||
it('processes an Atom feed (YouTube) that the old RSS-only parser dropped', function () {
|
||||
Carbon::setTestNow('2026-06-16 10:00:00');
|
||||
Http::fake(['1.1.1.1/*' => Http::response(file_get_contents(base_path('tests/fixtures/feeds/youtube_atom.xml')), 200)]);
|
||||
|
||||
$automation = Automation::factory()->active()->create();
|
||||
|
||||
// Watermark predates the single entry, so it counts as new.
|
||||
AutomationNodeState::create([
|
||||
'automation_id' => $automation->id,
|
||||
'node_id' => 'fetch_1',
|
||||
'data' => ['last_item_date' => '2026-01-01T00:00:00+00:00'],
|
||||
]);
|
||||
|
||||
$run = AutomationRun::factory()->for($automation)->create(['current_node_id' => 'fetch_1']);
|
||||
|
||||
$result = app(RunFetchRssNode::class)($run, ['feed_url' => 'https://1.1.1.1/feed']);
|
||||
|
||||
expect($result->status)->toBe(NodeRunStatus::Completed);
|
||||
expect($result->output['fetch']['count'])->toBe(1);
|
||||
expect($result->output['fetched']['title'])->toBe('Ninguém aguenta o TikTok Shop');
|
||||
expect($result->output['fetched']['link'])->toBe('https://www.youtube.com/watch?v=bIQVeW4sTcE');
|
||||
expect($result->output['fetched']['yt_videoId'])->toBe('bIQVeW4sTcE');
|
||||
});
|
||||
|
||||
it('fails when feed_url is missing', function () {
|
||||
|
|
|
|||
81
tests/Unit/Automation/FeedParserTest.php
Normal file
81
tests/Unit/Automation/FeedParserTest.php
Normal file
|
|
@ -0,0 +1,81 @@
|
|||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
use App\Services\Automation\FeedParser;
|
||||
|
||||
function feedFixture(string $name): string
|
||||
{
|
||||
return file_get_contents(base_path("tests/fixtures/feeds/{$name}.xml"));
|
||||
}
|
||||
|
||||
beforeEach(function () {
|
||||
$this->parser = new FeedParser;
|
||||
});
|
||||
|
||||
it('parses an Atom (YouTube) feed with aliases and raw namespaced fields', function () {
|
||||
$item = $this->parser->parse(feedFixture('youtube_atom'))[0];
|
||||
|
||||
// Aliases resolve cross-format: link comes from the Atom href attribute,
|
||||
// date from <published>, author from <author><name>.
|
||||
expect($item['title'])->toBe('Ninguém aguenta o TikTok Shop');
|
||||
expect($item['link'])->toBe('https://www.youtube.com/watch?v=bIQVeW4sTcE');
|
||||
expect($item['date'])->toBe('2026-06-15T20:24:32+00:00');
|
||||
expect($item['author'])->toBe('Beta Boechat');
|
||||
expect($item['key'])->toBe('yt:video:bIQVeW4sTcE');
|
||||
|
||||
// Raw namespaced fields stay reachable.
|
||||
expect($item['yt_videoId'])->toBe('bIQVeW4sTcE');
|
||||
expect(data_get($item, 'media_group.media_thumbnail.url'))
|
||||
->toBe('https://i3.ytimg.com/vi/bIQVeW4sTcE/hqdefault.jpg');
|
||||
expect(data_get($item, 'media_group.media_description'))
|
||||
->toBe('Descrição completa do vídeo aqui.');
|
||||
});
|
||||
|
||||
it('parses an RSS 2.0 feed with dc and content namespaces', function () {
|
||||
$item = $this->parser->parse(feedFixture('rss_namespaces'))[0];
|
||||
|
||||
expect($item['title'])->toBe('Polish is here');
|
||||
expect($item['link'])->toBe('https://feedback.changelogfy.com/en/polish-is-here');
|
||||
expect($item['author'])->toBe('Paulo Castellano');
|
||||
expect($item['key'])->toBe('16092');
|
||||
|
||||
// Extension fields the old parser threw away.
|
||||
expect($item['dc_creator'])->toBe('Paulo Castellano');
|
||||
expect($item['content_encoded'])->toBe('<p>You can now serve your changelogs fully translated in Polish.</p>');
|
||||
expect($item['categories'])->toBe(['Feature', 'i18n']);
|
||||
});
|
||||
|
||||
it('parses a podcast feed exposing the enclosure and itunes fields', function () {
|
||||
$item = $this->parser->parse(feedFixture('podcast'))[0];
|
||||
|
||||
expect(data_get($item, 'enclosure.url'))->toBe('https://cdn.changelog.com/the-changelog-682.mp3');
|
||||
expect(data_get($item, 'enclosure.type'))->toBe('audio/mpeg');
|
||||
expect($item['itunes_duration'])->toBe('1:46:28');
|
||||
expect($item['key'])->toBe('changelog.com/1/2829');
|
||||
});
|
||||
|
||||
it('preserves backward-compatible RSS 2.0 alias keys', function () {
|
||||
$item = $this->parser->parse(feedFixture('rss_namespaces'))[0];
|
||||
|
||||
// Automations saved before multi-format support reference these exact keys.
|
||||
expect($item)->toHaveKeys(['title', 'link', 'description', 'pubDate', 'key']);
|
||||
expect($item['pubDate'])->toBe($item['date']);
|
||||
});
|
||||
|
||||
it('keeps a falsy element from a tag that only carries attributes', function () {
|
||||
$item = $this->parser->parse(feedFixture('podcast'))[0];
|
||||
|
||||
// <itunes:image href="..."/> has no text — the attribute object is exposed.
|
||||
expect(data_get($item, 'itunes_image.href'))->toBe('https://cdn.changelog.com/cover.png');
|
||||
});
|
||||
|
||||
it('returns null for a body that is not a valid feed', function () {
|
||||
expect($this->parser->parse('this is not xml at all'))->toBeNull();
|
||||
});
|
||||
|
||||
it('returns an empty list for a valid but empty feed', function () {
|
||||
$empty = '<?xml version="1.0"?><rss version="2.0"><channel></channel></rss>';
|
||||
|
||||
expect($this->parser->parse($empty))->toBe([]);
|
||||
});
|
||||
20
tests/fixtures/feeds/podcast.xml
vendored
Normal file
20
tests/fixtures/feeds/podcast.xml
vendored
Normal file
|
|
@ -0,0 +1,20 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<rss xmlns:itunes="http://www.itunes.com/dtds/podcast-1.0.dtd" version="2.0">
|
||||
<channel>
|
||||
<title>The Changelog</title>
|
||||
<link>https://changelog.com/podcast</link>
|
||||
<description>Software news</description>
|
||||
<item>
|
||||
<title>From open source hits to OpenAI</title>
|
||||
<guid isPermaLink="false">changelog.com/1/2829</guid>
|
||||
<link>https://changelog.com/podcast/682</link>
|
||||
<pubDate>Fri, 05 Jun 2026 19:00:00 +0000</pubDate>
|
||||
<enclosure type="audio/mpeg" length="153465091" url="https://cdn.changelog.com/the-changelog-682.mp3"/>
|
||||
<description>This week I am talking with Max Stoiber.</description>
|
||||
<itunes:episodeType>full</itunes:episodeType>
|
||||
<itunes:image href="https://cdn.changelog.com/cover.png"/>
|
||||
<itunes:duration>1:46:28</itunes:duration>
|
||||
<itunes:explicit>false</itunes:explicit>
|
||||
</item>
|
||||
</channel>
|
||||
</rss>
|
||||
20
tests/fixtures/feeds/rss_namespaces.xml
vendored
Normal file
20
tests/fixtures/feeds/rss_namespaces.xml
vendored
Normal file
|
|
@ -0,0 +1,20 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<rss xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:content="http://purl.org/rss/1.0/modules/content/" xmlns:atom="http://www.w3.org/2005/Atom" version="2.0" xmlns:media="http://search.yahoo.com/mrss/">
|
||||
<channel>
|
||||
<title>Changelogfy</title>
|
||||
<description>Changelogfy</description>
|
||||
<link>https://feedback.changelogfy.com</link>
|
||||
<atom:link href="https://feedback.changelogfy.com/rss" rel="self" type="application/rss+xml"/>
|
||||
<item>
|
||||
<title><![CDATA[Polish is here]]></title>
|
||||
<description><![CDATA[You can now serve your changelogs in Polish.]]></description>
|
||||
<link>https://feedback.changelogfy.com/en/polish-is-here</link>
|
||||
<guid>16092</guid>
|
||||
<dc:creator>Paulo Castellano</dc:creator>
|
||||
<pubDate>Thu, 05 Mar 2026 16:20:09 +0000</pubDate>
|
||||
<category><![CDATA[Feature]]></category>
|
||||
<category><![CDATA[i18n]]></category>
|
||||
<content:encoded><![CDATA[<p>You can now serve your changelogs fully translated in Polish.</p>]]></content:encoded>
|
||||
</item>
|
||||
</channel>
|
||||
</rss>
|
||||
36
tests/fixtures/feeds/youtube_atom.xml
vendored
Normal file
36
tests/fixtures/feeds/youtube_atom.xml
vendored
Normal file
|
|
@ -0,0 +1,36 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<feed xmlns:yt="http://www.youtube.com/xml/schemas/2015" xmlns:media="http://search.yahoo.com/mrss/" xmlns="http://www.w3.org/2005/Atom">
|
||||
<link rel="self" href="http://www.youtube.com/feeds/videos.xml?channel_id=UC123"/>
|
||||
<id>yt:channel:123</id>
|
||||
<yt:channelId>123</yt:channelId>
|
||||
<title>Beta Boechat</title>
|
||||
<link rel="alternate" href="https://www.youtube.com/channel/UC123"/>
|
||||
<author>
|
||||
<name>Beta Boechat</name>
|
||||
<uri>https://www.youtube.com/channel/UC123</uri>
|
||||
</author>
|
||||
<published>2015-08-11T18:07:31+00:00</published>
|
||||
<entry>
|
||||
<id>yt:video:bIQVeW4sTcE</id>
|
||||
<yt:videoId>bIQVeW4sTcE</yt:videoId>
|
||||
<yt:channelId>123</yt:channelId>
|
||||
<title>Ninguém aguenta o TikTok Shop</title>
|
||||
<link rel="alternate" href="https://www.youtube.com/watch?v=bIQVeW4sTcE"/>
|
||||
<author>
|
||||
<name>Beta Boechat</name>
|
||||
<uri>https://www.youtube.com/channel/UC123</uri>
|
||||
</author>
|
||||
<published>2026-06-15T20:24:32+00:00</published>
|
||||
<updated>2026-06-15T21:15:11+00:00</updated>
|
||||
<media:group>
|
||||
<media:title>Ninguém aguenta o TikTok Shop</media:title>
|
||||
<media:content url="https://www.youtube.com/v/bIQVeW4sTcE?version=3" type="application/x-shockwave-flash" width="640" height="390"/>
|
||||
<media:thumbnail url="https://i3.ytimg.com/vi/bIQVeW4sTcE/hqdefault.jpg" width="480" height="360"/>
|
||||
<media:description>Descrição completa do vídeo aqui.</media:description>
|
||||
<media:community>
|
||||
<media:starRating count="3389" average="5.00" min="1" max="5"/>
|
||||
<media:statistics views="20085"/>
|
||||
</media:community>
|
||||
</media:group>
|
||||
</entry>
|
||||
</feed>
|
||||
Loading…
Reference in a new issue