415 lines
15 KiB
TypeScript
415 lines
15 KiB
TypeScript
#!/usr/bin/env tsx
|
|
/**
|
|
* Multi-provider comparison harness for the brand-analysis (onboarding)
|
|
* pipeline. Builds the production analysis context once (same prompt, same
|
|
* schema, same website excerpt that `analyzeBrand` would use) and runs every
|
|
* configured direct-API provider in parallel against it. Reports elapsed
|
|
* time, model, branded-prompt count, and tag vocabulary so you can compare
|
|
* providers without prompt drift.
|
|
*
|
|
* Providers compared (each must have its API key in env to participate):
|
|
* - openai-api: OpenAI Responses API + native web_search tool
|
|
* - openrouter: OpenRouter chat/completions + plugins:[{web,native}]
|
|
* - anthropic-api: Anthropic Messages API + web_search_20250305 tool
|
|
* - mistral-api: Mistral Conversations API + web_search tool
|
|
*
|
|
* Usage:
|
|
* pnpm --filter @workspace/lib compare:onboarding nike.com
|
|
* pnpm --filter @workspace/lib compare:onboarding nike.com --max-prompts 30 --max-competitors 12
|
|
* pnpm --filter @workspace/lib compare:onboarding nike.com --only anthropic-api,openrouter
|
|
* pnpm --filter @workspace/lib compare:onboarding nike.com --skip mistral-api
|
|
* pnpm --filter @workspace/lib compare:onboarding nike.com --env-file ~/code/Getcito/apps/web/.env
|
|
*
|
|
* Reads `<repo>/apps/web/.env` and `<repo>/.env` automatically; --env-file
|
|
* PATH overrides. Real env vars always win over .env entries.
|
|
*/
|
|
import { parseArgs } from "node:util";
|
|
import { readFile } from "node:fs/promises";
|
|
import { homedir } from "node:os";
|
|
import { join, resolve } from "node:path";
|
|
import { getProvider } from "../src/providers";
|
|
import { RESEARCH_PROVIDER_PREFERENCE, type ResearchProviderId } from "../src/onboarding/llm";
|
|
import {
|
|
buildAnalysisContext,
|
|
normalizeAnalysisResult,
|
|
type AnalysisContext,
|
|
type OnboardingSuggestion,
|
|
} from "../src/onboarding/analyze";
|
|
import { isPromptBranded } from "../src/tag-utils";
|
|
|
|
type ProviderId = ResearchProviderId;
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Helpers
|
|
// ---------------------------------------------------------------------------
|
|
|
|
function resolveHomePath(p: string): string {
|
|
if (p === "~") return homedir();
|
|
if (p.startsWith("~/")) return resolve(homedir(), p.slice(2));
|
|
return resolve(p);
|
|
}
|
|
|
|
/**
|
|
* Race a promise against a timeout and reject with a clear message if it
|
|
* didn't resolve in time. Note: the underlying LLM call keeps running and
|
|
* billing tokens — providers that accept an AbortSignal would need plumbing
|
|
* through StructuredResearchOptions to actually stop.
|
|
*/
|
|
async function withTimeout<T>(promise: Promise<T>, timeoutMs: number, label: string): Promise<T> {
|
|
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
const timeoutPromise = new Promise<never>((_, reject) => {
|
|
timer = setTimeout(
|
|
() => reject(new Error(`${label} timed out after ${(timeoutMs / 1000).toFixed(0)}s`)),
|
|
timeoutMs,
|
|
);
|
|
});
|
|
try {
|
|
return await Promise.race([promise, timeoutPromise]);
|
|
} finally {
|
|
if (timer) clearTimeout(timer);
|
|
}
|
|
}
|
|
|
|
async function loadDotEnv(path: string): Promise<void> {
|
|
let contents: string;
|
|
try {
|
|
contents = await readFile(path, "utf8");
|
|
} catch {
|
|
return;
|
|
}
|
|
for (const line of contents.split("\n")) {
|
|
const trimmed = line.trim();
|
|
if (!trimmed || trimmed.startsWith("#")) continue;
|
|
const eq = trimmed.indexOf("=");
|
|
if (eq < 0) continue;
|
|
const key = trimmed.slice(0, eq).trim();
|
|
let value = trimmed.slice(eq + 1).trim();
|
|
if ((value.startsWith('"') && value.endsWith('"')) || (value.startsWith("'") && value.endsWith("'"))) {
|
|
value = value.slice(1, -1);
|
|
}
|
|
if (process.env[key] === undefined) process.env[key] = value;
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Per-provider runner
|
|
// ---------------------------------------------------------------------------
|
|
|
|
interface RunResult {
|
|
providerId: ProviderId;
|
|
model: string;
|
|
elapsedMs: number;
|
|
suggestion: OnboardingSuggestion;
|
|
/** Number of suggestedPrompts production would classify as "branded". */
|
|
brandedCount: number;
|
|
}
|
|
|
|
interface RunFailure {
|
|
providerId: ProviderId;
|
|
error: string;
|
|
elapsedMs: number;
|
|
}
|
|
|
|
async function runProvider(providerId: ProviderId, ctx: AnalysisContext, timeoutMs: number): Promise<RunResult> {
|
|
const provider = getProvider(providerId);
|
|
if (!provider.isConfigured()) throw new Error(`${providerId}: not configured (missing API key in env)`);
|
|
if (!provider.runStructuredResearch) throw new Error(`${providerId}: provider doesn't implement structured research`);
|
|
|
|
const start = Date.now();
|
|
const result = await withTimeout(
|
|
provider.runStructuredResearch({ prompt: ctx.prompt, schema: ctx.schema }),
|
|
timeoutMs,
|
|
`${providerId} engine`,
|
|
);
|
|
const elapsedMs = Date.now() - start;
|
|
const suggestion = normalizeAnalysisResult(result.object, ctx);
|
|
const brandedCount = suggestion.suggestedPrompts.filter((p) =>
|
|
isPromptBranded(p.prompt, suggestion.brandName, suggestion.website),
|
|
).length;
|
|
return {
|
|
providerId,
|
|
model: result.modelVersion ?? "(unknown)",
|
|
elapsedMs,
|
|
suggestion,
|
|
brandedCount,
|
|
};
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Reporting
|
|
// ---------------------------------------------------------------------------
|
|
|
|
function formatMs(ms: number): string {
|
|
return `${(ms / 1000).toFixed(2)}s`;
|
|
}
|
|
|
|
function tagVocabOf(s: OnboardingSuggestion): string[] {
|
|
const set = new Set<string>();
|
|
for (const p of s.suggestedPrompts) for (const t of p.tags) set.add(t);
|
|
return [...set].sort();
|
|
}
|
|
|
|
function summary(r: RunResult): string {
|
|
const tags = tagVocabOf(r.suggestion);
|
|
return [
|
|
` provider: ${r.providerId}`,
|
|
` model: ${r.model}`,
|
|
` elapsed: ${formatMs(r.elapsedMs)}`,
|
|
` brand: ${r.suggestion.brandName}`,
|
|
` domains: ${r.suggestion.additionalDomains.length}`,
|
|
` aliases: ${r.suggestion.aliases.length}`,
|
|
` competit: ${r.suggestion.competitors.length}`,
|
|
` prompts: ${r.suggestion.suggestedPrompts.length} (${r.brandedCount} branded by name)`,
|
|
` tag vocab: ${tags.length} distinct — ${tags.join(", ") || "(none)"}`,
|
|
].join("\n");
|
|
}
|
|
|
|
/** RFC 4180-style CSV escaping: wrap in quotes if needed, double internal quotes. */
|
|
function csvEscape(value: string | number | undefined | null): string {
|
|
if (value === undefined || value === null) return "";
|
|
const s = String(value);
|
|
if (s.includes(",") || s.includes('"') || s.includes("\n") || s.includes("\r")) {
|
|
return `"${s.replace(/"/g, '""')}"`;
|
|
}
|
|
return s;
|
|
}
|
|
|
|
const CSV_HEADERS = [
|
|
"provider",
|
|
"status",
|
|
"model",
|
|
"elapsed_s",
|
|
"brand",
|
|
"domains",
|
|
"aliases",
|
|
"competitors",
|
|
"prompts",
|
|
"branded_by_name",
|
|
"tag_vocab_size",
|
|
"tags",
|
|
"error",
|
|
] as const;
|
|
|
|
function rowFromResult(r: RunResult): string {
|
|
const tags = tagVocabOf(r.suggestion);
|
|
const cells = [
|
|
r.providerId,
|
|
"ok",
|
|
r.model,
|
|
(r.elapsedMs / 1000).toFixed(2),
|
|
r.suggestion.brandName,
|
|
r.suggestion.additionalDomains.length,
|
|
r.suggestion.aliases.length,
|
|
r.suggestion.competitors.length,
|
|
r.suggestion.suggestedPrompts.length,
|
|
r.brandedCount,
|
|
tags.length,
|
|
tags.join(","),
|
|
"",
|
|
];
|
|
return cells.map(csvEscape).join(",");
|
|
}
|
|
|
|
function rowFromFailure(f: RunFailure): string {
|
|
// Header has 13 columns: provider, status, model, elapsed_s, then 8 empty
|
|
// metric/output cells, then error.
|
|
const cells: (string | number)[] = [
|
|
f.providerId,
|
|
"failed",
|
|
"", // model
|
|
(f.elapsedMs / 1000).toFixed(2),
|
|
...Array(8).fill(""),
|
|
f.error.split("\n")[0],
|
|
];
|
|
return cells.map(csvEscape).join(",");
|
|
}
|
|
|
|
const PROMPT_CSV_HEADERS = ["provider", "prompt_index", "prompt", "tags"] as const;
|
|
|
|
function promptRowsFromResult(r: RunResult): string[] {
|
|
return r.suggestion.suggestedPrompts.map((p, i) => {
|
|
const cells = [r.providerId, i + 1, p.prompt, p.tags.join(",")];
|
|
return cells.map(csvEscape).join(",");
|
|
});
|
|
}
|
|
|
|
function tabulatedComparison(results: RunResult[]): string {
|
|
if (results.length === 0) return "";
|
|
const sorted = [...results].sort((a, b) => a.elapsedMs - b.elapsedMs);
|
|
const lines = ["", "----- COMPARISON (sorted by elapsed time, fastest first) -----", ""];
|
|
const colName = "provider";
|
|
const nameWidth = Math.max(colName.length, ...sorted.map((r) => r.providerId.length));
|
|
lines.push(
|
|
` ${colName.padEnd(nameWidth)} ${"time".padStart(8)} ${"branded".padStart(8)} ${"competit".padStart(8)} ${"tag vocab".padStart(10)}`,
|
|
);
|
|
for (const r of sorted) {
|
|
const tags = tagVocabOf(r.suggestion);
|
|
lines.push(
|
|
` ${r.providerId.padEnd(nameWidth)} ${formatMs(r.elapsedMs).padStart(8)} ${String(r.brandedCount).padStart(8)} ${String(r.suggestion.competitors.length).padStart(8)} ${String(tags.length).padStart(10)}`,
|
|
);
|
|
}
|
|
return lines.join("\n");
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Main
|
|
// ---------------------------------------------------------------------------
|
|
|
|
function parseList(value: string | undefined): string[] {
|
|
return (value ?? "")
|
|
.split(",")
|
|
.map((s) => s.trim())
|
|
.filter(Boolean);
|
|
}
|
|
|
|
function isProviderId(id: string): id is ProviderId {
|
|
return (RESEARCH_PROVIDER_PREFERENCE as readonly string[]).includes(id);
|
|
}
|
|
|
|
async function main() {
|
|
const { values, positionals } = parseArgs({
|
|
args: process.argv.slice(2),
|
|
options: {
|
|
"max-prompts": { type: "string" },
|
|
"max-competitors": { type: "string" },
|
|
only: { type: "string" },
|
|
skip: { type: "string" },
|
|
"env-file": { type: "string" },
|
|
timeout: { type: "string", default: "180" },
|
|
help: { type: "boolean", short: "h" },
|
|
},
|
|
allowPositionals: true,
|
|
});
|
|
|
|
if (values.help || positionals.length === 0) {
|
|
console.error("Usage: tsx compare-onboarding.ts <website> [options]");
|
|
console.error("");
|
|
console.error("Options:");
|
|
console.error(" --max-prompts N Pass through to analyzeBrand (default: production default)");
|
|
console.error(" --max-competitors N Pass through to analyzeBrand (default: production default)");
|
|
console.error(` --only IDS Comma-separated provider IDs (default all of: ${RESEARCH_PROVIDER_PREFERENCE.join(",")})`);
|
|
console.error(" --skip IDS Comma-separated provider IDs to skip");
|
|
console.error(" --env-file PATH Load API keys from this .env before defaults");
|
|
console.error(" --timeout SECONDS Hard timeout per provider (default 180)");
|
|
console.error("");
|
|
console.error("Each provider needs its own API key in env to participate;");
|
|
console.error("missing-key providers are silently skipped, not treated as failures.");
|
|
console.error("");
|
|
console.error("Example:");
|
|
console.error(" pnpm --filter @workspace/lib compare:onboarding nike.com");
|
|
console.error(" pnpm --filter @workspace/lib compare:onboarding nike.com --only anthropic-api,openrouter");
|
|
console.error(" pnpm --filter @workspace/lib compare:onboarding nike.com --env-file ~/code/Getcito/apps/web/.env");
|
|
process.exit(values.help ? 0 : 1);
|
|
}
|
|
|
|
const website = positionals[0];
|
|
const maxPrompts = values["max-prompts"] !== undefined ? Number(values["max-prompts"]) : undefined;
|
|
const maxCompetitors = values["max-competitors"] !== undefined ? Number(values["max-competitors"]) : undefined;
|
|
const timeoutMs = Number(values.timeout) * 1000;
|
|
if (!Number.isFinite(timeoutMs) || timeoutMs < 1000) {
|
|
console.error("--timeout must be a positive number of seconds (>=1)");
|
|
process.exit(1);
|
|
}
|
|
|
|
if (values["env-file"]) {
|
|
const explicit = resolveHomePath(values["env-file"]);
|
|
try {
|
|
await readFile(explicit, "utf8");
|
|
} catch (err) {
|
|
console.error(`--env-file ${explicit} could not be read: ${err instanceof Error ? err.message : err}`);
|
|
process.exit(1);
|
|
}
|
|
await loadDotEnv(explicit);
|
|
}
|
|
const repoRoot = join(import.meta.dirname ?? __dirname, "..", "..", "..");
|
|
await loadDotEnv(join(repoRoot, "apps", "web", ".env"));
|
|
await loadDotEnv(join(repoRoot, ".env"));
|
|
|
|
// Determine which providers to run.
|
|
const onlyList = parseList(values.only);
|
|
const skipList = parseList(values.skip);
|
|
for (const id of [...onlyList, ...skipList]) {
|
|
if (!isProviderId(id)) {
|
|
console.error(`Unknown provider id: "${id}". Valid: ${RESEARCH_PROVIDER_PREFERENCE.join(", ")}`);
|
|
process.exit(1);
|
|
}
|
|
}
|
|
const targets = (onlyList.length > 0 ? onlyList : [...RESEARCH_PROVIDER_PREFERENCE]).filter(
|
|
(id): id is ProviderId => isProviderId(id) && !skipList.includes(id),
|
|
);
|
|
if (targets.length === 0) {
|
|
console.error("No providers to run after applying --only / --skip filters.");
|
|
process.exit(1);
|
|
}
|
|
|
|
// Build the same analysis context production would build — once. Every
|
|
// provider gets identical prompt + schema + excerpt.
|
|
const ctx = await buildAnalysisContext({
|
|
website,
|
|
...(maxCompetitors !== undefined ? { maxCompetitors } : {}),
|
|
...(maxPrompts !== undefined ? { maxPrompts } : {}),
|
|
});
|
|
|
|
const results: RunResult[] = [];
|
|
const failures: RunFailure[] = [];
|
|
|
|
const runs = targets.map(async (providerId): Promise<void> => {
|
|
const provider = getProvider(providerId);
|
|
if (!provider.isConfigured()) {
|
|
console.error(`○ ${providerId}: skipped (no API key configured)`);
|
|
return;
|
|
}
|
|
console.error(`→ ${providerId}: starting`);
|
|
const start = Date.now();
|
|
try {
|
|
const result = await runProvider(providerId, ctx, timeoutMs);
|
|
console.error(`✓ ${providerId}: done in ${formatMs(result.elapsedMs)}`);
|
|
results.push(result);
|
|
} catch (err) {
|
|
const elapsedMs = Date.now() - start;
|
|
const message = err instanceof Error ? err.message : String(err);
|
|
console.error(`✗ ${providerId}: failed after ${formatMs(elapsedMs)} — ${message.split("\n")[0]}`);
|
|
failures.push({ providerId, error: message, elapsedMs });
|
|
}
|
|
});
|
|
await Promise.all(runs);
|
|
|
|
// Print full per-provider summary blocks + tabulated comparison to stderr
|
|
// (human-readable). Stdout gets just the CSV so `> results.csv` captures
|
|
// clean parseable output.
|
|
if (results.length > 0) {
|
|
console.error("\n========== SUMMARY ==========");
|
|
for (const r of results) {
|
|
console.error(`\n[${r.providerId}]`);
|
|
console.error(summary(r));
|
|
}
|
|
}
|
|
for (const f of failures) {
|
|
console.error(`\n[${f.providerId}] FAILED`);
|
|
console.error(` ${f.error.split("\n").join("\n ")}`);
|
|
}
|
|
if (results.length >= 2) {
|
|
console.error(tabulatedComparison(results));
|
|
}
|
|
|
|
// Two CSV blocks on stdout, separated by a blank line:
|
|
// 1. Summary — one row per provider (success or failure)
|
|
// 2. Prompts — one row per (provider, prompt) so you can compare the
|
|
// actual prompt text + tags side-by-side across providers
|
|
// awk '/^$/{n++;next} {print > "out"n".csv"}' splits them; pandas can
|
|
// read one section at a time with skiprows + nrows.
|
|
console.log(CSV_HEADERS.join(","));
|
|
for (const r of results) console.log(rowFromResult(r));
|
|
for (const f of failures) console.log(rowFromFailure(f));
|
|
|
|
console.log("");
|
|
console.log(PROMPT_CSV_HEADERS.join(","));
|
|
for (const r of results) for (const row of promptRowsFromResult(r)) console.log(row);
|
|
|
|
if (results.length === 0) process.exit(1);
|
|
}
|
|
|
|
main().catch((err) => {
|
|
console.error("Error:", err instanceof Error ? err.stack || err.message : err);
|
|
process.exit(1);
|
|
});
|