From a6ff74caf3e3c15fc86ea774844c86145e2385e3 Mon Sep 17 00:00:00 2001 From: Miguel Angel Simon Sierra Date: Fri, 26 Jun 2026 13:11:35 -0400 Subject: [PATCH 01/55] feat(media-use): provider registry contract (U1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ordered, capability-based registry — search/generate/process slots, tried deterministically with heygen-CLI first. Disabled slots (fal aggregator, Iconify, voice TTS, local) carry their gate reason (B-Q1/B-Q2/U2) so enabling them later is a flag flip, not a refactor. resolve cascade now registry-driven; providers.mjs kept as a back-compat shim. Regenerated skills-manifest. --- skills-manifest.json | 44 +++---- skills/media-use/scripts/lib/providers.mjs | 34 +---- skills/media-use/scripts/lib/registry.mjs | 99 +++++++++++++++ .../media-use/scripts/lib/registry.test.mjs | 118 ++++++++++++++++++ skills/media-use/scripts/resolve.mjs | 18 +-- 5 files changed, 251 insertions(+), 62 deletions(-) create mode 100644 skills/media-use/scripts/lib/registry.mjs create mode 100644 skills/media-use/scripts/lib/registry.test.mjs diff --git a/skills-manifest.json b/skills-manifest.json index 7001b1b645..bd0ab6aede 100644 --- a/skills-manifest.json +++ b/skills-manifest.json @@ -2,71 +2,63 @@ "source": "heygen-com/hyperframes", "skills": { "embedded-captions": { - "hash": "62aec45830eda54b", + "hash": "d5561ac019e72ed5", "files": 144 }, "faceless-explainer": { - "hash": "844c54a06fd16d3c", + "hash": "d6beeca6029b815a", "files": 17 }, - "figma": { - "hash": "c462519102acc265", - "files": 1 - }, "general-video": { "hash": "a30225e30ec7b06c", "files": 1 }, "hyperframes": { - "hash": "2bd5cdf9f851b4d2", + "hash": "00ad363cda72268d", "files": 1 }, "hyperframes-animation": { - "hash": "19024009fab8e5d1", - "files": 116 + "hash": "ced7bd16b3cdb376", + "files": 108 }, "hyperframes-cli": { - "hash": "9b36a367a0e3a332", + "hash": "ed24d781c2ae462b", "files": 7 }, "hyperframes-core": { - "hash": "e978b3168748eb39", + "hash": "6243b6f09c94cce1", "files": 13 }, "hyperframes-creative": { - "hash": "18a14a79da6cbc06", - "files": 68 - }, - "hyperframes-keyframes": { - "hash": "47f20312033792c3", - "files": 3 + "hash": "bb248c1bc5cc28b5", + "files": 67 }, "hyperframes-media": { - "hash": "eb72d0765185b36d", - "files": 44 + "hash": "ce80c5a15ecaf0fc", + "files": 40 }, "hyperframes-registry": { "hash": "e3b389526834109d", "files": 10 }, "media-use": { - "hash": "fba6e0963e431b1e", - "files": 19 + "hash": "d266431492a08ead", + "files": 21 }, "motion-graphics": { - "hash": "be1d1f159d5eb0e4", + "hash": "5a256811f730f36e", "files": 23 }, "music-to-video": { - "hash": "0c5738fac0fe622f", + "hash": "abea9101f4322954", "files": 132 }, "pr-to-video": { - "hash": "132b44ddda774fef", + "hash": "d7acaeb6281f99ac", "files": 21 }, "product-launch-video": { - "hash": "32cab842cc7a6b76", + "hash": "5669874d3bdd5bd4", "files": 18 }, "remotion-to-hyperframes": { @@ -74,7 +66,7 @@ "files": 70 }, "slideshow": { - "hash": "63c685cd4a93bfa5", + "hash": "ae8d8f3093ef0c04", "files": 2 }, "talking-head-recut": { diff --git a/skills/media-use/scripts/lib/providers.mjs b/skills/media-use/scripts/lib/providers.mjs index f8924410d2..6220e88ee9 100644 --- a/skills/media-use/scripts/lib/providers.mjs +++ b/skills/media-use/scripts/lib/providers.mjs @@ -1,29 +1,5 @@ -import { sfxProvider } from "./sfx-provider.mjs"; -import { imageProvider, iconProvider } from "./image-provider.mjs"; -import { bgmProvider } from "./bgm-provider.mjs"; -import { brandProvider } from "./brand-provider.mjs"; - -const STUB = { - async search() { - return null; - }, -}; - -const registry = { - bgm: { ...bgmProvider, type: "bgm" }, - sfx: { ...sfxProvider, type: "sfx" }, - voice: { ...STUB, type: "voice" }, - image: { ...imageProvider, type: "image" }, - icon: { ...iconProvider, type: "icon" }, - brand: { ...brandProvider, type: "brand" }, -}; - -export function getProvider(type) { - const p = registry[type]; - if (!p) throw new Error(`unknown media type: ${type}`); - return p; -} - -export function listTypes() { - return Object.keys(registry); -} +// Back-compat surface for the v1 provider API. The ordered, capability-based +// registry now lives in registry.mjs; this re-exports the v1 helpers so existing +// callers keep working. New code should import from registry.mjs directly +// (getProviders / runCapability). +export { getProvider, listTypes } from "./registry.mjs"; diff --git a/skills/media-use/scripts/lib/registry.mjs b/skills/media-use/scripts/lib/registry.mjs new file mode 100644 index 0000000000..f167c7dad4 --- /dev/null +++ b/skills/media-use/scripts/lib/registry.mjs @@ -0,0 +1,99 @@ +// Provider registry — the v2 contract. +// +// Each media type maps to an ORDERED list of provider entries. Providers are +// tried in order; the first to return a non-null result wins, which keeps +// resolution deterministic (same request -> same provider -> same file -> +// reproducible renders). heygen-CLI is always first for the types it serves. +// +// An entry exposes any of three capability methods — search / generate / +// process — plus { name, enabled, gated? }. Disabled entries are the seams for +// work that is decided-but-not-shipped: the `fal` aggregator and Iconify wait on +// the Bin bundling decision (B-Q2); voice generation waits on B-Q1; the local +// slot is filled by U2/U4. They live in the registry (so enabling them later is +// a flag flip, not a refactor) but are skipped by getProviders() until enabled. + +import { bgmProvider } from "./bgm-provider.mjs"; +import { sfxProvider } from "./sfx-provider.mjs"; +import { imageProvider, iconProvider } from "./image-provider.mjs"; +import { brandProvider } from "./brand-provider.mjs"; + +const on = (name, caps) => ({ name, enabled: true, ...caps }); +const off = (name, gated, caps = {}) => ({ name, enabled: false, gated, ...caps }); + +// heygen-first for everything it serves. Aggregator (fal) / Iconify / local / +// voice slots are present but gated — see the file header. +const REGISTRY = { + bgm: [ + on("heygen.audio.sounds", { search: bgmProvider.search }), + off("fal", "B-Q2"), // aggregator music generation + off("local", "U2"), // local model generation + ], + sfx: [ + on("heygen.audio.sounds", { search: sfxProvider.search }), + off("fal", "B-Q2"), + off("local", "U2"), + ], + image: [ + on("heygen.asset.search", { search: imageProvider.search }), + off("fal", "B-Q2"), // aggregator image generation (Flux) + off("local", "U2"), + ], + icon: [ + on("heygen.asset.search", { search: iconProvider.search }), + off("iconify", "B-Q2"), // 200k+ open icons, fallback after heygen + ], + voice: [ + off("heygen.tts", "B-Q1"), // voice/TTS generation gated on Bin + off("local", "U2"), + ], + brand: [ + // Local design spec, not heygen — reads frame.md / design.md tokens. + on("design_spec", { search: brandProvider.search }), + ], +}; + +function listFor(type) { + const list = REGISTRY[type]; + if (!list) throw new Error(`unknown media type: ${type}`); + return list; +} + +/** Ordered providers for a type. Disabled (gated) entries are excluded unless asked for. */ +export function getProviders(type, { includeDisabled = false } = {}) { + const list = listFor(type); + return includeDisabled ? list.slice() : list.filter((p) => p.enabled); +} + +/** All declared media types. */ +export function listTypes() { + return Object.keys(REGISTRY); +} + +/** + * Back-compat shim for the v1 single-provider API. Returns the first declared + * provider for the type (tagged with `type`); throws for an unknown type. + */ +export function getProvider(type) { + const first = listFor(type)[0] || {}; + return { ...first, type }; +} + +/** + * Run a capability across an explicit ordered provider list. Tries each in + * order, returns the first non-null result, skips providers that don't expose + * the capability. Pure over its input — the unit-testable core of the cascade. + */ +export async function runProviders(providers, capability, intent, ctx) { + for (const p of providers) { + const fn = p[capability]; + if (typeof fn !== "function") continue; + const res = await fn(intent, ctx); + if (res) return res; + } + return null; +} + +/** Run a capability over the enabled providers for a type (deterministic, heygen-first). */ +export async function runCapability(type, capability, intent, ctx) { + return runProviders(getProviders(type), capability, intent, ctx); +} diff --git a/skills/media-use/scripts/lib/registry.test.mjs b/skills/media-use/scripts/lib/registry.test.mjs new file mode 100644 index 0000000000..0cb2050cae --- /dev/null +++ b/skills/media-use/scripts/lib/registry.test.mjs @@ -0,0 +1,118 @@ +import { strict as assert } from "node:assert"; +import { test } from "node:test"; +import { getProviders, getProvider, listTypes, runProviders, runCapability } from "./registry.mjs"; + +// --- registry shape ------------------------------------------------------- + +test("listTypes exposes the v2 media types", () => { + const types = listTypes(); + for (const t of ["bgm", "sfx", "image", "icon", "voice", "brand"]) { + assert.ok(types.includes(t), `missing type: ${t}`); + } +}); + +test("heygen provider is first for every type it serves", () => { + for (const t of ["bgm", "sfx", "image", "icon"]) { + const first = getProviders(t)[0]; + assert.ok(first, `no enabled provider for ${t}`); + assert.match(first.name, /^heygen/, `${t} first provider is ${first.name}`); + } +}); + +test("getProviders filters disabled by default, includes them on request", () => { + const enabled = getProviders("bgm"); + const all = getProviders("bgm", { includeDisabled: true }); + assert.ok(all.length > enabled.length, "expected gated providers to exist"); + assert.ok( + enabled.every((p) => p.enabled), + "enabled list must not contain disabled providers", + ); +}); + +test("gated providers carry a gate reason (B-Q1 / B-Q2 / U-id)", () => { + const all = getProviders("bgm", { includeDisabled: true }); + const disabled = all.filter((p) => !p.enabled); + assert.ok(disabled.length > 0, "bgm should have a disabled aggregator/local slot"); + assert.ok( + disabled.every((p) => typeof p.gated === "string" && p.gated.length > 0), + "every disabled provider names why it is gated", + ); +}); + +test("the aggregator slot exists for bgm but is disabled (B-Q2)", () => { + const all = getProviders("bgm", { includeDisabled: true }); + const agg = all.find((p) => p.name === "fal"); + assert.ok(agg, "fal aggregator slot must be present (built, not shipped)"); + assert.equal(agg.enabled, false); + assert.equal(agg.gated, "B-Q2"); +}); + +test("voice generation is gated on B-Q1 (no enabled provider yet)", () => { + assert.deepEqual(getProviders("voice"), []); + const all = getProviders("voice", { includeDisabled: true }); + assert.ok(all.some((p) => p.gated === "B-Q1")); +}); + +test("getProvider returns the first provider with its type, throws for unknown", () => { + const p = getProvider("bgm"); + assert.equal(p.type, "bgm"); + assert.equal(typeof p.search, "function"); + assert.throws(() => getProvider("unknown_type"), /unknown media type/); +}); + +test("getProviders throws for unknown type", () => { + assert.throws(() => getProviders("nope"), /unknown media type/); +}); + +// --- deterministic capability execution (runProviders core) --------------- + +test("runProviders calls providers in order and returns the first non-null", async () => { + const calls = []; + const providers = [ + { + name: "a", + enabled: true, + search: async () => { + calls.push("a"); + return null; + }, + }, + { + name: "b", + enabled: true, + search: async () => { + calls.push("b"); + return { hit: "b" }; + }, + }, + { + name: "c", + enabled: true, + search: async () => { + calls.push("c"); + return { hit: "c" }; + }, + }, + ]; + const res = await runProviders(providers, "search", "x", {}); + assert.deepEqual(res, { hit: "b" }); + assert.deepEqual(calls, ["a", "b"], "must stop at first non-null, never call c"); +}); + +test("runProviders skips providers missing the requested capability", async () => { + const providers = [ + { name: "a", enabled: true /* no search */ }, + { name: "b", enabled: true, search: async () => ({ hit: "b" }) }, + ]; + const res = await runProviders(providers, "search", "x", {}); + assert.deepEqual(res, { hit: "b" }); +}); + +test("runProviders returns null when no provider yields a result", async () => { + const providers = [{ name: "a", enabled: true, search: async () => null }]; + assert.equal(await runProviders(providers, "search", "x", {}), null); +}); + +test("runCapability('bgm','process') is null — process slot is graceful when unfilled", async () => { + assert.equal(await runCapability("bgm", "process", "x", {}), null); +}); diff --git a/skills/media-use/scripts/resolve.mjs b/skills/media-use/scripts/resolve.mjs index 0de752cdd1..47e32554e4 100644 --- a/skills/media-use/scripts/resolve.mjs +++ b/skills/media-use/scripts/resolve.mjs @@ -6,7 +6,7 @@ import { parseArgs } from "node:util"; import { appendRecord, findByPrompt, findByEntity, nextId, typeSubdir } from "./lib/manifest.mjs"; import { regenerateIndex } from "./lib/index-gen.mjs"; import { cacheGet, cacheGetByEntity, importFromCache } from "./lib/cache.mjs"; -import { getProvider, listTypes } from "./lib/providers.mjs"; +import { runCapability, listTypes } from "./lib/registry.mjs"; import { freezeUrl, freezeLocalFile } from "./lib/freeze.mjs"; import { findExistingAsset } from "./lib/adopt.mjs"; @@ -62,6 +62,11 @@ if (!args.type || !args.intent) { process.exit(2); } +if (!listTypes().includes(args.type)) { + console.error(`error: unknown media type: ${args.type} (known: ${listTypes().join(", ")})`); + process.exit(2); +} + const projectDir = resolve(args.project); const type = args.type; const intent = args.intent; @@ -128,19 +133,18 @@ async function run() { } } - // 3. provider search - const provider = getProvider(type); + // 3. provider search — registry tries providers in order (heygen-CLI first) let searchResult = null; try { - searchResult = await provider.search(intent, { entity, projectDir }); + searchResult = await runCapability(type, "search", intent, { entity, projectDir }); } catch { // search failed, try generate } - // 4. generate fallback - if (!searchResult && provider.generate) { + // 4. generate fallback — same ordered cascade for the generate capability + if (!searchResult) { try { - searchResult = await provider.generate(intent, { entity, projectDir }); + searchResult = await runCapability(type, "generate", intent, { entity, projectDir }); } catch { // generate failed too } From 675c892c8437502e81a1c53f1a3d89afd01a4d3a Mon Sep 17 00:00:00 2001 From: Miguel Angel Simon Sierra Date: Fri, 26 Jun 2026 13:20:30 -0400 Subject: [PATCH 02/55] feat(media-use): spec-gated local-model runtime (U2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit specs.mjs probes CPU/RAM/GPU/VRAM (Apple Silicon unified memory + nvidia-smi), injectable for tests. local-models.mjs is a declarative table of user-installed models per capability+tier (tts/asr/upscale) with size/needs/install/invoke; selectModel picks the highest runnable tier or recommends the CLI path. No license gate — models are user-installed, local-use-only. --- skills-manifest.json | 4 +- skills/media-use/scripts/lib/local-models.mjs | 128 ++++++++++++++++++ .../scripts/lib/local-models.test.mjs | 77 +++++++++++ skills/media-use/scripts/lib/specs.mjs | 47 +++++++ skills/media-use/scripts/lib/specs.test.mjs | 50 +++++++ 5 files changed, 304 insertions(+), 2 deletions(-) create mode 100644 skills/media-use/scripts/lib/local-models.mjs create mode 100644 skills/media-use/scripts/lib/local-models.test.mjs create mode 100644 skills/media-use/scripts/lib/specs.mjs create mode 100644 skills/media-use/scripts/lib/specs.test.mjs diff --git a/skills-manifest.json b/skills-manifest.json index bd0ab6aede..2acdb8b1f8 100644 --- a/skills-manifest.json +++ b/skills-manifest.json @@ -42,8 +42,8 @@ "files": 10 }, "media-use": { - "hash": "d266431492a08ead", - "files": 21 + "hash": "c57e7a83af6c0e4e", + "files": 25 }, "motion-graphics": { "hash": "5a256811f730f36e", diff --git a/skills/media-use/scripts/lib/local-models.mjs b/skills/media-use/scripts/lib/local-models.mjs new file mode 100644 index 0000000000..05b139ea05 --- /dev/null +++ b/skills/media-use/scripts/lib/local-models.mjs @@ -0,0 +1,128 @@ +// Declarative table of USER-INSTALLED local models, for the spec-gated fallback. +// +// These models run on the user's own machine for their own use — media-use +// recommends, spec-checks, and assists install; it does not bundle, redistribute, +// or sell them. Because nothing is redistributed, selection is purely by +// quality / size / spec-fit / word-timestamp support — there is deliberately NO +// license field gating availability. +// +// Tiers: `medium` = broad-compat, smaller (auto-install target ~<=2 GB); +// `large` = best quality, needs a strong machine. selectModel() picks the +// highest tier the machine can run, or returns a recommend-the-CLI result. +// +// Picks reflect the 2026 research pass (see the v2 plan). The large-tier TTS +// default (fish-speech) is the meeting's pick; final defaults are confirmed by +// the eval harness in U7 — this table is the shortlist + current default. + +export const CAPABILITIES = ["tts", "asr", "upscale"]; + +const MODELS = { + tts: [ + { + id: "kokoro", + tier: "medium", + sizeMB: 330, + needs: { ramMB: 2048, gpu: false }, + wordTimestamps: "native", + install: "pip install kokoro", + invoke: "python -m kokoro --text {text} --voice {voice} --out {out}", + notes: "CPU, faster-than-realtime, native per-word timestamps. Default floor.", + }, + { + id: "fish-speech", + tier: "large", + sizeMB: 1100, + needs: { ramMB: 16000, gpu: true, vramMB: 12000 }, + wordTimestamps: "whisperx", // needs forced alignment (run ASR over output) + install: "pip install fish-speech", + invoke: "fish-speech synth --text {text} --ref {ref} --out {out}", + notes: "Expressive zero-shot voice cloning; meeting pick. WhisperX for word timing.", + }, + ], + asr: [ + { + id: "whisperx", + tier: "medium", + sizeMB: 1500, + needs: { ramMB: 4096, gpu: false }, + wordTimestamps: "native", // faster-whisper + wav2vec2 forced alignment + install: "pip install whisperx", + invoke: "whisperx {audio} --output_format json --out {out}", + notes: "Sub-100ms word timestamps on CPU. Strict upgrade over plain whisper.", + }, + { + id: "parakeet", + tier: "large", + sizeMB: 2400, + needs: { ramMB: 8000, gpu: true, vramMB: 4000 }, + wordTimestamps: "native", + install: "pip install parakeet-mlx # NVIDIA: nemo-toolkit[asr]", + invoke: "parakeet {audio} --timestamps word --out {out}", + notes: "~1000x realtime; native word timestamps. Apple Silicon via parakeet-mlx.", + }, + ], + upscale: [ + { + id: "real-esrgan", + tier: "medium", + sizeMB: 70, + needs: { ramMB: 2048, gpu: false }, + wordTimestamps: false, + install: "brew install real-esrgan-ncnn-vulkan # or download the ncnn binary", + invoke: "realesrgan-ncnn-vulkan -i {in} -o {out} -s 4", + notes: "ncnn-vulkan binary, CPU-capable. GFPGAN for faces.", + }, + { + id: "seedvr2", + tier: "large", + sizeMB: 6000, + needs: { ramMB: 24000, gpu: true, vramMB: 16000 }, + wordTimestamps: false, + install: "pip install seedvr2", + invoke: "seedvr2 upscale --in {in} --out {out}", + notes: "Diffusion upscaler, GPU-only. Video2X for video.", + }, + ], +}; + +function tableFor(capability) { + const t = MODELS[capability]; + if (!t) throw new Error(`unknown local-model capability: ${capability}`); + return t; +} + +/** All local models for a capability. */ +export function listModels(capability) { + return tableFor(capability).slice(); +} + +/** Does this machine meet a model's needs? Apple Silicon unified memory counts as VRAM. */ +export function meetsSpecs(model, specs) { + const n = model.needs || {}; + if (n.ramMB && specs.ramMB < n.ramMB) return false; + if (n.gpu && !specs.gpu?.present) return false; + if (n.vramMB) { + const vram = specs.gpu?.vramMB ?? 0; + if (vram < n.vramMB) return false; + } + return true; +} + +/** + * Pick the best local model the machine can run for a capability. + * Prefers `large` unless preferTier pins `medium`. Returns + * `{ model, tier }`, or `{ recommend: "cli", reason }` when nothing fits. + */ +export function selectModel(capability, specs, { preferTier } = {}) { + const table = tableFor(capability); + const order = preferTier === "medium" ? ["medium"] : ["large", "medium"]; + for (const tier of order) { + const model = table.find((m) => m.tier === tier && meetsSpecs(m, specs)); + if (model) return { model, tier }; + } + const smallest = table.reduce((a, b) => (a.sizeMB <= b.sizeMB ? a : b)); + return { + recommend: "cli", + reason: `machine does not meet specs for any local ${capability} model (smallest needs ~${smallest.needs.ramMB}MB RAM${smallest.needs.gpu ? " + GPU" : ""}); use the CLI path instead`, + }; +} diff --git a/skills/media-use/scripts/lib/local-models.test.mjs b/skills/media-use/scripts/lib/local-models.test.mjs new file mode 100644 index 0000000000..7115265466 --- /dev/null +++ b/skills/media-use/scripts/lib/local-models.test.mjs @@ -0,0 +1,77 @@ +import { strict as assert } from "node:assert"; +import { test } from "node:test"; +import { listModels, meetsSpecs, selectModel, CAPABILITIES } from "./local-models.mjs"; + +const strongGpu = { + ramMB: 64000, + gpu: { present: true, kind: "nvidia", vramMB: 24000 }, + appleSilicon: false, +}; +const cpuOnly = { ramMB: 16000, gpu: { present: false, vramMB: 0 }, appleSilicon: false }; +const tiny = { ramMB: 1024, gpu: { present: false, vramMB: 0 }, appleSilicon: false }; + +test("every capability table is non-empty and well-formed", () => { + for (const cap of CAPABILITIES) { + const models = listModels(cap); + assert.ok(models.length > 0, `no models for ${cap}`); + for (const m of models) { + assert.ok(m.id && m.tier && m.needs, `${cap}/${m.id} missing fields`); + assert.ok(["medium", "large"].includes(m.tier), `${cap}/${m.id} bad tier`); + assert.equal(typeof m.install, "string", `${cap}/${m.id} needs an install command`); + assert.equal(typeof m.invoke, "string", `${cap}/${m.id} needs an invoke command`); + // user-installed, local-use-only: there is NO license gate on selection + assert.equal("license" in m, false, `${cap}/${m.id} must not carry a license gate`); + } + } +}); + +test("meetsSpecs enforces RAM, GPU presence, and VRAM", () => { + const gpuModel = { needs: { ramMB: 8000, gpu: true, vramMB: 12000 } }; + assert.equal(meetsSpecs(gpuModel, strongGpu), true); + assert.equal(meetsSpecs(gpuModel, cpuOnly), false, "no GPU -> fails a GPU model"); + const cpuModel = { needs: { ramMB: 2000, gpu: false } }; + assert.equal(meetsSpecs(cpuModel, cpuOnly), true); + assert.equal(meetsSpecs(cpuModel, tiny), false, "too little RAM"); +}); + +test("Apple Silicon unified memory counts as VRAM", () => { + const apple = { + ramMB: 24000, + appleSilicon: true, + gpu: { present: true, kind: "apple", vramMB: 24000 }, + }; + const gpuModel = { needs: { ramMB: 8000, gpu: true, vramMB: 16000 } }; + assert.equal(meetsSpecs(gpuModel, apple), true); +}); + +test("selectModel picks the large tier on a strong machine", () => { + const r = selectModel("tts", strongGpu); + assert.equal(r.tier, "large"); + assert.ok(r.model.id); +}); + +test("selectModel falls back to medium on a CPU-only machine", () => { + const r = selectModel("tts", cpuOnly); + assert.equal(r.tier, "medium"); + assert.equal(r.model.id, "kokoro", "Kokoro is the CPU/medium default (native word timestamps)"); +}); + +test("selectModel recommends the CLI path when no tier fits", () => { + const r = selectModel("tts", tiny); + assert.equal(r.recommend, "cli"); + assert.ok(r.reason && /spec/i.test(r.reason)); + assert.equal(r.model, undefined); +}); + +test("preferTier:'medium' avoids the large model even on a strong machine", () => { + const r = selectModel("tts", strongGpu, { preferTier: "medium" }); + assert.equal(r.tier, "medium"); +}); + +test("ASR offers word-timestamp-capable models (better than plain whisper)", () => { + const asr = listModels("asr"); + assert.ok( + asr.every((m) => m.wordTimestamps), + "every ASR model must support word timestamps", + ); +}); diff --git a/skills/media-use/scripts/lib/specs.mjs b/skills/media-use/scripts/lib/specs.mjs new file mode 100644 index 0000000000..aa68b37660 --- /dev/null +++ b/skills/media-use/scripts/lib/specs.mjs @@ -0,0 +1,47 @@ +// Machine-capability probe for the spec-gated local-model fallback. +// +// Local models are USER-INSTALLED and local-use-only — media-use recommends, +// spec-checks, and assists install, but never bundles or runs them as a service. +// This probe answers "what tier can this machine actually run?" so selection can +// offer a medium/large local model, or fall back to recommending the CLI path. +// +// `osMod` and `exec` are injectable for tests. `exec(cmd)` returns the command's +// stdout as a string, or throws / returns null on failure. + +import os from "node:os"; +import { execSync } from "node:child_process"; + +function defaultExec(cmd) { + return execSync(cmd, { encoding: "utf8", stdio: ["ignore", "pipe", "ignore"], timeout: 4000 }); +} + +function detectGpu(platform, arch, ramMB, exec) { + // Apple Silicon: Metal GPU with unified memory — VRAM tracks system RAM. + if (platform === "darwin" && arch === "arm64") { + return { present: true, kind: "apple", vramMB: ramMB }; + } + // NVIDIA: query total VRAM. Any failure (no driver, no GPU) -> no GPU. + try { + const out = exec("nvidia-smi --query-gpu=memory.total --format=csv,noheader,nounits"); + const mb = parseInt(String(out).trim().split(/\r?\n/)[0], 10); + if (Number.isFinite(mb) && mb > 0) return { present: true, kind: "nvidia", vramMB: mb }; + } catch { + // fall through — no usable GPU + } + return { present: false, kind: null, vramMB: 0 }; +} + +export function probeSpecs({ osMod = os, exec = defaultExec } = {}) { + const platform = osMod.platform(); + const arch = osMod.arch(); + const cpuCores = osMod.cpus().length; + const ramMB = Math.round(osMod.totalmem() / (1024 * 1024)); + return { + platform, + arch, + cpuCores, + ramMB, + appleSilicon: platform === "darwin" && arch === "arm64", + gpu: detectGpu(platform, arch, ramMB, exec), + }; +} diff --git a/skills/media-use/scripts/lib/specs.test.mjs b/skills/media-use/scripts/lib/specs.test.mjs new file mode 100644 index 0000000000..769b85e1c6 --- /dev/null +++ b/skills/media-use/scripts/lib/specs.test.mjs @@ -0,0 +1,50 @@ +import { strict as assert } from "node:assert"; +import { test } from "node:test"; +import { probeSpecs } from "./specs.mjs"; + +// Fake os module + exec so the probe is deterministic across CI machines. +const fakeOs = (over = {}) => ({ + platform: () => over.platform ?? "linux", + arch: () => over.arch ?? "x64", + cpus: () => Array.from({ length: over.cores ?? 8 }), + totalmem: () => (over.ramMB ?? 16384) * 1024 * 1024, +}); + +test("probeSpecs reports structured caps", () => { + const s = probeSpecs({ osMod: fakeOs({ cores: 12, ramMB: 32768 }), exec: () => null }); + assert.equal(s.cpuCores, 12); + assert.equal(s.ramMB, 32768); + assert.equal(s.platform, "linux"); + assert.equal(s.gpu.present, false); +}); + +test("Apple Silicon is detected as a unified-memory GPU", () => { + const s = probeSpecs({ + osMod: fakeOs({ platform: "darwin", arch: "arm64", ramMB: 24576 }), + exec: () => null, + }); + assert.equal(s.appleSilicon, true); + assert.equal(s.gpu.present, true); + assert.equal(s.gpu.kind, "apple"); + // unified memory: VRAM tracks system RAM + assert.equal(s.gpu.vramMB, 24576); +}); + +test("NVIDIA GPU is detected via nvidia-smi VRAM query", () => { + const exec = (cmd) => (cmd.includes("nvidia-smi") ? "24564" : null); + const s = probeSpecs({ osMod: fakeOs({ platform: "linux" }), exec }); + assert.equal(s.gpu.present, true); + assert.equal(s.gpu.kind, "nvidia"); + assert.equal(s.gpu.vramMB, 24564); +}); + +test("no GPU when nvidia-smi is absent / fails", () => { + const s = probeSpecs({ + osMod: fakeOs({ platform: "linux" }), + exec: () => { + throw new Error("command not found"); + }, + }); + assert.equal(s.gpu.present, false); + assert.equal(s.gpu.vramMB, 0); +}); From bff2343ed9e7deefbe5e4deea304ce5d81773944 Mon Sep 17 00:00:00 2001 From: Miguel Angel Simon Sierra Date: Fri, 26 Jun 2026 14:29:00 -0400 Subject: [PATCH 03/55] feat(media-use): flip-to-enable fal + voice adapters (#5) fal aggregator + ElevenLabs/HeyGen-TTS voice providers built as real CLI adapters, registered default-OFF and flipped on by env flag (MEDIA_USE_ENABLE_*) read at call time. Bin's answer becomes a flag, not a code change. Tests prove off-by-default + flip-on ordering (heygen stays first). --- skills-manifest.json | 4 +- .../scripts/lib/aggregator-provider.mjs | 43 ++++++++++++ skills/media-use/scripts/lib/registry.mjs | 65 +++++++++++-------- .../media-use/scripts/lib/registry.test.mjs | 32 +++++++++ .../media-use/scripts/lib/voice-provider.mjs | 49 ++++++++++++++ 5 files changed, 164 insertions(+), 29 deletions(-) create mode 100644 skills/media-use/scripts/lib/aggregator-provider.mjs create mode 100644 skills/media-use/scripts/lib/voice-provider.mjs diff --git a/skills-manifest.json b/skills-manifest.json index 2acdb8b1f8..e168424dc4 100644 --- a/skills-manifest.json +++ b/skills-manifest.json @@ -42,8 +42,8 @@ "files": 10 }, "media-use": { - "hash": "c57e7a83af6c0e4e", - "files": 25 + "hash": "691c25efaae78de9", + "files": 27 }, "motion-graphics": { "hash": "5a256811f730f36e", diff --git a/skills/media-use/scripts/lib/aggregator-provider.mjs b/skills/media-use/scripts/lib/aggregator-provider.mjs new file mode 100644 index 0000000000..4077cc8a82 --- /dev/null +++ b/skills/media-use/scripts/lib/aggregator-provider.mjs @@ -0,0 +1,43 @@ +import { execFileSync } from "node:child_process"; + +// fal aggregator (genmedia CLI). Default-OFF in the registry (B-Q2); enable with +// MEDIA_USE_ENABLE_FAL=1 once Bin approves bundling third-party cloud CLIs. +// Mirrors the heygen-search execFile pattern (argv array, no shell). +// +// ponytail: exact fal CLI flags/JSON shape are per fal's genmedia docs, not +// verified here (CLI not installed). Confirm the model ids + output keys against +// `fal run --help` when the flag is first flipped on. The wiring is the point. + +const MODEL = { image: "fal-ai/flux/schnell", bgm: "fal-ai/minimax-music", sfx: "fal-ai/mmaudio" }; + +export function falGenerate(kind) { + return async function generate(intent) { + const model = MODEL[kind]; + if (!model) return null; + let out; + try { + out = execFileSync("fal", ["run", model, "--input", JSON.stringify({ prompt: intent })], { + encoding: "utf8", + timeout: 120000, + stdio: ["pipe", "pipe", "pipe"], + }); + } catch (err) { + const detail = err.stderr?.toString().trim() || err.message; + console.error(`media-use: \`fal run ${model}\` failed: ${detail}`); + return null; + } + let parsed; + try { + parsed = JSON.parse(out); + } catch { + return null; + } + const url = parsed?.url || parsed?.images?.[0]?.url || parsed?.audio?.url || parsed?.video?.url; + if (!url) return null; + return { + url, + source: "generated", + metadata: { description: intent, provider: "fal", provenance: { model, prompt: intent } }, + }; + }; +} diff --git a/skills/media-use/scripts/lib/registry.mjs b/skills/media-use/scripts/lib/registry.mjs index f167c7dad4..51a867d6d7 100644 --- a/skills/media-use/scripts/lib/registry.mjs +++ b/skills/media-use/scripts/lib/registry.mjs @@ -6,62 +6,73 @@ // reproducible renders). heygen-CLI is always first for the types it serves. // // An entry exposes any of three capability methods — search / generate / -// process — plus { name, enabled, gated? }. Disabled entries are the seams for -// work that is decided-but-not-shipped: the `fal` aggregator and Iconify wait on -// the Bin bundling decision (B-Q2); voice generation waits on B-Q1; the local -// slot is filled by U2/U4. They live in the registry (so enabling them later is -// a flag flip, not a refactor) but are skipped by getProviders() until enabled. +// process — plus { name } and an enablement rule: +// - always:true -> always enabled (heygen, local design spec) +// - envFlag:"MEDIA_USE_ENABLE_X" -> enabled only when that env flag is set +// Flag-gated entries are decided-but-not-shipped seams: the `fal` aggregator and +// Iconify wait on the Bin bundling decision (B-Q2); voice (ElevenLabs / HeyGen +// TTS) waits on B-Q1. They are fully built — flipping the flag enables them with +// NO code change. `gated` records which decision gates them, for diagnostics. import { bgmProvider } from "./bgm-provider.mjs"; import { sfxProvider } from "./sfx-provider.mjs"; import { imageProvider, iconProvider } from "./image-provider.mjs"; import { brandProvider } from "./brand-provider.mjs"; +import { falGenerate } from "./aggregator-provider.mjs"; +import { elevenlabsGenerate, heygenTtsGenerate } from "./voice-provider.mjs"; -const on = (name, caps) => ({ name, enabled: true, ...caps }); -const off = (name, gated, caps = {}) => ({ name, enabled: false, gated, ...caps }); +const A = (name, caps) => ({ name, always: true, ...caps }); // always-on +const G = (name, envFlag, gated, caps = {}) => ({ name, envFlag, gated, ...caps }); // flag-gated -// heygen-first for everything it serves. Aggregator (fal) / Iconify / local / -// voice slots are present but gated — see the file header. +// heygen-first for everything it serves. fal / Iconify / voice slots are present +// but flag-gated — see the file header. const REGISTRY = { bgm: [ - on("heygen.audio.sounds", { search: bgmProvider.search }), - off("fal", "B-Q2"), // aggregator music generation - off("local", "U2"), // local model generation + A("heygen.audio.sounds", { search: bgmProvider.search }), + G("fal", "MEDIA_USE_ENABLE_FAL", "B-Q2", { generate: falGenerate("bgm") }), ], sfx: [ - on("heygen.audio.sounds", { search: sfxProvider.search }), - off("fal", "B-Q2"), - off("local", "U2"), + A("heygen.audio.sounds", { search: sfxProvider.search }), + G("fal", "MEDIA_USE_ENABLE_FAL", "B-Q2", { generate: falGenerate("sfx") }), ], image: [ - on("heygen.asset.search", { search: imageProvider.search }), - off("fal", "B-Q2"), // aggregator image generation (Flux) - off("local", "U2"), + A("heygen.asset.search", { search: imageProvider.search }), + G("fal", "MEDIA_USE_ENABLE_FAL", "B-Q2", { generate: falGenerate("image") }), ], icon: [ - on("heygen.asset.search", { search: iconProvider.search }), - off("iconify", "B-Q2"), // 200k+ open icons, fallback after heygen + A("heygen.asset.search", { search: iconProvider.search }), + G("iconify", "MEDIA_USE_ENABLE_ICONIFY", "B-Q2"), // 200k+ open icons, fallback after heygen ], voice: [ - off("heygen.tts", "B-Q1"), // voice/TTS generation gated on Bin - off("local", "U2"), + G("elevenlabs", "MEDIA_USE_ENABLE_ELEVENLABS", "B-Q1", { generate: elevenlabsGenerate }), + G("heygen.tts", "MEDIA_USE_ENABLE_HEYGEN_TTS", "B-Q1", { generate: heygenTtsGenerate }), ], brand: [ // Local design spec, not heygen — reads frame.md / design.md tokens. - on("design_spec", { search: brandProvider.search }), + A("design_spec", { search: brandProvider.search }), ], }; +const isEnabled = (p) => p.always === true || (p.envFlag ? envSet(p.envFlag) : false); + +function envSet(name) { + const v = process.env[name]; + return v === "1" || v === "true"; +} + function listFor(type) { const list = REGISTRY[type]; if (!list) throw new Error(`unknown media type: ${type}`); return list; } -/** Ordered providers for a type. Disabled (gated) entries are excluded unless asked for. */ +// Add the computed `enabled` so callers/tests see a flat boolean. +const tag = (p) => ({ ...p, enabled: isEnabled(p) }); + +/** Ordered providers for a type. Disabled (flag-gated) entries are excluded unless asked for. */ export function getProviders(type, { includeDisabled = false } = {}) { - const list = listFor(type); - return includeDisabled ? list.slice() : list.filter((p) => p.enabled); + const list = listFor(type).map(tag); + return includeDisabled ? list : list.filter((p) => p.enabled); } /** All declared media types. */ @@ -75,7 +86,7 @@ export function listTypes() { */ export function getProvider(type) { const first = listFor(type)[0] || {}; - return { ...first, type }; + return { ...tag(first), type }; } /** diff --git a/skills/media-use/scripts/lib/registry.test.mjs b/skills/media-use/scripts/lib/registry.test.mjs index 0cb2050cae..10c05ababa 100644 --- a/skills/media-use/scripts/lib/registry.test.mjs +++ b/skills/media-use/scripts/lib/registry.test.mjs @@ -116,3 +116,35 @@ test("runProviders returns null when no provider yields a result", async () => { test("runCapability('bgm','process') is null — process slot is graceful when unfilled", async () => { assert.equal(await runCapability("bgm", "process", "x", {}), null); }); + +// --- flip-to-enable: Bin's answer is a flag, not a code change ---------------- + +function withEnv(name, value, fn) { + const prev = process.env[name]; + process.env[name] = value; + try { + return fn(); + } finally { + if (prev === undefined) delete process.env[name]; + else process.env[name] = prev; + } +} + +test("MEDIA_USE_ENABLE_FAL=1 enables the fal aggregator after heygen — no code change", () => { + withEnv("MEDIA_USE_ENABLE_FAL", "1", () => { + const ps = getProviders("bgm"); + assert.equal(ps.length, 2); + assert.match(ps[0].name, /^heygen/, "heygen stays first"); + assert.equal(ps[1].name, "fal"); + assert.equal(ps[1].enabled, true); + assert.equal(typeof ps[1].generate, "function", "fal exposes generate when enabled"); + }); +}); + +test("MEDIA_USE_ENABLE_ELEVENLABS=1 enables voice generation (B-Q1 flip)", () => { + assert.deepEqual(getProviders("voice"), [], "off by default"); + withEnv("MEDIA_USE_ENABLE_ELEVENLABS", "1", () => { + const ps = getProviders("voice"); + assert.ok(ps.some((p) => p.name === "elevenlabs" && typeof p.generate === "function")); + }); +}); diff --git a/skills/media-use/scripts/lib/voice-provider.mjs b/skills/media-use/scripts/lib/voice-provider.mjs new file mode 100644 index 0000000000..05f6b00a0e --- /dev/null +++ b/skills/media-use/scripts/lib/voice-provider.mjs @@ -0,0 +1,49 @@ +import { execFileSync } from "node:child_process"; + +// Voice/TTS generation. Default-OFF (B-Q1); enable once Bin approves voiceover: +// ElevenLabs CLI -> MEDIA_USE_ENABLE_ELEVENLABS=1 +// HeyGen TTS -> MEDIA_USE_ENABLE_HEYGEN_TTS=1 +// Both shell their own CLI (CLI-only invariant: media-use holds no keys). +// +// ponytail: exact CLI flags/output aren't verified here (CLIs not installed). +// Confirm against each CLI's --help when first flipped on. Wiring is the point. + +function runJson(bin, argv) { + let out; + try { + out = execFileSync(bin, argv, { + encoding: "utf8", + timeout: 120000, + stdio: ["pipe", "pipe", "pipe"], + }); + } catch (err) { + console.error( + `media-use: \`${bin}\` tts failed: ${err.stderr?.toString().trim() || err.message}`, + ); + return null; + } + try { + return JSON.parse(out); + } catch { + return null; + } +} + +function result(url, provider, intent) { + if (!url) return null; + return { + url, + source: "generated", + metadata: { description: intent, provider, provenance: { prompt: intent } }, + }; +} + +export async function elevenlabsGenerate(intent) { + const p = runJson("elevenlabs", ["tts", "--text", intent, "--json"]); + return result(p?.url || p?.audio_url, "elevenlabs", intent); +} + +export async function heygenTtsGenerate(intent) { + const p = runJson("heygen", ["voice", "tts", "--text", intent]); + return result(p?.data?.audio_url || p?.audio_url, "heygen.tts", intent); +} From ff86a811c65eb4641643d0b4ec6960a6318e4933 Mon Sep 17 00:00:00 2001 From: Miguel Angel Simon Sierra Date: Fri, 26 Jun 2026 14:30:34 -0400 Subject: [PATCH 04/55] feat(media-use): local-model runner (#4) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit runLocalModel(capability) picks the best tier for the machine (selectModel), checks the tool is on PATH, fills + runs its invoke template, returns the output path — or a clear install/CLI recommendation when not runnable. Injectable which/exec; never throws. Consumed by the process verb. --- skills-manifest.json | 4 +- skills/media-use/scripts/lib/local-run.mjs | 64 +++++++++++++++++++ .../media-use/scripts/lib/local-run.test.mjs | 54 ++++++++++++++++ 3 files changed, 120 insertions(+), 2 deletions(-) create mode 100644 skills/media-use/scripts/lib/local-run.mjs create mode 100644 skills/media-use/scripts/lib/local-run.test.mjs diff --git a/skills-manifest.json b/skills-manifest.json index e168424dc4..adb8f95874 100644 --- a/skills-manifest.json +++ b/skills-manifest.json @@ -42,8 +42,8 @@ "files": 10 }, "media-use": { - "hash": "691c25efaae78de9", - "files": 27 + "hash": "d601731182828ee3", + "files": 29 }, "motion-graphics": { "hash": "5a256811f730f36e", diff --git a/skills/media-use/scripts/lib/local-run.mjs b/skills/media-use/scripts/lib/local-run.mjs new file mode 100644 index 0000000000..b8b1bc6c21 --- /dev/null +++ b/skills/media-use/scripts/lib/local-run.mjs @@ -0,0 +1,64 @@ +import { execFileSync } from "node:child_process"; +import { selectModel } from "./local-models.mjs"; +import { probeSpecs } from "./specs.mjs"; + +// Run a USER-INSTALLED local model for a capability (tts/asr/upscale). +// Picks the best tier the machine supports (selectModel), checks the tool is on +// PATH, fills the model's invoke template, and runs it. Returns: +// { model, tier, out } on success +// { recommend:"install", model, command, reason } when the tool isn't installed +// { recommend:"cli", reason } when no tier fits the machine +// `exec` / `which` are injectable for tests. +// +// ponytail: "installed" = the invoke's first token is on PATH (e.g. `whisperx`, +// `realesrgan-ncnn-vulkan`). For `python -m kokoro` this only proves python +// exists; good enough to gate — the recommend.command names the real package. +// Upgrade to a per-tool probe if a "python present but package missing" run ever +// produces a confusing error instead of a clean recommend. + +function defaultWhich(bin) { + execFileSync("command", ["-v", bin], { stdio: "ignore", shell: true }); +} + +function defaultExec(cmd) { + execFileSync(cmd, { stdio: ["ignore", "pipe", "pipe"], shell: true, timeout: 600000 }); +} + +const fill = (tpl, vars) => + tpl.replace(/\{(\w+)\}/g, (_, k) => (vars[k] != null ? String(vars[k]) : "")); + +export function runLocalModel(capability, opts = {}) { + const { + specs = probeSpecs(), + exec = defaultExec, + which = defaultWhich, + vars = {}, + preferTier, + } = opts; + const sel = selectModel(capability, specs, { preferTier }); + if (sel.recommend) return sel; // no tier fits -> recommend the CLI path + + const { model } = sel; + const bin = model.invoke.split(/\s+/)[0]; + try { + which(bin); + } catch { + return { + recommend: "install", + model: model.id, + command: model.install, + reason: `${model.id} not installed`, + }; + } + try { + exec(fill(model.invoke, vars)); + } catch (e) { + return { + recommend: "install", + model: model.id, + command: model.install, + reason: e.message || String(e), + }; + } + return { model: model.id, tier: sel.tier, out: vars.out }; +} diff --git a/skills/media-use/scripts/lib/local-run.test.mjs b/skills/media-use/scripts/lib/local-run.test.mjs new file mode 100644 index 0000000000..7e9c883dbc --- /dev/null +++ b/skills/media-use/scripts/lib/local-run.test.mjs @@ -0,0 +1,54 @@ +import { strict as assert } from "node:assert"; +import { test } from "node:test"; +import { runLocalModel } from "./local-run.mjs"; + +const strongCpu = { ramMB: 16000, gpu: { present: false, vramMB: 0 }, appleSilicon: false }; +const tiny = { ramMB: 512, gpu: { present: false, vramMB: 0 }, appleSilicon: false }; +const ok = () => {}; // which/exec that succeed + +test("recommends the CLI path when no local tier fits the machine", () => { + const r = runLocalModel("tts", { specs: tiny, which: ok, exec: ok }); + assert.equal(r.recommend, "cli"); +}); + +test("recommends install when the tool is not on PATH", () => { + const r = runLocalModel("tts", { + specs: strongCpu, + which: () => { + throw new Error("not found"); + }, + exec: ok, + vars: { text: "hi", out: "/tmp/v.wav" }, + }); + assert.equal(r.recommend, "install"); + assert.equal(r.model, "kokoro"); + assert.match(r.command, /pip install kokoro/); +}); + +test("runs the model and returns the output path when installed", () => { + let ran = ""; + const r = runLocalModel("tts", { + specs: strongCpu, + which: ok, + exec: (cmd) => { + ran = cmd; + }, + vars: { text: "hello world", voice: "af_heart", out: "/tmp/v.wav" }, + }); + assert.equal(r.model, "kokoro"); + assert.equal(r.out, "/tmp/v.wav"); + assert.match(ran, /hello world/, "invoke template filled with vars"); + assert.match(ran, /\/tmp\/v\.wav/); +}); + +test("a failing run degrades to an install recommendation, never throws", () => { + const r = runLocalModel("upscale", { + specs: strongCpu, + which: ok, + exec: () => { + throw new Error("boom"); + }, + vars: { in: "a.png", out: "b.png" }, + }); + assert.equal(r.recommend, "install"); +}); From bc946e531b0de994451aba0cd975457b2832c782 Mon Sep 17 00:00:00 2001 From: Miguel Angel Simon Sierra Date: Fri, 26 Jun 2026 14:34:26 -0400 Subject: [PATCH 05/55] feat(media-use): auto-promote + global cache + search + in-use detection (#11, #16) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Auto-promote every fetched asset into the global content-addressed cache so it's reusable across all hyperframes projects (cachePut now dedups by sha; resolve promotes on register, non-fatal). search.mjs: paginated text search (no vector DB). usage.mjs: tag/partition assets by in-use vs unused (scans compositions) — powers the Studio Asset-tab filter. --- skills-manifest.json | 4 +- skills/media-use/scripts/lib/cache.mjs | 5 ++ skills/media-use/scripts/lib/search.mjs | 24 +++++++++ skills/media-use/scripts/lib/search.test.mjs | 32 +++++++++++ skills/media-use/scripts/lib/usage.mjs | 56 ++++++++++++++++++++ skills/media-use/scripts/lib/usage.test.mjs | 54 +++++++++++++++++++ skills/media-use/scripts/resolve.mjs | 12 ++++- 7 files changed, 184 insertions(+), 3 deletions(-) create mode 100644 skills/media-use/scripts/lib/search.mjs create mode 100644 skills/media-use/scripts/lib/search.test.mjs create mode 100644 skills/media-use/scripts/lib/usage.mjs create mode 100644 skills/media-use/scripts/lib/usage.test.mjs diff --git a/skills-manifest.json b/skills-manifest.json index adb8f95874..148d2dc0ed 100644 --- a/skills-manifest.json +++ b/skills-manifest.json @@ -42,8 +42,8 @@ "files": 10 }, "media-use": { - "hash": "d601731182828ee3", - "files": 29 + "hash": "af2dd8c3f0526cde", + "files": 33 }, "motion-graphics": { "hash": "5a256811f730f36e", diff --git a/skills/media-use/scripts/lib/cache.mjs b/skills/media-use/scripts/lib/cache.mjs index e138e2bc01..0d0a1e4ae3 100644 --- a/skills/media-use/scripts/lib/cache.mjs +++ b/skills/media-use/scripts/lib/cache.mjs @@ -55,6 +55,11 @@ export function cacheGetByEntity(entity) { export function cachePut(filePath, record) { const sha = contentHash(filePath); + // Idempotent: same content already promoted -> don't duplicate the global + // record. ponytail: skips usage_count bump; add it when the metric is needed. + const existing = readGlobalManifest().find((r) => r.sha === sha); + if (existing) return { sha, cached_path: existing.cached_path, deduped: true }; + const dir = globalMediaDir(); const entryDir = cacheEntryDir(dir, sha); mkdirSync(entryDir, { recursive: true }); diff --git a/skills/media-use/scripts/lib/search.mjs b/skills/media-use/scripts/lib/search.mjs new file mode 100644 index 0000000000..b545b4852d --- /dev/null +++ b/skills/media-use/scripts/lib/search.mjs @@ -0,0 +1,24 @@ +// Paginated text search over a manifest (project or global) by id / description +// / entity. Plain substring match — no vector DB (B4): the agent's own file +// search covers semantics; this just keeps result pages small enough to not +// blow the context window. + +const PAGE = 10; + +export function searchRecords(records, query, { page = 1, pageSize = PAGE } = {}) { + const q = String(query || "") + .trim() + .toLowerCase(); + const matched = q + ? records.filter((r) => + [r.id, r.description, r.entity, r.type].some( + (f) => f && String(f).toLowerCase().includes(q), + ), + ) + : records.slice(); + const total = matched.length; + const pages = Math.max(1, Math.ceil(total / pageSize)); + const p = Math.min(Math.max(1, page), pages); + const start = (p - 1) * pageSize; + return { results: matched.slice(start, start + pageSize), total, page: p, pages, pageSize }; +} diff --git a/skills/media-use/scripts/lib/search.test.mjs b/skills/media-use/scripts/lib/search.test.mjs new file mode 100644 index 0000000000..8d68a68821 --- /dev/null +++ b/skills/media-use/scripts/lib/search.test.mjs @@ -0,0 +1,32 @@ +import { strict as assert } from "node:assert"; +import { test } from "node:test"; +import { searchRecords } from "./search.mjs"; + +const recs = Array.from({ length: 23 }, (_, i) => ({ + id: `bgm_${i}`, + type: i % 2 ? "bgm" : "sfx", + description: i === 5 ? "energetic tech whoosh" : `track ${i}`, +})); + +test("matches id / description / type, case-insensitively", () => { + assert.equal(searchRecords(recs, "ENERGETIC").results.length, 1); + assert.equal(searchRecords(recs, "energetic").results[0].id, "bgm_5"); + assert.ok(searchRecords(recs, "sfx").total > 0); +}); + +test("caps page size and reports pagination", () => { + const r = searchRecords(recs, "", { pageSize: 10 }); + assert.equal(r.results.length, 10, "never returns more than a page"); + assert.equal(r.total, 23); + assert.equal(r.pages, 3); + assert.equal(r.page, 1); +}); + +test("page navigation returns the right slice, clamps out-of-range", () => { + assert.equal(searchRecords(recs, "", { page: 3, pageSize: 10 }).results.length, 3); + assert.equal(searchRecords(recs, "", { page: 99, pageSize: 10 }).page, 3, "clamped to last page"); +}); + +test("empty query returns everything (paginated)", () => { + assert.equal(searchRecords(recs, "").total, 23); +}); diff --git a/skills/media-use/scripts/lib/usage.mjs b/skills/media-use/scripts/lib/usage.mjs new file mode 100644 index 0000000000..98166fb995 --- /dev/null +++ b/skills/media-use/scripts/lib/usage.mjs @@ -0,0 +1,56 @@ +import { readFileSync, readdirSync, existsSync } from "node:fs"; +import { join } from "node:path"; + +// Asset in-use detection: which manifest assets are actually referenced by the +// project's compositions. Powers the Studio Asset tab's in-use / unused filter +// and "is this safe to prune?" — nothing else tells you that today. +// +// ponytail: substring match of each asset's filename against the .html text +// (covers src= / href= / url() / data-* without parsing HTML). False positives +// only if a filename literally appears in prose; fine for a filter. Upgrade to +// attribute parsing if that ever bites. + +function compositionHtml(projectDir) { + let files = []; + try { + files = readdirSync(projectDir).filter((f) => f.endsWith(".html")); + } catch { + return ""; + } + const sub = join(projectDir, "compositions"); + if (existsSync(sub)) { + try { + for (const f of readdirSync(sub)) + if (f.endsWith(".html")) files.push(join("compositions", f)); + } catch { + // ignore unreadable subdir + } + } + return files + .map((f) => { + try { + return readFileSync(join(projectDir, f), "utf8"); + } catch { + return ""; + } + }) + .join("\n"); +} + +/** Tag each record with `inUse` (referenced by some composition). */ +export function tagUsage(records, projectDir) { + const html = compositionHtml(projectDir); + return records.map((r) => { + const file = r.path ? r.path.split("/").pop() : r.id; + return { ...r, inUse: Boolean(file) && html.includes(file) }; + }); +} + +/** Split records into { used, unused } for the filter. */ +export function partitionUsage(records, projectDir) { + const tagged = tagUsage(records, projectDir); + return { + used: tagged.filter((r) => r.inUse), + unused: tagged.filter((r) => !r.inUse), + }; +} diff --git a/skills/media-use/scripts/lib/usage.test.mjs b/skills/media-use/scripts/lib/usage.test.mjs new file mode 100644 index 0000000000..271e707627 --- /dev/null +++ b/skills/media-use/scripts/lib/usage.test.mjs @@ -0,0 +1,54 @@ +import { strict as assert } from "node:assert"; +import { test } from "node:test"; +import { mkdtempSync, writeFileSync, rmSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { tagUsage, partitionUsage } from "./usage.mjs"; + +function project(html) { + const dir = mkdtempSync(join(tmpdir(), "mu-usage-")); + writeFileSync(join(dir, "index.html"), html); + return dir; +} + +const records = [ + { id: "bgm_001", path: ".media/audio/bgm/bgm_001.wav", description: "used track" }, + { id: "bgm_002", path: ".media/audio/bgm/bgm_002.wav", description: "orphan track" }, +]; + +test("an asset referenced by a composition is marked in-use", () => { + const dir = project(``); + try { + const tagged = tagUsage(records, dir); + assert.equal(tagged.find((r) => r.id === "bgm_001").inUse, true); + assert.equal(tagged.find((r) => r.id === "bgm_002").inUse, false); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("partitionUsage splits used vs unused for the filter", () => { + const dir = project(``); + try { + const { used, unused } = partitionUsage(records, dir); + assert.deepEqual( + used.map((r) => r.id), + ["bgm_001"], + ); + assert.deepEqual( + unused.map((r) => r.id), + ["bgm_002"], + ); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("no compositions -> everything reads as unused (safe default)", () => { + const dir = mkdtempSync(join(tmpdir(), "mu-usage-")); + try { + assert.equal(partitionUsage(records, dir).used.length, 0); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); diff --git a/skills/media-use/scripts/resolve.mjs b/skills/media-use/scripts/resolve.mjs index 47e32554e4..f3ca12b95b 100644 --- a/skills/media-use/scripts/resolve.mjs +++ b/skills/media-use/scripts/resolve.mjs @@ -5,7 +5,7 @@ import { resolve, join, extname } from "node:path"; import { parseArgs } from "node:util"; import { appendRecord, findByPrompt, findByEntity, nextId, typeSubdir } from "./lib/manifest.mjs"; import { regenerateIndex } from "./lib/index-gen.mjs"; -import { cacheGet, cacheGetByEntity, importFromCache } from "./lib/cache.mjs"; +import { cacheGet, cacheGetByEntity, importFromCache, cachePut } from "./lib/cache.mjs"; import { runCapability, listTypes } from "./lib/registry.mjs"; import { freezeUrl, freezeLocalFile } from "./lib/freeze.mjs"; import { findExistingAsset } from "./lib/adopt.mjs"; @@ -198,6 +198,16 @@ async function run() { appendRecord(projectDir, record); regenerateIndex(projectDir); + // Auto-promote: surface every fetched asset in the global cache so it's + // reusable across all hyperframes projects (B3). Non-fatal; dedup by sha. + // ponytail: promotes search/generate/ingest assets (the ones media-use + // fetched), not bulk --adopt imports — add those if cross-project reuse of + // pre-existing project assets is wanted. + try { + cachePut(fullPath, record); + } catch { + // promotion is best-effort; a resolve still succeeds locally + } return result(record, searchResult.source || "search"); } From c5556898e052ccda5c5d93cab2c35eccb99d4441 Mon Sep 17 00:00:00 2001 From: Miguel Angel Simon Sierra Date: Fri, 26 Jun 2026 14:37:20 -0400 Subject: [PATCH 06/55] feat(media-use): ingest --from + brand upsell (#9, #10) resolve --from freezes a user-supplied asset + registers + auto-promotes it. isDirectMediaUrl rejects platform pages (no yt-dlp). brand resolve with no frame.md/design.md now upsells the HyperFrames design flow instead of a generic miss. --- skills-manifest.json | 4 +- skills/media-use/scripts/lib/freeze.mjs | 20 ++++++ skills/media-use/scripts/lib/freeze.test.mjs | 22 +++++++ skills/media-use/scripts/resolve.mjs | 65 ++++++++++++++++++-- 4 files changed, 103 insertions(+), 8 deletions(-) create mode 100644 skills/media-use/scripts/lib/freeze.test.mjs diff --git a/skills-manifest.json b/skills-manifest.json index 148d2dc0ed..51776ecfc8 100644 --- a/skills-manifest.json +++ b/skills-manifest.json @@ -42,8 +42,8 @@ "files": 10 }, "media-use": { - "hash": "af2dd8c3f0526cde", - "files": 33 + "hash": "26b00482d105a3ad", + "files": 34 }, "motion-graphics": { "hash": "5a256811f730f36e", diff --git a/skills/media-use/scripts/lib/freeze.mjs b/skills/media-use/scripts/lib/freeze.mjs index aa4704201d..0b7239fc1e 100644 --- a/skills/media-use/scripts/lib/freeze.mjs +++ b/skills/media-use/scripts/lib/freeze.mjs @@ -24,3 +24,23 @@ export function freezeLocalFile(srcPath, destPath) { mkdirSync(dirname(destPath), { recursive: true }); copyFileSync(srcPath, destPath); } + +// Ingest accepts a DIRECT public media URL only — not a platform page. yt-dlp is +// deliberately out (cloud IPs get blocked, and it's brittle); the supported case +// is "user points at their own file or a direct asset link". A direct URL is a +// non-platform host whose path ends in a known media extension. +const PLATFORM_HOSTS = + /(^|\.)(youtube\.com|youtu\.be|vimeo\.com|tiktok\.com|instagram\.com|twitter\.com|x\.com|facebook\.com|dailymotion\.com)$/i; +const MEDIA_EXT = /\.(mp3|wav|m4a|aac|ogg|flac|mp4|mov|webm|mkv|png|jpe?g|webp|gif|svg|avif)$/i; + +export function isDirectMediaUrl(u) { + let url; + try { + url = new URL(u); + } catch { + return false; + } + if (url.protocol !== "http:" && url.protocol !== "https:") return false; + if (PLATFORM_HOSTS.test(url.hostname)) return false; + return MEDIA_EXT.test(url.pathname); +} diff --git a/skills/media-use/scripts/lib/freeze.test.mjs b/skills/media-use/scripts/lib/freeze.test.mjs new file mode 100644 index 0000000000..20b0e36145 --- /dev/null +++ b/skills/media-use/scripts/lib/freeze.test.mjs @@ -0,0 +1,22 @@ +import { strict as assert } from "node:assert"; +import { test } from "node:test"; +import { isDirectMediaUrl } from "./freeze.mjs"; + +test("accepts direct public media URLs", () => { + assert.equal(isDirectMediaUrl("https://cdn.example.com/clip.mp4"), true); + assert.equal(isDirectMediaUrl("https://example.com/a/b/track.mp3"), true); + assert.equal(isDirectMediaUrl("http://example.com/logo.svg"), true); +}); + +test("rejects platform pages (no yt-dlp)", () => { + assert.equal(isDirectMediaUrl("https://www.youtube.com/watch?v=abc"), false); + assert.equal(isDirectMediaUrl("https://youtu.be/abc"), false); + assert.equal(isDirectMediaUrl("https://vimeo.com/12345"), false); + assert.equal(isDirectMediaUrl("https://x.com/u/status/1"), false); +}); + +test("rejects non-direct / non-media URLs", () => { + assert.equal(isDirectMediaUrl("https://example.com/page"), false, "no media extension"); + assert.equal(isDirectMediaUrl("ftp://example.com/a.mp4"), false, "non-http(s)"); + assert.equal(isDirectMediaUrl("not a url"), false); +}); diff --git a/skills/media-use/scripts/resolve.mjs b/skills/media-use/scripts/resolve.mjs index f3ca12b95b..a9599d0ddd 100644 --- a/skills/media-use/scripts/resolve.mjs +++ b/skills/media-use/scripts/resolve.mjs @@ -1,13 +1,13 @@ #!/usr/bin/env node import { existsSync } from "node:fs"; -import { resolve, join, extname } from "node:path"; +import { resolve, join, extname, basename } from "node:path"; import { parseArgs } from "node:util"; import { appendRecord, findByPrompt, findByEntity, nextId, typeSubdir } from "./lib/manifest.mjs"; import { regenerateIndex } from "./lib/index-gen.mjs"; import { cacheGet, cacheGetByEntity, importFromCache, cachePut } from "./lib/cache.mjs"; import { runCapability, listTypes } from "./lib/registry.mjs"; -import { freezeUrl, freezeLocalFile } from "./lib/freeze.mjs"; +import { freezeUrl, freezeLocalFile, isDirectMediaUrl } from "./lib/freeze.mjs"; import { findExistingAsset } from "./lib/adopt.mjs"; const { values: args } = parseArgs({ @@ -17,6 +17,7 @@ const { values: args } = parseArgs({ entity: { type: "string", short: "e" }, project: { type: "string", short: "p", default: "." }, adopt: { type: "boolean", default: false }, + from: { type: "string" }, json: { type: "boolean", default: false }, help: { type: "boolean", short: "h", default: false }, }, @@ -57,6 +58,12 @@ if (args.adopt) { process.exit(0); } +// Ingest: freeze a user-supplied local file or direct public URL (no search). +if (args.from) { + await ingest(args.from); + process.exit(0); +} + if (!args.type || !args.intent) { console.error("error: --type and --intent are required"); process.exit(2); @@ -151,12 +158,16 @@ async function run() { } if (!searchResult) { + // brand stays local: no frame.md/design.md -> upsell the HyperFrames design + // flow rather than reporting a generic miss (B5). + const msg = + type === "brand" + ? "no brand spec found — add a frame.md or design.md (colors/font/logo) to this project. Run the HyperFrames design flow to create one; brand tokens are read locally for deterministic rendering." + : `no provider could resolve ${type}: "${intent}"`; if (args.json) { - console.log( - JSON.stringify({ ok: false, error: `no provider could resolve ${type}: "${intent}"` }), - ); + console.log(JSON.stringify({ ok: false, error: msg })); } else { - console.error(`error: no provider could resolve ${type}: "${intent}"`); + console.error(`error: ${msg}`); } process.exit(1); } @@ -211,6 +222,48 @@ async function run() { return result(record, searchResult.source || "search"); } +async function ingest(src) { + const projectDir = resolve(args.project); + const type = args.type; + if (!type || !listTypes().includes(type)) { + console.error(`error: --from requires --type (one of: ${listTypes().join(", ")})`); + process.exit(2); + } + const isUrl = /^https?:\/\//i.test(src); + if (isUrl && !isDirectMediaUrl(src)) { + console.error( + `error: --from takes a direct public media URL or a local file; "${src}" is not a direct media link (no platform pages / yt-dlp)`, + ); + process.exit(2); + } + if (!isUrl && !existsSync(resolve(src))) { + console.error(`error: file not found: ${src}`); + process.exit(2); + } + const id = nextId(projectDir, type); + const ext = extname(isUrl ? new URL(src).pathname : src) || defaultExt(type); + const localPath = `.media/${typeSubdir(type)}/${id}${ext}`; + const fullPath = join(projectDir, localPath); + if (isUrl) await freezeUrl(src, fullPath); + else freezeLocalFile(resolve(src), fullPath); + const record = { + id, + type, + path: localPath, + source: "ingested", + description: basename(src.split("?")[0]), + provenance: { provider: "local", from: src }, + }; + appendRecord(projectDir, record); + regenerateIndex(projectDir); + try { + cachePut(fullPath, record); // surface ingested assets globally too (B3) + } catch { + // best-effort + } + result(record, "ingested"); +} + function result(record, source) { if (args.json) { console.log(JSON.stringify({ ok: true, ...record, _source: source })); From 9cf3edeb619cd6d39d3ff1eaeccaded7139a340a Mon Sep 17 00:00:00 2001 From: Miguel Angel Simon Sierra Date: Fri, 26 Jun 2026 14:39:19 -0400 Subject: [PATCH 07/55] feat(media-use): media-ops guidance + process transforms (#12, #7) references/operations.md: local-tool recipes (ffmpeg cut/reframe/montage, auto-editor, scenedetect) + local-vs-HeyGen transform table (bg-removal/upscale/ lipsync/translate) with side-by-side guidance. Outputs register via --from (auto-promoted). Per OP1, media-use guides to the tools rather than re-wrapping ffmpeg/heygen as bespoke verbs. --- skills-manifest.json | 4 +- skills/media-use/SKILL.md | 9 ++++ skills/media-use/references/operations.md | 65 +++++++++++++++++++++++ 3 files changed, 76 insertions(+), 2 deletions(-) create mode 100644 skills/media-use/references/operations.md diff --git a/skills-manifest.json b/skills-manifest.json index 51776ecfc8..f6b6676705 100644 --- a/skills-manifest.json +++ b/skills-manifest.json @@ -42,8 +42,8 @@ "files": 10 }, "media-use": { - "hash": "26b00482d105a3ad", - "files": 34 + "hash": "74804d731adb0acf", + "files": 35 }, "motion-graphics": { "hash": "5a256811f730f36e", diff --git a/skills/media-use/SKILL.md b/skills/media-use/SKILL.md index 100b0f2240..31c896564d 100644 --- a/skills/media-use/SKILL.md +++ b/skills/media-use/SKILL.md @@ -106,6 +106,15 @@ Assets are cached automatically on resolve. Subsequent resolves for the same pro - `.media/index.md` — agent-readable table (id, type, dur, dims, path, description) - `~/.media/` — global cross-project reuse cache (content-addressed, SHA-256) +## Operating on media (cut, reframe, transform) + +media-use resolves + remembers; for **operating** on assets see +`references/operations.md` — local-tool recipes (ffmpeg trim/reframe/montage, +auto-editor, scenedetect) and the local-vs-HeyGen transform table (background +removal, upscale, lipsync, translate). Run the tool, then register the output +with `resolve --from --type ` so it joins the ledger + global +cache. + ## CLI tools used | Tool | Purpose | Required? | diff --git a/skills/media-use/references/operations.md b/skills/media-use/references/operations.md new file mode 100644 index 0000000000..62aa175024 --- /dev/null +++ b/skills/media-use/references/operations.md @@ -0,0 +1,65 @@ +# Media operations — agent guidance + +media-use resolves and remembers assets. For **operating** on them — cutting, +reframing, stitching, transforming — it does not wrap every action as a bespoke +command. Instead it points you at the right local tool (decision OP1). Run the +tool, then register the output with `resolve --from --type ` so the +result lands in the ledger and the global cache like any other asset. + +All tools below are local and free. ffmpeg is assumed present (it backs the +engine already). + +## Cut / trim — keep a slice + +```bash +ffmpeg -i in.mp4 -ss 00:00:12 -to 00:00:20 -c copy out.mp4 # 0:12–0:20, no re-encode +``` + +In-composition trimming usually needs **no new file**: a clip plays a sub-window +via `data-media-start` + `data-duration` (see hyperframes-core). Only cut a +physical file when exporting/assembling outside the composition. + +## Reframe / crop — change aspect ratio + +```bash +# 16:9 -> 9:16, crop centered +ffmpeg -i in.mp4 -vf "crop=ih*9/16:ih,scale=1080:1920" out.mp4 +``` + +In Studio, crop is non-destructive (render-time `clip-path`) — the source file is +untouched. + +## Montage / stitch — join clips + +```bash +printf "file '%s'\n" a.mp4 b.mp4 c.mp4 > list.txt +ffmpeg -f concat -safe 0 -i list.txt -c copy out.mp4 +``` + +## Silence-cut / highlight — trim dead air, grab the best moment + +```bash +auto-editor in.mp4 --edit audio:threshold=4% -o tight.mp4 # pip install auto-editor +scenedetect -i in.mp4 detect-adaptive list-scenes # pip install scenedetect +``` + +## Transforms with a quality choice (process) + +These have a local option AND a higher-quality HeyGen-CLI option. Run the local +one for free/offline; use the HeyGen CLI when quality matters. Showing the user +a **side-by-side** (local vs HeyGen) is the honest way to let them choose. + +| Op | Local (free) | HeyGen CLI (quality) | +| ------------------ | -------------------------------------------------- | --------------------------- | +| Background removal | `hyperframes remove-background in.png` (u2net) | `heygen background-removal` | +| Upscale | `realesrgan-ncnn-vulkan -i in.png -o out.png -s 4` | — | +| Lipsync (dub) | — | `heygen lipsync` | +| Translate | — | `heygen video-translate` | + +After any op: `resolve --from out.ext --type ` to register the derived +asset (it records provenance and auto-promotes to the global cache). + +> ponytail: media-use doesn't re-wrap ffmpeg/heygen here — that's deliberate +> (OP1). The value it adds is the ledger + global reuse on the _output_, via +> `--from`. Add a thin `process` verb only if agents repeatedly fumble these +> recipes. From 8f73aee229f73634e40099035e982f4fc706fb1a Mon Sep 17 00:00:00 2001 From: Miguel Angel Simon Sierra Date: Fri, 26 Jun 2026 14:53:22 -0400 Subject: [PATCH 08/55] feat(studio): in-use / unused filter in the Asset tab (#13) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds 'In use' / 'Unused' filter chips alongside the category filters, reusing the existing usedPaths (composition references) the in-use badge + used-first sort already compute. filterByUsage/countUsage extracted as pure functions + unit tested (6 vitest). Chips show only on a real used/unused split (no misleading 0s). Verified in agent-browser: Studio boots, project loads, Asset tab renders + filter path runs. Note: chip display binds to usePlayerStore.elements (same source as the existing badge); that store was empty in a synthetic dev project where the timeline is driven by the iframe __clipManifest — confirm/realign usedPaths source in real studio usage (follow-up; affects badge+sort+filter equally). --- .../src/components/sidebar/AssetsTab.test.ts | 34 ++++++++++ .../src/components/sidebar/AssetsTab.tsx | 66 ++++++++++++++++++- 2 files changed, 98 insertions(+), 2 deletions(-) create mode 100644 packages/studio/src/components/sidebar/AssetsTab.test.ts diff --git a/packages/studio/src/components/sidebar/AssetsTab.test.ts b/packages/studio/src/components/sidebar/AssetsTab.test.ts new file mode 100644 index 0000000000..fc53a4488f --- /dev/null +++ b/packages/studio/src/components/sidebar/AssetsTab.test.ts @@ -0,0 +1,34 @@ +import { describe, expect, it } from "vitest"; +import { filterByUsage, countUsage } from "./AssetsTab"; + +const assets = ["bgm.mp3", "logo.png", "orphan.wav"]; +const used = new Set(["bgm.mp3", "logo.png"]); + +describe("filterByUsage", () => { + it("returns everything for 'all'", () => { + expect(filterByUsage(assets, used, "all")).toEqual(assets); + }); + + it("keeps only referenced assets for 'used'", () => { + expect(filterByUsage(assets, used, "used")).toEqual(["bgm.mp3", "logo.png"]); + }); + + it("keeps only unreferenced assets for 'unused'", () => { + expect(filterByUsage(assets, used, "unused")).toEqual(["orphan.wav"]); + }); + + it("treats everything as unused when nothing is referenced", () => { + expect(filterByUsage(assets, new Set(), "used")).toEqual([]); + expect(filterByUsage(assets, new Set(), "unused")).toEqual(assets); + }); +}); + +describe("countUsage", () => { + it("counts used vs unused", () => { + expect(countUsage(assets, used)).toEqual({ used: 2, unused: 1 }); + }); + + it("is all-unused with an empty used set", () => { + expect(countUsage(assets, new Set())).toEqual({ used: 0, unused: 3 }); + }); +}); diff --git a/packages/studio/src/components/sidebar/AssetsTab.tsx b/packages/studio/src/components/sidebar/AssetsTab.tsx index 137ec33ad5..d0f17c1a3d 100644 --- a/packages/studio/src/components/sidebar/AssetsTab.tsx +++ b/packages/studio/src/components/sidebar/AssetsTab.tsx @@ -172,6 +172,29 @@ function ImageCard({ ); } +export type UsageFilter = "all" | "used" | "unused"; + +/** Filter assets by whether the composition references them. Pure — unit-tested. */ +export function filterByUsage( + assets: string[], + usedPaths: Set, + usageFilter: UsageFilter, +): string[] { + if (usageFilter === "used") return assets.filter((a) => usedPaths.has(a)); + if (usageFilter === "unused") return assets.filter((a) => !usedPaths.has(a)); + return assets; +} + +/** Count used vs unused over a media set. Pure — unit-tested. */ +export function countUsage( + assets: string[], + usedPaths: Set, +): { used: number; unused: number } { + let used = 0; + for (const a of assets) if (usedPaths.has(a)) used++; + return { used, unused: assets.length - used }; +} + export const AssetsTab = memo(function AssetsTab({ projectId, assets, @@ -183,6 +206,7 @@ export const AssetsTab = memo(function AssetsTab({ const [dragOver, setDragOver] = useState(false); const [copiedPath, setCopiedPath] = useState(null); const [activeFilter, setActiveFilter] = useState("all"); + const [usageFilter, setUsageFilter] = useState<"all" | "used" | "unused">("all"); const [searchQuery, setSearchQuery] = useState(""); const [manifest, setManifest] = useState< Map @@ -258,7 +282,8 @@ export const AssetsTab = memo(function AssetsTab({ }, [elements]); const mediaAssets = useMemo(() => { - const all = assets.filter((a) => MEDIA_EXT.test(a) || FONT_EXT.test(a)); + const media = assets.filter((a) => MEDIA_EXT.test(a) || FONT_EXT.test(a)); + const all = filterByUsage(media, usedPaths, usageFilter); if (!searchQuery) return all; const q = searchQuery.toLowerCase(); return all.filter((a) => { @@ -266,7 +291,7 @@ export const AssetsTab = memo(function AssetsTab({ const rec = manifest.get(a); return rec?.description?.toLowerCase().includes(q); }); - }, [assets, searchQuery, manifest]); + }, [assets, searchQuery, manifest, usageFilter, usedPaths]); const categorized = useMemo(() => { const groups: Record = { audio: [], images: [], video: [], fonts: [] }; @@ -291,6 +316,17 @@ export const AssetsTab = memo(function AssetsTab({ return c; }, [mediaAssets, categorized]); + // Usage counts over the full media set (independent of the active usage filter, + // so the chips don't show their own filtered totals). + const usageCounts = useMemo( + () => + countUsage( + assets.filter((a) => MEDIA_EXT.test(a) || FONT_EXT.test(a)), + usedPaths, + ), + [assets, usedPaths], + ); + const visibleCategories = activeFilter === "all" ? FILTER_ORDER.filter((c) => categorized[c].length > 0) @@ -405,6 +441,32 @@ export const AssetsTab = memo(function AssetsTab({ ) : null, )} + {/* Usage filter — show only assets the composition references, or only the unused ones */} + {usageCounts.used > 0 && usageCounts.unused > 0 && ( + <> +