Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion packages/cli/src/audio/providers.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
/**
* Which voice / music engine a workflow will actually use, and whether
* its local dependencies are present. Mirrors the resolution order the
* hyperframes-media skill scripts use, so `auth status` and `doctor`
* media-use skill scripts use, so `auth status` and `doctor`
* report the same engine the render pipeline would pick:
*
* voice: HeyGen Starfish → ElevenLabs (key + `elevenlabs`) → Kokoro (local)
Expand Down
2 changes: 1 addition & 1 deletion packages/cli/src/commands/auth/status-guidance.ts
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ function offlineEngineLines(engines?: OfflineEngineLine[]): string[] {
* so it's left to the docs — not dangled here as a command a fresh machine
* can't run. Names the local fallback so "no key" never reads as a failure,
* and never steers users toward a per-repo `.env`. Mirrors the
* hyperframes-media skill's Preflight section.
* media-use skill's Preflight section.
*/
export function buildUnconfiguredLines(
ctx: UnconfiguredContext,
Expand Down
58 changes: 48 additions & 10 deletions packages/cli/src/commands/transcribe.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
// fallow-ignore-file code-duplication
import { defineCommand } from "citty";
import type { Example } from "./_examples.js";
import { existsSync, writeFileSync } from "node:fs";
import { findParakeet, transcribeWithParakeet } from "../whisper/parakeet.js";

type CaptionExportFormat = "srt" | "vtt";

Expand Down Expand Up @@ -41,6 +43,12 @@ export default defineCommand({
description: "Project directory (default: current directory)",
alias: "d",
},
engine: {
type: "string",
description:
"ASR engine: auto (Parakeet if installed, else whisper), parakeet, or whisper. Default: auto. Parakeet is more accurate and faster; enable with `uv pip install parakeet-mlx`.",
alias: "e",
},
model: {
type: "string",
description: `Whisper model (default: ${DEFAULT_MODEL}). Options: tiny.en, base.en, small.en, medium.en, large-v3`,
Expand Down Expand Up @@ -111,8 +119,9 @@ export default defineCommand({
return importTranscript(inputPath, dir, args.json);
}

// ── Transcribe mode: run whisper ─────────────────────────────────────
// ── Transcribe mode: run the ASR engine ──────────────────────────────
return transcribeAudio(inputPath, dir, {
engine: args.engine,
model: args.model,
language: args.language,
json: args.json,
Expand Down Expand Up @@ -213,25 +222,45 @@ async function exportTranscript(
// Transcribe audio/video with whisper
// ---------------------------------------------------------------------------

// fallow-ignore-next-line complexity
async function transcribeAudio(
inputPath: string,
dir: string,
opts: { model?: string; language?: string; json?: boolean; optional?: boolean },
opts: { engine?: string; model?: string; language?: string; json?: boolean; optional?: boolean },
): Promise<void> {
const { transcribe } = await import("../whisper/transcribe.js");
const { loadTranscript, patchCaptionHtml, stripBeforeOnset } =
await import("../whisper/normalize.js");

// Engine: auto (Parakeet if installed, else whisper), or forced parakeet/whisper.
const engine = (opts.engine ?? "auto").toLowerCase();
if (engine !== "auto" && engine !== "parakeet" && engine !== "whisper") {
failWith(`Unknown --engine: ${opts.engine}. Use auto, parakeet, or whisper.`, !!opts.json);
}
const useParakeet = engine === "parakeet" || (engine === "auto" && !!findParakeet());

const model = opts.model ?? DEFAULT_MODEL;
// --model selects the whisper model only; Parakeet uses its own fixed model.
if (useParakeet && opts.model && !opts.json) {
console.error(
c.dim(` Note: --model applies to the whisper engine only; ignored under Parakeet.`),
);
}
const label = useParakeet ? "Parakeet" : model;
const spin = opts.json ? null : clack.spinner();
spin?.start(`Transcribing with ${c.accent(model)}...`);
spin?.start(`Transcribing with ${c.accent(label)}...`);

try {
const result = await transcribe(inputPath, dir, {
model,
language: opts.language,
onProgress: spin ? (msg) => spin.message(msg) : undefined,
});
const result = useParakeet
? transcribeWithParakeet(inputPath, dir, {
language: opts.language,
onProgress: spin ? (msg) => spin.message(msg) : undefined,
})
: await transcribe(inputPath, dir, {
model,
language: opts.language,
onProgress: spin ? (msg) => spin.message(msg) : undefined,
});

let { words } = loadTranscript(result.transcriptPath);

Expand All @@ -253,7 +282,8 @@ async function transcribeAudio(
console.log(
JSON.stringify({
ok: true,
model,
engine: useParakeet ? "parakeet" : "whisper",
model: useParakeet ? "parakeet-tdt-0.6b-v3" : model,
wordCount: words.length,
durationSeconds: result.durationSeconds,
speechOnsetSeconds: result.speechOnsetSeconds,
Expand All @@ -272,7 +302,15 @@ async function transcribeAudio(
);
}
} catch (err) {
const message = err instanceof Error ? err.message : String(err);
// Surface the last few lines of the ASR subprocess's stderr, which
// execFileSync captures but otherwise drops on the floor — that's where
// parakeet-mlx / whisper report the actual failure cause.
const stderr =
err && typeof err === "object" && "stderr" in err && err.stderr
? String(err.stderr).trim().split("\n").slice(-3).join("\n")
: "";
const base = err instanceof Error ? err.message : String(err);
const message = stderr ? `${base}\n${stderr}` : base;

// whisper-cpp is an optional prerequisite, not part of the CLI. When it is
// simply unavailable (no binary, no toolchain to build one), that is a setup
Expand Down
4 changes: 3 additions & 1 deletion packages/cli/src/commands/tts.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
// fallow-ignore-file code-duplication
import { defineCommand } from "citty";
import type { Example } from "./_examples.js";
import { existsSync, readFileSync } from "node:fs";
Expand Down Expand Up @@ -48,7 +49,7 @@ export default defineCommand({
output: {
type: "string",
description: "Output file path (default: speech.wav in current directory)",
alias: "o",
alias: ["o", "out"],
},
voice: {
type: "string",
Expand Down Expand Up @@ -76,6 +77,7 @@ export default defineCommand({
default: false,
},
},
// fallow-ignore-next-line complexity
async run({ args }) {
// ── List voices mode ──────────────────────────────────────────────
if (args.list) {
Expand Down
2 changes: 1 addition & 1 deletion packages/cli/src/templates/_shared/AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@

**Porting an existing composition?** `/remotion-to-hyperframes` translates a Remotion (React) composition into HyperFrames HTML — a source migration, separate from the creation workflows above.

The domain skills (`/hyperframes-core`, `/hyperframes-animation`, `/hyperframes-creative`, `/hyperframes-cli`, `/hyperframes-media`, `/hyperframes-registry`) and the full capability map live inside `/hyperframes` — it is the single source of truth for which skill handles which intent.
The domain skills (`/hyperframes-core`, `/hyperframes-animation`, `/hyperframes-creative`, `/hyperframes-cli`, `/media-use`, `/hyperframes-registry`) and the full capability map live inside `/hyperframes` — it is the single source of truth for which skill handles which intent.

> **Tailwind v4 projects** (`hyperframes init --tailwind`): see `/hyperframes-core` → `references/tailwind.md`.

Expand Down
2 changes: 1 addition & 1 deletion packages/cli/src/templates/_shared/CLAUDE.md
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@

**Porting an existing composition?** `/remotion-to-hyperframes` translates a Remotion (React) composition into HyperFrames HTML — a source migration, separate from the creation workflows above.

The domain skills (`/hyperframes-core`, `/hyperframes-animation`, `/hyperframes-creative`, `/hyperframes-cli`, `/hyperframes-media`, `/hyperframes-registry`) and the full capability map live inside `/hyperframes` — it is the single source of truth for which skill handles which intent.
The domain skills (`/hyperframes-core`, `/hyperframes-animation`, `/hyperframes-creative`, `/hyperframes-cli`, `/media-use`, `/hyperframes-registry`) and the full capability map live inside `/hyperframes` — it is the single source of truth for which skill handles which intent.

> **Tailwind v4 projects** (`hyperframes init --tailwind`): see `/hyperframes-core` → `references/tailwind.md`.

Expand Down
15 changes: 14 additions & 1 deletion packages/cli/src/tts/python.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,21 @@

import { execFileSync } from "node:child_process";

/** Locate a `python3` (or `python`) on PATH that reports as Python 3. */
/** Locate a Python 3: `HYPERFRAMES_PYTHON` env override first, then PATH. */
export function findPython(): string | undefined {
const override = process.env.HYPERFRAMES_PYTHON;
if (override) {
try {
const version = execFileSync(override, ["--version"], {
encoding: "utf-8",
stdio: ["pipe", "pipe", "pipe"],
timeout: 5000,
});
if (/Python 3/.test(version)) return override;
} catch {
// fall through to the PATH probe
}
}
for (const name of ["python3", "python"]) {
try {
const cmd = process.platform === "win32" ? "where" : "which";
Expand Down
5 changes: 3 additions & 2 deletions packages/cli/src/tts/synthesize.ts
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,7 @@ export interface SynthesizeResult {
/**
* Synthesize text to speech using Kokoro-82M via kokoro-onnx.
*/
// fallow-ignore-next-line complexity
export async function synthesize(
text: string,
outputPath: string,
Expand All @@ -124,13 +125,13 @@ export async function synthesize(
const python = findPython();
if (!python) {
throw new Error(
"Python 3 is required for text-to-speech. Install Python 3.8+ and run: pip install kokoro-onnx soundfile",
"Python 3 is required for text-to-speech. Install Python 3.10+ and run: pip install kokoro-onnx soundfile (or point HYPERFRAMES_PYTHON at a venv python that has them)",
);
}

if (!hasPythonPackage(python, "kokoro_onnx")) {
throw new Error(
"The kokoro-onnx package is not installed. Run: pip install kokoro-onnx soundfile",
"The kokoro-onnx package is not installed. Run: pip install kokoro-onnx soundfile (or point HYPERFRAMES_PYTHON at a venv python that has them)",
);
}

Expand Down
37 changes: 37 additions & 0 deletions packages/cli/src/whisper/parakeet.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
import { describe, expect, it } from "vitest";
import { mergeTokensToWords } from "./parakeet.js";

describe("mergeTokensToWords", () => {
it("joins Parakeet sub-word tokens into words on the space boundary", () => {
const words = mergeTokensToWords({
text: "Hello everyone. Um,",
sentences: [
{
tokens: [
{ text: " H", start: 0.0, end: 0.24 },
{ text: "ello", start: 0.24, end: 0.48 },
{ text: " everyone.", start: 0.48, end: 1.28 },
{ text: " Um,", start: 1.28, end: 1.92 },
],
},
],
});
expect(words).toEqual([
{ text: "Hello", start: 0.0, end: 0.48 },
{ text: "everyone.", start: 0.48, end: 1.28 },
{ text: "Um,", start: 1.28, end: 1.92 },
]);
});

it("spans sentences and tolerates missing tokens", () => {
expect(mergeTokensToWords({}).length).toBe(0);
const words = mergeTokensToWords({
sentences: [
{ tokens: [{ text: "Hi", start: 0, end: 0.2 }] },
{ tokens: [{ text: " there", start: 0.5, end: 0.9 }] },
],
});
expect(words.map((w) => w.text)).toEqual(["Hi", "there"]);
expect(words[1]!.start).toBe(0.5);
});
});
Loading
Loading