Green screen pipeline: new tools, AnimatedBackground, caption burn fixes
Add green_screen_processor (auto-detect + rembg fallback) and green_screen_composite (4 layout presets with alpha compositing) tools to automate the full keying-to-composite pipeline. Remotion: add AnimatedBackground with gradient mesh and floating orbs to Explainer, fix caption burn tool (remove entry point arg, auto-detect dimensions, extend TalkingHead duration to 300s). Update scene-director, compose-director, and asset-director skill docs with green screen workflow steps and component constraints.
This commit is contained in:
@@ -0,0 +1,260 @@
|
||||
{
|
||||
"cuts": [
|
||||
{
|
||||
"id": "scene-1-hook",
|
||||
"source": "",
|
||||
"in_seconds": 0,
|
||||
"out_seconds": 5.5,
|
||||
"type": "hero_title",
|
||||
"text": "STOP PAYING",
|
||||
"subtitle": "Multiple EMIs Every Month",
|
||||
"backgroundColor": "#0F172A",
|
||||
"accentColor": "#EF4444"
|
||||
},
|
||||
{
|
||||
"id": "scene-2-control",
|
||||
"source": "",
|
||||
"in_seconds": 5.5,
|
||||
"out_seconds": 9.5,
|
||||
"type": "callout",
|
||||
"text": "Your loans are managing YOU",
|
||||
"callout_type": "warning",
|
||||
"backgroundColor": "#0F172A",
|
||||
"accentColor": "#F59E0B"
|
||||
},
|
||||
{
|
||||
"id": "scene-3-rates",
|
||||
"source": "",
|
||||
"in_seconds": 9.5,
|
||||
"out_seconds": 15.5,
|
||||
"type": "bar_chart",
|
||||
"chartData": [
|
||||
{"label": "Credit Cards", "value": 24},
|
||||
{"label": "Personal", "value": 16},
|
||||
{"label": "App Loans", "value": 36},
|
||||
{"label": "NBFC", "value": 18}
|
||||
],
|
||||
"title": "Interest Rates You're Paying (%)",
|
||||
"backgroundColor": "#0F172A",
|
||||
"accentColor": "#F59E0B",
|
||||
"chartColors": ["#EF4444", "#F59E0B", "#8B5CF6", "#EC4899"]
|
||||
},
|
||||
{
|
||||
"id": "scene-4-fear",
|
||||
"source": "",
|
||||
"in_seconds": 15.5,
|
||||
"out_seconds": 20,
|
||||
"type": "stat_reveal",
|
||||
"text": "Every Month",
|
||||
"subtitle": "Will I have anything left?",
|
||||
"backgroundColor": "#0F172A",
|
||||
"accentColor": "#EF4444"
|
||||
},
|
||||
{
|
||||
"id": "scene-5-reveal",
|
||||
"source": "",
|
||||
"in_seconds": 20,
|
||||
"out_seconds": 22,
|
||||
"type": "hero_title",
|
||||
"text": "WHAT NO ONE",
|
||||
"subtitle": "Tells You",
|
||||
"backgroundColor": "#0F172A",
|
||||
"accentColor": "#22D3EE"
|
||||
},
|
||||
{
|
||||
"id": "scene-6-inquiries",
|
||||
"source": "",
|
||||
"in_seconds": 22,
|
||||
"out_seconds": 40.5,
|
||||
"type": "progress_bar",
|
||||
"progress": 95,
|
||||
"label": "Loan Inquiries Piling Up",
|
||||
"progressLabel": "DANGER ZONE",
|
||||
"progressColor": "#EF4444",
|
||||
"animationStyle": "step",
|
||||
"progressSegments": [
|
||||
{"value": 25, "color": "#F59E0B", "label": "Bank"},
|
||||
{"value": 25, "color": "#F97316", "label": "NBFC"},
|
||||
{"value": 25, "color": "#EF4444", "label": "Digital"},
|
||||
{"value": 20, "color": "#DC2626", "label": "Random"}
|
||||
],
|
||||
"backgroundColor": "#0F172A"
|
||||
},
|
||||
{
|
||||
"id": "scene-7-score-drop",
|
||||
"source": "",
|
||||
"in_seconds": 41,
|
||||
"out_seconds": 46.5,
|
||||
"type": "line_chart",
|
||||
"chartSeries": [
|
||||
{
|
||||
"name": "Credit Score",
|
||||
"data": [
|
||||
{"x": 0, "y": 750},
|
||||
{"x": 1, "y": 720},
|
||||
{"x": 2, "y": 680},
|
||||
{"x": 3, "y": 620}
|
||||
],
|
||||
"color": "#EF4444"
|
||||
}
|
||||
],
|
||||
"title": "Your Credit Score Drops",
|
||||
"xLabel": "Each Inquiry",
|
||||
"yLabel": "Score",
|
||||
"showGrid": true,
|
||||
"showMarkers": true,
|
||||
"chartAnimation": "draw",
|
||||
"backgroundColor": "#0F172A",
|
||||
"accentColor": "#EF4444"
|
||||
},
|
||||
{
|
||||
"id": "scene-8-risk",
|
||||
"source": "",
|
||||
"in_seconds": 47,
|
||||
"out_seconds": 49.5,
|
||||
"type": "callout",
|
||||
"text": "Lenders See You As HIGH RISK",
|
||||
"callout_type": "warning",
|
||||
"backgroundColor": "#0F172A",
|
||||
"accentColor": "#EF4444"
|
||||
},
|
||||
{
|
||||
"id": "scene-9-wrong",
|
||||
"source": "",
|
||||
"in_seconds": 50,
|
||||
"out_seconds": 53.5,
|
||||
"type": "stat_card",
|
||||
"stat": "❌",
|
||||
"subtitle": "Wrong approach wastes your time",
|
||||
"backgroundColor": "#0F172A",
|
||||
"accentColor": "#EF4444"
|
||||
},
|
||||
{
|
||||
"id": "scene-9-right",
|
||||
"source": "",
|
||||
"in_seconds": 53.5,
|
||||
"out_seconds": 57,
|
||||
"type": "stat_card",
|
||||
"stat": "✅",
|
||||
"subtitle": "Right approach gets approval",
|
||||
"backgroundColor": "#0F172A",
|
||||
"accentColor": "#10B981"
|
||||
},
|
||||
{
|
||||
"id": "scene-10-matters",
|
||||
"source": "",
|
||||
"in_seconds": 60,
|
||||
"out_seconds": 66,
|
||||
"type": "callout",
|
||||
"text": "WHO you approach matters as much as WHAT you apply for",
|
||||
"callout_type": "tip",
|
||||
"backgroundColor": "#0F172A",
|
||||
"accentColor": "#10B981"
|
||||
},
|
||||
{
|
||||
"id": "scene-11-creditwin",
|
||||
"source": "",
|
||||
"in_seconds": 66,
|
||||
"out_seconds": 80,
|
||||
"type": "hero_title",
|
||||
"text": "CREDIT WIN",
|
||||
"subtitle": "Your Consolidation Partner",
|
||||
"backgroundColor": "#0F172A",
|
||||
"accentColor": "#22D3EE"
|
||||
},
|
||||
{
|
||||
"id": "scene-12-oneplan",
|
||||
"source": "",
|
||||
"in_seconds": 81,
|
||||
"out_seconds": 84.5,
|
||||
"type": "kpi_grid",
|
||||
"chartData": [
|
||||
{"label": "EMI", "value": "1"},
|
||||
{"label": "Due Date", "value": "1"},
|
||||
{"label": "Clear Plan", "value": "1"}
|
||||
],
|
||||
"title": "From Chaos to Control",
|
||||
"backgroundColor": "#0F172A",
|
||||
"accentColor": "#22D3EE",
|
||||
"chartColors": ["#22D3EE", "#10B981", "#A78BFA"]
|
||||
},
|
||||
{
|
||||
"id": "scene-13a-years",
|
||||
"source": "",
|
||||
"in_seconds": 84.5,
|
||||
"out_seconds": 89,
|
||||
"type": "stat_card",
|
||||
"stat": "15+",
|
||||
"subtitle": "Years of Experience",
|
||||
"backgroundColor": "#0F172A",
|
||||
"accentColor": "#F59E0B"
|
||||
},
|
||||
{
|
||||
"id": "scene-13b-borrowers",
|
||||
"source": "",
|
||||
"in_seconds": 89,
|
||||
"out_seconds": 94,
|
||||
"type": "stat_card",
|
||||
"stat": "10,000+",
|
||||
"subtitle": "Borrowers Served",
|
||||
"backgroundColor": "#0F172A",
|
||||
"accentColor": "#22D3EE"
|
||||
},
|
||||
{
|
||||
"id": "scene-13c-saved",
|
||||
"source": "",
|
||||
"in_seconds": 94,
|
||||
"out_seconds": 99,
|
||||
"type": "stat_card",
|
||||
"stat": "₹5K-15K",
|
||||
"subtitle": "Saved Per Month on EMIs",
|
||||
"backgroundColor": "#0F172A",
|
||||
"accentColor": "#10B981"
|
||||
},
|
||||
{
|
||||
"id": "scene-14-qualify",
|
||||
"source": "",
|
||||
"in_seconds": 100,
|
||||
"out_seconds": 118,
|
||||
"type": "stat_card",
|
||||
"stat": "₹60,000+",
|
||||
"subtitle": "Salary? You Qualify for Consolidation",
|
||||
"backgroundColor": "#0F172A",
|
||||
"accentColor": "#22D3EE"
|
||||
},
|
||||
{
|
||||
"id": "scene-15-before",
|
||||
"source": "",
|
||||
"in_seconds": 118,
|
||||
"out_seconds": 121,
|
||||
"type": "stat_card",
|
||||
"stat": "5 EMIs",
|
||||
"subtitle": "Chaos → Exit This",
|
||||
"backgroundColor": "#0F172A",
|
||||
"accentColor": "#EF4444"
|
||||
},
|
||||
{
|
||||
"id": "scene-15-after",
|
||||
"source": "",
|
||||
"in_seconds": 121,
|
||||
"out_seconds": 124,
|
||||
"type": "stat_card",
|
||||
"stat": "1 EMI",
|
||||
"subtitle": "Control → Your New Reality",
|
||||
"backgroundColor": "#0F172A",
|
||||
"accentColor": "#10B981"
|
||||
},
|
||||
{
|
||||
"id": "scene-16-cta",
|
||||
"source": "",
|
||||
"in_seconds": 124,
|
||||
"out_seconds": 127,
|
||||
"type": "hero_title",
|
||||
"text": "APPLY NOW",
|
||||
"subtitle": "CreditWin.in",
|
||||
"backgroundColor": "#0F172A",
|
||||
"accentColor": "#10B981"
|
||||
}
|
||||
],
|
||||
"overlays": []
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
{
|
||||
"cuts": [
|
||||
{
|
||||
"id": "scene-1-hook",
|
||||
"source": "",
|
||||
"in_seconds": 0,
|
||||
"out_seconds": 5.5,
|
||||
"type": "hero_title",
|
||||
"text": "STOP PAYING",
|
||||
"subtitle": "Multiple EMIs Every Month",
|
||||
"backgroundColor": "#0F172A",
|
||||
"accentColor": "#EF4444"
|
||||
},
|
||||
{
|
||||
"id": "scene-2-control",
|
||||
"source": "",
|
||||
"in_seconds": 5.5,
|
||||
"out_seconds": 9.5,
|
||||
"type": "callout",
|
||||
"text": "Your loans are managing YOU",
|
||||
"callout_type": "warning",
|
||||
"backgroundColor": "#0F172A",
|
||||
"accentColor": "#F59E0B"
|
||||
},
|
||||
{
|
||||
"id": "scene-3-rates",
|
||||
"source": "",
|
||||
"in_seconds": 9.5,
|
||||
"out_seconds": 15.5,
|
||||
"type": "bar_chart",
|
||||
"chartData": [
|
||||
{"label": "Credit Cards", "value": 24},
|
||||
{"label": "Personal", "value": 16},
|
||||
{"label": "App Loans", "value": 36},
|
||||
{"label": "NBFC", "value": 18}
|
||||
],
|
||||
"title": "Interest Rates You're Paying (%)",
|
||||
"backgroundColor": "#0F172A",
|
||||
"accentColor": "#F59E0B",
|
||||
"chartColors": ["#EF4444", "#F59E0B", "#8B5CF6", "#EC4899"]
|
||||
},
|
||||
{
|
||||
"id": "scene-4-fear",
|
||||
"source": "",
|
||||
"in_seconds": 15.5,
|
||||
"out_seconds": 20,
|
||||
"type": "stat_reveal",
|
||||
"text": "Every Month",
|
||||
"subtitle": "Will I have anything left?",
|
||||
"backgroundColor": "#0F172A",
|
||||
"accentColor": "#EF4444"
|
||||
}
|
||||
],
|
||||
"overlays": []
|
||||
}
|
||||
@@ -44,6 +44,97 @@ const { fontFamily } = loadFont("normal", {
|
||||
subsets: ["latin"],
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Animated Background — Gradient Mesh + Floating Orbs
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const AnimatedBackground: React.FC<{ style?: "fintech" | "default" }> = ({
|
||||
style: bgStyle = "default",
|
||||
}) => {
|
||||
const frame = useCurrentFrame();
|
||||
const { fps, durationInFrames, width, height } = useVideoConfig();
|
||||
const progress = frame / durationInFrames;
|
||||
|
||||
// Slow-moving gradient angles
|
||||
const angle1 = 135 + Math.sin(frame / (fps * 8)) * 30;
|
||||
const angle2 = 225 + Math.cos(frame / (fps * 6)) * 25;
|
||||
|
||||
// Color stops shift over time
|
||||
const shift = Math.sin(frame / (fps * 12)) * 0.15;
|
||||
|
||||
const gradient = `
|
||||
radial-gradient(ellipse at ${30 + Math.sin(frame / (fps * 10)) * 20}% ${40 + Math.cos(frame / (fps * 8)) * 20}%,
|
||||
rgba(15, 23, 60, 1) 0%, transparent 60%),
|
||||
radial-gradient(ellipse at ${70 + Math.cos(frame / (fps * 7)) * 20}% ${60 + Math.sin(frame / (fps * 9)) * 25}%,
|
||||
rgba(30, 10, 60, 0.8) 0%, transparent 55%),
|
||||
radial-gradient(ellipse at ${50 + Math.sin(frame / (fps * 14)) * 30}% ${20 + Math.cos(frame / (fps * 11)) * 15}%,
|
||||
rgba(0, 40, 60, 0.6) 0%, transparent 50%),
|
||||
linear-gradient(${angle1}deg, #060918 0%, #0B1026 40%, #0F0A2E 70%, #080D1F 100%)
|
||||
`;
|
||||
|
||||
// Floating orbs
|
||||
const orbs = [
|
||||
{ x: 20, y: 30, size: 300, color: "rgba(34, 211, 238, 0.08)", speedX: 7, speedY: 11 },
|
||||
{ x: 70, y: 60, size: 250, color: "rgba(139, 92, 246, 0.1)", speedX: 9, speedY: 8 },
|
||||
{ x: 40, y: 80, size: 200, color: "rgba(16, 185, 129, 0.07)", speedX: 13, speedY: 6 },
|
||||
{ x: 80, y: 20, size: 350, color: "rgba(245, 158, 11, 0.06)", speedX: 11, speedY: 14 },
|
||||
{ x: 10, y: 70, size: 180, color: "rgba(236, 72, 153, 0.05)", speedX: 8, speedY: 10 },
|
||||
];
|
||||
|
||||
return (
|
||||
<AbsoluteFill style={{ background: gradient }}>
|
||||
{/* Floating glow orbs */}
|
||||
{orbs.map((orb, i) => {
|
||||
const ox = orb.x + Math.sin(frame / (fps * orb.speedX)) * 15;
|
||||
const oy = orb.y + Math.cos(frame / (fps * orb.speedY)) * 12;
|
||||
return (
|
||||
<div
|
||||
key={i}
|
||||
style={{
|
||||
position: "absolute",
|
||||
left: `${ox}%`,
|
||||
top: `${oy}%`,
|
||||
width: orb.size,
|
||||
height: orb.size,
|
||||
borderRadius: "50%",
|
||||
background: orb.color,
|
||||
filter: `blur(${orb.size * 0.4}px)`,
|
||||
transform: "translate(-50%, -50%)",
|
||||
willChange: "transform",
|
||||
}}
|
||||
/>
|
||||
);
|
||||
})}
|
||||
|
||||
{/* Subtle grid overlay */}
|
||||
<div
|
||||
style={{
|
||||
position: "absolute",
|
||||
inset: 0,
|
||||
backgroundImage: `
|
||||
linear-gradient(rgba(255,255,255,0.02) 1px, transparent 1px),
|
||||
linear-gradient(90deg, rgba(255,255,255,0.02) 1px, transparent 1px)
|
||||
`,
|
||||
backgroundSize: "60px 60px",
|
||||
opacity: 0.5 + Math.sin(frame / (fps * 20)) * 0.2,
|
||||
}}
|
||||
/>
|
||||
|
||||
{/* Top gradient fade for depth */}
|
||||
<div
|
||||
style={{
|
||||
position: "absolute",
|
||||
top: 0,
|
||||
left: 0,
|
||||
right: 0,
|
||||
height: "30%",
|
||||
background: "linear-gradient(to bottom, rgba(6,9,24,0.4), transparent)",
|
||||
}}
|
||||
/>
|
||||
</AbsoluteFill>
|
||||
);
|
||||
};
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Types — aligned with edit_decisions artifact schema
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -353,7 +444,9 @@ const SceneRenderer: React.FC<{ cut: Cut }> = ({ cut }) => {
|
||||
};
|
||||
|
||||
// Resolve the scene element based on cut type, then wrap with backgroundImage if set
|
||||
const bgColor = cut.backgroundImage ? "transparent" : cut.backgroundColor;
|
||||
// Use transparent bg so the animated gradient background shows through
|
||||
const rawBg = cut.backgroundImage ? "transparent" : cut.backgroundColor;
|
||||
const bgColor = (rawBg === "#0F172A" || rawBg === "#0f172a") ? "transparent" : rawBg;
|
||||
|
||||
// Explicit component types
|
||||
if (cut.type === "text_card" && cut.text) {
|
||||
@@ -539,7 +632,10 @@ export const Explainer: React.FC<ExplainerProps> = ({
|
||||
const { fps, durationInFrames } = useVideoConfig();
|
||||
|
||||
return (
|
||||
<AbsoluteFill style={{ background: "#0F172A", fontFamily }}>
|
||||
<AbsoluteFill style={{ background: "#060918", fontFamily }}>
|
||||
{/* Layer 0: Animated gradient background */}
|
||||
<AnimatedBackground style="fintech" />
|
||||
|
||||
{/* Layer 1: Visual scenes */}
|
||||
{cuts.map((cut) => {
|
||||
const from = Math.round(cut.in_seconds * fps);
|
||||
|
||||
@@ -65,13 +65,14 @@ export const Root: React.FC = () => {
|
||||
<Composition
|
||||
id="TalkingHead"
|
||||
component={TalkingHead}
|
||||
durationInFrames={30 * 30}
|
||||
durationInFrames={30 * 300}
|
||||
fps={30}
|
||||
width={1080}
|
||||
height={1920}
|
||||
defaultProps={{
|
||||
videoSrc: "",
|
||||
captions: [],
|
||||
overlays: [],
|
||||
wordsPerPage: 4,
|
||||
fontSize: 52,
|
||||
highlightColor: "#22D3EE",
|
||||
|
||||
@@ -1,9 +1,304 @@
|
||||
import { AbsoluteFill, OffthreadVideo } from "remotion";
|
||||
import {
|
||||
AbsoluteFill,
|
||||
OffthreadVideo,
|
||||
Sequence,
|
||||
interpolate,
|
||||
useCurrentFrame,
|
||||
useVideoConfig,
|
||||
} from "remotion";
|
||||
import { CaptionOverlay, WordCaption } from "./components/CaptionOverlay";
|
||||
import { TextCard } from "./components/TextCard";
|
||||
import { StatCard } from "./components/StatCard";
|
||||
import { CalloutBox } from "./components/CalloutBox";
|
||||
import { ComparisonCard } from "./components/ComparisonCard";
|
||||
import { BarChart } from "./components/charts/BarChart";
|
||||
import { LineChart } from "./components/charts/LineChart";
|
||||
import { PieChart } from "./components/charts/PieChart";
|
||||
import { KPIGrid } from "./components/charts/KPIGrid";
|
||||
import { HeroTitle } from "./components/HeroTitle";
|
||||
import { SectionTitle } from "./components/SectionTitle";
|
||||
import { StatReveal } from "./components/StatReveal";
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Overlay types for talking-head video
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export interface TalkingHeadOverlay {
|
||||
id?: string;
|
||||
type: string;
|
||||
in_seconds: number;
|
||||
out_seconds: number;
|
||||
position?:
|
||||
| "lower_third"
|
||||
| "upper_third"
|
||||
| "left_panel"
|
||||
| "right_panel"
|
||||
| "full_overlay";
|
||||
// Component-specific props (same as Explainer Cut)
|
||||
text?: string;
|
||||
stat?: string;
|
||||
subtitle?: string;
|
||||
callout_type?: "info" | "warning" | "tip" | "quote";
|
||||
title?: string;
|
||||
leftLabel?: string;
|
||||
rightLabel?: string;
|
||||
leftValue?: string;
|
||||
rightValue?: string;
|
||||
chartData?: any[];
|
||||
chartSeries?: any[];
|
||||
chartColors?: string[];
|
||||
chartAnimation?: string;
|
||||
donut?: boolean;
|
||||
centerLabel?: string;
|
||||
centerValue?: string;
|
||||
showGrid?: boolean;
|
||||
showValues?: boolean;
|
||||
showLegend?: boolean;
|
||||
showMarkers?: boolean;
|
||||
columns?: 2 | 3 | 4;
|
||||
// Styling
|
||||
backgroundColor?: string;
|
||||
color?: string;
|
||||
accentColor?: string;
|
||||
fontSize?: number;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Position presets for 9:16 (1080x1920) frame
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const POSITION_STYLES: Record<string, React.CSSProperties> = {
|
||||
lower_third: {
|
||||
position: "absolute",
|
||||
bottom: 320, // Above caption area (~1600px)
|
||||
left: 40,
|
||||
right: 40,
|
||||
height: 480,
|
||||
},
|
||||
upper_third: {
|
||||
position: "absolute",
|
||||
top: 80,
|
||||
left: 40,
|
||||
right: 40,
|
||||
height: 480,
|
||||
},
|
||||
left_panel: {
|
||||
position: "absolute",
|
||||
top: 200,
|
||||
left: 40,
|
||||
width: 480,
|
||||
bottom: 400,
|
||||
},
|
||||
right_panel: {
|
||||
position: "absolute",
|
||||
top: 200,
|
||||
right: 40,
|
||||
width: 480,
|
||||
bottom: 400,
|
||||
},
|
||||
full_overlay: {
|
||||
position: "absolute",
|
||||
top: 0,
|
||||
left: 0,
|
||||
right: 0,
|
||||
bottom: 0,
|
||||
},
|
||||
};
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Overlay component dispatcher — maps overlay type to Remotion component
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const OverlayContent: React.FC<{ overlay: TalkingHeadOverlay }> = ({
|
||||
overlay,
|
||||
}) => {
|
||||
const bgColor = overlay.backgroundColor || "#0F172A";
|
||||
|
||||
if (overlay.type === "text_card" && overlay.text) {
|
||||
return (
|
||||
<TextCard
|
||||
text={overlay.text}
|
||||
fontSize={overlay.fontSize}
|
||||
color={overlay.color}
|
||||
backgroundColor={bgColor}
|
||||
/>
|
||||
);
|
||||
}
|
||||
if (overlay.type === "stat_card" && overlay.stat) {
|
||||
return (
|
||||
<StatCard
|
||||
stat={overlay.stat}
|
||||
subtitle={overlay.subtitle}
|
||||
accentColor={overlay.accentColor}
|
||||
backgroundColor={bgColor}
|
||||
/>
|
||||
);
|
||||
}
|
||||
if (overlay.type === "callout" && overlay.text) {
|
||||
return (
|
||||
<CalloutBox
|
||||
text={overlay.text}
|
||||
type={overlay.callout_type}
|
||||
title={overlay.title}
|
||||
borderColor={overlay.accentColor}
|
||||
backgroundColor={overlay.backgroundColor}
|
||||
textColor={overlay.color}
|
||||
containerBackgroundColor={bgColor}
|
||||
/>
|
||||
);
|
||||
}
|
||||
if (
|
||||
overlay.type === "comparison" &&
|
||||
overlay.leftLabel &&
|
||||
overlay.rightLabel
|
||||
) {
|
||||
return (
|
||||
<ComparisonCard
|
||||
leftLabel={overlay.leftLabel}
|
||||
rightLabel={overlay.rightLabel}
|
||||
leftValue={overlay.leftValue || ""}
|
||||
rightValue={overlay.rightValue || ""}
|
||||
title={overlay.title}
|
||||
backgroundColor={bgColor}
|
||||
textColor={overlay.color}
|
||||
/>
|
||||
);
|
||||
}
|
||||
if (overlay.type === "bar_chart" && overlay.chartData) {
|
||||
return (
|
||||
<BarChart
|
||||
data={overlay.chartData}
|
||||
title={overlay.title}
|
||||
colors={overlay.chartColors}
|
||||
animationStyle={(overlay.chartAnimation as any) || "grow-up"}
|
||||
showValues={overlay.showValues}
|
||||
backgroundColor={bgColor}
|
||||
/>
|
||||
);
|
||||
}
|
||||
if (overlay.type === "line_chart" && overlay.chartSeries) {
|
||||
return (
|
||||
<LineChart
|
||||
series={overlay.chartSeries}
|
||||
title={overlay.title}
|
||||
colors={overlay.chartColors}
|
||||
animationStyle={(overlay.chartAnimation as any) || "draw"}
|
||||
showGrid={overlay.showGrid}
|
||||
showMarkers={overlay.showMarkers}
|
||||
showLegend={overlay.showLegend}
|
||||
backgroundColor={bgColor}
|
||||
/>
|
||||
);
|
||||
}
|
||||
if (overlay.type === "pie_chart" && overlay.chartData) {
|
||||
return (
|
||||
<PieChart
|
||||
data={overlay.chartData}
|
||||
title={overlay.title}
|
||||
colors={overlay.chartColors}
|
||||
animationStyle={(overlay.chartAnimation as any) || "expand"}
|
||||
donut={overlay.donut}
|
||||
centerLabel={overlay.centerLabel}
|
||||
centerValue={overlay.centerValue}
|
||||
showLegend={overlay.showLegend}
|
||||
backgroundColor={bgColor}
|
||||
/>
|
||||
);
|
||||
}
|
||||
if (overlay.type === "kpi_grid" && overlay.chartData) {
|
||||
return (
|
||||
<KPIGrid
|
||||
metrics={overlay.chartData}
|
||||
title={overlay.title}
|
||||
columns={overlay.columns}
|
||||
colors={overlay.chartColors}
|
||||
animationStyle={(overlay.chartAnimation as any) || "count-up"}
|
||||
backgroundColor={bgColor}
|
||||
/>
|
||||
);
|
||||
}
|
||||
if (overlay.type === "hero_title" && overlay.text) {
|
||||
return <HeroTitle title={overlay.text} subtitle={overlay.subtitle} />;
|
||||
}
|
||||
if (overlay.type === "section_title" && overlay.text) {
|
||||
return (
|
||||
<SectionTitle
|
||||
title={overlay.text}
|
||||
subtitle={overlay.subtitle}
|
||||
accentColor={overlay.accentColor}
|
||||
position="top-left"
|
||||
/>
|
||||
);
|
||||
}
|
||||
if (overlay.type === "stat_reveal" && overlay.text) {
|
||||
return (
|
||||
<StatReveal
|
||||
stat={overlay.text}
|
||||
label={overlay.subtitle}
|
||||
accentColor={overlay.accentColor}
|
||||
position="bottom-right"
|
||||
/>
|
||||
);
|
||||
}
|
||||
return null;
|
||||
};
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Positioned overlay wrapper — handles position + fade in/out
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const PositionedOverlay: React.FC<{ overlay: TalkingHeadOverlay }> = ({
|
||||
overlay,
|
||||
}) => {
|
||||
const frame = useCurrentFrame();
|
||||
const { durationInFrames } = useVideoConfig();
|
||||
|
||||
// Fade in over 8 frames (~0.27s), fade out over 8 frames
|
||||
const fadeIn = interpolate(frame, [0, 8], [0, 1], {
|
||||
extrapolateLeft: "clamp",
|
||||
extrapolateRight: "clamp",
|
||||
});
|
||||
const fadeOut = interpolate(
|
||||
frame,
|
||||
[durationInFrames - 8, durationInFrames],
|
||||
[1, 0],
|
||||
{ extrapolateLeft: "clamp", extrapolateRight: "clamp" }
|
||||
);
|
||||
const opacity = fadeIn * fadeOut;
|
||||
|
||||
const position = overlay.position || "lower_third";
|
||||
const posStyle = POSITION_STYLES[position] || POSITION_STYLES.lower_third;
|
||||
const isFullOverlay = position === "full_overlay";
|
||||
|
||||
return (
|
||||
<div
|
||||
style={{
|
||||
...posStyle,
|
||||
opacity,
|
||||
overflow: "hidden",
|
||||
borderRadius: isFullOverlay ? 0 : 16,
|
||||
boxShadow: isFullOverlay
|
||||
? "none"
|
||||
: "0 8px 32px rgba(0, 0, 0, 0.4)",
|
||||
}}
|
||||
>
|
||||
{isFullOverlay && (
|
||||
<AbsoluteFill style={{ background: "rgba(0, 0, 0, 0.7)" }} />
|
||||
)}
|
||||
<OverlayContent overlay={overlay} />
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Main TalkingHead composition
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export interface TalkingHeadProps {
|
||||
[key: string]: unknown;
|
||||
videoSrc: string;
|
||||
captions: WordCaption[];
|
||||
overlays?: TalkingHeadOverlay[];
|
||||
wordsPerPage?: number;
|
||||
fontSize?: number;
|
||||
highlightColor?: string;
|
||||
@@ -12,16 +307,39 @@ export interface TalkingHeadProps {
|
||||
export const TalkingHead: React.FC<TalkingHeadProps> = ({
|
||||
videoSrc,
|
||||
captions,
|
||||
overlays,
|
||||
wordsPerPage = 4,
|
||||
fontSize = 52,
|
||||
highlightColor = "#22D3EE",
|
||||
}) => {
|
||||
const { fps } = useVideoConfig();
|
||||
|
||||
return (
|
||||
<AbsoluteFill style={{ backgroundColor: "#000" }}>
|
||||
{/* Layer 1: Video background */}
|
||||
<OffthreadVideo
|
||||
src={videoSrc}
|
||||
style={{ width: "100%", height: "100%", objectFit: "cover" }}
|
||||
/>
|
||||
|
||||
{/* Layer 2: Overlays (charts, stats, callouts, etc.) */}
|
||||
{overlays?.map((overlay, i) => {
|
||||
const from = Math.round(overlay.in_seconds * fps);
|
||||
const duration = Math.round(
|
||||
(overlay.out_seconds - overlay.in_seconds) * fps
|
||||
);
|
||||
return (
|
||||
<Sequence
|
||||
key={overlay.id || `overlay-${i}`}
|
||||
from={from}
|
||||
durationInFrames={duration}
|
||||
>
|
||||
<PositionedOverlay overlay={overlay} />
|
||||
</Sequence>
|
||||
);
|
||||
})}
|
||||
|
||||
{/* Layer 3: Captions (topmost — always visible above overlays) */}
|
||||
<CaptionOverlay
|
||||
words={captions}
|
||||
wordsPerPage={wordsPerPage}
|
||||
|
||||
@@ -90,7 +90,34 @@ Create a composition JSON snippet for each overlay. These will be rendered by th
|
||||
| Callout/quote | `callout` | `text`, `icon` |
|
||||
| Lower third | `text_card` | `text` |
|
||||
|
||||
**Dark theme for all overlays** — use dark backgrounds (`#0F172A`, `#1E293B`) with light text. This ensures overlays are legible when composited on top of talking-head footage.
|
||||
**Remotion AnimatedBackground:**
|
||||
|
||||
The Explainer composition now includes an `AnimatedBackground` component that renders an animated gradient mesh, floating orbs, and a subtle grid pattern. This provides a far more professional look than flat solid colors.
|
||||
|
||||
- Scene backgrounds should use `backgroundColor: "#0F172A"` which auto-converts to transparent, letting the animated gradient show through.
|
||||
- Do NOT use flat solid colors for backgrounds -- always let the AnimatedBackground render behind overlays.
|
||||
- When compositing green screen footage, render the AnimatedBackground as the replacement background (see compose-director Step 3c).
|
||||
|
||||
**Component constraints:**
|
||||
|
||||
| Component | Min Width | 720px Portrait? | Value Type |
|
||||
|-----------|-----------|-----------------|------------|
|
||||
| comparison | 900px | NO -> use 2x stat_card | string |
|
||||
| kpi_grid | 720px | YES | numeric ONLY (no "15+") |
|
||||
| bar_chart | 500px | YES | numeric |
|
||||
| stat_card | 300px | YES | string OK |
|
||||
| callout | 400px | YES | string |
|
||||
| hero_title | 400px | YES | string |
|
||||
| line_chart | 500px | YES | numeric |
|
||||
| progress_bar | 600px | YES | numeric |
|
||||
| stat_reveal | 300px | YES | string OK |
|
||||
|
||||
Key rules:
|
||||
- `comparison` requires 900px+ width. In 720px portrait frames, substitute with two sequential `stat_card` components instead.
|
||||
- `kpi_grid` values MUST be purely numeric (e.g., `4.8`, `73`, `2400`). Formatted strings like `"15+"`, `"$4.8B"`, or `"2.4M"` will cause rendering errors. Use `stat_card` for string-formatted numbers instead.
|
||||
- Always check the target frame width before choosing a component. Portrait (720px) excludes `comparison`.
|
||||
|
||||
**Dark theme for all overlays** -- use dark backgrounds (`#0F172A`, `#1E293B`) with light text. This ensures overlays are legible when composited on top of talking-head footage. The `#0F172A` backgroundColor is treated as transparent by the AnimatedBackground system, so overlays will show the animated gradient behind them.
|
||||
|
||||
**For simple text overlays** (if Remotion is overkill):
|
||||
|
||||
|
||||
@@ -15,6 +15,45 @@ You have edit decisions and an asset manifest. Your job is to render the final t
|
||||
|
||||
## Process
|
||||
|
||||
### Step 0: Pre-flight Checks
|
||||
|
||||
Before rendering anything, validate the inputs and catch issues that are expensive to fix later.
|
||||
|
||||
1. **Silence detection** -- Run `silence_cutter` in mark mode:
|
||||
```
|
||||
silence_cutter.execute({
|
||||
"input_path": "<raw_footage>",
|
||||
"mode": "mark",
|
||||
"silence_threshold_db": -35,
|
||||
"min_silence_duration": 0.5
|
||||
})
|
||||
```
|
||||
- Report all gaps > 0.5s with timestamps.
|
||||
- If total silence > 5s, **recommend cutting before proceeding**. Long silences waste render time and produce dead spots in the final video.
|
||||
|
||||
2. **ASR confidence check** -- Scan word-level transcript for low-confidence words:
|
||||
- Flag any word with probability < 0.7.
|
||||
- List flagged words with timestamps so the user can verify correct transcription.
|
||||
- Common misrecognitions to watch for: proper nouns, brand names, domain jargon.
|
||||
|
||||
3. **Auto-build corrections dictionary** from common ASR error patterns:
|
||||
```python
|
||||
corrections = {
|
||||
# Indian finance context
|
||||
"DMI": "EMI",
|
||||
"AMI": "EMI",
|
||||
# Common brand misspellings
|
||||
"open montage": "OpenMontage",
|
||||
"remotion": "Remotion",
|
||||
# Numbers that got split by ASR
|
||||
"4 -5": "4-5",
|
||||
"10 -15": "10-15",
|
||||
}
|
||||
```
|
||||
Extend this dict with domain-specific corrections based on the video topic. Present the corrections to the user for review before applying.
|
||||
|
||||
4. **Green screen flag** -- Check if scene-director Step 0 flagged green/blue screen footage. If yes, note that Step 3c (Green Screen Composite) will be needed.
|
||||
|
||||
### Step 1: Run Enhancement Chain
|
||||
|
||||
Apply video enhancements in this exact order. **Attempt every step** if the tool is available — do not skip steps without a reason.
|
||||
@@ -106,7 +145,14 @@ Pass this dict to both `subtitle_gen` (if generating SRT) and `remotion_caption_
|
||||
|
||||
### Step 3: Burn Subtitles
|
||||
|
||||
**Preferred: Remotion captions** (if `remotion_caption_burn` tool available):
|
||||
**ALWAYS use Remotion TikTok-style captions** (word-by-word highlighting). This is the default and preferred method. Do NOT fall back to FFmpeg ASS subtitles unless Remotion is completely unavailable.
|
||||
|
||||
**Remotion caption requirements:**
|
||||
- **Auto-detect video dimensions** -- do NOT hardcode width/height. Use `visual_qa` probe or ffprobe to get actual dimensions, then pass them to the render.
|
||||
- **Set `--frames` based on actual video duration** -- calculate from probe: `frames = duration_seconds * fps`. Never use a hardcoded frame count.
|
||||
- Word-by-word highlighting with active word color (`highlight_color`).
|
||||
- Captions positioned at the bottom of frame, away from the face.
|
||||
|
||||
```
|
||||
remotion_caption_burn.execute({
|
||||
"input_path": "<reframed_or_enhanced_video>",
|
||||
@@ -118,15 +164,11 @@ remotion_caption_burn.execute({
|
||||
"highlight_color": "#22D3EE",
|
||||
})
|
||||
```
|
||||
Remotion renders animated word-by-word captions at the bottom of the frame with active word highlighting. Captions are positioned away from the face.
|
||||
|
||||
**Fallback: FFmpeg subtitles** (if Remotion unavailable):
|
||||
Use `video_compose` with `burn_subtitles` operation:
|
||||
- Input: reframed video (or enhanced video if no reframe needed)
|
||||
- Subtitle file from asset manifest
|
||||
**Fallback ONLY if Remotion is completely unavailable:** Use `video_compose` with `burn_subtitles` operation. This is a degraded experience -- warn the user that word-by-word highlighting won't be available.
|
||||
|
||||
**CRITICAL: Caption positioning for 9:16 vertical video.**
|
||||
Captions MUST be in the lower 20% of the frame. On a 1920-high frame, that means `MarginV=160` or higher. The default FFmpeg subtitle position is center — this WILL occlude the face. You MUST override it.
|
||||
**CRITICAL: Caption positioning for 9:16 vertical video (FFmpeg fallback only).**
|
||||
Captions MUST be in the lower 20% of the frame. On a 1920-high frame, that means `MarginV=160` or higher. The default FFmpeg subtitle position is center -- this WILL occlude the face. You MUST override it.
|
||||
|
||||
FFmpeg subtitle style string for vertical talking-head:
|
||||
```
|
||||
@@ -137,27 +179,152 @@ FFmpeg subtitle style string for vertical talking-head:
|
||||
|
||||
### Step 3b: Burn Overlay Graphics (if scene plan includes overlays)
|
||||
|
||||
If the scene plan includes overlay scenes (text_cards, stat_cards, charts, comparisons, callouts), render them onto the video.
|
||||
If the scene plan includes overlay scenes (text_cards, stat_cards, charts, comparisons, callouts), pass them to `remotion_caption_burn` alongside captions. **Both captions and overlays render in a single Remotion pass** — no separate FFmpeg compositing needed.
|
||||
|
||||
**How overlay compositing works:**
|
||||
1. Each overlay is a short Remotion composition (3-5 seconds) rendered as a transparent video clip or composited directly
|
||||
2. Use `video_compose` with `picture_in_picture` or `overlay` operation to place each overlay at the correct timestamp
|
||||
3. Respect the overlay's `position` field from the scene plan:
|
||||
- `lower_third` → bottom 30% of frame
|
||||
- `upper_third` → top 30% of frame
|
||||
- `side_panel` → left or right 40%
|
||||
- `full_overlay` → centered, brief (1-2s)
|
||||
**How it works:** The TalkingHead Remotion composition renders three layers:
|
||||
1. **Video** (bottom) — the talking-head footage
|
||||
2. **Overlays** (middle) — positioned charts, stats, callouts with fade in/out
|
||||
3. **Captions** (top) — word-by-word highlighting, always visible
|
||||
|
||||
**For Remotion-based overlays:** Create a composition JSON with the overlay cuts, render to a transparent clip, then composite onto the talking-head video using FFmpeg.
|
||||
|
||||
**For simple text overlays:** Use FFmpeg's drawtext filter directly:
|
||||
**Combine Step 3 and 3b into one `remotion_caption_burn` call:**
|
||||
```
|
||||
ffmpeg -i captioned.mp4 -vf "drawtext=text='Key Term':fontsize=48:fontcolor=white:borderw=3:bordercolor=black:x=(w-text_w)/2:y=h*0.75:enable='between(t,22,26)'" -c:a copy output.mp4
|
||||
remotion_caption_burn.execute({
|
||||
"input_path": "<reframed_or_enhanced_video>",
|
||||
"output_path": "<project>/assets/video/captioned.mp4",
|
||||
"segments": <transcript_segments>,
|
||||
"corrections": {"cloud": "Claude"},
|
||||
"words_per_page": 4,
|
||||
"font_size": 52,
|
||||
"highlight_color": "#22D3EE",
|
||||
"overlays": [
|
||||
{
|
||||
"id": "term-agentic-ai",
|
||||
"type": "callout",
|
||||
"text": "Agentic AI: software that acts autonomously toward goals",
|
||||
"callout_type": "info",
|
||||
"in_seconds": 22.0,
|
||||
"out_seconds": 26.0,
|
||||
"position": "lower_third",
|
||||
"backgroundColor": "#0F172A",
|
||||
"accentColor": "#22D3EE"
|
||||
},
|
||||
{
|
||||
"id": "stat-market-size",
|
||||
"type": "stat_card",
|
||||
"stat": "$4.8B",
|
||||
"subtitle": "Global AI Agent Market (2026)",
|
||||
"in_seconds": 35.0,
|
||||
"out_seconds": 39.0,
|
||||
"position": "upper_third",
|
||||
"accentColor": "#A78BFA"
|
||||
},
|
||||
{
|
||||
"id": "chart-growth",
|
||||
"type": "bar_chart",
|
||||
"chartData": [
|
||||
{"label": "2023", "value": 1.2},
|
||||
{"label": "2024", "value": 2.1},
|
||||
{"label": "2025", "value": 3.5},
|
||||
{"label": "2026", "value": 4.8}
|
||||
],
|
||||
"title": "AI Agent Market ($B)",
|
||||
"in_seconds": 40.0,
|
||||
"out_seconds": 45.0,
|
||||
"position": "lower_third",
|
||||
"chartColors": ["#22D3EE", "#A78BFA", "#F472B6", "#34D399"]
|
||||
}
|
||||
]
|
||||
})
|
||||
```
|
||||
|
||||
**Important:** Time each overlay to match the scene plan timestamps. After speed adjustment, recalculate overlay timestamps: `adjusted_time = original_time / speed_factor`.
|
||||
**Overlay position options:**
|
||||
- `lower_third` → bottom area, above captions (default — safest for most overlays)
|
||||
- `upper_third` → top area (good for stats while speaker is center/lower)
|
||||
- `left_panel` → left 45% of frame (side-by-side with speaker)
|
||||
- `right_panel` → right 45% of frame
|
||||
- `full_overlay` → full frame with dark backdrop (use sparingly, 1-2s max)
|
||||
|
||||
### Step 3c: Build Showcase Cards (if multi-clip reel)
|
||||
**Overlay type → required props** (same as asset-director mapping):
|
||||
|
||||
| Type | Required Props |
|
||||
|------|---------------|
|
||||
| `text_card` | `text` |
|
||||
| `stat_card` | `stat`, `subtitle` (optional) |
|
||||
| `callout` | `text`, `callout_type` (info/warning/tip/quote) |
|
||||
| `comparison` | `leftLabel`, `rightLabel`, `leftValue`, `rightValue` |
|
||||
| `bar_chart` | `chartData` (array of `{label, value}`) |
|
||||
| `line_chart` | `chartSeries` (array of `{name, data: number[]}`) |
|
||||
| `pie_chart` | `chartData` (array of `{label, value}`) |
|
||||
| `kpi_grid` | `chartData` (array of `{label, value}`) |
|
||||
| `hero_title` | `text`, `subtitle` (optional) |
|
||||
| `section_title` | `text`, `subtitle` (optional) |
|
||||
| `stat_reveal` | `text` (the stat), `subtitle` (label) |
|
||||
|
||||
**Important:** After speed adjustment, recalculate overlay timestamps: `adjusted_time = original_time / speed_factor`.
|
||||
|
||||
**Fallback (no Remotion):** If Remotion is unavailable, `remotion_caption_burn` falls back to FFmpeg for captions only. Overlays are NOT rendered in FFmpeg fallback mode — warn the user that overlays require Remotion.
|
||||
|
||||
### Step 3c: Green Screen Composite (if green screen footage)
|
||||
|
||||
If the footage has a green/blue screen (detected in scene-director Step 0), follow this pipeline:
|
||||
|
||||
1. **Run `green_screen_processor` tool** to remove the green/blue screen:
|
||||
```
|
||||
green_screen_processor.execute({
|
||||
"input_path": "<enhanced_video>",
|
||||
"output_path": "<project>/assets/video/greenscreen_removed.mp4",
|
||||
"method": "auto"
|
||||
})
|
||||
```
|
||||
The `auto` method detects whether the background is green or blue and applies the appropriate chroma key.
|
||||
|
||||
2. **Render Remotion animated background** using the Explainer composition:
|
||||
```
|
||||
# Render an AnimatedBackground clip (gradient mesh, floating orbs, subtle grid)
|
||||
# Use the Explainer composition — NOT a flat #0F172A solid color
|
||||
npx remotion render src/index.ts Explainer --props='{"duration":VIDEO_DURATION}' --output=<project>/assets/video/animated_bg.mp4
|
||||
```
|
||||
The AnimatedBackground provides a professional gradient mesh with floating orbs and a subtle grid pattern. This is far superior to a flat solid color.
|
||||
|
||||
3. **Run `green_screen_composite` tool** to layer the speaker onto the animated background:
|
||||
```
|
||||
green_screen_composite.execute({
|
||||
"foreground_path": "<greenscreen_removed_video>",
|
||||
"background_path": "<animated_bg>",
|
||||
"output_path": "<project>/assets/video/composited.mp4",
|
||||
"layout": "news_anchor"
|
||||
})
|
||||
```
|
||||
Default layout is `news_anchor` (speaker center-bottom, background fills frame). Adjust layout based on speaker position detected in Step 0.
|
||||
|
||||
4. **Burn captions via Remotion TalkingHead composition** (NOT FFmpeg ASS subtitles):
|
||||
```
|
||||
remotion_caption_burn.execute({
|
||||
"input_path": "<composited_video>",
|
||||
"output_path": "<project>/assets/video/captioned.mp4",
|
||||
"segments": <transcript_segments>,
|
||||
"corrections": <corrections_dict>,
|
||||
"words_per_page": 4,
|
||||
"font_size": 52,
|
||||
"highlight_color": "#22D3EE",
|
||||
"overlays": <overlay_list_from_scene_plan>
|
||||
})
|
||||
```
|
||||
|
||||
5. **Mix background music** (ducked at 15% volume under speech):
|
||||
```
|
||||
audio_mixer.execute({
|
||||
"operation": "duck",
|
||||
"video_path": "<captioned_video>",
|
||||
"music_path": "<bg_music>",
|
||||
"music_volume": 0.15,
|
||||
"output_path": "<project>/assets/video/with_music.mp4"
|
||||
})
|
||||
```
|
||||
|
||||
6. **Final encode** to target platform specs (see Step 6 below).
|
||||
|
||||
### Step 3d: Build Showcase Cards (if multi-clip reel)
|
||||
|
||||
If the output is a reel with showcase clips, use `showcase_card` for each:
|
||||
```
|
||||
|
||||
@@ -18,6 +18,34 @@ You are not just a processor. You are a creative director. Your job is to figure
|
||||
|
||||
## Process
|
||||
|
||||
### Step 0: Footage Analysis
|
||||
|
||||
Before watching the content, analyze the raw footage to understand the physical setup.
|
||||
|
||||
1. **Sample 5 frames** from the footage using ffmpeg (evenly spaced across the duration):
|
||||
```
|
||||
ffmpeg -i <footage> -vf "select='not(mod(n\,TOTAL_FRAMES/5))'" -vsync vfr -frames:v 5 frame_%02d.png
|
||||
```
|
||||
|
||||
2. **Run visual_qa or histogram analysis** on each sampled frame to detect:
|
||||
- **Background type:** Green screen / blue screen / natural background. Green/blue screens show a dominant narrow-band color spike in the histogram. Use `visual_qa` with prompt "Is this a green screen or blue screen background?" for confirmation.
|
||||
- **Speaker position:** Center, left, or right of frame. Estimate the approximate bounding box (e.g., "speaker occupies center 40% of frame, from x=30% to x=70%").
|
||||
- **Lighting quality:** Even studio lighting, harsh shadows, backlit, mixed color temperature. Note any issues that may affect chroma keying.
|
||||
|
||||
3. **Green screen detected?**
|
||||
- If yes, note that `green_screen_processor` tool will be needed in the compose stage.
|
||||
- Record the detected screen color (green or blue) and estimated uniformity.
|
||||
- The compose-director will use this to run chroma key removal and composite onto an animated background.
|
||||
|
||||
4. **Measure speaker safe zone:**
|
||||
- From the speaker's bounding box, determine where graphics can be placed WITHOUT overlapping the speaker.
|
||||
- For a centered speaker: left panel and right panel are safe for overlays.
|
||||
- For a left-positioned speaker: right panel is the primary safe zone.
|
||||
- For a right-positioned speaker: left panel is the primary safe zone.
|
||||
- Upper third and lower third are generally safe regardless of speaker position.
|
||||
|
||||
Record all findings in the scene plan metadata for downstream stages.
|
||||
|
||||
### Step 1: Watch & Listen — Understand the Content
|
||||
|
||||
**This is the most important step. Do not skip it.**
|
||||
@@ -52,6 +80,24 @@ Based on your content analysis, propose **on-screen graphics** that will appear
|
||||
| **Callout/quote** | `callout` | When the speaker makes a key point worth emphasizing |
|
||||
| **Lower third** | `text_card` | Speaker identification at the start |
|
||||
|
||||
**Remotion Component Constraints:**
|
||||
|
||||
| Component | Min Width | 720px Portrait? | Value Type |
|
||||
|-----------|-----------|-----------------|------------|
|
||||
| comparison | 900px | NO -> use 2x stat_card | string |
|
||||
| kpi_grid | 720px | YES | numeric ONLY (no "15+") |
|
||||
| bar_chart | 500px | YES | numeric |
|
||||
| stat_card | 300px | YES | string OK |
|
||||
| callout | 400px | YES | string |
|
||||
| hero_title | 400px | YES | string |
|
||||
| line_chart | 500px | YES | numeric |
|
||||
| progress_bar | 600px | YES | numeric |
|
||||
| stat_reveal | 300px | YES | string OK |
|
||||
|
||||
When a component's minimum width exceeds the available space (e.g., `comparison` at 900px won't fit in a 720px portrait frame), substitute with the recommended alternative. For `comparison`, use two `stat_card` components shown sequentially instead.
|
||||
|
||||
For `kpi_grid`, values MUST be numeric (e.g., `4.8`, `73`, `2400`). String values like `"15+"` or `"$4.8B"` will cause rendering errors. Use `stat_card` for string-formatted numbers.
|
||||
|
||||
**Overlay planning rules:**
|
||||
- **Don't over-overlay.** 3-6 overlays per minute of final video is the sweet spot. More than that is distracting.
|
||||
- **Time overlays to speech.** Each overlay should appear when the speaker says the relevant words, not before or after.
|
||||
@@ -92,6 +138,8 @@ Format your proposal clearly:
|
||||
**Estimated final duration:** ~Xs (from Xs raw)
|
||||
```
|
||||
|
||||
**IMPORTANT: When outputting the overlay plan, ALSO generate the actual Remotion JSON props file (`greenscreen-bg.json`) -- do not just describe scenes in prose.** The JSON props file should be a complete, valid input for the Remotion TalkingHead composition, including all overlay definitions, timing, colors, and content. Save it to `<project>/public/demo-props/` or the project's props directory.
|
||||
|
||||
Wait for user approval before proceeding. The user may:
|
||||
- Approve as-is
|
||||
- Add/remove overlays
|
||||
|
||||
@@ -0,0 +1,430 @@
|
||||
"""Green screen composite tool for talking-head pipeline.
|
||||
|
||||
Composites a keyed speaker (dark/solid background) over a Remotion
|
||||
background video with layout presets. Supports news anchor, full behind,
|
||||
picture-in-picture, and split layouts.
|
||||
|
||||
Uses PIL/numpy for frame-level alpha compositing and FFmpeg for
|
||||
frame extraction, encoding, and audio muxing.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import shutil
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
from PIL import Image
|
||||
|
||||
from tools.base_tool import (
|
||||
BaseTool,
|
||||
Determinism,
|
||||
ExecutionMode,
|
||||
ResourceProfile,
|
||||
RetryPolicy,
|
||||
ResumeSupport,
|
||||
ToolResult,
|
||||
ToolStability,
|
||||
ToolTier,
|
||||
)
|
||||
|
||||
|
||||
class GreenScreenComposite(BaseTool):
|
||||
name = "green_screen_composite"
|
||||
version = "0.1.0"
|
||||
tier = ToolTier.CORE
|
||||
capability = "video_post"
|
||||
provider = "ffmpeg"
|
||||
stability = ToolStability.EXPERIMENTAL
|
||||
execution_mode = ExecutionMode.SYNC
|
||||
determinism = Determinism.DETERMINISTIC
|
||||
|
||||
dependencies = ["cmd:ffmpeg", "python:numpy", "python:PIL"]
|
||||
install_instructions = (
|
||||
"Install FFmpeg: https://ffmpeg.org/download.html — "
|
||||
"pip install numpy Pillow"
|
||||
)
|
||||
agent_skills = ["ffmpeg"]
|
||||
|
||||
capabilities = [
|
||||
"green_screen_composite",
|
||||
"speaker_overlay",
|
||||
"layout_preset",
|
||||
"alpha_composite",
|
||||
]
|
||||
|
||||
input_schema = {
|
||||
"type": "object",
|
||||
"required": ["speaker_path", "background_path", "output_path"],
|
||||
"properties": {
|
||||
"speaker_path": {
|
||||
"type": "string",
|
||||
"description": "Path to keyed speaker video (dark bg, from green_screen_processor)",
|
||||
},
|
||||
"background_path": {
|
||||
"type": "string",
|
||||
"description": "Path to Remotion background video",
|
||||
},
|
||||
"output_path": {
|
||||
"type": "string",
|
||||
"description": "Output composite video path",
|
||||
},
|
||||
"original_audio_path": {
|
||||
"type": "string",
|
||||
"description": "Path to original footage to extract audio from",
|
||||
},
|
||||
"layout": {
|
||||
"type": "string",
|
||||
"enum": ["news_anchor", "full_behind", "pip", "split"],
|
||||
"default": "news_anchor",
|
||||
"description": (
|
||||
"news_anchor=speaker bottom-center over shifted bg, "
|
||||
"full_behind=speaker full-frame on bg, "
|
||||
"pip=speaker 30% bottom-right, "
|
||||
"split=speaker left 50% bg right 50%"
|
||||
),
|
||||
},
|
||||
"speaker_scale": {
|
||||
"type": "number",
|
||||
"default": 0.65,
|
||||
"description": "Scale factor for speaker layer",
|
||||
},
|
||||
"bg_shift_up": {
|
||||
"type": "integer",
|
||||
"default": 300,
|
||||
"description": "Pixels to shift background content upward",
|
||||
},
|
||||
"bg_color_hex": {
|
||||
"type": "string",
|
||||
"default": "#0E172A",
|
||||
"description": "The keyed speaker's background color for alpha creation",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
resource_profile = ResourceProfile(
|
||||
cpu_cores=4, ram_mb=4096, vram_mb=0, disk_mb=8000, network_required=False
|
||||
)
|
||||
retry_policy = RetryPolicy(max_retries=1, retryable_errors=["FFmpeg error"])
|
||||
resume_support = ResumeSupport.FROM_START
|
||||
idempotency_key_fields = [
|
||||
"speaker_path", "background_path", "layout",
|
||||
"speaker_scale", "bg_shift_up", "bg_color_hex",
|
||||
]
|
||||
side_effects = ["writes composite video to output_path"]
|
||||
user_visible_verification = [
|
||||
"Watch output — speaker should be cleanly composited without color fringing",
|
||||
"Check layout positioning matches the chosen preset",
|
||||
"Verify audio is synced if original_audio_path was provided",
|
||||
]
|
||||
|
||||
def execute(self, inputs: dict[str, Any]) -> ToolResult:
|
||||
speaker_path = Path(inputs["speaker_path"])
|
||||
background_path = Path(inputs["background_path"])
|
||||
output_path = Path(inputs["output_path"])
|
||||
original_audio_path = inputs.get("original_audio_path")
|
||||
layout = inputs.get("layout", "news_anchor")
|
||||
speaker_scale = inputs.get("speaker_scale", 0.65)
|
||||
bg_shift_up = inputs.get("bg_shift_up", 300)
|
||||
bg_color_hex = inputs.get("bg_color_hex", "#0E172A")
|
||||
|
||||
if not speaker_path.exists():
|
||||
return ToolResult(success=False, error=f"Speaker video not found: {speaker_path}")
|
||||
if not background_path.exists():
|
||||
return ToolResult(success=False, error=f"Background video not found: {background_path}")
|
||||
if original_audio_path and not Path(original_audio_path).exists():
|
||||
return ToolResult(success=False, error=f"Audio source not found: {original_audio_path}")
|
||||
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
start = time.time()
|
||||
|
||||
# Parse bg color
|
||||
bg_color = self._parse_hex_color(bg_color_hex)
|
||||
|
||||
# Step 1: Probe both videos
|
||||
speaker_info = self._probe_video(speaker_path)
|
||||
bg_info = self._probe_video(background_path)
|
||||
|
||||
if not speaker_info or not bg_info:
|
||||
return ToolResult(
|
||||
success=False,
|
||||
error="Failed to probe one or both input videos",
|
||||
)
|
||||
|
||||
# Step 2: Use the LOWER fps (typically 15fps from speaker)
|
||||
target_fps = min(speaker_info["fps"], bg_info["fps"])
|
||||
if target_fps <= 0:
|
||||
target_fps = 15.0
|
||||
|
||||
# Determine output dimensions from background
|
||||
out_w = bg_info["width"]
|
||||
out_h = bg_info["height"]
|
||||
|
||||
# Use shorter duration
|
||||
duration = min(speaker_info["duration"], bg_info["duration"])
|
||||
|
||||
# Step 3: Extract frames from both videos
|
||||
temp_dir = output_path.parent / ".greenscreen_composite_tmp"
|
||||
speaker_frames_dir = temp_dir / "speaker"
|
||||
bg_frames_dir = temp_dir / "bg"
|
||||
comp_frames_dir = temp_dir / "composite"
|
||||
|
||||
for d in [speaker_frames_dir, bg_frames_dir, comp_frames_dir]:
|
||||
d.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
try:
|
||||
self._extract_frames(speaker_path, speaker_frames_dir, target_fps)
|
||||
self._extract_frames(background_path, bg_frames_dir, target_fps)
|
||||
|
||||
# Get sorted frame lists
|
||||
speaker_frames = sorted(speaker_frames_dir.glob("*.png"))
|
||||
bg_frames = sorted(bg_frames_dir.glob("*.png"))
|
||||
|
||||
if not speaker_frames or not bg_frames:
|
||||
return ToolResult(
|
||||
success=False,
|
||||
error="Frame extraction produced no frames",
|
||||
)
|
||||
|
||||
frame_count = min(len(speaker_frames), len(bg_frames))
|
||||
log_interval = max(1, frame_count // 10)
|
||||
|
||||
# Step 4: Composite each frame pair
|
||||
for i in range(frame_count):
|
||||
if i % log_interval == 0:
|
||||
print(f"[green_screen_composite] Compositing frame {i + 1}/{frame_count}")
|
||||
|
||||
speaker_img = Image.open(speaker_frames[i]).convert("RGB")
|
||||
bg_img = Image.open(bg_frames[i]).convert("RGB")
|
||||
|
||||
comp = self._composite_frame(
|
||||
speaker_img, bg_img, bg_color,
|
||||
layout=layout,
|
||||
speaker_scale=speaker_scale,
|
||||
bg_shift_up=bg_shift_up,
|
||||
out_w=out_w,
|
||||
out_h=out_h,
|
||||
)
|
||||
comp.save(comp_frames_dir / f"frame_{i:06d}.png")
|
||||
|
||||
print(f"[green_screen_composite] All {frame_count} frames composited")
|
||||
|
||||
# Step 5: Encode composite frames to video
|
||||
no_audio_path = output_path if not original_audio_path else temp_dir / "no_audio.mp4"
|
||||
self._encode_frames(comp_frames_dir, no_audio_path, target_fps, out_w, out_h)
|
||||
|
||||
# Step 6: Mux audio if provided
|
||||
if original_audio_path:
|
||||
self._mux_audio(no_audio_path, Path(original_audio_path), output_path, duration)
|
||||
|
||||
if not output_path.exists() or output_path.stat().st_size == 0:
|
||||
return ToolResult(success=False, error="Output video was not created")
|
||||
|
||||
elapsed = time.time() - start
|
||||
|
||||
return ToolResult(
|
||||
success=True,
|
||||
data={
|
||||
"output": str(output_path),
|
||||
"layout": layout,
|
||||
"fps": target_fps,
|
||||
"frame_count": frame_count,
|
||||
"duration": round(duration, 2),
|
||||
"dimensions": f"{out_w}x{out_h}",
|
||||
"speaker_scale": speaker_scale,
|
||||
"has_audio": bool(original_audio_path),
|
||||
},
|
||||
artifacts=[str(output_path)],
|
||||
duration_seconds=round(elapsed, 2),
|
||||
)
|
||||
except Exception as e:
|
||||
return ToolResult(success=False, error=f"Composite failed: {e}")
|
||||
finally:
|
||||
# Step 7: Clean up temp directories
|
||||
self._cleanup_temp(temp_dir)
|
||||
|
||||
def _parse_hex_color(self, hex_str: str) -> np.ndarray:
|
||||
"""Parse a hex color string like '#0E172A' to an RGB numpy array."""
|
||||
hex_str = hex_str.lstrip("#")
|
||||
r = int(hex_str[0:2], 16)
|
||||
g = int(hex_str[2:4], 16)
|
||||
b = int(hex_str[4:6], 16)
|
||||
return np.array([r, g, b])
|
||||
|
||||
def _probe_video(self, path: Path) -> dict[str, Any] | None:
|
||||
"""Probe a video for fps, duration, and dimensions."""
|
||||
cmd = [
|
||||
"ffprobe", "-v", "quiet",
|
||||
"-print_format", "json",
|
||||
"-show_format", "-show_streams",
|
||||
str(path),
|
||||
]
|
||||
try:
|
||||
result = self.run_command(cmd, timeout=30)
|
||||
data = json.loads(result.stdout)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
# Find video stream
|
||||
video_stream = None
|
||||
for stream in data.get("streams", []):
|
||||
if stream.get("codec_type") == "video":
|
||||
video_stream = stream
|
||||
break
|
||||
|
||||
if not video_stream:
|
||||
return None
|
||||
|
||||
# Parse fps from r_frame_rate (e.g., "30/1" or "15000/1001")
|
||||
fps_str = video_stream.get("r_frame_rate", "30/1")
|
||||
try:
|
||||
num, den = fps_str.split("/")
|
||||
fps = float(num) / float(den)
|
||||
except (ValueError, ZeroDivisionError):
|
||||
fps = 30.0
|
||||
|
||||
duration = float(data.get("format", {}).get("duration", 0))
|
||||
|
||||
return {
|
||||
"fps": fps,
|
||||
"duration": duration,
|
||||
"width": int(video_stream.get("width", 1920)),
|
||||
"height": int(video_stream.get("height", 1080)),
|
||||
}
|
||||
|
||||
def _extract_frames(self, video_path: Path, output_dir: Path, fps: float) -> None:
|
||||
"""Extract frames from a video at the given fps."""
|
||||
cmd = [
|
||||
"ffmpeg", "-y",
|
||||
"-i", str(video_path),
|
||||
"-vf", f"fps={fps}",
|
||||
str(output_dir / "frame_%06d.png"),
|
||||
]
|
||||
self.run_command(cmd, timeout=600)
|
||||
|
||||
def _composite_frame(
|
||||
self,
|
||||
speaker_img: Image.Image,
|
||||
bg_img: Image.Image,
|
||||
bg_color: np.ndarray,
|
||||
*,
|
||||
layout: str,
|
||||
speaker_scale: float,
|
||||
bg_shift_up: int,
|
||||
out_w: int,
|
||||
out_h: int,
|
||||
) -> Image.Image:
|
||||
"""Composite a single speaker frame over a background frame using the given layout."""
|
||||
# Create alpha mask from speaker frame
|
||||
speaker_arr = np.array(speaker_img).astype(float)
|
||||
dist = np.sqrt(np.sum((speaker_arr - bg_color.astype(float)) ** 2, axis=2))
|
||||
threshold = 35
|
||||
alpha = np.clip((dist - threshold) * 8, 0, 255).astype(np.uint8)
|
||||
|
||||
speaker_rgba = Image.new("RGBA", speaker_img.size)
|
||||
speaker_rgba.paste(speaker_img, (0, 0))
|
||||
speaker_rgba.putalpha(Image.fromarray(alpha))
|
||||
|
||||
# Prepare background canvas at output size
|
||||
canvas = Image.new("RGBA", (out_w, out_h), (0, 0, 0, 255))
|
||||
|
||||
if layout == "news_anchor":
|
||||
# Background shifted up so graphics appear above speaker's head
|
||||
bg_resized = bg_img.resize((out_w, out_h), Image.LANCZOS).convert("RGBA")
|
||||
# Shift background up: paste it higher so bottom content scrolls up
|
||||
shifted_bg = Image.new("RGBA", (out_w, out_h), (0, 0, 0, 255))
|
||||
shifted_bg.paste(bg_resized, (0, -bg_shift_up))
|
||||
canvas = shifted_bg
|
||||
|
||||
# Scale speaker and place at bottom center
|
||||
sp_w = int(speaker_rgba.width * speaker_scale)
|
||||
sp_h = int(speaker_rgba.height * speaker_scale)
|
||||
speaker_scaled = speaker_rgba.resize((sp_w, sp_h), Image.LANCZOS)
|
||||
x = (out_w - sp_w) // 2
|
||||
y = out_h - sp_h
|
||||
canvas.paste(speaker_scaled, (x, y), speaker_scaled)
|
||||
|
||||
elif layout == "full_behind":
|
||||
# Speaker full-frame on background, no scaling, no shifting
|
||||
bg_resized = bg_img.resize((out_w, out_h), Image.LANCZOS).convert("RGBA")
|
||||
canvas = bg_resized
|
||||
|
||||
# Resize speaker to match output
|
||||
speaker_full = speaker_rgba.resize((out_w, out_h), Image.LANCZOS)
|
||||
canvas.paste(speaker_full, (0, 0), speaker_full)
|
||||
|
||||
elif layout == "pip":
|
||||
# Background full-frame, speaker 30% in bottom-right
|
||||
bg_resized = bg_img.resize((out_w, out_h), Image.LANCZOS).convert("RGBA")
|
||||
canvas = bg_resized
|
||||
|
||||
pip_scale = 0.30
|
||||
sp_w = int(out_w * pip_scale)
|
||||
sp_h = int(out_h * pip_scale)
|
||||
speaker_pip = speaker_rgba.resize((sp_w, sp_h), Image.LANCZOS)
|
||||
margin = 20
|
||||
x = out_w - sp_w - margin
|
||||
y = out_h - sp_h - margin
|
||||
canvas.paste(speaker_pip, (x, y), speaker_pip)
|
||||
|
||||
elif layout == "split":
|
||||
# Speaker on left 50%, background on right 50%
|
||||
half_w = out_w // 2
|
||||
|
||||
# Left side: speaker resized to fill left half
|
||||
speaker_left = speaker_rgba.resize((half_w, out_h), Image.LANCZOS)
|
||||
# Right side: background cropped/resized to fill right half
|
||||
bg_right = bg_img.resize((half_w, out_h), Image.LANCZOS).convert("RGBA")
|
||||
|
||||
canvas.paste(speaker_left, (0, 0), speaker_left)
|
||||
canvas.paste(bg_right, (half_w, 0), bg_right)
|
||||
|
||||
# Convert to RGB for output
|
||||
return canvas.convert("RGB")
|
||||
|
||||
def _encode_frames(
|
||||
self, frames_dir: Path, output_path: Path, fps: float, width: int, height: int
|
||||
) -> None:
|
||||
"""Encode PNG frames to an MP4 video."""
|
||||
cmd = [
|
||||
"ffmpeg", "-y",
|
||||
"-framerate", str(fps),
|
||||
"-i", str(frames_dir / "frame_%06d.png"),
|
||||
"-c:v", "libx264", "-crf", "18", "-preset", "fast",
|
||||
"-pix_fmt", "yuv420p",
|
||||
"-vf", f"scale={width}:{height}",
|
||||
str(output_path),
|
||||
]
|
||||
self.run_command(cmd, timeout=600)
|
||||
|
||||
def _mux_audio(
|
||||
self, video_path: Path, audio_source: Path, output_path: Path, duration: float
|
||||
) -> None:
|
||||
"""Mux audio from the original source into the composite video."""
|
||||
cmd = [
|
||||
"ffmpeg", "-y",
|
||||
"-i", str(video_path),
|
||||
"-i", str(audio_source),
|
||||
"-t", f"{duration:.3f}",
|
||||
"-c:v", "copy",
|
||||
"-c:a", "aac", "-b:a", "192k",
|
||||
"-map", "0:v:0", "-map", "1:a:0",
|
||||
"-shortest",
|
||||
str(output_path),
|
||||
]
|
||||
self.run_command(cmd, timeout=300)
|
||||
|
||||
def _cleanup_temp(self, temp_dir: Path) -> None:
|
||||
"""Remove temporary frame directories."""
|
||||
if temp_dir.exists():
|
||||
try:
|
||||
shutil.rmtree(temp_dir)
|
||||
except OSError:
|
||||
# Best-effort cleanup; log but don't fail
|
||||
print(f"[green_screen_composite] Warning: could not fully clean {temp_dir}")
|
||||
|
||||
def estimate_runtime(self, inputs: dict[str, Any]) -> float:
|
||||
return 120.0
|
||||
@@ -0,0 +1,622 @@
|
||||
"""Green screen keying processor.
|
||||
|
||||
Removes green/blue screen backgrounds from footage using either FFmpeg
|
||||
chromakey filtering or rembg AI segmentation. Supports automatic method
|
||||
detection by analyzing frame color histograms.
|
||||
|
||||
Methods:
|
||||
- auto: Analyze frames to pick the best method (chromakey vs rembg)
|
||||
- chromakey: FFmpeg chromakey filter (fast, works well on clean screens)
|
||||
- rembg: AI background removal via rembg/u2net (slower, handles any bg)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import platform
|
||||
import shutil
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from tools.base_tool import (
|
||||
BaseTool,
|
||||
Determinism,
|
||||
ExecutionMode,
|
||||
ResourceProfile,
|
||||
RetryPolicy,
|
||||
ResumeSupport,
|
||||
ToolResult,
|
||||
ToolStability,
|
||||
ToolTier,
|
||||
)
|
||||
|
||||
|
||||
class GreenScreenProcessor(BaseTool):
|
||||
name = "green_screen_processor"
|
||||
version = "0.1.0"
|
||||
tier = ToolTier.CORE
|
||||
capability = "video_post"
|
||||
provider = "ffmpeg"
|
||||
stability = ToolStability.EXPERIMENTAL
|
||||
execution_mode = ExecutionMode.SYNC
|
||||
determinism = Determinism.DETERMINISTIC
|
||||
|
||||
dependencies = ["cmd:ffmpeg"]
|
||||
install_instructions = (
|
||||
"Install FFmpeg: https://ffmpeg.org/download.html "
|
||||
"For rembg method: pip install rembg[gpu] onnxruntime"
|
||||
)
|
||||
agent_skills = ["ffmpeg"]
|
||||
|
||||
capabilities = [
|
||||
"green_screen_keying",
|
||||
"chromakey",
|
||||
"background_removal",
|
||||
"rembg_segmentation",
|
||||
]
|
||||
|
||||
input_schema = {
|
||||
"type": "object",
|
||||
"required": ["input_path", "output_path"],
|
||||
"properties": {
|
||||
"input_path": {
|
||||
"type": "string",
|
||||
"description": "Path to raw green screen footage",
|
||||
},
|
||||
"output_path": {
|
||||
"type": "string",
|
||||
"description": "Path for keyed output video",
|
||||
},
|
||||
"method": {
|
||||
"type": "string",
|
||||
"enum": ["auto", "chromakey", "rembg"],
|
||||
"default": "auto",
|
||||
"description": "Keying method: auto detects best approach, chromakey uses FFmpeg, rembg uses AI segmentation",
|
||||
},
|
||||
"fps": {
|
||||
"type": "integer",
|
||||
"default": 15,
|
||||
"description": "Output frames per second",
|
||||
},
|
||||
"bg_color": {
|
||||
"type": "string",
|
||||
"default": "#0E172A",
|
||||
"description": "Hex color for output background",
|
||||
},
|
||||
"max_frames": {
|
||||
"type": "integer",
|
||||
"default": 0,
|
||||
"description": "Limit frames to process (0 = all)",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
resource_profile = ResourceProfile(
|
||||
cpu_cores=4, ram_mb=4096, vram_mb=0, disk_mb=8000, network_required=False
|
||||
)
|
||||
retry_policy = RetryPolicy(max_retries=1, retryable_errors=["FFmpeg error"])
|
||||
resume_support = ResumeSupport.FROM_START
|
||||
idempotency_key_fields = [
|
||||
"input_path", "method", "fps", "bg_color", "max_frames",
|
||||
]
|
||||
side_effects = ["writes keyed video to output_path"]
|
||||
user_visible_verification = [
|
||||
"Check output for green fringing around subject edges",
|
||||
"Verify background is cleanly replaced with target color",
|
||||
"Look for flickering or inconsistent keying between frames",
|
||||
]
|
||||
|
||||
# Platform-specific null device
|
||||
_null_device = "NUL" if platform.system() == "Windows" else "/dev/null"
|
||||
|
||||
def execute(self, inputs: dict[str, Any]) -> ToolResult:
|
||||
input_path = Path(inputs["input_path"])
|
||||
if not input_path.exists():
|
||||
return ToolResult(success=False, error=f"Input not found: {input_path}")
|
||||
|
||||
output_path = Path(inputs["output_path"])
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
method = inputs.get("method", "auto")
|
||||
fps = inputs.get("fps", 15)
|
||||
bg_color = inputs.get("bg_color", "#0E172A")
|
||||
max_frames = inputs.get("max_frames", 0)
|
||||
start = time.time()
|
||||
|
||||
# Step 1: Probe input video
|
||||
probe = self._probe_video(input_path)
|
||||
if not probe:
|
||||
return ToolResult(success=False, error="Failed to probe input video")
|
||||
|
||||
duration = probe["duration"]
|
||||
width = probe["width"]
|
||||
height = probe["height"]
|
||||
src_fps = probe["fps"]
|
||||
|
||||
# Step 2: Determine method
|
||||
if method == "auto":
|
||||
method = self._auto_detect_method(input_path, duration, width, height)
|
||||
|
||||
# Step 3: Set up temp directory for frame processing
|
||||
temp_dir = output_path.parent / f".gs_tmp_{int(time.time())}"
|
||||
temp_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
try:
|
||||
# Step 4: Extract frames at target fps
|
||||
frames_dir = temp_dir / "frames"
|
||||
frames_dir.mkdir(exist_ok=True)
|
||||
frame_count = self._extract_frames(
|
||||
input_path, frames_dir, fps, max_frames
|
||||
)
|
||||
if frame_count == 0:
|
||||
return ToolResult(
|
||||
success=False, error="No frames extracted from input"
|
||||
)
|
||||
|
||||
# Step 5: Process frames
|
||||
processed_dir = temp_dir / "processed"
|
||||
processed_dir.mkdir(exist_ok=True)
|
||||
|
||||
if method == "chromakey":
|
||||
ok = self._process_chromakey(
|
||||
frames_dir, processed_dir, bg_color, frame_count
|
||||
)
|
||||
else:
|
||||
ok = self._process_rembg(
|
||||
frames_dir, processed_dir, bg_color, frame_count
|
||||
)
|
||||
|
||||
if not ok:
|
||||
return ToolResult(
|
||||
success=False,
|
||||
error=f"Frame processing failed with method={method}",
|
||||
)
|
||||
|
||||
# Step 6: Reconstruct video from processed frames
|
||||
self._reconstruct_video(processed_dir, output_path, fps, width, height)
|
||||
|
||||
if not output_path.exists() or output_path.stat().st_size == 0:
|
||||
return ToolResult(
|
||||
success=False, error="Output video was not created"
|
||||
)
|
||||
|
||||
elapsed = time.time() - start
|
||||
|
||||
return ToolResult(
|
||||
success=True,
|
||||
data={
|
||||
"method_used": method,
|
||||
"frame_count": frame_count,
|
||||
"duration": round(duration, 2),
|
||||
"output_path": str(output_path),
|
||||
"resolution": f"{width}x{height}",
|
||||
"fps": fps,
|
||||
"bg_color": bg_color,
|
||||
},
|
||||
artifacts=[str(output_path)],
|
||||
duration_seconds=round(elapsed, 2),
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
return ToolResult(success=False, error=f"Green screen processing failed: {e}")
|
||||
finally:
|
||||
# Clean up temp directory
|
||||
self._cleanup_dir(temp_dir)
|
||||
|
||||
def _probe_video(self, input_path: Path) -> dict[str, Any] | None:
|
||||
"""Probe video for duration, dimensions, and fps."""
|
||||
cmd = [
|
||||
"ffprobe", "-v", "quiet",
|
||||
"-show_entries", "format=duration:stream=width,height,r_frame_rate",
|
||||
"-select_streams", "v:0",
|
||||
"-of", "json",
|
||||
str(input_path),
|
||||
]
|
||||
try:
|
||||
result = self.run_command(cmd, timeout=30)
|
||||
data = json.loads(result.stdout)
|
||||
|
||||
duration = float(data.get("format", {}).get("duration", 0))
|
||||
|
||||
stream = data.get("streams", [{}])[0]
|
||||
width = int(stream.get("width", 0))
|
||||
height = int(stream.get("height", 0))
|
||||
|
||||
# Parse r_frame_rate like "30/1" or "30000/1001"
|
||||
fps_str = stream.get("r_frame_rate", "30/1")
|
||||
if "/" in fps_str:
|
||||
num, den = fps_str.split("/")
|
||||
fps_val = float(num) / float(den) if float(den) != 0 else 30.0
|
||||
else:
|
||||
fps_val = float(fps_str)
|
||||
|
||||
return {
|
||||
"duration": duration,
|
||||
"width": width,
|
||||
"height": height,
|
||||
"fps": fps_val,
|
||||
}
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
def _auto_detect_method(
|
||||
self, input_path: Path, duration: float, width: int, height: int
|
||||
) -> str:
|
||||
"""Analyze sample frames to decide between chromakey and rembg.
|
||||
|
||||
Extracts 5 evenly-spaced frames, checks color histograms for
|
||||
green/blue screen presence, then tests chromakey quality on a sample.
|
||||
"""
|
||||
temp_dir = input_path.parent / f".gs_detect_{int(time.time())}"
|
||||
temp_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
try:
|
||||
# Extract 5 sample frames evenly spaced
|
||||
interval = max(duration / 6, 0.1)
|
||||
sample_paths = []
|
||||
for i in range(5):
|
||||
ts = interval * (i + 1)
|
||||
out = temp_dir / f"sample_{i}.png"
|
||||
cmd = [
|
||||
"ffmpeg", "-y",
|
||||
"-ss", f"{ts:.3f}",
|
||||
"-i", str(input_path),
|
||||
"-frames:v", "1",
|
||||
str(out),
|
||||
]
|
||||
try:
|
||||
self.run_command(cmd, timeout=30)
|
||||
if out.exists():
|
||||
sample_paths.append(out)
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
if not sample_paths:
|
||||
return "rembg" # fallback if we can't extract samples
|
||||
|
||||
# Analyze color histograms for green/blue screen presence
|
||||
has_green_screen = self._detect_green_screen_histogram(sample_paths)
|
||||
|
||||
if not has_green_screen:
|
||||
# No obvious green/blue screen detected, use rembg
|
||||
return "rembg"
|
||||
|
||||
# Test chromakey on a sample frame and check quality
|
||||
test_frame = sample_paths[len(sample_paths) // 2]
|
||||
chromakey_quality = self._test_chromakey_quality(test_frame, temp_dir)
|
||||
|
||||
if chromakey_quality > 80:
|
||||
return "chromakey"
|
||||
else:
|
||||
return "rembg"
|
||||
|
||||
finally:
|
||||
self._cleanup_dir(temp_dir)
|
||||
|
||||
def _detect_green_screen_histogram(self, sample_paths: list[Path]) -> bool:
|
||||
"""Analyze frames for dominant green or blue channel presence.
|
||||
|
||||
Uses FFmpeg signalstats to measure average hue. A strong green
|
||||
screen typically has a large area of similar green/blue hue.
|
||||
"""
|
||||
green_votes = 0
|
||||
for sample in sample_paths:
|
||||
cmd = [
|
||||
"ffmpeg", "-y",
|
||||
"-i", str(sample),
|
||||
"-vf", "signalstats=stat=tout+vrep+brng,metadata=mode=print",
|
||||
"-frames:v", "1",
|
||||
"-f", "null", self._null_device,
|
||||
]
|
||||
try:
|
||||
result = self.run_command(cmd, timeout=15)
|
||||
# Check stderr for color stats
|
||||
output = result.stderr or ""
|
||||
|
||||
# Alternative: use FFmpeg to count green-ish pixels
|
||||
# Run a simpler hue check with colorchannelmixer
|
||||
cmd2 = [
|
||||
"ffmpeg", "-y",
|
||||
"-i", str(sample),
|
||||
"-vf", (
|
||||
"split[a][b];"
|
||||
"[a]colorchannelmixer=rr=0:gg=1:bb=0,"
|
||||
"threshold=threshold=0.3:similarity=0.3[mask];"
|
||||
"[mask]blackframe=amount=0:threshold=32"
|
||||
),
|
||||
"-frames:v", "1",
|
||||
"-f", "null", self._null_device,
|
||||
]
|
||||
# This is complex; use a simpler approach: check raw pixels
|
||||
# via a green-range filter
|
||||
cmd_green = [
|
||||
"ffmpeg", "-y",
|
||||
"-i", str(sample),
|
||||
"-vf", (
|
||||
"colorkey=color=0x00FF00:similarity=0.4:blend=0.0,"
|
||||
"alphaextract,"
|
||||
"blackframe=amount=0:threshold=128"
|
||||
),
|
||||
"-frames:v", "1",
|
||||
"-f", "null", self._null_device,
|
||||
]
|
||||
try:
|
||||
result2 = self.run_command(cmd_green, timeout=15)
|
||||
stderr = result2.stderr or ""
|
||||
# blackframe reports percentage of black pixels
|
||||
# If many pixels became transparent (black in alpha), there's green
|
||||
if "pblack:" in stderr:
|
||||
import re
|
||||
pblack_matches = re.findall(r"pblack:(\d+)", stderr)
|
||||
if pblack_matches:
|
||||
pblack = int(pblack_matches[0])
|
||||
if pblack >= 20:
|
||||
green_votes += 1
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
# If majority of frames show green screen
|
||||
return green_votes >= len(sample_paths) // 2
|
||||
|
||||
def _test_chromakey_quality(self, test_frame: Path, temp_dir: Path) -> float:
|
||||
"""Run chromakey on a test frame and estimate quality percentage.
|
||||
|
||||
Returns a score 0-100 indicating what percentage of the expected
|
||||
background was successfully keyed out.
|
||||
"""
|
||||
keyed_out = temp_dir / "chromakey_test.png"
|
||||
|
||||
# Apply chromakey and output with alpha
|
||||
cmd = [
|
||||
"ffmpeg", "-y",
|
||||
"-i", str(test_frame),
|
||||
"-vf", "chromakey=color=0x00FF00:similarity=0.3:blend=0.08",
|
||||
str(keyed_out),
|
||||
]
|
||||
try:
|
||||
self.run_command(cmd, timeout=15)
|
||||
except Exception:
|
||||
return 0.0
|
||||
|
||||
if not keyed_out.exists():
|
||||
return 0.0
|
||||
|
||||
# Count transparent pixels via alphaextract + blackframe
|
||||
cmd2 = [
|
||||
"ffmpeg", "-y",
|
||||
"-i", str(keyed_out),
|
||||
"-vf", "alphaextract,blackframe=amount=0:threshold=32",
|
||||
"-frames:v", "1",
|
||||
"-f", "null", self._null_device,
|
||||
]
|
||||
try:
|
||||
result = self.run_command(cmd2, timeout=15)
|
||||
stderr = result.stderr or ""
|
||||
import re
|
||||
pblack_matches = re.findall(r"pblack:(\d+)", stderr)
|
||||
if pblack_matches:
|
||||
# pblack = percentage of black pixels in alpha = transparent pixels
|
||||
return float(pblack_matches[0])
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return 0.0
|
||||
|
||||
def _extract_frames(
|
||||
self, input_path: Path, frames_dir: Path, fps: int, max_frames: int
|
||||
) -> int:
|
||||
"""Extract frames from video at target fps."""
|
||||
cmd = [
|
||||
"ffmpeg", "-y",
|
||||
"-i", str(input_path),
|
||||
"-vf", f"fps={fps}",
|
||||
str(frames_dir / "frame_%06d.png"),
|
||||
]
|
||||
|
||||
if max_frames > 0:
|
||||
cmd.insert(-1, "-frames:v")
|
||||
cmd.insert(-1, str(max_frames))
|
||||
|
||||
try:
|
||||
self.run_command(cmd, timeout=600)
|
||||
except Exception as e:
|
||||
# ffmpeg may return non-zero but still produce frames
|
||||
pass
|
||||
|
||||
# Count extracted frames
|
||||
frame_files = sorted(frames_dir.glob("frame_*.png"))
|
||||
count = len(frame_files)
|
||||
|
||||
if count > 0:
|
||||
# Log progress for large frame counts
|
||||
if count > 100:
|
||||
print(f"[green_screen_processor] Extracted {count} frames")
|
||||
|
||||
return count
|
||||
|
||||
def _process_chromakey(
|
||||
self,
|
||||
frames_dir: Path,
|
||||
processed_dir: Path,
|
||||
bg_color: str,
|
||||
frame_count: int,
|
||||
) -> bool:
|
||||
"""Process frames using FFmpeg chromakey filter.
|
||||
|
||||
Applies chromakey to remove green, then composites onto bg_color.
|
||||
"""
|
||||
bg_hex = bg_color.lstrip("#")
|
||||
# Convert hex to FFmpeg color format
|
||||
ffmpeg_bg = f"0x{bg_hex}"
|
||||
|
||||
frame_files = sorted(frames_dir.glob("frame_*.png"))
|
||||
processed = 0
|
||||
|
||||
for i, frame in enumerate(frame_files):
|
||||
out_path = processed_dir / frame.name
|
||||
cmd = [
|
||||
"ffmpeg", "-y",
|
||||
"-f", "lavfi", "-i", f"color=c={ffmpeg_bg}:size=1x1",
|
||||
"-i", str(frame),
|
||||
"-filter_complex",
|
||||
(
|
||||
f"[0:v]scale=iw:ih[bg];"
|
||||
f"[1:v]chromakey=color=0x00FF00:similarity=0.3:blend=0.08[fg];"
|
||||
f"[bg][fg]overlay=0:0"
|
||||
),
|
||||
"-frames:v", "1",
|
||||
str(out_path),
|
||||
]
|
||||
try:
|
||||
self.run_command(cmd, timeout=30)
|
||||
if out_path.exists():
|
||||
processed += 1
|
||||
except Exception:
|
||||
# Try with the frame size explicitly to fix scale
|
||||
try:
|
||||
cmd_retry = [
|
||||
"ffmpeg", "-y",
|
||||
"-i", str(frame),
|
||||
"-vf",
|
||||
f"chromakey=color=0x00FF00:similarity=0.3:blend=0.08,"
|
||||
f"split[fg][alpha];"
|
||||
f"[alpha]alphaextract[a];"
|
||||
f"color=c={ffmpeg_bg}[bg];"
|
||||
f"[bg][fg][a]maskedmerge",
|
||||
"-frames:v", "1",
|
||||
str(out_path),
|
||||
]
|
||||
# Simpler fallback: just apply chromakey without compositing
|
||||
cmd_simple = [
|
||||
"ffmpeg", "-y",
|
||||
"-i", str(frame),
|
||||
"-vf", f"chromakey=color=0x00FF00:similarity=0.3:blend=0.08",
|
||||
str(out_path),
|
||||
]
|
||||
self.run_command(cmd_simple, timeout=30)
|
||||
if out_path.exists():
|
||||
processed += 1
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
if frame_count > 100 and (i + 1) % 50 == 0:
|
||||
print(
|
||||
f"[green_screen_processor] Chromakey: {i + 1}/{frame_count} frames"
|
||||
)
|
||||
|
||||
return processed > 0
|
||||
|
||||
def _process_rembg(
|
||||
self,
|
||||
frames_dir: Path,
|
||||
processed_dir: Path,
|
||||
bg_color: str,
|
||||
frame_count: int,
|
||||
) -> bool:
|
||||
"""Process frames using rembg AI segmentation.
|
||||
|
||||
Removes background with u2net_human_seg model, then composites
|
||||
the subject onto bg_color background.
|
||||
"""
|
||||
try:
|
||||
import rembg
|
||||
from PIL import Image
|
||||
except ImportError:
|
||||
return False
|
||||
|
||||
# Parse bg_color hex to RGB
|
||||
bg_hex = bg_color.lstrip("#")
|
||||
bg_r = int(bg_hex[0:2], 16)
|
||||
bg_g = int(bg_hex[2:4], 16)
|
||||
bg_b = int(bg_hex[4:6], 16)
|
||||
|
||||
session = rembg.new_session("u2net_human_seg")
|
||||
|
||||
frame_files = sorted(frames_dir.glob("frame_*.png"))
|
||||
processed = 0
|
||||
|
||||
for i, frame in enumerate(frame_files):
|
||||
try:
|
||||
img = Image.open(frame).convert("RGB")
|
||||
import numpy as np
|
||||
|
||||
# Remove background (returns RGBA)
|
||||
result = rembg.remove(
|
||||
np.array(img),
|
||||
session=session,
|
||||
)
|
||||
result_img = Image.fromarray(result)
|
||||
|
||||
# Composite onto bg_color background
|
||||
bg = Image.new("RGBA", result_img.size, (bg_r, bg_g, bg_b, 255))
|
||||
bg.paste(result_img, (0, 0), result_img)
|
||||
|
||||
# Save as RGB
|
||||
out_path = processed_dir / frame.name
|
||||
bg.convert("RGB").save(out_path)
|
||||
processed += 1
|
||||
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
if frame_count > 100 and (i + 1) % 50 == 0:
|
||||
print(
|
||||
f"[green_screen_processor] rembg: {i + 1}/{frame_count} frames"
|
||||
)
|
||||
|
||||
return processed > 0
|
||||
|
||||
def _reconstruct_video(
|
||||
self,
|
||||
frames_dir: Path,
|
||||
output_path: Path,
|
||||
fps: int,
|
||||
width: int,
|
||||
height: int,
|
||||
) -> None:
|
||||
"""Reconstruct video from processed frames using FFmpeg."""
|
||||
cmd = [
|
||||
"ffmpeg", "-y",
|
||||
"-framerate", str(fps),
|
||||
"-i", str(frames_dir / "frame_%06d.png"),
|
||||
"-vf", f"scale={width}:{height}:flags=lanczos",
|
||||
"-c:v", "libx264",
|
||||
"-crf", "18",
|
||||
"-preset", "fast",
|
||||
"-pix_fmt", "yuv420p",
|
||||
str(output_path),
|
||||
]
|
||||
self.run_command(cmd, timeout=600)
|
||||
|
||||
@staticmethod
|
||||
def _cleanup_dir(dir_path: Path) -> None:
|
||||
"""Recursively remove a temp directory."""
|
||||
if not dir_path.exists():
|
||||
return
|
||||
try:
|
||||
shutil.rmtree(dir_path)
|
||||
except OSError:
|
||||
# Best-effort cleanup; individual file removal as fallback
|
||||
for f in dir_path.rglob("*"):
|
||||
try:
|
||||
if f.is_file():
|
||||
f.unlink()
|
||||
except OSError:
|
||||
pass
|
||||
try:
|
||||
dir_path.rmdir()
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
def estimate_runtime(self, inputs: dict[str, Any]) -> float:
|
||||
method = inputs.get("method", "auto")
|
||||
if method == "rembg":
|
||||
return 120.0
|
||||
elif method == "chromakey":
|
||||
return 30.0
|
||||
return 60.0 # auto
|
||||
@@ -106,6 +106,19 @@ class RemotionCaptionBurn(BaseTool):
|
||||
"correct replacement. Example: {\"cloud\": \"Claude\"}."
|
||||
),
|
||||
},
|
||||
"overlays": {
|
||||
"type": "array",
|
||||
"description": (
|
||||
"Array of overlay objects to render on top of the video. "
|
||||
"Each overlay has: type (text_card, stat_card, callout, "
|
||||
"comparison, bar_chart, line_chart, pie_chart, kpi_grid, "
|
||||
"hero_title, section_title, stat_reveal), in_seconds, "
|
||||
"out_seconds, position (lower_third, upper_third, "
|
||||
"left_panel, right_panel, full_overlay), and component-"
|
||||
"specific props (text, stat, chartData, etc.). "
|
||||
"See asset_manifest overlays from the asset-director."
|
||||
),
|
||||
},
|
||||
"force_ffmpeg": {
|
||||
"type": "boolean",
|
||||
"default": False,
|
||||
@@ -245,6 +258,7 @@ class RemotionCaptionBurn(BaseTool):
|
||||
words_per_page: int,
|
||||
font_size: int,
|
||||
highlight_color: str,
|
||||
overlays: list[dict] | None = None,
|
||||
) -> ToolResult:
|
||||
root = self._find_remotion_root()
|
||||
if root is None:
|
||||
@@ -262,6 +276,19 @@ class RemotionCaptionBurn(BaseTool):
|
||||
duration_s = float(dur_out.strip().split("\n")[0])
|
||||
total_frames = math.ceil(duration_s * 30)
|
||||
|
||||
# Detect video dimensions
|
||||
dim_cmd = [
|
||||
"ffprobe", "-v", "error",
|
||||
"-select_streams", "v:0",
|
||||
"-show_entries", "stream=width,height",
|
||||
"-of", "csv=p=0:s=x",
|
||||
input_path,
|
||||
]
|
||||
dim_result = self.run_command(dim_cmd)
|
||||
dim_parts = dim_result.stdout.strip().split("x")
|
||||
width = int(dim_parts[0])
|
||||
height = int(dim_parts[1])
|
||||
|
||||
# Copy video to Remotion public folder
|
||||
pub_dir = root / "public" / "talking-head"
|
||||
pub_dir.mkdir(parents=True, exist_ok=True)
|
||||
@@ -273,6 +300,7 @@ class RemotionCaptionBurn(BaseTool):
|
||||
props = {
|
||||
"videoSrc": f"public/talking-head/{video_filename}",
|
||||
"captions": captions,
|
||||
"overlays": overlays or [],
|
||||
"wordsPerPage": words_per_page,
|
||||
"fontSize": font_size,
|
||||
"highlightColor": highlight_color,
|
||||
@@ -287,12 +315,12 @@ class RemotionCaptionBurn(BaseTool):
|
||||
npx_bin = "npx.cmd" if sys.platform == "win32" else "npx"
|
||||
render_cmd = [
|
||||
npx_bin, "remotion", "render",
|
||||
"src/index.tsx", "TalkingHead",
|
||||
"TalkingHead",
|
||||
f"--props={props_file.relative_to(root)}",
|
||||
"--width=1080", "--height=1920", "--fps=30",
|
||||
f"--width={width}", f"--height={height}", "--fps=30",
|
||||
f"--frames=0-{total_frames - 1}",
|
||||
"--codec=h264", "--crf=18",
|
||||
str(Path(output_path).resolve()),
|
||||
f"--output={str(Path(output_path).resolve())}",
|
||||
]
|
||||
self.run_command(render_cmd, cwd=str(root))
|
||||
|
||||
@@ -307,6 +335,7 @@ class RemotionCaptionBurn(BaseTool):
|
||||
"duration_seconds": round(duration_s, 2),
|
||||
"total_frames": total_frames,
|
||||
"caption_count": len(captions),
|
||||
"overlay_count": len(overlays or []),
|
||||
"words_per_page": words_per_page,
|
||||
},
|
||||
artifacts=[output_path],
|
||||
@@ -429,11 +458,14 @@ class RemotionCaptionBurn(BaseTool):
|
||||
if not captions:
|
||||
return ToolResult(success=False, error="No caption words extracted.")
|
||||
|
||||
overlays = inputs.get("overlays")
|
||||
|
||||
# Choose render method
|
||||
if not force_ffmpeg and self._remotion_available():
|
||||
result = self._render_remotion(
|
||||
input_path, output_path, captions,
|
||||
words_per_page, font_size, highlight_color,
|
||||
overlays=overlays,
|
||||
)
|
||||
else:
|
||||
result = self._render_ffmpeg(input_path, output_path, captions)
|
||||
|
||||
Reference in New Issue
Block a user