Green screen pipeline: new tools, AnimatedBackground, caption burn fixes

Add green_screen_processor (auto-detect + rembg fallback) and
green_screen_composite (4 layout presets with alpha compositing) tools
to automate the full keying-to-composite pipeline.

Remotion: add AnimatedBackground with gradient mesh and floating orbs
to Explainer, fix caption burn tool (remove entry point arg, auto-detect
dimensions, extend TalkingHead duration to 300s).

Update scene-director, compose-director, and asset-director skill docs
with green screen workflow steps and component constraints.
This commit is contained in:
calesthio
2026-04-02 11:54:30 -07:00
parent 942244ca54
commit a7e5f7498b
11 changed files with 2087 additions and 31 deletions
@@ -0,0 +1,260 @@
{
"cuts": [
{
"id": "scene-1-hook",
"source": "",
"in_seconds": 0,
"out_seconds": 5.5,
"type": "hero_title",
"text": "STOP PAYING",
"subtitle": "Multiple EMIs Every Month",
"backgroundColor": "#0F172A",
"accentColor": "#EF4444"
},
{
"id": "scene-2-control",
"source": "",
"in_seconds": 5.5,
"out_seconds": 9.5,
"type": "callout",
"text": "Your loans are managing YOU",
"callout_type": "warning",
"backgroundColor": "#0F172A",
"accentColor": "#F59E0B"
},
{
"id": "scene-3-rates",
"source": "",
"in_seconds": 9.5,
"out_seconds": 15.5,
"type": "bar_chart",
"chartData": [
{"label": "Credit Cards", "value": 24},
{"label": "Personal", "value": 16},
{"label": "App Loans", "value": 36},
{"label": "NBFC", "value": 18}
],
"title": "Interest Rates You're Paying (%)",
"backgroundColor": "#0F172A",
"accentColor": "#F59E0B",
"chartColors": ["#EF4444", "#F59E0B", "#8B5CF6", "#EC4899"]
},
{
"id": "scene-4-fear",
"source": "",
"in_seconds": 15.5,
"out_seconds": 20,
"type": "stat_reveal",
"text": "Every Month",
"subtitle": "Will I have anything left?",
"backgroundColor": "#0F172A",
"accentColor": "#EF4444"
},
{
"id": "scene-5-reveal",
"source": "",
"in_seconds": 20,
"out_seconds": 22,
"type": "hero_title",
"text": "WHAT NO ONE",
"subtitle": "Tells You",
"backgroundColor": "#0F172A",
"accentColor": "#22D3EE"
},
{
"id": "scene-6-inquiries",
"source": "",
"in_seconds": 22,
"out_seconds": 40.5,
"type": "progress_bar",
"progress": 95,
"label": "Loan Inquiries Piling Up",
"progressLabel": "DANGER ZONE",
"progressColor": "#EF4444",
"animationStyle": "step",
"progressSegments": [
{"value": 25, "color": "#F59E0B", "label": "Bank"},
{"value": 25, "color": "#F97316", "label": "NBFC"},
{"value": 25, "color": "#EF4444", "label": "Digital"},
{"value": 20, "color": "#DC2626", "label": "Random"}
],
"backgroundColor": "#0F172A"
},
{
"id": "scene-7-score-drop",
"source": "",
"in_seconds": 41,
"out_seconds": 46.5,
"type": "line_chart",
"chartSeries": [
{
"name": "Credit Score",
"data": [
{"x": 0, "y": 750},
{"x": 1, "y": 720},
{"x": 2, "y": 680},
{"x": 3, "y": 620}
],
"color": "#EF4444"
}
],
"title": "Your Credit Score Drops",
"xLabel": "Each Inquiry",
"yLabel": "Score",
"showGrid": true,
"showMarkers": true,
"chartAnimation": "draw",
"backgroundColor": "#0F172A",
"accentColor": "#EF4444"
},
{
"id": "scene-8-risk",
"source": "",
"in_seconds": 47,
"out_seconds": 49.5,
"type": "callout",
"text": "Lenders See You As HIGH RISK",
"callout_type": "warning",
"backgroundColor": "#0F172A",
"accentColor": "#EF4444"
},
{
"id": "scene-9-wrong",
"source": "",
"in_seconds": 50,
"out_seconds": 53.5,
"type": "stat_card",
"stat": "❌",
"subtitle": "Wrong approach wastes your time",
"backgroundColor": "#0F172A",
"accentColor": "#EF4444"
},
{
"id": "scene-9-right",
"source": "",
"in_seconds": 53.5,
"out_seconds": 57,
"type": "stat_card",
"stat": "✅",
"subtitle": "Right approach gets approval",
"backgroundColor": "#0F172A",
"accentColor": "#10B981"
},
{
"id": "scene-10-matters",
"source": "",
"in_seconds": 60,
"out_seconds": 66,
"type": "callout",
"text": "WHO you approach matters as much as WHAT you apply for",
"callout_type": "tip",
"backgroundColor": "#0F172A",
"accentColor": "#10B981"
},
{
"id": "scene-11-creditwin",
"source": "",
"in_seconds": 66,
"out_seconds": 80,
"type": "hero_title",
"text": "CREDIT WIN",
"subtitle": "Your Consolidation Partner",
"backgroundColor": "#0F172A",
"accentColor": "#22D3EE"
},
{
"id": "scene-12-oneplan",
"source": "",
"in_seconds": 81,
"out_seconds": 84.5,
"type": "kpi_grid",
"chartData": [
{"label": "EMI", "value": "1"},
{"label": "Due Date", "value": "1"},
{"label": "Clear Plan", "value": "1"}
],
"title": "From Chaos to Control",
"backgroundColor": "#0F172A",
"accentColor": "#22D3EE",
"chartColors": ["#22D3EE", "#10B981", "#A78BFA"]
},
{
"id": "scene-13a-years",
"source": "",
"in_seconds": 84.5,
"out_seconds": 89,
"type": "stat_card",
"stat": "15+",
"subtitle": "Years of Experience",
"backgroundColor": "#0F172A",
"accentColor": "#F59E0B"
},
{
"id": "scene-13b-borrowers",
"source": "",
"in_seconds": 89,
"out_seconds": 94,
"type": "stat_card",
"stat": "10,000+",
"subtitle": "Borrowers Served",
"backgroundColor": "#0F172A",
"accentColor": "#22D3EE"
},
{
"id": "scene-13c-saved",
"source": "",
"in_seconds": 94,
"out_seconds": 99,
"type": "stat_card",
"stat": "₹5K-15K",
"subtitle": "Saved Per Month on EMIs",
"backgroundColor": "#0F172A",
"accentColor": "#10B981"
},
{
"id": "scene-14-qualify",
"source": "",
"in_seconds": 100,
"out_seconds": 118,
"type": "stat_card",
"stat": "₹60,000+",
"subtitle": "Salary? You Qualify for Consolidation",
"backgroundColor": "#0F172A",
"accentColor": "#22D3EE"
},
{
"id": "scene-15-before",
"source": "",
"in_seconds": 118,
"out_seconds": 121,
"type": "stat_card",
"stat": "5 EMIs",
"subtitle": "Chaos → Exit This",
"backgroundColor": "#0F172A",
"accentColor": "#EF4444"
},
{
"id": "scene-15-after",
"source": "",
"in_seconds": 121,
"out_seconds": 124,
"type": "stat_card",
"stat": "1 EMI",
"subtitle": "Control → Your New Reality",
"backgroundColor": "#0F172A",
"accentColor": "#10B981"
},
{
"id": "scene-16-cta",
"source": "",
"in_seconds": 124,
"out_seconds": 127,
"type": "hero_title",
"text": "APPLY NOW",
"subtitle": "CreditWin.in",
"backgroundColor": "#0F172A",
"accentColor": "#10B981"
}
],
"overlays": []
}
@@ -0,0 +1,55 @@
{
"cuts": [
{
"id": "scene-1-hook",
"source": "",
"in_seconds": 0,
"out_seconds": 5.5,
"type": "hero_title",
"text": "STOP PAYING",
"subtitle": "Multiple EMIs Every Month",
"backgroundColor": "#0F172A",
"accentColor": "#EF4444"
},
{
"id": "scene-2-control",
"source": "",
"in_seconds": 5.5,
"out_seconds": 9.5,
"type": "callout",
"text": "Your loans are managing YOU",
"callout_type": "warning",
"backgroundColor": "#0F172A",
"accentColor": "#F59E0B"
},
{
"id": "scene-3-rates",
"source": "",
"in_seconds": 9.5,
"out_seconds": 15.5,
"type": "bar_chart",
"chartData": [
{"label": "Credit Cards", "value": 24},
{"label": "Personal", "value": 16},
{"label": "App Loans", "value": 36},
{"label": "NBFC", "value": 18}
],
"title": "Interest Rates You're Paying (%)",
"backgroundColor": "#0F172A",
"accentColor": "#F59E0B",
"chartColors": ["#EF4444", "#F59E0B", "#8B5CF6", "#EC4899"]
},
{
"id": "scene-4-fear",
"source": "",
"in_seconds": 15.5,
"out_seconds": 20,
"type": "stat_reveal",
"text": "Every Month",
"subtitle": "Will I have anything left?",
"backgroundColor": "#0F172A",
"accentColor": "#EF4444"
}
],
"overlays": []
}
+98 -2
View File
@@ -44,6 +44,97 @@ const { fontFamily } = loadFont("normal", {
subsets: ["latin"],
});
// ---------------------------------------------------------------------------
// Animated Background — Gradient Mesh + Floating Orbs
// ---------------------------------------------------------------------------
const AnimatedBackground: React.FC<{ style?: "fintech" | "default" }> = ({
style: bgStyle = "default",
}) => {
const frame = useCurrentFrame();
const { fps, durationInFrames, width, height } = useVideoConfig();
const progress = frame / durationInFrames;
// Slow-moving gradient angles
const angle1 = 135 + Math.sin(frame / (fps * 8)) * 30;
const angle2 = 225 + Math.cos(frame / (fps * 6)) * 25;
// Color stops shift over time
const shift = Math.sin(frame / (fps * 12)) * 0.15;
const gradient = `
radial-gradient(ellipse at ${30 + Math.sin(frame / (fps * 10)) * 20}% ${40 + Math.cos(frame / (fps * 8)) * 20}%,
rgba(15, 23, 60, 1) 0%, transparent 60%),
radial-gradient(ellipse at ${70 + Math.cos(frame / (fps * 7)) * 20}% ${60 + Math.sin(frame / (fps * 9)) * 25}%,
rgba(30, 10, 60, 0.8) 0%, transparent 55%),
radial-gradient(ellipse at ${50 + Math.sin(frame / (fps * 14)) * 30}% ${20 + Math.cos(frame / (fps * 11)) * 15}%,
rgba(0, 40, 60, 0.6) 0%, transparent 50%),
linear-gradient(${angle1}deg, #060918 0%, #0B1026 40%, #0F0A2E 70%, #080D1F 100%)
`;
// Floating orbs
const orbs = [
{ x: 20, y: 30, size: 300, color: "rgba(34, 211, 238, 0.08)", speedX: 7, speedY: 11 },
{ x: 70, y: 60, size: 250, color: "rgba(139, 92, 246, 0.1)", speedX: 9, speedY: 8 },
{ x: 40, y: 80, size: 200, color: "rgba(16, 185, 129, 0.07)", speedX: 13, speedY: 6 },
{ x: 80, y: 20, size: 350, color: "rgba(245, 158, 11, 0.06)", speedX: 11, speedY: 14 },
{ x: 10, y: 70, size: 180, color: "rgba(236, 72, 153, 0.05)", speedX: 8, speedY: 10 },
];
return (
<AbsoluteFill style={{ background: gradient }}>
{/* Floating glow orbs */}
{orbs.map((orb, i) => {
const ox = orb.x + Math.sin(frame / (fps * orb.speedX)) * 15;
const oy = orb.y + Math.cos(frame / (fps * orb.speedY)) * 12;
return (
<div
key={i}
style={{
position: "absolute",
left: `${ox}%`,
top: `${oy}%`,
width: orb.size,
height: orb.size,
borderRadius: "50%",
background: orb.color,
filter: `blur(${orb.size * 0.4}px)`,
transform: "translate(-50%, -50%)",
willChange: "transform",
}}
/>
);
})}
{/* Subtle grid overlay */}
<div
style={{
position: "absolute",
inset: 0,
backgroundImage: `
linear-gradient(rgba(255,255,255,0.02) 1px, transparent 1px),
linear-gradient(90deg, rgba(255,255,255,0.02) 1px, transparent 1px)
`,
backgroundSize: "60px 60px",
opacity: 0.5 + Math.sin(frame / (fps * 20)) * 0.2,
}}
/>
{/* Top gradient fade for depth */}
<div
style={{
position: "absolute",
top: 0,
left: 0,
right: 0,
height: "30%",
background: "linear-gradient(to bottom, rgba(6,9,24,0.4), transparent)",
}}
/>
</AbsoluteFill>
);
};
// ---------------------------------------------------------------------------
// Types — aligned with edit_decisions artifact schema
// ---------------------------------------------------------------------------
@@ -353,7 +444,9 @@ const SceneRenderer: React.FC<{ cut: Cut }> = ({ cut }) => {
};
// Resolve the scene element based on cut type, then wrap with backgroundImage if set
const bgColor = cut.backgroundImage ? "transparent" : cut.backgroundColor;
// Use transparent bg so the animated gradient background shows through
const rawBg = cut.backgroundImage ? "transparent" : cut.backgroundColor;
const bgColor = (rawBg === "#0F172A" || rawBg === "#0f172a") ? "transparent" : rawBg;
// Explicit component types
if (cut.type === "text_card" && cut.text) {
@@ -539,7 +632,10 @@ export const Explainer: React.FC<ExplainerProps> = ({
const { fps, durationInFrames } = useVideoConfig();
return (
<AbsoluteFill style={{ background: "#0F172A", fontFamily }}>
<AbsoluteFill style={{ background: "#060918", fontFamily }}>
{/* Layer 0: Animated gradient background */}
<AnimatedBackground style="fintech" />
{/* Layer 1: Visual scenes */}
{cuts.map((cut) => {
const from = Math.round(cut.in_seconds * fps);
+2 -1
View File
@@ -65,13 +65,14 @@ export const Root: React.FC = () => {
<Composition
id="TalkingHead"
component={TalkingHead}
durationInFrames={30 * 30}
durationInFrames={30 * 300}
fps={30}
width={1080}
height={1920}
defaultProps={{
videoSrc: "",
captions: [],
overlays: [],
wordsPerPage: 4,
fontSize: 52,
highlightColor: "#22D3EE",
+319 -1
View File
@@ -1,9 +1,304 @@
import { AbsoluteFill, OffthreadVideo } from "remotion";
import {
AbsoluteFill,
OffthreadVideo,
Sequence,
interpolate,
useCurrentFrame,
useVideoConfig,
} from "remotion";
import { CaptionOverlay, WordCaption } from "./components/CaptionOverlay";
import { TextCard } from "./components/TextCard";
import { StatCard } from "./components/StatCard";
import { CalloutBox } from "./components/CalloutBox";
import { ComparisonCard } from "./components/ComparisonCard";
import { BarChart } from "./components/charts/BarChart";
import { LineChart } from "./components/charts/LineChart";
import { PieChart } from "./components/charts/PieChart";
import { KPIGrid } from "./components/charts/KPIGrid";
import { HeroTitle } from "./components/HeroTitle";
import { SectionTitle } from "./components/SectionTitle";
import { StatReveal } from "./components/StatReveal";
// ---------------------------------------------------------------------------
// Overlay types for talking-head video
// ---------------------------------------------------------------------------
export interface TalkingHeadOverlay {
id?: string;
type: string;
in_seconds: number;
out_seconds: number;
position?:
| "lower_third"
| "upper_third"
| "left_panel"
| "right_panel"
| "full_overlay";
// Component-specific props (same as Explainer Cut)
text?: string;
stat?: string;
subtitle?: string;
callout_type?: "info" | "warning" | "tip" | "quote";
title?: string;
leftLabel?: string;
rightLabel?: string;
leftValue?: string;
rightValue?: string;
chartData?: any[];
chartSeries?: any[];
chartColors?: string[];
chartAnimation?: string;
donut?: boolean;
centerLabel?: string;
centerValue?: string;
showGrid?: boolean;
showValues?: boolean;
showLegend?: boolean;
showMarkers?: boolean;
columns?: 2 | 3 | 4;
// Styling
backgroundColor?: string;
color?: string;
accentColor?: string;
fontSize?: number;
}
// ---------------------------------------------------------------------------
// Position presets for 9:16 (1080x1920) frame
// ---------------------------------------------------------------------------
const POSITION_STYLES: Record<string, React.CSSProperties> = {
lower_third: {
position: "absolute",
bottom: 320, // Above caption area (~1600px)
left: 40,
right: 40,
height: 480,
},
upper_third: {
position: "absolute",
top: 80,
left: 40,
right: 40,
height: 480,
},
left_panel: {
position: "absolute",
top: 200,
left: 40,
width: 480,
bottom: 400,
},
right_panel: {
position: "absolute",
top: 200,
right: 40,
width: 480,
bottom: 400,
},
full_overlay: {
position: "absolute",
top: 0,
left: 0,
right: 0,
bottom: 0,
},
};
// ---------------------------------------------------------------------------
// Overlay component dispatcher — maps overlay type to Remotion component
// ---------------------------------------------------------------------------
const OverlayContent: React.FC<{ overlay: TalkingHeadOverlay }> = ({
overlay,
}) => {
const bgColor = overlay.backgroundColor || "#0F172A";
if (overlay.type === "text_card" && overlay.text) {
return (
<TextCard
text={overlay.text}
fontSize={overlay.fontSize}
color={overlay.color}
backgroundColor={bgColor}
/>
);
}
if (overlay.type === "stat_card" && overlay.stat) {
return (
<StatCard
stat={overlay.stat}
subtitle={overlay.subtitle}
accentColor={overlay.accentColor}
backgroundColor={bgColor}
/>
);
}
if (overlay.type === "callout" && overlay.text) {
return (
<CalloutBox
text={overlay.text}
type={overlay.callout_type}
title={overlay.title}
borderColor={overlay.accentColor}
backgroundColor={overlay.backgroundColor}
textColor={overlay.color}
containerBackgroundColor={bgColor}
/>
);
}
if (
overlay.type === "comparison" &&
overlay.leftLabel &&
overlay.rightLabel
) {
return (
<ComparisonCard
leftLabel={overlay.leftLabel}
rightLabel={overlay.rightLabel}
leftValue={overlay.leftValue || ""}
rightValue={overlay.rightValue || ""}
title={overlay.title}
backgroundColor={bgColor}
textColor={overlay.color}
/>
);
}
if (overlay.type === "bar_chart" && overlay.chartData) {
return (
<BarChart
data={overlay.chartData}
title={overlay.title}
colors={overlay.chartColors}
animationStyle={(overlay.chartAnimation as any) || "grow-up"}
showValues={overlay.showValues}
backgroundColor={bgColor}
/>
);
}
if (overlay.type === "line_chart" && overlay.chartSeries) {
return (
<LineChart
series={overlay.chartSeries}
title={overlay.title}
colors={overlay.chartColors}
animationStyle={(overlay.chartAnimation as any) || "draw"}
showGrid={overlay.showGrid}
showMarkers={overlay.showMarkers}
showLegend={overlay.showLegend}
backgroundColor={bgColor}
/>
);
}
if (overlay.type === "pie_chart" && overlay.chartData) {
return (
<PieChart
data={overlay.chartData}
title={overlay.title}
colors={overlay.chartColors}
animationStyle={(overlay.chartAnimation as any) || "expand"}
donut={overlay.donut}
centerLabel={overlay.centerLabel}
centerValue={overlay.centerValue}
showLegend={overlay.showLegend}
backgroundColor={bgColor}
/>
);
}
if (overlay.type === "kpi_grid" && overlay.chartData) {
return (
<KPIGrid
metrics={overlay.chartData}
title={overlay.title}
columns={overlay.columns}
colors={overlay.chartColors}
animationStyle={(overlay.chartAnimation as any) || "count-up"}
backgroundColor={bgColor}
/>
);
}
if (overlay.type === "hero_title" && overlay.text) {
return <HeroTitle title={overlay.text} subtitle={overlay.subtitle} />;
}
if (overlay.type === "section_title" && overlay.text) {
return (
<SectionTitle
title={overlay.text}
subtitle={overlay.subtitle}
accentColor={overlay.accentColor}
position="top-left"
/>
);
}
if (overlay.type === "stat_reveal" && overlay.text) {
return (
<StatReveal
stat={overlay.text}
label={overlay.subtitle}
accentColor={overlay.accentColor}
position="bottom-right"
/>
);
}
return null;
};
// ---------------------------------------------------------------------------
// Positioned overlay wrapper — handles position + fade in/out
// ---------------------------------------------------------------------------
const PositionedOverlay: React.FC<{ overlay: TalkingHeadOverlay }> = ({
overlay,
}) => {
const frame = useCurrentFrame();
const { durationInFrames } = useVideoConfig();
// Fade in over 8 frames (~0.27s), fade out over 8 frames
const fadeIn = interpolate(frame, [0, 8], [0, 1], {
extrapolateLeft: "clamp",
extrapolateRight: "clamp",
});
const fadeOut = interpolate(
frame,
[durationInFrames - 8, durationInFrames],
[1, 0],
{ extrapolateLeft: "clamp", extrapolateRight: "clamp" }
);
const opacity = fadeIn * fadeOut;
const position = overlay.position || "lower_third";
const posStyle = POSITION_STYLES[position] || POSITION_STYLES.lower_third;
const isFullOverlay = position === "full_overlay";
return (
<div
style={{
...posStyle,
opacity,
overflow: "hidden",
borderRadius: isFullOverlay ? 0 : 16,
boxShadow: isFullOverlay
? "none"
: "0 8px 32px rgba(0, 0, 0, 0.4)",
}}
>
{isFullOverlay && (
<AbsoluteFill style={{ background: "rgba(0, 0, 0, 0.7)" }} />
)}
<OverlayContent overlay={overlay} />
</div>
);
};
// ---------------------------------------------------------------------------
// Main TalkingHead composition
// ---------------------------------------------------------------------------
export interface TalkingHeadProps {
[key: string]: unknown;
videoSrc: string;
captions: WordCaption[];
overlays?: TalkingHeadOverlay[];
wordsPerPage?: number;
fontSize?: number;
highlightColor?: string;
@@ -12,16 +307,39 @@ export interface TalkingHeadProps {
export const TalkingHead: React.FC<TalkingHeadProps> = ({
videoSrc,
captions,
overlays,
wordsPerPage = 4,
fontSize = 52,
highlightColor = "#22D3EE",
}) => {
const { fps } = useVideoConfig();
return (
<AbsoluteFill style={{ backgroundColor: "#000" }}>
{/* Layer 1: Video background */}
<OffthreadVideo
src={videoSrc}
style={{ width: "100%", height: "100%", objectFit: "cover" }}
/>
{/* Layer 2: Overlays (charts, stats, callouts, etc.) */}
{overlays?.map((overlay, i) => {
const from = Math.round(overlay.in_seconds * fps);
const duration = Math.round(
(overlay.out_seconds - overlay.in_seconds) * fps
);
return (
<Sequence
key={overlay.id || `overlay-${i}`}
from={from}
durationInFrames={duration}
>
<PositionedOverlay overlay={overlay} />
</Sequence>
);
})}
{/* Layer 3: Captions (topmost — always visible above overlays) */}
<CaptionOverlay
words={captions}
wordsPerPage={wordsPerPage}
@@ -90,7 +90,34 @@ Create a composition JSON snippet for each overlay. These will be rendered by th
| Callout/quote | `callout` | `text`, `icon` |
| Lower third | `text_card` | `text` |
**Dark theme for all overlays** — use dark backgrounds (`#0F172A`, `#1E293B`) with light text. This ensures overlays are legible when composited on top of talking-head footage.
**Remotion AnimatedBackground:**
The Explainer composition now includes an `AnimatedBackground` component that renders an animated gradient mesh, floating orbs, and a subtle grid pattern. This provides a far more professional look than flat solid colors.
- Scene backgrounds should use `backgroundColor: "#0F172A"` which auto-converts to transparent, letting the animated gradient show through.
- Do NOT use flat solid colors for backgrounds -- always let the AnimatedBackground render behind overlays.
- When compositing green screen footage, render the AnimatedBackground as the replacement background (see compose-director Step 3c).
**Component constraints:**
| Component | Min Width | 720px Portrait? | Value Type |
|-----------|-----------|-----------------|------------|
| comparison | 900px | NO -> use 2x stat_card | string |
| kpi_grid | 720px | YES | numeric ONLY (no "15+") |
| bar_chart | 500px | YES | numeric |
| stat_card | 300px | YES | string OK |
| callout | 400px | YES | string |
| hero_title | 400px | YES | string |
| line_chart | 500px | YES | numeric |
| progress_bar | 600px | YES | numeric |
| stat_reveal | 300px | YES | string OK |
Key rules:
- `comparison` requires 900px+ width. In 720px portrait frames, substitute with two sequential `stat_card` components instead.
- `kpi_grid` values MUST be purely numeric (e.g., `4.8`, `73`, `2400`). Formatted strings like `"15+"`, `"$4.8B"`, or `"2.4M"` will cause rendering errors. Use `stat_card` for string-formatted numbers instead.
- Always check the target frame width before choosing a component. Portrait (720px) excludes `comparison`.
**Dark theme for all overlays** -- use dark backgrounds (`#0F172A`, `#1E293B`) with light text. This ensures overlays are legible when composited on top of talking-head footage. The `#0F172A` backgroundColor is treated as transparent by the AnimatedBackground system, so overlays will show the animated gradient behind them.
**For simple text overlays** (if Remotion is overkill):
+190 -23
View File
@@ -15,6 +15,45 @@ You have edit decisions and an asset manifest. Your job is to render the final t
## Process
### Step 0: Pre-flight Checks
Before rendering anything, validate the inputs and catch issues that are expensive to fix later.
1. **Silence detection** -- Run `silence_cutter` in mark mode:
```
silence_cutter.execute({
"input_path": "<raw_footage>",
"mode": "mark",
"silence_threshold_db": -35,
"min_silence_duration": 0.5
})
```
- Report all gaps > 0.5s with timestamps.
- If total silence > 5s, **recommend cutting before proceeding**. Long silences waste render time and produce dead spots in the final video.
2. **ASR confidence check** -- Scan word-level transcript for low-confidence words:
- Flag any word with probability < 0.7.
- List flagged words with timestamps so the user can verify correct transcription.
- Common misrecognitions to watch for: proper nouns, brand names, domain jargon.
3. **Auto-build corrections dictionary** from common ASR error patterns:
```python
corrections = {
# Indian finance context
"DMI": "EMI",
"AMI": "EMI",
# Common brand misspellings
"open montage": "OpenMontage",
"remotion": "Remotion",
# Numbers that got split by ASR
"4 -5": "4-5",
"10 -15": "10-15",
}
```
Extend this dict with domain-specific corrections based on the video topic. Present the corrections to the user for review before applying.
4. **Green screen flag** -- Check if scene-director Step 0 flagged green/blue screen footage. If yes, note that Step 3c (Green Screen Composite) will be needed.
### Step 1: Run Enhancement Chain
Apply video enhancements in this exact order. **Attempt every step** if the tool is available — do not skip steps without a reason.
@@ -106,7 +145,14 @@ Pass this dict to both `subtitle_gen` (if generating SRT) and `remotion_caption_
### Step 3: Burn Subtitles
**Preferred: Remotion captions** (if `remotion_caption_burn` tool available):
**ALWAYS use Remotion TikTok-style captions** (word-by-word highlighting). This is the default and preferred method. Do NOT fall back to FFmpeg ASS subtitles unless Remotion is completely unavailable.
**Remotion caption requirements:**
- **Auto-detect video dimensions** -- do NOT hardcode width/height. Use `visual_qa` probe or ffprobe to get actual dimensions, then pass them to the render.
- **Set `--frames` based on actual video duration** -- calculate from probe: `frames = duration_seconds * fps`. Never use a hardcoded frame count.
- Word-by-word highlighting with active word color (`highlight_color`).
- Captions positioned at the bottom of frame, away from the face.
```
remotion_caption_burn.execute({
"input_path": "<reframed_or_enhanced_video>",
@@ -118,15 +164,11 @@ remotion_caption_burn.execute({
"highlight_color": "#22D3EE",
})
```
Remotion renders animated word-by-word captions at the bottom of the frame with active word highlighting. Captions are positioned away from the face.
**Fallback: FFmpeg subtitles** (if Remotion unavailable):
Use `video_compose` with `burn_subtitles` operation:
- Input: reframed video (or enhanced video if no reframe needed)
- Subtitle file from asset manifest
**Fallback ONLY if Remotion is completely unavailable:** Use `video_compose` with `burn_subtitles` operation. This is a degraded experience -- warn the user that word-by-word highlighting won't be available.
**CRITICAL: Caption positioning for 9:16 vertical video.**
Captions MUST be in the lower 20% of the frame. On a 1920-high frame, that means `MarginV=160` or higher. The default FFmpeg subtitle position is center this WILL occlude the face. You MUST override it.
**CRITICAL: Caption positioning for 9:16 vertical video (FFmpeg fallback only).**
Captions MUST be in the lower 20% of the frame. On a 1920-high frame, that means `MarginV=160` or higher. The default FFmpeg subtitle position is center -- this WILL occlude the face. You MUST override it.
FFmpeg subtitle style string for vertical talking-head:
```
@@ -137,27 +179,152 @@ FFmpeg subtitle style string for vertical talking-head:
### Step 3b: Burn Overlay Graphics (if scene plan includes overlays)
If the scene plan includes overlay scenes (text_cards, stat_cards, charts, comparisons, callouts), render them onto the video.
If the scene plan includes overlay scenes (text_cards, stat_cards, charts, comparisons, callouts), pass them to `remotion_caption_burn` alongside captions. **Both captions and overlays render in a single Remotion pass** — no separate FFmpeg compositing needed.
**How overlay compositing works:**
1. Each overlay is a short Remotion composition (3-5 seconds) rendered as a transparent video clip or composited directly
2. Use `video_compose` with `picture_in_picture` or `overlay` operation to place each overlay at the correct timestamp
3. Respect the overlay's `position` field from the scene plan:
- `lower_third` → bottom 30% of frame
- `upper_third` → top 30% of frame
- `side_panel` → left or right 40%
- `full_overlay` → centered, brief (1-2s)
**How it works:** The TalkingHead Remotion composition renders three layers:
1. **Video** (bottom) — the talking-head footage
2. **Overlays** (middle) — positioned charts, stats, callouts with fade in/out
3. **Captions** (top) — word-by-word highlighting, always visible
**For Remotion-based overlays:** Create a composition JSON with the overlay cuts, render to a transparent clip, then composite onto the talking-head video using FFmpeg.
**For simple text overlays:** Use FFmpeg's drawtext filter directly:
**Combine Step 3 and 3b into one `remotion_caption_burn` call:**
```
ffmpeg -i captioned.mp4 -vf "drawtext=text='Key Term':fontsize=48:fontcolor=white:borderw=3:bordercolor=black:x=(w-text_w)/2:y=h*0.75:enable='between(t,22,26)'" -c:a copy output.mp4
remotion_caption_burn.execute({
"input_path": "<reframed_or_enhanced_video>",
"output_path": "<project>/assets/video/captioned.mp4",
"segments": <transcript_segments>,
"corrections": {"cloud": "Claude"},
"words_per_page": 4,
"font_size": 52,
"highlight_color": "#22D3EE",
"overlays": [
{
"id": "term-agentic-ai",
"type": "callout",
"text": "Agentic AI: software that acts autonomously toward goals",
"callout_type": "info",
"in_seconds": 22.0,
"out_seconds": 26.0,
"position": "lower_third",
"backgroundColor": "#0F172A",
"accentColor": "#22D3EE"
},
{
"id": "stat-market-size",
"type": "stat_card",
"stat": "$4.8B",
"subtitle": "Global AI Agent Market (2026)",
"in_seconds": 35.0,
"out_seconds": 39.0,
"position": "upper_third",
"accentColor": "#A78BFA"
},
{
"id": "chart-growth",
"type": "bar_chart",
"chartData": [
{"label": "2023", "value": 1.2},
{"label": "2024", "value": 2.1},
{"label": "2025", "value": 3.5},
{"label": "2026", "value": 4.8}
],
"title": "AI Agent Market ($B)",
"in_seconds": 40.0,
"out_seconds": 45.0,
"position": "lower_third",
"chartColors": ["#22D3EE", "#A78BFA", "#F472B6", "#34D399"]
}
]
})
```
**Important:** Time each overlay to match the scene plan timestamps. After speed adjustment, recalculate overlay timestamps: `adjusted_time = original_time / speed_factor`.
**Overlay position options:**
- `lower_third` → bottom area, above captions (default — safest for most overlays)
- `upper_third` → top area (good for stats while speaker is center/lower)
- `left_panel` → left 45% of frame (side-by-side with speaker)
- `right_panel` → right 45% of frame
- `full_overlay` → full frame with dark backdrop (use sparingly, 1-2s max)
### Step 3c: Build Showcase Cards (if multi-clip reel)
**Overlay type → required props** (same as asset-director mapping):
| Type | Required Props |
|------|---------------|
| `text_card` | `text` |
| `stat_card` | `stat`, `subtitle` (optional) |
| `callout` | `text`, `callout_type` (info/warning/tip/quote) |
| `comparison` | `leftLabel`, `rightLabel`, `leftValue`, `rightValue` |
| `bar_chart` | `chartData` (array of `{label, value}`) |
| `line_chart` | `chartSeries` (array of `{name, data: number[]}`) |
| `pie_chart` | `chartData` (array of `{label, value}`) |
| `kpi_grid` | `chartData` (array of `{label, value}`) |
| `hero_title` | `text`, `subtitle` (optional) |
| `section_title` | `text`, `subtitle` (optional) |
| `stat_reveal` | `text` (the stat), `subtitle` (label) |
**Important:** After speed adjustment, recalculate overlay timestamps: `adjusted_time = original_time / speed_factor`.
**Fallback (no Remotion):** If Remotion is unavailable, `remotion_caption_burn` falls back to FFmpeg for captions only. Overlays are NOT rendered in FFmpeg fallback mode — warn the user that overlays require Remotion.
### Step 3c: Green Screen Composite (if green screen footage)
If the footage has a green/blue screen (detected in scene-director Step 0), follow this pipeline:
1. **Run `green_screen_processor` tool** to remove the green/blue screen:
```
green_screen_processor.execute({
"input_path": "<enhanced_video>",
"output_path": "<project>/assets/video/greenscreen_removed.mp4",
"method": "auto"
})
```
The `auto` method detects whether the background is green or blue and applies the appropriate chroma key.
2. **Render Remotion animated background** using the Explainer composition:
```
# Render an AnimatedBackground clip (gradient mesh, floating orbs, subtle grid)
# Use the Explainer composition — NOT a flat #0F172A solid color
npx remotion render src/index.ts Explainer --props='{"duration":VIDEO_DURATION}' --output=<project>/assets/video/animated_bg.mp4
```
The AnimatedBackground provides a professional gradient mesh with floating orbs and a subtle grid pattern. This is far superior to a flat solid color.
3. **Run `green_screen_composite` tool** to layer the speaker onto the animated background:
```
green_screen_composite.execute({
"foreground_path": "<greenscreen_removed_video>",
"background_path": "<animated_bg>",
"output_path": "<project>/assets/video/composited.mp4",
"layout": "news_anchor"
})
```
Default layout is `news_anchor` (speaker center-bottom, background fills frame). Adjust layout based on speaker position detected in Step 0.
4. **Burn captions via Remotion TalkingHead composition** (NOT FFmpeg ASS subtitles):
```
remotion_caption_burn.execute({
"input_path": "<composited_video>",
"output_path": "<project>/assets/video/captioned.mp4",
"segments": <transcript_segments>,
"corrections": <corrections_dict>,
"words_per_page": 4,
"font_size": 52,
"highlight_color": "#22D3EE",
"overlays": <overlay_list_from_scene_plan>
})
```
5. **Mix background music** (ducked at 15% volume under speech):
```
audio_mixer.execute({
"operation": "duck",
"video_path": "<captioned_video>",
"music_path": "<bg_music>",
"music_volume": 0.15,
"output_path": "<project>/assets/video/with_music.mp4"
})
```
6. **Final encode** to target platform specs (see Step 6 below).
### Step 3d: Build Showcase Cards (if multi-clip reel)
If the output is a reel with showcase clips, use `showcase_card` for each:
```
@@ -18,6 +18,34 @@ You are not just a processor. You are a creative director. Your job is to figure
## Process
### Step 0: Footage Analysis
Before watching the content, analyze the raw footage to understand the physical setup.
1. **Sample 5 frames** from the footage using ffmpeg (evenly spaced across the duration):
```
ffmpeg -i <footage> -vf "select='not(mod(n\,TOTAL_FRAMES/5))'" -vsync vfr -frames:v 5 frame_%02d.png
```
2. **Run visual_qa or histogram analysis** on each sampled frame to detect:
- **Background type:** Green screen / blue screen / natural background. Green/blue screens show a dominant narrow-band color spike in the histogram. Use `visual_qa` with prompt "Is this a green screen or blue screen background?" for confirmation.
- **Speaker position:** Center, left, or right of frame. Estimate the approximate bounding box (e.g., "speaker occupies center 40% of frame, from x=30% to x=70%").
- **Lighting quality:** Even studio lighting, harsh shadows, backlit, mixed color temperature. Note any issues that may affect chroma keying.
3. **Green screen detected?**
- If yes, note that `green_screen_processor` tool will be needed in the compose stage.
- Record the detected screen color (green or blue) and estimated uniformity.
- The compose-director will use this to run chroma key removal and composite onto an animated background.
4. **Measure speaker safe zone:**
- From the speaker's bounding box, determine where graphics can be placed WITHOUT overlapping the speaker.
- For a centered speaker: left panel and right panel are safe for overlays.
- For a left-positioned speaker: right panel is the primary safe zone.
- For a right-positioned speaker: left panel is the primary safe zone.
- Upper third and lower third are generally safe regardless of speaker position.
Record all findings in the scene plan metadata for downstream stages.
### Step 1: Watch & Listen — Understand the Content
**This is the most important step. Do not skip it.**
@@ -52,6 +80,24 @@ Based on your content analysis, propose **on-screen graphics** that will appear
| **Callout/quote** | `callout` | When the speaker makes a key point worth emphasizing |
| **Lower third** | `text_card` | Speaker identification at the start |
**Remotion Component Constraints:**
| Component | Min Width | 720px Portrait? | Value Type |
|-----------|-----------|-----------------|------------|
| comparison | 900px | NO -> use 2x stat_card | string |
| kpi_grid | 720px | YES | numeric ONLY (no "15+") |
| bar_chart | 500px | YES | numeric |
| stat_card | 300px | YES | string OK |
| callout | 400px | YES | string |
| hero_title | 400px | YES | string |
| line_chart | 500px | YES | numeric |
| progress_bar | 600px | YES | numeric |
| stat_reveal | 300px | YES | string OK |
When a component's minimum width exceeds the available space (e.g., `comparison` at 900px won't fit in a 720px portrait frame), substitute with the recommended alternative. For `comparison`, use two `stat_card` components shown sequentially instead.
For `kpi_grid`, values MUST be numeric (e.g., `4.8`, `73`, `2400`). String values like `"15+"` or `"$4.8B"` will cause rendering errors. Use `stat_card` for string-formatted numbers.
**Overlay planning rules:**
- **Don't over-overlay.** 3-6 overlays per minute of final video is the sweet spot. More than that is distracting.
- **Time overlays to speech.** Each overlay should appear when the speaker says the relevant words, not before or after.
@@ -92,6 +138,8 @@ Format your proposal clearly:
**Estimated final duration:** ~Xs (from Xs raw)
```
**IMPORTANT: When outputting the overlay plan, ALSO generate the actual Remotion JSON props file (`greenscreen-bg.json`) -- do not just describe scenes in prose.** The JSON props file should be a complete, valid input for the Remotion TalkingHead composition, including all overlay definitions, timing, colors, and content. Save it to `<project>/public/demo-props/` or the project's props directory.
Wait for user approval before proceeding. The user may:
- Approve as-is
- Add/remove overlays
+430
View File
@@ -0,0 +1,430 @@
"""Green screen composite tool for talking-head pipeline.
Composites a keyed speaker (dark/solid background) over a Remotion
background video with layout presets. Supports news anchor, full behind,
picture-in-picture, and split layouts.
Uses PIL/numpy for frame-level alpha compositing and FFmpeg for
frame extraction, encoding, and audio muxing.
"""
from __future__ import annotations
import json
import shutil
import time
from pathlib import Path
from typing import Any
import numpy as np
from PIL import Image
from tools.base_tool import (
BaseTool,
Determinism,
ExecutionMode,
ResourceProfile,
RetryPolicy,
ResumeSupport,
ToolResult,
ToolStability,
ToolTier,
)
class GreenScreenComposite(BaseTool):
name = "green_screen_composite"
version = "0.1.0"
tier = ToolTier.CORE
capability = "video_post"
provider = "ffmpeg"
stability = ToolStability.EXPERIMENTAL
execution_mode = ExecutionMode.SYNC
determinism = Determinism.DETERMINISTIC
dependencies = ["cmd:ffmpeg", "python:numpy", "python:PIL"]
install_instructions = (
"Install FFmpeg: https://ffmpeg.org/download.html — "
"pip install numpy Pillow"
)
agent_skills = ["ffmpeg"]
capabilities = [
"green_screen_composite",
"speaker_overlay",
"layout_preset",
"alpha_composite",
]
input_schema = {
"type": "object",
"required": ["speaker_path", "background_path", "output_path"],
"properties": {
"speaker_path": {
"type": "string",
"description": "Path to keyed speaker video (dark bg, from green_screen_processor)",
},
"background_path": {
"type": "string",
"description": "Path to Remotion background video",
},
"output_path": {
"type": "string",
"description": "Output composite video path",
},
"original_audio_path": {
"type": "string",
"description": "Path to original footage to extract audio from",
},
"layout": {
"type": "string",
"enum": ["news_anchor", "full_behind", "pip", "split"],
"default": "news_anchor",
"description": (
"news_anchor=speaker bottom-center over shifted bg, "
"full_behind=speaker full-frame on bg, "
"pip=speaker 30% bottom-right, "
"split=speaker left 50% bg right 50%"
),
},
"speaker_scale": {
"type": "number",
"default": 0.65,
"description": "Scale factor for speaker layer",
},
"bg_shift_up": {
"type": "integer",
"default": 300,
"description": "Pixels to shift background content upward",
},
"bg_color_hex": {
"type": "string",
"default": "#0E172A",
"description": "The keyed speaker's background color for alpha creation",
},
},
}
resource_profile = ResourceProfile(
cpu_cores=4, ram_mb=4096, vram_mb=0, disk_mb=8000, network_required=False
)
retry_policy = RetryPolicy(max_retries=1, retryable_errors=["FFmpeg error"])
resume_support = ResumeSupport.FROM_START
idempotency_key_fields = [
"speaker_path", "background_path", "layout",
"speaker_scale", "bg_shift_up", "bg_color_hex",
]
side_effects = ["writes composite video to output_path"]
user_visible_verification = [
"Watch output — speaker should be cleanly composited without color fringing",
"Check layout positioning matches the chosen preset",
"Verify audio is synced if original_audio_path was provided",
]
def execute(self, inputs: dict[str, Any]) -> ToolResult:
speaker_path = Path(inputs["speaker_path"])
background_path = Path(inputs["background_path"])
output_path = Path(inputs["output_path"])
original_audio_path = inputs.get("original_audio_path")
layout = inputs.get("layout", "news_anchor")
speaker_scale = inputs.get("speaker_scale", 0.65)
bg_shift_up = inputs.get("bg_shift_up", 300)
bg_color_hex = inputs.get("bg_color_hex", "#0E172A")
if not speaker_path.exists():
return ToolResult(success=False, error=f"Speaker video not found: {speaker_path}")
if not background_path.exists():
return ToolResult(success=False, error=f"Background video not found: {background_path}")
if original_audio_path and not Path(original_audio_path).exists():
return ToolResult(success=False, error=f"Audio source not found: {original_audio_path}")
output_path.parent.mkdir(parents=True, exist_ok=True)
start = time.time()
# Parse bg color
bg_color = self._parse_hex_color(bg_color_hex)
# Step 1: Probe both videos
speaker_info = self._probe_video(speaker_path)
bg_info = self._probe_video(background_path)
if not speaker_info or not bg_info:
return ToolResult(
success=False,
error="Failed to probe one or both input videos",
)
# Step 2: Use the LOWER fps (typically 15fps from speaker)
target_fps = min(speaker_info["fps"], bg_info["fps"])
if target_fps <= 0:
target_fps = 15.0
# Determine output dimensions from background
out_w = bg_info["width"]
out_h = bg_info["height"]
# Use shorter duration
duration = min(speaker_info["duration"], bg_info["duration"])
# Step 3: Extract frames from both videos
temp_dir = output_path.parent / ".greenscreen_composite_tmp"
speaker_frames_dir = temp_dir / "speaker"
bg_frames_dir = temp_dir / "bg"
comp_frames_dir = temp_dir / "composite"
for d in [speaker_frames_dir, bg_frames_dir, comp_frames_dir]:
d.mkdir(parents=True, exist_ok=True)
try:
self._extract_frames(speaker_path, speaker_frames_dir, target_fps)
self._extract_frames(background_path, bg_frames_dir, target_fps)
# Get sorted frame lists
speaker_frames = sorted(speaker_frames_dir.glob("*.png"))
bg_frames = sorted(bg_frames_dir.glob("*.png"))
if not speaker_frames or not bg_frames:
return ToolResult(
success=False,
error="Frame extraction produced no frames",
)
frame_count = min(len(speaker_frames), len(bg_frames))
log_interval = max(1, frame_count // 10)
# Step 4: Composite each frame pair
for i in range(frame_count):
if i % log_interval == 0:
print(f"[green_screen_composite] Compositing frame {i + 1}/{frame_count}")
speaker_img = Image.open(speaker_frames[i]).convert("RGB")
bg_img = Image.open(bg_frames[i]).convert("RGB")
comp = self._composite_frame(
speaker_img, bg_img, bg_color,
layout=layout,
speaker_scale=speaker_scale,
bg_shift_up=bg_shift_up,
out_w=out_w,
out_h=out_h,
)
comp.save(comp_frames_dir / f"frame_{i:06d}.png")
print(f"[green_screen_composite] All {frame_count} frames composited")
# Step 5: Encode composite frames to video
no_audio_path = output_path if not original_audio_path else temp_dir / "no_audio.mp4"
self._encode_frames(comp_frames_dir, no_audio_path, target_fps, out_w, out_h)
# Step 6: Mux audio if provided
if original_audio_path:
self._mux_audio(no_audio_path, Path(original_audio_path), output_path, duration)
if not output_path.exists() or output_path.stat().st_size == 0:
return ToolResult(success=False, error="Output video was not created")
elapsed = time.time() - start
return ToolResult(
success=True,
data={
"output": str(output_path),
"layout": layout,
"fps": target_fps,
"frame_count": frame_count,
"duration": round(duration, 2),
"dimensions": f"{out_w}x{out_h}",
"speaker_scale": speaker_scale,
"has_audio": bool(original_audio_path),
},
artifacts=[str(output_path)],
duration_seconds=round(elapsed, 2),
)
except Exception as e:
return ToolResult(success=False, error=f"Composite failed: {e}")
finally:
# Step 7: Clean up temp directories
self._cleanup_temp(temp_dir)
def _parse_hex_color(self, hex_str: str) -> np.ndarray:
"""Parse a hex color string like '#0E172A' to an RGB numpy array."""
hex_str = hex_str.lstrip("#")
r = int(hex_str[0:2], 16)
g = int(hex_str[2:4], 16)
b = int(hex_str[4:6], 16)
return np.array([r, g, b])
def _probe_video(self, path: Path) -> dict[str, Any] | None:
"""Probe a video for fps, duration, and dimensions."""
cmd = [
"ffprobe", "-v", "quiet",
"-print_format", "json",
"-show_format", "-show_streams",
str(path),
]
try:
result = self.run_command(cmd, timeout=30)
data = json.loads(result.stdout)
except Exception:
return None
# Find video stream
video_stream = None
for stream in data.get("streams", []):
if stream.get("codec_type") == "video":
video_stream = stream
break
if not video_stream:
return None
# Parse fps from r_frame_rate (e.g., "30/1" or "15000/1001")
fps_str = video_stream.get("r_frame_rate", "30/1")
try:
num, den = fps_str.split("/")
fps = float(num) / float(den)
except (ValueError, ZeroDivisionError):
fps = 30.0
duration = float(data.get("format", {}).get("duration", 0))
return {
"fps": fps,
"duration": duration,
"width": int(video_stream.get("width", 1920)),
"height": int(video_stream.get("height", 1080)),
}
def _extract_frames(self, video_path: Path, output_dir: Path, fps: float) -> None:
"""Extract frames from a video at the given fps."""
cmd = [
"ffmpeg", "-y",
"-i", str(video_path),
"-vf", f"fps={fps}",
str(output_dir / "frame_%06d.png"),
]
self.run_command(cmd, timeout=600)
def _composite_frame(
self,
speaker_img: Image.Image,
bg_img: Image.Image,
bg_color: np.ndarray,
*,
layout: str,
speaker_scale: float,
bg_shift_up: int,
out_w: int,
out_h: int,
) -> Image.Image:
"""Composite a single speaker frame over a background frame using the given layout."""
# Create alpha mask from speaker frame
speaker_arr = np.array(speaker_img).astype(float)
dist = np.sqrt(np.sum((speaker_arr - bg_color.astype(float)) ** 2, axis=2))
threshold = 35
alpha = np.clip((dist - threshold) * 8, 0, 255).astype(np.uint8)
speaker_rgba = Image.new("RGBA", speaker_img.size)
speaker_rgba.paste(speaker_img, (0, 0))
speaker_rgba.putalpha(Image.fromarray(alpha))
# Prepare background canvas at output size
canvas = Image.new("RGBA", (out_w, out_h), (0, 0, 0, 255))
if layout == "news_anchor":
# Background shifted up so graphics appear above speaker's head
bg_resized = bg_img.resize((out_w, out_h), Image.LANCZOS).convert("RGBA")
# Shift background up: paste it higher so bottom content scrolls up
shifted_bg = Image.new("RGBA", (out_w, out_h), (0, 0, 0, 255))
shifted_bg.paste(bg_resized, (0, -bg_shift_up))
canvas = shifted_bg
# Scale speaker and place at bottom center
sp_w = int(speaker_rgba.width * speaker_scale)
sp_h = int(speaker_rgba.height * speaker_scale)
speaker_scaled = speaker_rgba.resize((sp_w, sp_h), Image.LANCZOS)
x = (out_w - sp_w) // 2
y = out_h - sp_h
canvas.paste(speaker_scaled, (x, y), speaker_scaled)
elif layout == "full_behind":
# Speaker full-frame on background, no scaling, no shifting
bg_resized = bg_img.resize((out_w, out_h), Image.LANCZOS).convert("RGBA")
canvas = bg_resized
# Resize speaker to match output
speaker_full = speaker_rgba.resize((out_w, out_h), Image.LANCZOS)
canvas.paste(speaker_full, (0, 0), speaker_full)
elif layout == "pip":
# Background full-frame, speaker 30% in bottom-right
bg_resized = bg_img.resize((out_w, out_h), Image.LANCZOS).convert("RGBA")
canvas = bg_resized
pip_scale = 0.30
sp_w = int(out_w * pip_scale)
sp_h = int(out_h * pip_scale)
speaker_pip = speaker_rgba.resize((sp_w, sp_h), Image.LANCZOS)
margin = 20
x = out_w - sp_w - margin
y = out_h - sp_h - margin
canvas.paste(speaker_pip, (x, y), speaker_pip)
elif layout == "split":
# Speaker on left 50%, background on right 50%
half_w = out_w // 2
# Left side: speaker resized to fill left half
speaker_left = speaker_rgba.resize((half_w, out_h), Image.LANCZOS)
# Right side: background cropped/resized to fill right half
bg_right = bg_img.resize((half_w, out_h), Image.LANCZOS).convert("RGBA")
canvas.paste(speaker_left, (0, 0), speaker_left)
canvas.paste(bg_right, (half_w, 0), bg_right)
# Convert to RGB for output
return canvas.convert("RGB")
def _encode_frames(
self, frames_dir: Path, output_path: Path, fps: float, width: int, height: int
) -> None:
"""Encode PNG frames to an MP4 video."""
cmd = [
"ffmpeg", "-y",
"-framerate", str(fps),
"-i", str(frames_dir / "frame_%06d.png"),
"-c:v", "libx264", "-crf", "18", "-preset", "fast",
"-pix_fmt", "yuv420p",
"-vf", f"scale={width}:{height}",
str(output_path),
]
self.run_command(cmd, timeout=600)
def _mux_audio(
self, video_path: Path, audio_source: Path, output_path: Path, duration: float
) -> None:
"""Mux audio from the original source into the composite video."""
cmd = [
"ffmpeg", "-y",
"-i", str(video_path),
"-i", str(audio_source),
"-t", f"{duration:.3f}",
"-c:v", "copy",
"-c:a", "aac", "-b:a", "192k",
"-map", "0:v:0", "-map", "1:a:0",
"-shortest",
str(output_path),
]
self.run_command(cmd, timeout=300)
def _cleanup_temp(self, temp_dir: Path) -> None:
"""Remove temporary frame directories."""
if temp_dir.exists():
try:
shutil.rmtree(temp_dir)
except OSError:
# Best-effort cleanup; log but don't fail
print(f"[green_screen_composite] Warning: could not fully clean {temp_dir}")
def estimate_runtime(self, inputs: dict[str, Any]) -> float:
return 120.0
+622
View File
@@ -0,0 +1,622 @@
"""Green screen keying processor.
Removes green/blue screen backgrounds from footage using either FFmpeg
chromakey filtering or rembg AI segmentation. Supports automatic method
detection by analyzing frame color histograms.
Methods:
- auto: Analyze frames to pick the best method (chromakey vs rembg)
- chromakey: FFmpeg chromakey filter (fast, works well on clean screens)
- rembg: AI background removal via rembg/u2net (slower, handles any bg)
"""
from __future__ import annotations
import json
import os
import platform
import shutil
import time
from pathlib import Path
from typing import Any
from tools.base_tool import (
BaseTool,
Determinism,
ExecutionMode,
ResourceProfile,
RetryPolicy,
ResumeSupport,
ToolResult,
ToolStability,
ToolTier,
)
class GreenScreenProcessor(BaseTool):
name = "green_screen_processor"
version = "0.1.0"
tier = ToolTier.CORE
capability = "video_post"
provider = "ffmpeg"
stability = ToolStability.EXPERIMENTAL
execution_mode = ExecutionMode.SYNC
determinism = Determinism.DETERMINISTIC
dependencies = ["cmd:ffmpeg"]
install_instructions = (
"Install FFmpeg: https://ffmpeg.org/download.html "
"For rembg method: pip install rembg[gpu] onnxruntime"
)
agent_skills = ["ffmpeg"]
capabilities = [
"green_screen_keying",
"chromakey",
"background_removal",
"rembg_segmentation",
]
input_schema = {
"type": "object",
"required": ["input_path", "output_path"],
"properties": {
"input_path": {
"type": "string",
"description": "Path to raw green screen footage",
},
"output_path": {
"type": "string",
"description": "Path for keyed output video",
},
"method": {
"type": "string",
"enum": ["auto", "chromakey", "rembg"],
"default": "auto",
"description": "Keying method: auto detects best approach, chromakey uses FFmpeg, rembg uses AI segmentation",
},
"fps": {
"type": "integer",
"default": 15,
"description": "Output frames per second",
},
"bg_color": {
"type": "string",
"default": "#0E172A",
"description": "Hex color for output background",
},
"max_frames": {
"type": "integer",
"default": 0,
"description": "Limit frames to process (0 = all)",
},
},
}
resource_profile = ResourceProfile(
cpu_cores=4, ram_mb=4096, vram_mb=0, disk_mb=8000, network_required=False
)
retry_policy = RetryPolicy(max_retries=1, retryable_errors=["FFmpeg error"])
resume_support = ResumeSupport.FROM_START
idempotency_key_fields = [
"input_path", "method", "fps", "bg_color", "max_frames",
]
side_effects = ["writes keyed video to output_path"]
user_visible_verification = [
"Check output for green fringing around subject edges",
"Verify background is cleanly replaced with target color",
"Look for flickering or inconsistent keying between frames",
]
# Platform-specific null device
_null_device = "NUL" if platform.system() == "Windows" else "/dev/null"
def execute(self, inputs: dict[str, Any]) -> ToolResult:
input_path = Path(inputs["input_path"])
if not input_path.exists():
return ToolResult(success=False, error=f"Input not found: {input_path}")
output_path = Path(inputs["output_path"])
output_path.parent.mkdir(parents=True, exist_ok=True)
method = inputs.get("method", "auto")
fps = inputs.get("fps", 15)
bg_color = inputs.get("bg_color", "#0E172A")
max_frames = inputs.get("max_frames", 0)
start = time.time()
# Step 1: Probe input video
probe = self._probe_video(input_path)
if not probe:
return ToolResult(success=False, error="Failed to probe input video")
duration = probe["duration"]
width = probe["width"]
height = probe["height"]
src_fps = probe["fps"]
# Step 2: Determine method
if method == "auto":
method = self._auto_detect_method(input_path, duration, width, height)
# Step 3: Set up temp directory for frame processing
temp_dir = output_path.parent / f".gs_tmp_{int(time.time())}"
temp_dir.mkdir(parents=True, exist_ok=True)
try:
# Step 4: Extract frames at target fps
frames_dir = temp_dir / "frames"
frames_dir.mkdir(exist_ok=True)
frame_count = self._extract_frames(
input_path, frames_dir, fps, max_frames
)
if frame_count == 0:
return ToolResult(
success=False, error="No frames extracted from input"
)
# Step 5: Process frames
processed_dir = temp_dir / "processed"
processed_dir.mkdir(exist_ok=True)
if method == "chromakey":
ok = self._process_chromakey(
frames_dir, processed_dir, bg_color, frame_count
)
else:
ok = self._process_rembg(
frames_dir, processed_dir, bg_color, frame_count
)
if not ok:
return ToolResult(
success=False,
error=f"Frame processing failed with method={method}",
)
# Step 6: Reconstruct video from processed frames
self._reconstruct_video(processed_dir, output_path, fps, width, height)
if not output_path.exists() or output_path.stat().st_size == 0:
return ToolResult(
success=False, error="Output video was not created"
)
elapsed = time.time() - start
return ToolResult(
success=True,
data={
"method_used": method,
"frame_count": frame_count,
"duration": round(duration, 2),
"output_path": str(output_path),
"resolution": f"{width}x{height}",
"fps": fps,
"bg_color": bg_color,
},
artifacts=[str(output_path)],
duration_seconds=round(elapsed, 2),
)
except Exception as e:
return ToolResult(success=False, error=f"Green screen processing failed: {e}")
finally:
# Clean up temp directory
self._cleanup_dir(temp_dir)
def _probe_video(self, input_path: Path) -> dict[str, Any] | None:
"""Probe video for duration, dimensions, and fps."""
cmd = [
"ffprobe", "-v", "quiet",
"-show_entries", "format=duration:stream=width,height,r_frame_rate",
"-select_streams", "v:0",
"-of", "json",
str(input_path),
]
try:
result = self.run_command(cmd, timeout=30)
data = json.loads(result.stdout)
duration = float(data.get("format", {}).get("duration", 0))
stream = data.get("streams", [{}])[0]
width = int(stream.get("width", 0))
height = int(stream.get("height", 0))
# Parse r_frame_rate like "30/1" or "30000/1001"
fps_str = stream.get("r_frame_rate", "30/1")
if "/" in fps_str:
num, den = fps_str.split("/")
fps_val = float(num) / float(den) if float(den) != 0 else 30.0
else:
fps_val = float(fps_str)
return {
"duration": duration,
"width": width,
"height": height,
"fps": fps_val,
}
except Exception:
return None
def _auto_detect_method(
self, input_path: Path, duration: float, width: int, height: int
) -> str:
"""Analyze sample frames to decide between chromakey and rembg.
Extracts 5 evenly-spaced frames, checks color histograms for
green/blue screen presence, then tests chromakey quality on a sample.
"""
temp_dir = input_path.parent / f".gs_detect_{int(time.time())}"
temp_dir.mkdir(parents=True, exist_ok=True)
try:
# Extract 5 sample frames evenly spaced
interval = max(duration / 6, 0.1)
sample_paths = []
for i in range(5):
ts = interval * (i + 1)
out = temp_dir / f"sample_{i}.png"
cmd = [
"ffmpeg", "-y",
"-ss", f"{ts:.3f}",
"-i", str(input_path),
"-frames:v", "1",
str(out),
]
try:
self.run_command(cmd, timeout=30)
if out.exists():
sample_paths.append(out)
except Exception:
continue
if not sample_paths:
return "rembg" # fallback if we can't extract samples
# Analyze color histograms for green/blue screen presence
has_green_screen = self._detect_green_screen_histogram(sample_paths)
if not has_green_screen:
# No obvious green/blue screen detected, use rembg
return "rembg"
# Test chromakey on a sample frame and check quality
test_frame = sample_paths[len(sample_paths) // 2]
chromakey_quality = self._test_chromakey_quality(test_frame, temp_dir)
if chromakey_quality > 80:
return "chromakey"
else:
return "rembg"
finally:
self._cleanup_dir(temp_dir)
def _detect_green_screen_histogram(self, sample_paths: list[Path]) -> bool:
"""Analyze frames for dominant green or blue channel presence.
Uses FFmpeg signalstats to measure average hue. A strong green
screen typically has a large area of similar green/blue hue.
"""
green_votes = 0
for sample in sample_paths:
cmd = [
"ffmpeg", "-y",
"-i", str(sample),
"-vf", "signalstats=stat=tout+vrep+brng,metadata=mode=print",
"-frames:v", "1",
"-f", "null", self._null_device,
]
try:
result = self.run_command(cmd, timeout=15)
# Check stderr for color stats
output = result.stderr or ""
# Alternative: use FFmpeg to count green-ish pixels
# Run a simpler hue check with colorchannelmixer
cmd2 = [
"ffmpeg", "-y",
"-i", str(sample),
"-vf", (
"split[a][b];"
"[a]colorchannelmixer=rr=0:gg=1:bb=0,"
"threshold=threshold=0.3:similarity=0.3[mask];"
"[mask]blackframe=amount=0:threshold=32"
),
"-frames:v", "1",
"-f", "null", self._null_device,
]
# This is complex; use a simpler approach: check raw pixels
# via a green-range filter
cmd_green = [
"ffmpeg", "-y",
"-i", str(sample),
"-vf", (
"colorkey=color=0x00FF00:similarity=0.4:blend=0.0,"
"alphaextract,"
"blackframe=amount=0:threshold=128"
),
"-frames:v", "1",
"-f", "null", self._null_device,
]
try:
result2 = self.run_command(cmd_green, timeout=15)
stderr = result2.stderr or ""
# blackframe reports percentage of black pixels
# If many pixels became transparent (black in alpha), there's green
if "pblack:" in stderr:
import re
pblack_matches = re.findall(r"pblack:(\d+)", stderr)
if pblack_matches:
pblack = int(pblack_matches[0])
if pblack >= 20:
green_votes += 1
except Exception:
pass
except Exception:
continue
# If majority of frames show green screen
return green_votes >= len(sample_paths) // 2
def _test_chromakey_quality(self, test_frame: Path, temp_dir: Path) -> float:
"""Run chromakey on a test frame and estimate quality percentage.
Returns a score 0-100 indicating what percentage of the expected
background was successfully keyed out.
"""
keyed_out = temp_dir / "chromakey_test.png"
# Apply chromakey and output with alpha
cmd = [
"ffmpeg", "-y",
"-i", str(test_frame),
"-vf", "chromakey=color=0x00FF00:similarity=0.3:blend=0.08",
str(keyed_out),
]
try:
self.run_command(cmd, timeout=15)
except Exception:
return 0.0
if not keyed_out.exists():
return 0.0
# Count transparent pixels via alphaextract + blackframe
cmd2 = [
"ffmpeg", "-y",
"-i", str(keyed_out),
"-vf", "alphaextract,blackframe=amount=0:threshold=32",
"-frames:v", "1",
"-f", "null", self._null_device,
]
try:
result = self.run_command(cmd2, timeout=15)
stderr = result.stderr or ""
import re
pblack_matches = re.findall(r"pblack:(\d+)", stderr)
if pblack_matches:
# pblack = percentage of black pixels in alpha = transparent pixels
return float(pblack_matches[0])
except Exception:
pass
return 0.0
def _extract_frames(
self, input_path: Path, frames_dir: Path, fps: int, max_frames: int
) -> int:
"""Extract frames from video at target fps."""
cmd = [
"ffmpeg", "-y",
"-i", str(input_path),
"-vf", f"fps={fps}",
str(frames_dir / "frame_%06d.png"),
]
if max_frames > 0:
cmd.insert(-1, "-frames:v")
cmd.insert(-1, str(max_frames))
try:
self.run_command(cmd, timeout=600)
except Exception as e:
# ffmpeg may return non-zero but still produce frames
pass
# Count extracted frames
frame_files = sorted(frames_dir.glob("frame_*.png"))
count = len(frame_files)
if count > 0:
# Log progress for large frame counts
if count > 100:
print(f"[green_screen_processor] Extracted {count} frames")
return count
def _process_chromakey(
self,
frames_dir: Path,
processed_dir: Path,
bg_color: str,
frame_count: int,
) -> bool:
"""Process frames using FFmpeg chromakey filter.
Applies chromakey to remove green, then composites onto bg_color.
"""
bg_hex = bg_color.lstrip("#")
# Convert hex to FFmpeg color format
ffmpeg_bg = f"0x{bg_hex}"
frame_files = sorted(frames_dir.glob("frame_*.png"))
processed = 0
for i, frame in enumerate(frame_files):
out_path = processed_dir / frame.name
cmd = [
"ffmpeg", "-y",
"-f", "lavfi", "-i", f"color=c={ffmpeg_bg}:size=1x1",
"-i", str(frame),
"-filter_complex",
(
f"[0:v]scale=iw:ih[bg];"
f"[1:v]chromakey=color=0x00FF00:similarity=0.3:blend=0.08[fg];"
f"[bg][fg]overlay=0:0"
),
"-frames:v", "1",
str(out_path),
]
try:
self.run_command(cmd, timeout=30)
if out_path.exists():
processed += 1
except Exception:
# Try with the frame size explicitly to fix scale
try:
cmd_retry = [
"ffmpeg", "-y",
"-i", str(frame),
"-vf",
f"chromakey=color=0x00FF00:similarity=0.3:blend=0.08,"
f"split[fg][alpha];"
f"[alpha]alphaextract[a];"
f"color=c={ffmpeg_bg}[bg];"
f"[bg][fg][a]maskedmerge",
"-frames:v", "1",
str(out_path),
]
# Simpler fallback: just apply chromakey without compositing
cmd_simple = [
"ffmpeg", "-y",
"-i", str(frame),
"-vf", f"chromakey=color=0x00FF00:similarity=0.3:blend=0.08",
str(out_path),
]
self.run_command(cmd_simple, timeout=30)
if out_path.exists():
processed += 1
except Exception:
continue
if frame_count > 100 and (i + 1) % 50 == 0:
print(
f"[green_screen_processor] Chromakey: {i + 1}/{frame_count} frames"
)
return processed > 0
def _process_rembg(
self,
frames_dir: Path,
processed_dir: Path,
bg_color: str,
frame_count: int,
) -> bool:
"""Process frames using rembg AI segmentation.
Removes background with u2net_human_seg model, then composites
the subject onto bg_color background.
"""
try:
import rembg
from PIL import Image
except ImportError:
return False
# Parse bg_color hex to RGB
bg_hex = bg_color.lstrip("#")
bg_r = int(bg_hex[0:2], 16)
bg_g = int(bg_hex[2:4], 16)
bg_b = int(bg_hex[4:6], 16)
session = rembg.new_session("u2net_human_seg")
frame_files = sorted(frames_dir.glob("frame_*.png"))
processed = 0
for i, frame in enumerate(frame_files):
try:
img = Image.open(frame).convert("RGB")
import numpy as np
# Remove background (returns RGBA)
result = rembg.remove(
np.array(img),
session=session,
)
result_img = Image.fromarray(result)
# Composite onto bg_color background
bg = Image.new("RGBA", result_img.size, (bg_r, bg_g, bg_b, 255))
bg.paste(result_img, (0, 0), result_img)
# Save as RGB
out_path = processed_dir / frame.name
bg.convert("RGB").save(out_path)
processed += 1
except Exception:
continue
if frame_count > 100 and (i + 1) % 50 == 0:
print(
f"[green_screen_processor] rembg: {i + 1}/{frame_count} frames"
)
return processed > 0
def _reconstruct_video(
self,
frames_dir: Path,
output_path: Path,
fps: int,
width: int,
height: int,
) -> None:
"""Reconstruct video from processed frames using FFmpeg."""
cmd = [
"ffmpeg", "-y",
"-framerate", str(fps),
"-i", str(frames_dir / "frame_%06d.png"),
"-vf", f"scale={width}:{height}:flags=lanczos",
"-c:v", "libx264",
"-crf", "18",
"-preset", "fast",
"-pix_fmt", "yuv420p",
str(output_path),
]
self.run_command(cmd, timeout=600)
@staticmethod
def _cleanup_dir(dir_path: Path) -> None:
"""Recursively remove a temp directory."""
if not dir_path.exists():
return
try:
shutil.rmtree(dir_path)
except OSError:
# Best-effort cleanup; individual file removal as fallback
for f in dir_path.rglob("*"):
try:
if f.is_file():
f.unlink()
except OSError:
pass
try:
dir_path.rmdir()
except OSError:
pass
def estimate_runtime(self, inputs: dict[str, Any]) -> float:
method = inputs.get("method", "auto")
if method == "rembg":
return 120.0
elif method == "chromakey":
return 30.0
return 60.0 # auto
+35 -3
View File
@@ -106,6 +106,19 @@ class RemotionCaptionBurn(BaseTool):
"correct replacement. Example: {\"cloud\": \"Claude\"}."
),
},
"overlays": {
"type": "array",
"description": (
"Array of overlay objects to render on top of the video. "
"Each overlay has: type (text_card, stat_card, callout, "
"comparison, bar_chart, line_chart, pie_chart, kpi_grid, "
"hero_title, section_title, stat_reveal), in_seconds, "
"out_seconds, position (lower_third, upper_third, "
"left_panel, right_panel, full_overlay), and component-"
"specific props (text, stat, chartData, etc.). "
"See asset_manifest overlays from the asset-director."
),
},
"force_ffmpeg": {
"type": "boolean",
"default": False,
@@ -245,6 +258,7 @@ class RemotionCaptionBurn(BaseTool):
words_per_page: int,
font_size: int,
highlight_color: str,
overlays: list[dict] | None = None,
) -> ToolResult:
root = self._find_remotion_root()
if root is None:
@@ -262,6 +276,19 @@ class RemotionCaptionBurn(BaseTool):
duration_s = float(dur_out.strip().split("\n")[0])
total_frames = math.ceil(duration_s * 30)
# Detect video dimensions
dim_cmd = [
"ffprobe", "-v", "error",
"-select_streams", "v:0",
"-show_entries", "stream=width,height",
"-of", "csv=p=0:s=x",
input_path,
]
dim_result = self.run_command(dim_cmd)
dim_parts = dim_result.stdout.strip().split("x")
width = int(dim_parts[0])
height = int(dim_parts[1])
# Copy video to Remotion public folder
pub_dir = root / "public" / "talking-head"
pub_dir.mkdir(parents=True, exist_ok=True)
@@ -273,6 +300,7 @@ class RemotionCaptionBurn(BaseTool):
props = {
"videoSrc": f"public/talking-head/{video_filename}",
"captions": captions,
"overlays": overlays or [],
"wordsPerPage": words_per_page,
"fontSize": font_size,
"highlightColor": highlight_color,
@@ -287,12 +315,12 @@ class RemotionCaptionBurn(BaseTool):
npx_bin = "npx.cmd" if sys.platform == "win32" else "npx"
render_cmd = [
npx_bin, "remotion", "render",
"src/index.tsx", "TalkingHead",
"TalkingHead",
f"--props={props_file.relative_to(root)}",
"--width=1080", "--height=1920", "--fps=30",
f"--width={width}", f"--height={height}", "--fps=30",
f"--frames=0-{total_frames - 1}",
"--codec=h264", "--crf=18",
str(Path(output_path).resolve()),
f"--output={str(Path(output_path).resolve())}",
]
self.run_command(render_cmd, cwd=str(root))
@@ -307,6 +335,7 @@ class RemotionCaptionBurn(BaseTool):
"duration_seconds": round(duration_s, 2),
"total_frames": total_frames,
"caption_count": len(captions),
"overlay_count": len(overlays or []),
"words_per_page": words_per_page,
},
artifacts=[output_path],
@@ -429,11 +458,14 @@ class RemotionCaptionBurn(BaseTool):
if not captions:
return ToolResult(success=False, error="No caption words extracted.")
overlays = inputs.get("overlays")
# Choose render method
if not force_ffmpeg and self._remotion_available():
result = self._render_remotion(
input_path, output_path, captions,
words_per_page, font_size, highlight_color,
overlays=overlays,
)
else:
result = self._render_ffmpeg(input_path, output_path, captions)