From 8fbb7f176e31d78dd09ec5cd42608d5394c62002 Mon Sep 17 00:00:00 2001 From: nite Date: Tue, 14 Jul 2026 12:16:17 +0800 Subject: [PATCH] feat: preserve multi-track audio in clip extraction MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SKILL.md: - Step 6: two clip modes — video-only (frame extraction) vs all-streams (editor use) - Pitfall #10: warn about dropping audio tracks with default -map 0:v:0 - Note on keyframe accuracy vs frame-exact re-encode extract_frames.py: - New --all-streams flag on clip subcommand - -map 0 preserves video + all audio tracks + subtitles - Default remains -map 0:v:0 for fast frame extraction --- SKILL.md | 13 +++++++++++++ scripts/frames/extract_frames.py | 10 +++++++++- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/SKILL.md b/SKILL.md index 42a0ee5..25f8525 100644 --- a/SKILL.md +++ b/SKILL.md @@ -171,10 +171,22 @@ uv run --directory SKILL_DIR python scripts/index/manage_index.py check-range -- **Clip extraction** (keyframe-accurate, `-c copy`): +For frame extraction purposes, only the video stream is needed: + ```bash ffmpeg -hide_banner -y -ss START -to END -i VIDEO_PATH -c copy -map 0:v:0 CLIP_PATH ``` +For clips that the user may want to import into a video editor or play back, **preserve all audio tracks** by mapping all streams instead of only the video: + +```bash +ffmpeg -hide_banner -y -ss START -to END -i VIDEO_PATH -c copy -map 0 CLIP_PATH +``` + +`-map 0` copies all streams (video + all audio tracks + subtitles). The agent should ask the user whether they need a full-multitrack clip or a video-only clip for frame extraction. When in doubt, produce both: a video-only clip for fast frame extraction, and a full-multitrack clip if the user plans to use it in an editor. + +**Note:** When using `-c copy` with `-ss` before `-i`, seek accuracy is keyframe-level. If the user needs frame-exact cuts, re-encode is required (significantly slower). Keyframe accuracy is sufficient for planning purposes. + **Frame extraction** (scene detection + uniform sampling, hardware-accelerated, downsampled): ```bash @@ -315,6 +327,7 @@ This creates an isolated venv with `funasr`, `torch`, `torchaudio`. If the user 7. **Mixing funasr-script flags.** The bundled `funasr-nano`/`funasr-fast` use `--track STREAM_INDEX` and `--output-dir DIR`. Do not use the legacy `~/.local/bin/funasr-transcribe` wrapper's flags. 8. **Hardcoding tool names.** This skill is agent-agnostic. Do not assume specific MCP tools or APIs exist. Use whatever the runtime provides for vision analysis, background execution, and user notification. 9. **Hardcoding output language.** The edit plan table and all user-facing text must follow the user's language, not a fixed language. The English table in Step 8 is a template — translate columns and content to match the user's language at runtime. +10. **Dropping audio tracks during clip extraction.** The default clip command maps only `0:v:0` (video-only) for fast frame extraction. If the user plans to import the clip into a video editor, use `--all-streams` (or `-map 0`) to preserve all audio tracks. Always ask the user which they need. ## Verification checklist diff --git a/scripts/frames/extract_frames.py b/scripts/frames/extract_frames.py index 1ce988d..dfcfc50 100644 --- a/scripts/frames/extract_frames.py +++ b/scripts/frames/extract_frames.py @@ -65,13 +65,19 @@ def cmd_clip(args: argparse.Namespace) -> None: if not video.exists(): sys.exit(f"Video not found: {video}") + # Build stream mapping: video-only for frame extraction, or all streams for editor use + if args.all_streams: + stream_map = ["-map", "0"] # all streams (video + all audio + subtitles) + else: + stream_map = ["-map", "0:v:0"] # video only (faster, smaller for frame extraction) + cmd = [ "ffmpeg", "-hide_banner", "-y", "-ss", str(args.start), "-to", str(args.end), "-i", str(video), "-c", "copy", - "-map", "0:v:0", + *stream_map, str(output), ] @@ -192,6 +198,8 @@ def main() -> None: p_clip.add_argument("--start", required=True, type=float, help="Start time in seconds") p_clip.add_argument("--end", required=True, type=float, help="End time in seconds") p_clip.add_argument("--output", required=True, help="Output clip path") + p_clip.add_argument("--all-streams", action="store_true", default=False, + help="Preserve all streams (video + all audio tracks + subtitles). Default: video only (for frame extraction)") # frames p_frames = sub.add_parser("frames", help="Extract frames from a video/clip")