feat(provider): support native video input modality

astrbot/core/agent/message.py

@@ -21,7 +21,7 @@ class ContentPart(BaseModel):
 
     __content_part_registry: ClassVar[dict[str, type["ContentPart"]]] = {}
 
-    type: Literal["text", "think", "image_url", "audio_url"]
+    type: Literal["text", "think", "image_url", "audio_url", "video_url"]
     _no_save: bool = PrivateAttr(default=False)
 
     def __init_subclass__(cls, **kwargs: Any) -> None:
@@ -141,6 +141,22 @@ class AudioURL(BaseModel):
     audio_url: AudioURL
 
 
+class VideoURLPart(ContentPart):
+    """
+    >>> VideoURLPart(video_url=VideoURLPart.VideoURL(url="https://example.com/clip.mp4")).model_dump()
+    {'type': 'video_url', 'video_url': {'url': 'https://example.com/clip.mp4', 'id': None}}
+    """
+
+    class VideoURL(BaseModel):
+        url: str
+        """The URL of the video, can be data URI scheme like `data:video/mp4;base64,...`."""
+        id: str | None = None
+        """The ID of the video, to allow LLMs to distinguish different videos."""
+
+    type: str = "video_url"
+    video_url: VideoURL
+
+
 class ToolCall(BaseModel):
     """
     A tool call requested by the assistant.

astrbot/core/agent/runners/tool_loop_agent_runner.py

@@ -42,6 +42,7 @@
     ToolCallsResult,
 )
 from astrbot.core.provider.modalities import (
+    has_video_blocks,
     log_context_sanitize_stats,
     sanitize_contexts_by_modalities,
 )
@@ -317,6 +318,7 @@ async def reset(
             request.prompt is not None
             or request.image_urls
             or request.audio_urls
+            or request.video_urls
             or request.extra_user_content_parts
         ):
             m = await self._assemble_request_context_for_provider(request)
@@ -342,17 +344,26 @@ async def _assemble_request_context_for_provider(
     ) -> dict[str, T.Any]:
         modalities = self.provider.provider_config.get("modalities", None)
         if not modalities:  # Unconfigured (None or empty list) defaults to support all modalities for backward compatibility
-            return await request.assemble_context()
+            # Video is excluded from that default. It postdates the "supports
+            # everything" convention and is not universally supported, so a
+            # provider that never opted in would fail the whole request rather
+            # than degrade. Video is therefore strictly opt-in via the
+            # `modalities` list.
+            if not request.video_urls:
+                return await request.assemble_context()
+            return await replace(request, video_urls=[]).assemble_context()
 
         supports_image = "image" in modalities
         supports_audio = "audio" in modalities
-        if supports_image and supports_audio:
+        supports_video = "video" in modalities
+        if supports_image and supports_audio and supports_video:
             return await request.assemble_context()
 
         adjusted_request = replace(
             request,
             image_urls=request.image_urls if supports_image else [],
             audio_urls=request.audio_urls if supports_audio else [],
+            video_urls=request.video_urls if supports_video else [],
         )
         context = await adjusted_request.assemble_context()
         content = context.get("content")
@@ -369,6 +380,9 @@ async def _assemble_request_context_for_provider(
         if not supports_audio:
             for _ in request.audio_urls:
                 content_blocks.append({"type": "text", "text": "[Audio]"})
+        if not supports_video:
+            for _ in request.video_urls:
+                content_blocks.append({"type": "text", "text": "[Video]"})
 
         return {"role": "user", "content": content_blocks}
 
@@ -663,13 +677,17 @@ def _sanitize_contexts_for_provider(
         contexts: list[Message] | list[dict[str, T.Any]],
     ) -> list[Message] | list[dict[str, T.Any]]:
         modalities = self.provider.provider_config.get("modalities", None)
-        if (
-            not modalities
-        ):  # Unconfigured (None or empty list) defaults to support all modalities
+        if not modalities and not has_video_blocks(contexts):
+            # Unconfigured (None or empty list) defaults to supporting every
+            # modality, so the context objects are handed back untouched. Video
+            # is the exception: it is strictly opt-in (see
+            # `_assemble_request_context_for_provider`), and this path also runs
+            # when a request falls back to another provider, whose context was
+            # assembled for the primary one and may already carry a `video_url`.
             return contexts
         sanitized_contexts, stats = sanitize_contexts_by_modalities(
             contexts,
-            self.provider.provider_config.get("modalities", None),
+            modalities,
         )
         log_context_sanitize_stats(stats)
         return sanitized_contexts

astrbot/core/astr_main_agent.py

@@ -877,6 +877,10 @@ async def _append_video_attachment(
     else:
         text = f"[Video Attachment: name {video_name}, path {video_path}]"
 
+    # Collect the resolved path so providers that declare the `video` modality
+    # receive the video itself. The text part above is kept for every other
+    # provider, and for the persisted history where the video block is dropped.
+    req.video_urls.append(video_path)
     req.extra_user_content_parts.append(TextPart(text=text))
 
 
@@ -1821,6 +1825,7 @@ async def build_main_agent(
         )
     req.image_urls = normalize_and_dedupe_strings(req.image_urls)
     req.audio_urls = normalize_and_dedupe_strings(req.audio_urls)
+    req.video_urls = normalize_and_dedupe_strings(req.video_urls)
 
     if config.file_extract_enabled:
         try:

astrbot/core/config/default.py

@@ -2247,8 +2247,8 @@ def get_local_permission_defaults(system: str | None = None) -> dict:
                         "description": "模型能力",
                         "type": "list",
                         "items": {"type": "string"},
-                        "options": ["text", "image", "audio", "tool_use"],
-                        "labels": ["文本", "图像", "音频", "工具使用"],
+                        "options": ["text", "image", "audio", "video", "tool_use"],
+                        "labels": ["文本", "图像", "音频", "视频", "工具使用"],
                         "render_type": "checkbox",
                         "hint": "模型支持的模态及能力。",
                     },

astrbot/core/provider/entities.py

@@ -101,6 +101,8 @@ class ProviderRequest:
     """图片 URL 列表"""
     audio_urls: list[str] = field(default_factory=list)
     """音频 URL 列表，也支持本地路径"""
+    video_urls: list[str] = field(default_factory=list)
+    """Video URL list. Local paths are supported as well."""
     extra_user_content_parts: list[ContentPart] = field(default_factory=list)
     """额外的用户消息内容部分列表，用于在用户消息后添加额外的内容块（如系统提醒、指令等）。支持 dict 或 ContentPart 对象"""
     func_tool: ToolSet | None = None
@@ -124,6 +126,7 @@ def __repr__(self) -> str:
             f"ProviderRequest(prompt={self.prompt}, session_id={self.session_id}, "
             f"image_count={len(self.image_urls or [])}, "
             f"audio_count={len(self.audio_urls or [])}, "
+            f"video_count={len(self.video_urls or [])}, "
             f"func_tool={self.func_tool}, "
             f"contexts={self._print_friendly_context()}, "
             f"system_prompt={self.system_prompt}, "
@@ -146,7 +149,8 @@ def _print_friendly_context(self):
         if not self.contexts:
             return (
                 f"prompt: {self.prompt}, image_count: {len(self.image_urls or [])}, "
-                f"audio_count: {len(self.audio_urls or [])}"
+                f"audio_count: {len(self.audio_urls or [])}, "
+                f"video_count: {len(self.video_urls or [])}"
             )
 
         result_parts = []
@@ -163,6 +167,7 @@ def _print_friendly_context(self):
                 msg_parts = []
                 image_count = 0
                 audio_count = 0
+                video_count = 0
 
                 for item in content:
                     item_type = item.get("type", "")
@@ -173,6 +178,8 @@ def _print_friendly_context(self):
                         image_count += 1
                     elif item_type == "audio_url":
                         audio_count += 1
+                    elif item_type == "video_url":
+                        video_count += 1
 
                 if image_count > 0:
                     if msg_parts:
@@ -184,6 +191,11 @@ def _print_friendly_context(self):
                         msg_parts.append(f"[+{audio_count} audios]")
                     else:
                         msg_parts.append(f"[{audio_count} audios]")
+                if video_count > 0:
+                    if msg_parts:
+                        msg_parts.append(f"[+{video_count} videos]")
+                    else:
+                        msg_parts.append(f"[{video_count} videos]")
 
                 result_parts.append(f"{role}: {''.join(msg_parts)}")
 
@@ -198,6 +210,7 @@ async def assemble_context(self) -> dict:
         # 构建内容块列表
         content_blocks = []
         image_capture_failed = False
+        transient_video_added = False
 
         # 1. 用户原始发言（OpenAI 建议：用户发言在前）
         if self.prompt and self.prompt.strip():
@@ -208,6 +221,9 @@ async def assemble_context(self) -> dict:
         elif self.audio_urls:
             # 如果没有文本但有音频，添加占位文本
             content_blocks.append({"type": "text", "text": "[音频]"})
+        elif self.video_urls:
+            # If there is no text but there is video, add a placeholder text.
+            content_blocks.append({"type": "text", "text": "[Video]"})
 
         # 2. 额外的内容块（系统提醒、指令等）
         if self.extra_user_content_parts:
@@ -245,6 +261,40 @@ async def assemble_context(self) -> dict:
                                 "url": resolved.to_data_url(),
                             },
                         }
+                # A video supplied through `extra_user_content_parts` is handled
+                # like `video_urls` below: resolve it to a portable data URI and
+                # flag it transient. Persisting it would replay the whole payload
+                # on every later request, and a local path may be gone by then.
+                if isinstance(dumped, dict) and dumped.get("type") == "video_url":
+                    video_url = dumped.get("video_url")
+                    url = video_url.get("url") if isinstance(video_url, dict) else None
+                    if isinstance(url, str) and url:
+                        try:
+                            resolved_video = await MediaResolver(
+                                url,
+                                media_type="video",
+                                default_suffix=".mp4",
+                            ).to_base64_data(strict=True)
+                        except Exception as exc:
+                            logger.warning(
+                                "Video preprocessing failed; skipping it. Error: %s",
+                                exc,
+                            )
+                            continue
+                        if not resolved_video:
+                            logger.warning(
+                                "Video preprocessing returned no data; skipping it."
+                            )
+                            continue
+                        dumped = {
+                            **dumped,
+                            "video_url": {
+                                **video_url,
+                                "url": resolved_video.to_data_url(),
+                            },
+                            "_no_save": True,
+                        }
+                        transient_video_added = True
                 content_blocks.append(dumped)
 
         # 3. Read image references without resizing or transcoding.
@@ -299,19 +349,57 @@ async def assemble_context(self) -> dict:
                     },
                 )
 
+        # 5. Video content. The block is flagged `_no_save` so that the
+        # multi-megabyte data URI is dropped when the message is written back to
+        # the conversation history; otherwise it would be replayed on every later
+        # request in the session. Providers still receive it, because they
+        # serialize messages with `model_dump()`, which does not emit `_no_save`.
+        if self.video_urls:
+            for video_url in self.video_urls:
+                try:
+                    video_data = await MediaResolver(
+                        video_url,
+                        media_type="video",
+                        default_suffix=".mp4",
+                    ).to_base64_data(strict=True)
+                except Exception as exc:
+                    logger.warning(
+                        "Video preprocessing failed; skipping it. Error: %s", exc
+                    )
+                    continue
+                if not video_data:
+                    logger.warning("Video preprocessing returned no data; skipping it.")
+                    continue
+                content_blocks.append(
+                    {
+                        "type": "video_url",
+                        "video_url": {"url": video_data.to_data_url()},
+                        "_no_save": True,
+                    },
+                )
+
         if image_capture_failed and not any(
             block.get("type") != "text" or block.get("text", "").strip()
             for block in content_blocks
         ):
             content_blocks = [{"type": "text", "text": "[Image unavailable]"}]
 
+        # A message made only of transient blocks would be persisted with an
+        # empty content list and replayed that way on every later request, so
+        # keep a text placeholder for the attachment.
+        if transient_video_added and not any(
+            not block.get("_no_save") for block in content_blocks
+        ):
+            content_blocks.insert(0, {"type": "text", "text": "[Video]"})
+
         # 只有当只有一个来自 prompt 的文本块且没有额外内容块时，才降级为简单格式以保持向后兼容
         if (
             len(content_blocks) == 1
             and content_blocks[0]["type"] == "text"
             and not self.extra_user_content_parts
             and not self.image_urls
             and not self.audio_urls
+            and not self.video_urls
         ):
             return {"role": "user", "content": content_blocks[0]["text"]}