Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
ProgressPort,
)
from windup_ai_engine.postprocess import align_bottom_center, frame_durations
from windup_ai_engine.postprocess.pack import ANCHOR_CENTROID
from windup_ai_engine.prompt import PROMPT_VERSION
from windup_ai_engine.slicing import (
dead_frame_indices,
Expand All @@ -39,6 +40,7 @@
ROUTE_MATRIX,
DerivationStrategy,
is_cyclic,
vertical_anchor,
)

# ── 进度刻度:整条生产线只有一个 total ────────────────────────────────────────
Expand Down Expand Up @@ -183,8 +185,8 @@ def _finish(
"请调小 n_frames 或加长视频。"
)

# 最后一公里:脚线对齐成原地序列帧(直接对齐到调用方要的画布尺寸)
aligned = self._lastmile(frames, progress, canvas)
# 最后一公里:对齐成原地序列帧(直接对齐到调用方要的画布尺寸)
aligned = self._lastmile(frames, action, progress, canvas)

# 量交付成色。在**对齐之后**量,量的是用户真正会看到的那组帧:抠图 / 像素化 /
# 对齐都会改像素,在中间任何一步量出来的数都描述不了交付物。
Expand Down Expand Up @@ -225,10 +227,11 @@ def _assess(self, frames: list[Image.Image], action: ActionSpec) -> ActionQualit
def _lastmile(
self,
frames: list[bytes],
action: ActionSpec,
progress: ProgressPort,
canvas: tuple[int, int] | None = None,
) -> list[Image.Image]:
"""脚线对齐:把各帧对齐成原地序列帧(消除逐帧画布漂移,Issue #21)。
"""对齐成原地序列帧(消除逐帧画布漂移,Issue #21);锚点随动作有无地面接触

返回 PIL 而不是 PNG bytes:紧接着的成色测量要按图看帧,再编码回 PNG 只为了
让上一句话好听、下一句话又得解码回来。编码统一在 ``generate`` 出参那一步做。
Expand All @@ -241,7 +244,11 @@ def _lastmile(
原尺寸居中贴进 512 画布,于是这里刚对齐好的脚线 0.92 被挪到 0.709(2026-08-11
实测),角色不站在地上、跨动作对齐也失效。在这里一次出到位就没有那一步了。
"""
progress.step("lastmile", _TICK_LASTMILE, _TOTAL, "脚线对齐(原地)")
anchor = vertical_anchor(action)
progress.step(
"lastmile", _TICK_LASTMILE, _TOTAL,
f"{'质心' if anchor == ANCHOR_CENTROID else '脚线'}对齐(原地)",
)
# 空帧不再静默跳过:未实现的路线现在在 strategy / 装配表处就抛错(见 generate),
# 走到这里还有空帧说明 provider 或抠图吐了坏数据,同样要炸而不是原样放行。
if not frames:
Expand All @@ -260,6 +267,6 @@ def _lastmile(
# TODO(dev, #21): tail_match 循环闭合(净位移动作先锚点再匹配帧)
ref = float(np.median(hs)) if hs else None
if canvas is None:
return align_bottom_center(imgs, ref_height=ref)
return align_bottom_center(imgs, ref_height=ref, anchor=anchor)
cw, ch = canvas
return align_bottom_center(imgs, cell=cw, cell_h=ch, ref_height=ref)
return align_bottom_center(imgs, cell=cw, cell_h=ch, ref_height=ref, anchor=anchor)
60 changes: 48 additions & 12 deletions backend/packages/ai_engine/src/windup_ai_engine/postprocess/pack.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,10 @@

_logger = logging.getLogger(__name__)

__all__ = ["CELL", "CORE_THICKNESS", "DRIFT_CX_TOL", "DRIFT_FOOT_TOL",
"FILL_H", "FILL_W", "FOOT_LINE", "drifted_frames",
"align_bottom_center", "core_span", "sprite_sheet", "save_gif"]
__all__ = ["ANCHOR_CENTROID", "ANCHOR_FOOT", "CELL", "CORE_THICKNESS",
"DRIFT_CX_TOL", "DRIFT_FOOT_TOL", "FILL_H", "FILL_W", "FOOT_LINE",
"drifted_frames", "align_bottom_center", "core_span", "sprite_sheet",
"save_gif"]

# 交付画布的几何 —— 提成模块常量而不是只当默认参数,是因为**入口预检要按同一套几何
# 判母版能不能装下**(见 master_check.REJECT_ASPECT)。抄一份数字过去就等于埋下
Expand All @@ -29,6 +30,12 @@
# 0.25 之下是延展物(尾巴、翅膀、披风、举起的武器)—— 它们细,本体厚。
CORE_THICKNESS = 0.25

# 垂直对齐锚点。脚线那档对站在地上的动作成立;飞 / 游 / 攀全程没有地面接触,它们的包围盒
# 底边是尾羽与爪子、逐帧在变,钉死底边等于让身体跟着延展物上下浮动(#534)。
ANCHOR_FOOT = "foot"
ANCHOR_CENTROID = "centroid"
_ANCHORS = (ANCHOR_FOOT, ANCHOR_CENTROID)


def core_span(frame: Image.Image, thickness: float = CORE_THICKNESS) -> tuple[float, float] | None:
"""本体的 (高, 宽),单位=该帧像素。空帧返回 ``None``。
Expand All @@ -54,12 +61,9 @@ def core_span(frame: Image.Image, thickness: float = CORE_THICKNESS) -> tuple[fl
# · 它又让**大量列**只有 10px 高 → 列方向若以中位数为基准,中位数被压到 10、门槛低到
# 2,翅膀整条算进本体,量出的本体宽 209px(真值 49)。故列用 **max**。
# 判据不对称是数据形态决定的,不是漏了统一。
def span(counts: np.ndarray, base: float) -> float:
keep = np.flatnonzero(counts >= base * thickness)
return float(keep.max() - keep.min())

nz_rows = rows[rows > 0]
return (span(rows, float(np.median(nz_rows))), span(cols, float(cols.max())))
r0, r1 = _core_rows(m, thickness)
keep = np.flatnonzero(cols >= float(cols.max()) * thickness)
return (float(r1 - r0), float(keep.max() - keep.min()))


# 单调漂移的判定门槛(整段首尾相对变化)。低于它的不动 —— 真实身高起伏实测约 4%,
Expand Down Expand Up @@ -152,6 +156,7 @@ def align_bottom_center(
preserve_lift: bool = False,
ref_height: float | None = None,
cell_h: int | None = None,
anchor: str = ANCHOR_FOOT,
) -> list[Image.Image]:
"""按脚线对齐到统一画布,消除逐帧画布漂移(Issue #21)。

Expand All @@ -160,7 +165,7 @@ def align_bottom_center(
缩小。统一缩放后帧间只剩真实姿态差,尺度稳定。

水平方向按**主体水平中心**对齐(不含挥出的武器会更好,当前用整体包围盒中心兜底);
垂直方向按**脚线**(包围盒底边)对齐到 ``foot_line``。
垂直方向按 ``anchor`` 选锚点,默认脚线(包围盒底边)对齐到 ``foot_line``。

``ref_height``:**跨动作一致性的关键**,单位=传入帧的像素高。给定时按它定标,否则按本
序列最高帧。按最高帧定标会让"举过头顶"的动作整段被缩小去迁就那一帧 —— 实测攻击时
Expand All @@ -170,6 +175,10 @@ def align_bottom_center(
``preserve_lift``:腾空位移**默认不烘进像素**(业界:位移交引擎 root motion)。仅在要把
位移画进序列帧时才开;开启后以序列里最低的脚线为地面基准,保留每帧相对地面的抬升量。

``anchor``:``"foot"`` 给有地面接触的动作,``"centroid"`` 给飞 / 游 / 攀 —— 后者的底边
是尾羽与爪子、逐帧在变,钉死底边身体反而上下浮动(#534)。质心那档落在
``foot_line - fill_h/2``,即参考姿态包围盒的纵向中心,故换锚点不改变构图。

``cell``/``cell_h``:交付画布的宽与高,``cell_h=None`` 即方形 ``cell×cell``(默认,
行为与加这个参数之前逐像素相同)。**要能出非方形画布,是为了让引擎一次就出到项目
要的 sprite 尺寸、不必在上层再缩一次。** 上层那次二次缩放不是"糊一点"那么简单:
Expand All @@ -189,6 +198,14 @@ def align_bottom_center(
# 不静默出一张 0×0:PIL 允许建 0 边长的图,后面 alpha_composite 也不报错,
# 错产物要到落库/前端才暴露。
raise ValueError(f"交付画布尺寸必须为正,收到 cell={cell} cell_h={cell_h}")
if anchor not in _ANCHORS:
# 拼错一个字母不静默回落到脚线:那正好是本参数要修的那个错,而帧数 / 尺寸 / 成色
# 全部正常,要靠人看动图才发现。
raise ValueError(f"anchor 只能是 {_ANCHORS} 之一,收到 {anchor!r}")
if preserve_lift and anchor == ANCHOR_CENTROID:
# 质心对齐把每帧摆到同一条线上,抬升量随之被抹平 —— 两个都开等于 preserve_lift
# 是个空操作。
raise ValueError("preserve_lift 与 anchor='centroid' 互斥:质心对齐会抹平抬升量")

boxes: list[tuple[int, int, int, int] | None] = []
for f in frames:
Expand Down Expand Up @@ -277,10 +294,16 @@ def align_bottom_center(
fs = scale * per_frame[idx]
w = max(1, round(crop.width * fs))
h = max(1, round(crop.height * fs))
if anchor == ANCHOR_CENTROID:
# 锚点在缩放之前量:本体判据看的是原始像素,resize 的取整会把细延展物抹掉或加粗。
r0, r1 = _core_rows(np.asarray(crop)[:, :, 3] > 128)
top = round(ch * (foot_line - fill_h / 2) - (r0 + r1) / 2 * fs)
else:
lift = round((ground - box[3]) * fs) if preserve_lift else 0
top = int(ch * foot_line) - h - lift
crop = crop.resize((w, h), Image.NEAREST)
lift = round((ground - box[3]) * fs) if preserve_lift else 0
canvas = Image.new("RGBA", (cw, ch), (0, 0, 0, 0))
canvas.alpha_composite(crop, (cw // 2 - w // 2, int(ch * foot_line) - h - lift))
canvas.alpha_composite(crop, (cw // 2 - w // 2, top))
out.append(canvas)
return out

Expand Down Expand Up @@ -357,6 +380,19 @@ def _core_columns(mask) -> tuple[int, int] | None:
return int(keep.min()), int(keep.max())


def _core_rows(mask, thickness: float = CORE_THICKNESS) -> tuple[int, int]:
"""本体占据的首尾行;``mask`` 须有主体像素。

基准取行宽中位数而不是最厚那行,理由见 :func:`core_span` 里"延展物对行、列的污染方向
相反"那段 —— 与它共用一把尺子,免得本体的**位置**和**跨度**各按各的判据算。
"""
import numpy as np

rows = mask.sum(1)
keep = np.flatnonzero(rows >= float(np.median(rows[rows > 0])) * thickness)
return int(keep.min()), int(keep.max())


def _outliers(obs: list[tuple[int, float]], tol: float) -> set[int]:
"""按中位数判离群;整段有净位移时先把这条直线除掉。

Expand Down
13 changes: 13 additions & 0 deletions backend/packages/ai_engine/src/windup_ai_engine/strategy/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
from windup_common.models import ActionSpec, ActionType, CharacterCard, GenRoute

from windup_ai_engine.ports import ProgressPort
from windup_ai_engine.postprocess.pack import ANCHOR_CENTROID, ANCHOR_FOOT

# 动作类型 → 生成路线(架构决策,写死为契约)
ROUTE_MATRIX: dict[ActionType, GenRoute] = {
Expand Down Expand Up @@ -59,6 +60,18 @@ def is_cyclic(action: ActionSpec) -> bool:
return bool(action.cyclic)
return action.action in CYCLIC_ACTIONS


def vertical_anchor(action: ActionSpec) -> str:
"""这次生成按什么对齐垂直方向(见 postprocess.pack.align_bottom_center)。

**jump 不算无地面接触** —— 它腾空但要回地,按质心对齐会让落地帧悬在半空。全程无地面
的飞 / 游 / 攀只能以 CUSTOM 进来,故由调用方显式声明:按描述文字猜错是静默的(#534)。
"""
if action.action is ActionType.CUSTOM and action.ground_contact is False:
return ANCHOR_CENTROID
return ANCHOR_FOOT


# 本矩阵的形状本身有个已知边界,记录在此以免后来者按错误前提扩展:
# 它是「动作类型 → 路线」的一对一映射,隐含前提是"路线由动作的物理性质唯一决定"。
# 该前提对逐帧 / 视频两条路线成立(有无连续步态是动作固有属性),但对渲染出帧路线不成立
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -338,7 +338,14 @@ def _produce_action(
# 缺 loop 时兜成一次性,依据是失败代价不对称:一次性误当循环会让末帧接回首帧
# 抽搐、产物不可用;反之只是不无缝闭环、仍可用。不从描述文字猜。
cyclic = False if input.loop is None else bool(input.loop)
extra = {"custom_action": input.custom_prompt or "", "cyclic": cyclic}
# 缺 ground_contact 时兜成"有地面接触":绝大多数自述动作有脚踩地,而误判成
# 全程离地会让角色不站在地上 —— 比飞行动作上下浮动严重。
grounded = True if input.ground_contact is None else bool(input.ground_contact)
extra = {
"custom_action": input.custom_prompt or "",
"cyclic": cyclic,
"ground_contact": grounded,
}
action = ActionSpec(
action=engine_action,
poses=[""] * input.num_frames,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,10 @@ class CharacterActionInput:
# 这个动作是否循环播放。``None`` 原样往下传,由编排层兜成一次性:本层替调用方填默认值
# 的话,"没给"和"明确给了 False"从这里起就再也分不开了。
loop: bool | None = None
# 这个动作有没有地面接触。``None`` 原样往下传,由编排层兜成"有" —— 本层替调用方填默认
# 值的话,"没给"与"明确给了 True"从这里起就分不开了。飞 / 游 / 攀传 False,垂直对齐
# 改按躯干中心走(#534);jump 不属此列,它腾空但要回地。
ground_contact: bool | None = None
# 视频模型。``None`` = 用部署配置的默认值。取值域为
# ``ModelRegistry.chain(CHARACTER_ACTION)``(部署默认 + fallbacks);不在链上 → 入口
# 报错,不到付费调用才失败。选中的型号表示这次从它开始试,由 Gateway 读 start_from_model。
Expand Down
15 changes: 15 additions & 0 deletions backend/packages/app/src/windup_app/web/api/generation.py
Original file line number Diff line number Diff line change
Expand Up @@ -198,6 +198,10 @@ class CharacterActionGenerateRequest(BaseModel):
# 不对称:一次性动作被当成循环会让末帧接回首帧抽搐、产物不可用,反之只是不无缝闭环、
# 仍可用。而且猜错是静默的,帧数/时长/成色全部正常、没有任何一道会红。
loop: bool | None = None
# 这个动作有没有地面接触。飞 / 游 / 攀全程离地,它们的包围盒底边是尾羽与爪子、逐帧在变,
# 按脚线对齐反而让身体上下浮动(#534)。不给按"有"处理:误判成离地会让角色不站在地上,
# 比浮动严重。jump 不走这个字段 —— 它腾空但要回地。
ground_contact: bool | None = None
# 视频模型。None = 用部署默认。取值域见 ModelRegistry.chain(CHARACTER_ACTION);
# 非法值在入口就报错,不到付费调用才失败。选中的型号表示这次从它开始试。
video_model: str | None = None
Expand All @@ -220,6 +224,16 @@ def require_custom_prompt(self):
self.custom_prompt = prompt
return self

@model_validator(mode="after")
def ground_contact_belongs_to_custom_only(self):
# 写死的那几个动作都有地面接触,收下这个字段等于让调用方以为自己能改它。
if self.action_type is not ActionType.CUSTOM and self.ground_contact is not None:
raise ValueError(
f"action_type={self.action_type.value} 不该带 ground_contact:"
"它只对 custom 动作有意义,写死的动作都有地面接触"
)
return self

@model_validator(mode="after")
def drop_blank_reference_image_urls(self):
# 执行阶段只取 reference_image_urls[0]。留着空串会让"有母版"的判定成立,
Expand Down Expand Up @@ -425,6 +439,7 @@ def submit_action_generation(
action_type=body.action_type,
custom_prompt=body.custom_prompt,
loop=body.loop,
ground_contact=body.ground_contact,
video_model=body.video_model,
reference_video_url=body.reference_video_url,
reference_image_urls=body.reference_image_urls,
Expand Down
1 change: 1 addition & 0 deletions backend/packages/app/src/windup_app/worker/handlers.py
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,7 @@ def _action_input(payload: dict) -> CharacterActionInput:
reference_image_urls=list(payload.get("reference_image_urls") or []),
num_frames=int(payload.get("num_frames") or 16),
loop=payload.get("loop"),
ground_contact=payload.get("ground_contact"),
video_model=payload.get("video_model"),
outfit_id=payload.get("outfit_id"),
model_3d_url=payload.get("model_3d_url"),
Expand Down
11 changes: 11 additions & 0 deletions backend/packages/common/src/windup_common/models/character.py
Original file line number Diff line number Diff line change
Expand Up @@ -241,6 +241,12 @@ class ActionSpec(BaseModel):
# 这里不给默认值,否则同一个缺省被两处各写一份,改一处另一处静默不动。
archetype: AttackArchetype | None = None

# 这个动作有没有地面接触。``False`` = 飞 / 游 / 攀这类**全程离地**,垂直对齐改按躯干
# 中心走(见 ai_engine.strategy.base.vertical_anchor);jump 不属此列 —— 它腾空但要回地。
# 不给按有地面接触处理:代价不对称,把有脚的动作当离地会让角色不站在地上,而反过来
# 只是身体跟着尾羽上下浮动、产物仍可用。
ground_contact: bool | None = None

@model_validator(mode="after")
def _archetype_belongs_to_attack_only(self) -> ActionSpec:
"""非攻击动作带 archetype 要炸:它只被攻击提示词消费,传了不会生效。"""
Expand Down Expand Up @@ -275,6 +281,11 @@ def _custom_needs_its_own_settings(self) -> ActionSpec:
f"action={self.action.value} 不该带 cyclic;循环性由 CYCLIC_ACTIONS 写死,"
"传了不会生效"
)
if self.ground_contact is not None:
raise ValueError(
f"action={self.action.value} 不该带 ground_contact;写死的那几个动作都有"
"地面接触(jump 腾空但要回地),传了不会生效"
)
return self

# 这里**没有** ``route`` 字段:路线选择整个在 server —— 走不走三渲二取决于"这个造型
Expand Down
51 changes: 51 additions & 0 deletions backend/tests/test_generation_api.py
Original file line number Diff line number Diff line change
Expand Up @@ -467,6 +467,57 @@ def test_stance_omitted_stays_none_not_biped(auth_client, monkeypatch):
assert inputs and inputs[0].stance is None


def test_ground_contact_from_request_reaches_the_task_input(auth_client, monkeypatch):
"""飞 / 游 / 攀的声明断在请求层的话,对齐那一步永远走脚线分支(#534)。"""
from windup_app.web.api import generation as gen_api
from windup_app.server.orchestrator.model import CharacterActionInput

dispatched = _capture_action_input(gen_api, monkeypatch)
project = _create_project(auth_client)
character = _create_character(auth_client, project["id"])

auth_client.post(
"/generation/action",
json=_action_payload(
project["id"], character["id"],
action_type="custom", custom_prompt="flies forward", ground_contact=False,
),
)

inputs = [a for args in dispatched for a in args if isinstance(a, CharacterActionInput)]
assert inputs, "任务没被收下"
assert inputs[0].ground_contact is False


def test_ground_contact_omitted_stays_none_not_true(auth_client, monkeypatch):
"""不给就原样传 None —— 在这层填默认值,"没给"与"明确有地面接触"就分不开了。"""
from windup_app.web.api import generation as gen_api
from windup_app.server.orchestrator.model import CharacterActionInput

dispatched = _capture_action_input(gen_api, monkeypatch)
project = _create_project(auth_client)
character = _create_character(auth_client, project["id"])

auth_client.post(
"/generation/action", json=_action_payload(project["id"], character["id"]),
)

inputs = [a for args in dispatched for a in args if isinstance(a, CharacterActionInput)]
assert inputs and inputs[0].ground_contact is None


def test_ground_contact_on_a_fixed_action_is_rejected_at_the_entrance(auth_client):
"""walk 收下这个字段等于让调用方以为自己能改它,而它不会生效。"""
project = _create_project(auth_client)
character = _create_character(auth_client, project["id"])
body = auth_client.post(
"/generation/action",
json=_action_payload(project["id"], character["id"], ground_contact=False),
).json()
assert body["code"] == 400, body
assert "ground_contact" in body["message"], body["message"]


def test_illegal_stance_is_rejected_at_the_entrance(auth_client):
project = _create_project(auth_client)
character = _create_character(auth_client, project["id"])
Expand Down
Loading
Loading