Skip to content

Commit f01ae1f

Browse files
dkackmanclaude
andcommitted
Merge feat/471-a-h3-latent-upscale-rebuild: #499 H3 latent upscaler, relanded with the node's normalization
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2 parents 9c708a1 + 924a973 commit f01ae1f

11 files changed

Lines changed: 1713 additions & 20 deletions

‎docs/WORKFLOW_GUIDE.md‎

Lines changed: 228 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1587,6 +1587,234 @@ it — see [workflows/templates/minimax/last-frame-only.json](../workflows/templ
15871587
See [workflows/templates/minimax/music.json](../workflows/templates/minimax/music.json) and
15881588
[workflows/templates/minimax/video-with-audio.json](../workflows/templates/minimax/video-with-audio.json) for full examples.
15891589

1590+
### Promoting an H3 take to 768p in latent space: upscale_h3_latents and decode_h3_latents
1591+
1592+
Once a 960x544 MiniMax-H3 take reads the way it should, `upscale_h3_latents` and
1593+
`decode_h3_latents` promote it to 1344x768 without denoising it again - cheaper than a
1594+
native 768p render, since only a small 3D-convolution network and a VAE decode run,
1595+
not the transformer. There is no refine pass over the upscaled latents, so the result
1596+
is sharper than the 544p take but cannot show detail the base pass never generated.
1597+
This is a measurement path, not a catalog template: the catalog's native 768p render is
1598+
`templates/minimax/video-with-audio-768p`.
1599+
1600+
Target `width`/`height` must be multiples of 16, each between 1x and 4x the base
1601+
latents' own size, and within H3's 1344x768 (or portrait 768x1344) canvas. The latents
1602+
pass from `base` to `up` to `decode` entirely in memory. A `previous_result:` property
1603+
on a modular step names a key of the dict its `output` list returns, spelled as the
1604+
pipeline spells it: `base.latents`, `base.audio` and `base.sampling_rate` - not
1605+
`sample_rate`, which is what the saved AudioVideo calls it, and which the dict does
1606+
not carry. A property no result carries is an error rather than an empty list, so a
1607+
misspelt one fails the step instead of skipping the mux that reads it. The base
1608+
step carries no `result` at all, so nothing beyond its return value is written -
1609+
upscaling only makes sense for a take chosen from something already reviewed, so the
1610+
544p pass that produced it is not itself a deliverable here.
1611+
1612+
The upscaler weights (~691 MB, `LBH-123-AI/Minimax_h3_latent_Upscaler`, MIT-licensed,
1613+
read at a pinned revision) and the H3 VAE download on first use, the same as any other
1614+
model. Neither is counted in `plan.downloads_required`, since that walk collects
1615+
`from_pretrained_arguments` sources on pipeline steps and does not see a task
1616+
argument naming a Hugging Face repo - a box that has run the base pass before but never
1617+
this task can still stall mid-run pulling the upscaler.
1618+
1619+
```json
1620+
{
1621+
"id": "H3LatentUpscalePreview",
1622+
"description": "Promote a MiniMax-H3 take from 960x544 to 1344x768 in latent space, no refine pass.",
1623+
"variables": {
1624+
"prompt": "prompt:minimax/fox_dawn_context_ir",
1625+
"num_frames": 124,
1626+
"num_inference_steps": 9,
1627+
"video_shift": 12.0,
1628+
"audio_shift": 3.0,
1629+
"weights_dtype": "{int4}",
1630+
"lora_scale": 1.0,
1631+
"lora_alpha": null,
1632+
"lora_model_name": "lightx2v/Minimax-h3-Turbo",
1633+
"lora_weight_name": "minimax_h3_fl2v_turbo_8step_v1.0_bf16.safetensors",
1634+
"lora_adapter_name": "turbo",
1635+
"seed": 42
1636+
},
1637+
"seed": "variable:seed",
1638+
"steps": [
1639+
{
1640+
"name": "base",
1641+
"pipeline": {
1642+
"configuration": {
1643+
"component_type": "ModularPipeline",
1644+
"pre_load_modules": [
1645+
"sdnq"
1646+
],
1647+
"load_components": {
1648+
"dtype": "torch.bfloat16",
1649+
"quantization_config": {
1650+
"transformer": {
1651+
"configuration": {
1652+
"config_type": "sdnq.SDNQConfig"
1653+
},
1654+
"arguments": {
1655+
"weights_dtype": "variable:weights_dtype",
1656+
"quantization_device": "cuda",
1657+
"return_device": "cpu",
1658+
"use_quantized_matmul": true,
1659+
"dequantize_fp32": false,
1660+
"modules_to_not_convert": [
1661+
"proj_in",
1662+
"audio_proj_in",
1663+
"context_embedder",
1664+
"time_embedder",
1665+
"time_proj",
1666+
"token_refiner",
1667+
"norm_out",
1668+
"proj_out",
1669+
"audio_proj_out"
1670+
]
1671+
}
1672+
},
1673+
"text_encoder": {
1674+
"configuration": {
1675+
"config_type": "sdnq.SDNQConfig"
1676+
},
1677+
"arguments": {
1678+
"weights_dtype": "variable:weights_dtype",
1679+
"quantization_device": "cuda",
1680+
"return_device": "cpu",
1681+
"dequantize_fp32": false,
1682+
"modules_to_not_convert": [
1683+
".model.visual",
1684+
"lm_head"
1685+
]
1686+
}
1687+
},
1688+
"vae": {
1689+
"configuration": {
1690+
"config_type": "sdnq.SDNQConfig"
1691+
},
1692+
"arguments": {
1693+
"weights_dtype": "{int8}",
1694+
"quant_conv": true,
1695+
"use_quantized_matmul_conv": true,
1696+
"quantization_device": "cuda",
1697+
"return_device": "cpu",
1698+
"dequantize_fp32": false
1699+
}
1700+
}
1701+
}
1702+
},
1703+
"components": {
1704+
"transformer": {
1705+
"group_offload": {
1706+
"offload_type": "block_level",
1707+
"num_blocks_per_group": 2,
1708+
"use_stream": true,
1709+
"record_stream": true,
1710+
"low_cpu_mem_usage": true
1711+
}
1712+
},
1713+
"text_encoder": {
1714+
"remove_modules": [
1715+
"lm_head"
1716+
]
1717+
},
1718+
"text_encoder.model": {
1719+
"truncate_layers": {
1720+
"language_model.layers": 51
1721+
},
1722+
"group_offload": {
1723+
"offload_type": "leaf_level"
1724+
}
1725+
},
1726+
"vae": {
1727+
"device": "cuda",
1728+
"residency": "on_demand"
1729+
},
1730+
"audio_vae": {
1731+
"device": "cuda",
1732+
"residency": "on_demand"
1733+
}
1734+
}
1735+
},
1736+
"from_pretrained_arguments": {
1737+
"model_name": "MiniMaxAI/MiniMax-H3",
1738+
"workflow": "t2va"
1739+
},
1740+
"loras": [
1741+
{
1742+
"model_name": "variable:lora_model_name",
1743+
"weight_name": "variable:lora_weight_name",
1744+
"adapter_name": "variable:lora_adapter_name",
1745+
"scale": "variable:lora_scale",
1746+
"alpha": "variable:lora_alpha"
1747+
}
1748+
],
1749+
"scheduler": {
1750+
"shift": "variable:video_shift"
1751+
},
1752+
"audio_scheduler": {
1753+
"shift": "variable:audio_shift"
1754+
},
1755+
"arguments": {
1756+
"prompt": "variable:prompt",
1757+
"num_frames": "variable:num_frames",
1758+
"width": 960,
1759+
"height": 544,
1760+
"num_inference_steps": "variable:num_inference_steps",
1761+
"output": [
1762+
"videos",
1763+
"audio",
1764+
"sampling_rate",
1765+
"latents"
1766+
]
1767+
}
1768+
}
1769+
},
1770+
{
1771+
"name": "up",
1772+
"task": {
1773+
"command": "upscale_h3_latents",
1774+
"arguments": {
1775+
"latents": "previous_result:base.latents",
1776+
"width": 1344,
1777+
"height": 768
1778+
}
1779+
}
1780+
},
1781+
{
1782+
"name": "decode",
1783+
"task": {
1784+
"command": "decode_h3_latents",
1785+
"arguments": {
1786+
"latents": "previous_result:up"
1787+
}
1788+
}
1789+
},
1790+
{
1791+
"name": "mux",
1792+
"task": {
1793+
"command": "pair_audio",
1794+
"arguments": {
1795+
"video": "previous_result:decode",
1796+
"audio": "previous_result:base.audio",
1797+
"sample_rate": "previous_result:base.sampling_rate",
1798+
"fit": "video"
1799+
}
1800+
},
1801+
"result": {
1802+
"content_type": "video/mp4",
1803+
"subfolder": "final"
1804+
}
1805+
}
1806+
]
1807+
}
1808+
```
1809+
1810+
To keep the 544p take beside the promotion for comparison, add one more `pair_audio`
1811+
step with `"video": "previous_result:base.videos"` and the same `audio`/`sample_rate`,
1812+
saved with `"result": {"content_type": "video/mp4", "fps": 24, "subfolder":
1813+
"intermediate"}`. `base.videos` is the pipeline's batch - a list holding the one video -
1814+
and `pair_audio` unwraps a batch of one (a batch of several is refused, since one track
1815+
goes under one video). The dict's frames carry no frame rate, so the result's `fps`
1816+
says it; without it the file is written at the 8 fps fallback.
1817+
15901818
### Chained Video Generation
15911819

15921820
Video pipelines generate short clips - a `chain` block on a pipeline step runs the

‎dw/locations.py‎

Lines changed: 53 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -469,27 +469,29 @@ def validate_model_name(name, base_dir=None):
469469
# relative path and nothing else
470470
WEIGHT_NAME_KEY = "weight_name"
471471

472-
# Tasks that read their weights with safetensors and nothing else. A pickle
473-
# format is refused by name rather than left to fail in the loader: the
474-
# refusal is the documented contract, not an accident of which loader runs
475-
SAFETENSORS_ONLY_COMMANDS = ()
472+
# Tasks that read their weights with safetensors and nothing else, from a file
473+
# named bare (no subfolder: the task knows its default repo's layout). A
474+
# pickle format is refused by name rather than left to fail in the loader:
475+
# the refusal is the documented contract, not an accident of which loader runs
476+
SAFETENSORS_ONLY_COMMANDS = ("upscale_h3_latents",)
476477
SAFETENSORS_SUFFIX = ".safetensors"
477478

478479

479-
def validate_weight_name(name, suffixes=None, what="weight_name"):
480+
def validate_weight_name(name, suffixes=None, what="weight_name", subfolders=True):
480481
"""A file name inside a Hub repo: relative, and never climbing out of it.
481482
482483
`hf_hub_download(filename=...)` joins the name onto the local cache
483484
directory, so a name shaped like a path is a path on this machine. A
484-
subfolder is legitimate (the H3 upscaler's default lives under one);
485-
an absolute path, a backslash, a drive, an empty segment or a '.'/'..'
485+
subfolder is legitimate where `subfolders` allows one (an IP adapter's
486+
lives under one); an absolute path, a backslash, a drive, an empty segment or a '.'/'..'
486487
segment is not. The refusal names only what the caller wrote, never a
487488
directory on the server.
488489
489490
Args:
490491
name: The file name the workflow supplied
491492
suffixes: The file endings allowed, or None for any
492493
what: Short phrase naming the argument, for the error message
494+
subfolders: Whether the name may hold a '/' at all
493495
494496
Raises:
495497
PathTraversalError: On a name that is not a plain relative path
@@ -510,6 +512,10 @@ def validate_weight_name(name, suffixes=None, what="weight_name"):
510512
f"Refusing a {what} of '{name}': it has an empty, '.' or '..' "
511513
f"path segment, so it does not name a file inside the model repo."
512514
)
515+
if not subfolders and "/" in name:
516+
raise PathTraversalError(
517+
f"Refusing a {what} of '{name}': it is a file name, not a path - no '/'."
518+
)
513519
if suffixes and not name.lower().endswith(tuple(suffixes)):
514520
raise InvalidInputError(
515521
f"Refusing a {what} of '{name}': only "
@@ -583,31 +589,40 @@ def location_errors(definition, source_indices=None, base_dir=None):
583589
if not isinstance(step, dict):
584590
continue
585591
source = references.author_index(source_indices, index)
586-
_walk(step, f"steps[{source}]", base_dir, errors, _weight_suffixes(step))
592+
_walk(step, f"steps[{source}]", base_dir, errors, _weight_rules(step))
587593
return errors
588594

589595

590-
def _weight_suffixes(step):
591-
"""The weight file endings a step's task reads, or None for any."""
596+
def _weight_rules(step):
597+
"""validate_weight_name's (suffixes, subfolders) for a step's task, and
598+
whether its model_name must be a Hub repo id (a task that downloads its
599+
weights reads no local directory, so one is refused here, not at run)."""
592600
task = step.get("task")
593601
command = task.get("command") if isinstance(task, dict) else None
594602
if command in SAFETENSORS_ONLY_COMMANDS:
595-
return (SAFETENSORS_SUFFIX,)
596-
return None
603+
return (SAFETENSORS_SUFFIX,), False, True
604+
return None, True, False
597605

598606

599-
def _walk(node, path, base_dir, errors, weight_suffixes=None):
607+
def _walk(node, path, base_dir, errors, weight_rules=(None, True, False)):
600608
if isinstance(node, dict):
601609
for key, value in node.items():
602610
here = f"{path}.{key}"
603611
if key == "model_name" and isinstance(value, str):
604-
message = _model_name_message(value, base_dir)
612+
message = _model_name_message(value, base_dir, weight_rules[2])
605613
if message:
606614
errors.append({"path": here, "message": message})
607615
continue
608616
if key == WEIGHT_NAME_KEY and isinstance(value, str):
609617
if not _deferred(value):
610-
message = _refusal(validate_weight_name, value, weight_suffixes)
618+
suffixes, subfolders, _ = weight_rules
619+
message = _refusal(
620+
validate_weight_name,
621+
value,
622+
suffixes,
623+
WEIGHT_NAME_KEY,
624+
subfolders,
625+
)
611626
if message:
612627
errors.append({"path": here, "message": message})
613628
continue
@@ -642,10 +657,10 @@ def _walk(node, path, base_dir, errors, weight_suffixes=None):
642657
message = _check(item, base_dir, f"'{key}'")
643658
if message:
644659
errors.append({"path": sub_path, "message": message})
645-
_walk(value, here, base_dir, errors, weight_suffixes)
660+
_walk(value, here, base_dir, errors, weight_rules)
646661
elif isinstance(node, list):
647662
for index, item in enumerate(node):
648-
_walk(item, f"{path}[{index}]", base_dir, errors, weight_suffixes)
663+
_walk(item, f"{path}[{index}]", base_dir, errors, weight_rules)
649664

650665

651666
def _each(value, path):
@@ -660,12 +675,32 @@ def _glob_message(pattern, base_dir):
660675
return _refusal(validate_media_glob, pattern, base_dir)
661676

662677

663-
def _model_name_message(name, base_dir):
678+
def _model_name_message(name, base_dir, hub_only=False):
664679
if _deferred(name):
665680
return None
681+
if hub_only:
682+
return _refusal(validate_hub_repo_id, name)
666683
return _refusal(validate_model_name, name, base_dir)
667684

668685

686+
def validate_hub_repo_id(name, what="model_name"):
687+
"""A Hugging Face repo id and nothing else - for a task that downloads.
688+
689+
Raises:
690+
InvalidInputError: On a path, a URL, or any other non-repo-id
691+
"""
692+
from huggingface_hub.utils import HFValidationError, validate_repo_id
693+
694+
try:
695+
validate_repo_id(str(name))
696+
except HFValidationError:
697+
raise InvalidInputError(
698+
f"Refusing a {what} of '{name}': this task downloads its weights, "
699+
f"so it takes a Hugging Face repo id ('owner/name'), not a path."
700+
) from None
701+
return str(name)
702+
703+
669704
def _refusal(check, *args):
670705
"""The message `check` refused its argument with, or None if it allowed it."""
671706
try:

‎dw/task_domains.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -150,6 +150,7 @@
150150
"fps": POSITIVE,
151151
},
152152
"loop_frames": {"num_frames": POSITIVE},
153+
"upscale_h3_latents": {"width": POSITIVE, "height": POSITIVE},
153154
"frame_grid": {"count": POSITIVE, "columns": POSITIVE, "tile_width": POSITIVE},
154155
"dissolve_videos": {
155156
"dissolve_frames": NON_NEGATIVE,

0 commit comments

Comments
 (0)