From 73aa966d7853bbfc514b4afc3221369dc0f7f9d0 Mon Sep 17 00:00:00 2001 From: Urvang Joshi Date: Fri, 14 Aug 2026 10:58:05 -0700 Subject: [PATCH 1/2] Use encoder-only method for reduced-tx-part-set option - Always enable all tx types at sequence level - Disable 4way/5way tx types only during RDO Fixes https://github.com/AOMediaCodec/av2-spec-internal/issues/1262 using encoder-only method. --- av2/encoder/encoder.c | 2 +- av2/encoder/tx_search.c | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/av2/encoder/encoder.c b/av2/encoder/encoder.c index df551c3cfc..41e0e3d67a 100644 --- a/av2/encoder/encoder.c +++ b/av2/encoder/encoder.c @@ -577,7 +577,7 @@ void av2_init_seq_coding_tools(AV2_COMP *cpi, SequenceHeader *seq, seq->enable_mhccp = oxcf->intra_mode_cfg.enable_mhccp; seq->enable_inter_ddt = seq->single_picture_header_flag ? 0 : oxcf->txfm_cfg.enable_inter_ddt; - seq->reduced_tx_part_set = oxcf->txfm_cfg.reduced_tx_part_set; + seq->reduced_tx_part_set = 0; seq->enable_cctx = oxcf->txfm_cfg.enable_cctx; seq->enable_ibp = oxcf->intra_mode_cfg.enable_ibp; seq->enable_adaptive_mvd = diff --git a/av2/encoder/tx_search.c b/av2/encoder/tx_search.c index 559c7d1da9..31d3c2b8fc 100644 --- a/av2/encoder/tx_search.c +++ b/av2/encoder/tx_search.c @@ -3048,8 +3048,7 @@ static void select_tx_partition_type( uint8_t full_blk_skip[MAX_TX_PARTITIONS] = { 0 }; for (TX_PARTITION_TYPE type = 0; type < TX_PARTITION_TYPES; ++type) { - if (cpi->common.seq_params.reduced_tx_part_set && - type > TX_PARTITION_VERT) { + if (cpi->oxcf.txfm_cfg.reduced_tx_part_set && type > TX_PARTITION_VERT) { break; } // Skip any illegal partitions for this block size @@ -3362,8 +3361,7 @@ static void choose_tx_size_type_from_rd(const AV2_COMP *const cpi, int64_t cur_rd = INT64_MAX; const bool is_rect = is_rect_tx(max_tx_size); for (TX_PARTITION_TYPE type = 0; type < TX_PARTITION_TYPES; ++type) { - if (cpi->common.seq_params.reduced_tx_part_set && - type > TX_PARTITION_VERT) { + if (cpi->oxcf.txfm_cfg.reduced_tx_part_set && type > TX_PARTITION_VERT) { break; } // Skip any illegal partitions for this block size From bd27a660fd358f72994161831762ea027d9f7d82 Mon Sep 17 00:00:00 2001 From: Urvang Joshi Date: Tue, 25 Aug 2026 10:50:44 -0700 Subject: [PATCH 2/2] add some comments --- av2/common/blockd.h | 3 ++- av2/encoder/encoder.c | 5 +++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/av2/common/blockd.h b/av2/common/blockd.h index 3805e61707..67bd4c293a 100644 --- a/av2/common/blockd.h +++ b/av2/common/blockd.h @@ -2331,7 +2331,8 @@ typedef struct macroblockd { /*! \brief Flag to decide whether CFL is allowed for a particular chroma * block as passed down from the parent tree */ CFL_ALLOWED_FOR_SDP_TYPE is_cfl_allowed_in_sdp; - /*! \brief Flag to indicate whether reduced transform partition mode is on */ + /*! \brief Flag to indicate whether reduced transform partition mode is on and + * is the same as `cm->seq_params.reduced_tx_part_set` */ int reduced_tx_part_set; } MACROBLOCKD; diff --git a/av2/encoder/encoder.c b/av2/encoder/encoder.c index 41e0e3d67a..f8791f1ebb 100644 --- a/av2/encoder/encoder.c +++ b/av2/encoder/encoder.c @@ -577,6 +577,11 @@ void av2_init_seq_coding_tools(AV2_COMP *cpi, SequenceHeader *seq, seq->enable_mhccp = oxcf->intra_mode_cfg.enable_mhccp; seq->enable_inter_ddt = seq->single_picture_header_flag ? 0 : oxcf->txfm_cfg.enable_inter_ddt; + // Note: we always keep the sequence header flag `reduced_tx_part_set` off, + // ignoring the encoder option `oxcf->txfm_cfg.reduced_tx_part_set`. This is + // because the restricting the transform partition types during encoder RDO + // only has been found to be more compression-efficient than using the + // sequence header option. seq->reduced_tx_part_set = 0; seq->enable_cctx = oxcf->txfm_cfg.enable_cctx; seq->enable_ibp = oxcf->intra_mode_cfg.enable_ibp;