From 8c5cb8b0ac2b08dcf80e31ecc0c347e6e5f42778 Mon Sep 17 00:00:00 2001 From: Developer Date: Wed, 15 Jul 2026 11:02:50 +0000 Subject: [PATCH 16/17] ppc: rewrite iadst16_8col as a direct scalar-C port, fixing wrong output Same bug class as the iadst8_vmx rewrite (54009dd85), now in the 16-point inverse ADST used by vp9_iht16x16_256_add for tx_type 1/2/3 (ADST_DCT, DCT_ADST, ADST_ADST). iadst16_c (vpx_dsp/inv_txfm.c) keeps its entire working set (x0..x15, s0..s15) in tran_high_t (32-bit int) for the whole function body -- never narrowed to a 16-bit type until output[] is written -- and WRAPLOW() is a genuine no-op in this build (no CONFIG_EMULATE_HARDWARE, no CONFIG_COEFFICIENT_RANGE_CHECKING). The original iadst16_8col packed intermediate stage results back into vector signed short between stages and combined/multiplied them in 16-bit vector arithmetic. With 4 stages instead of iadst8's 3, this bug was present in more places, not fewer: stage 2's x8..x15 (already round-shifted, so already possibly out of int16 range) get multiplied by cospi constants, stage 3's x4..x7/x12..x15 (post-combine, un-shifted) get multiplied by cospi constants, and stage 4 computes cospi_16_64*(x2+x3)-shaped expressions four times over (x2/x3, x6/x7, x10/x11, x14/x15) -- the same expression shape that was shown in the iadst8 rewrite to reach ~2.7e9 in magnitude for adversarial inputs. Rewrote the whole function as a direct, line-for-line port of iadst16_c, reusing the vi8 helper toolkit introduced for iadst8_vmx (a lo/hi pair of vector signed int carrying 8 columns of genuinely-wide 32-bit state): one vi8 per reference scalar (x0..x15, s0..s15), stage 1 multiplies fresh tran_low_t-range input via the narrow widening multiply (vi8_mul_narrow), and every subsequent wide-value combine/multiply (stages 2-4) goes through vi8_add/vi8_sub/vi8_mul_wide, only narrowing to vector signed short at the very end for output. No 16-lane wide struct was needed -- the existing 8-lane vi8 maps 1:1 onto how this "8col" function already processes 8 columns per call, exactly like iadst8_vmx. Verified against the QEMU/AltiVec cross-check harness (full 23000-check run, all sizes/tx_types): vp9_iht16x16_256_add tx_type=1/2/3 dropped from 499/500 failures to 45-65/500, both unaligned and 4-byte-aligned stride variants -- matching tx_type=0's (pure DCT, untouched by this change) existing 51-62/500 baseline, the same separately-tracked pixel-reconstruction rounding edge case noise floor documented in 54009dd85. All other tests in the full harness run are unchanged from their pre-existing baseline counts (e.g. vp9_iht8x8_64_add stays at 8-10/500 across all tx_types, vpx_idct16x16_256_add stays at ~52-71/500). Co-Authored-By: Claude Sonnet 5 --- vpx_dsp/ppc/vpx_idct_altivec.c | 692 ++++++++++----------------------- 1 file changed, 214 insertions(+), 478 deletions(-) diff --git a/vpx_dsp/ppc/vpx_idct_altivec.c b/vpx_dsp/ppc/vpx_idct_altivec.c index 0818ae460..d15e48e0c 100644 --- a/vpx_dsp/ppc/vpx_idct_altivec.c +++ b/vpx_dsp/ppc/vpx_idct_altivec.c @@ -3260,485 +3260,221 @@ static inline void write_buffer_8x16(uint8_t *dest, vector signed short *in, int // It's time for INTRINSIC HELL. // First tarpit: performing an 16x16 1-D inverse ADST on eight columns. +// +// Direct, line-for-line port of iadst16_c's x0..x15/s0..s15 algebra +// (vpx_dsp/inv_txfm.c), vectorized across the 8 rows/columns processed in +// parallel per vi8 half -- exactly the same rewrite as iadst8_vmx above +// (see the big vi8 comment block for why every stage needs genuine 32-bit +// (wide) headroom, not just narrow int16 arithmetic): iadst16_c also keeps +// its ENTIRE working set (x0..x15, s0..s15) in tran_high_t for the whole +// function body, with WRAPLOW() a genuine no-op in this build, and stage 4 +// alone computes things like cospi_16_64*(x2+x3) which can overflow int32 +// -- the same bug class, just with a 4th stage and twice the lanes. Each +// reference scalar xN/sN maps to one vi8 here (a lo/hi pair of vector +// signed int covering the 8 columns processed by this "8col" call), so the +// mapping is mechanically checkable line by line against the reference. static void iadst16_8col(vector signed short *in) { - vector signed short s[16], x[16], u[32]; - vector signed int v[32], w[32]; - vector signed short k__cospi_p01_p31 = short_pair_a(cospi_1_64, cospi_31_64); - vector signed short k__cospi_p31_m01 = short_pair_a(cospi_31_64, -cospi_1_64); - vector signed short k__cospi_p05_p27 = short_pair_a(cospi_5_64, cospi_27_64); - vector signed short k__cospi_p27_m05 = short_pair_a(cospi_27_64, -cospi_5_64); - vector signed short k__cospi_p09_p23 = short_pair_a(cospi_9_64, cospi_23_64); - vector signed short k__cospi_p23_m09 = short_pair_a(cospi_23_64, -cospi_9_64); - vector signed short k__cospi_p13_p19 = short_pair_a(cospi_13_64, cospi_19_64); - vector signed short k__cospi_p19_m13 = short_pair_a(cospi_19_64, -cospi_13_64); - vector signed short k__cospi_p17_p15 = short_pair_a(cospi_17_64, cospi_15_64); - vector signed short k__cospi_p15_m17 = short_pair_a(cospi_15_64, -cospi_17_64); - vector signed short k__cospi_p21_p11 = short_pair_a(cospi_21_64, cospi_11_64); - vector signed short k__cospi_p11_m21 = short_pair_a(cospi_11_64, -cospi_21_64); - vector signed short k__cospi_p25_p07 = short_pair_a(cospi_25_64, cospi_7_64); - vector signed short k__cospi_p07_m25 = short_pair_a(cospi_7_64, -cospi_25_64); - vector signed short k__cospi_p29_p03 = short_pair_a(cospi_29_64, cospi_3_64); - vector signed short k__cospi_p03_m29 = short_pair_a(cospi_3_64, -cospi_29_64); - vector signed short k__cospi_p04_p28 = short_pair_a(cospi_4_64, cospi_28_64); - vector signed short k__cospi_p28_m04 = short_pair_a(cospi_28_64, -cospi_4_64); - vector signed short k__cospi_p20_p12 = short_pair_a(cospi_20_64, cospi_12_64); - vector signed short k__cospi_p12_m20 = short_pair_a(cospi_12_64, -cospi_20_64); - vector signed short k__cospi_m28_p04 = short_pair_a(-cospi_28_64, cospi_4_64); - vector signed short k__cospi_m12_p20 = short_pair_a(-cospi_12_64, cospi_20_64); - vector signed short k__cospi_p08_p24 = short_pair_a(cospi_8_64, cospi_24_64); - vector signed short k__cospi_p24_m08 = short_pair_a(cospi_24_64, -cospi_8_64); - vector signed short k__cospi_m24_p08 = short_pair_a(-cospi_24_64, cospi_8_64); - vector signed short k__cospi_m16_m16; - vector signed short k__cospi_p16_p16; - vector signed short k__cospi_p16_m16 = short_pair_a(cospi_16_64, -cospi_16_64); - vector signed short k__cospi_m16_p16 = short_pair_a(-cospi_16_64, cospi_16_64); - vector signed short kZero = vec_splat_s16(0); - vector signed int mzero = vec_splat_s32(0); - - signed short a_s = ((int16_t)-cospi_16_64); - signed short b_s = ((int16_t)cospi_16_64); - k__cospi_m16_m16 = ss_splat(&a_s); - k__cospi_p16_p16 = ss_splat(&b_s); - - // Stage 1. - u[0] = vec_mergeh(in[15], in[0]); - u[1] = vec_mergel(in[15], in[0]); - u[2] = vec_mergeh(in[13], in[2]); - u[3] = vec_mergel(in[13], in[2]); - u[4] = vec_mergeh(in[11], in[4]); - u[5] = vec_mergel(in[11], in[4]); - u[6] = vec_mergeh(in[9], in[6]); - u[7] = vec_mergel(in[9], in[6]); - u[8] = vec_mergeh(in[7], in[8]); - u[9] = vec_mergel(in[7], in[8]); - u[10] = vec_mergeh(in[5], in[10]); - u[11] = vec_mergel(in[5], in[10]); - u[12] = vec_mergeh(in[3], in[12]); - u[13] = vec_mergel(in[3], in[12]); - u[14] = vec_mergeh(in[1], in[14]); - u[15] = vec_mergel(in[1], in[14]); - - // Unfortunately no post-add is possible. - v[0] = vec_msum(u[0], k__cospi_p01_p31, mzero); - v[1] = vec_msum(u[1], k__cospi_p01_p31, mzero); - v[2] = vec_msum(u[0], k__cospi_p31_m01, mzero); - v[3] = vec_msum(u[1], k__cospi_p31_m01, mzero); - v[4] = vec_msum(u[2], k__cospi_p05_p27, mzero); - v[5] = vec_msum(u[3], k__cospi_p05_p27, mzero); - v[6] = vec_msum(u[2], k__cospi_p27_m05, mzero); - v[7] = vec_msum(u[3], k__cospi_p27_m05, mzero); - v[8] = vec_msum(u[4], k__cospi_p09_p23, mzero); - v[9] = vec_msum(u[5], k__cospi_p09_p23, mzero); - v[10] = vec_msum(u[4], k__cospi_p23_m09, mzero); - v[11] = vec_msum(u[5], k__cospi_p23_m09, mzero); - v[12] = vec_msum(u[6], k__cospi_p13_p19, mzero); - v[13] = vec_msum(u[7], k__cospi_p13_p19, mzero); - v[14] = vec_msum(u[6], k__cospi_p19_m13, mzero); - v[15] = vec_msum(u[7], k__cospi_p19_m13, mzero); - v[16] = vec_msum(u[8], k__cospi_p17_p15, mzero); - v[17] = vec_msum(u[9], k__cospi_p17_p15, mzero); - v[18] = vec_msum(u[8], k__cospi_p15_m17, mzero); - v[19] = vec_msum(u[9], k__cospi_p15_m17, mzero); - v[20] = vec_msum(u[10], k__cospi_p21_p11, mzero); - v[21] = vec_msum(u[11], k__cospi_p21_p11, mzero); - v[22] = vec_msum(u[10], k__cospi_p11_m21, mzero); - v[23] = vec_msum(u[11], k__cospi_p11_m21, mzero); - v[24] = vec_msum(u[12], k__cospi_p25_p07, mzero); - v[25] = vec_msum(u[13], k__cospi_p25_p07, mzero); - v[26] = vec_msum(u[12], k__cospi_p07_m25, mzero); - v[27] = vec_msum(u[13], k__cospi_p07_m25, mzero); - v[28] = vec_msum(u[14], k__cospi_p29_p03, mzero); - v[29] = vec_msum(u[15], k__cospi_p29_p03, mzero); - v[30] = vec_msum(u[14], k__cospi_p03_m29, mzero); - v[31] = vec_msum(u[15], k__cospi_p03_m29, mzero); - - w[0] = vec_add(v[0], v[16]); - w[1] = vec_add(v[1], v[17]); - w[2] = vec_add(v[2], v[18]); - w[3] = vec_add(v[3], v[19]); - w[4] = vec_add(v[4], v[20]); - w[5] = vec_add(v[5], v[21]); - w[6] = vec_add(v[6], v[22]); - w[7] = vec_add(v[7], v[23]); - w[8] = vec_add(v[8], v[24]); - w[9] = vec_add(v[9], v[25]); - w[10] = vec_add(v[10], v[26]); - w[11] = vec_add(v[11], v[27]); - w[12] = vec_add(v[12], v[28]); - w[13] = vec_add(v[13], v[29]); - w[14] = vec_add(v[14], v[30]); - w[15] = vec_add(v[15], v[31]); - w[16] = vec_sub(v[0], v[16]); - w[17] = vec_sub(v[1], v[17]); - w[18] = vec_sub(v[2], v[18]); - w[19] = vec_sub(v[3], v[19]); - w[20] = vec_sub(v[4], v[20]); - w[21] = vec_sub(v[5], v[21]); - w[22] = vec_sub(v[6], v[22]); - w[23] = vec_sub(v[7], v[23]); - w[24] = vec_sub(v[8], v[24]); - w[25] = vec_sub(v[9], v[25]); - w[26] = vec_sub(v[10], v[26]); - w[27] = vec_sub(v[11], v[27]); - w[28] = vec_sub(v[12], v[28]); - w[29] = vec_sub(v[13], v[29]); - w[30] = vec_sub(v[14], v[30]); - w[31] = vec_sub(v[15], v[31]); - - v[0] = vec_add(w[0], dct_rounding_vec); - v[1] = vec_add(w[1], dct_rounding_vec); - v[2] = vec_add(w[2], dct_rounding_vec); - v[3] = vec_add(w[3], dct_rounding_vec); - v[4] = vec_add(w[4], dct_rounding_vec); - v[5] = vec_add(w[5], dct_rounding_vec); - v[6] = vec_add(w[6], dct_rounding_vec); - v[7] = vec_add(w[7], dct_rounding_vec); - v[8] = vec_add(w[8], dct_rounding_vec); - v[9] = vec_add(w[9], dct_rounding_vec); - v[10] = vec_add(w[10], dct_rounding_vec); - v[11] = vec_add(w[11], dct_rounding_vec); - v[12] = vec_add(w[12], dct_rounding_vec); - v[13] = vec_add(w[13], dct_rounding_vec); - v[14] = vec_add(w[14], dct_rounding_vec); - v[15] = vec_add(w[15], dct_rounding_vec); - v[16] = vec_add(w[16], dct_rounding_vec); - v[17] = vec_add(w[17], dct_rounding_vec); - v[18] = vec_add(w[18], dct_rounding_vec); - v[19] = vec_add(w[19], dct_rounding_vec); - v[20] = vec_add(w[20], dct_rounding_vec); - v[21] = vec_add(w[21], dct_rounding_vec); - v[22] = vec_add(w[22], dct_rounding_vec); - v[23] = vec_add(w[23], dct_rounding_vec); - v[24] = vec_add(w[24], dct_rounding_vec); - v[25] = vec_add(w[25], dct_rounding_vec); - v[26] = vec_add(w[26], dct_rounding_vec); - v[27] = vec_add(w[27], dct_rounding_vec); - v[28] = vec_add(w[28], dct_rounding_vec); - v[29] = vec_add(w[29], dct_rounding_vec); - v[30] = vec_add(w[30], dct_rounding_vec); - v[31] = vec_add(w[31], dct_rounding_vec); - - w[0] = vec_sra(v[0], dct_bitshift_vec); - w[1] = vec_sra(v[1], dct_bitshift_vec); - w[2] = vec_sra(v[2], dct_bitshift_vec); - w[3] = vec_sra(v[3], dct_bitshift_vec); - w[4] = vec_sra(v[4], dct_bitshift_vec); - w[5] = vec_sra(v[5], dct_bitshift_vec); - w[6] = vec_sra(v[6], dct_bitshift_vec); - w[7] = vec_sra(v[7], dct_bitshift_vec); - w[8] = vec_sra(v[8], dct_bitshift_vec); - w[9] = vec_sra(v[9], dct_bitshift_vec); - w[10] = vec_sra(v[10], dct_bitshift_vec); - w[11] = vec_sra(v[11], dct_bitshift_vec); - w[12] = vec_sra(v[12], dct_bitshift_vec); - w[13] = vec_sra(v[13], dct_bitshift_vec); - w[14] = vec_sra(v[14], dct_bitshift_vec); - w[15] = vec_sra(v[15], dct_bitshift_vec); - w[16] = vec_sra(v[16], dct_bitshift_vec); - w[17] = vec_sra(v[17], dct_bitshift_vec); - w[18] = vec_sra(v[18], dct_bitshift_vec); - w[19] = vec_sra(v[19], dct_bitshift_vec); - w[20] = vec_sra(v[20], dct_bitshift_vec); - w[21] = vec_sra(v[21], dct_bitshift_vec); - w[22] = vec_sra(v[22], dct_bitshift_vec); - w[23] = vec_sra(v[23], dct_bitshift_vec); - w[24] = vec_sra(v[24], dct_bitshift_vec); - w[25] = vec_sra(v[25], dct_bitshift_vec); - w[26] = vec_sra(v[26], dct_bitshift_vec); - w[27] = vec_sra(v[27], dct_bitshift_vec); - w[28] = vec_sra(v[28], dct_bitshift_vec); - w[29] = vec_sra(v[29], dct_bitshift_vec); - w[30] = vec_sra(v[30], dct_bitshift_vec); - w[31] = vec_sra(v[31], dct_bitshift_vec); - - s[0] = vec_pack(w[0], w[1]); - s[1] = vec_pack(w[2], w[3]); - s[2] = vec_pack(w[4], w[5]); - s[3] = vec_pack(w[6], w[7]); - s[4] = vec_pack(w[8], w[9]); - s[5] = vec_pack(w[10], w[11]); - s[6] = vec_pack(w[12], w[13]); - s[7] = vec_pack(w[14], w[15]); - s[8] = vec_pack(w[16], w[17]); - s[9] = vec_pack(w[18], w[19]); - s[10] = vec_pack(w[20], w[21]); - s[11] = vec_pack(w[22], w[23]); - s[12] = vec_pack(w[24], w[25]); - s[13] = vec_pack(w[26], w[27]); - s[14] = vec_pack(w[28], w[29]); - s[15] = vec_pack(w[30], w[31]); - - // Stage 2. - u[0] = vec_mergeh(s[8], s[9]); - u[1] = vec_mergel(s[8], s[9]); - u[2] = vec_mergeh(s[10], s[11]); - u[3] = vec_mergel(s[10], s[11]); - u[4] = vec_mergeh(s[12], s[13]); - u[5] = vec_mergel(s[12], s[13]); - u[6] = vec_mergeh(s[14], s[15]); - u[7] = vec_mergel(s[14], s[15]); - - v[0] = vec_msum(u[0], k__cospi_p04_p28, mzero); - v[1] = vec_msum(u[1], k__cospi_p04_p28, mzero); - v[2] = vec_msum(u[0], k__cospi_p28_m04, mzero); - v[3] = vec_msum(u[1], k__cospi_p28_m04, mzero); - v[4] = vec_msum(u[2], k__cospi_p20_p12, mzero); - v[5] = vec_msum(u[3], k__cospi_p20_p12, mzero); - v[6] = vec_msum(u[2], k__cospi_p12_m20, mzero); - v[7] = vec_msum(u[3], k__cospi_p12_m20, mzero); - v[8] = vec_msum(u[4], k__cospi_m28_p04, mzero); - v[9] = vec_msum(u[5], k__cospi_m28_p04, mzero); - v[10] = vec_msum(u[4], k__cospi_p04_p28, mzero); - v[11] = vec_msum(u[5], k__cospi_p04_p28, mzero); - v[12] = vec_msum(u[6], k__cospi_m12_p20, mzero); - v[13] = vec_msum(u[7], k__cospi_m12_p20, mzero); - v[14] = vec_msum(u[6], k__cospi_p20_p12, mzero); - v[15] = vec_msum(u[7], k__cospi_p20_p12, mzero); - - w[0] = vec_add(v[0], v[8]); - w[1] = vec_add(v[1], v[9]); - w[2] = vec_add(v[2], v[10]); - w[3] = vec_add(v[3], v[11]); - w[4] = vec_add(v[4], v[12]); - w[5] = vec_add(v[5], v[13]); - w[6] = vec_add(v[6], v[14]); - w[7] = vec_add(v[7], v[15]); - w[8] = vec_sub(v[0], v[8]); - w[9] = vec_sub(v[1], v[9]); - w[10] = vec_sub(v[2], v[10]); - w[11] = vec_sub(v[3], v[11]); - w[12] = vec_sub(v[4], v[12]); - w[13] = vec_sub(v[5], v[13]); - w[14] = vec_sub(v[6], v[14]); - w[15] = vec_sub(v[7], v[15]); - - v[0] = vec_add(w[0], dct_rounding_vec); - v[1] = vec_add(w[1], dct_rounding_vec); - v[2] = vec_add(w[2], dct_rounding_vec); - v[3] = vec_add(w[3], dct_rounding_vec); - v[4] = vec_add(w[4], dct_rounding_vec); - v[5] = vec_add(w[5], dct_rounding_vec); - v[6] = vec_add(w[6], dct_rounding_vec); - v[7] = vec_add(w[7], dct_rounding_vec); - v[8] = vec_add(w[8], dct_rounding_vec); - v[9] = vec_add(w[9], dct_rounding_vec); - v[10] = vec_add(w[10], dct_rounding_vec); - v[11] = vec_add(w[11], dct_rounding_vec); - v[12] = vec_add(w[12], dct_rounding_vec); - v[13] = vec_add(w[13], dct_rounding_vec); - v[14] = vec_add(w[14], dct_rounding_vec); - v[15] = vec_add(w[15], dct_rounding_vec); - - w[0] = vec_sra(v[0], dct_bitshift_vec); - w[1] = vec_sra(v[1], dct_bitshift_vec); - w[2] = vec_sra(v[2], dct_bitshift_vec); - w[3] = vec_sra(v[3], dct_bitshift_vec); - w[4] = vec_sra(v[4], dct_bitshift_vec); - w[5] = vec_sra(v[5], dct_bitshift_vec); - w[6] = vec_sra(v[6], dct_bitshift_vec); - w[7] = vec_sra(v[7], dct_bitshift_vec); - w[8] = vec_sra(v[8], dct_bitshift_vec); - w[9] = vec_sra(v[9], dct_bitshift_vec); - w[10] = vec_sra(v[10], dct_bitshift_vec); - w[11] = vec_sra(v[11], dct_bitshift_vec); - w[12] = vec_sra(v[12], dct_bitshift_vec); - w[13] = vec_sra(v[13], dct_bitshift_vec); - w[14] = vec_sra(v[14], dct_bitshift_vec); - w[15] = vec_sra(v[15], dct_bitshift_vec); - - x[0] = vec_add(s[0], s[4]); - x[1] = vec_add(s[1], s[5]); - x[2] = vec_add(s[2], s[6]); - x[3] = vec_add(s[3], s[7]); - x[4] = vec_sub(s[0], s[4]); - x[5] = vec_sub(s[1], s[5]); - x[6] = vec_sub(s[2], s[6]); - x[7] = vec_sub(s[3], s[7]); - - x[8] = vec_pack(w[0], w[1]); - x[9] = vec_pack(w[2], w[3]); - x[10] = vec_pack(w[4], w[5]); - x[11] = vec_pack(w[6], w[7]); - x[12] = vec_pack(w[8], w[9]); - x[13] = vec_pack(w[10], w[11]); - x[14] = vec_pack(w[12], w[13]); - x[15] = vec_pack(w[14], w[15]); - - // Stage 3. - u[0] = vec_mergeh(x[4], x[5]); - u[1] = vec_mergel(x[4], x[5]); - u[2] = vec_mergeh(x[6], x[7]); - u[3] = vec_mergel(x[6], x[7]); - u[4] = vec_mergeh(x[12], x[13]); - u[5] = vec_mergel(x[12], x[13]); - u[6] = vec_mergeh(x[14], x[15]); - u[7] = vec_mergel(x[14], x[15]); - - v[0] = vec_msum(u[0], k__cospi_p08_p24, mzero); - v[1] = vec_msum(u[1], k__cospi_p08_p24, mzero); - v[2] = vec_msum(u[0], k__cospi_p24_m08, mzero); - v[3] = vec_msum(u[1], k__cospi_p24_m08, mzero); - v[4] = vec_msum(u[2], k__cospi_m24_p08, mzero); - v[5] = vec_msum(u[3], k__cospi_m24_p08, mzero); - v[6] = vec_msum(u[2], k__cospi_p08_p24, mzero); - v[7] = vec_msum(u[3], k__cospi_p08_p24, mzero); - v[8] = vec_msum(u[4], k__cospi_p08_p24, mzero); - v[9] = vec_msum(u[5], k__cospi_p08_p24, mzero); - v[10] = vec_msum(u[4], k__cospi_p24_m08, mzero); - v[11] = vec_msum(u[5], k__cospi_p24_m08, mzero); - v[12] = vec_msum(u[6], k__cospi_m24_p08, mzero); - v[13] = vec_msum(u[7], k__cospi_m24_p08, mzero); - v[14] = vec_msum(u[6], k__cospi_p08_p24, mzero); - v[15] = vec_msum(u[7], k__cospi_p08_p24, mzero); - - w[0] = vec_add(v[0], v[4]); - w[1] = vec_add(v[1], v[5]); - w[2] = vec_add(v[2], v[6]); - w[3] = vec_add(v[3], v[7]); - w[4] = vec_sub(v[0], v[4]); - w[5] = vec_sub(v[1], v[5]); - w[6] = vec_sub(v[2], v[6]); - w[7] = vec_sub(v[3], v[7]); - w[8] = vec_add(v[8], v[12]); - w[9] = vec_add(v[9], v[13]); - w[10] = vec_add(v[10], v[14]); - w[11] = vec_add(v[11], v[15]); - w[12] = vec_sub(v[8], v[12]); - w[13] = vec_sub(v[9] , v[13]); - w[14] = vec_sub(v[10], v[14]); - w[15] = vec_sub(v[11], v[15]); - - w[0] = vec_add(w[0], dct_rounding_vec); - w[1] = vec_add(w[1], dct_rounding_vec); - w[2] = vec_add(w[2], dct_rounding_vec); - w[3] = vec_add(w[3], dct_rounding_vec); - w[4] = vec_add(w[4], dct_rounding_vec); - w[5] = vec_add(w[5], dct_rounding_vec); - w[6] = vec_add(w[6], dct_rounding_vec); - w[7] = vec_add(w[7], dct_rounding_vec); - w[8] = vec_add(w[8], dct_rounding_vec); - w[9] = vec_add(w[9], dct_rounding_vec); - w[10] = vec_add(w[10], dct_rounding_vec); - w[11] = vec_add(w[11], dct_rounding_vec); - w[12] = vec_add(w[12], dct_rounding_vec); - w[13] = vec_add(w[13], dct_rounding_vec); - w[14] = vec_add(w[14], dct_rounding_vec); - w[15] = vec_add(w[15], dct_rounding_vec); - - v[0] = vec_sra(w[0], dct_bitshift_vec); - v[1] = vec_sra(w[1], dct_bitshift_vec); - v[2] = vec_sra(w[2], dct_bitshift_vec); - v[3] = vec_sra(w[3], dct_bitshift_vec); - v[4] = vec_sra(w[4], dct_bitshift_vec); - v[5] = vec_sra(w[5], dct_bitshift_vec); - v[6] = vec_sra(w[6], dct_bitshift_vec); - v[7] = vec_sra(w[7], dct_bitshift_vec); - v[8] = vec_sra(w[8], dct_bitshift_vec); - v[9] = vec_sra(w[9], dct_bitshift_vec); - v[10] = vec_sra(w[10], dct_bitshift_vec); - v[11] = vec_sra(w[11], dct_bitshift_vec); - v[12] = vec_sra(w[12], dct_bitshift_vec); - v[13] = vec_sra(w[13], dct_bitshift_vec); - v[14] = vec_sra(w[14], dct_bitshift_vec); - v[15] = vec_sra(w[15], dct_bitshift_vec); - - s[0] = vec_add(x[0], x[2]); - s[1] = vec_add(x[1], x[3]); - s[2] = vec_sub(x[0], x[2]); - s[3] = vec_sub(x[1], x[3]); - - s[4] = vec_pack(v[0], v[1]); - s[5] = vec_pack(v[2], v[3]); - s[6] = vec_pack(v[4], v[5]); - s[7] = vec_pack(v[6], v[7]); - - s[8] = vec_add(x[8], x[10]); - s[9] = vec_add(x[9], x[11]); - s[10] = vec_sub(x[8], x[10]); - s[11] = vec_sub(x[9], x[11]); - - s[12] = vec_pack(v[8], v[9]); - s[13] = vec_pack(v[10], v[11]); - s[14] = vec_pack(v[12], v[13]); - s[15] = vec_pack(v[14], v[15]); - - // Stage 4. - u[0] = vec_mergeh(s[2], s[3]); - u[1] = vec_mergel(s[2], s[3]); - u[2] = vec_mergeh(s[6], s[7]); - u[3] = vec_mergel(s[6], s[7]); - u[4] = vec_mergeh(s[10], s[11]); - u[5] = vec_mergel(s[10], s[11]); - u[6] = vec_mergeh(s[14], s[15]); - u[7] = vec_mergel(s[14], s[15]); - - v[0] = vec_msum(u[0], k__cospi_m16_m16, mzero); - v[1] = vec_msum(u[1], k__cospi_m16_m16, mzero); - v[2] = vec_msum(u[0], k__cospi_p16_m16, mzero); - v[3] = vec_msum(u[1], k__cospi_p16_m16, mzero); - v[4] = vec_msum(u[2], k__cospi_p16_p16, mzero); - v[5] = vec_msum(u[3], k__cospi_p16_p16, mzero); - v[6] = vec_msum(u[2], k__cospi_m16_p16, mzero); - v[7] = vec_msum(u[3], k__cospi_m16_p16, mzero); - v[8] = vec_msum(u[4], k__cospi_p16_p16, mzero); - v[9] = vec_msum(u[5], k__cospi_p16_p16, mzero); - v[10] = vec_msum(u[4], k__cospi_m16_p16, mzero); - v[11] = vec_msum(u[5], k__cospi_m16_p16, mzero); - v[12] = vec_msum(u[6], k__cospi_m16_m16, mzero); - v[13] = vec_msum(u[7], k__cospi_m16_m16, mzero); - v[14] = vec_msum(u[6], k__cospi_p16_m16, mzero); - v[15] = vec_msum(u[7], k__cospi_p16_m16, mzero); - - w[0] = vec_add(v[0], dct_rounding_vec); - w[1] = vec_add(v[1], dct_rounding_vec); - w[2] = vec_add(v[2], dct_rounding_vec); - w[3] = vec_add(v[3], dct_rounding_vec); - w[4] = vec_add(v[4], dct_rounding_vec); - w[5] = vec_add(v[5], dct_rounding_vec); - w[6] = vec_add(v[6], dct_rounding_vec); - w[7] = vec_add(v[7], dct_rounding_vec); - w[8] = vec_add(v[8], dct_rounding_vec); - w[9] = vec_add(v[9], dct_rounding_vec); - w[10] = vec_add(v[10], dct_rounding_vec); - w[11] = vec_add(v[11], dct_rounding_vec); - w[12] = vec_add(v[12], dct_rounding_vec); - w[13] = vec_add(v[13], dct_rounding_vec); - w[14] = vec_add(v[14], dct_rounding_vec); - w[15] = vec_add(v[15], dct_rounding_vec); - - v[0] = vec_sra(w[0], dct_bitshift_vec); - v[1] = vec_sra(w[1], dct_bitshift_vec); - v[2] = vec_sra(w[2], dct_bitshift_vec); - v[3] = vec_sra(w[3], dct_bitshift_vec); - v[4] = vec_sra(w[4], dct_bitshift_vec); - v[5] = vec_sra(w[5], dct_bitshift_vec); - v[6] = vec_sra(w[6], dct_bitshift_vec); - v[7] = vec_sra(w[7], dct_bitshift_vec); - v[8] = vec_sra(w[8], dct_bitshift_vec); - v[9] = vec_sra(w[9], dct_bitshift_vec); - v[10] = vec_sra(w[10], dct_bitshift_vec); - v[11] = vec_sra(w[11], dct_bitshift_vec); - v[12] = vec_sra(w[12], dct_bitshift_vec); - v[13] = vec_sra(w[13], dct_bitshift_vec); - v[14] = vec_sra(w[14], dct_bitshift_vec); - v[15] = vec_sra(w[15], dct_bitshift_vec); - - in[0] = s[0]; - in[1] = vec_sub(kZero, s[8]); - in[2] = s[12]; - in[3] = vec_sub(kZero, s[4]); - in[4] = vec_pack(v[4], v[5]); - in[5] = vec_pack(v[12], v[13]); - in[6] = vec_pack(v[8], v[9]); - in[7] = vec_pack(v[0], v[1]); - in[8] = vec_pack(v[2], v[3]); - in[9] = vec_pack(v[10], v[11]); - in[10] = vec_pack(v[14], v[15]); - in[11] = vec_pack(v[6], v[7]); - in[12] = s[5]; - in[13] = vec_sub(kZero, s[13]); - in[14] = s[9]; - in[15] = vec_sub(kZero, s[1]); + vector signed short k__const_0 = vec_splat_s16(0); + vi8 s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, s12, s13, s14, s15; + vi8 x0, x1, x2, x3, x4, x5, x6, x7, x8, x9, x10, x11, x12, x13, x14, x15; + + // tran_high_t x0 = input[15]; x1 = input[0]; x2 = input[13]; x3 = input[2]; + // x4 = input[11]; x5 = input[4]; x6 = input[9]; x7 = input[6]; + // x8 = input[7]; x9 = input[8]; x10 = input[5]; x11 = input[10]; + // x12 = input[3]; x13 = input[12]; x14 = input[1]; x15 = input[14]; + // (in[] here is the "input" array indexed the same way the reference + // indexes input[]; no permutation needed beyond picking the right in[N] + // below, matching the original code's u[]-merge input pairing.) + + // Stage 1. Inputs are genuine tran_low_t-range values straight from the + // caller, so these multiplies use the narrow (int16 operand) widening + // multiply. + // s0 = x0 * cospi_1_64 + x1 * cospi_31_64; + s0 = vi8_add(vi8_mul_narrow(in[15], cospi_1_64), vi8_mul_narrow(in[0], cospi_31_64)); + // s1 = x0 * cospi_31_64 - x1 * cospi_1_64; + s1 = vi8_sub(vi8_mul_narrow(in[15], cospi_31_64), vi8_mul_narrow(in[0], cospi_1_64)); + // s2 = x2 * cospi_5_64 + x3 * cospi_27_64; + s2 = vi8_add(vi8_mul_narrow(in[13], cospi_5_64), vi8_mul_narrow(in[2], cospi_27_64)); + // s3 = x2 * cospi_27_64 - x3 * cospi_5_64; + s3 = vi8_sub(vi8_mul_narrow(in[13], cospi_27_64), vi8_mul_narrow(in[2], cospi_5_64)); + // s4 = x4 * cospi_9_64 + x5 * cospi_23_64; + s4 = vi8_add(vi8_mul_narrow(in[11], cospi_9_64), vi8_mul_narrow(in[4], cospi_23_64)); + // s5 = x4 * cospi_23_64 - x5 * cospi_9_64; + s5 = vi8_sub(vi8_mul_narrow(in[11], cospi_23_64), vi8_mul_narrow(in[4], cospi_9_64)); + // s6 = x6 * cospi_13_64 + x7 * cospi_19_64; + s6 = vi8_add(vi8_mul_narrow(in[9], cospi_13_64), vi8_mul_narrow(in[6], cospi_19_64)); + // s7 = x6 * cospi_19_64 - x7 * cospi_13_64; + s7 = vi8_sub(vi8_mul_narrow(in[9], cospi_19_64), vi8_mul_narrow(in[6], cospi_13_64)); + // s8 = x8 * cospi_17_64 + x9 * cospi_15_64; + s8 = vi8_add(vi8_mul_narrow(in[7], cospi_17_64), vi8_mul_narrow(in[8], cospi_15_64)); + // s9 = x8 * cospi_15_64 - x9 * cospi_17_64; + s9 = vi8_sub(vi8_mul_narrow(in[7], cospi_15_64), vi8_mul_narrow(in[8], cospi_17_64)); + // s10 = x10 * cospi_21_64 + x11 * cospi_11_64; + s10 = vi8_add(vi8_mul_narrow(in[5], cospi_21_64), vi8_mul_narrow(in[10], cospi_11_64)); + // s11 = x10 * cospi_11_64 - x11 * cospi_21_64; + s11 = vi8_sub(vi8_mul_narrow(in[5], cospi_11_64), vi8_mul_narrow(in[10], cospi_21_64)); + // s12 = x12 * cospi_25_64 + x13 * cospi_7_64; + s12 = vi8_add(vi8_mul_narrow(in[3], cospi_25_64), vi8_mul_narrow(in[12], cospi_7_64)); + // s13 = x12 * cospi_7_64 - x13 * cospi_25_64; + s13 = vi8_sub(vi8_mul_narrow(in[3], cospi_7_64), vi8_mul_narrow(in[12], cospi_25_64)); + // s14 = x14 * cospi_29_64 + x15 * cospi_3_64; + s14 = vi8_add(vi8_mul_narrow(in[1], cospi_29_64), vi8_mul_narrow(in[14], cospi_3_64)); + // s15 = x14 * cospi_3_64 - x15 * cospi_29_64; + s15 = vi8_sub(vi8_mul_narrow(in[1], cospi_3_64), vi8_mul_narrow(in[14], cospi_29_64)); + + // x0 = RS(s0+s8); ... x7 = RS(s7+s15); x8 = RS(s0-s8); ... x15 = RS(s7-s15); + x0 = vi8_round_shift(vi8_add(s0, s8)); + x1 = vi8_round_shift(vi8_add(s1, s9)); + x2 = vi8_round_shift(vi8_add(s2, s10)); + x3 = vi8_round_shift(vi8_add(s3, s11)); + x4 = vi8_round_shift(vi8_add(s4, s12)); + x5 = vi8_round_shift(vi8_add(s5, s13)); + x6 = vi8_round_shift(vi8_add(s6, s14)); + x7 = vi8_round_shift(vi8_add(s7, s15)); + x8 = vi8_round_shift(vi8_sub(s0, s8)); + x9 = vi8_round_shift(vi8_sub(s1, s9)); + x10 = vi8_round_shift(vi8_sub(s2, s10)); + x11 = vi8_round_shift(vi8_sub(s3, s11)); + x12 = vi8_round_shift(vi8_sub(s4, s12)); + x13 = vi8_round_shift(vi8_sub(s5, s13)); + x14 = vi8_round_shift(vi8_sub(s6, s14)); + x15 = vi8_round_shift(vi8_sub(s7, s15)); + + // Stage 2. x0..x7 are wide pass-through; x8..x15 feed wide-by-cospi + // multiplies (x8..x15 are stage-1's round-shifted outputs, which can + // already exceed int16 range -- must use vi8_mul_wide, not narrow). + s0 = x0; s1 = x1; s2 = x2; s3 = x3; s4 = x4; s5 = x5; s6 = x6; s7 = x7; + // s8 = x8 * cospi_4_64 + x9 * cospi_28_64; + s8 = vi8_add(vi8_mul_wide(x8, cospi_4_64), vi8_mul_wide(x9, cospi_28_64)); + // s9 = x8 * cospi_28_64 - x9 * cospi_4_64; + s9 = vi8_sub(vi8_mul_wide(x8, cospi_28_64), vi8_mul_wide(x9, cospi_4_64)); + // s10 = x10 * cospi_20_64 + x11 * cospi_12_64; + s10 = vi8_add(vi8_mul_wide(x10, cospi_20_64), vi8_mul_wide(x11, cospi_12_64)); + // s11 = x10 * cospi_12_64 - x11 * cospi_20_64; + s11 = vi8_sub(vi8_mul_wide(x10, cospi_12_64), vi8_mul_wide(x11, cospi_20_64)); + // s12 = -x12 * cospi_28_64 + x13 * cospi_4_64; + s12 = vi8_add(vi8_mul_wide(x12, (int16_t)(-cospi_28_64)), vi8_mul_wide(x13, cospi_4_64)); + // s13 = x12 * cospi_4_64 + x13 * cospi_28_64; + s13 = vi8_add(vi8_mul_wide(x12, cospi_4_64), vi8_mul_wide(x13, cospi_28_64)); + // s14 = -x14 * cospi_12_64 + x15 * cospi_20_64; + s14 = vi8_add(vi8_mul_wide(x14, (int16_t)(-cospi_12_64)), vi8_mul_wide(x15, cospi_20_64)); + // s15 = x14 * cospi_20_64 + x15 * cospi_12_64; + s15 = vi8_add(vi8_mul_wide(x14, cospi_20_64), vi8_mul_wide(x15, cospi_12_64)); + + // x0 = s0+s4; x1 = s1+s5; x2 = s2+s6; x3 = s3+s7; (no round-shift, wide) + // x4 = s0-s4; x5 = s1-s5; x6 = s2-s6; x7 = s3-s7; + x0 = vi8_add(s0, s4); + x1 = vi8_add(s1, s5); + x2 = vi8_add(s2, s6); + x3 = vi8_add(s3, s7); + x4 = vi8_sub(s0, s4); + x5 = vi8_sub(s1, s5); + x6 = vi8_sub(s2, s6); + x7 = vi8_sub(s3, s7); + // x8 = RS(s8+s12); x9 = RS(s9+s13); x10 = RS(s10+s14); x11 = RS(s11+s15); + // x12 = RS(s8-s12); x13 = RS(s9-s13); x14 = RS(s10-s14); x15 = RS(s11-s15); + x8 = vi8_round_shift(vi8_add(s8, s12)); + x9 = vi8_round_shift(vi8_add(s9, s13)); + x10 = vi8_round_shift(vi8_add(s10, s14)); + x11 = vi8_round_shift(vi8_add(s11, s15)); + x12 = vi8_round_shift(vi8_sub(s8, s12)); + x13 = vi8_round_shift(vi8_sub(s9, s13)); + x14 = vi8_round_shift(vi8_sub(s10, s14)); + x15 = vi8_round_shift(vi8_sub(s11, s15)); + + // Stage 3. s0..s3 pass through wide; s4..s7/s12..s15 are wide-by-cospi + // multiplies of stage-2 outputs (already possibly out of int16 range); + // s8..s11 pass through wide. + s0 = x0; s1 = x1; s2 = x2; s3 = x3; + // s4 = x4 * cospi_8_64 + x5 * cospi_24_64; + s4 = vi8_add(vi8_mul_wide(x4, cospi_8_64), vi8_mul_wide(x5, cospi_24_64)); + // s5 = x4 * cospi_24_64 - x5 * cospi_8_64; + s5 = vi8_sub(vi8_mul_wide(x4, cospi_24_64), vi8_mul_wide(x5, cospi_8_64)); + // s6 = -x6 * cospi_24_64 + x7 * cospi_8_64; + s6 = vi8_add(vi8_mul_wide(x6, (int16_t)(-cospi_24_64)), vi8_mul_wide(x7, cospi_8_64)); + // s7 = x6 * cospi_8_64 + x7 * cospi_24_64; + s7 = vi8_add(vi8_mul_wide(x6, cospi_8_64), vi8_mul_wide(x7, cospi_24_64)); + s8 = x8; s9 = x9; s10 = x10; s11 = x11; + // s12 = x12 * cospi_8_64 + x13 * cospi_24_64; + s12 = vi8_add(vi8_mul_wide(x12, cospi_8_64), vi8_mul_wide(x13, cospi_24_64)); + // s13 = x12 * cospi_24_64 - x13 * cospi_8_64; + s13 = vi8_sub(vi8_mul_wide(x12, cospi_24_64), vi8_mul_wide(x13, cospi_8_64)); + // s14 = -x14 * cospi_24_64 + x15 * cospi_8_64; + s14 = vi8_add(vi8_mul_wide(x14, (int16_t)(-cospi_24_64)), vi8_mul_wide(x15, cospi_8_64)); + // s15 = x14 * cospi_8_64 + x15 * cospi_24_64; + s15 = vi8_add(vi8_mul_wide(x14, cospi_8_64), vi8_mul_wide(x15, cospi_24_64)); + + // x0 = s0+s2; x1 = s1+s3; x2 = s0-s2; x3 = s1-s3; (wide, no round-shift) + x0 = vi8_add(s0, s2); + x1 = vi8_add(s1, s3); + x2 = vi8_sub(s0, s2); + x3 = vi8_sub(s1, s3); + // x4 = RS(s4+s6); x5 = RS(s5+s7); x6 = RS(s4-s6); x7 = RS(s5-s7); + x4 = vi8_round_shift(vi8_add(s4, s6)); + x5 = vi8_round_shift(vi8_add(s5, s7)); + x6 = vi8_round_shift(vi8_sub(s4, s6)); + x7 = vi8_round_shift(vi8_sub(s5, s7)); + // x8 = s8+s10; x9 = s9+s11; x10 = s8-s10; x11 = s9-s11; (wide) + x8 = vi8_add(s8, s10); + x9 = vi8_add(s9, s11); + x10 = vi8_sub(s8, s10); + x11 = vi8_sub(s9, s11); + // x12 = RS(s12+s14); x13 = RS(s13+s15); x14 = RS(s12-s14); x15 = RS(s13-s15); + x12 = vi8_round_shift(vi8_add(s12, s14)); + x13 = vi8_round_shift(vi8_add(s13, s15)); + x14 = vi8_round_shift(vi8_sub(s12, s14)); + x15 = vi8_round_shift(vi8_sub(s13, s15)); + + // Stage 4. Every combine here (x2+x3 etc.) and every multiply by + // cospi_16_64 must be wide -- this is the stage that, in iadst8_c, + // was shown to reach ~2.7e9 in magnitude for adversarial inputs; the + // same shape of expression appears 4 times here (x2/x3, x6/x7, + // x10/x11, x14/x15), so all 4 must go through vi8_mul_wide. + // s2 = -cospi_16_64 * (x2 + x3); s3 = cospi_16_64 * (x2 - x3); + s2 = vi8_mul_wide(vi8_add(x2, x3), (int16_t)(-cospi_16_64)); + s3 = vi8_mul_wide(vi8_sub(x2, x3), cospi_16_64); + // s6 = cospi_16_64 * (x6 + x7); s7 = cospi_16_64 * (-x6 + x7); + s6 = vi8_mul_wide(vi8_add(x6, x7), cospi_16_64); + s7 = vi8_mul_wide(vi8_sub(x7, x6), cospi_16_64); + // s10 = cospi_16_64 * (x10 + x11); s11 = cospi_16_64 * (-x10 + x11); + s10 = vi8_mul_wide(vi8_add(x10, x11), cospi_16_64); + s11 = vi8_mul_wide(vi8_sub(x11, x10), cospi_16_64); + // s14 = -cospi_16_64 * (x14 + x15); s15 = cospi_16_64 * (x14 - x15); + s14 = vi8_mul_wide(vi8_add(x14, x15), (int16_t)(-cospi_16_64)); + s15 = vi8_mul_wide(vi8_sub(x14, x15), cospi_16_64); + + // x2 = RS(s2); x3 = RS(s3); x6 = RS(s6); x7 = RS(s7); + // x10 = RS(s10); x11 = RS(s11); x14 = RS(s14); x15 = RS(s15); + x2 = vi8_round_shift(s2); + x3 = vi8_round_shift(s3); + x6 = vi8_round_shift(s6); + x7 = vi8_round_shift(s7); + x10 = vi8_round_shift(s10); + x11 = vi8_round_shift(s11); + x14 = vi8_round_shift(s14); + x15 = vi8_round_shift(s15); + + // output[0]=x0; output[1]=-x8; output[2]=x12; output[3]=-x4; + // output[4]=x6; output[5]=x14; output[6]=x10; output[7]=x2; + // output[8]=x3; output[9]=x11; output[10]=x15; output[11]=x7; + // output[12]=x5; output[13]=-x13; output[14]=x9; output[15]=-x1; + // Narrow to int16 only now (matches output[] being tran_low_t). + in[0] = vi8_pack(x0); + in[1] = vec_sub(k__const_0, vi8_pack(x8)); + in[2] = vi8_pack(x12); + in[3] = vec_sub(k__const_0, vi8_pack(x4)); + in[4] = vi8_pack(x6); + in[5] = vi8_pack(x14); + in[6] = vi8_pack(x10); + in[7] = vi8_pack(x2); + in[8] = vi8_pack(x3); + in[9] = vi8_pack(x11); + in[10] = vi8_pack(x15); + in[11] = vi8_pack(x7); + in[12] = vi8_pack(x5); + in[13] = vec_sub(k__const_0, vi8_pack(x13)); + in[14] = vi8_pack(x9); + in[15] = vec_sub(k__const_0, vi8_pack(x1)); } // Second tarpit: 16x16 IDCT, same eight columns. -- 2.43.0