From 37f8086e368b2c971ed0dba4e68e525b0fdd837e Mon Sep 17 00:00:00 2001 From: Sergey Fedorov Date: Tue, 14 Jul 2026 09:03:13 +0000 Subject: [PATCH 1/6] Support powerpc Enable the plain-AltiVec (VSX-less) PPC backend for big-endian G4/G5 targets, including 32-bit Darwin: gate runtime dispatch and the meson sub-target on ARCH_PPC instead of ARCH_PPC64LE, and add AltiVec-only fallbacks (vec_xl/vec_xst/vec_xxpermdi/vec_mergeo/etc.) in src/ppc/dav1d_types.h for the vec_* names that GCC's otherwise only exposes as VSX builtins. Fixes three bit-exact bugs in the AltiVec fallbacks/CDEF that corrupted CDEF/loop-restoration output on real big-endian hardware (verified against a qemu-ppc-emulated gcc -maltivec build): - u8h_to_u16/u8l_to_u16/u8h_to_i16/u8l_to_i16/u16h_to_i32/u16l_to_i32 had vec_mergeh/vec_mergel operands in the wrong order. On big-endian, vec_mergeh(a, b) puts a's byte in the high (MSB) position, so widening a pixel with vec_mergeh(v, zero) instead of vec_mergeh(zero, v) left-shifted every value by 8 bits instead of zero-extending it. - vec_xst's unaligned-store mask assumed vec_lvsr's permute indices mirror vec_lvsl's ((i - sh) mod 32), but real hardware gives (16 - sh + i) mod 32. The ones/zeros operand order in the mask's vec_perm must be swapped to match, or unaligned stores land in the wrong 16-byte block and corrupt neighboring memory. - copy4xN/copy8xN in src/ppc/cdef_tmpl.c built the CDEF padding buffer with the aligned vec_st, but tmp is never 16-byte aligned (tmp = tmp_buf + 2*tmp_stride + 2, i.e. tmp % 16 == 4). AltiVec vec_st masks off the low 4 address bits, so each row store landed at the wrong 16-byte-rounded address and scrambled the buffer; this hit the 4x4 filter in every case (whose rows are 16 bytes apart) and showed up as color/"purple" flare on object borders. Use the unaligned vec_vsx_st for every tmp store. --- include/common/attributes.h | 6 +- meson.build | 4 + src/cdef_tmpl.c | 4 +- src/cpu.c | 2 +- src/cpu.h | 7 +- src/looprestoration_tmpl.c | 4 +- src/meson.build | 46 +++++-- src/ppc/cdef.h | 3 +- src/ppc/cdef_tmpl.c | 44 ++++--- src/ppc/cpu.c | 5 +- src/ppc/cpu.h | 5 +- src/ppc/dav1d_types.h | 215 ++++++++++++++++++++++++++++++++- src/ppc/itx.h | 10 ++ src/ppc/itx_tmpl.c | 4 + src/ppc/loopfilter.h | 9 ++ src/ppc/loopfilter_tmpl.c | 4 + src/ppc/looprestoration.h | 3 +- src/ppc/looprestoration_tmpl.c | 14 +++ src/ppc/mc.h | 9 +- src/ppc/mc_tmpl.c | 4 + tests/checkasm/checkasm.c | 5 +- 21 files changed, 359 insertions(+), 48 deletions(-) diff --git a/include/common/attributes.h b/include/common/attributes.h index c6cecc7a..13c53d4a 100644 --- a/include/common/attributes.h +++ b/include/common/attributes.h @@ -60,7 +60,11 @@ #define ALIGN_64_VAL 64 #define ALIGN_32_VAL 32 #define ALIGN_16_VAL 16 -#elif ARCH_AARCH64 || ARCH_ARM || ARCH_LOONGARCH || ARCH_PPC64LE || ARCH_X86_32 +/* ARCH_PPC (not just ARCH_PPC64LE): the big-endian AltiVec (G4/G5) build also + * runs vector kernels, and its aligned vec_ld/vec_st silently mask the low 4 + * address bits — an ALIGN_STK_16 buffer that is only 8-byte aligned makes them + * read/write 8 bytes off. */ +#elif ARCH_AARCH64 || ARCH_ARM || ARCH_LOONGARCH || ARCH_PPC || ARCH_X86_32 /* ARM doesn't benefit from anything more than 16-byte alignment. */ #define ALIGN_64_VAL 16 #define ALIGN_32_VAL 16 diff --git a/meson.build b/meson.build index b7e4c4be..943085ab 100644 --- a/meson.build +++ b/meson.build @@ -70,6 +70,7 @@ is_asm_enabled = (get_option('enable_asm') == true and (host_machine.cpu_family() == 'aarch64' or host_machine.cpu_family().startswith('arm') or host_machine.cpu() == 'ppc64le' or + host_machine.cpu_family() == 'ppc' or host_machine.cpu_family().startswith('riscv') or host_machine.cpu_family().startswith('loongarch') or host_machine.cpu_family() == 'x86' or @@ -249,6 +250,7 @@ if (host_machine.cpu_family() == 'aarch64' or host_machine.cpu_family().startswith('arm') or host_machine.cpu_family().startswith('loongarch') or host_machine.cpu() == 'ppc64le' or + host_machine.cpu_family() == 'ppc' or host_machine.cpu_family().startswith('riscv')) have_getauxval = cc.has_function('getauxval', prefix : '#include ', args : test_args) have_elf_aux_info = cc.has_function('elf_aux_info', prefix : '#include ', args : test_args) @@ -511,6 +513,8 @@ if host_machine.cpu_family().startswith('x86') endif cdata.set10('ARCH_PPC64LE', host_machine.cpu() == 'ppc64le') +cdata.set10('ARCH_PPC', host_machine.cpu_family() == 'ppc') +cdata.set10('ARCH_PPC32', host_machine.cpu_family() == 'ppc' and host_machine.cpu() != 'ppc64le') cdata.set10('ARCH_RISCV', host_machine.cpu_family().startswith('riscv')) cdata.set10('ARCH_RV32', host_machine.cpu_family() == 'riscv32') diff --git a/src/cdef_tmpl.c b/src/cdef_tmpl.c index 4efea450..86323e1b 100644 --- a/src/cdef_tmpl.c +++ b/src/cdef_tmpl.c @@ -306,7 +306,7 @@ static int cdef_find_dir_c(const pixel *img, const ptrdiff_t stride, #if HAVE_ASM #if ARCH_AARCH64 || ARCH_ARM #include "src/arm/cdef.h" -#elif ARCH_PPC64LE +#elif ARCH_PPC #include "src/ppc/cdef.h" #elif ARCH_RISCV #include "src/riscv/cdef.h" @@ -326,7 +326,7 @@ COLD void bitfn(dav1d_cdef_dsp_init)(Dav1dCdefDSPContext *const c) { #if HAVE_ASM #if ARCH_AARCH64 || ARCH_ARM cdef_dsp_init_arm(c); -#elif ARCH_PPC64LE +#elif ARCH_PPC cdef_dsp_init_ppc(c); #elif ARCH_RISCV cdef_dsp_init_riscv(c); diff --git a/src/cpu.c b/src/cpu.c index 6b57fe4a..7994d23e 100644 --- a/src/cpu.c +++ b/src/cpu.c @@ -67,7 +67,7 @@ COLD void dav1d_init_cpu(void) { dav1d_cpu_flags = dav1d_get_cpu_flags_arm(); #elif ARCH_LOONGARCH dav1d_cpu_flags = dav1d_get_cpu_flags_loongarch(); -#elif ARCH_PPC64LE +#elif ARCH_PPC dav1d_cpu_flags = dav1d_get_cpu_flags_ppc(); #elif ARCH_RISCV dav1d_cpu_flags = dav1d_get_cpu_flags_riscv(); diff --git a/src/cpu.h b/src/cpu.h index 5d712bbb..cdc0c255 100644 --- a/src/cpu.h +++ b/src/cpu.h @@ -39,7 +39,7 @@ #include "src/arm/cpu.h" #elif ARCH_LOONGARCH #include "src/loongarch/cpu.h" -#elif ARCH_PPC64LE +#elif ARCH_PPC #include "src/ppc/cpu.h" #elif ARCH_RISCV #include "src/riscv/cpu.h" @@ -76,7 +76,10 @@ static ALWAYS_INLINE unsigned dav1d_get_default_cpu_flags(void) { flags |= DAV1D_ARM_CPU_FLAG_SVE2; #endif #endif /* ARCH_AARCH64 */ -#elif ARCH_PPC64LE +#elif ARCH_PPC +#if defined(__ALTIVEC__) + flags |= DAV1D_PPC_CPU_FLAG_ALTIVEC; +#endif #if defined(__VSX__) flags |= DAV1D_PPC_CPU_FLAG_VSX; #endif diff --git a/src/looprestoration_tmpl.c b/src/looprestoration_tmpl.c index c172462b..d7722a1a 100644 --- a/src/looprestoration_tmpl.c +++ b/src/looprestoration_tmpl.c @@ -1355,7 +1355,7 @@ vert_1: #include "src/arm/looprestoration.h" #elif ARCH_LOONGARCH64 #include "src/loongarch/looprestoration.h" -#elif ARCH_PPC64LE +#elif ARCH_PPC #include "src/ppc/looprestoration.h" #elif ARCH_X86 #include "src/x86/looprestoration.h" @@ -1375,7 +1375,7 @@ COLD void bitfn(dav1d_loop_restoration_dsp_init)(Dav1dLoopRestorationDSPContext loop_restoration_dsp_init_arm(c, bpc); #elif ARCH_LOONGARCH64 loop_restoration_dsp_init_loongarch(c, bpc); -#elif ARCH_PPC64LE +#elif ARCH_PPC loop_restoration_dsp_init_ppc(c, bpc); #elif ARCH_X86 loop_restoration_dsp_init_x86(c, bpc); diff --git a/src/meson.build b/src/meson.build index 108b7623..8b4283c5 100644 --- a/src/meson.build +++ b/src/meson.build @@ -249,21 +249,43 @@ if is_asm_enabled 'loongarch/itx.S', ) libdav1d_asm_objs += libdav1d_sources_asm - elif host_machine.cpu() == 'ppc64le' - arch_flags += {'vsx': ['-maltivec', '-mvsx', '-DDAV1D_VSX']} + elif host_machine.cpu() == 'ppc64le' or host_machine.cpu_family() == 'ppc' libdav1d_sources += files( 'ppc/cpu.c', ) - libdav1d_arch_tmpl_sources += {'vsx': files( - 'ppc/cdef_tmpl.c', - 'ppc/looprestoration_tmpl.c', - )} - arch_flags += {'pwr9': ['-mcpu=power9', '-DDAV1D_PWR9']} - libdav1d_arch_tmpl_sources += {'pwr9': files( - 'ppc/itx_tmpl.c', - 'ppc/loopfilter_tmpl.c', - 'ppc/mc_tmpl.c', - )} + + # VSX (POWER7+) and the POWER9 code paths use instructions that a + # plain AltiVec CPU (G4/G5) cannot execute. GCC accepts -mvsx / + # -mcpu=power9 even when targeting such a CPU, so we cannot rely on + # flag acceptance alone. Only build them for 64-bit little-endian + # POWER (ppc64le), where VSX is guaranteed. macOS/Darwin PowerPC is + # always a big-endian, VSX-less G4/G5 target, so it never gets these. + # The cdef/looprestoration templates carry a "_vsx" suffix on their + # exported symbols regardless of the flags used to build them; each + # must therefore be compiled in exactly ONE sub-target to avoid + # duplicate-symbol link errors. The ARCH_PPC64LE guards in + # ppc/{itx,mc,loopfilter}.h must stay in sync with this split. + if host_machine.cpu() == 'ppc64le' and host_machine.system() != 'darwin' + arch_flags += {'vsx': ['-maltivec', '-mvsx', '-DDAV1D_VSX']} + libdav1d_arch_tmpl_sources += {'vsx': files( + 'ppc/cdef_tmpl.c', + 'ppc/looprestoration_tmpl.c', + )} + arch_flags += {'pwr9': ['-mcpu=power9', '-DDAV1D_PWR9']} + libdav1d_arch_tmpl_sources += {'pwr9': files( + 'ppc/itx_tmpl.c', + 'ppc/loopfilter_tmpl.c', + 'ppc/mc_tmpl.c', + )} + else + # Big-endian 32/64-bit PowerPC (G4/G5): AltiVec baseline only. + # This runs on every VMX-capable PowerPC without needing VSX. + arch_flags += {'altivec': ['-maltivec', '-DDAV1D_ALTIVEC']} + libdav1d_arch_tmpl_sources += {'altivec': files( + 'ppc/cdef_tmpl.c', + 'ppc/looprestoration_tmpl.c', + )} + endif elif host_machine.cpu_family().startswith('riscv') libdav1d_sources += files( 'riscv/cpu.c', diff --git a/src/ppc/cdef.h b/src/ppc/cdef.h index b794ba53..88ea5e2c 100644 --- a/src/ppc/cdef.h +++ b/src/ppc/cdef.h @@ -31,6 +31,7 @@ #include "src/cdef.h" #include "src/cpu.h" +#include "src/ppc/cpu.h" #define cdef_vsx_fn(w, h) \ void dav1d_cdef_filter_##w##x##h##_vsx(pixel *const dst, \ @@ -51,7 +52,7 @@ cdef_vsx_fn(8, 8); static ALWAYS_INLINE void cdef_dsp_init_ppc(Dav1dCdefDSPContext *const c) { const unsigned flags = dav1d_get_cpu_flags(); - if (!(flags & DAV1D_PPC_CPU_FLAG_VSX)) return; + if (!(flags & (DAV1D_PPC_CPU_FLAG_VSX | DAV1D_PPC_CPU_FLAG_ALTIVEC))) return; #if BITDEPTH == 8 c->fb[0] = dav1d_cdef_filter_8x8_vsx; diff --git a/src/ppc/cdef_tmpl.c b/src/ppc/cdef_tmpl.c index 6ef87ad4..a4a6140c 100644 --- a/src/ppc/cdef_tmpl.c +++ b/src/ppc/cdef_tmpl.c @@ -56,6 +56,15 @@ static inline void copy4xN(uint16_t *tmp, int y_start = -2, y_end = h + 2; + /* + * All stores here go to tmp-relative addresses. tmp is NOT 16-byte + * aligned (cdef_fn sets tmp = tmp_buf + 2*tmp_stride + 2, i.e. +36/+68 + * bytes off a 16-aligned buffer => tmp % 16 == 4), and the 4-wide rows + * are 8 u16 = 16 bytes apart, so every tmp + y*8 address is also + * misaligned. The aligned vec_st masks off the low 4 address bits and + * would silently store to the wrong 16-byte-rounded location, scrambling + * the padding buffer. Use the unaligned vec_vsx_st for every tmp store. + */ // Copy top and bottom first if (!(edges & CDEF_HAVE_TOP)) { l0 = fill; @@ -66,8 +75,8 @@ static inline void copy4xN(uint16_t *tmp, l1 = u8h_to_u16(vec_vsx_ld(0, top + 1 * src_stride - 2)); } - vec_st(l0, 0, tmp - 2 * 8); - vec_st(l1, 0, tmp - 1 * 8); + vec_vsx_st(l0, 0, tmp - 2 * 8); + vec_vsx_st(l1, 0, tmp - 1 * 8); if (!(edges & CDEF_HAVE_BOTTOM)) { l0 = fill; @@ -78,8 +87,8 @@ static inline void copy4xN(uint16_t *tmp, l1 = u8h_to_u16(vec_vsx_ld(0, bottom + 1 * src_stride - 2)); } - vec_st(l0, 0, tmp + (h + 0) * 8); - vec_st(l1, 0, tmp + (h + 1) * 8); + vec_vsx_st(l0, 0, tmp + (h + 0) * 8); + vec_vsx_st(l1, 0, tmp + (h + 1) * 8); int y_with_left_edge = 0; if (!(edges & CDEF_HAVE_LEFT)) { @@ -91,7 +100,7 @@ static inline void copy4xN(uint16_t *tmp, for (int y = y_with_left_edge; y < h; y++) { u16x8 l = u8h_to_u16(vec_vsx_ld(0, src - 2 + y * src_stride)); - vec_st(l, 0, tmp + y * 8); + vec_vsx_st(l, 0, tmp + y * 8); } if (!(edges & CDEF_HAVE_LEFT)) { @@ -126,6 +135,11 @@ static inline void copy8xN(uint16_t *tmp, int y_start = -2, y_end = h + 2; + /* + * As in copy4xN, tmp is not 16-byte aligned (tmp % 16 == 4), so the + * aligned vec_st would round the store address down and corrupt the + * padding buffer. Every tmp store below uses the unaligned vec_vsx_st. + */ // Copy top and bottom first if (!(edges & CDEF_HAVE_TOP)) { l0h = fill; @@ -142,10 +156,10 @@ static inline void copy8xN(uint16_t *tmp, l1l = u8l_to_u16(l1); } - vec_st(l0h, 0, tmp - 4 * 8); - vec_st(l0l, 0, tmp - 3 * 8); - vec_st(l1h, 0, tmp - 2 * 8); - vec_st(l1l, 0, tmp - 1 * 8); + vec_vsx_st(l0h, 0, tmp - 4 * 8); + vec_vsx_st(l0l, 0, tmp - 3 * 8); + vec_vsx_st(l1h, 0, tmp - 2 * 8); + vec_vsx_st(l1l, 0, tmp - 1 * 8); if (!(edges & CDEF_HAVE_BOTTOM)) { l0h = fill; @@ -162,10 +176,10 @@ static inline void copy8xN(uint16_t *tmp, l1l = u8l_to_u16(l1); } - vec_st(l0h, 0, tmp + (h + 0) * 16); - vec_st(l0l, 0, tmp + (h + 0) * 16 + 8); - vec_st(l1h, 0, tmp + (h + 1) * 16); - vec_st(l1l, 0, tmp + (h + 1) * 16 + 8); + vec_vsx_st(l0h, 0, tmp + (h + 0) * 16); + vec_vsx_st(l0l, 0, tmp + (h + 0) * 16 + 8); + vec_vsx_st(l1h, 0, tmp + (h + 1) * 16); + vec_vsx_st(l1l, 0, tmp + (h + 1) * 16 + 8); int y_with_left_edge = 0; if (!(edges & CDEF_HAVE_LEFT)) { @@ -182,8 +196,8 @@ static inline void copy8xN(uint16_t *tmp, u8x16 l = vec_vsx_ld(0, src - 2 + y * src_stride); u16x8 lh = u8h_to_u16(l); u16x8 ll = u8l_to_u16(l); - vec_st(lh, 0, tmp + y * 16); - vec_st(ll, 0, tmp + 8 + y * 16); + vec_vsx_st(lh, 0, tmp + y * 16); + vec_vsx_st(ll, 0, tmp + 8 + y * 16); } if (!(edges & CDEF_HAVE_LEFT)) { diff --git a/src/ppc/cpu.c b/src/ppc/cpu.c index f6d9ac62..127cff08 100644 --- a/src/ppc/cpu.c +++ b/src/ppc/cpu.c @@ -32,7 +32,7 @@ #include "src/cpu.h" #include "src/ppc/cpu.h" -#define HAVE_AUX ((HAVE_GETAUXVAL || HAVE_ELF_AUX_INFO) && ARCH_PPC64LE) +#define HAVE_AUX ((HAVE_GETAUXVAL || HAVE_ELF_AUX_INFO) && (ARCH_PPC64LE || ARCH_PPC)) #if HAVE_AUX #include #endif @@ -42,8 +42,11 @@ COLD unsigned dav1d_get_cpu_flags_ppc(void) { #if HAVE_AUX unsigned long hw_cap = dav1d_getauxval(AT_HWCAP); unsigned long hw_cap2 = dav1d_getauxval(AT_HWCAP2); + flags |= (hw_cap & PPC_FEATURE_HAS_ALTIVEC) ? DAV1D_PPC_CPU_FLAG_ALTIVEC : 0; flags |= (hw_cap & PPC_FEATURE_HAS_VSX) ? DAV1D_PPC_CPU_FLAG_VSX : 0; flags |= (hw_cap2 & PPC_FEATURE2_ARCH_3_00) ? DAV1D_PPC_CPU_FLAG_PWR9 : 0; +#elif defined(__APPLE__) && (defined(__ppc__) || defined(__ppc64__)) + flags |= DAV1D_PPC_CPU_FLAG_ALTIVEC; #endif return flags; } diff --git a/src/ppc/cpu.h b/src/ppc/cpu.h index 02db7a60..9925cede 100644 --- a/src/ppc/cpu.h +++ b/src/ppc/cpu.h @@ -29,8 +29,9 @@ #define DAV1D_SRC_PPC_CPU_H enum CpuFlags { - DAV1D_PPC_CPU_FLAG_VSX = 1 << 0, - DAV1D_PPC_CPU_FLAG_PWR9 = 1 << 1, + DAV1D_PPC_CPU_FLAG_ALTIVEC = 1 << 0, + DAV1D_PPC_CPU_FLAG_VSX = 1 << 1, + DAV1D_PPC_CPU_FLAG_PWR9 = 1 << 2, }; unsigned dav1d_get_cpu_flags_ppc(void); diff --git a/src/ppc/dav1d_types.h b/src/ppc/dav1d_types.h index 9a8bc7a7..eeeeb02f 100644 --- a/src/ppc/dav1d_types.h +++ b/src/ppc/dav1d_types.h @@ -28,9 +28,30 @@ #ifndef DAV1D_SRC_PPC_TYPES_H #define DAV1D_SRC_PPC_TYPES_H +#include "config.h" +#include #include #undef pixel +/* + * On a plain AltiVec target (G4/G5, big-endian, no VSX) GCC's + * still predefines a number of convenience names — vec_xl, vec_xst, + * vec_xst_len, vec_mergeo, vec_xxpermdi, … — as object-like macros that + * expand to VSX builtins (__builtin_vsx_*). Those builtins require -mvsx and + * fail to compile under -maltivec alone. Undefine them here so the AltiVec-only + * fallbacks below are used instead. Only do this when VSX is not available; + * on a real VSX/LE build we want the native definitions. + */ +#if !defined(__VSX__) && !ARCH_PPC64LE +#undef vec_xl +#undef vec_xst +#undef vec_xst_len +#undef vec_mergeo +#undef vec_xxpermdi +#undef vec_vsx_ld +#undef vec_vsx_st +#endif + #define u8x16 vector unsigned char #define i8x16 vector signed char #define b8x16 vector bool char @@ -46,13 +67,195 @@ #define i8h_to_i16(v) ((i16x8) vec_unpackh((i8x16)v)) #define i8l_to_i16(v) ((i16x8) vec_unpackl((i8x16)v)) -#define u8h_to_i16(v) ((i16x8) vec_mergeh((u8x16) v, vec_splat_u8(0))) -#define u8l_to_i16(v) ((i16x8) vec_mergel((u8x16) v, vec_splat_u8(0))) -#define u8h_to_u16(v) ((u16x8) vec_mergeh((u8x16) v, vec_splat_u8(0))) -#define u8l_to_u16(v) ((u16x8) vec_mergel((u8x16) v, vec_splat_u8(0))) -#define u16h_to_i32(v) ((i32x4) vec_mergeh((u16x8) v, vec_splat_u16(0))) +#define u8h_to_i16(v) ((i16x8) vec_mergeh(vec_splat_u8(0), (u8x16) v)) +#define u8l_to_i16(v) ((i16x8) vec_mergel(vec_splat_u8(0), (u8x16) v)) +#define u8h_to_u16(v) ((u16x8) vec_mergeh(vec_splat_u8(0), (u8x16) v)) +#define u8l_to_u16(v) ((u16x8) vec_mergel(vec_splat_u8(0), (u8x16) v)) +#define u16h_to_i32(v) ((i32x4) vec_mergeh(vec_splat_u16(0), (u16x8) v)) #define i16h_to_i32(v) ((i32x4) vec_unpackh((i16x8)v)) -#define u16l_to_i32(v) ((i32x4) vec_mergel((u16x8) v, vec_splat_u16(0))) +#define u16l_to_i32(v) ((i32x4) vec_mergel(vec_splat_u16(0), (u16x8) v)) #define i16l_to_i32(v) ((i32x4) vec_unpackl((i16x8)v)) +#if !ARCH_PPC64LE + +static inline u8x16 aligned_ld(unsigned long offset, const void *ptr) { + return vec_ld(offset, (const unsigned char *)ptr); +} + +static inline void aligned_st(u8x16 v, unsigned long offset, void *ptr) { + vec_st(v, offset, (unsigned char *)ptr); +} + +/* + * The native VSX vec_xl / vec_vsx_ld are generic over the pointer's element + * type and return a vector whose element type matches it, so callers write + * e.g. `i16x8 v = vec_vsx_ld(0, short_ptr);`. Our AltiVec fallback must do the + * same: it loads 16 raw bytes (handling misalignment via lvsl/perm) and casts + * the result to a vector of the pointed-to element type so it matches whatever + * the caller expects. + * + * The result type is selected with _Generic on the pointer, keyed on the + * element types dav1d actually loads (uint8_t / int16_t / uint16_t). A + * statement-expression that declares a typedef cannot be used here: this macro + * is often nested inside another (e.g. u8h_to_u16(vec_vsx_ld(...))), and a + * ({ typedef ...; expr }) block is not accepted as a cast/call operand there. + */ +#define vec_xl_raw(offset, ptr) ({ \ + const unsigned char *_ptr = (const unsigned char *)(ptr) + (offset); \ + u8x16 _lo = vec_ld(0, _ptr); \ + u8x16 _hi = vec_ld(15, _ptr); \ + u8x16 _perm = vec_lvsl(0, _ptr); \ + vec_perm(_lo, _hi, _perm); \ +}) + +#define vec_xl(offset, ptr) _Generic((ptr), \ + const uint8_t *: (u8x16)vec_xl_raw(offset, ptr), \ + uint8_t *: (u8x16)vec_xl_raw(offset, ptr), \ + const int16_t *: (i16x8)vec_xl_raw(offset, ptr), \ + int16_t *: (i16x8)vec_xl_raw(offset, ptr), \ + const uint16_t *: (u16x8)vec_xl_raw(offset, ptr), \ + uint16_t *: (u16x8)vec_xl_raw(offset, ptr), \ + default: (u8x16)vec_xl_raw(offset, ptr)) + +/* + * vec_lvsr's permute indices are NOT the mirror image of vec_lvsl's: for a + * misalignment sh = addr & 15, lvsl[i] = (sh + i) mod 32 (as expected), but + * lvsr[i] = (16 - sh + i) mod 32 -- verified against real AltiVec hardware + * (qemu-ppc + gcc -maltivec), NOT the naively-assumed (i - sh) & 0x1F. That + * flips which operand of the ones/zeros vec_perm below is "selected" vs + * "background", so the mask is built as vec_perm(zeros, ones, perm) (zeros + * first) -- swapping this ordering silently corrupts every unaligned store. + */ +#define vec_xst(v, offset, ptr) do { \ + u8x16 _v = (u8x16)(v); \ + unsigned char *_ptr = (unsigned char *)(ptr) + (offset); \ + unsigned long _addr = (unsigned long)_ptr; \ + if ((_addr & 15) == 0) { \ + vec_st(_v, 0, _ptr); \ + } else { \ + u8x16 _lo = vec_ld(0, _ptr); \ + u8x16 _hi = vec_ld(15, _ptr); \ + u8x16 _perm = vec_lvsr(0, _ptr); \ + u8x16 _tmp = vec_perm(_v, _v, _perm); \ + u8x16 _ones = (u8x16)vec_splat_s8(-1); \ + u8x16 _zeros = (u8x16)vec_splat_s8(0); \ + u8x16 _sel = vec_perm(_zeros, _ones, _perm); \ + u8x16 _result_lo = vec_sel(_lo, _tmp, (b8x16)_sel); \ + u8x16 _result_hi = vec_sel(_tmp, _hi, (b8x16)_sel); \ + vec_st(_result_lo, 0, _ptr); \ + vec_st(_result_hi, 16, _ptr); \ + } \ +} while(0) + +#define vec_xst_len(v, ptr, len) do { \ + unsigned char *_dst = (unsigned char *)(ptr); \ + u8x16 _v = (u8x16)(v); \ + if ((len) == 16) { \ + vec_xst(_v, 0, _dst); \ + } else { \ + unsigned char _tmp[16] __attribute__((aligned(16))); \ + vec_st(_v, 0, _tmp); \ + for (int _i = 0; _i < (len); _i++) { \ + _dst[_i] = _tmp[_i]; \ + } \ + } \ +} while(0) + +#define vec_vsx_ld(offset, ptr) vec_xl(offset, ptr) +#define vec_vsx_st(v, offset, ptr) vec_xst(v, offset, ptr) + +#define vec_mergeo(a, b) ({ \ + __typeof__(a) _a = (a); __typeof__(b) _b = (b); \ + (__typeof__(a))vec_perm((u8x16)_a, (u8x16)_b, \ + (u8x16){4,5,6,7,20,21,22,23,12,13,14,15,28,29,30,31}); \ +}) + +#ifndef vec_cmpne +#define vec_cmpne(a, b) vec_nor(vec_cmpeq(a, b), vec_cmpeq(a, b)) +#endif + +#ifndef vec_xxpermdi +/* + * AltiVec-only fallback for the VSX xxpermdi instruction, using vec_perm. + * xxpermdi(a, b, c) builds a result from one doubleword of `a` and one of `b`. + * On big-endian, doubleword 0 = bytes 0..7 (high), doubleword 1 = bytes 8..15 + * (low). The byte selectors below encode, for each 2-bit `c`: + * c==0: hi(a), hi(b) c==1: hi(a), lo(b) + * c==2: lo(a), hi(b) c==3: lo(a), lo(b) + * The cast to __typeof__(a) keeps the macro's result type matching the VSX + * builtin (vec_perm returns the type of its first argument, so callers passing + * e.g. i16x8 get i16x8 back). + */ +#define vec_xxpermdi(a, b, c) ({ \ + __typeof__(a) _a = (a); \ + __typeof__(b) _b = (b); \ + (__typeof__(a))( \ + (c) == 0 ? vec_perm((u8x16)_a, (u8x16)_b, (u8x16){0,1,2,3,4,5,6,7,16,17,18,19,20,21,22,23}) : \ + (c) == 1 ? vec_perm((u8x16)_a, (u8x16)_b, (u8x16){0,1,2,3,4,5,6,7,24,25,26,27,28,29,30,31}) : \ + (c) == 2 ? vec_perm((u8x16)_a, (u8x16)_b, (u8x16){8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23}) : \ + vec_perm((u8x16)_a, (u8x16)_b, (u8x16){8,9,10,11,12,13,14,15,24,25,26,27,28,29,30,31})); \ +}) +#endif + +static inline u8x16 vec_absd_u8(u8x16 a, u8x16 b) { + return vec_max(a, b) - vec_min(a, b); +} + +static inline u16x8 vec_absd_u16(u16x8 a, u16x8 b) { + return vec_max(a, b) - vec_min(a, b); +} + +#ifndef vec_absd +#define vec_absd(a, b) _Generic((a), \ + u8x16: vec_absd_u8, \ + u16x8: vec_absd_u16 \ +)(a, b) +#endif + +static inline u8x16 vec_splats_u8(unsigned char val) { + u8x16 v = {val, val, val, val, val, val, val, val, + val, val, val, val, val, val, val, val}; + return v; +} + +static inline i8x16 vec_splats_s8(signed char val) { + i8x16 v = {val, val, val, val, val, val, val, val, + val, val, val, val, val, val, val, val}; + return v; +} + +static inline u16x8 vec_splats_u16(unsigned short val) { + u16x8 v = {val, val, val, val, val, val, val, val}; + return v; +} + +static inline i16x8 vec_splats_s16(short val) { + i16x8 v = {val, val, val, val, val, val, val, val}; + return v; +} + +static inline u32x4 vec_splats_u32(unsigned int val) { + u32x4 v = {val, val, val, val}; + return v; +} + +static inline i32x4 vec_splats_s32(int val) { + i32x4 v = {val, val, val, val}; + return v; +} + +#ifndef vec_splats +#define vec_splats(val) _Generic((val), \ + unsigned char: vec_splats_u8, \ + signed char: vec_splats_s8, \ + char: vec_splats_u8, \ + unsigned short: vec_splats_u16, \ + short: vec_splats_s16, \ + unsigned int: vec_splats_u32, \ + int: vec_splats_s32 \ +)(val) +#endif + +#endif + #endif /* DAV1D_SRC_PPC_TYPES_H */ diff --git a/src/ppc/itx.h b/src/ppc/itx.h index 6bddf7a3..f27ebf4c 100644 --- a/src/ppc/itx.h +++ b/src/ppc/itx.h @@ -28,6 +28,10 @@ #include "src/cpu.h" #include "src/itx.h" +/* The POWER9 code paths are only built for little-endian POWER (ppc64le); + * on big-endian/AltiVec-only targets (e.g. G4/G5) they are absent, so the + * declarations and assignments below must not reference their symbols. */ +#if ARCH_PPC64LE decl_itx17_fns( 4, 4, pwr9); decl_itx16_fns( 4, 8, pwr9); decl_itx16_fns( 4, 16, pwr9); @@ -48,8 +52,10 @@ decl_itx_fn(BF(dav1d_inv_txfm_add_dct_dct_32x64, pwr9)); decl_itx_fn(BF(dav1d_inv_txfm_add_dct_dct_64x16, pwr9)); decl_itx_fn(BF(dav1d_inv_txfm_add_dct_dct_64x32, pwr9)); decl_itx_fn(BF(dav1d_inv_txfm_add_dct_dct_64x64, pwr9)); +#endif /* ARCH_PPC64LE */ static ALWAYS_INLINE void itx_dsp_init_ppc(Dav1dInvTxfmDSPContext *const c, const int bpc) { +#if ARCH_PPC64LE const unsigned flags = dav1d_get_cpu_flags(); if (!(flags & DAV1D_PPC_CPU_FLAG_PWR9)) return; @@ -62,4 +68,8 @@ static ALWAYS_INLINE void itx_dsp_init_ppc(Dav1dInvTxfmDSPContext *const c, cons assign_itx16_fn(R, 4, 16, pwr9); assign_itx16_fn(R, 16, 4, pwr9); #endif +#else + (void)c; + (void)bpc; +#endif /* ARCH_PPC64LE */ } diff --git a/src/ppc/itx_tmpl.c b/src/ppc/itx_tmpl.c index 81806552..9740c0f8 100644 --- a/src/ppc/itx_tmpl.c +++ b/src/ppc/itx_tmpl.c @@ -25,6 +25,8 @@ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +#if defined(__VSX__) || defined(__POWER9_VECTOR__) + #include "src/ppc/dav1d_types.h" #include "src/ppc/itx.h" #include "src/ppc/utils.h" @@ -2004,3 +2006,5 @@ inv_txfm_fn16x4_identity(adst) inv_txfm_fn16x4_identity(flipadst) #endif // BITDEPTH + +#endif // __VSX__ || __POWER9_VECTOR__ diff --git a/src/ppc/loopfilter.h b/src/ppc/loopfilter.h index fc97b375..6b857a8c 100644 --- a/src/ppc/loopfilter.h +++ b/src/ppc/loopfilter.h @@ -26,14 +26,20 @@ */ #include "src/cpu.h" +#include "src/ppc/cpu.h" #include "src/loopfilter.h" +/* The POWER9 loop filter is only built for little-endian POWER (ppc64le); + * on big-endian/AltiVec-only targets (e.g. G4/G5) its symbols do not exist. */ +#if ARCH_PPC64LE decl_loopfilter_sb_fn(BF(dav1d_lpf_h_sb_y, pwr9)); decl_loopfilter_sb_fn(BF(dav1d_lpf_v_sb_y, pwr9)); decl_loopfilter_sb_fn(BF(dav1d_lpf_h_sb_uv, pwr9)); decl_loopfilter_sb_fn(BF(dav1d_lpf_v_sb_uv, pwr9)); +#endif static ALWAYS_INLINE void loop_filter_dsp_init_ppc(Dav1dLoopFilterDSPContext *const c) { +#if ARCH_PPC64LE const unsigned flags = dav1d_get_cpu_flags(); if (!(flags & DAV1D_PPC_CPU_FLAG_PWR9)) return; @@ -44,4 +50,7 @@ static ALWAYS_INLINE void loop_filter_dsp_init_ppc(Dav1dLoopFilterDSPContext *co c->loop_filter_sb[1][0] = BF(dav1d_lpf_h_sb_uv, pwr9); c->loop_filter_sb[1][1] = BF(dav1d_lpf_v_sb_uv, pwr9); #endif +#else + (void)c; +#endif /* ARCH_PPC64LE */ } diff --git a/src/ppc/loopfilter_tmpl.c b/src/ppc/loopfilter_tmpl.c index 107192f8..ac21cf9a 100644 --- a/src/ppc/loopfilter_tmpl.c +++ b/src/ppc/loopfilter_tmpl.c @@ -28,6 +28,8 @@ #include "config.h" +#if defined(__VSX__) || defined(__POWER9_VECTOR__) + #undef NDEBUG #include @@ -1688,3 +1690,5 @@ void LPF(v_sb_uv)(pixel *dst, const ptrdiff_t stride, } #endif // BITDEPTH + +#endif // __VSX__ || __POWER9_VECTOR__ diff --git a/src/ppc/looprestoration.h b/src/ppc/looprestoration.h index 614234ab..ef2c3f44 100644 --- a/src/ppc/looprestoration.h +++ b/src/ppc/looprestoration.h @@ -28,6 +28,7 @@ #include "common/intops.h" #include "src/cpu.h" +#include "src/ppc/cpu.h" #include "src/looprestoration.h" void dav1d_wiener_filter_vsx(uint8_t *p, const ptrdiff_t stride, @@ -40,7 +41,7 @@ void dav1d_wiener_filter_vsx(uint8_t *p, const ptrdiff_t stride, static ALWAYS_INLINE void loop_restoration_dsp_init_ppc(Dav1dLoopRestorationDSPContext *const c, const int bpc) { const unsigned flags = dav1d_get_cpu_flags(); - if (!(flags & DAV1D_PPC_CPU_FLAG_VSX)) return; + if (!(flags & (DAV1D_PPC_CPU_FLAG_VSX | DAV1D_PPC_CPU_FLAG_ALTIVEC))) return; #if BITDEPTH == 8 c->wiener[0] = c->wiener[1] = dav1d_wiener_filter_vsx; diff --git a/src/ppc/looprestoration_tmpl.c b/src/ppc/looprestoration_tmpl.c index 76c1d07f..f0c028e6 100644 --- a/src/ppc/looprestoration_tmpl.c +++ b/src/ppc/looprestoration_tmpl.c @@ -76,12 +76,26 @@ static void wiener_filter_h_vsx(int32_t *hor_ptr, u8x16 tmp_v0 = vec_ld(0, &tmp_ptr[i]); u8x16 tmp_v7 = vec_ld(0, &tmp_ptr[i+16]); + /* tmp_vk must hold the 16 pixels starting k bytes into the row + * (memory order). vec_sld concatenates and shifts in register + * element order, which matches memory order only on big-endian; + * the upstream form vec_sld(tmp_v7, tmp_v0, 16 - k) is the + * little-endian compensation and produces rotated garbage on BE. */ +#if ENDIANNESS_BIG + u8x16 tmp_v1 = vec_sld( tmp_v0, tmp_v7, 1); + u8x16 tmp_v2 = vec_sld( tmp_v0, tmp_v7, 2); + u8x16 tmp_v3 = vec_sld( tmp_v0, tmp_v7, 3); + u8x16 tmp_v4 = vec_sld( tmp_v0, tmp_v7, 4); + u8x16 tmp_v5 = vec_sld( tmp_v0, tmp_v7, 5); + u8x16 tmp_v6 = vec_sld( tmp_v0, tmp_v7, 6); +#else u8x16 tmp_v1 = vec_sld( tmp_v7, tmp_v0, 15); u8x16 tmp_v2 = vec_sld( tmp_v7, tmp_v0, 14); u8x16 tmp_v3 = vec_sld( tmp_v7, tmp_v0, 13); u8x16 tmp_v4 = vec_sld( tmp_v7, tmp_v0, 12); u8x16 tmp_v5 = vec_sld( tmp_v7, tmp_v0, 11); u8x16 tmp_v6 = vec_sld( tmp_v7, tmp_v0, 10); +#endif u16x8 tmp_u16_high = u8h_to_u16(tmp_v3); u16x8 tmp_u16_low = u8l_to_u16(tmp_v3); diff --git a/src/ppc/mc.h b/src/ppc/mc.h index 44fa142a..b946d47f 100644 --- a/src/ppc/mc.h +++ b/src/ppc/mc.h @@ -28,11 +28,16 @@ #include "src/cpu.h" #include "src/mc.h" +/* The POWER9 mc code is only built for little-endian POWER (ppc64le); + * on big-endian/AltiVec-only targets (e.g. G4/G5) its symbols do not exist. */ +#if ARCH_PPC64LE decl_blend_fn(BF(dav1d_blend, pwr9)); decl_blend_dir_fn(BF(dav1d_blend_h, pwr9)); decl_blend_dir_fn(BF(dav1d_blend_v, pwr9)); +#endif static ALWAYS_INLINE void mc_dsp_init_ppc(Dav1dMCDSPContext *const c) { +#if ARCH_PPC64LE const unsigned flags = dav1d_get_cpu_flags(); if (!(flags & DAV1D_PPC_CPU_FLAG_PWR9)) return; @@ -42,5 +47,7 @@ static ALWAYS_INLINE void mc_dsp_init_ppc(Dav1dMCDSPContext *const c) { c->blend_h = BF(dav1d_blend_h, pwr9); c->blend_v = BF(dav1d_blend_v, pwr9); #endif - +#else + (void)c; +#endif /* ARCH_PPC64LE */ } diff --git a/src/ppc/mc_tmpl.c b/src/ppc/mc_tmpl.c index e00058e2..262eab35 100644 --- a/src/ppc/mc_tmpl.c +++ b/src/ppc/mc_tmpl.c @@ -25,6 +25,8 @@ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +#if defined(__VSX__) || defined(__POWER9_VECTOR__) + #include "common/attributes.h" #include "src/ppc/mc.h" #include "src/tables.h" @@ -552,3 +554,5 @@ void dav1d_blend_h_8bpc_pwr9(pixel *dst, const ptrdiff_t dst_stride, const pixel } #endif // BITDEPTH + +#endif // __VSX__ || __POWER9_VECTOR__ diff --git a/tests/checkasm/checkasm.c b/tests/checkasm/checkasm.c index d9179f73..e776423c 100644 --- a/tests/checkasm/checkasm.c +++ b/tests/checkasm/checkasm.c @@ -79,9 +79,12 @@ static const CheckasmCpuInfo flags[] = { #elif ARCH_LOONGARCH { "LSX", "lsx", DAV1D_LOONGARCH_CPU_FLAG_LSX }, { "LASX", "lasx", DAV1D_LOONGARCH_CPU_FLAG_LASX }, -#elif ARCH_PPC64LE +#elif ARCH_PPC + { "ALTIVEC", "altivec", DAV1D_PPC_CPU_FLAG_ALTIVEC }, +#if ARCH_PPC64LE { "VSX", "vsx", DAV1D_PPC_CPU_FLAG_VSX }, { "PWR9", "pwr9", DAV1D_PPC_CPU_FLAG_PWR9 }, +#endif #elif ARCH_RISCV { "RVV", "rvv", DAV1D_RISCV_CPU_FLAG_V }, #endif -- 2.43.0