media-libs/x265: allow building with c++17, small ebuild fixes

The new x265-3.5-r5-cpp-std.patch disabled setting c++98 or c++11 as versions.
This made newer compilers default to c++17 and thus `register` a unused and
reserved keyword.

While on it fix some obvious things that might cause problems:
	one shadowed variable
  use std::abs	in place of abs so we don't accidentally truncate values
	use boolean instead of bitwise comparison when comparing bool

Move ENABLE_CLI=OFF for non native multilib builds to make it clearer.
Don't set ENABLE_TESTS=no on platforms where we set ENABLE_ASSEMBLY=OFF, because
it requires it to be ON and this the option is never created.

Closes: https://bugs.gentoo.org/938515
Signed-off-by: Paul Zander <negril.nx+gentoo@gmail.com>
Closes: https://github.com/gentoo/gentoo/pull/38289
Signed-off-by: Sam James <sam@gentoo.org>

5 files changed, 217 insertions, 6 deletions

diff --git a/media-libs/x265/files/x265-3.6-cmake-cleanup.patch b/media-libs/x265/files/x265-3.6-cmake-cleanup.patch
new file mode 100644
index 000000000000..7cbe3722f1e0
--- /dev/null
+++ b/media-libs/x265/files/x265-3.6-cmake-cleanup.patch
@@ -0,0 +1,47 @@
+From d6b009be2aa62c679ea994125c02310d4b8d401b Mon Sep 17 00:00:00 2001
+From: Paul Zander <negril.nx+gentoo@gmail.com>
+Date: Mon, 26 Aug 2024 14:23:49 +0200
+Subject: [PATCH 1/2] cmake: cleanup old policies
+
+Signed-off-by: Paul Zander <negril.nx+gentoo@gmail.com>
+
+--- a/CMakeLists.txt
++++ b/CMakeLists.txt
+@@ -6,18 +6,9 @@ if(NOT CMAKE_BUILD_TYPE)
+         FORCE)
+ endif()
+ message(STATUS "cmake version ${CMAKE_VERSION}")
+-if(POLICY CMP0025)
+-    cmake_policy(SET CMP0025 OLD) # report Apple's Clang as just Clang
+-endif()
+-if(POLICY CMP0042)
+-    cmake_policy(SET CMP0042 NEW) # MACOSX_RPATH
+-endif()
+-if(POLICY CMP0054)
+-    cmake_policy(SET CMP0054 OLD) # Only interpret if() arguments as variables or keywords when unquoted
+-endif()
+ 
++cmake_minimum_required (VERSION 3.28) # OBJECT libraries require 2.8.8
+ project (x265)
+-cmake_minimum_required (VERSION 2.8.8) # OBJECT libraries require 2.8.8
+ include(CheckIncludeFiles)
+ include(CheckFunctionExists)
+ include(CheckSymbolExists)
+
+From 701cc9679e9453cc94afac7904657262109e56b5 Mon Sep 17 00:00:00 2001
+From: Paul Zander <negril.nx+gentoo@gmail.com>
+Date: Mon, 26 Aug 2024 14:23:59 +0200
+Subject: [PATCH 2/2] cmake: fix nasm warning
+
+Signed-off-by: Paul Zander <negril.nx+gentoo@gmail.com>
+
+--- a/cmake/FindNasm.cmake
++++ b/cmake/FindNasm.cmake
+@@ -20,6 +20,6 @@ if(NASM_EXECUTABLE)
+ endif()
+ 
+ # Provide standardized success/failure messages
+-find_package_handle_standard_args(nasm
++find_package_handle_standard_args(Nasm
+     REQUIRED_VARS NASM_EXECUTABLE
+     VERSION_VAR NASM_VERSION_STRING)
diff --git a/media-libs/x265/files/x265-3.6-code-cleanup.patch b/media-libs/x265/files/x265-3.6-code-cleanup.patch
new file mode 100644
index 000000000000..dbea7ef6c1ff
--- /dev/null
+++ b/media-libs/x265/files/x265-3.6-code-cleanup.patch
@@ -0,0 +1,140 @@
+From bca0b4b72977683794d441e57adc8c279ff4bfca Mon Sep 17 00:00:00 2001
+From: Paul Zander <negril.nx+gentoo@gmail.com>
+Date: Mon, 26 Aug 2024 14:25:35 +0200
+Subject: [PATCH 1/4] use boolean compare instead of bitwise
+
+Signed-off-by: Paul Zander <negril.nx+gentoo@gmail.com>
+
+diff --git a/common/cudata.cpp b/common/cudata.cpp
+index 19281de..d5b295f 100644
+--- a/common/cudata.cpp
++++ b/common/cudata.cpp
+@@ -73,7 +73,7 @@ inline bool isEqualRow(int addrA, int addrB)
+ /* Check whether 2 addresses point to the same row or column */
+ inline bool isEqualRowOrCol(int addrA, int addrB)
+ {
+-    return isEqualCol(addrA, addrB) | isEqualRow(addrA, addrB);
++    return isEqualCol(addrA, addrB) || isEqualRow(addrA, addrB);
+ }
+ 
+ /* Check whether one address points to the first column */
+
+From da3eb3ea55be74e440b272e24d7d8e67cb7a76db Mon Sep 17 00:00:00 2001
+From: Paul Zander <negril.nx+gentoo@gmail.com>
+Date: Mon, 26 Aug 2024 14:44:55 +0200
+Subject: [PATCH 2/4] fix variable shadowing
+
+Signed-off-by: Paul Zander <negril.nx+gentoo@gmail.com>
+
+diff --git a/common/cudata.h b/common/cudata.h
+index 8397f05..7f132e6 100644
+--- a/common/cudata.h
++++ b/common/cudata.h
+@@ -48,7 +48,7 @@ enum PartSize
+     SIZE_2NxnD, // asymmetric motion partition, 2Nx(3N/2) + 2Nx( N/2)
+     SIZE_nLx2N, // asymmetric motion partition, ( N/2)x2N + (3N/2)x2N
+     SIZE_nRx2N, // asymmetric motion partition, (3N/2)x2N + ( N/2)x2N
+-    NUM_SIZES
++    PART_NUM_SIZES
+ };
+ 
+ enum PredMode
+
+From 4ec09af244e2cfe3dfb739d74af7640ac114e775 Mon Sep 17 00:00:00 2001
+From: Paul Zander <negril.nx+gentoo@gmail.com>
+Date: Mon, 26 Aug 2024 14:45:33 +0200
+Subject: [PATCH 3/4] register is a unused and reserved keyword in c++-17
+
+Signed-off-by: Paul Zander <negril.nx+gentoo@gmail.com>
+
+diff --git a/common/md5.cpp b/common/md5.cpp
+index a9042f4..7e638e7 100644
+--- a/common/md5.cpp
++++ b/common/md5.cpp
+@@ -185,7 +185,10 @@ void MD5Final(MD5Context *ctx, uint8_t *digest)
+  */
+ void MD5Transform(uint32_t *buf, uint32_t *in)
+ {
+-    register uint32_t a, b, c, d;
++#if __cplusplus < 201703L
++    register
++#endif
++    uint32_t a, b, c, d;
+ 
+     a = buf[0];
+     b = buf[1];
+
+From 09379bbfe797e54f91ef5702c802f75aad604067 Mon Sep 17 00:00:00 2001
+From: Paul Zander <negril.nx+gentoo@gmail.com>
+Date: Mon, 26 Aug 2024 14:26:55 +0200
+Subject: [PATCH 4/4] use std::abs instead of abs to avoid truncating values
+
+Signed-off-by: Paul Zander <negril.nx+gentoo@gmail.com>
+
+diff --git a/common/pixel.cpp b/common/pixel.cpp
+index 3cd074c..62410f3 100644
+--- a/common/pixel.cpp
++++ b/common/pixel.cpp
+@@ -124,10 +124,10 @@ int ads_x4(int encDC[4], uint32_t *sums, int delta, uint16_t *costMvX, int16_t *
+     int nmv = 0;
+     for (int16_t i = 0; i < width; i++, sums++)
+     {
+-        int ads = abs(encDC[0] - long(sums[0]))
+-            + abs(encDC[1] - long(sums[lx >> 1]))
+-            + abs(encDC[2] - long(sums[delta]))
+-            + abs(encDC[3] - long(sums[delta + (lx >> 1)]))
++        int ads = std::abs(encDC[0] - long(sums[0]))
++            + std::abs(encDC[1] - long(sums[lx >> 1]))
++            + std::abs(encDC[2] - long(sums[delta]))
++            + std::abs(encDC[3] - long(sums[delta + (lx >> 1)]))
+             + costMvX[i];
+         if (ads < thresh)
+             mvs[nmv++] = i;
+@@ -141,8 +141,8 @@ int ads_x2(int encDC[2], uint32_t *sums, int delta, uint16_t *costMvX, int16_t *
+     int nmv = 0;
+     for (int16_t i = 0; i < width; i++, sums++)
+     {
+-        int ads = abs(encDC[0] - long(sums[0]))
+-            + abs(encDC[1] - long(sums[delta]))
++        int ads = std::abs(encDC[0] - long(sums[0]))
++            + std::abs(encDC[1] - long(sums[delta]))
+             + costMvX[i];
+         if (ads < thresh)
+             mvs[nmv++] = i;
+@@ -156,7 +156,7 @@ int ads_x1(int encDC[1], uint32_t *sums, int, uint16_t *costMvX, int16_t *mvs, i
+     int nmv = 0;
+     for (int16_t i = 0; i < width; i++, sums++)
+     {
+-        int ads = abs(encDC[0] - long(sums[0]))
++        int ads = std::abs(encDC[0] - long(sums[0]))
+             + costMvX[i];
+         if (ads < thresh)
+             mvs[nmv++] = i;
+diff --git a/encoder/analysis.cpp b/encoder/analysis.cpp
+index aabf386..127032d 100644
+--- a/encoder/analysis.cpp
++++ b/encoder/analysis.cpp
+@@ -2692,8 +2692,8 @@ void Analysis::classifyCU(const CUData& ctu, const CUGeom& cuGeom, const Mode& b
+             {
+                 offset = (depth * X265_REFINE_INTER_LEVELS) + i;
+                 /* Calculate distance values */
+-                diffRefine[i] = abs((int64_t)(trainData.cuVariance - m_frame->m_classifyVariance[offset]));
+-                diffRefineRd[i] = abs((int64_t)(cuCost - m_frame->m_classifyRd[offset]));
++                diffRefine[i] = std::abs((int64_t)(trainData.cuVariance - m_frame->m_classifyVariance[offset]));
++                diffRefineRd[i] = std::abs((int64_t)(cuCost - m_frame->m_classifyRd[offset]));
+ 
+                 /* Calculate prior probability - ranges between 0 and 1 */
+                 if (trainingCount)
+diff --git a/encoder/ratecontrol.cpp b/encoder/ratecontrol.cpp
+index 9f2b8d9..7732ccd 100644
+--- a/encoder/ratecontrol.cpp
++++ b/encoder/ratecontrol.cpp
+@@ -1891,7 +1891,7 @@ double RateControl::tuneQScaleForGrain(double rcOverflow)
+     int newQp = rcOverflow > 1.1 ? curQp + 2 : rcOverflow > 1 ? curQp + 1 : curQp - 1 ;
+     double projectedBitrate =  int(m_fps + 0.5) * m_qpToEncodedBits[newQp];
+     if (curBitrate > 0 && projectedBitrate > 0)
+-        q =  abs(projectedBitrate - m_bitrate) < abs (curBitrate - m_bitrate) ? x265_qp2qScale(newQp) : m_lastQScaleFor[P_SLICE];
++        q =  std::abs(projectedBitrate - m_bitrate) < std::abs (curBitrate - m_bitrate) ? x265_qp2qScale(newQp) : m_lastQScaleFor[P_SLICE];
+     else
+         q = rcOverflow > 1 ? qScaleAvg * qpstep : rcOverflow < 1 ?  qScaleAvg / qpstep : m_lastQScaleFor[P_SLICE];
+     return q;
diff --git a/media-libs/x265/x265-3.5-r5.ebuild b/media-libs/x265/x265-3.5-r5.ebuild
index ad78f487735a..c9db10613904 100644
--- a/media-libs/x265/x265-3.5-r5.ebuild
+++ b/media-libs/x265/x265-3.5-r5.ebuild
@@ -42,6 +42,8 @@ PATCHES=(
 	"${FILESDIR}/${PN}-3.5-r5-cpp-std.patch"
 	"${FILESDIR}/${PN}-3.5-r5-gcc15.patch"
 	"${FILESDIR}/${PN}-3.5-r5-test-ns_2.patch"
+	"${FILESDIR}/${PN}-3.6-cmake-cleanup.patch"
+	"${FILESDIR}/${PN}-3.6-code-cleanup.patch"
 )
 
 pkg_setup() {
@@ -99,7 +101,6 @@ multilib_src_configure() {
 	filter-lto
 
 	local mycmakeargs=(
-		$(multilib_is_native_abi || echo "-DENABLE_CLI=OFF")
 		-DENABLE_PIC=ON
 		-DENABLE_LIBNUMA="$(usex numa)"
 		-DENABLE_SVT_HEVC="no" # missing
@@ -107,6 +108,11 @@ multilib_src_configure() {
 		-DGIT_ARCHETYPE=1 #814116
 		-DLIB_INSTALL_DIR="$(get_libdir)"
 	)
+	if multilib_is_native_abi; then
+		mycmakeargs+=(
+			-DENABLE_CLI="no"
+		)
+	fi
 
 	# Unfortunately, the asm for x86/x32/arm isn't PIC-safe.
 	# x86 # Bug #528202, bug #913412
@@ -114,7 +120,9 @@ multilib_src_configure() {
 	if [[ ${ABI} = x86 ]] || [[ ${ABI} = x32 ]] || [[ ${ABI} = arm ]] ; then
 		mycmakeargs+=(
 			-DENABLE_ASSEMBLY=OFF
-			-DENABLE_TESTS="no" #728748
+			# ENABLE_TESTS requires ENABLE_ASSEMBLY=ON to be visible
+			# source/CMakeLists.txt:858
+			# -DENABLE_TESTS="no" #728748
 		)
 	else
 		mycmakeargs+=(
diff --git a/media-libs/x265/x265-3.6.ebuild b/media-libs/x265/x265-3.6.ebuild
index 064bd95be46a..ba013c73d378 100644
--- a/media-libs/x265/x265-3.6.ebuild
+++ b/media-libs/x265/x265-3.6.ebuild
@@ -42,6 +42,8 @@ PATCHES=(
 	"${FILESDIR}/${PN}-3.5-r5-cpp-std.patch"
 	"${FILESDIR}/${PN}-3.5-r5-gcc15.patch"
 	"${FILESDIR}/${PN}-3.6-test-ns_2.patch"
+	"${FILESDIR}/${PN}-3.6-cmake-cleanup.patch"
+	"${FILESDIR}/${PN}-3.6-code-cleanup.patch"
 )
 
 pkg_setup() {
@@ -99,7 +101,6 @@ multilib_src_configure() {
 	filter-lto
 
 	local mycmakeargs=(
-		$(multilib_is_native_abi || echo "-DENABLE_CLI=OFF")
 		-DENABLE_PIC=ON
 		-DENABLE_LIBNUMA="$(usex numa)"
 		-DENABLE_SVT_HEVC="no" # missing
@@ -107,6 +108,11 @@ multilib_src_configure() {
 		-DGIT_ARCHETYPE=1 #814116
 		-DLIB_INSTALL_DIR="$(get_libdir)"
 	)
+	if multilib_is_native_abi; then
+		mycmakeargs+=(
+			-DENABLE_CLI="no"
+		)
+	fi
 
 	# Unfortunately, the asm for x86/x32/arm isn't PIC-safe.
 	# x86 # Bug #528202, bug #913412
@@ -114,7 +120,9 @@ multilib_src_configure() {
 	if [[ ${ABI} = x86 ]] || [[ ${ABI} = x32 ]] || [[ ${ABI} = arm ]] ; then
 		mycmakeargs+=(
 			-DENABLE_ASSEMBLY=OFF
-			-DENABLE_TESTS="no" #728748
+			# ENABLE_TESTS requires ENABLE_ASSEMBLY=ON to be visible
+			# source/CMakeLists.txt:858
+			# -DENABLE_TESTS="no" #728748
 		)
 	else
 		mycmakeargs+=(
diff --git a/media-libs/x265/x265-9999.ebuild b/media-libs/x265/x265-9999.ebuild
index 9e4c71595c06..bf2855a23f64 100644
--- a/media-libs/x265/x265-9999.ebuild
+++ b/media-libs/x265/x265-9999.ebuild
@@ -41,6 +41,8 @@ PATCHES=(
 	"${FILESDIR}/${PN}-3.5-r5-cpp-std.patch"
 	"${FILESDIR}/${PN}-3.5-r5-gcc15.patch"
 	"${FILESDIR}/${PN}-9999-test-ns.patch"
+	"${FILESDIR}/${PN}-3.6-cmake-cleanup.patch"
+	"${FILESDIR}/${PN}-3.6-code-cleanup.patch"
 )
 
 pkg_setup() {
@@ -98,7 +100,6 @@ multilib_src_configure() {
 	filter-lto
 
 	local mycmakeargs=(
-		$(multilib_is_native_abi || echo "-DENABLE_CLI=OFF")
 		-DENABLE_PIC=ON
 		-DENABLE_LIBNUMA="$(usex numa)"
 		-DENABLE_MULTIVIEW="yes"
@@ -107,6 +108,11 @@ multilib_src_configure() {
 		-DGIT_ARCHETYPE=1 #814116
 		-DLIB_INSTALL_DIR="$(get_libdir)"
 	)
+	if multilib_is_native_abi; then
+		mycmakeargs+=(
+			-DENABLE_CLI="no"
+		)
+	fi
 
 	# Unfortunately, the asm for x86/x32/arm isn't PIC-safe.
 	# x86 # Bug #528202, bug #913412
@@ -114,7 +120,9 @@ multilib_src_configure() {
 	if [[ ${ABI} = x86 ]] || [[ ${ABI} = x32 ]] || [[ ${ABI} = arm ]] ; then
 		mycmakeargs+=(
 			-DENABLE_ASSEMBLY=OFF
-			-DENABLE_TESTS="no" #728748
+			# ENABLE_TESTS requires ENABLE_ASSEMBLY=ON to be visible
+			# source/CMakeLists.txt:858
+			# -DENABLE_TESTS="no" #728748
 		)
 	else
 		mycmakeargs+=(