[Bug target/110838] New: [14 Regression] wrong code on x265-3.5, -O3, sign extraction

["slyfox at gcc dot gnu.org" <gcc-bugzilla@gcc.gnu.org>]

https://gcc.gnu.org/bugzilla/show_bug.cgi?id=110838

            Bug ID: 110838
           Summary: [14 Regression] wrong code on x265-3.5, -O3, sign
                    extraction
           Product: gcc
           Version: unknown
            Status: UNCONFIRMED
          Severity: normal
          Priority: P3
         Component: target
          Assignee: unassigned at gcc dot gnu.org
          Reporter: slyfox at gcc dot gnu.org
  Target Milestone: ---

Initially observed as SSE4 test failures on x365-3.5 on gcc
r14-2829-g31d18ff44244d1.

Extracted self-contained reproducer:

// $ cat bug.cc
typedef unsigned  int uint32_t;
typedef unsigned char uint8_t;
typedef   signed char  int8_t;
typedef uint8_t pixel;

/* get the sign of input variable (TODO: this is a dup, make common) */
inline int8_t signOf(int x)
{
    return (x >> 31) | ((int)((((uint32_t)-x)) >> 31));
}

__attribute__((noipa))
static void calSign_bug(int8_t *dst, const pixel *src1, const pixel *src2,
const int endX)
{
    for (int x = 0; x < endX; x++)
        dst[x] = signOf(src1[x] - src2[x]);
}

__attribute__((noipa, optimize(0)))
static void calSign_ok(int8_t *dst, const pixel *src1, const pixel *src2, const
int endX)
{
    for (int x = 0; x < endX; x++)
        dst[x] = signOf(src1[x] - src2[x]);
}

__attribute__((noipa, optimize(0)))
int main(void)
{
    const pixel s1[9] = { 0xcd, 0x33, 0xd4, 0x3e, 0xb0, 0xfb, 0x95, 0x64, 0x70,
};
    const pixel s2[9] = { 0xba, 0x9f, 0xab, 0xa1, 0x3b, 0x29, 0xb1, 0xbd, 0x64,
};
    int endX = 9;
    int8_t dst[9];
    int8_t dst_ok[9];

    calSign_bug(dst, s1, s2, endX);
    calSign_ok(dst_ok, s1, s2, endX);

    if (__builtin_memcmp(dst, dst_ok, endX) != 0) __builtin_trap();
}

Triggering:

$ g++ bug.cc -o bug -O2 && ./bug
# ok
$ g++ bug.cc -o bug -O3 && ./bug
Illegal instruction (core dumped)

$ g++ -v
Using built-in specs.
COLLECT_GCC=/<<NIX>>/gcc-14.0.0/bin/g++
COLLECT_LTO_WRAPPER=/<<NIX>>/gcc-14.0.0/libexec/gcc/x86_64-unknown-linux-gnu/14.0.0/lto-wrapper
Target: x86_64-unknown-linux-gnu
Configured with: ../source/configure --prefix=/<<NIX>>/gcc-14.0.0
--with-gmp-include=/<<NIX>>/gmp-6.2.1-dev/include
--with-gmp-lib=/<<NIX>>/gmp-6.2.1/lib
--with-mpfr-include=/<<NIX>>/mpfr-4.2.0-dev/include
--with-mpfr-lib=/<<NIX>>/mpfr-4.2.0/lib --with-mpc=/<<NIX>>/libmpc-1.3.1
--with-native-system-header-dir=/<<NIX>>/glibc-2.37-8-dev/include
--with-build-sysroot=/ --program-prefix= --enable-lto --disable-libstdcxx-pch
--without-included-gettext --with-system-zlib --enable-checking=release
--enable-static --enable-languages=c,c++ --disable-multilib --enable-plugin
--disable-libcc1 --with-isl=/<<NIX>>/isl-0.20 --disable-bootstrap
--build=x86_64-unknown-linux-gnu --host=x86_64-unknown-linux-gnu
--target=x86_64-unknown-linux-gnu
Thread model: posix
Supported LTO compression algorithms: zlib
gcc version 14.0.0 99999999 (experimental) (GCC)