[Bug tree-optimization/105368] [10/11/12 Regression] ICE: SIGSEGV in powi_lookup_cost (tree-ssa-math-opts.cc:1441) with -Ofast and __builtin_powf()

["zsojka at seznam dot cz" <gcc-bugzilla@gcc.gnu.org>]

https://gcc.gnu.org/bugzilla/show_bug.cgi?id=105368

--- Comment #2 from Zdenek Sojka <zsojka at seznam dot cz> ---
(gdb) disas
Dump of assembler code for function powi_cost(long):
   0x0000000001538160 <+0>:       push   %r14
   0x0000000001538162 <+2>:       mov    $0x20,%ecx
   0x0000000001538167 <+7>:       push   %r13
   0x0000000001538169 <+9>:       push   %r12
   0x000000000153816b <+11>:      push   %rbp
   0x000000000153816c <+12>:      push   %rbx
   0x000000000153816d <+13>:      mov    %rdi,%rbx
   0x0000000001538170 <+16>:      sub    $0x100,%rsp
   0x0000000001538177 <+23>:      neg    %rbx
   0x000000000153817a <+26>:      cmovs  %rdi,%rbx
   0x000000000153817e <+30>:      mov    %rsp,%r13
   0x0000000001538181 <+33>:      xor    %eax,%eax
   0x0000000001538183 <+35>:      xor    %r12d,%r12d
   0x0000000001538186 <+38>:      mov    %r13,%rdi
   0x0000000001538189 <+41>:      rep stos %rax,%es:(%rdi)
   0x000000000153818c <+44>:      movb   $0x1,0x1(%rsp)
=> 0x0000000001538191 <+49>:      cmp    $0xff,%rbx
   0x0000000001538198 <+56>:      jg     0x15381b0 <powi_cost(long)+80>
   0x000000000153819a <+58>:      jmp    0x1538207 <powi_cost(long)+167>

1463    
1464      /* Ignore the reciprocal when calculating the cost.  */
1465      val = (n < 0) ? -n : n;
   0x000000000153816d <+13>:      mov    %rdi,%rbx

1454    powi_cost (HOST_WIDE_INT n)
   0x0000000001538170 <+16>:      sub    $0x100,%rsp

1455    {
1456      bool cache[POWI_TABLE_SIZE];
1457      unsigned HOST_WIDE_INT digit;
1458      unsigned HOST_WIDE_INT val;
1459      int result;
1460    
1461      if (n == 0)
1462        return 0;
--Type <RET> for more, q to quit, c to continue without paging--
1463    
1464      /* Ignore the reciprocal when calculating the cost.  */
1465      val = (n < 0) ? -n : n;
   0x0000000001538177 <+23>:      neg    %rbx
   0x000000000153817a <+26>:      cmovs  %rdi,%rbx

...

/repo/gcc-trunk/gcc/tree-ssa-math-opts.cc:
1469      cache[1] = true;
   0x000000000153818c <+44>:      movb   $0x1,0x1(%rsp)

1472    
1473      while (val >= POWI_TABLE_SIZE)
=> 0x0000000001538191 <+49>:      cmp    $0xff,%rbx
   0x0000000001538198 <+56>:      jg     0x15381b0 <powi_cost(long)+80>
   0x000000000153819a <+58>:      jmp    0x1538207 <powi_cost(long)+167>


(gdb) p/x n
$19 = 0x8000000000000000
(gdb) p/x val
$20 = 0x8000000000000000
(gdb) p/x $rbx
$22 = 0x8000000000000000

The "while (val >= POWI_TABLE_SIZE)" check seems to be done in signed
arithmetics ("jg" conditional jump), so the value is not reduced and cache[i]
accesses out of bounds.

This happens with both boostrapped and non-bootstrapped compiler (using host
gcc-11.3.0)