[Bug tree-optimization/108751] New: Removing dead code results in worse optimization at -Os

["theodort at inf dot ethz.ch" <gcc-bugzilla@gcc.gnu.org>]

https://gcc.gnu.org/bugzilla/show_bug.cgi?id=108751

            Bug ID: 108751
           Summary: Removing dead code results in worse optimization at
                    -Os
           Product: gcc
           Version: 13.0
            Status: UNCONFIRMED
          Severity: normal
          Priority: P3
         Component: tree-optimization
          Assignee: unassigned at gcc dot gnu.org
          Reporter: theodort at inf dot ethz.ch
  Target Milestone: ---

I found this case where slight changes in the program that, in theory, should
not affect the output (or affect it trivially) cause the compiler to generate
worse code: 

static int a = 0;
static int b = 1;
int main() {
  char c = 0;
  for (;;) {
    if (c)
      break;
    for (; a; a++) { // a is 0, this loop is dead
      if (b) // this is always true
        continue;
      else
        return 2; // this program will never return 2
    }
    c = 10;
  }
  return 3;
}

compiled with gcc-trunk -Os: 

main:
.L2:
        movl    a(%rip), %eax
        testl   %eax, %eax
        je      .L6
        incl    %eax
        movl    %eax, a(%rip)
        jmp     .L2
.L6:
        movl    $3, %eax
        ret

Clearly, the compiler has figured out that "return 2;" will never be executed.
But if I remove it from the source:

static int a = 0;
static int b = 1;
int main() {
  char c = 0;
  for (;;) {
    if (c)
      break;
    for (; a; a++) {
      if (b)
        continue;
      //else
      // return 2;
    }
    c = 10;
  }
  return 3;
}

and compile with gcc-trunk -Os again:

main:
        movl    a(%rip), %eax
        xorl    %edx, %edx
.L2:
        testl   %eax, %eax
        jne     .L4
        testb   %dl, %dl
        je      .L7
        xorl    %eax, %eax
        movl    %eax, a(%rip)
        jmp     .L7
.L4:
        incl    %eax
        movb    $1, %dl
        jmp     .L2
.L7:
        movl    $3, %eax
        ret

the generated code is worse. 

The same thing happens if the return value is changed:

static int a = 0;
static int b = 1;
int main() {
  char c = 0;
  for (;;) {
    if (c)
      break;
    for (; a; a++) {
      if (b)
        continue;
      else
        return 2;
    }
    c = 10;
  }
  return 1; // changed from return 3
}

gcc-trunk -Os: 

main:
        movl    a(%rip), %eax
        xorl    %edx, %edx
.L2:
        testl   %eax, %eax
        jne     .L4
        testb   %dl, %dl
        je      .L7
        xorl    %eax, %eax
        movl    %eax, a(%rip)
        jmp     .L7
.L4:
        incl    %eax
        movb    $1, %dl
        jmp     .L2
.L7:
        movl    $1, %eax
        ret

and if we constant propagate b:

static int a = 0;
int main() {
  char c = 0;
  for (;;) {
    if (c)
      break;
    for (; a; a++) {
      if (1) // this was if (b) before
        continue;
      else
        return 2;
    }
    c = 10;
  }
  return 1;
}

gcc-trunk -Os:

main:
        movl    a(%rip), %eax
        xorl    %edx, %edx
.L2:
        testl   %eax, %eax
        jne     .L12
        testb   %dl, %dl
        je      .L7
        xorl    %eax, %eax
        movl    %eax, a(%rip)
        jmp     .L7
.L12:
        incl    %eax
        movb    $1, %dl
        jmp     .L2
.L7:
        movl    $1, %eax
        ret