[Bug middle-end/108410] x264 averaging loop not optimized well for avx512

["rguenth at gcc dot gnu.org" <gcc-bugzilla@gcc.gnu.org>]

https://gcc.gnu.org/bugzilla/show_bug.cgi?id=108410

--- Comment #4 from Richard Biener <rguenth at gcc dot gnu.org> ---
Adding fully masked AVX512 and AVX512 with a masked epilog data:

size   scalar     128     256     512    512e    512f
    1    9.42   11.32    9.35   11.17   15.13   16.89
    2    5.72    6.53    6.66    6.66    7.62    8.56
    3    4.49    5.10    5.10    5.74    5.08    5.73
    4    4.10    4.33    4.29    5.21    3.79    4.25
    6    3.78    3.85    3.86    4.76    2.54    2.85
    8    3.64    1.89    3.76    4.50    1.92    2.16
   12    3.56    2.21    3.75    4.26    1.26    1.42
   16    3.36    0.83    1.06    4.16    0.95    1.07
   20    3.39    1.42    1.33    4.07    0.75    0.85
   24    3.23    0.66    1.72    4.22    0.62    0.70
   28    3.18    1.09    2.04    4.20    0.54    0.61
   32    3.16    0.47    0.41    0.41    0.47    0.53
   34    3.16    0.67    0.61    0.56    0.44    0.50
   38    3.19    0.95    0.95    0.82    0.40    0.45
   42    3.09    0.58    1.21    1.13    0.36    0.40

text sizes are not much different:

         1389    1837    2125    1629    1721    1689

the AVX2 size is large because we completely peel the scalar epilogue,
same for the SSE case.  The scalar epilogue of the 512 loop iterates
32 times (too many for peeling), the masked loop/epilogue are quite
large due to the EVEX encoded instructions so the saved scalar/vector
epilogues do not show.

The AVX512 masked epilogue case now looks like:

        .p2align 3
.L5:
        vmovdqu8        (%r8,%rax), %zmm0
        vpavgb  (%rsi,%rax), %zmm0, %zmm0
        vmovdqu8        %zmm0, (%rdi,%rax)
        addq    $64, %rax
        cmpq    %rcx, %rax
        jne     .L5
        movl    %edx, %ecx
        andl    $-64, %ecx
        testb   $63, %dl
        je      .L19
.L4:
        movl    %ecx, %eax
        subl    %ecx, %edx
        movl    $255, %ecx
        cmpl    %ecx, %edx
        cmova   %ecx, %edx
        vpbroadcastb    %edx, %zmm0
        vpcmpub $6, .LC0(%rip), %zmm0, %k1
        vmovdqu8        (%rsi,%rax), %zmm0{%k1}{z}
        vmovdqu8        (%r8,%rax), %zmm1{%k1}{z}
        vpavgb  %zmm1, %zmm0, %zmm0
        vmovdqu8        %zmm0, (%rdi,%rax){%k1}
.L19:
        vzeroupper
        ret

where there's a missed optimization around the saturation to 255.

The fully masked AVX512 loop is

        vmovdqa64       .LC0(%rip), %zmm3
        movl    $255, %eax
        cmpl    %eax, %ecx 
        cmovbe  %ecx, %eax
        vpbroadcastb    %eax, %zmm0
        vpcmpub $6, %zmm3, %zmm0, %k1
        .p2align 4
        .p2align 3
.L4:
        vmovdqu8        (%rsi,%rax), %zmm1{%k1}
        vmovdqu8        (%r8,%rax), %zmm2{%k1}
        movl    %r10d, %edx
        movl    $255, %ecx
        subl    %eax, %edx
        cmpl    %ecx, %edx
        cmova   %ecx, %edx
        vpavgb  %zmm2, %zmm1, %zmm0
        vmovdqu8        %zmm0, (%rdi,%rax){%k1}
        vpbroadcastb    %edx, %zmm0
        addq    $64, %rax
        movl    %r9d, %edx
        subl    %eax, %edx
        vpcmpub $6, %zmm3, %zmm0, %k1
        cmpl    $64, %edx
        ja      .L4
        vzeroupper
        ret

which is a much larger loop body due to the mask creation.  At least
that interleaves nicely (dependence wise) with the loop control and
vectorized stmts.  What needs to be optimized somehow is what IVOPTs
makes out of the decreasing remaining scalar iters IV with the 
IV required for the memory accesses.  Without IVOPTs the body looks
like

.L4:
        vmovdqu8        (%rsi), %zmm1{%k1}
        vmovdqu8        (%rdx), %zmm2{%k1}
        movl    $255, %eax
        movl    %ecx, %r8d
        subl    $64, %ecx
        addq    $64, %rsi
        addq    $64, %rdx
        vpavgb  %zmm2, %zmm1, %zmm0
        vmovdqu8        %zmm0, (%rdi){%k1}
        addq    $64, %rdi
        cmpl    %eax, %ecx
        cmovbe  %ecx, %eax
        vpbroadcastb    %eax, %zmm0
        vpcmpub $6, %zmm3, %zmm0, %k1
        cmpl    $64, %r8d
        ja      .L4

and the key thing to optimize is

  ivtmp_78 = ivtmp_77 + 4294967232; // -64
  _79 = MIN_EXPR <ivtmp_78, 255>;
  _80 = (unsigned char) _79;
  _81 = {_80, _80, _80, _80, _80, _80, _80, _80, _80, _80, _80, _80, _80, _80,
_80, _80, _80, _80, _80, _80, _80, _80, _80, _80, _80, _80, _80, _80, _80, _80,
_80, _80, _80, _80, _80, _80, _80, _80, _80, _80, _80, _80, _80, _80, _80, _80,
_80, _80, _80, _80, _80, _80, _80, _80, _80, _80, _80, _80, _80, _80, _80, _80,
_80, _80};

that is we want to broadcast a saturated (to vector element precision) value.