[PATCH] Fix hybrid SLP vectorization

[PATCH] Fix hybrid SLP vectorization

[Richard Biener]

The following patch catches more similar cases as in the
187.facerec case and adds checking to vect_transform_stmt.

Bootstrapped and tested on x86_64-unknown-linux-gnu.

Richard.

2015-06-25  Richard Biener  <rguenther@suse.de>

	* tree-vect-stmts.c (vectorizable_conversion): Do not set
	STMT_VINFO_VEC_STMT for SLP.
	(vectorizable_store): Likewise.
	(vectorizable_load): Likewise.
	(vect_transform_stmt): Catch SLP vectorization clobbering
	STMT_VINFO_VEC_STMT.

Index: gcc/tree-vect-stmts.c
===================================================================
*** gcc/tree-vect-stmts.c	(revision 224893)
--- gcc/tree-vect-stmts.c	(working copy)
*************** vectorizable_conversion (gimple stmt, gi
*** 3841,3853 ****
  	      vect_finish_stmt_generation (stmt, new_stmt, gsi);
  	      if (slp_node)
  		SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt);
  	    }
- 
- 	  if (j == 0)
- 	    STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt;
- 	  else
- 	    STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt;
- 	  prev_stmt_info = vinfo_for_stmt (new_stmt);
  	}
        break;
  
--- 3841,3855 ----
  	      vect_finish_stmt_generation (stmt, new_stmt, gsi);
  	      if (slp_node)
  		SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt);
+ 	      else
+ 		{
+ 		  if (!prev_stmt_info)
+ 		    STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt;
+ 		  else
+ 		    STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt;
+ 		  prev_stmt_info = vinfo_for_stmt (new_stmt);
+ 		}
  	    }
  	}
        break;
  
*************** vectorizable_store (gimple stmt, gimple_
*** 5400,5409 ****
  		  vect_finish_stmt_generation (stmt, incr, gsi);
  
  		  running_off = newoff;
! 		  if (g == group_size - 1)
  		    {
  		      if (j == 0 && i == 0)
! 			STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = assign;
  		      else
  			STMT_VINFO_RELATED_STMT (prev_stmt_info) = assign;
  		      prev_stmt_info = vinfo_for_stmt (assign);
--- 5402,5413 ----
  		  vect_finish_stmt_generation (stmt, incr, gsi);
  
  		  running_off = newoff;
! 		  if (g == group_size - 1
! 		      && !slp)
  		    {
  		      if (j == 0 && i == 0)
! 			STMT_VINFO_VEC_STMT (stmt_info)
! 			    = *vec_stmt = assign;
  		      else
  			STMT_VINFO_RELATED_STMT (prev_stmt_info) = assign;
  		      prev_stmt_info = vinfo_for_stmt (assign);
*************** vectorizable_load (gimple stmt, gimple_s
*** 6409,6419 ****
  	      if (slp_perm)
  		dr_chain.quick_push (gimple_assign_lhs (new_stmt));
  	    }
- 	  if (j == 0)
- 	    STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt;
  	  else
! 	    STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt;
! 	  prev_stmt_info = vinfo_for_stmt (new_stmt);
  	}
        if (slp_perm)
  	vect_transform_slp_perm_load (slp_node, dr_chain, gsi, vf,
--- 6413,6426 ----
  	      if (slp_perm)
  		dr_chain.quick_push (gimple_assign_lhs (new_stmt));
  	    }
  	  else
! 	    {
! 	      if (j == 0)
! 		STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt;
! 	      else
! 		STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt;
! 	      prev_stmt_info = vinfo_for_stmt (new_stmt);
! 	    }
  	}
        if (slp_perm)
  	vect_transform_slp_perm_load (slp_node, dr_chain, gsi, vf,
*************** vect_transform_stmt (gimple stmt, gimple
*** 7523,7528 ****
--- 7530,7537 ----
    stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
    bool done;
  
+   gimple old_vec_stmt = STMT_VINFO_VEC_STMT (stmt_info);
+ 
    switch (STMT_VINFO_TYPE (stmt_info))
      {
      case type_demotion_vec_info_type:
*************** vect_transform_stmt (gimple stmt, gimple
*** 7610,7615 ****
--- 7619,7636 ----
  	}
      }
  
+   /* Verify SLP vectorization doesn't mess with STMT_VINFO_VEC_STMT.
+      This would break hybrid SLP vectorization.  */
+   if (slp_node)
+     {
+       if (PURE_SLP_STMT (stmt_info))
+ 	gcc_assert (!old_vec_stmt && !vec_stmt
+ 		    && !STMT_VINFO_VEC_STMT (stmt_info));
+       else if (HYBRID_SLP_STMT (stmt_info))
+ 	gcc_assert (!vec_stmt
+ 		    && STMT_VINFO_VEC_STMT (stmt_info) == old_vec_stmt);
+     }
+ 
    /* Handle inner-loop stmts whose DEF is used in the loop-nest that
       is being vectorized, but outside the immediately enclosing loop.  */
    if (vec_stmt