[PATCH 4/9] Rewrite all compare-and-swap in terms of expand_atomic_compare_and_swap.

[Richard Henderson <rth@redhat.com>]

---
 gcc/builtins.c |   39 +++++---
 gcc/expr.h     |    4 -
 gcc/optabs.c   |  267 ++++++++++++++++++++-----------------------------------
 gcc/optabs.h   |    4 +
 4 files changed, 126 insertions(+), 188 deletions(-)

diff --git a/gcc/builtins.c b/gcc/builtins.c
index cb9da83..756070f 100644
--- a/gcc/builtins.c
+++ b/gcc/builtins.c
@@ -5196,10 +5196,13 @@ expand_builtin_compare_and_swap (enum machine_mode mode, tree exp,
   old_val = expand_expr_force_mode (CALL_EXPR_ARG (exp, 1), mode);
   new_val = expand_expr_force_mode (CALL_EXPR_ARG (exp, 2), mode);
 
-  if (is_bool)
-    return expand_bool_compare_and_swap (mem, old_val, new_val, target);
-  else
-    return expand_val_compare_and_swap (mem, old_val, new_val, target);
+  if (!expand_atomic_compare_and_swap ((is_bool ? &target : NULL),
+				       (is_bool ? NULL : &target),
+				       mem, old_val, new_val, false,
+				       MEMMODEL_SEQ_CST, MEMMODEL_SEQ_CST))
+    return NULL_RTX;
+
+  return target;
 }
 
 /* Expand the __sync_lock_test_and_set intrinsic.  Note that the most
@@ -5293,8 +5296,10 @@ static rtx
 expand_builtin_atomic_compare_exchange (enum machine_mode mode, tree exp, 
 					rtx target)
 {
-  rtx expect, desired, mem, weak;
+  rtx expect, desired, mem, oldval;
   enum memmodel success, failure;
+  tree weak;
+  bool is_weak;
 
   success = get_memmodel (CALL_EXPR_ARG (exp, 4));
   failure = get_memmodel (CALL_EXPR_ARG (exp, 5));
@@ -5307,24 +5312,31 @@ expand_builtin_atomic_compare_exchange (enum machine_mode mode, tree exp,
 
   if (failure > success)
     {
-      error ("failure memory model cannot be stronger than success memory model for %<__atomic_compare_exchange%>");
+      error ("failure memory model cannot be stronger than success "
+	     "memory model for %<__atomic_compare_exchange%>");
       return NULL_RTX;
     }
   
   /* Expand the operands.  */
   mem = get_builtin_sync_mem (CALL_EXPR_ARG (exp, 0), mode);
 
-  expect = expand_expr (CALL_EXPR_ARG (exp, 1), NULL_RTX, ptr_mode, 
-			EXPAND_NORMAL);
+  expect = expand_normal (CALL_EXPR_ARG (exp, 1));
   expect = convert_memory_address (Pmode, expect);
-
   desired = expand_expr_force_mode (CALL_EXPR_ARG (exp, 2), mode);
 
-  weak = expand_expr (CALL_EXPR_ARG (exp, 3), NULL_RTX, ptr_mode,
-		      EXPAND_NORMAL);
+  weak = CALL_EXPR_ARG (exp, 3);
+  is_weak = false;
+  if (host_integerp (weak, 0) && tree_low_cst (weak, 0) != 0)
+    is_weak = true;
 
-  return expand_atomic_compare_exchange (target, mem, expect, desired, weak,
-					 success, failure);
+  oldval = copy_to_reg (gen_rtx_MEM (mode, expect));
+
+  if (!expand_atomic_compare_and_swap (&target, &oldval, mem, oldval,
+				       desired, is_weak, success, failure))
+    return NULL_RTX;
+
+  emit_move_insn (gen_rtx_MEM (mode, expect), oldval);
+  return target;
 }
 
 /* Expand the __atomic_load intrinsic:
@@ -5442,7 +5454,6 @@ fold_builtin_atomic_always_lock_free (tree arg)
 {
   int size;
   enum machine_mode mode;
-  enum insn_code icode;
 
   if (TREE_CODE (arg) != INTEGER_CST)
     return NULL_TREE;
diff --git a/gcc/expr.h b/gcc/expr.h
index 8cae22b..1623ad9 100644
--- a/gcc/expr.h
+++ b/gcc/expr.h
@@ -212,14 +212,10 @@ int can_conditionally_move_p (enum machine_mode mode);
 rtx emit_conditional_add (rtx, enum rtx_code, rtx, rtx, enum machine_mode,
 			  rtx, rtx, enum machine_mode, int);
 
-rtx expand_val_compare_and_swap (rtx, rtx, rtx, rtx);
-rtx expand_bool_compare_and_swap (rtx, rtx, rtx, rtx);
 rtx expand_sync_operation (rtx, rtx, enum rtx_code);
 rtx expand_sync_fetch_operation (rtx, rtx, enum rtx_code, bool, rtx);
 
 rtx expand_atomic_exchange (rtx, rtx, rtx, enum memmodel);
-rtx expand_atomic_compare_exchange (rtx, rtx, rtx, rtx, rtx, enum memmodel, 
-				      enum memmodel);
 rtx expand_atomic_load (rtx, rtx, enum memmodel);
 rtx expand_atomic_store (rtx, rtx, enum memmodel);
 rtx expand_atomic_fetch_op (rtx, rtx, rtx, enum rtx_code, enum memmodel, 
diff --git a/gcc/optabs.c b/gcc/optabs.c
index 9b4d6cf..b7c00be 100644
--- a/gcc/optabs.c
+++ b/gcc/optabs.c
@@ -6799,45 +6799,6 @@ can_compare_and_swap_p (enum machine_mode mode)
   return false;
 }
 
-/* This is an internal subroutine of the other compare_and_swap expanders.
-   MEM, OLD_VAL and NEW_VAL are as you'd expect for a compare-and-swap
-   operation.  TARGET is an optional place to store the value result of
-   the operation.  ICODE is the particular instruction to expand.  Return
-   the result of the operation.  */
-
-static rtx
-expand_val_compare_and_swap_1 (rtx mem, rtx old_val, rtx new_val,
-			       rtx target, enum insn_code icode)
-{
-  struct expand_operand ops[4];
-  enum machine_mode mode = GET_MODE (mem);
-
-  create_output_operand (&ops[0], target, mode);
-  create_fixed_operand (&ops[1], mem);
-  /* OLD_VAL and NEW_VAL may have been promoted to a wider mode.
-     Shrink them if so.  */
-  create_convert_operand_to (&ops[2], old_val, mode, true);
-  create_convert_operand_to (&ops[3], new_val, mode, true);
-  if (maybe_expand_insn (icode, 4, ops))
-    return ops[0].value;
-  return NULL_RTX;
-}
-
-/* Expand a compare-and-swap operation and return its value.  */
-
-rtx
-expand_val_compare_and_swap (rtx mem, rtx old_val, rtx new_val, rtx target)
-{
-  enum machine_mode mode = GET_MODE (mem);
-  enum insn_code icode
-    = direct_optab_handler (sync_compare_and_swap_optab, mode);
-
-  if (icode == CODE_FOR_nothing)
-    return NULL_RTX;
-
-  return expand_val_compare_and_swap_1 (mem, old_val, new_val, target, icode);
-}
-
 /* Helper function to find the MODE_CC set in a sync_compare_and_swap
    pattern.  */
 
@@ -6853,58 +6814,6 @@ find_cc_set (rtx x, const_rtx pat, void *data)
     }
 }
 
-/* Expand a compare-and-swap operation and store true into the result if
-   the operation was successful and false otherwise.  Return the result.
-   Unlike other routines, TARGET is not optional.  */
-
-rtx
-expand_bool_compare_and_swap (rtx mem, rtx old_val, rtx new_val, rtx target)
-{
-  enum machine_mode mode = GET_MODE (mem);
-  enum insn_code icode;
-  rtx subtarget, seq, cc_reg;
-
-  /* If the target supports a compare-and-swap pattern that simultaneously
-     sets some flag for success, then use it.  Otherwise use the regular
-     compare-and-swap and follow that immediately with a compare insn.  */
-  icode = direct_optab_handler (sync_compare_and_swap_optab, mode);
-  if (icode == CODE_FOR_nothing)
-    return NULL_RTX;
-
-  do_pending_stack_adjust ();
-  do
-    {
-      start_sequence ();
-      subtarget = expand_val_compare_and_swap_1 (mem, old_val, new_val,
-					         NULL_RTX, icode);
-      cc_reg = NULL_RTX;
-      if (subtarget == NULL_RTX)
-	{
-	  end_sequence ();
-	  return NULL_RTX;
-	}
-
-      if (have_insn_for (COMPARE, CCmode))
-	note_stores (PATTERN (get_last_insn ()), find_cc_set, &cc_reg);
-      seq = get_insns ();
-      end_sequence ();
-
-      /* We might be comparing against an old value.  Try again. :-(  */
-      if (!cc_reg && MEM_P (old_val))
-	{
-	  seq = NULL_RTX;
-	  old_val = force_reg (mode, old_val);
-        }
-    }
-  while (!seq);
-
-  emit_insn (seq);
-  if (cc_reg)
-    return emit_store_flag_force (target, EQ, cc_reg, const0_rtx, VOIDmode, 0, 1);
-  else
-    return emit_store_flag_force (target, EQ, subtarget, old_val, VOIDmode, 1, 1);
-}
-
 /* This is a helper function for the other atomic operations.  This function
    emits a loop that contains SEQ that iterates until a compare-and-swap
    operation at the end succeeds.  MEM is the memory to be modified.  SEQ is
@@ -6918,8 +6827,7 @@ static bool
 expand_compare_and_swap_loop (rtx mem, rtx old_reg, rtx new_reg, rtx seq)
 {
   enum machine_mode mode = GET_MODE (mem);
-  enum insn_code icode;
-  rtx label, cmp_reg, subtarget, cc_reg;
+  rtx label, cmp_reg, success, oldval;
 
   /* The loop we want to generate looks like
 
@@ -6927,8 +6835,8 @@ expand_compare_and_swap_loop (rtx mem, rtx old_reg, rtx new_reg, rtx seq)
       label:
         old_reg = cmp_reg;
 	seq;
-	cmp_reg = compare-and-swap(mem, old_reg, new_reg)
-	if (cmp_reg != old_reg)
+	(success, cmp_reg) = compare-and-swap(mem, old_reg, new_reg)
+	if (success)
 	  goto label;
 
      Note that we only do the plain load from memory once.  Subsequent
@@ -6943,35 +6851,19 @@ expand_compare_and_swap_loop (rtx mem, rtx old_reg, rtx new_reg, rtx seq)
   if (seq)
     emit_insn (seq);
 
-  /* If the target supports a compare-and-swap pattern that simultaneously
-     sets some flag for success, then use it.  Otherwise use the regular
-     compare-and-swap and follow that immediately with a compare insn.  */
-  icode = direct_optab_handler (sync_compare_and_swap_optab, mode);
-  if (icode == CODE_FOR_nothing)
+  success = NULL_RTX;
+  oldval = cmp_reg;
+  if (!expand_atomic_compare_and_swap (&success, &oldval, mem, old_reg,
+				       new_reg, false, MEMMODEL_SEQ_CST,
+				       MEMMODEL_RELAXED))
     return false;
 
-  subtarget = expand_val_compare_and_swap_1 (mem, old_reg, new_reg,
-					     cmp_reg, icode);
-  if (subtarget == NULL_RTX)
-    return false;
-
-  cc_reg = NULL_RTX;
-  if (have_insn_for (COMPARE, CCmode))
-    note_stores (PATTERN (get_last_insn ()), find_cc_set, &cc_reg);
-  if (cc_reg)
-    {
-      cmp_reg = cc_reg;
-      old_reg = const0_rtx;
-    }
-  else
-    {
-      if (subtarget != cmp_reg)
-	emit_move_insn (cmp_reg, subtarget);
-    }
+  if (oldval != cmp_reg)
+    emit_move_insn (cmp_reg, oldval);
 
   /* ??? Mark this jump predicted not taken?  */
-  emit_cmp_and_jump_insns (cmp_reg, old_reg, NE, const0_rtx, GET_MODE (cmp_reg), 1,
-			   label);
+  emit_cmp_and_jump_insns (success, const0_rtx, EQ, const0_rtx,
+			   GET_MODE (success), 1, label);
   return true;
 }
 
@@ -7050,77 +6942,112 @@ expand_atomic_exchange (rtx target, rtx mem, rtx val, enum memmodel model)
 
 /* This function expands the atomic compare exchange operation:
 
+   *PTARGET_BOOL is an optional place to store the boolean success/failure.
+   *PTARGET_OVAL is an optional place to store the old value from memory.
+   Both target parameters may be NULL to indicate that we do not care about
+   that return value.  Both target parameters are updated on success to
+   the actual location of the corresponding result.
+
    MEMMODEL is the memory model variant to use.
-   TARGET is an option place to stick the return value.  */
 
-rtx
-expand_atomic_compare_exchange (rtx target, rtx mem, rtx expected, 
-				rtx desired, rtx weak, enum memmodel success, 
-				enum memmodel failure)
+   The return value of the function is true for success.  */
+
+bool
+expand_atomic_compare_and_swap (rtx *ptarget_bool, rtx *ptarget_oval,
+				rtx mem, rtx expected, rtx desired,
+				bool is_weak, enum memmodel succ_model,
+				enum memmodel fail_model)
 {
   enum machine_mode mode = GET_MODE (mem);
-  enum insn_code icode = CODE_FOR_nothing;
-  rtx expected_val, CAS_val;
+  struct expand_operand ops[8];
+  enum insn_code icode;
+  rtx target_bool, target_oval;
 
-  /* Load '*expected into a register for the compare and swap */
-  expected_val = gen_reg_rtx (mode);
-  emit_move_insn (expected_val, gen_rtx_MEM (mode, expected));
+  /* Load expected into a register for the compare and swap.  */
+  if (MEM_P (expected))
+    expected = copy_to_reg (expected);
+
+  /* Make sure we always have some place to put the return oldval.
+     Further, make sure that place is distinct from the input expected,
+     just in case we need that path down below.  */
+  if (ptarget_oval == NULL
+      || (target_oval = *ptarget_oval) == NULL
+      || reg_overlap_mentioned_p (expected, target_oval))
+    target_oval = gen_reg_rtx (mode);
 
   icode = direct_optab_handler (atomic_compare_and_swap_optab, mode);
-    
   if (icode != CODE_FOR_nothing)
     {
-      struct expand_operand ops[8];
-      rtx original_val = gen_reg_rtx (mode);
-      enum machine_mode target_mode;
-      int is_weak;
-
-      target_mode = insn_data[icode].operand[0].mode;
-
-      /* Either WEAK is explcitly 1, or assume strong to be safe.  */
-      is_weak = (weak == const1_rtx);
+      enum machine_mode bool_mode = insn_data[icode].operand[0].mode;
 
-      if (target == NULL_RTX || target == const0_rtx 
-	  || GET_MODE (target) != target_mode)
-        target = gen_reg_rtx (target_mode);
+      /* Make sure we always have a place for the bool operand.  */
+      if (ptarget_bool == NULL
+	  || (target_bool = *ptarget_bool) == NULL
+	  || GET_MODE (target_bool) != bool_mode)
+	target_bool = gen_reg_rtx (bool_mode);
 
       /* Emit the compare_and_swap.  */
-      create_output_operand (&ops[0], target, target_mode);
-      create_output_operand (&ops[1], original_val, mode);
+      create_output_operand (&ops[0], target_bool, bool_mode);
+      create_output_operand (&ops[1], target_oval, mode);
       create_fixed_operand (&ops[2], mem);
-      create_convert_operand_to (&ops[3], expected_val, mode, true);
+      create_convert_operand_to (&ops[3], expected, mode, true);
       create_convert_operand_to (&ops[4], desired, mode, true);
       create_integer_operand (&ops[5], is_weak);
-      create_integer_operand (&ops[6], success);
-      create_integer_operand (&ops[7], failure);
+      create_integer_operand (&ops[6], succ_model);
+      create_integer_operand (&ops[7], fail_model);
       expand_insn (icode, 8, ops);
 
-      /* Store *expected = original_val  */
-      emit_move_insn (gen_rtx_MEM (mode, expected), original_val);
-
       /* Return success/failure.  */
-      return ops[0].value;
+      target_bool = ops[0].value;
+      target_oval = ops[1].value;
+      goto success;
     }
 
-
-  /* Otherwise fall back to the original __sync_val_compare_and_swap which is
-     always seq-cst.  */
+  /* Otherwise fall back to the original __sync_val_compare_and_swap
+     which is always seq-cst.  */
   icode = direct_optab_handler (sync_compare_and_swap_optab, mode);
-  if (icode == CODE_FOR_nothing)
-      return NULL_RTX;
+  if (icode != CODE_FOR_nothing)
+    {
+      rtx cc_reg;
+
+      create_output_operand (&ops[0], target_oval, mode);
+      create_fixed_operand (&ops[1], mem);
+      create_convert_operand_to (&ops[2], expected, mode, true);
+      create_convert_operand_to (&ops[3], desired, mode, true);
+      if (!maybe_expand_insn (icode, 4, ops))
+	return false;
 
-  /* Issue the compare and swap */
-  CAS_val = expand_val_compare_and_swap_1 (mem, expected_val, desired, NULL_RTX,
-					   icode);
+      target_oval = ops[0].value;
+      target_bool = NULL_RTX;
 
-  /* Always store the result back into 'expected'.  If the result is true, its
-     an extra store, but makes the flow better.  */
-  emit_move_insn (gen_rtx_MEM (mode, expected), CAS_val);
+      /* If the caller isn't interested in the boolean return value,
+	 skip the computation of it.  */
+      if (ptarget_bool == NULL)
+	goto success;
 
-  return emit_store_flag_force (target, EQ, CAS_val, expected_val,
-				VOIDmode, 1, 1);
-}
+      /* Otherwise, work out if the compare-and-swap succeeded.  */
+      cc_reg = NULL_RTX;
+      if (have_insn_for (COMPARE, CCmode))
+	note_stores (PATTERN (get_last_insn ()), find_cc_set, &cc_reg);
 
+      target_bool
+	= (cc_reg
+	   ? emit_store_flag_force (target_bool, EQ, cc_reg,
+				    const0_rtx, VOIDmode, 0, 1)
+	   : emit_store_flag_force (target_bool, EQ, target_oval,
+				    expected, VOIDmode, 1, 1));
+      goto success;
+    }
+  return false;
+
+ success:
+  /* Make sure that the oval output winds up where the caller asked.  */
+  if (ptarget_oval)
+    *ptarget_oval = target_oval;
+  if (ptarget_bool)
+    *ptarget_bool = target_bool;
+  return true;
+}
 
 /* This function expands the atomic load operation:
    return the atomically loaded value in MEM.
@@ -7150,13 +7077,13 @@ expand_atomic_load (rtx target, rtx mem, enum memmodel model)
   /* If there is no load pattern, default to a move with barriers. If the size
      of the object is greater than word size on this target, a default load 
      will not be atomic.  */
-  if (GET_MODE_PRECISION(mode) > BITS_PER_WORD)
+  if (GET_MODE_PRECISION (mode) > BITS_PER_WORD)
     {
-      /* Issue val = compare_and_swap (mem, 0 , 0).
+      /* Issue val = compare_and_swap (mem, 0, 0).
 	 This may cause the occasional harmless store of 0 when the value is
 	 already 0, but do it anyway until its determined to be invalid.  */
-      target = expand_val_compare_and_swap (mem, const0_rtx, const0_rtx, 
-					    target);
+      expand_atomic_compare_and_swap (NULL, &target, mem, const0_rtx,
+				      const0_rtx, false, model, model);
       return target;
     }
 
diff --git a/gcc/optabs.h b/gcc/optabs.h
index eabc994..2ca0fcd 100644
--- a/gcc/optabs.h
+++ b/gcc/optabs.h
@@ -955,6 +955,10 @@ enum insn_code can_float_p (enum machine_mode, enum machine_mode, int);
 /* Return true if there is an inline compare and swap pattern.  */
 extern bool can_compare_and_swap_p (enum machine_mode);
 
+/* Generate code for a compare and swap.  */
+extern bool expand_atomic_compare_and_swap (rtx *, rtx *, rtx, rtx, rtx, bool,
+					    enum memmodel, enum memmodel);
+
 /* Generate code for a FIX_EXPR.  */
 extern void expand_fix (rtx, rtx, int);
 
-- 
1.7.6.4