From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: (qmail 14310 invoked by alias); 28 Oct 2011 04:08:34 -0000 Received: (qmail 13918 invoked by uid 22791); 28 Oct 2011 04:08:28 -0000 X-SWARE-Spam-Status: No, hits=-2.4 required=5.0 tests=AWL,BAYES_00,DKIM_SIGNED,DKIM_VALID,FREEMAIL_ENVFROM_END_DIGIT,FREEMAIL_FROM,RCVD_IN_DNSWL_LOW,T_TO_NO_BRKTS_FREEMAIL X-Spam-Check-By: sourceware.org Received: from mail-gx0-f175.google.com (HELO mail-gx0-f175.google.com) (209.85.161.175) by sourceware.org (qpsmtpd/0.43rc1) with ESMTP; Fri, 28 Oct 2011 04:07:52 +0000 Received: by ggnj1 with SMTP id j1so3779666ggn.20 for ; Thu, 27 Oct 2011 21:07:52 -0700 (PDT) Received: by 10.236.124.17 with SMTP id w17mr781163yhh.126.1319774872317; Thu, 27 Oct 2011 21:07:52 -0700 (PDT) Received: from localhost.localdomain (c-98-203-235-125.hsd1.wa.comcast.net. [98.203.235.125]) by mx.google.com with ESMTPS id j25sm10849016yhm.12.2011.10.27.21.07.51 (version=TLSv1/SSLv3 cipher=OTHER); Thu, 27 Oct 2011 21:07:51 -0700 (PDT) From: Richard Henderson To: gcc-patches@gcc.gnu.org Cc: amacleod@redhat.com Subject: [PATCH 4/9] Rewrite all compare-and-swap in terms of expand_atomic_compare_and_swap. Date: Fri, 28 Oct 2011 05:40:00 -0000 Message-Id: <1319774858-9181-5-git-send-email-rth@redhat.com> In-Reply-To: <1319774858-9181-1-git-send-email-rth@redhat.com> References: <1319774858-9181-1-git-send-email-rth@redhat.com> X-IsSubscribed: yes Mailing-List: contact gcc-patches-help@gcc.gnu.org; run by ezmlm Precedence: bulk List-Id: List-Archive: List-Post: List-Help: Sender: gcc-patches-owner@gcc.gnu.org X-SW-Source: 2011-10/txt/msg02612.txt.bz2 --- gcc/builtins.c | 39 +++++--- gcc/expr.h | 4 - gcc/optabs.c | 267 ++++++++++++++++++++----------------------------------- gcc/optabs.h | 4 + 4 files changed, 126 insertions(+), 188 deletions(-) diff --git a/gcc/builtins.c b/gcc/builtins.c index cb9da83..756070f 100644 --- a/gcc/builtins.c +++ b/gcc/builtins.c @@ -5196,10 +5196,13 @@ expand_builtin_compare_and_swap (enum machine_mode mode, tree exp, old_val = expand_expr_force_mode (CALL_EXPR_ARG (exp, 1), mode); new_val = expand_expr_force_mode (CALL_EXPR_ARG (exp, 2), mode); - if (is_bool) - return expand_bool_compare_and_swap (mem, old_val, new_val, target); - else - return expand_val_compare_and_swap (mem, old_val, new_val, target); + if (!expand_atomic_compare_and_swap ((is_bool ? &target : NULL), + (is_bool ? NULL : &target), + mem, old_val, new_val, false, + MEMMODEL_SEQ_CST, MEMMODEL_SEQ_CST)) + return NULL_RTX; + + return target; } /* Expand the __sync_lock_test_and_set intrinsic. Note that the most @@ -5293,8 +5296,10 @@ static rtx expand_builtin_atomic_compare_exchange (enum machine_mode mode, tree exp, rtx target) { - rtx expect, desired, mem, weak; + rtx expect, desired, mem, oldval; enum memmodel success, failure; + tree weak; + bool is_weak; success = get_memmodel (CALL_EXPR_ARG (exp, 4)); failure = get_memmodel (CALL_EXPR_ARG (exp, 5)); @@ -5307,24 +5312,31 @@ expand_builtin_atomic_compare_exchange (enum machine_mode mode, tree exp, if (failure > success) { - error ("failure memory model cannot be stronger than success memory model for %<__atomic_compare_exchange%>"); + error ("failure memory model cannot be stronger than success " + "memory model for %<__atomic_compare_exchange%>"); return NULL_RTX; } /* Expand the operands. */ mem = get_builtin_sync_mem (CALL_EXPR_ARG (exp, 0), mode); - expect = expand_expr (CALL_EXPR_ARG (exp, 1), NULL_RTX, ptr_mode, - EXPAND_NORMAL); + expect = expand_normal (CALL_EXPR_ARG (exp, 1)); expect = convert_memory_address (Pmode, expect); - desired = expand_expr_force_mode (CALL_EXPR_ARG (exp, 2), mode); - weak = expand_expr (CALL_EXPR_ARG (exp, 3), NULL_RTX, ptr_mode, - EXPAND_NORMAL); + weak = CALL_EXPR_ARG (exp, 3); + is_weak = false; + if (host_integerp (weak, 0) && tree_low_cst (weak, 0) != 0) + is_weak = true; - return expand_atomic_compare_exchange (target, mem, expect, desired, weak, - success, failure); + oldval = copy_to_reg (gen_rtx_MEM (mode, expect)); + + if (!expand_atomic_compare_and_swap (&target, &oldval, mem, oldval, + desired, is_weak, success, failure)) + return NULL_RTX; + + emit_move_insn (gen_rtx_MEM (mode, expect), oldval); + return target; } /* Expand the __atomic_load intrinsic: @@ -5442,7 +5454,6 @@ fold_builtin_atomic_always_lock_free (tree arg) { int size; enum machine_mode mode; - enum insn_code icode; if (TREE_CODE (arg) != INTEGER_CST) return NULL_TREE; diff --git a/gcc/expr.h b/gcc/expr.h index 8cae22b..1623ad9 100644 --- a/gcc/expr.h +++ b/gcc/expr.h @@ -212,14 +212,10 @@ int can_conditionally_move_p (enum machine_mode mode); rtx emit_conditional_add (rtx, enum rtx_code, rtx, rtx, enum machine_mode, rtx, rtx, enum machine_mode, int); -rtx expand_val_compare_and_swap (rtx, rtx, rtx, rtx); -rtx expand_bool_compare_and_swap (rtx, rtx, rtx, rtx); rtx expand_sync_operation (rtx, rtx, enum rtx_code); rtx expand_sync_fetch_operation (rtx, rtx, enum rtx_code, bool, rtx); rtx expand_atomic_exchange (rtx, rtx, rtx, enum memmodel); -rtx expand_atomic_compare_exchange (rtx, rtx, rtx, rtx, rtx, enum memmodel, - enum memmodel); rtx expand_atomic_load (rtx, rtx, enum memmodel); rtx expand_atomic_store (rtx, rtx, enum memmodel); rtx expand_atomic_fetch_op (rtx, rtx, rtx, enum rtx_code, enum memmodel, diff --git a/gcc/optabs.c b/gcc/optabs.c index 9b4d6cf..b7c00be 100644 --- a/gcc/optabs.c +++ b/gcc/optabs.c @@ -6799,45 +6799,6 @@ can_compare_and_swap_p (enum machine_mode mode) return false; } -/* This is an internal subroutine of the other compare_and_swap expanders. - MEM, OLD_VAL and NEW_VAL are as you'd expect for a compare-and-swap - operation. TARGET is an optional place to store the value result of - the operation. ICODE is the particular instruction to expand. Return - the result of the operation. */ - -static rtx -expand_val_compare_and_swap_1 (rtx mem, rtx old_val, rtx new_val, - rtx target, enum insn_code icode) -{ - struct expand_operand ops[4]; - enum machine_mode mode = GET_MODE (mem); - - create_output_operand (&ops[0], target, mode); - create_fixed_operand (&ops[1], mem); - /* OLD_VAL and NEW_VAL may have been promoted to a wider mode. - Shrink them if so. */ - create_convert_operand_to (&ops[2], old_val, mode, true); - create_convert_operand_to (&ops[3], new_val, mode, true); - if (maybe_expand_insn (icode, 4, ops)) - return ops[0].value; - return NULL_RTX; -} - -/* Expand a compare-and-swap operation and return its value. */ - -rtx -expand_val_compare_and_swap (rtx mem, rtx old_val, rtx new_val, rtx target) -{ - enum machine_mode mode = GET_MODE (mem); - enum insn_code icode - = direct_optab_handler (sync_compare_and_swap_optab, mode); - - if (icode == CODE_FOR_nothing) - return NULL_RTX; - - return expand_val_compare_and_swap_1 (mem, old_val, new_val, target, icode); -} - /* Helper function to find the MODE_CC set in a sync_compare_and_swap pattern. */ @@ -6853,58 +6814,6 @@ find_cc_set (rtx x, const_rtx pat, void *data) } } -/* Expand a compare-and-swap operation and store true into the result if - the operation was successful and false otherwise. Return the result. - Unlike other routines, TARGET is not optional. */ - -rtx -expand_bool_compare_and_swap (rtx mem, rtx old_val, rtx new_val, rtx target) -{ - enum machine_mode mode = GET_MODE (mem); - enum insn_code icode; - rtx subtarget, seq, cc_reg; - - /* If the target supports a compare-and-swap pattern that simultaneously - sets some flag for success, then use it. Otherwise use the regular - compare-and-swap and follow that immediately with a compare insn. */ - icode = direct_optab_handler (sync_compare_and_swap_optab, mode); - if (icode == CODE_FOR_nothing) - return NULL_RTX; - - do_pending_stack_adjust (); - do - { - start_sequence (); - subtarget = expand_val_compare_and_swap_1 (mem, old_val, new_val, - NULL_RTX, icode); - cc_reg = NULL_RTX; - if (subtarget == NULL_RTX) - { - end_sequence (); - return NULL_RTX; - } - - if (have_insn_for (COMPARE, CCmode)) - note_stores (PATTERN (get_last_insn ()), find_cc_set, &cc_reg); - seq = get_insns (); - end_sequence (); - - /* We might be comparing against an old value. Try again. :-( */ - if (!cc_reg && MEM_P (old_val)) - { - seq = NULL_RTX; - old_val = force_reg (mode, old_val); - } - } - while (!seq); - - emit_insn (seq); - if (cc_reg) - return emit_store_flag_force (target, EQ, cc_reg, const0_rtx, VOIDmode, 0, 1); - else - return emit_store_flag_force (target, EQ, subtarget, old_val, VOIDmode, 1, 1); -} - /* This is a helper function for the other atomic operations. This function emits a loop that contains SEQ that iterates until a compare-and-swap operation at the end succeeds. MEM is the memory to be modified. SEQ is @@ -6918,8 +6827,7 @@ static bool expand_compare_and_swap_loop (rtx mem, rtx old_reg, rtx new_reg, rtx seq) { enum machine_mode mode = GET_MODE (mem); - enum insn_code icode; - rtx label, cmp_reg, subtarget, cc_reg; + rtx label, cmp_reg, success, oldval; /* The loop we want to generate looks like @@ -6927,8 +6835,8 @@ expand_compare_and_swap_loop (rtx mem, rtx old_reg, rtx new_reg, rtx seq) label: old_reg = cmp_reg; seq; - cmp_reg = compare-and-swap(mem, old_reg, new_reg) - if (cmp_reg != old_reg) + (success, cmp_reg) = compare-and-swap(mem, old_reg, new_reg) + if (success) goto label; Note that we only do the plain load from memory once. Subsequent @@ -6943,35 +6851,19 @@ expand_compare_and_swap_loop (rtx mem, rtx old_reg, rtx new_reg, rtx seq) if (seq) emit_insn (seq); - /* If the target supports a compare-and-swap pattern that simultaneously - sets some flag for success, then use it. Otherwise use the regular - compare-and-swap and follow that immediately with a compare insn. */ - icode = direct_optab_handler (sync_compare_and_swap_optab, mode); - if (icode == CODE_FOR_nothing) + success = NULL_RTX; + oldval = cmp_reg; + if (!expand_atomic_compare_and_swap (&success, &oldval, mem, old_reg, + new_reg, false, MEMMODEL_SEQ_CST, + MEMMODEL_RELAXED)) return false; - subtarget = expand_val_compare_and_swap_1 (mem, old_reg, new_reg, - cmp_reg, icode); - if (subtarget == NULL_RTX) - return false; - - cc_reg = NULL_RTX; - if (have_insn_for (COMPARE, CCmode)) - note_stores (PATTERN (get_last_insn ()), find_cc_set, &cc_reg); - if (cc_reg) - { - cmp_reg = cc_reg; - old_reg = const0_rtx; - } - else - { - if (subtarget != cmp_reg) - emit_move_insn (cmp_reg, subtarget); - } + if (oldval != cmp_reg) + emit_move_insn (cmp_reg, oldval); /* ??? Mark this jump predicted not taken? */ - emit_cmp_and_jump_insns (cmp_reg, old_reg, NE, const0_rtx, GET_MODE (cmp_reg), 1, - label); + emit_cmp_and_jump_insns (success, const0_rtx, EQ, const0_rtx, + GET_MODE (success), 1, label); return true; } @@ -7050,77 +6942,112 @@ expand_atomic_exchange (rtx target, rtx mem, rtx val, enum memmodel model) /* This function expands the atomic compare exchange operation: + *PTARGET_BOOL is an optional place to store the boolean success/failure. + *PTARGET_OVAL is an optional place to store the old value from memory. + Both target parameters may be NULL to indicate that we do not care about + that return value. Both target parameters are updated on success to + the actual location of the corresponding result. + MEMMODEL is the memory model variant to use. - TARGET is an option place to stick the return value. */ -rtx -expand_atomic_compare_exchange (rtx target, rtx mem, rtx expected, - rtx desired, rtx weak, enum memmodel success, - enum memmodel failure) + The return value of the function is true for success. */ + +bool +expand_atomic_compare_and_swap (rtx *ptarget_bool, rtx *ptarget_oval, + rtx mem, rtx expected, rtx desired, + bool is_weak, enum memmodel succ_model, + enum memmodel fail_model) { enum machine_mode mode = GET_MODE (mem); - enum insn_code icode = CODE_FOR_nothing; - rtx expected_val, CAS_val; + struct expand_operand ops[8]; + enum insn_code icode; + rtx target_bool, target_oval; - /* Load '*expected into a register for the compare and swap */ - expected_val = gen_reg_rtx (mode); - emit_move_insn (expected_val, gen_rtx_MEM (mode, expected)); + /* Load expected into a register for the compare and swap. */ + if (MEM_P (expected)) + expected = copy_to_reg (expected); + + /* Make sure we always have some place to put the return oldval. + Further, make sure that place is distinct from the input expected, + just in case we need that path down below. */ + if (ptarget_oval == NULL + || (target_oval = *ptarget_oval) == NULL + || reg_overlap_mentioned_p (expected, target_oval)) + target_oval = gen_reg_rtx (mode); icode = direct_optab_handler (atomic_compare_and_swap_optab, mode); - if (icode != CODE_FOR_nothing) { - struct expand_operand ops[8]; - rtx original_val = gen_reg_rtx (mode); - enum machine_mode target_mode; - int is_weak; - - target_mode = insn_data[icode].operand[0].mode; - - /* Either WEAK is explcitly 1, or assume strong to be safe. */ - is_weak = (weak == const1_rtx); + enum machine_mode bool_mode = insn_data[icode].operand[0].mode; - if (target == NULL_RTX || target == const0_rtx - || GET_MODE (target) != target_mode) - target = gen_reg_rtx (target_mode); + /* Make sure we always have a place for the bool operand. */ + if (ptarget_bool == NULL + || (target_bool = *ptarget_bool) == NULL + || GET_MODE (target_bool) != bool_mode) + target_bool = gen_reg_rtx (bool_mode); /* Emit the compare_and_swap. */ - create_output_operand (&ops[0], target, target_mode); - create_output_operand (&ops[1], original_val, mode); + create_output_operand (&ops[0], target_bool, bool_mode); + create_output_operand (&ops[1], target_oval, mode); create_fixed_operand (&ops[2], mem); - create_convert_operand_to (&ops[3], expected_val, mode, true); + create_convert_operand_to (&ops[3], expected, mode, true); create_convert_operand_to (&ops[4], desired, mode, true); create_integer_operand (&ops[5], is_weak); - create_integer_operand (&ops[6], success); - create_integer_operand (&ops[7], failure); + create_integer_operand (&ops[6], succ_model); + create_integer_operand (&ops[7], fail_model); expand_insn (icode, 8, ops); - /* Store *expected = original_val */ - emit_move_insn (gen_rtx_MEM (mode, expected), original_val); - /* Return success/failure. */ - return ops[0].value; + target_bool = ops[0].value; + target_oval = ops[1].value; + goto success; } - - /* Otherwise fall back to the original __sync_val_compare_and_swap which is - always seq-cst. */ + /* Otherwise fall back to the original __sync_val_compare_and_swap + which is always seq-cst. */ icode = direct_optab_handler (sync_compare_and_swap_optab, mode); - if (icode == CODE_FOR_nothing) - return NULL_RTX; + if (icode != CODE_FOR_nothing) + { + rtx cc_reg; + + create_output_operand (&ops[0], target_oval, mode); + create_fixed_operand (&ops[1], mem); + create_convert_operand_to (&ops[2], expected, mode, true); + create_convert_operand_to (&ops[3], desired, mode, true); + if (!maybe_expand_insn (icode, 4, ops)) + return false; - /* Issue the compare and swap */ - CAS_val = expand_val_compare_and_swap_1 (mem, expected_val, desired, NULL_RTX, - icode); + target_oval = ops[0].value; + target_bool = NULL_RTX; - /* Always store the result back into 'expected'. If the result is true, its - an extra store, but makes the flow better. */ - emit_move_insn (gen_rtx_MEM (mode, expected), CAS_val); + /* If the caller isn't interested in the boolean return value, + skip the computation of it. */ + if (ptarget_bool == NULL) + goto success; - return emit_store_flag_force (target, EQ, CAS_val, expected_val, - VOIDmode, 1, 1); -} + /* Otherwise, work out if the compare-and-swap succeeded. */ + cc_reg = NULL_RTX; + if (have_insn_for (COMPARE, CCmode)) + note_stores (PATTERN (get_last_insn ()), find_cc_set, &cc_reg); + target_bool + = (cc_reg + ? emit_store_flag_force (target_bool, EQ, cc_reg, + const0_rtx, VOIDmode, 0, 1) + : emit_store_flag_force (target_bool, EQ, target_oval, + expected, VOIDmode, 1, 1)); + goto success; + } + return false; + + success: + /* Make sure that the oval output winds up where the caller asked. */ + if (ptarget_oval) + *ptarget_oval = target_oval; + if (ptarget_bool) + *ptarget_bool = target_bool; + return true; +} /* This function expands the atomic load operation: return the atomically loaded value in MEM. @@ -7150,13 +7077,13 @@ expand_atomic_load (rtx target, rtx mem, enum memmodel model) /* If there is no load pattern, default to a move with barriers. If the size of the object is greater than word size on this target, a default load will not be atomic. */ - if (GET_MODE_PRECISION(mode) > BITS_PER_WORD) + if (GET_MODE_PRECISION (mode) > BITS_PER_WORD) { - /* Issue val = compare_and_swap (mem, 0 , 0). + /* Issue val = compare_and_swap (mem, 0, 0). This may cause the occasional harmless store of 0 when the value is already 0, but do it anyway until its determined to be invalid. */ - target = expand_val_compare_and_swap (mem, const0_rtx, const0_rtx, - target); + expand_atomic_compare_and_swap (NULL, &target, mem, const0_rtx, + const0_rtx, false, model, model); return target; } diff --git a/gcc/optabs.h b/gcc/optabs.h index eabc994..2ca0fcd 100644 --- a/gcc/optabs.h +++ b/gcc/optabs.h @@ -955,6 +955,10 @@ enum insn_code can_float_p (enum machine_mode, enum machine_mode, int); /* Return true if there is an inline compare and swap pattern. */ extern bool can_compare_and_swap_p (enum machine_mode); +/* Generate code for a compare and swap. */ +extern bool expand_atomic_compare_and_swap (rtx *, rtx *, rtx, rtx, rtx, bool, + enum memmodel, enum memmodel); + /* Generate code for a FIX_EXPR. */ extern void expand_fix (rtx, rtx, int); -- 1.7.6.4