From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: (qmail 20801 invoked by alias); 13 Aug 2007 13:52:18 -0000 Received: (qmail 20732 invoked by uid 22791); 13 Aug 2007 13:52:17 -0000 X-Spam-Check-By: sourceware.org Received: from sunsite.ms.mff.cuni.cz (HELO sunsite.mff.cuni.cz) (195.113.15.26) by sourceware.org (qpsmtpd/0.31) with ESMTP; Mon, 13 Aug 2007 13:52:04 +0000 Received: from sunsite.mff.cuni.cz (localhost.localdomain [127.0.0.1]) by sunsite.mff.cuni.cz (8.13.8/8.13.8) with ESMTP id l7DDvfao006549; Mon, 13 Aug 2007 15:57:41 +0200 Received: (from jakub@localhost) by sunsite.mff.cuni.cz (8.13.8/8.13.8/Submit) id l7DDvfsD006548; Mon, 13 Aug 2007 15:57:41 +0200 Date: Mon, 13 Aug 2007 13:52:00 -0000 From: Jakub Jelinek To: Ulrich Drepper Cc: Glibc hackers Subject: [PATCH] i?86 fixes Message-ID: <20070813135741.GM4603@sunsite.mff.cuni.cz> Reply-To: Jakub Jelinek Mime-Version: 1.0 Content-Type: text/plain; charset=us-ascii Content-Disposition: inline User-Agent: Mutt/1.4.2.2i Mailing-List: contact libc-hacker-help@sourceware.org; run by ezmlm Precedence: bulk List-Id: List-Subscribe: List-Archive: List-Post: List-Help: , Sender: libc-hacker-owner@sourceware.org X-SW-Source: 2007-08/txt/msg00013.txt.bz2 Hi! The patch I sent last Monday didn't adjust i[456]86 pthread_cond_*.S so i?86 port was broken on kernels with private futex. The following should fix that together with other issues I found when building --enable-kernel=2.6.9 and --enable-kernel=2.6.22 i686 and x86_64 glibcs. The x86_64 pthread_cond_broadcast change is just an optimization, the dep_mutex(cvptr) value is already loaded in %r8, so it is faster to compare just the register. The x86_64 pthread_cond_signal change is a real bugfix, by xoring FUTEX_WAKE_OP resp. FUTEX_WAKE_OP|FUTEX_PRIVATE_FLAG with (FUTEX_WAKE_OP|FUTEX_WAKE) we would do FUTEX_WAIT resp. FUTEX_WAIT|FUTEX_PRIVATE_FLAG rather than the desired FUTEX_WAKE (or private wake). It would be best to rewrite the hardcoded .eh_frame into .cfi_* directives, but that can wait for later, in the mean time using DW_CFA_advance_loc4 is safe no matter how many insns were inserted (had to debug weird failures because some DW_CFA_advance_loc opcode embedded args grew over 0x3f) and gas optimizes it into DW_CFA_advance_loc{,1,2} when the value is small enough (unlike explicit DW_CFA_advance_loc{,1,2} which is never optimized by gas). 2007-08-13 Jakub Jelinek * sysdeps/unix/sysv/linux/i386/lowlevellock.h (__lll_private_flag): Fix a pasto. * sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S (__pthread_cond_broadcast): Pass LLL_PRIVATE to lll_* and or FUTEX_PRIVATE_FLAG into SYS_futex op if cv is process private. Don't use FUTEX_CMP_REQUEUE if dep_mutex is not process private. * sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S (__pthread_cond_signal): Pass LLL_PRIVATE to lll_* and or FUTEX_PRIVATE_FLAG into SYS_futex op if cv is process private. * sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S: Include kernel-features.h. (__pthread_cond_wait, __condvar_w_cleanup): Pass LLL_PRIVATE to lll_* and or FUTEX_PRIVATE_FLAG into SYS_futex op if cv is process private. Switch DW_CFA_advance_loc1 and some DW_CFA_advance_loc .eh_frame opcodes to DW_CFA_advance_loc4. * sysdeps/unix/sysv/linux/i386/i486/pthread_cond_timedwait.S (__pthread_cond_timedwait, __condvar_tw_cleanup): Pass LLL_PRIVATE to lll_* and or FUTEX_PRIVATE_FLAG into SYS_futex op if cv is process private. Switch DW_CFA_advance_loc{1,2} and some DW_CFA_advance_loc .eh_frame opcodes to DW_CFA_advance_loc4. * sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_rdlock.S: Use #ifdef __ASSUME_PRIVATE_FUTEX instead of #if __ASSUME_PRIVATE_FUTEX. * sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedwrlock.S: Likewise. * sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_unlock.S: Likewise. * sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_wrlock.S: Likewise. * sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedrdlock.S: Likewise. * sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S (__pthread_cond_broadcast): Compare %r8 instead of dep_mutex-cond_*(%rdi) with $-1. * sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S (__pthread_cond_signal): Xor FUTEX_WAKE_OP with FUTEX_WAKE instead of oring. --- libc/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h.jj 2007-08-01 10:50:19.000000000 +0200 +++ libc/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h 2007-08-13 14:20:47.000000000 +0200 @@ -83,7 +83,7 @@ ? ((fl) | THREAD_GETMEM (THREAD_SELF, header.private_futex)) \ : (fl)) \ : ({ unsigned int __fl = ((private) ^ FUTEX_PRIVATE_FLAG); \ - asm ("andl %%fs:%P1, %0" : "+r" (__fl) \ + asm ("andl %%gs:%P1, %0" : "+r" (__fl) \ : "i" (offsetof (struct pthread, header.private_futex))); \ __fl | (fl); })) # endif --- libc/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S.jj 2007-08-01 10:50:19.000000000 +0200 +++ libc/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S 2007-08-13 14:05:05.000000000 +0200 @@ -22,6 +22,7 @@ #include #include #include +#include .text @@ -100,7 +101,20 @@ __pthread_cond_wait: 4: call __pthread_enable_asynccancel movl %eax, (%esp) - movl %esi, %ecx /* movl $FUTEX_WAIT, %ecx */ +#if FUTEX_PRIVATE_FLAG > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex(%ebx) + sete %cl + subl $1, %ecx +#ifdef __ASSUME_PRIVATE_FUTEX + andl $FUTEX_PRIVATE_FLAG, %ecx +#else + andl %gs:PRIVATE_FUTEX, %ecx +#endif +#if FUTEX_WAIT != 0 + addl $FUTEX_WAIT, %ecx +#endif movl %edi, %edx addl $cond_futex, %ebx .Ladd_cond_futex: @@ -161,7 +175,18 @@ __pthread_cond_wait: addl $cond_nwaiters, %ebx movl $SYS_futex, %eax - movl $FUTEX_WAKE, %ecx +#if FUTEX_PRIVATE_FLAG > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex-cond_nwaiters(%ebx) + sete %cl + subl $1, %ecx +#ifdef __ASSUME_PRIVATE_FUTEX + andl $FUTEX_PRIVATE_FLAG, %ecx +#else + andl %gs:PRIVATE_FUTEX, %ecx +#endif + addl $FUTEX_WAKE, %ecx movl $1, %edx ENTER_KERNEL subl $cond_nwaiters, %ebx @@ -197,8 +222,16 @@ __pthread_cond_wait: #else leal cond_lock(%ebx), %edx #endif - /* XYZ */ - movl $LLL_SHARED, %ecx +#if (LLL_SHARED-LLL_PRIVATE) > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex(%ebx) + setne %cl + subl $1, %ecx + andl $(LLL_SHARED-LLL_PRIVATE), %ecx +#if LLL_PRIVATE != 0 + addl $LLL_PRIVATE, %ecx +#endif call __lll_lock_wait jmp 2b @@ -210,8 +243,16 @@ __pthread_cond_wait: #else leal cond_lock(%ebx), %eax #endif - /* XYZ */ - movl $LLL_SHARED, %ecx +#if (LLL_SHARED-LLL_PRIVATE) > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex(%ebx) + setne %cl + subl $1, %ecx + andl $(LLL_SHARED-LLL_PRIVATE), %ecx +#if LLL_PRIVATE != 0 + addl $LLL_PRIVATE, %ecx +#endif call __lll_unlock_wake jmp 4b @@ -222,8 +263,16 @@ __pthread_cond_wait: #else leal cond_lock(%ebx), %edx #endif - /* XYZ */ - movl $LLL_SHARED, %ecx +#if (LLL_SHARED-LLL_PRIVATE) > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex(%ebx) + setne %cl + subl $1, %ecx + andl $(LLL_SHARED-LLL_PRIVATE), %ecx +#if LLL_PRIVATE != 0 + addl $LLL_PRIVATE, %ecx +#endif call __lll_lock_wait jmp 6b @@ -234,8 +283,16 @@ __pthread_cond_wait: #else leal cond_lock(%ebx), %eax #endif - /* XYZ */ - movl $LLL_SHARED, %ecx +#if (LLL_SHARED-LLL_PRIVATE) > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex(%ebx) + setne %cl + subl $1, %ecx + andl $(LLL_SHARED-LLL_PRIVATE), %ecx +#if LLL_PRIVATE != 0 + addl $LLL_PRIVATE, %ecx +#endif call __lll_unlock_wake jmp 11b @@ -256,8 +313,16 @@ __pthread_cond_wait: #else leal cond_lock(%ebx), %eax #endif - /* XYZ */ - movl $LLL_SHARED, %ecx +#if (LLL_SHARED-LLL_PRIVATE) > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex(%ebx) + setne %cl + subl $1, %ecx + andl $(LLL_SHARED-LLL_PRIVATE), %ecx +#if LLL_PRIVATE != 0 + addl $LLL_PRIVATE, %ecx +#endif call __lll_unlock_wake movl %esi, %eax @@ -292,8 +357,16 @@ __condvar_w_cleanup: #else leal cond_lock(%ebx), %edx #endif - /* XYZ */ - movl $LLL_SHARED, %ecx +#if (LLL_SHARED-LLL_PRIVATE) > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex(%ebx) + setne %cl + subl $1, %ecx + andl $(LLL_SHARED-LLL_PRIVATE), %ecx +#if LLL_PRIVATE != 0 + addl $LLL_PRIVATE, %ecx +#endif call __lll_lock_wait 1: movl broadcast_seq(%ebx), %eax @@ -332,7 +405,18 @@ __condvar_w_cleanup: addl $cond_nwaiters, %ebx movl $SYS_futex, %eax - movl $FUTEX_WAKE, %ecx +#if FUTEX_PRIVATE_FLAG > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex-cond_nwaiters(%ebx) + sete %cl + subl $1, %ecx +#ifdef __ASSUME_PRIVATE_FUTEX + andl $FUTEX_PRIVATE_FLAG, %ecx +#else + andl %gs:PRIVATE_FUTEX, %ecx +#endif + addl $FUTEX_WAKE, %ecx movl $1, %edx ENTER_KERNEL subl $cond_nwaiters, %ebx @@ -351,15 +435,34 @@ __condvar_w_cleanup: #else leal cond_lock(%ebx), %eax #endif - /* XYZ */ - movl $LLL_SHARED, %ecx +#if (LLL_SHARED-LLL_PRIVATE) > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex(%ebx) + setne %cl + subl $1, %ecx + andl $(LLL_SHARED-LLL_PRIVATE), %ecx +#if LLL_PRIVATE != 0 + addl $LLL_PRIVATE, %ecx +#endif call __lll_unlock_wake /* Wake up all waiters to make sure no signal gets lost. */ 2: testl %edi, %edi jnz 5f addl $cond_futex, %ebx - movl $FUTEX_WAKE, %ecx +#if FUTEX_PRIVATE_FLAG > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex-cond_futex(%ebx) + sete %cl + subl $1, %ecx +#ifdef __ASSUME_PRIVATE_FUTEX + andl $FUTEX_PRIVATE_FLAG, %ecx +#else + andl %gs:PRIVATE_FUTEX, %ecx +#endif + addl $FUTEX_WAKE, %ecx movl $SYS_futex, %eax movl $0x7fffffff, %edx ENTER_KERNEL @@ -473,12 +576,12 @@ __condvar_w_cleanup: .uleb128 16 .byte 0x83 # DW_CFA_offset %ebx .uleb128 4 - .byte 2 # DW_CFA_advance_loc1 - .byte .Lsubl-.Lpush_ebx + .byte 4 # DW_CFA_advance_loc4 + .4byte .Lsubl-.Lpush_ebx .byte 14 # DW_CFA_def_cfa_offset .uleb128 16+FRAME_SIZE - .byte 2 # DW_CFA_advance_loc1 - .byte .Laddl-.Lsubl + .byte 4 # DW_CFA_advance_loc4 + .4byte .Laddl-.Lsubl .byte 14 # DW_CFA_def_cfa_offset .uleb128 16 .byte 0x40+ .Lpop_ebx-.Laddl # DW_CFA_advance_loc+N @@ -502,13 +605,16 @@ __condvar_w_cleanup: .uleb128 3 .byte 0x83 # DW_CFA_offset %ebx .uleb128 4 - .byte 0x40+.LSbl2-.LSbl1 # DW_CFA_advance_loc+N + .byte 4 # DW_CFA_advance_loc4 + .4byte .LSbl2-.LSbl1 .byte 14 # DW_CFA_def_cfa_offset .uleb128 16+FRAME_SIZE - .byte 0x40+.LSbl3-.LSbl2 # DW_CFA_advance_loc+N + .byte 4 # DW_CFA_advance_loc4 + .4byte .LSbl3-.LSbl2 .byte 14 # DW_CFA_def_cfa_offset .uleb128 16 - .byte 0x40+.LSbl4-.LSbl3 # DW_CFA_advance_loc+N + .byte 4 # DW_CFA_advance_loc4 + .4byte .LSbl4-.LSbl3 .byte 14 # DW_CFA_def_cfa_offset .uleb128 16+FRAME_SIZE .align 4 --- libc/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_rdlock.S.jj 2007-08-13 08:54:48.000000000 +0200 +++ libc/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_rdlock.S 2007-08-13 10:44:59.000000000 +0200 @@ -69,7 +69,7 @@ __pthread_rwlock_rdlock: jne 10f 11: -#if __ASSUME_PRIVATE_FUTEX +#ifdef __ASSUME_PRIVATE_FUTEX movzbl PSHARED(%ebx), %ecx xorl $FUTEX_PRIVATE_FLAG|FUTEX_WAIT, %ecx #else --- libc/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedwrlock.S.jj 2007-08-13 08:55:11.000000000 +0200 +++ libc/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedwrlock.S 2007-08-13 10:45:08.000000000 +0200 @@ -98,7 +98,7 @@ pthread_rwlock_timedwrlock: movl %edx, 4(%esp) movl %esi, %edx -#if __ASSUME_PRIVATE_FUTEX +#ifdef __ASSUME_PRIVATE_FUTEX movzbl PSHARED(%ebp), %ecx xorl $FUTEX_PRIVATE_FLAG|FUTEX_WAIT, %ecx #else --- libc/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_unlock.S.jj 2007-08-13 08:54:40.000000000 +0200 +++ libc/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_unlock.S 2007-08-13 10:44:55.000000000 +0200 @@ -74,7 +74,7 @@ __pthread_rwlock_unlock: jne 7f 8: -#if __ASSUME_PRIVATE_FUTEX +#ifdef __ASSUME_PRIVATE_FUTEX movzbl PSHARED(%edi), %ecx xorl $FUTEX_PRIVATE_FLAG|FUTEX_WAKE, %ecx #else --- libc/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_wrlock.S.jj 2007-08-13 08:55:18.000000000 +0200 +++ libc/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_wrlock.S 2007-08-13 10:45:12.000000000 +0200 @@ -67,7 +67,7 @@ __pthread_rwlock_wrlock: jne 10f 11: -#if __ASSUME_PRIVATE_FUTEX +#ifdef __ASSUME_PRIVATE_FUTEX movzbl PSHARED(%ebx), %ecx xorl $FUTEX_PRIVATE_FLAG|FUTEX_WAIT, %ecx #else --- libc/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedrdlock.S.jj 2007-08-13 08:55:03.000000000 +0200 +++ libc/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedrdlock.S 2007-08-13 10:45:03.000000000 +0200 @@ -100,7 +100,7 @@ pthread_rwlock_timedrdlock: movl %edx, 4(%esp) movl %esi, %edx -#if __ASSUME_PRIVATE_FUTEX +#ifdef __ASSUME_PRIVATE_FUTEX movzbl PSHARED(%ebp), %ecx xorl $FUTEX_PRIVATE_FLAG|FUTEX_WAIT, %ecx #else --- libc/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S.jj 2007-08-01 10:50:19.000000000 +0200 +++ libc/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S 2007-08-13 10:43:10.000000000 +0200 @@ -83,11 +83,18 @@ __pthread_cond_broadcast: je 9f /* XXX: The kernel so far doesn't support requeue to PI futex. */ - testl $PI_BIT, MUTEX_KIND(%edi) + /* XXX: The kernel only supports FUTEX_CMP_REQUEUE to the same + type of futex (private resp. shared). */ + testl $(PI_BIT | PS_BIT), MUTEX_KIND(%edi) jne 9f /* Wake up all threads. */ - movl $FUTEX_CMP_REQUEUE, %ecx +#ifdef __ASSUME_PRIVATE_FUTEX + movl $(FUTEX_CMP_REQUEUE|FUTEX_PRIVATE_FLAG), %ecx +#else + movl %gs:PRIVATE_FUTEX, %ecx + orl $FUTEX_CMP_REQUEUE, %ecx +#endif movl $SYS_futex, %eax movl $0x7fffffff, %esi movl $1, %edx @@ -132,28 +139,63 @@ __pthread_cond_broadcast: #else leal cond_lock(%ebx), %edx #endif - /* XYZ */ - movl $LLL_SHARED, %ecx +#if (LLL_SHARED-LLL_PRIVATE) > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex(%ebx) + setne %cl + subl $1, %ecx + andl $(LLL_SHARED-LLL_PRIVATE), %ecx +#if LLL_PRIVATE != 0 + addl $LLL_PRIVATE, %ecx +#endif call __lll_lock_wait jmp 2b /* Unlock in loop requires waekup. */ 5: leal cond_lock-cond_futex(%ebx), %eax - /* XYZ */ - movl $LLL_SHARED, %ecx +#if (LLL_SHARED-LLL_PRIVATE) > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex-cond_futex(%ebx) + setne %cl + subl $1, %ecx + andl $(LLL_SHARED-LLL_PRIVATE), %ecx +#if LLL_PRIVATE != 0 + addl $LLL_PRIVATE, %ecx +#endif call __lll_unlock_wake jmp 6b /* Unlock in loop requires waekup. */ 7: leal cond_lock-cond_futex(%ebx), %eax - /* XYZ */ - movl $LLL_SHARED, %ecx +#if (LLL_SHARED-LLL_PRIVATE) > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex-cond_futex(%ebx) + setne %cl + subl $1, %ecx + andl $(LLL_SHARED-LLL_PRIVATE), %ecx +#if LLL_PRIVATE != 0 + addl $LLL_PRIVATE, %ecx +#endif call __lll_unlock_wake jmp 8b 9: /* The futex requeue functionality is not available. */ movl $0x7fffffff, %edx - movl $FUTEX_WAKE, %ecx +#if FUTEX_PRIVATE_FLAG > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex-cond_futex(%ebx) + sete %cl + subl $1, %ecx +#ifdef __ASSUME_PRIVATE_FUTEX + andl $FUTEX_PRIVATE_FLAG, %ecx +#else + andl %gs:PRIVATE_FUTEX, %ecx +#endif + addl $FUTEX_WAKE, %ecx movl $SYS_futex, %eax ENTER_KERNEL jmp 10b --- libc/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S.jj 2007-08-01 10:50:19.000000000 +0200 +++ libc/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S 2007-08-13 10:43:20.000000000 +0200 @@ -70,7 +70,18 @@ __pthread_cond_signal: /* Wake up one thread. */ pushl %esi pushl %ebp - movl $FUTEX_WAKE_OP, %ecx +#if FUTEX_PRIVATE_FLAG > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex-cond_futex(%ebx) + sete %cl + subl $1, %ecx +#ifdef __ASSUME_PRIVATE_FUTEX + andl $FUTEX_PRIVATE_FLAG, %ecx +#else + andl %gs:PRIVATE_FUTEX, %ecx +#endif + addl $FUTEX_WAKE_OP, %ecx movl $SYS_futex, %eax movl $1, %edx movl $1, %esi @@ -92,7 +103,9 @@ __pthread_cond_signal: popl %ebx ret -7: movl $FUTEX_WAKE, %ecx +7: /* %ecx should be either FUTEX_WAKE_OP or + FUTEX_WAKE_OP|FUTEX_PRIVATE_FLAG from the previous syscall. */ + xorl $(FUTEX_WAKE ^ FUTEX_WAKE_OP), %ecx movl $SYS_futex, %eax /* %edx should be 1 already from $FUTEX_WAKE_OP syscall. movl $1, %edx */ @@ -106,8 +119,16 @@ __pthread_cond_signal: /* Unlock in loop requires wakeup. */ 5: movl %edi, %eax - /* XYZ */ - movl $LLL_SHARED, %ecx +#if (LLL_SHARED-LLL_PRIVATE) > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex-cond_futex(%ebx) + setne %cl + subl $1, %ecx + andl $(LLL_SHARED-LLL_PRIVATE), %ecx +#if LLL_PRIVATE != 0 + addl $LLL_PRIVATE, %ecx +#endif call __lll_unlock_wake jmp 6b @@ -118,8 +139,16 @@ __pthread_cond_signal: #else leal cond_lock(%edi), %edx #endif - /* XYZ */ - movl $LLL_SHARED, %ecx +#if (LLL_SHARED-LLL_PRIVATE) > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex(%edi) + setne %cl + subl $1, %ecx + andl $(LLL_SHARED-LLL_PRIVATE), %ecx +#if LLL_PRIVATE != 0 + addl $LLL_PRIVATE, %ecx +#endif call __lll_lock_wait jmp 2b --- libc/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_timedwait.S.jj 2007-08-13 08:54:56.000000000 +0200 +++ libc/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_timedwait.S 2007-08-13 14:12:17.000000000 +0200 @@ -158,7 +158,20 @@ __pthread_cond_timedwait: movl %eax, (%esp) leal 4(%esp), %esi - xorl %ecx, %ecx /* movl $FUTEX_WAIT, %ecx */ +#if FUTEX_PRIVATE_FLAG > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex(%ebx) + sete %cl + subl $1, %ecx +#ifdef __ASSUME_PRIVATE_FUTEX + andl $FUTEX_PRIVATE_FLAG, %ecx +#else + andl %gs:PRIVATE_FUTEX, %ecx +#endif +#if FUTEX_WAIT != 0 + addl $FUTEX_WAIT, %ecx +#endif movl %edi, %edx addl $cond_futex, %ebx .Ladd_cond_futex: @@ -232,7 +245,18 @@ __pthread_cond_timedwait: addl $cond_nwaiters, %ebx movl $SYS_futex, %eax - movl $FUTEX_WAKE, %ecx +#if FUTEX_PRIVATE_FLAG > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex-cond_nwaiters(%ebx) + sete %cl + subl $1, %ecx +#ifdef __ASSUME_PRIVATE_FUTEX + andl $FUTEX_PRIVATE_FLAG, %ecx +#else + andl %gs:PRIVATE_FUTEX, %ecx +#endif + addl $FUTEX_WAKE, %ecx movl $1, %edx ENTER_KERNEL subl $cond_nwaiters, %ebx @@ -280,8 +304,16 @@ __pthread_cond_timedwait: #else leal cond_lock(%ebx), %edx #endif - /* XYZ */ - movl $LLL_SHARED, %ecx +#if (LLL_SHARED-LLL_PRIVATE) > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex(%ebx) + setne %cl + subl $1, %ecx + andl $(LLL_SHARED-LLL_PRIVATE), %ecx +#if LLL_PRIVATE != 0 + addl $LLL_PRIVATE, %ecx +#endif call __lll_lock_wait jmp 2b @@ -293,8 +325,16 @@ __pthread_cond_timedwait: #else leal cond_lock(%ebx), %eax #endif - /* XYZ */ - movl $LLL_SHARED, %ecx +#if (LLL_SHARED-LLL_PRIVATE) > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex(%ebx) + setne %cl + subl $1, %ecx + andl $(LLL_SHARED-LLL_PRIVATE), %ecx +#if LLL_PRIVATE != 0 + addl $LLL_PRIVATE, %ecx +#endif call __lll_unlock_wake jmp 4b @@ -305,8 +345,16 @@ __pthread_cond_timedwait: #else leal cond_lock(%ebx), %edx #endif - /* XYZ */ - movl $LLL_SHARED, %ecx +#if (LLL_SHARED-LLL_PRIVATE) > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex(%ebx) + setne %cl + subl $1, %ecx + andl $(LLL_SHARED-LLL_PRIVATE), %ecx +#if LLL_PRIVATE != 0 + addl $LLL_PRIVATE, %ecx +#endif call __lll_lock_wait jmp 6b @@ -317,8 +365,16 @@ __pthread_cond_timedwait: #else leal cond_lock(%ebx), %eax #endif - /* XYZ */ - movl $LLL_SHARED, %ecx +#if (LLL_SHARED-LLL_PRIVATE) > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex(%ebx) + setne %cl + subl $1, %ecx + andl $(LLL_SHARED-LLL_PRIVATE), %ecx +#if LLL_PRIVATE != 0 + addl $LLL_PRIVATE, %ecx +#endif call __lll_unlock_wake jmp 11b @@ -339,8 +395,16 @@ __pthread_cond_timedwait: #else leal cond_lock(%ebx), %eax #endif - /* XYZ */ - movl $LLL_SHARED, %ecx +#if (LLL_SHARED-LLL_PRIVATE) > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex(%ebx) + setne %cl + subl $1, %ecx + andl $(LLL_SHARED-LLL_PRIVATE), %ecx +#if LLL_PRIVATE != 0 + addl $LLL_PRIVATE, %ecx +#endif call __lll_unlock_wake movl %esi, %eax @@ -401,8 +465,16 @@ __condvar_tw_cleanup: #else leal cond_lock(%ebx), %edx #endif - /* XYZ */ - movl $LLL_SHARED, %ecx +#if (LLL_SHARED-LLL_PRIVATE) > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex(%ebx) + setne %cl + subl $1, %ecx + andl $(LLL_SHARED-LLL_PRIVATE), %ecx +#if LLL_PRIVATE != 0 + addl $LLL_PRIVATE, %ecx +#endif call __lll_lock_wait 1: movl broadcast_seq(%ebx), %eax @@ -441,7 +513,18 @@ __condvar_tw_cleanup: addl $cond_nwaiters, %ebx movl $SYS_futex, %eax - movl $FUTEX_WAKE, %ecx +#if FUTEX_PRIVATE_FLAG > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex-cond_nwaiters(%ebx) + sete %cl + subl $1, %ecx +#ifdef __ASSUME_PRIVATE_FUTEX + andl $FUTEX_PRIVATE_FLAG, %ecx +#else + andl %gs:PRIVATE_FUTEX, %ecx +#endif + addl $FUTEX_WAKE, %ecx movl $1, %edx ENTER_KERNEL subl $cond_nwaiters, %ebx @@ -460,15 +543,34 @@ __condvar_tw_cleanup: #else leal cond_lock(%ebx), %eax #endif - /* XYZ */ - movl $LLL_SHARED, %ecx +#if (LLL_SHARED-LLL_PRIVATE) > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex(%ebx) + setne %cl + subl $1, %ecx + andl $(LLL_SHARED-LLL_PRIVATE), %ecx +#if LLL_PRIVATE != 0 + addl $LLL_PRIVATE, %ecx +#endif call __lll_unlock_wake /* Wake up all waiters to make sure no signal gets lost. */ 2: testl %edi, %edi jnz 5f addl $cond_futex, %ebx - movl $FUTEX_WAKE, %ecx +#if FUTEX_PRIVATE_FLAG > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex-cond_futex(%ebx) + sete %cl + subl $1, %ecx +#ifdef __ASSUME_PRIVATE_FUTEX + andl $FUTEX_PRIVATE_FLAG, %ecx +#else + andl %gs:PRIVATE_FUTEX, %ecx +#endif + addl $FUTEX_WAKE, %ecx movl $SYS_futex, %eax movl $0x7fffffff, %edx ENTER_KERNEL @@ -588,12 +690,12 @@ __condvar_tw_cleanup: .uleb128 20 .byte 0x83 # DW_CFA_offset %ebx .uleb128 5 - .byte 2 # DW_CFA_advance_loc1 - .byte .Lsubl-.Lpush_ebx + .byte 4 # DW_CFA_advance_loc4 + .4byte .Lsubl-.Lpush_ebx .byte 14 # DW_CFA_def_cfa_offset .uleb128 20+FRAME_SIZE - .byte 3 # DW_CFA_advance_loc2 - .2byte .Laddl-.Lsubl + .byte 4 # DW_CFA_advance_loc4 + .4byte .Laddl-.Lsubl .byte 14 # DW_CFA_def_cfa_offset .uleb128 20 .byte 0x40+.Lpop_ebx-.Laddl # DW_CFA_advance_loc+N @@ -615,7 +717,8 @@ __condvar_tw_cleanup: .byte 0x40+.LSbl1-.Lpop_edi # DW_CFA_advance_loc+N .byte 14 # DW_CFA_def_cfa_offset .uleb128 20 - .byte 0x40+.LSbl2-.LSbl1 # DW_CFA_advance_loc+N + .byte 4 # DW_CFA_advance_loc4 + .4byte .LSbl2-.LSbl1 .byte 14 # DW_CFA_def_cfa_offset .uleb128 20+FRAME_SIZE .byte 0x85 # DW_CFA_offset %ebp @@ -626,14 +729,15 @@ __condvar_tw_cleanup: .uleb128 4 .byte 0x83 # DW_CFA_offset %ebx .uleb128 5 - .byte 0x40+.LSbl3-.LSbl2 # DW_CFA_advance_loc+N + .byte 4 # DW_CFA_advance_loc4 + .4byte .LSbl3-.LSbl2 .byte 14 # DW_CFA_def_cfa_offset .uleb128 20 + .byte 4 # DW_CFA_advance_loc4 #if defined __NR_clock_gettime && !defined __ASSUME_POSIX_TIMERS - .byte 0x40+.LSbl4-.LSbl3 # DW_CFA_advance_loc+N + .4byte .LSbl4-.LSbl3 #else - .byte 4 # DW_CFA_advance_loc4 - .long .LSbl5-.LSbl3 + .4byte .LSbl5-.LSbl3 #endif .byte 14 # DW_CFA_def_cfa_offset .uleb128 20+FRAME_SIZE --- libc/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S.jj 2007-08-13 08:35:02.000000000 +0200 +++ libc/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S 2007-08-13 09:50:33.000000000 +0200 @@ -132,7 +132,7 @@ __pthread_cond_broadcast: /* Unlock in loop requires wakeup. */ 7: addq $cond_lock-cond_futex, %rdi - cmpq $-1, dep_mutex-cond_lock(%rdi) + cmpq $-1, %r8 movl $LLL_PRIVATE, %eax movl $LLL_SHARED, %esi cmovne %eax, %esi @@ -141,7 +141,7 @@ __pthread_cond_broadcast: jmp 8b 9: /* The futex requeue functionality is not available. */ - cmpq $-1, dep_mutex-cond_futex(%rdi) + cmpq $-1, %r8 movl $0x7fffffff, %edx #ifdef __ASSUME_PRIVATE_FUTEX movl $FUTEX_WAKE, %eax --- libc/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S.jj 2007-08-13 08:35:02.000000000 +0200 +++ libc/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S 2007-08-13 10:04:35.000000000 +0200 @@ -87,7 +87,7 @@ __pthread_cond_signal: 7: /* %esi should be either FUTEX_WAKE_OP or FUTEX_WAKE_OP|FUTEX_PRIVATE_FLAG from the previous syscall. */ - xorl $(FUTEX_WAKE | FUTEX_WAKE_OP), %esi + xorl $(FUTEX_WAKE ^ FUTEX_WAKE_OP), %esi movl $SYS_futex, %eax /* %rdx should be 1 already from $FUTEX_WAKE_OP syscall. movl $1, %edx */ Jakub