From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mail-ej1-x62e.google.com (mail-ej1-x62e.google.com [IPv6:2a00:1450:4864:20::62e]) by sourceware.org (Postfix) with ESMTPS id BC9693858D32 for ; Wed, 1 Nov 2023 22:17:06 +0000 (GMT) DMARC-Filter: OpenDMARC Filter v1.4.2 sourceware.org BC9693858D32 Authentication-Results: sourceware.org; dmarc=pass (p=none dis=none) header.from=gmail.com Authentication-Results: sourceware.org; spf=pass smtp.mailfrom=gmail.com ARC-Filter: OpenARC Filter v1.0.0 sourceware.org BC9693858D32 Authentication-Results: server2.sourceware.org; arc=none smtp.remote-ip=2a00:1450:4864:20::62e ARC-Seal: i=1; a=rsa-sha256; d=sourceware.org; s=key; t=1698877030; cv=none; b=ZgKB8JkRUVvEhzhk+1TQFKgFZ2+/uWmQ8+YQ+iHnOd8SN+LLDr+nFVQVe8WmRUrqkU1Sh6IvI3mvhnOBEP1OBEh4VDc0JAVeUf3qmIHm1NkqPlGZTwGTc/Y2NVpDBGGiCN2sr78dYnLL+X05Exx9JJ12iFfG3E9mi6TWbl4siY8= ARC-Message-Signature: i=1; a=rsa-sha256; d=sourceware.org; s=key; t=1698877030; c=relaxed/simple; bh=k2iOVhnTuFWQbTWioXqXYvXAGVzKnXhV+/SsRV0PPc0=; h=DKIM-Signature:From:To:Subject:Date:Message-Id:MIME-Version; b=uijlK+f6PCz5s2WKLZBFRLOQqpJSrfTOaRiAXD2FD1QFLx3eNVc2pK06q+WZvFkzzwRAXBHLBDun8/wzVc63lhFvk/97cSSWIxihMChVndiuanNvSMYfreCwK6VLLblp261i//QC4PnHDIT0LY+zIlp3kR2OuTNpZMICJE9eFmI= ARC-Authentication-Results: i=1; server2.sourceware.org Received: by mail-ej1-x62e.google.com with SMTP id a640c23a62f3a-9d465d1c86bso43957566b.3 for ; Wed, 01 Nov 2023 15:17:06 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20230601; t=1698877025; x=1699481825; darn=sourceware.org; h=content-transfer-encoding:mime-version:message-id:date:subject:cc :to:from:from:to:cc:subject:date:message-id:reply-to; bh=YHhQGGo5aqDtFBswCialUdkAwMmaIzKAAAAq8F3I6Kg=; b=eeMrp1oZJIRA0M44iQb/SAUlO3fJDqoi0Gw6YvLM7UdgYOWXy9D4wipXpiBAudO7jU NLSLAIhHYK8E7D7rz7AxNIEvUjDLBzYs0NbjwGk8ELJuUhOLGE4PIicdu1leC5j90Mkv S3y2HdxTMEddM60oLAEEbMemmq4pOyFAdcFTigS++d84NjGBNRDGTJWybCHPb1YXhdPy 858Jl+wW+oBh6wnAuA9mKEgia4hQVHofrWILqS6asLtD1XGizFr2M7m9JhK+lrG95pvT D30TShG1/zo4zCgPGubFO47hvu3NSJEiXidkmLRgc3v+ATOJ3p1QT6mvTjzFUn9FV+CY 8wTg== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20230601; t=1698877025; x=1699481825; h=content-transfer-encoding:mime-version:message-id:date:subject:cc :to:from:x-gm-message-state:from:to:cc:subject:date:message-id :reply-to; bh=YHhQGGo5aqDtFBswCialUdkAwMmaIzKAAAAq8F3I6Kg=; b=UZFGbqqpJWtV9pZfjVTffL/tEABD8txfhz16miyWv9VoaGjFtwPNGZzCweZTt5Xi6N KWyhtpO9Y5DoBEb7zVEajFrEDAOm78hmKf+tYDSZYkU4iyWCyB56DCBRdp0wxHhFpvhs mLEriZiGbGgjxFW8TdXR86IBoMTvK3/WK36FPXyaCDtk2kRuf6MnpJwb5jAYMueVADKi G/UviaS9Vrv4fLxuXfMKB5q4mHjnjta8EXZO+b/zlwph/3rSboSN9uVpkvkZ4Sz7Rm+C w47NbL1eMz17IUq4HSgKMp9pvjicruosdsRxXDYN01JklqvtBUj0wxkPWTBICYwGxjk5 qbKQ== X-Gm-Message-State: AOJu0Yw4siZV7IhLzi1G8bk8woDhUnlOcQgOVmOl4WGO15wIt7vP/1Jl 2ispAFPwsRzaMM6XekchxRkDrMo1aOc= X-Google-Smtp-Source: AGHT+IH6MdPssId3hxiDQN+2vifc5CEOlcbQ+L1xM12tBsaEYrnCQVyHjebp0ZFLldLprILwP8MsGQ== X-Received: by 2002:a17:906:6a23:b0:9bf:2f84:5de7 with SMTP id qw35-20020a1709066a2300b009bf2f845de7mr2974801ejc.4.1698877024522; Wed, 01 Nov 2023 15:17:04 -0700 (PDT) Received: from noahgold-desk.intel.com ([192.55.54.53]) by smtp.gmail.com with ESMTPSA id lx8-20020a170906af0800b00993664a9987sm406787ejb.103.2023.11.01.15.17.02 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Wed, 01 Nov 2023 15:17:03 -0700 (PDT) From: Noah Goldstein To: libc-alpha@sourceware.org Cc: goldstein.w.n@gmail.com, hjl.tools@gmail.com, carlos@systemhalted.org Subject: x86: Fix unchecked AVX512-VBMI2 usage in strrchr-evex-base.S Date: Wed, 1 Nov 2023 17:16:57 -0500 Message-Id: <20231101221657.311121-1-goldstein.w.n@gmail.com> X-Mailer: git-send-email 2.34.1 MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Spam-Status: No, score=-12.1 required=5.0 tests=BAYES_00,DKIM_SIGNED,DKIM_VALID,DKIM_VALID_AU,DKIM_VALID_EF,FREEMAIL_FROM,GIT_PATCH_0,RCVD_IN_DNSWL_NONE,SPF_HELO_NONE,SPF_PASS,TXREP,T_SCC_BODY_TEXT_LINE autolearn=ham autolearn_force=no version=3.4.6 X-Spam-Checker-Version: SpamAssassin 3.4.6 (2021-04-09) on server2.sourceware.org List-Id: strrchr-evex-base used `vpcompress{b|d}` in the page cross logic but was missing the CPU_FEATURE checks for VBMI2 in the ifunc/ifunc-impl-list. The fix is either to add those checks or change the logic to not use `vpcompress{b|d}`. Choosing the latter here so that the strrchr-evex implementation is usable on SKX. New implementation is a bit slower, but this is in a cold path so its probably okay. --- sysdeps/x86_64/multiarch/strrchr-evex-base.S | 64 +++++++++++++------- 1 file changed, 43 insertions(+), 21 deletions(-) diff --git a/sysdeps/x86_64/multiarch/strrchr-evex-base.S b/sysdeps/x86_64/multiarch/strrchr-evex-base.S index cd6a0a870a..b174d43208 100644 --- a/sysdeps/x86_64/multiarch/strrchr-evex-base.S +++ b/sysdeps/x86_64/multiarch/strrchr-evex-base.S @@ -71,7 +71,7 @@ ENTRY_P2ALIGN(STRRCHR, 6) andl $(PAGE_SIZE - 1), %eax cmpl $(PAGE_SIZE - VEC_SIZE), %eax jg L(cross_page_boundary) - +L(page_cross_continue): VMOVU (%rdi), %VMM(1) /* k0 has a 1 for each zero CHAR in YMM1. */ VPTESTN %VMM(1), %VMM(1), %k0 @@ -79,10 +79,11 @@ ENTRY_P2ALIGN(STRRCHR, 6) test %VGPR(rsi), %VGPR(rsi) jz L(aligned_more) /* fallthrough: zero CHAR in first VEC. */ -L(page_cross_return): + /* K1 has a 1 for each search CHAR match in VEC(1). */ VPCMPEQ %VMATCH, %VMM(1), %k1 KMOV %k1, %VGPR(rax) +L(page_cross_return): /* Build mask up until first zero CHAR (used to mask of potential search CHAR matches past the end of the string). */ blsmsk %VGPR(rsi), %VGPR(rsi) @@ -167,7 +168,6 @@ L(first_vec_x1_return): .p2align 4,, 12 L(aligned_more): -L(page_cross_continue): /* Need to keep original pointer incase VEC(1) has last match. */ movq %rdi, %rsi andq $-VEC_SIZE, %rdi @@ -340,34 +340,56 @@ L(return_new_match_ret): leaq (VEC_SIZE * 2)(%rdi, %rax, CHAR_SIZE), %rax ret - .p2align 4,, 4 L(cross_page_boundary): + /* eax contains all the page offset bits of src (rdi). `xor rdi, + rax` sets pointer will all page offset bits cleared so + offset of (PAGE_SIZE - VEC_SIZE) will get last aligned VEC + before page cross (guaranteed to be safe to read). Doing this + as opposed to `movq %rdi, %rax; andq $-VEC_SIZE, %rax` saves + a bit of code size. */ xorq %rdi, %rax - mov $-1, %VRDX - VMOVU (PAGE_SIZE - VEC_SIZE)(%rax), %VMM(6) - VPTESTN %VMM(6), %VMM(6), %k0 + VMOVU (PAGE_SIZE - VEC_SIZE)(%rax), %VMM(1) + VPTESTN %VMM(1), %VMM(1), %k0 KMOV %k0, %VRSI + /* Shift out zero CHAR matches that are before the beginning of + src (rdi). */ # ifdef USE_AS_WCSRCHR movl %edi, %ecx - and $(VEC_SIZE - 1), %ecx + andl $(VEC_SIZE - 1), %ecx shrl $2, %ecx # endif - shlx %SHIFT_REG, %VRDX, %VRDX + shrx %SHIFT_REG, %VRSI, %VRSI -# ifdef USE_AS_WCSRCHR - kmovw %edx, %k1 + test %VRSI, %VRSI + jz L(page_cross_continue) + + /* Found zero CHAR so need to test for search CHAR. */ + VPCMP $0, %VMATCH, %VMM(1), %k1 + KMOV %k1, %VRAX + /* Shift out search CHAR matches that are before the beginning of + src (rdi). */ + shrx %SHIFT_REG, %VRAX, %VRAX + /* For VEC_SIZE == 64 we are just at the end of a cacheline here + so to save code-size just re-use return logic for first VEC. + This is relatively cold code (page cross). */ +# if VEC_SIZE == 64 + jmp L(page_cross_return) + /* 6 bytes from cache-line. */ # else - KMOV %VRDX, %k1 + /* Check if any search CHAR match in range. */ + blsmsk %VRSI, %VRSI + and %VRSI, %VRAX + jz L(ret2) + bsr %VRAX, %VRAX +# ifdef USE_AS_WCSRCHR + leaq (%rdi, %rax, CHAR_SIZE), %rax +# else + addq %rdi, %rax +# endif +L(ret2): + ret + /* 3 bytes from cache-line. */ # endif - - VPCOMPRESS %VMM(6), %VMM(1){%k1}{z} - /* We could technically just jmp back after the vpcompress but - it doesn't save any 16-byte blocks. */ - shrx %SHIFT_REG, %VRSI, %VRSI - test %VRSI, %VRSI - jnz L(page_cross_return) - jmp L(page_cross_continue) - /* 1-byte from cache line. */ END(STRRCHR) #endif -- 2.34.1