Aren
/
NetBSD
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

replace the str*() functions by speed optimized versions, submitted by 

J.T. Conklin per PR port-amd64/25411
This commit is contained in:
drochner 2004-07-19 20:04:41 +00:00
parent 26fc0dfe09
commit 07d87c5c6e
9 changed files with 808 additions and 246 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
lib/libc/arch/x86_64/string/index.S
View File

@ -1,29 +1,4 @@
/*
* Written by J.T. Conklin <jtc@NetBSD.org>.
* Public domain.
* Adapted for NetBSD/x86_64 by Frank van der Linden <fvdl@wasabisystems.com>
*/
/* $NetBSD: index.S,v 1.3 2004/07/19 20:04:41 drochner Exp $ */
#include <machine/asm.h>
#if defined(LIBC_SCCS)
RCSID("$NetBSD: index.S,v 1.2 2003/07/26 19:24:38 salo Exp $")
#endif
#ifdef STRCHR
ENTRY(strchr)
#else
ENTRY(index)
#endif
movq %rdi,%rax
movb %sil,%cl
L1:
movb (%rax),%dl
cmpb %dl,%cl /* found char? */
je L2
incq %rax
testb %dl,%dl /* null terminator? */
jnz L1
xorq %rax,%rax
L2:
ret
#define INDEX
#include "strchr.S"

112
lib/libc/arch/x86_64/string/memchr.S
View File

@ -1,25 +1,111 @@
/*
* Written by J.T. Conklin <jtc@NetBSD.org>.
* Written by J.T. Conklin <jtc@acorntoolworks.com>
* Public domain.
* Adapted for NetBSD/x86_64 by Frank van der Linden <fvdl@wasabisystems.com>
*/
#include <machine/asm.h>
#if defined(LIBC_SCCS)
RCSID("$NetBSD: memchr.S,v 1.2 2003/07/26 19:24:39 salo Exp $")
RCSID("$NetBSD: memchr.S,v 1.3 2004/07/19 20:04:41 drochner Exp $")
#endif
ENTRY(memchr)
movb %sil,%al /* set character to search for */
movq %rdx,%rcx /* set length of search */
testq %rcx,%rcx /* test for len == 0 */
jz L1
cld /* set search forward */
repne /* search! */
scasb
jne L1 /* scan failed, return null */
leaq -1(%rdi),%rax /* adjust result of scan */
movzbq %sil,%rcx
/*
* Align to word boundry
* Consider unrolling loop?
*/
testq %rdx,%rdx /* nbytes == 0? */
je .Lzero
.Lalign:
testb $7,%dil
je .Lword_aligned
movq %rdi,%rax
cmpb (%rdi),%cl
je .Ldone
incq %rdi
decq %rdx
jnz .Lalign
jmp .Lzero
.Lword_aligned:
/* copy char to all bytes in word */
movb %cl,%ch
movq %rcx,%rsi
salq $16,%rcx
orq %rsi,%rcx
movq %rcx,%rsi
salq $32,%rcx
orq %rsi,%rcx
movabsq $0x0101010101010101,%r8
movabsq $0x8080808080808080,%r9
_ALIGN_TEXT
.Lloop:
cmpq $7,%rdx /* nbytes > 8 */
jbe .Lbyte
movq (%rdi),%rsi
addq $8,%rdi
xorq %rcx,%rsi
subq $8,%rdx
subq %r8,%rsi
testq %r9,%rsi
je .Lloop
/*
* In rare cases, the above loop may exit prematurely. We must
* return to the loop if none of the bytes in the word are
* equal to ch.
*/
leaq -8(%rdi),%rax
cmpb -8(%rdi),%cl /* 1st byte == ch? */
je .Ldone
leaq -7(%rdi),%rax
cmpb -7(%rdi),%cl /* 2nd byte == ch? */
je .Ldone
leaq -6(%rdi),%rax
cmpb -6(%rdi),%cl /* 3rd byte == ch? */
je .Ldone
leaq -5(%rdi),%rax
cmpb -5(%rdi),%cl /* 4th byte == ch? */
je .Ldone
leaq -4(%rdi),%rax
cmpb -4(%rdi),%cl /* 5th byte == ch? */
je .Ldone
leaq -3(%rdi),%rax
cmpb -3(%rdi),%cl /* 6th byte == ch? */
je .Ldone
leaq -2(%rdi),%rax
cmpb -2(%rdi),%cl /* 7th byte == ch? */
je .Ldone
leaq -1(%rdi),%rax
cmpb -1(%rdi),%cl /* 7th byte == ch? */
jne .Lloop
ret
L1: xorq %rax,%rax
.Lbyte:
testq %rdx,%rdx
je .Lzero
.Lbyte_loop:
movq %rdi,%rax
cmpb (%rdi),%cl
je .Ldone
incq %rdi
decq %rdx
jnz .Lbyte_loop
.Lzero:
xorq %rax,%rax
.Ldone:
ret

31
lib/libc/arch/x86_64/string/rindex.S
View File

@ -1,29 +1,4 @@
/*
* Written by J.T. Conklin <jtc@NetBSD.org>.
* Public domain.
* Adapted for NetBSD/x86_64 by Frank van der Linden <fvdl@wasabisystems.com>
*/
/* $NetBSD: rindex.S,v 1.3 2004/07/19 20:04:41 drochner Exp $ */
#include <machine/asm.h>
#if defined(LIBC_SCCS)
RCSID("$NetBSD: rindex.S,v 1.2 2003/07/26 19:24:39 salo Exp $")
#endif
#ifdef STRRCHR
ENTRY(strrchr)
#else
ENTRY(rindex)
#endif
movb %sil,%cl
xorq %rax,%rax /* init pointer to null */
L1:
movb (%rdi),%dl
cmpb %dl,%cl
jne L2
movq %rdi,%rax
L2:
incq %rdi
testb %dl,%dl /* null terminator??? */
jnz L1
ret
#define RINDEX
#include "strrchr.S"

199
lib/libc/arch/x86_64/string/strcat.S
View File

@ -1,65 +1,164 @@
/*
* Written by J.T. Conklin <jtc@NetBSD.org>.
* Written by J.T. Conklin <jtc@acorntoolworks.com>
* Public domain.
* Adapted for NetBSD/x86_64 by Frank van der Linden <fvdl@wasabisystems.com>
*/
#include <machine/asm.h>
#if defined(LIBC_SCCS)
RCSID("$NetBSD: strcat.S,v 1.2 2003/07/26 19:24:39 salo Exp $")
RCSID("$NetBSD: strcat.S,v 1.3 2004/07/19 20:04:41 drochner Exp $")
#endif
/*
* NOTE: I've unrolled the loop eight times: large enough to make a
* significant difference, and small enough not to totally trash the
* cache.
ENTRY(strcat)
movq %rdi,%rax
movabsq $0x0101010101010101,%r8
movabsq $0x8080808080808080,%r9
/*
* Align destination to word boundary.
* Consider unrolling loop?
*/
.Lscan:
.Lscan_align:
testb $7,%dil
je .Lscan_aligned
cmpb $0,(%rdi)
je .Lcopy
incq %rdi
jmp .Lscan_align
_ALIGN_TEXT
.Lscan_aligned:
.Lscan_loop:
movq (%rdi),%rdx
addq $8,%rdi
subq %r8,%rdx
testq %r9,%rdx
je .Lscan_loop
/*
* In rare cases, the above loop may exit prematurely. We must
* return to the loop if none of the bytes in the word equal 0.
*/
ENTRY(strcat)
movq %rdi,%r11
cmpb $0,-8(%rdi) /* 1st byte == 0? */
jne 1f
subq $8,%rdi
jmp .Lcopy
cld /* set search forward */
xorl %eax,%eax /* set search for null terminator */
movq $-1,%rcx /* set search for lots of characters */
repne /* search! */
scasb
1: cmpb $0,-7(%rdi) /* 2nd byte == 0? */
jne 1f
subq $7,%rdi
jmp .Lcopy
decq %rdi
1: cmpb $0,-6(%rdi) /* 3rd byte == 0? */
jne 1f
subq $6,%rdi
jmp .Lcopy
L1: movb (%rsi),%al /* unroll loop, but not too much */
movb %al,(%rdi)
testb %al,%al
jz L2
movb 1(%rsi),%al
movb %al,1(%rdi)
testb %al,%al
jz L2
movb 2(%rsi),%al
movb %al,2(%rdi)
testb %al,%al
jz L2
movb 3(%rsi),%al
movb %al,3(%rdi)
testb %al,%al
jz L2
movb 4(%rsi),%al
movb %al,4(%rdi)
testb %al,%al
jz L2
movb 5(%rsi),%al
movb %al,5(%rdi)
testb %al,%al
jz L2
movb 6(%rsi),%al
movb %al,6(%rdi)
testb %al,%al
jz L2
movb 7(%rsi),%al
movb %al,7(%rdi)
addq $8,%rsi
addq $8,%rdi
testb %al,%al
jnz L1
L2: movq %r11,%rax
1: cmpb $0,-5(%rdi) /* 4th byte == 0? */
jne 1f
subq $5,%rdi
jmp .Lcopy
1: cmpb $0,-4(%rdi) /* 5th byte == 0? */
jne 1f
subq $4,%rdi
jmp .Lcopy
1: cmpb $0,-3(%rdi) /* 6th byte == 0? */
jne 1f
subq $3,%rdi
jmp .Lcopy
1: cmpb $0,-2(%rdi) /* 7th byte == 0? */
jne 1f
subq $2,%rdi
jmp .Lcopy
1: cmpb $0,-1(%rdi) /* 8th byte == 0? */
jne .Lscan_loop
subq $1,%rdi
/*
* Align source to a word boundary.
* Consider unrolling loop?
*/
.Lcopy:
.Lcopy_align:
testb $3,%sil
je .Lcopy_aligned
movb (%rsi),%dl
incq %rsi
movb %dl,(%rdi)
incq %rdi
testb %dl,%dl
jne .Lcopy_align
ret
_ALIGN_TEXT
.Lcopy_loop:
movq %rdx,(%rdi)
addq $8,%rdi
.Lcopy_aligned:
movq (%rsi),%rdx
movq %rdx,%rcx
addq $8,%rsi
subq %r8,%rcx
testq %r9,%rcx
je .Lcopy_loop
/*
* In rare cases, the above loop may exit prematurely. We must
* return to the loop if none of the bytes in the word equal 0.
*/
movb %dl,(%rdi)
incq %rdi
testb %dl,%dl /* 1st byte == 0? */
je .Ldone
shrq $8,%rdx
movb %dl,(%rdi)
incq %rdi
testb %dl,%dl /* 2nd byte == 0? */
je .Ldone
shrq $8,%rdx
movb %dl,(%rdi)
incq %rdi
testb %dl,%dl /* 3rd byte == 0? */
je .Ldone
shrq $8,%rdx
movb %dl,(%rdi)
incq %rdi
testb %dl,%dl /* 4th byte == 0? */
je .Ldone
shrq $8,%rdx
movb %dl,(%rdi)
incq %rdi
testb %dl,%dl /* 5th byte == 0? */
je .Ldone
shrq $8,%rdx
movb %dl,(%rdi)
incq %rdi
testb %dl,%dl /* 6th byte == 0? */
je .Ldone
shrq $8,%rdx
movb %dl,(%rdi)
incq %rdi
testb %dl,%dl /* 7th byte == 0? */
je .Ldone
shrq $8,%rdx
movb %dl,(%rdi)
incq %rdi
testb %dl,%dl /* 8th byte == 0? */
jne .Lcopy_aligned
.Ldone:
ret

138
lib/libc/arch/x86_64/string/strchr.S
View File

@ -1,4 +1,136 @@
/* $NetBSD: strchr.S,v 1.1 2001/06/19 00:25:05 fvdl Exp $ */
/*
* Written by J.T. Conklin <jtc@acorntoolworks.com>
* Public domain.
*/
#define STRCHR
#include "index.S"
#include <machine/asm.h>
#if defined(LIBC_SCCS)
RCSID("$NetBSD: strchr.S,v 1.2 2004/07/19 20:04:41 drochner Exp $")
#endif
#ifdef INDEX
ENTRY(index)
#else
ENTRY(strchr)
#endif
movzbq %sil,%rcx
/*
* Align to word boundary.
* Consider unrolling loop?
*/
.Lalign:
testb $7,%dil
je .Lword_aligned
movb (%rdi),%dl
cmpb %cl,%dl
je .Ldone
incq %rdi
testb %dl,%dl
jne .Lalign
jmp .Lzero
.Lword_aligned:
/* copy char to all bytes in word */
movb %cl,%ch
movq %rcx,%rdx
salq $16,%rcx
orq %rdx,%rcx
movq %rcx,%rdx
salq $32,%rcx
orq %rdx,%rcx
movabsq $0x0101010101010101,%r8
movabsq $0x8080808080808080,%r9
/* Check whether any byte in the word is equal to ch or 0. */
_ALIGN_TEXT
.Lloop:
movq (%rdi),%rdx
addq $8,%rdi
movq %rdx,%rsi
subq %r8,%rdx
xorq %rcx,%rsi
subq %r8,%rsi
orq %rsi,%rdx
testq %r9,%rdx
je .Lloop
/*
* In rare cases, the above loop may exit prematurely. We must
* return to the loop if none of the bytes in the word match
* ch or are equal to 0.
*/
movb -8(%rdi),%dl
cmpb %cl,%dl /* 1st byte == ch? */
jne 1f
subq $8,%rdi
jmp .Ldone
1: testb %dl,%dl /* 1st byte == 0? */
je .Lzero
movb -7(%rdi),%dl
cmpb %cl,%dl /* 2nd byte == ch? */
jne 1f
subq $7,%rdi
jmp .Ldone
1: testb %dl,%dl /* 2nd byte == 0? */
je .Lzero
movb -6(%rdi),%dl
cmpb %cl,%dl /* 3rd byte == ch? */
jne 1f
subq $6,%rdi
jmp .Ldone
1: testb %dl,%dl /* 3rd byte == 0? */
je .Lzero
movb -5(%rdi),%dl
cmpb %cl,%dl /* 4th byte == ch? */
jne 1f
subq $5,%rdi
jmp .Ldone
1: testb %dl,%dl /* 4th byte == 0? */
je .Lzero
movb -4(%rdi),%dl
cmpb %cl,%dl /* 5th byte == ch? */
jne 1f
subq $4,%rdi
jmp .Ldone
1: testb %dl,%dl /* 5th byte == 0? */
je .Lzero
movb -3(%rdi),%dl
cmpb %cl,%dl /* 6th byte == ch? */
jne 1f
subq $3,%rdi
jmp .Ldone
1: testb %dl,%dl /* 6th byte == 0? */
je .Lzero
movb -2(%rdi),%dl
cmpb %cl,%dl /* 7th byte == ch? */
jne 1f
subq $2,%rdi
jmp .Ldone
1: testb %dl,%dl /* 7th byte == 0? */
je .Lzero
movb -1(%rdi),%dl
cmpb %cl,%dl /* 8th byte == ch? */
jne 1f
subq $1,%rdi
jmp .Ldone
1: testb %dl,%dl /* 8th byte == 0? */
jne .Lloop
.Lzero:
/* If a ch wasn't found, return 0. */
xorq %rdi,%rdi
.Ldone:
movq %rdi,%rax
ret

120
lib/libc/arch/x86_64/string/strcmp.S
View File

@ -1,88 +1,72 @@
/*
* Written by J.T. Conklin <jtc@NetBSD.org>.
* Written by J.T. Conklin <jtc@acorntoolworks.com>
* Public domain.
* Adapted for NetBSD/x86_64 by Frank van der Linden <fvdl@wasabisystems.com>
*/
#include <machine/asm.h>
#if defined(LIBC_SCCS)
RCSID("$NetBSD: strcmp.S,v 1.2 2003/07/26 19:24:40 salo Exp $")
RCSID("$NetBSD: strcmp.S,v 1.3 2004/07/19 20:04:41 drochner Exp $")
#endif
/*
* NOTE: I've unrolled the loop eight times: large enough to make a
* significant difference, and small enough not to totally trash the
* cache.
*/
ENTRY(strcmp)
jmp L2 /* Jump into the loop. */
L1: incq %rdi
incq %rsi
L2: movb (%rdi),%cl
testb %cl,%cl /* null terminator */
jz L3
cmpb %cl,(%rsi) /* chars match */
jne L3
/*
* Align s1 to word boundary.
* Consider unrolling loop?
*/
.Ls1align:
testb $7,%dil
je .Ls1aligned
movb (%rdi),%al
incq %rdi
movb (%rsi),%dl
incq %rsi
movb (%rdi),%cl
testb %cl,%cl
jz L3
cmpb %cl,(%rsi)
jne L3
testb %al,%al
je .Ldone
cmpb %al,%dl
je .Ls1align
jmp .Ldone
incq %rdi
incq %rsi
movb (%rdi),%cl
testb %cl,%cl
jz L3
cmpb %cl,(%rsi)
jne L3
/*
* Check whether s2 is aligned to a word boundry. If it is, we
* can compare by words. Otherwise we have to compare by bytes.
*/
.Ls1aligned:
testb $7,%sil
jne .Lbyte_loop
incq %rdi
incq %rsi
movb (%rdi),%cl
testb %cl,%cl
jz L3
cmpb %cl,(%rsi)
jne L3
movabsq $0x0101010101010101,%r8
subq $8,%rdi
movabsq $0x8080808080808080,%r9
subq $8,%rsi
incq %rdi
incq %rsi
movb (%rdi),%cl
testb %cl,%cl
jz L3
cmpb %cl,(%rsi)
jne L3
_ALIGN_TEXT
.Lword_loop:
movq 8(%rdi),%rax
addq $8,%rdi
movq 8(%rsi),%rdx
addq $8,%rsi
cmpq %rax,%rdx
jne .Lbyte_loop
subq %r8,%rdx
notq %rax
andq %rax,%rdx
testq %r9,%rdx
je .Lword_loop
_ALIGN_TEXT
.Lbyte_loop:
movb (%rdi),%al
incq %rdi
movb (%rsi),%dl
incq %rsi
movb (%rdi),%cl
testb %cl,%cl
jz L3
cmpb %cl,(%rsi)
jne L3
testb %al,%al
je .Ldone
cmpb %al,%dl
je .Lbyte_loop
incq %rdi
incq %rsi
movb (%rdi),%cl
testb %cl,%cl
jz L3
cmpb %cl,(%rsi)
jne L3
incq %rdi
incq %rsi
movb (%rdi),%cl
testb %cl,%cl
jz L3
cmpb %cl,(%rsi)
je L1
L3: movzbl (%rdi),%eax /* unsigned comparison */
movzbl (%rsi),%edx
subl %edx,%eax
.Ldone:
movzbq %al,%rax
movzbq %dl,%rdx
subq %rdx,%rax
ret

138
lib/libc/arch/x86_64/string/strcpy.S
View File

@ -1,57 +1,111 @@
/*
* Written by J.T. Conklin <jtc@NetBSD.org>.
* Written by J.T. Conklin <jtc@acorntoolworks.com>
* Public domain.
* Adapted for NetBSD/x86_64 by Frank van der Linden <fvdl@wasabisystems.com>
*/
#include <machine/asm.h>
#if defined(LIBC_SCCS)
RCSID("$NetBSD: strcpy.S,v 1.2 2003/07/26 19:24:40 salo Exp $")
RCSID("$NetBSD: strcpy.S,v 1.3 2004/07/19 20:04:41 drochner Exp $")
#endif
/*
* NOTE: I've unrolled the loop eight times: large enough to make a
* significant difference, and small enough not to totally trash the
* cache.
* This strcpy implementation copies a byte at a time until the
* source pointer is aligned to a word boundary, it then copies by
* words until it finds a word containing a zero byte, and finally
* copies by bytes until the end of the string is reached.
*
* While this may result in unaligned stores if the source and
* destination pointers are unaligned with respect to each other,
* it is still faster than either byte copies or the overhead of
* an implementation suitable for machines with strict alignment
* requirements.
*/
ENTRY(strcpy)
movq %rdi,%r11
movq %rdi,%rax
movabsq $0x0101010101010101,%r8
movabsq $0x8080808080808080,%r9
L1: movb (%rsi),%al /* unroll loop, but not too much */
movb %al,(%rdi)
testb %al,%al
jz L2
movb 1(%rsi),%al
movb %al,1(%rdi)
testb %al,%al
jz L2
movb 2(%rsi),%al
movb %al,2(%rdi)
testb %al,%al
jz L2
movb 3(%rsi),%al
movb %al,3(%rdi)
testb %al,%al
jz L2
movb 4(%rsi),%al
movb %al,4(%rdi)
testb %al,%al
jz L2
movb 5(%rsi),%al
movb %al,5(%rdi)
testb %al,%al
jz L2
movb 6(%rsi),%al
movb %al,6(%rdi)
testb %al,%al
jz L2
movb 7(%rsi),%al
movb %al,7(%rdi)
addq $8,%rsi
addq $8,%rdi
testb %al,%al
jnz L1
L2: movq %r11,%rax
/*
* Align source to a word boundary.
* Consider unrolling loop?
*/
_ALIGN_TEXT
.Lalign:
testb $7,%sil
je .Lword_aligned
movb (%rsi),%dl
incq %rsi
movb %dl,(%rdi)
incq %rdi
testb %dl,%dl
jne .Lalign
ret
_ALIGN_TEXT
.Lloop:
movq %rdx,(%rdi)
addq $8,%rdi
.Lword_aligned:
movq (%rsi),%rdx
movq %rdx,%rcx
addq $8,%rsi
subq %r8,%rcx
testq %r9,%rcx
je .Lloop
/*
* In rare cases, the above loop may exit prematurely. We must
* return to the loop if none of the bytes in the word equal 0.
*/
movb %dl,(%rdi)
incq %rdi
testb %dl,%dl /* 1st byte == 0? */
je .Ldone
shrq $8,%rdx
movb %dl,(%rdi)
incq %rdi
testb %dl,%dl /* 2nd byte == 0? */
je .Ldone
shrq $8,%rdx
movb %dl,(%rdi)
incq %rdi
testb %dl,%dl /* 3rd byte == 0? */
je .Ldone
shrq $8,%rdx
movb %dl,(%rdi)
incq %rdi
testb %dl,%dl /* 4th byte == 0? */
je .Ldone
shrq $8,%rdx
movb %dl,(%rdi)
incq %rdi
testb %dl,%dl /* 5th byte == 0? */
je .Ldone
shrq $8,%rdx
movb %dl,(%rdi)
incq %rdi
testb %dl,%dl /* 6th byte == 0? */
je .Ldone
shrq $8,%rdx
movb %dl,(%rdi)
incq %rdi
testb %dl,%dl /* 7th byte == 0? */
je .Ldone
shrq $8,%rdx
movb %dl,(%rdi)
incq %rdi
testb %dl,%dl /* 8th byte == 0? */
jne .Lword_aligned
.Ldone:
ret

155
lib/libc/arch/x86_64/string/strlen.S
View File

@ -1,21 +1,156 @@
/*
* Written by J.T. Conklin <jtc@NetBSD.org>.
* Written by J.T. Conklin <jtc@acorntoolworks.com>
* Public domain.
* Adapted for NetBSD/x86_64 by Frank van der Linden <fvdl@wasabisystems.com>
*/
#include <machine/asm.h>
#if defined(LIBC_SCCS)
RCSID("$NetBSD: strlen.S,v 1.2 2003/07/26 19:24:40 salo Exp $")
RCSID("$NetBSD: strlen.S,v 1.3 2004/07/19 20:04:41 drochner Exp $")
#endif
ENTRY(strlen)
cld /* set search forward */
xorl %eax,%eax /* set search for null terminator */
movq $-1,%rcx /* set search for lots of characters */
repne /* search! */
scasb
notq %rcx /* get length by taking complement */
leaq -1(%rcx),%rax /* and subtracting one */
movq %rdi,%rax
negq %rdi
.Lalign:
/* Consider unrolling loop? */
testb $7,%al
je .Lword_aligned
cmpb $0,(%rax)
jne 1f
leaq (%rdi,%rax),%rax
ret
1: incq %rax
jmp .Lalign
/*
* There are many well known branch-free sequences which are used
* for determining whether a zero-byte is contained within a word.
* These sequences are generally much more efficent than loading
* and comparing each byte individually.
*
* The expression [1,2]:
*
* (1) ~(((x & 0x7f....7f) + 0x7f....7f) | (x | 0x7f....7f))
*
* evaluates to a non-zero value if any of the bytes in the
* original word is zero.
*
* It also has the useful property that bytes in the result word
* that coorespond to non-zero bytes in the original word have
* the value 0x00, while bytes cooresponding to zero bytes have
* the value 0x80. This allows calculation of the first (and
* last) occurance of a zero byte within the word (useful for C's
* str* primitives) by counting the number of leading (or
* trailing) zeros and dividing the result by 8. On machines
* without (or with slow) clz() / ctz() instructions, testing
* each byte in the result word for zero is necessary.
*
* This typically takes 4 instructions (5 on machines without
* "not-or") not including those needed to load the constant.
*
*
* The expression:
*
* (2) ((x - 0x01....01) & ~x & 0x80....80)
*
* evaluates to a non-zero value if any of the bytes in the
* original word is zero.
*
* On little endian machines, the first byte in the result word
* that cooresponds to a zero byte in the original byte is 0x80,
* so clz() can be used as above. On big endian machines, and
* little endian machines without (or with a slow) clz() insn,
* testing each byte in the original for zero is necessary
*
* This typically takes 3 instructions (4 on machines without
* "and with complement") not including those needed to load
* constants.
*
*
* The expression:
*
* (3) ((x - 0x01....01) & 0x80....80)
*
* always evaluates to a non-zero value if any of the bytes in
* the original word is zero. However, in rare cases, it also
* evaluates to a non-zero value when none of the bytes in the
* original word is zero.
*
* To account for possible false positives, each byte of the
* original word must be checked when the expression evaluates to
* a non-zero value. However, because it is simpler than those
* presented above, code that uses it will be faster as long as
* the rate of false positives is low.
*
* This is likely, because the the false positive can only occur
* if the most siginificant bit of a byte within the word is set.
* The expression will never fail for typical 7-bit ASCII strings.
*
* This typically takes 2 instructions not including those needed
* to load constants.
*
*
* [1] Henry S. Warren Jr., "Hacker's Delight", Addison-Westley 2003
*
* [2] International Business Machines, "The PowerPC Compiler Writer's
* Guide", Warthman Associates, 1996
*/
_ALIGN_TEXT
.Lword_aligned:
movabsq $0x0101010101010101,%r8
movabsq $0x8080808080808080,%r9
.Lloop:
movq (%rax),%rdx
addq $8,%rax
subq %r8,%rdx
testq %r9,%rdx
je .Lloop
/*
* In rare cases, the above loop may exit prematurely. We must
* return to the loop if none of the bytes in the word equal 0.
*/
cmpb $0,-8(%rax) /* 1st byte == 0? */
je .Lsub8
cmpb $0,-7(%rax) /* 2nd byte == 0? */
je .Lsub7
cmpb $0,-6(%rax) /* 3rd byte == 0? */
je .Lsub6
cmpb $0,-5(%rax) /* 4th byte == 0? */
je .Lsub5
cmpb $0,-4(%rax) /* 5th byte == 0? */
je .Lsub4
cmpb $0,-3(%rax) /* 6th byte == 0? */
je .Lsub3
cmpb $0,-2(%rax) /* 7th byte == 0? */
je .Lsub2
cmpb $0,-1(%rax) /* 8th byte == 0? */
jne .Lloop
.Lsub1:
leaq -1(%rdi,%rax),%rax
ret
.Lsub2:
leaq -2(%rdi,%rax),%rax
ret
.Lsub3:
leaq -3(%rdi,%rax),%rax
ret
.Lsub4:
leaq -4(%rdi,%rax),%rax
ret
.Lsub5:
leaq -5(%rdi,%rax),%rax
ret
.Lsub6:
leaq -6(%rdi,%rax),%rax
ret
.Lsub7:
leaq -7(%rdi,%rax),%rax
ret
.Lsub8:
leaq -8(%rdi,%rax),%rax
ret

128
lib/libc/arch/x86_64/string/strrchr.S
View File

@ -1,4 +1,126 @@
/* $NetBSD: strrchr.S,v 1.1 2001/06/19 00:25:05 fvdl Exp $ */
/*
* Written by J.T. Conklin <jtc@acorntoolworks.com>
* Public domain.
*/
#define STRRCHR
#include "rindex.S"
#include <machine/asm.h>
#if defined(LIBC_SCCS)
RCSID("$NetBSD: strrchr.S,v 1.2 2004/07/19 20:04:41 drochner Exp $")
#endif
#ifdef RINDEX
ENTRY(rindex)
#else
ENTRY(strrchr)
#endif
movzbq %sil,%rcx
/* zero return value */
xorq %rax,%rax
/*
* Align to word boundary.
* Consider unrolling loop?
*/
.Lalign:
testb $7,%dil
je .Lword_aligned
movb (%rdi),%dl
cmpb %cl,%dl
cmoveq %rdi,%rax
incq %rdi
testb %dl,%dl
jne .Lalign
jmp .Ldone
.Lword_aligned:
/* copy char to all bytes in word */
movb %cl,%ch
movq %rcx,%rdx
salq $16,%rcx
orq %rdx,%rcx
movq %rcx,%rdx
salq $32,%rcx
orq %rdx,%rcx
movabsq $0x0101010101010101,%r8
movabsq $0x8080808080808080,%r9
/* Check whether any byte in the word is equal to ch or 0. */
_ALIGN_TEXT
.Lloop:
movq (%rdi),%rdx
addq $8,%rdi
movq %rdx,%rsi
subq %r8,%rdx
xorq %rcx,%rsi
subq %r8,%rsi
orq %rsi,%rdx
testq %r9,%rdx
je .Lloop
/*
* In rare cases, the above loop may exit prematurely. We must
* return to the loop if none of the bytes in the word match
* ch or are equal to 0.
*/
movb -8(%rdi),%dl
cmpb %cl,%dl /* 1st byte == ch? */
jne 1f
leaq -8(%rdi),%rax
1: testb %dl,%dl /* 1st byte == 0? */
je .Ldone
movb -7(%rdi),%dl
cmpb %cl,%dl /* 2nd byte == ch? */
jne 1f
leaq -7(%rdi),%rax
1: testb %dl,%dl /* 2nd byte == 0? */
je .Ldone
movb -6(%rdi),%dl
cmpb %cl,%dl /* 3rd byte == ch? */
jne 1f
leaq -6(%rdi),%rax
1: testb %dl,%dl /* 3rd byte == 0? */
je .Ldone
movb -5(%rdi),%dl
cmpb %cl,%dl /* 4th byte == ch? */
jne 1f
leaq -5(%rdi),%rax
1: testb %dl,%dl /* 4th byte == 0? */
je .Ldone
movb -4(%rdi),%dl
cmpb %cl,%dl /* 5th byte == ch? */
jne 1f
leaq -4(%rdi),%rax
1: testb %dl,%dl /* 5th byte == 0? */
je .Ldone
movb -3(%rdi),%dl
cmpb %cl,%dl /* 6th byte == ch? */
jne 1f
leaq -3(%rdi),%rax
1: testb %dl,%dl /* 6th byte == 0? */
je .Ldone
movb -2(%rdi),%dl
cmpb %cl,%dl /* 7th byte == ch? */
jne 1f
leaq -2(%rdi),%rax
1: testb %dl,%dl /* 7th byte == 0? */
je .Ldone
movb -1(%rdi),%dl
cmpb %cl,%dl /* 8th byte == ch? */
jne 1f
leaq -1(%rdi),%rax
1: testb %dl,%dl /* 8th byte == 0? */
jne .Lloop
.Ldone:
ret