1 /* SPDX-License-Identifier: GPL-2.0 */ 1 /* SPDX-License-Identifier: GPL-2.0 */
2 #include <linux/export.h> 2 #include <linux/export.h>
3 #include <linux/linkage.h> 3 #include <linux/linkage.h>
4 4
5 #include <asm/asm.h> !! 5 .text
6 !! 6 .align 32
7 /* !! 7 ENTRY(__arch_hweight8)
8 * unsigned int __sw_hweight32(unsigned int w) !! 8 sethi %hi(__sw_hweight8), %g1
9 * %rdi: w !! 9 jmpl %g1 + %lo(__sw_hweight8), %g0
10 */ !! 10 nop
11 SYM_FUNC_START(__sw_hweight32) !! 11 ENDPROC(__arch_hweight8)
12 !! 12 EXPORT_SYMBOL(__arch_hweight8)
13 #ifdef CONFIG_X86_64 !! 13 .section .popc_3insn_patch, "ax"
14 movl %edi, %eax !! 14 .word __arch_hweight8
15 #endif !! 15 sllx %o0, 64-8, %g1
16 __ASM_SIZE(push,) %__ASM_REG(dx) !! 16 retl
17 movl %eax, %edx !! 17 popc %g1, %o0
18 shrl %edx !! 18 .previous
19 andl $0x55555555, %edx !! 19
20 subl %edx, %eax !! 20 ENTRY(__arch_hweight16)
21 !! 21 sethi %hi(__sw_hweight16), %g1
22 movl %eax, %edx !! 22 jmpl %g1 + %lo(__sw_hweight16), %g0
23 shrl $2, %eax !! 23 nop
24 andl $0x33333333, %edx !! 24 ENDPROC(__arch_hweight16)
25 andl $0x33333333, %eax !! 25 EXPORT_SYMBOL(__arch_hweight16)
26 addl %edx, %eax !! 26 .section .popc_3insn_patch, "ax"
27 !! 27 .word __arch_hweight16
28 movl %eax, %edx !! 28 sllx %o0, 64-16, %g1
29 shrl $4, %edx !! 29 retl
30 addl %edx, %eax !! 30 popc %g1, %o0
31 andl $0x0f0f0f0f, %eax !! 31 .previous
32 imull $0x01010101, %eax, %eax !! 32
33 shrl $24, %eax !! 33 ENTRY(__arch_hweight32)
34 __ASM_SIZE(pop,) %__ASM_REG(dx) !! 34 sethi %hi(__sw_hweight32), %g1
35 RET !! 35 jmpl %g1 + %lo(__sw_hweight32), %g0
36 SYM_FUNC_END(__sw_hweight32) !! 36 nop
37 EXPORT_SYMBOL(__sw_hweight32) !! 37 ENDPROC(__arch_hweight32)
38 !! 38 EXPORT_SYMBOL(__arch_hweight32)
39 /* !! 39 .section .popc_3insn_patch, "ax"
40 * No 32-bit variant, because it's implemented !! 40 .word __arch_hweight32
41 * on top of __arch_hweight32(): !! 41 sllx %o0, 64-32, %g1
42 */ !! 42 retl
43 #ifdef CONFIG_X86_64 !! 43 popc %g1, %o0
44 SYM_FUNC_START(__sw_hweight64) !! 44 .previous
45 pushq %rdi !! 45
46 pushq %rdx !! 46 ENTRY(__arch_hweight64)
47 !! 47 sethi %hi(__sw_hweight64), %g1
48 movq %rdi, %rdx !! 48 jmpl %g1 + %lo(__sw_hweight64), %g0
49 movabsq $0x5555555555555555, %rax !! 49 nop
50 shrq %rdx !! 50 ENDPROC(__arch_hweight64)
51 andq %rdx, %rax !! 51 EXPORT_SYMBOL(__arch_hweight64)
52 movabsq $0x3333333333333333, %rdx !! 52 .section .popc_3insn_patch, "ax"
53 subq %rax, %rdi !! 53 .word __arch_hweight64
54 !! 54 retl
55 movq %rdi, %rax !! 55 popc %o0, %o0
56 shrq $2, %rdi !! 56 nop
57 andq %rdx, %rax !! 57 .previous
58 andq %rdi, %rdx <<
59 addq %rdx, %rax <<
60 <<
61 movq %rax, %rdx <<
62 shrq $4, %rdx <<
63 addq %rdx, %rax <<
64 movabsq $0x0f0f0f0f0f0f0f0f, %rdx <<
65 andq %rdx, %rax <<
66 movabsq $0x0101010101010101, %rdx <<
67 imulq %rdx, %rax <<
68 shrq $56, %rax <<
69 <<
70 popq %rdx <<
71 popq %rdi <<
72 RET <<
73 SYM_FUNC_END(__sw_hweight64) <<
74 EXPORT_SYMBOL(__sw_hweight64) <<
75 #endif <<
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.