| Thomas Gleixner | 457c899 | 2019-05-19 13:08:55 +0100 | [diff] [blame] | 1 | /* SPDX-License-Identifier: GPL-2.0-only */ |
| Jan Beulich | 8d379da | 2006-09-26 10:52:32 +0200 | [diff] [blame] | 2 | #include <linux/linkage.h> |
| Al Viro | 784d569 | 2016-01-11 11:04:34 -0500 | [diff] [blame] | 3 | #include <asm/export.h> |
| Jan Beulich | 8d379da | 2006-09-26 10:52:32 +0200 | [diff] [blame] | 4 | |
| Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 5 | /* |
| Borislav Petkov | 6620ef2 | 2015-01-18 12:57:41 +0100 | [diff] [blame] | 6 | * Most CPUs support enhanced REP MOVSB/STOSB instructions. It is |
| 7 | * recommended to use this when possible and we do use them by default. |
| 8 | * If enhanced REP MOVSB/STOSB is not available, try to use fast string. |
| 9 | * Otherwise, use original. |
| 10 | */ |
| 11 | |
| 12 | /* |
| 13 | * Zero a page. |
| 14 | * %rdi - page |
| 15 | */ |
| Jiri Slaby | 6dcc562 | 2019-10-11 13:51:04 +0200 | [diff] [blame] | 16 | SYM_FUNC_START(clear_page_rep) |
| Jan Beulich | 8d379da | 2006-09-26 10:52:32 +0200 | [diff] [blame] | 17 | movl $4096/8,%ecx |
| 18 | xorl %eax,%eax |
| 19 | rep stosq |
| Peter Zijlstra | f94909c | 2021-12-04 14:43:40 +0100 | [diff] [blame] | 20 | RET |
| Jiri Slaby | 6dcc562 | 2019-10-11 13:51:04 +0200 | [diff] [blame] | 21 | SYM_FUNC_END(clear_page_rep) |
| Borislav Petkov | f25d384 | 2017-02-09 01:34:49 +0100 | [diff] [blame] | 22 | EXPORT_SYMBOL_GPL(clear_page_rep) |
| Jan Beulich | 8d379da | 2006-09-26 10:52:32 +0200 | [diff] [blame] | 23 | |
| Jiri Slaby | 6dcc562 | 2019-10-11 13:51:04 +0200 | [diff] [blame] | 24 | SYM_FUNC_START(clear_page_orig) |
| Andi Kleen | 7bcd3f3 | 2006-02-03 21:51:02 +0100 | [diff] [blame] | 25 | xorl %eax,%eax |
| 26 | movl $4096/64,%ecx |
| 27 | .p2align 4 |
| 28 | .Lloop: |
| 29 | decl %ecx |
| 30 | #define PUT(x) movq %rax,x*8(%rdi) |
| 31 | movq %rax,(%rdi) |
| 32 | PUT(1) |
| 33 | PUT(2) |
| 34 | PUT(3) |
| 35 | PUT(4) |
| 36 | PUT(5) |
| 37 | PUT(6) |
| 38 | PUT(7) |
| 39 | leaq 64(%rdi),%rdi |
| 40 | jnz .Lloop |
| 41 | nop |
| Peter Zijlstra | f94909c | 2021-12-04 14:43:40 +0100 | [diff] [blame] | 42 | RET |
| Jiri Slaby | 6dcc562 | 2019-10-11 13:51:04 +0200 | [diff] [blame] | 43 | SYM_FUNC_END(clear_page_orig) |
| Borislav Petkov | f25d384 | 2017-02-09 01:34:49 +0100 | [diff] [blame] | 44 | EXPORT_SYMBOL_GPL(clear_page_orig) |
| Andi Kleen | 7bcd3f3 | 2006-02-03 21:51:02 +0100 | [diff] [blame] | 45 | |
| Jiri Slaby | 6dcc562 | 2019-10-11 13:51:04 +0200 | [diff] [blame] | 46 | SYM_FUNC_START(clear_page_erms) |
| Borislav Petkov | 6620ef2 | 2015-01-18 12:57:41 +0100 | [diff] [blame] | 47 | movl $4096,%ecx |
| 48 | xorl %eax,%eax |
| 49 | rep stosb |
| Peter Zijlstra | f94909c | 2021-12-04 14:43:40 +0100 | [diff] [blame] | 50 | RET |
| Jiri Slaby | 6dcc562 | 2019-10-11 13:51:04 +0200 | [diff] [blame] | 51 | SYM_FUNC_END(clear_page_erms) |
| Borislav Petkov | f25d384 | 2017-02-09 01:34:49 +0100 | [diff] [blame] | 52 | EXPORT_SYMBOL_GPL(clear_page_erms) |