benchmark: do not clobber callee-saved NEON registers

Make the ARM assembly code save and restore registers q4-q7 when they are used, since these are callee-saved registers. This is not needed in the Linux kernel, or in other contexts where FPU and NEON code generation is disabled. IIRC, I had used -mfloat-abi=soft when testing this code originally. But that option is not actually being set by the build system, and building with that option is not always possible anyway since some systems don't have the softfp library. I'm now seeing a test failure due to this, so let's fix it.
google · Dec 25, 2022 · 0874175 · 0874175
1 parent 818709d
commit 0874175
Show file tree

Hide file tree

Showing 11 changed files with 70 additions and 9 deletions.
diff --git a/benchmark/src/arm/cham-neon-core.S b/benchmark/src/arm/cham-neon-core.S
@@ -200,7 +200,6 @@
 
 .macro _cham_xts_crypt	decrypting
 	push		{r4-r7}
-	mov		r7, sp
 
 	/*
 	 * The first four parameters were passed in registers r0-r3.  Load the
@@ -209,6 +208,11 @@
 	ldr		NBYTES, [sp, #16]
 	ldr		TWEAK, [sp, #20]
 
+	// Save the callee-saved NEON registers.
+	vstmdb		sp!, {q4-q7}
+
+	mov		r7, sp
+
 	// Load the round keys
 	vld1.8		{ROUND_KEY_A, ROUND_KEY_B}, [ROUND_KEYS]
 
@@ -343,6 +347,7 @@
 	vst1.8		{TWEAKV}, [TWEAK]
 
 	mov		sp, r7
+	vldmia		sp!, {q4-q7}
 	pop		{r4-r7}
 	bx		lr
 .endm

diff --git a/benchmark/src/arm/chaskey-lts-neon-core.S b/benchmark/src/arm/chaskey-lts-neon-core.S
@@ -215,14 +215,18 @@
 
 .macro _chaskey_lts_xts_crypt	decrypting
 	push		{r4-r5}
-	mov		r5, sp
 
 	/*
 	 * The first four parameters were passed in registers r0-r3.  Load the
 	 * additional parameter, which was passed on the stack.
 	 */
 	ldr		TWEAK, [sp, #8]
 
+	// Save the callee-saved NEON registers.
+	vstmdb		sp!, {q4-q7}
+
+	mov		r5, sp
+
 	// Load key
 	vld1.8		{KEYV}, [KEY]
 
@@ -358,6 +362,7 @@
 	vst1.8		{TWEAKV}, [TWEAK]
 
 	mov		sp, r5
+	vldmia		sp!, {q4-q7}
 	pop		{r4-r5}
 	bx		lr
 .endm

diff --git a/benchmark/src/arm/lea-neon-core.S b/benchmark/src/arm/lea-neon-core.S
@@ -293,7 +293,6 @@
 
 .macro _lea_xts_crypt	is_lea128, decrypting
 	push		{r4-r7}
-	mov		r7, sp
 
 	/*
 	 * The first four parameters were passed in registers r0-r3.  Load the
@@ -302,6 +301,11 @@
 	ldr		NBYTES, [sp, #16]
 	ldr		TWEAK, [sp, #20]
 
+	// Save the callee-saved NEON registers.
+	vstmdb		sp!, {q4-q7}
+
+	mov		r7, sp
+
 	/*
 	 * Allocate stack space to store 128 bytes worth of tweaks.  For
 	 * performance, this space is aligned to a 16-byte boundary so that we
@@ -419,6 +423,7 @@
 	bne		.Lnext_128bytes_\@
 
 	mov		sp, r7
+	vldmia		sp!, {q4-q7}
 	pop		{r4-r7}
 	bx		lr
 .endm

diff --git a/benchmark/src/arm/nh-neon-core.S b/benchmark/src/arm/nh-neon-core.S
@@ -78,6 +78,9 @@
  */
 ENTRY(nh_neon)
 
+	// Save the callee-saved NEON registers.
+	vstmdb		sp!, {q4-q7}
+
 	vld1.32		{K0,K1}, [KEY]!
 	  vmov.u64	PASS0_SUMS, #0
 	  vmov.u64	PASS1_SUMS, #0
@@ -115,5 +118,6 @@ ENTRY(nh_neon)
 	vadd.u64	T1_L, PASS2_SUM_A, PASS2_SUM_B
 	vadd.u64	T1_H, PASS3_SUM_A, PASS3_SUM_B
 	vst1.8		{T0-T1}, [HASH]
+	vldmia		sp!, {q4-q7}
 	bx		lr
 ENDPROC(nh_neon)
diff --git a/benchmark/src/arm/noekeon-neon-core.S b/benchmark/src/arm/noekeon-neon-core.S
@@ -269,14 +269,18 @@
 
 .macro _noekeon_xts_crypt	decrypting
 	push		{r4-r5}
-	mov		r5, sp
 
 	/*
 	 * The first four parameters were passed in registers r0-r3.  Load the
 	 * additional parameter, which was passed on the stack.
 	 */
 	ldr		TWEAK, [sp, #8]
 
+	// Save the callee-saved NEON registers.
+	vstmdb		sp!, {q4-q7}
+
+	mov		r5, sp
+
 	// Load the key
 	vld1.32		{K}, [KEY]
 
@@ -456,6 +460,7 @@
 	bne		.Lnext_128bytes_\@
 
 	mov		sp, r5
+	vldmia		sp!, {q4-q7}
 	pop		{r4-r5}
 	bx		lr
 .endm

diff --git a/benchmark/src/arm/rc5-neon-core.S b/benchmark/src/arm/rc5-neon-core.S
@@ -262,7 +262,6 @@
  */
 .macro _rc5_xts_crypt	n, decrypting
 	push		{r4-r9}
-	mov		r9, sp
 
 	/*
 	 * The first four parameters were passed in registers r0-r3.  Load the
@@ -271,6 +270,11 @@
 	ldr		NBYTES, [sp, #24]
 	ldr		TWEAK, [sp, #28]
 
+	// Save the callee-saved NEON registers.
+	vstmdb		sp!, {q4-q7}
+
+	mov		r9, sp
+
 	mov		CONST_N_MINUS_1, #(\n - 1)
 	mov		CONST_N, #\n
 
@@ -460,6 +464,7 @@
 	bne		.Lnext_128bytes_\@
 
 	mov		sp, r9
+	vldmia		sp!, {q4-q7}
 	pop		{r4-r9}
 	bx		lr
 .endm

diff --git a/benchmark/src/arm/rc6-neon-core.S b/benchmark/src/arm/rc6-neon-core.S
@@ -229,14 +229,18 @@
 
 .macro _rc6_xts_crypt	decrypting
 	push		{r4-r9}
-	mov		r9, sp
 
 	/*
 	 * The first four parameters were passed in registers r0-r3.  Load the
 	 * additional parameter, which was passed on the stack.
 	 */
 	ldr		TWEAK, [sp, #24]
 
+	// Save the callee-saved NEON registers.
+	vstmdb		sp!, {q4-q7}
+
+	mov		r9, sp
+
 	mov		ONE, #1
 	mov		THIRTY_ONE, #31
 	mov		THIRTY_TWO, #32
@@ -403,6 +407,7 @@
 	bne		.Lnext_128bytes_\@
 
 	mov		sp, r9
+	vldmia		sp!, {q4-q7}
 	pop		{r4-r9}
 	bx		lr
 .endm

diff --git a/benchmark/src/arm/speck-neon-core.S b/benchmark/src/arm/speck-neon-core.S
@@ -221,7 +221,6 @@
  */
 .macro _speck_xts_crypt	n, decrypting
 	push		{r4-r7}
-	mov		r7, sp
 
 	/*
 	 * The first four parameters were passed in registers r0-r3.  Load the
@@ -230,6 +229,11 @@
 	ldr		NBYTES, [sp, #16]
 	ldr		TWEAK, [sp, #20]
 
+	// Save the callee-saved NEON registers.
+	vstmdb		sp!, {q4-q7}
+
+	mov		r7, sp
+
 	/*
 	 * If decrypting, modify the ROUND_KEYS parameter to point to the last
 	 * round key rather than the first, since for decryption the round keys
@@ -430,6 +434,7 @@
 .endif
 
 	mov		sp, r7
+	vldmia		sp!, {q4-q7}
 	pop		{r4-r7}
 	bx		lr
 .endm

diff --git a/benchmark/src/arm/xtea-neon-core.S b/benchmark/src/arm/xtea-neon-core.S
@@ -138,14 +138,18 @@
 
 .macro _xtea_xts_crypt	decrypting
 	push		{r4-r5}
-	mov		r5, sp
 
 	/*
 	 * The first four parameters were passed in registers r0-r3.  Load the
 	 * additional parameter, which was passed on the stack.
 	 */
 	ldr		TWEAK, [sp, #8]
 
+	// Save the callee-saved NEON registers.
+	vstmdb		sp!, {q4-q7}
+
+	mov		r5, sp
+
 	// Load the key
 	vld1.8		{KEYV}, [KEY]
 
@@ -324,6 +328,7 @@
 	bne		.Lnext_128bytes_\@
 
 	mov		sp, r5
+	vldmia		sp!, {q4-q7}
 	pop		{r4-r5}
 	bx		lr
 .endm

diff --git a/third_party/linux-kernel/arm/aes-neonbs-core.S b/third_party/linux-kernel/arm/aes-neonbs-core.S
@@ -442,6 +442,7 @@ M0:	.quad		0x02060a0e03070b0f, 0x0004080c0105090d
 	 * void aesbs_convert_key(u8 out[], u32 const rk[], int rounds)
 	 */
 ENTRY(aesbs_convert_key)
+	vstmdb		sp!, {q4-q7}		// Save callee-saved NEON registers
 	vld1.32		{q7}, [r1]!		// load round 0 key
 	vld1.32		{q15}, [r1]!		// load round 1 key
 
@@ -485,6 +486,7 @@ ENTRY(aesbs_convert_key)
 	vmov.i8		q7, #0x63		// compose .L63
 	veor		q15, q15, q7
 	vst1.8		{q15}, [r0, :128]
+	vldmia		sp!, {q4-q7}
 	bx		lr
 ENDPROC(aesbs_convert_key)
 
@@ -960,8 +962,9 @@ ENDPROC(__xts_prepare8)
 
 	.macro		__xts_crypt, do8, o0, o1, o2, o3, o4, o5, o6, o7
 	push		{r4-r8, lr}
-	mov		r5, sp			// preserve sp
 	ldrd		r6, r7, [sp, #24]	// get blocks and iv args
+	vstmdb		sp!, {q4-q7}		// Save callee-saved NEON registers
+	mov		r5, sp			// preserve sp
 	sub		ip, sp, #128		// make room for 8x tweak
 	bic		ip, ip, #0xf		// align sp to 16 bytes
 	mov		sp, ip
@@ -1013,6 +1016,7 @@ ENDPROC(__xts_prepare8)
 	bgt		99b
 
 	mov		sp, r5
+	vldmia		sp!, {q4-q7}	// Restore callee-saved NEON registers
 	pop		{r4-r8, pc}
 	.endm
 

diff --git a/third_party/linux-kernel/arm/chacha-neon-core.S b/third_party/linux-kernel/arm/chacha-neon-core.S
@@ -546,6 +546,9 @@ ENTRY(chacha_block_xor_neon)
 	// r2: 1 data block input, i
 	// r3: nrounds
 
+	// Save the callee-saved NEON registers.
+	vstmdb		sp!, {q4-q7}
+
 	// x0..3 = s0..3
 	add		ip, r0, #0x20
 	vld1.32		{q0-q1}, [r0]
@@ -582,6 +585,7 @@ ENTRY(chacha_block_xor_neon)
 	vst1.8		{q0-q1}, [r1]
 	vst1.8		{q2-q3}, [ip]
 
+	vldmia		sp!, {q4-q7}
 	bx		lr
 ENDPROC(chacha_block_xor_neon)
 
@@ -591,6 +595,7 @@ ENDPROC(chacha_block_xor_neon)
 
 	.align		5
 ENTRY(chacha_4block_xor_neon)
+	vstmdb		sp!, {q4-q7}		// Save callee-saved NEON registers
 	push		{r4-r5}
 	mov		r4, sp			// preserve the stack pointer
 	sub		ip, sp, #0x20		// allocate a 32 byte buffer
@@ -746,10 +751,14 @@ ENTRY(chacha_4block_xor_neon)
 	vst1.8		{q0-q1}, [r1]
 
 	pop		{r4-r5}
+	vldmia		sp!, {q4-q7}
 	bx		lr
 ENDPROC(chacha_4block_xor_neon)
 
 ENTRY(chacha_perm_neon)
+	// Save the callee-saved NEON registers that are used.
+	vstmdb		sp!, {q4-q5}
+
 	// r0: Input state matrix, s
 	// r1: nrounds
 	add		ip, r0, #0x20
@@ -762,6 +771,7 @@ ENTRY(chacha_perm_neon)
 	vst1.8		{q0-q1}, [r0]!
 	vst1.8		{q2-q3}, [r0]
 
+	vldmia		sp!, {q4-q5}
 	bx		lr
 ENDPROC(chacha_perm_neon)
 
@@ -782,6 +792,8 @@ ENDPROC(chacha_perm_neon)
  * r3 = nrounds
  */
 .macro _chacha_mem_crypt_4block_neon	decrypting
+	// Save the callee-saved NEON registers.
+	vstmdb		sp!, {q4-q7}
 
 	// 32-byte align the stack
 	push		{r4}
@@ -991,6 +1003,7 @@ ENDPROC(chacha_perm_neon)
 
 	mov		sp, r4
 	pop		{r4}
+	vldmia		sp!, {q4-q7}
 	bx		lr
 .endm