| /* SPDX-License-Identifier: GPL-2.0-or-later */ |
| /* |
| * SM4 Cipher Algorithm for ARMv8 with Crypto Extensions |
| * as specified in |
| * https://tools.ietf.org/id/draft-ribose-cfrg-sm4-10.html |
| * |
| * Copyright (C) 2022, Alibaba Group. |
| * Copyright (C) 2022 Tianjia Zhang <tianjia.zhang@linux.alibaba.com> |
| */ |
| |
| #include <linux/linkage.h> |
| #include <asm/assembler.h> |
| #include "sm4-ce-asm.h" |
| |
| .arch armv8-a+crypto |
| |
| .irp b, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, \ |
| 20, 24, 25, 26, 27, 28, 29, 30, 31 |
| .set .Lv\b\().4s, \b |
| .endr |
| |
| .macro sm4e, vd, vn |
| .inst 0xcec08400 | (.L\vn << 5) | .L\vd |
| .endm |
| |
| .macro sm4ekey, vd, vn, vm |
| .inst 0xce60c800 | (.L\vm << 16) | (.L\vn << 5) | .L\vd |
| .endm |
| |
| /* Register macros */ |
| |
| #define RTMP0 v16 |
| #define RTMP1 v17 |
| #define RTMP2 v18 |
| #define RTMP3 v19 |
| |
| #define RIV v20 |
| #define RMAC v20 |
| #define RMASK v21 |
| |
| |
| .align 3 |
| SYM_FUNC_START(sm4_ce_expand_key) |
| /* input: |
| * x0: 128-bit key |
| * x1: rkey_enc |
| * x2: rkey_dec |
| * x3: fk array |
| * x4: ck array |
| */ |
| ld1 {v0.16b}, [x0]; |
| rev32 v0.16b, v0.16b; |
| ld1 {v1.16b}, [x3]; |
| /* load ck */ |
| ld1 {v24.16b-v27.16b}, [x4], #64; |
| ld1 {v28.16b-v31.16b}, [x4]; |
| |
| /* input ^ fk */ |
| eor v0.16b, v0.16b, v1.16b; |
| |
| sm4ekey v0.4s, v0.4s, v24.4s; |
| sm4ekey v1.4s, v0.4s, v25.4s; |
| sm4ekey v2.4s, v1.4s, v26.4s; |
| sm4ekey v3.4s, v2.4s, v27.4s; |
| sm4ekey v4.4s, v3.4s, v28.4s; |
| sm4ekey v5.4s, v4.4s, v29.4s; |
| sm4ekey v6.4s, v5.4s, v30.4s; |
| sm4ekey v7.4s, v6.4s, v31.4s; |
| |
| adr_l x5, .Lbswap128_mask |
| ld1 {v24.16b}, [x5] |
| |
| st1 {v0.16b-v3.16b}, [x1], #64; |
| st1 {v4.16b-v7.16b}, [x1]; |
| |
| tbl v16.16b, {v7.16b}, v24.16b |
| tbl v17.16b, {v6.16b}, v24.16b |
| tbl v18.16b, {v5.16b}, v24.16b |
| tbl v19.16b, {v4.16b}, v24.16b |
| tbl v20.16b, {v3.16b}, v24.16b |
| tbl v21.16b, {v2.16b}, v24.16b |
| tbl v22.16b, {v1.16b}, v24.16b |
| tbl v23.16b, {v0.16b}, v24.16b |
| |
| st1 {v16.16b-v19.16b}, [x2], #64 |
| st1 {v20.16b-v23.16b}, [x2] |
| |
| ret; |
| SYM_FUNC_END(sm4_ce_expand_key) |
| |
| .align 3 |
| SYM_FUNC_START(sm4_ce_crypt_block) |
| /* input: |
| * x0: round key array, CTX |
| * x1: dst |
| * x2: src |
| */ |
| SM4_PREPARE(x0) |
| |
| ld1 {v0.16b}, [x2]; |
| SM4_CRYPT_BLK(v0); |
| st1 {v0.16b}, [x1]; |
| |
| ret; |
| SYM_FUNC_END(sm4_ce_crypt_block) |
| |
| .align 3 |
| SYM_FUNC_START(sm4_ce_crypt) |
| /* input: |
| * x0: round key array, CTX |
| * x1: dst |
| * x2: src |
| * w3: nblocks |
| */ |
| SM4_PREPARE(x0) |
| |
| .Lcrypt_loop_blk: |
| sub w3, w3, #8; |
| tbnz w3, #31, .Lcrypt_tail8; |
| |
| ld1 {v0.16b-v3.16b}, [x2], #64; |
| ld1 {v4.16b-v7.16b}, [x2], #64; |
| |
| SM4_CRYPT_BLK8(v0, v1, v2, v3, v4, v5, v6, v7); |
| |
| st1 {v0.16b-v3.16b}, [x1], #64; |
| st1 {v4.16b-v7.16b}, [x1], #64; |
| |
| cbz w3, .Lcrypt_end; |
| b .Lcrypt_loop_blk; |
| |
| .Lcrypt_tail8: |
| add w3, w3, #8; |
| cmp w3, #4; |
| blt .Lcrypt_tail4; |
| |
| sub w3, w3, #4; |
| |
| ld1 {v0.16b-v3.16b}, [x2], #64; |
| SM4_CRYPT_BLK4(v0, v1, v2, v3); |
| st1 {v0.16b-v3.16b}, [x1], #64; |
| |
| cbz w3, .Lcrypt_end; |
| |
| .Lcrypt_tail4: |
| sub w3, w3, #1; |
| |
| ld1 {v0.16b}, [x2], #16; |
| SM4_CRYPT_BLK(v0); |
| st1 {v0.16b}, [x1], #16; |
| |
| cbnz w3, .Lcrypt_tail4; |
| |
| .Lcrypt_end: |
| ret; |
| SYM_FUNC_END(sm4_ce_crypt) |
| |
| .align 3 |
| SYM_FUNC_START(sm4_ce_cbc_enc) |
| /* input: |
| * x0: round key array, CTX |
| * x1: dst |
| * x2: src |
| * x3: iv (big endian, 128 bit) |
| * w4: nblocks |
| */ |
| SM4_PREPARE(x0) |
| |
| ld1 {RIV.16b}, [x3] |
| |
| .Lcbc_enc_loop_4x: |
| cmp w4, #4 |
| blt .Lcbc_enc_loop_1x |
| |
| sub w4, w4, #4 |
| |
| ld1 {v0.16b-v3.16b}, [x2], #64 |
| |
| eor v0.16b, v0.16b, RIV.16b |
| SM4_CRYPT_BLK(v0) |
| eor v1.16b, v1.16b, v0.16b |
| SM4_CRYPT_BLK(v1) |
| eor v2.16b, v2.16b, v1.16b |
| SM4_CRYPT_BLK(v2) |
| eor v3.16b, v3.16b, v2.16b |
| SM4_CRYPT_BLK(v3) |
| |
| st1 {v0.16b-v3.16b}, [x1], #64 |
| mov RIV.16b, v3.16b |
| |
| cbz w4, .Lcbc_enc_end |
| b .Lcbc_enc_loop_4x |
| |
| .Lcbc_enc_loop_1x: |
| sub w4, w4, #1 |
| |
| ld1 {v0.16b}, [x2], #16 |
| |
| eor RIV.16b, RIV.16b, v0.16b |
| SM4_CRYPT_BLK(RIV) |
| |
| st1 {RIV.16b}, [x1], #16 |
| |
| cbnz w4, .Lcbc_enc_loop_1x |
| |
| .Lcbc_enc_end: |
| /* store new IV */ |
| st1 {RIV.16b}, [x3] |
| |
| ret |
| SYM_FUNC_END(sm4_ce_cbc_enc) |
| |
| .align 3 |
| SYM_FUNC_START(sm4_ce_cbc_dec) |
| /* input: |
| * x0: round key array, CTX |
| * x1: dst |
| * x2: src |
| * x3: iv (big endian, 128 bit) |
| * w4: nblocks |
| */ |
| SM4_PREPARE(x0) |
| |
| ld1 {RIV.16b}, [x3] |
| |
| .Lcbc_dec_loop_8x: |
| sub w4, w4, #8 |
| tbnz w4, #31, .Lcbc_dec_4x |
| |
| ld1 {v0.16b-v3.16b}, [x2], #64 |
| ld1 {v4.16b-v7.16b}, [x2], #64 |
| |
| rev32 v8.16b, v0.16b |
| rev32 v9.16b, v1.16b |
| rev32 v10.16b, v2.16b |
| rev32 v11.16b, v3.16b |
| rev32 v12.16b, v4.16b |
| rev32 v13.16b, v5.16b |
| rev32 v14.16b, v6.16b |
| rev32 v15.16b, v7.16b |
| |
| SM4_CRYPT_BLK8_BE(v8, v9, v10, v11, v12, v13, v14, v15) |
| |
| eor v8.16b, v8.16b, RIV.16b |
| eor v9.16b, v9.16b, v0.16b |
| eor v10.16b, v10.16b, v1.16b |
| eor v11.16b, v11.16b, v2.16b |
| eor v12.16b, v12.16b, v3.16b |
| eor v13.16b, v13.16b, v4.16b |
| eor v14.16b, v14.16b, v5.16b |
| eor v15.16b, v15.16b, v6.16b |
| |
| st1 {v8.16b-v11.16b}, [x1], #64 |
| st1 {v12.16b-v15.16b}, [x1], #64 |
| |
| mov RIV.16b, v7.16b |
| |
| cbz w4, .Lcbc_dec_end |
| b .Lcbc_dec_loop_8x |
| |
| .Lcbc_dec_4x: |
| add w4, w4, #8 |
| cmp w4, #4 |
| blt .Lcbc_dec_loop_1x |
| |
| sub w4, w4, #4 |
| |
| ld1 {v0.16b-v3.16b}, [x2], #64 |
| |
| rev32 v8.16b, v0.16b |
| rev32 v9.16b, v1.16b |
| rev32 v10.16b, v2.16b |
| rev32 v11.16b, v3.16b |
| |
| SM4_CRYPT_BLK4_BE(v8, v9, v10, v11) |
| |
| eor v8.16b, v8.16b, RIV.16b |
| eor v9.16b, v9.16b, v0.16b |
| eor v10.16b, v10.16b, v1.16b |
| eor v11.16b, v11.16b, v2.16b |
| |
| st1 {v8.16b-v11.16b}, [x1], #64 |
| |
| mov RIV.16b, v3.16b |
| |
| cbz w4, .Lcbc_dec_end |
| |
| .Lcbc_dec_loop_1x: |
| sub w4, w4, #1 |
| |
| ld1 {v0.16b}, [x2], #16 |
| |
| rev32 v8.16b, v0.16b |
| |
| SM4_CRYPT_BLK_BE(v8) |
| |
| eor v8.16b, v8.16b, RIV.16b |
| st1 {v8.16b}, [x1], #16 |
| |
| mov RIV.16b, v0.16b |
| |
| cbnz w4, .Lcbc_dec_loop_1x |
| |
| .Lcbc_dec_end: |
| /* store new IV */ |
| st1 {RIV.16b}, [x3] |
| |
| ret |
| SYM_FUNC_END(sm4_ce_cbc_dec) |
| |
| .align 3 |
| SYM_FUNC_START(sm4_ce_cbc_cts_enc) |
| /* input: |
| * x0: round key array, CTX |
| * x1: dst |
| * x2: src |
| * x3: iv (big endian, 128 bit) |
| * w4: nbytes |
| */ |
| SM4_PREPARE(x0) |
| |
| sub w5, w4, #16 |
| uxtw x5, w5 |
| |
| ld1 {RIV.16b}, [x3] |
| |
| ld1 {v0.16b}, [x2] |
| eor RIV.16b, RIV.16b, v0.16b |
| SM4_CRYPT_BLK(RIV) |
| |
| /* load permute table */ |
| adr_l x6, .Lcts_permute_table |
| add x7, x6, #32 |
| add x6, x6, x5 |
| sub x7, x7, x5 |
| ld1 {v3.16b}, [x6] |
| ld1 {v4.16b}, [x7] |
| |
| /* overlapping loads */ |
| add x2, x2, x5 |
| ld1 {v1.16b}, [x2] |
| |
| /* create Cn from En-1 */ |
| tbl v0.16b, {RIV.16b}, v3.16b |
| /* padding Pn with zeros */ |
| tbl v1.16b, {v1.16b}, v4.16b |
| |
| eor v1.16b, v1.16b, RIV.16b |
| SM4_CRYPT_BLK(v1) |
| |
| /* overlapping stores */ |
| add x5, x1, x5 |
| st1 {v0.16b}, [x5] |
| st1 {v1.16b}, [x1] |
| |
| ret |
| SYM_FUNC_END(sm4_ce_cbc_cts_enc) |
| |
| .align 3 |
| SYM_FUNC_START(sm4_ce_cbc_cts_dec) |
| /* input: |
| * x0: round key array, CTX |
| * x1: dst |
| * x2: src |
| * x3: iv (big endian, 128 bit) |
| * w4: nbytes |
| */ |
| SM4_PREPARE(x0) |
| |
| sub w5, w4, #16 |
| uxtw x5, w5 |
| |
| ld1 {RIV.16b}, [x3] |
| |
| /* load permute table */ |
| adr_l x6, .Lcts_permute_table |
| add x7, x6, #32 |
| add x6, x6, x5 |
| sub x7, x7, x5 |
| ld1 {v3.16b}, [x6] |
| ld1 {v4.16b}, [x7] |
| |
| /* overlapping loads */ |
| ld1 {v0.16b}, [x2], x5 |
| ld1 {v1.16b}, [x2] |
| |
| SM4_CRYPT_BLK(v0) |
| /* select the first Ln bytes of Xn to create Pn */ |
| tbl v2.16b, {v0.16b}, v3.16b |
| eor v2.16b, v2.16b, v1.16b |
| |
| /* overwrite the first Ln bytes with Cn to create En-1 */ |
| tbx v0.16b, {v1.16b}, v4.16b |
| SM4_CRYPT_BLK(v0) |
| eor v0.16b, v0.16b, RIV.16b |
| |
| /* overlapping stores */ |
| add x5, x1, x5 |
| st1 {v2.16b}, [x5] |
| st1 {v0.16b}, [x1] |
| |
| ret |
| SYM_FUNC_END(sm4_ce_cbc_cts_dec) |
| |
| .align 3 |
| SYM_FUNC_START(sm4_ce_cfb_enc) |
| /* input: |
| * x0: round key array, CTX |
| * x1: dst |
| * x2: src |
| * x3: iv (big endian, 128 bit) |
| * w4: nblocks |
| */ |
| SM4_PREPARE(x0) |
| |
| ld1 {RIV.16b}, [x3] |
| |
| .Lcfb_enc_loop_4x: |
| cmp w4, #4 |
| blt .Lcfb_enc_loop_1x |
| |
| sub w4, w4, #4 |
| |
| ld1 {v0.16b-v3.16b}, [x2], #64 |
| |
| rev32 v8.16b, RIV.16b |
| SM4_CRYPT_BLK_BE(v8) |
| eor v0.16b, v0.16b, v8.16b |
| |
| rev32 v8.16b, v0.16b |
| SM4_CRYPT_BLK_BE(v8) |
| eor v1.16b, v1.16b, v8.16b |
| |
| rev32 v8.16b, v1.16b |
| SM4_CRYPT_BLK_BE(v8) |
| eor v2.16b, v2.16b, v8.16b |
| |
| rev32 v8.16b, v2.16b |
| SM4_CRYPT_BLK_BE(v8) |
| eor v3.16b, v3.16b, v8.16b |
| |
| st1 {v0.16b-v3.16b}, [x1], #64 |
| mov RIV.16b, v3.16b |
| |
| cbz w4, .Lcfb_enc_end |
| b .Lcfb_enc_loop_4x |
| |
| .Lcfb_enc_loop_1x: |
| sub w4, w4, #1 |
| |
| ld1 {v0.16b}, [x2], #16 |
| |
| SM4_CRYPT_BLK(RIV) |
| eor RIV.16b, RIV.16b, v0.16b |
| |
| st1 {RIV.16b}, [x1], #16 |
| |
| cbnz w4, .Lcfb_enc_loop_1x |
| |
| .Lcfb_enc_end: |
| /* store new IV */ |
| st1 {RIV.16b}, [x3] |
| |
| ret |
| SYM_FUNC_END(sm4_ce_cfb_enc) |
| |
| .align 3 |
| SYM_FUNC_START(sm4_ce_cfb_dec) |
| /* input: |
| * x0: round key array, CTX |
| * x1: dst |
| * x2: src |
| * x3: iv (big endian, 128 bit) |
| * w4: nblocks |
| */ |
| SM4_PREPARE(x0) |
| |
| ld1 {RIV.16b}, [x3] |
| |
| .Lcfb_dec_loop_8x: |
| sub w4, w4, #8 |
| tbnz w4, #31, .Lcfb_dec_4x |
| |
| ld1 {v0.16b-v3.16b}, [x2], #64 |
| ld1 {v4.16b-v7.16b}, [x2], #64 |
| |
| rev32 v8.16b, RIV.16b |
| rev32 v9.16b, v0.16b |
| rev32 v10.16b, v1.16b |
| rev32 v11.16b, v2.16b |
| rev32 v12.16b, v3.16b |
| rev32 v13.16b, v4.16b |
| rev32 v14.16b, v5.16b |
| rev32 v15.16b, v6.16b |
| |
| SM4_CRYPT_BLK8_BE(v8, v9, v10, v11, v12, v13, v14, v15) |
| |
| mov RIV.16b, v7.16b |
| |
| eor v0.16b, v0.16b, v8.16b |
| eor v1.16b, v1.16b, v9.16b |
| eor v2.16b, v2.16b, v10.16b |
| eor v3.16b, v3.16b, v11.16b |
| eor v4.16b, v4.16b, v12.16b |
| eor v5.16b, v5.16b, v13.16b |
| eor v6.16b, v6.16b, v14.16b |
| eor v7.16b, v7.16b, v15.16b |
| |
| st1 {v0.16b-v3.16b}, [x1], #64 |
| st1 {v4.16b-v7.16b}, [x1], #64 |
| |
| cbz w4, .Lcfb_dec_end |
| b .Lcfb_dec_loop_8x |
| |
| .Lcfb_dec_4x: |
| add w4, w4, #8 |
| cmp w4, #4 |
| blt .Lcfb_dec_loop_1x |
| |
| sub w4, w4, #4 |
| |
| ld1 {v0.16b-v3.16b}, [x2], #64 |
| |
| rev32 v8.16b, RIV.16b |
| rev32 v9.16b, v0.16b |
| rev32 v10.16b, v1.16b |
| rev32 v11.16b, v2.16b |
| |
| SM4_CRYPT_BLK4_BE(v8, v9, v10, v11) |
| |
| mov RIV.16b, v3.16b |
| |
| eor v0.16b, v0.16b, v8.16b |
| eor v1.16b, v1.16b, v9.16b |
| eor v2.16b, v2.16b, v10.16b |
| eor v3.16b, v3.16b, v11.16b |
| |
| st1 {v0.16b-v3.16b}, [x1], #64 |
| |
| cbz w4, .Lcfb_dec_end |
| |
| .Lcfb_dec_loop_1x: |
| sub w4, w4, #1 |
| |
| ld1 {v0.16b}, [x2], #16 |
| |
| SM4_CRYPT_BLK(RIV) |
| |
| eor RIV.16b, RIV.16b, v0.16b |
| st1 {RIV.16b}, [x1], #16 |
| |
| mov RIV.16b, v0.16b |
| |
| cbnz w4, .Lcfb_dec_loop_1x |
| |
| .Lcfb_dec_end: |
| /* store new IV */ |
| st1 {RIV.16b}, [x3] |
| |
| ret |
| SYM_FUNC_END(sm4_ce_cfb_dec) |
| |
| .align 3 |
| SYM_FUNC_START(sm4_ce_ctr_enc) |
| /* input: |
| * x0: round key array, CTX |
| * x1: dst |
| * x2: src |
| * x3: ctr (big endian, 128 bit) |
| * w4: nblocks |
| */ |
| SM4_PREPARE(x0) |
| |
| ldp x7, x8, [x3] |
| rev x7, x7 |
| rev x8, x8 |
| |
| .Lctr_loop_8x: |
| sub w4, w4, #8 |
| tbnz w4, #31, .Lctr_4x |
| |
| #define inc_le128(vctr) \ |
| mov vctr.d[1], x8; \ |
| mov vctr.d[0], x7; \ |
| adds x8, x8, #1; \ |
| rev64 vctr.16b, vctr.16b; \ |
| adc x7, x7, xzr; |
| |
| /* construct CTRs */ |
| inc_le128(v0) /* +0 */ |
| inc_le128(v1) /* +1 */ |
| inc_le128(v2) /* +2 */ |
| inc_le128(v3) /* +3 */ |
| inc_le128(v4) /* +4 */ |
| inc_le128(v5) /* +5 */ |
| inc_le128(v6) /* +6 */ |
| inc_le128(v7) /* +7 */ |
| |
| ld1 {v8.16b-v11.16b}, [x2], #64 |
| ld1 {v12.16b-v15.16b}, [x2], #64 |
| |
| SM4_CRYPT_BLK8(v0, v1, v2, v3, v4, v5, v6, v7) |
| |
| eor v0.16b, v0.16b, v8.16b |
| eor v1.16b, v1.16b, v9.16b |
| eor v2.16b, v2.16b, v10.16b |
| eor v3.16b, v3.16b, v11.16b |
| eor v4.16b, v4.16b, v12.16b |
| eor v5.16b, v5.16b, v13.16b |
| eor v6.16b, v6.16b, v14.16b |
| eor v7.16b, v7.16b, v15.16b |
| |
| st1 {v0.16b-v3.16b}, [x1], #64 |
| st1 {v4.16b-v7.16b}, [x1], #64 |
| |
| cbz w4, .Lctr_end |
| b .Lctr_loop_8x |
| |
| .Lctr_4x: |
| add w4, w4, #8 |
| cmp w4, #4 |
| blt .Lctr_loop_1x |
| |
| sub w4, w4, #4 |
| |
| /* construct CTRs */ |
| inc_le128(v0) /* +0 */ |
| inc_le128(v1) /* +1 */ |
| inc_le128(v2) /* +2 */ |
| inc_le128(v3) /* +3 */ |
| |
| ld1 {v8.16b-v11.16b}, [x2], #64 |
| |
| SM4_CRYPT_BLK4(v0, v1, v2, v3) |
| |
| eor v0.16b, v0.16b, v8.16b |
| eor v1.16b, v1.16b, v9.16b |
| eor v2.16b, v2.16b, v10.16b |
| eor v3.16b, v3.16b, v11.16b |
| |
| st1 {v0.16b-v3.16b}, [x1], #64 |
| |
| cbz w4, .Lctr_end |
| |
| .Lctr_loop_1x: |
| sub w4, w4, #1 |
| |
| /* construct CTRs */ |
| inc_le128(v0) |
| |
| ld1 {v8.16b}, [x2], #16 |
| |
| SM4_CRYPT_BLK(v0) |
| |
| eor v0.16b, v0.16b, v8.16b |
| st1 {v0.16b}, [x1], #16 |
| |
| cbnz w4, .Lctr_loop_1x |
| |
| .Lctr_end: |
| /* store new CTR */ |
| rev x7, x7 |
| rev x8, x8 |
| stp x7, x8, [x3] |
| |
| ret |
| SYM_FUNC_END(sm4_ce_ctr_enc) |
| |
| |
| #define tweak_next(vt, vin, RTMP) \ |
| sshr RTMP.2d, vin.2d, #63; \ |
| and RTMP.16b, RTMP.16b, RMASK.16b; \ |
| add vt.2d, vin.2d, vin.2d; \ |
| ext RTMP.16b, RTMP.16b, RTMP.16b, #8; \ |
| eor vt.16b, vt.16b, RTMP.16b; |
| |
| .align 3 |
| SYM_FUNC_START(sm4_ce_xts_enc) |
| /* input: |
| * x0: round key array, CTX |
| * x1: dst |
| * x2: src |
| * x3: tweak (big endian, 128 bit) |
| * w4: nbytes |
| * x5: round key array for IV |
| */ |
| ld1 {v8.16b}, [x3] |
| |
| cbz x5, .Lxts_enc_nofirst |
| |
| SM4_PREPARE(x5) |
| |
| /* Generate first tweak */ |
| SM4_CRYPT_BLK(v8) |
| |
| .Lxts_enc_nofirst: |
| SM4_PREPARE(x0) |
| |
| ands w5, w4, #15 |
| lsr w4, w4, #4 |
| sub w6, w4, #1 |
| csel w4, w4, w6, eq |
| uxtw x5, w5 |
| |
| movi RMASK.2s, #0x1 |
| movi RTMP0.2s, #0x87 |
| uzp1 RMASK.4s, RMASK.4s, RTMP0.4s |
| |
| cbz w4, .Lxts_enc_cts |
| |
| .Lxts_enc_loop_8x: |
| sub w4, w4, #8 |
| tbnz w4, #31, .Lxts_enc_4x |
| |
| tweak_next( v9, v8, RTMP0) |
| tweak_next(v10, v9, RTMP1) |
| tweak_next(v11, v10, RTMP2) |
| tweak_next(v12, v11, RTMP3) |
| tweak_next(v13, v12, RTMP0) |
| tweak_next(v14, v13, RTMP1) |
| tweak_next(v15, v14, RTMP2) |
| |
| ld1 {v0.16b-v3.16b}, [x2], #64 |
| ld1 {v4.16b-v7.16b}, [x2], #64 |
| eor v0.16b, v0.16b, v8.16b |
| eor v1.16b, v1.16b, v9.16b |
| eor v2.16b, v2.16b, v10.16b |
| eor v3.16b, v3.16b, v11.16b |
| eor v4.16b, v4.16b, v12.16b |
| eor v5.16b, v5.16b, v13.16b |
| eor v6.16b, v6.16b, v14.16b |
| eor v7.16b, v7.16b, v15.16b |
| |
| SM4_CRYPT_BLK8(v0, v1, v2, v3, v4, v5, v6, v7) |
| |
| eor v0.16b, v0.16b, v8.16b |
| eor v1.16b, v1.16b, v9.16b |
| eor v2.16b, v2.16b, v10.16b |
| eor v3.16b, v3.16b, v11.16b |
| eor v4.16b, v4.16b, v12.16b |
| eor v5.16b, v5.16b, v13.16b |
| eor v6.16b, v6.16b, v14.16b |
| eor v7.16b, v7.16b, v15.16b |
| st1 {v0.16b-v3.16b}, [x1], #64 |
| st1 {v4.16b-v7.16b}, [x1], #64 |
| |
| tweak_next(v8, v15, RTMP3) |
| |
| cbz w4, .Lxts_enc_cts |
| b .Lxts_enc_loop_8x |
| |
| .Lxts_enc_4x: |
| add w4, w4, #8 |
| cmp w4, #4 |
| blt .Lxts_enc_loop_1x |
| |
| sub w4, w4, #4 |
| |
| tweak_next( v9, v8, RTMP0) |
| tweak_next(v10, v9, RTMP1) |
| tweak_next(v11, v10, RTMP2) |
| |
| ld1 {v0.16b-v3.16b}, [x2], #64 |
| eor v0.16b, v0.16b, v8.16b |
| eor v1.16b, v1.16b, v9.16b |
| eor v2.16b, v2.16b, v10.16b |
| eor v3.16b, v3.16b, v11.16b |
| |
| SM4_CRYPT_BLK4(v0, v1, v2, v3) |
| |
| eor v0.16b, v0.16b, v8.16b |
| eor v1.16b, v1.16b, v9.16b |
| eor v2.16b, v2.16b, v10.16b |
| eor v3.16b, v3.16b, v11.16b |
| st1 {v0.16b-v3.16b}, [x1], #64 |
| |
| tweak_next(v8, v11, RTMP3) |
| |
| cbz w4, .Lxts_enc_cts |
| |
| .Lxts_enc_loop_1x: |
| sub w4, w4, #1 |
| |
| ld1 {v0.16b}, [x2], #16 |
| eor v0.16b, v0.16b, v8.16b |
| |
| SM4_CRYPT_BLK(v0) |
| |
| eor v0.16b, v0.16b, v8.16b |
| st1 {v0.16b}, [x1], #16 |
| |
| tweak_next(v8, v8, RTMP0) |
| |
| cbnz w4, .Lxts_enc_loop_1x |
| |
| .Lxts_enc_cts: |
| cbz x5, .Lxts_enc_end |
| |
| /* cipher text stealing */ |
| |
| tweak_next(v9, v8, RTMP0) |
| ld1 {v0.16b}, [x2] |
| eor v0.16b, v0.16b, v8.16b |
| SM4_CRYPT_BLK(v0) |
| eor v0.16b, v0.16b, v8.16b |
| |
| /* load permute table */ |
| adr_l x6, .Lcts_permute_table |
| add x7, x6, #32 |
| add x6, x6, x5 |
| sub x7, x7, x5 |
| ld1 {v3.16b}, [x6] |
| ld1 {v4.16b}, [x7] |
| |
| /* overlapping loads */ |
| add x2, x2, x5 |
| ld1 {v1.16b}, [x2] |
| |
| /* create Cn from En-1 */ |
| tbl v2.16b, {v0.16b}, v3.16b |
| /* padding Pn with En-1 at the end */ |
| tbx v0.16b, {v1.16b}, v4.16b |
| |
| eor v0.16b, v0.16b, v9.16b |
| SM4_CRYPT_BLK(v0) |
| eor v0.16b, v0.16b, v9.16b |
| |
| |
| /* overlapping stores */ |
| add x5, x1, x5 |
| st1 {v2.16b}, [x5] |
| st1 {v0.16b}, [x1] |
| |
| b .Lxts_enc_ret |
| |
| .Lxts_enc_end: |
| /* store new tweak */ |
| st1 {v8.16b}, [x3] |
| |
| .Lxts_enc_ret: |
| ret |
| SYM_FUNC_END(sm4_ce_xts_enc) |
| |
| .align 3 |
| SYM_FUNC_START(sm4_ce_xts_dec) |
| /* input: |
| * x0: round key array, CTX |
| * x1: dst |
| * x2: src |
| * x3: tweak (big endian, 128 bit) |
| * w4: nbytes |
| * x5: round key array for IV |
| */ |
| ld1 {v8.16b}, [x3] |
| |
| cbz x5, .Lxts_dec_nofirst |
| |
| SM4_PREPARE(x5) |
| |
| /* Generate first tweak */ |
| SM4_CRYPT_BLK(v8) |
| |
| .Lxts_dec_nofirst: |
| SM4_PREPARE(x0) |
| |
| ands w5, w4, #15 |
| lsr w4, w4, #4 |
| sub w6, w4, #1 |
| csel w4, w4, w6, eq |
| uxtw x5, w5 |
| |
| movi RMASK.2s, #0x1 |
| movi RTMP0.2s, #0x87 |
| uzp1 RMASK.4s, RMASK.4s, RTMP0.4s |
| |
| cbz w4, .Lxts_dec_cts |
| |
| .Lxts_dec_loop_8x: |
| sub w4, w4, #8 |
| tbnz w4, #31, .Lxts_dec_4x |
| |
| tweak_next( v9, v8, RTMP0) |
| tweak_next(v10, v9, RTMP1) |
| tweak_next(v11, v10, RTMP2) |
| tweak_next(v12, v11, RTMP3) |
| tweak_next(v13, v12, RTMP0) |
| tweak_next(v14, v13, RTMP1) |
| tweak_next(v15, v14, RTMP2) |
| |
| ld1 {v0.16b-v3.16b}, [x2], #64 |
| ld1 {v4.16b-v7.16b}, [x2], #64 |
| eor v0.16b, v0.16b, v8.16b |
| eor v1.16b, v1.16b, v9.16b |
| eor v2.16b, v2.16b, v10.16b |
| eor v3.16b, v3.16b, v11.16b |
| eor v4.16b, v4.16b, v12.16b |
| eor v5.16b, v5.16b, v13.16b |
| eor v6.16b, v6.16b, v14.16b |
| eor v7.16b, v7.16b, v15.16b |
| |
| SM4_CRYPT_BLK8(v0, v1, v2, v3, v4, v5, v6, v7) |
| |
| eor v0.16b, v0.16b, v8.16b |
| eor v1.16b, v1.16b, v9.16b |
| eor v2.16b, v2.16b, v10.16b |
| eor v3.16b, v3.16b, v11.16b |
| eor v4.16b, v4.16b, v12.16b |
| eor v5.16b, v5.16b, v13.16b |
| eor v6.16b, v6.16b, v14.16b |
| eor v7.16b, v7.16b, v15.16b |
| st1 {v0.16b-v3.16b}, [x1], #64 |
| st1 {v4.16b-v7.16b}, [x1], #64 |
| |
| tweak_next(v8, v15, RTMP3) |
| |
| cbz w4, .Lxts_dec_cts |
| b .Lxts_dec_loop_8x |
| |
| .Lxts_dec_4x: |
| add w4, w4, #8 |
| cmp w4, #4 |
| blt .Lxts_dec_loop_1x |
| |
| sub w4, w4, #4 |
| |
| tweak_next( v9, v8, RTMP0) |
| tweak_next(v10, v9, RTMP1) |
| tweak_next(v11, v10, RTMP2) |
| |
| ld1 {v0.16b-v3.16b}, [x2], #64 |
| eor v0.16b, v0.16b, v8.16b |
| eor v1.16b, v1.16b, v9.16b |
| eor v2.16b, v2.16b, v10.16b |
| eor v3.16b, v3.16b, v11.16b |
| |
| SM4_CRYPT_BLK4(v0, v1, v2, v3) |
| |
| eor v0.16b, v0.16b, v8.16b |
| eor v1.16b, v1.16b, v9.16b |
| eor v2.16b, v2.16b, v10.16b |
| eor v3.16b, v3.16b, v11.16b |
| st1 {v0.16b-v3.16b}, [x1], #64 |
| |
| tweak_next(v8, v11, RTMP3) |
| |
| cbz w4, .Lxts_dec_cts |
| |
| .Lxts_dec_loop_1x: |
| sub w4, w4, #1 |
| |
| ld1 {v0.16b}, [x2], #16 |
| eor v0.16b, v0.16b, v8.16b |
| |
| SM4_CRYPT_BLK(v0) |
| |
| eor v0.16b, v0.16b, v8.16b |
| st1 {v0.16b}, [x1], #16 |
| |
| tweak_next(v8, v8, RTMP0) |
| |
| cbnz w4, .Lxts_dec_loop_1x |
| |
| .Lxts_dec_cts: |
| cbz x5, .Lxts_dec_end |
| |
| /* cipher text stealing */ |
| |
| tweak_next(v9, v8, RTMP0) |
| ld1 {v0.16b}, [x2] |
| eor v0.16b, v0.16b, v9.16b |
| SM4_CRYPT_BLK(v0) |
| eor v0.16b, v0.16b, v9.16b |
| |
| /* load permute table */ |
| adr_l x6, .Lcts_permute_table |
| add x7, x6, #32 |
| add x6, x6, x5 |
| sub x7, x7, x5 |
| ld1 {v3.16b}, [x6] |
| ld1 {v4.16b}, [x7] |
| |
| /* overlapping loads */ |
| add x2, x2, x5 |
| ld1 {v1.16b}, [x2] |
| |
| /* create Cn from En-1 */ |
| tbl v2.16b, {v0.16b}, v3.16b |
| /* padding Pn with En-1 at the end */ |
| tbx v0.16b, {v1.16b}, v4.16b |
| |
| eor v0.16b, v0.16b, v8.16b |
| SM4_CRYPT_BLK(v0) |
| eor v0.16b, v0.16b, v8.16b |
| |
| |
| /* overlapping stores */ |
| add x5, x1, x5 |
| st1 {v2.16b}, [x5] |
| st1 {v0.16b}, [x1] |
| |
| b .Lxts_dec_ret |
| |
| .Lxts_dec_end: |
| /* store new tweak */ |
| st1 {v8.16b}, [x3] |
| |
| .Lxts_dec_ret: |
| ret |
| SYM_FUNC_END(sm4_ce_xts_dec) |
| |
| .align 3 |
| SYM_FUNC_START(sm4_ce_mac_update) |
| /* input: |
| * x0: round key array, CTX |
| * x1: digest |
| * x2: src |
| * w3: nblocks |
| * w4: enc_before |
| * w5: enc_after |
| */ |
| SM4_PREPARE(x0) |
| |
| ld1 {RMAC.16b}, [x1] |
| |
| cbz w4, .Lmac_update |
| |
| SM4_CRYPT_BLK(RMAC) |
| |
| .Lmac_update: |
| cbz w3, .Lmac_ret |
| |
| sub w6, w3, #1 |
| cmp w5, wzr |
| csel w3, w3, w6, ne |
| |
| cbz w3, .Lmac_end |
| |
| .Lmac_loop_4x: |
| cmp w3, #4 |
| blt .Lmac_loop_1x |
| |
| sub w3, w3, #4 |
| |
| ld1 {v0.16b-v3.16b}, [x2], #64 |
| |
| eor RMAC.16b, RMAC.16b, v0.16b |
| SM4_CRYPT_BLK(RMAC) |
| eor RMAC.16b, RMAC.16b, v1.16b |
| SM4_CRYPT_BLK(RMAC) |
| eor RMAC.16b, RMAC.16b, v2.16b |
| SM4_CRYPT_BLK(RMAC) |
| eor RMAC.16b, RMAC.16b, v3.16b |
| SM4_CRYPT_BLK(RMAC) |
| |
| cbz w3, .Lmac_end |
| b .Lmac_loop_4x |
| |
| .Lmac_loop_1x: |
| sub w3, w3, #1 |
| |
| ld1 {v0.16b}, [x2], #16 |
| |
| eor RMAC.16b, RMAC.16b, v0.16b |
| SM4_CRYPT_BLK(RMAC) |
| |
| cbnz w3, .Lmac_loop_1x |
| |
| |
| .Lmac_end: |
| cbnz w5, .Lmac_ret |
| |
| ld1 {v0.16b}, [x2], #16 |
| eor RMAC.16b, RMAC.16b, v0.16b |
| |
| .Lmac_ret: |
| st1 {RMAC.16b}, [x1] |
| ret |
| SYM_FUNC_END(sm4_ce_mac_update) |
| |
| |
| .section ".rodata", "a" |
| .align 4 |
| .Lbswap128_mask: |
| .byte 0x0c, 0x0d, 0x0e, 0x0f, 0x08, 0x09, 0x0a, 0x0b |
| .byte 0x04, 0x05, 0x06, 0x07, 0x00, 0x01, 0x02, 0x03 |
| |
| .Lcts_permute_table: |
| .byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff |
| .byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff |
| .byte 0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7 |
| .byte 0x8, 0x9, 0xa, 0xb, 0xc, 0xd, 0xe, 0xf |
| .byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff |
| .byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff |
