blob: 571485502ec8d25e041c0b7503fc575d86798638 [file] [log] [blame]
Jussi Kivilinna8280daa2011-09-26 16:47:25 +03001/*
2 * Glue Code for 3-way parallel assembler optimized version of Twofish
3 *
4 * Copyright (c) 2011 Jussi Kivilinna <jussi.kivilinna@mbnet.fi>
5 *
Jussi Kivilinna8280daa2011-09-26 16:47:25 +03006 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; either version 2 of the License, or
9 * (at your option) any later version.
10 *
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
15 *
16 * You should have received a copy of the GNU General Public License
17 * along with this program; if not, write to the Free Software
18 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
19 * USA
20 *
21 */
22
Eric Biggers37992fa2018-02-19 23:48:09 -080023#include <asm/crypto/glue_helper.h>
24#include <asm/crypto/twofish.h>
25#include <crypto/algapi.h>
26#include <crypto/b128ops.h>
27#include <crypto/internal/skcipher.h>
28#include <crypto/twofish.h>
Jussi Kivilinna8280daa2011-09-26 16:47:25 +030029#include <linux/crypto.h>
30#include <linux/init.h>
31#include <linux/module.h>
32#include <linux/types.h>
Jussi Kivilinna81559f92011-10-18 13:33:02 +030033
Johannes Goetzfried107778b52012-05-28 15:54:24 +020034EXPORT_SYMBOL_GPL(__twofish_enc_blk_3way);
Johannes Goetzfried107778b52012-05-28 15:54:24 +020035EXPORT_SYMBOL_GPL(twofish_dec_blk_3way);
Jussi Kivilinna8280daa2011-09-26 16:47:25 +030036
Eric Biggers37992fa2018-02-19 23:48:09 -080037static int twofish_setkey_skcipher(struct crypto_skcipher *tfm,
38 const u8 *key, unsigned int keylen)
39{
40 return twofish_setkey(&tfm->base, key, keylen);
41}
42
Jussi Kivilinna8280daa2011-09-26 16:47:25 +030043static inline void twofish_enc_blk_3way(struct twofish_ctx *ctx, u8 *dst,
44 const u8 *src)
45{
46 __twofish_enc_blk_3way(ctx, dst, src, false);
47}
48
49static inline void twofish_enc_blk_xor_3way(struct twofish_ctx *ctx, u8 *dst,
50 const u8 *src)
51{
52 __twofish_enc_blk_3way(ctx, dst, src, true);
53}
54
Jussi Kivilinnaa7378d42012-06-18 14:07:39 +030055void twofish_dec_blk_cbc_3way(void *ctx, u128 *dst, const u128 *src)
Jussi Kivilinna8280daa2011-09-26 16:47:25 +030056{
Jussi Kivilinna414cb5e2012-06-18 14:07:34 +030057 u128 ivs[2];
Jussi Kivilinna8280daa2011-09-26 16:47:25 +030058
Jussi Kivilinna414cb5e2012-06-18 14:07:34 +030059 ivs[0] = src[0];
60 ivs[1] = src[1];
Jussi Kivilinna8280daa2011-09-26 16:47:25 +030061
Jussi Kivilinna414cb5e2012-06-18 14:07:34 +030062 twofish_dec_blk_3way(ctx, (u8 *)dst, (u8 *)src);
Jussi Kivilinna8280daa2011-09-26 16:47:25 +030063
Jussi Kivilinna414cb5e2012-06-18 14:07:34 +030064 u128_xor(&dst[1], &dst[1], &ivs[0]);
65 u128_xor(&dst[2], &dst[2], &ivs[1]);
66}
Jussi Kivilinnaa7378d42012-06-18 14:07:39 +030067EXPORT_SYMBOL_GPL(twofish_dec_blk_cbc_3way);
Jussi Kivilinna8280daa2011-09-26 16:47:25 +030068
Jussi Kivilinna58990982012-10-20 15:06:36 +030069void twofish_enc_blk_ctr(void *ctx, u128 *dst, const u128 *src, le128 *iv)
Jussi Kivilinna414cb5e2012-06-18 14:07:34 +030070{
71 be128 ctrblk;
Jussi Kivilinna8280daa2011-09-26 16:47:25 +030072
Jussi Kivilinna414cb5e2012-06-18 14:07:34 +030073 if (dst != src)
74 *dst = *src;
Jussi Kivilinna8280daa2011-09-26 16:47:25 +030075
Jussi Kivilinna58990982012-10-20 15:06:36 +030076 le128_to_be128(&ctrblk, iv);
77 le128_inc(iv);
Jussi Kivilinna8280daa2011-09-26 16:47:25 +030078
Jussi Kivilinna414cb5e2012-06-18 14:07:34 +030079 twofish_enc_blk(ctx, (u8 *)&ctrblk, (u8 *)&ctrblk);
80 u128_xor(dst, dst, (u128 *)&ctrblk);
81}
Jussi Kivilinnaa7378d42012-06-18 14:07:39 +030082EXPORT_SYMBOL_GPL(twofish_enc_blk_ctr);
Jussi Kivilinna8280daa2011-09-26 16:47:25 +030083
Jussi Kivilinnaa7378d42012-06-18 14:07:39 +030084void twofish_enc_blk_ctr_3way(void *ctx, u128 *dst, const u128 *src,
Jussi Kivilinna58990982012-10-20 15:06:36 +030085 le128 *iv)
Jussi Kivilinna414cb5e2012-06-18 14:07:34 +030086{
87 be128 ctrblks[3];
88
89 if (dst != src) {
90 dst[0] = src[0];
91 dst[1] = src[1];
92 dst[2] = src[2];
Jussi Kivilinna8280daa2011-09-26 16:47:25 +030093 }
94
Jussi Kivilinna58990982012-10-20 15:06:36 +030095 le128_to_be128(&ctrblks[0], iv);
96 le128_inc(iv);
97 le128_to_be128(&ctrblks[1], iv);
98 le128_inc(iv);
99 le128_to_be128(&ctrblks[2], iv);
100 le128_inc(iv);
Jussi Kivilinna414cb5e2012-06-18 14:07:34 +0300101
102 twofish_enc_blk_xor_3way(ctx, (u8 *)dst, (u8 *)ctrblks);
Jussi Kivilinna8280daa2011-09-26 16:47:25 +0300103}
Jussi Kivilinnaa7378d42012-06-18 14:07:39 +0300104EXPORT_SYMBOL_GPL(twofish_enc_blk_ctr_3way);
Jussi Kivilinna8280daa2011-09-26 16:47:25 +0300105
Jussi Kivilinna414cb5e2012-06-18 14:07:34 +0300106static const struct common_glue_ctx twofish_enc = {
107 .num_funcs = 2,
108 .fpu_blocks_limit = -1,
109
110 .funcs = { {
111 .num_blocks = 3,
112 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_enc_blk_3way) }
113 }, {
114 .num_blocks = 1,
115 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_enc_blk) }
116 } }
117};
118
119static const struct common_glue_ctx twofish_ctr = {
120 .num_funcs = 2,
121 .fpu_blocks_limit = -1,
122
123 .funcs = { {
124 .num_blocks = 3,
125 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_enc_blk_ctr_3way) }
126 }, {
127 .num_blocks = 1,
128 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_enc_blk_ctr) }
129 } }
130};
131
132static const struct common_glue_ctx twofish_dec = {
133 .num_funcs = 2,
134 .fpu_blocks_limit = -1,
135
136 .funcs = { {
137 .num_blocks = 3,
138 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_dec_blk_3way) }
139 }, {
140 .num_blocks = 1,
141 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_dec_blk) }
142 } }
143};
144
145static const struct common_glue_ctx twofish_dec_cbc = {
146 .num_funcs = 2,
147 .fpu_blocks_limit = -1,
148
149 .funcs = { {
150 .num_blocks = 3,
151 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_dec_blk_cbc_3way) }
152 }, {
153 .num_blocks = 1,
154 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_dec_blk) }
155 } }
156};
157
Eric Biggers37992fa2018-02-19 23:48:09 -0800158static int ecb_encrypt(struct skcipher_request *req)
Jussi Kivilinna8280daa2011-09-26 16:47:25 +0300159{
Eric Biggers37992fa2018-02-19 23:48:09 -0800160 return glue_ecb_req_128bit(&twofish_enc, req);
Jussi Kivilinna8280daa2011-09-26 16:47:25 +0300161}
162
Eric Biggers37992fa2018-02-19 23:48:09 -0800163static int ecb_decrypt(struct skcipher_request *req)
Jussi Kivilinna8280daa2011-09-26 16:47:25 +0300164{
Eric Biggers37992fa2018-02-19 23:48:09 -0800165 return glue_ecb_req_128bit(&twofish_dec, req);
Jussi Kivilinna8280daa2011-09-26 16:47:25 +0300166}
167
Eric Biggers37992fa2018-02-19 23:48:09 -0800168static int cbc_encrypt(struct skcipher_request *req)
Jussi Kivilinna8280daa2011-09-26 16:47:25 +0300169{
Eric Biggers37992fa2018-02-19 23:48:09 -0800170 return glue_cbc_encrypt_req_128bit(GLUE_FUNC_CAST(twofish_enc_blk),
171 req);
Jussi Kivilinna8280daa2011-09-26 16:47:25 +0300172}
173
Eric Biggers37992fa2018-02-19 23:48:09 -0800174static int cbc_decrypt(struct skcipher_request *req)
Jussi Kivilinna8280daa2011-09-26 16:47:25 +0300175{
Eric Biggers37992fa2018-02-19 23:48:09 -0800176 return glue_cbc_decrypt_req_128bit(&twofish_dec_cbc, req);
Jussi Kivilinna8280daa2011-09-26 16:47:25 +0300177}
178
Eric Biggers37992fa2018-02-19 23:48:09 -0800179static int ctr_crypt(struct skcipher_request *req)
Jussi Kivilinna8280daa2011-09-26 16:47:25 +0300180{
Eric Biggers37992fa2018-02-19 23:48:09 -0800181 return glue_ctr_req_128bit(&twofish_ctr, req);
Jussi Kivilinna8280daa2011-09-26 16:47:25 +0300182}
183
Eric Biggers37992fa2018-02-19 23:48:09 -0800184static struct skcipher_alg tf_skciphers[] = {
185 {
186 .base.cra_name = "ecb(twofish)",
187 .base.cra_driver_name = "ecb-twofish-3way",
188 .base.cra_priority = 300,
189 .base.cra_blocksize = TF_BLOCK_SIZE,
190 .base.cra_ctxsize = sizeof(struct twofish_ctx),
191 .base.cra_module = THIS_MODULE,
192 .min_keysize = TF_MIN_KEY_SIZE,
193 .max_keysize = TF_MAX_KEY_SIZE,
194 .setkey = twofish_setkey_skcipher,
195 .encrypt = ecb_encrypt,
196 .decrypt = ecb_decrypt,
197 }, {
198 .base.cra_name = "cbc(twofish)",
199 .base.cra_driver_name = "cbc-twofish-3way",
200 .base.cra_priority = 300,
201 .base.cra_blocksize = TF_BLOCK_SIZE,
202 .base.cra_ctxsize = sizeof(struct twofish_ctx),
203 .base.cra_module = THIS_MODULE,
204 .min_keysize = TF_MIN_KEY_SIZE,
205 .max_keysize = TF_MAX_KEY_SIZE,
206 .ivsize = TF_BLOCK_SIZE,
207 .setkey = twofish_setkey_skcipher,
208 .encrypt = cbc_encrypt,
209 .decrypt = cbc_decrypt,
210 }, {
211 .base.cra_name = "ctr(twofish)",
212 .base.cra_driver_name = "ctr-twofish-3way",
213 .base.cra_priority = 300,
214 .base.cra_blocksize = 1,
215 .base.cra_ctxsize = sizeof(struct twofish_ctx),
216 .base.cra_module = THIS_MODULE,
217 .min_keysize = TF_MIN_KEY_SIZE,
218 .max_keysize = TF_MAX_KEY_SIZE,
219 .ivsize = TF_BLOCK_SIZE,
220 .chunksize = TF_BLOCK_SIZE,
221 .setkey = twofish_setkey_skcipher,
222 .encrypt = ctr_crypt,
223 .decrypt = ctr_crypt,
Jussi Kivilinna53709dd2012-02-17 22:48:43 +0200224 },
Eric Biggers37992fa2018-02-19 23:48:09 -0800225};
Jussi Kivilinnabae6d302011-10-18 13:33:43 +0300226
Jussi Kivilinnaa522ee82011-12-20 12:20:16 +0200227static bool is_blacklisted_cpu(void)
228{
229 if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL)
230 return false;
231
232 if (boot_cpu_data.x86 == 0x06 &&
233 (boot_cpu_data.x86_model == 0x1c ||
234 boot_cpu_data.x86_model == 0x26 ||
235 boot_cpu_data.x86_model == 0x36)) {
236 /*
237 * On Atom, twofish-3way is slower than original assembler
238 * implementation. Twofish-3way trades off some performance in
239 * storing blocks in 64bit registers to allow three blocks to
240 * be processed parallel. Parallel operation then allows gaining
241 * more performance than was trade off, on out-of-order CPUs.
242 * However Atom does not benefit from this parallellism and
243 * should be blacklisted.
244 */
245 return true;
246 }
247
248 if (boot_cpu_data.x86 == 0x0f) {
249 /*
250 * On Pentium 4, twofish-3way is slower than original assembler
251 * implementation because excessive uses of 64bit rotate and
252 * left-shifts (which are really slow on P4) needed to store and
253 * handle 128bit block in two 64bit registers.
254 */
255 return true;
256 }
257
258 return false;
259}
260
261static int force;
262module_param(force, int, 0);
263MODULE_PARM_DESC(force, "Force module load, ignore CPU blacklist");
264
Jussi Kivilinnaff0a70f2012-03-15 22:11:57 +0200265static int __init init(void)
Jussi Kivilinna8280daa2011-09-26 16:47:25 +0300266{
Jussi Kivilinnaa522ee82011-12-20 12:20:16 +0200267 if (!force && is_blacklisted_cpu()) {
268 printk(KERN_INFO
269 "twofish-x86_64-3way: performance on this CPU "
270 "would be suboptimal: disabling "
271 "twofish-x86_64-3way.\n");
272 return -ENODEV;
273 }
274
Eric Biggers37992fa2018-02-19 23:48:09 -0800275 return crypto_register_skciphers(tf_skciphers,
276 ARRAY_SIZE(tf_skciphers));
Jussi Kivilinna8280daa2011-09-26 16:47:25 +0300277}
278
Jussi Kivilinnaff0a70f2012-03-15 22:11:57 +0200279static void __exit fini(void)
Jussi Kivilinna8280daa2011-09-26 16:47:25 +0300280{
Eric Biggers37992fa2018-02-19 23:48:09 -0800281 crypto_unregister_skciphers(tf_skciphers, ARRAY_SIZE(tf_skciphers));
Jussi Kivilinna8280daa2011-09-26 16:47:25 +0300282}
283
284module_init(init);
285module_exit(fini);
286
287MODULE_LICENSE("GPL");
288MODULE_DESCRIPTION("Twofish Cipher Algorithm, 3-way parallel asm optimized");
Kees Cook5d26a102014-11-20 17:05:53 -0800289MODULE_ALIAS_CRYPTO("twofish");
290MODULE_ALIAS_CRYPTO("twofish-asm");