Tom Lendacky | 2b78943 | 2013-11-12 11:46:28 -0600 | [diff] [blame] | 1 | /* |
| 2 | * AMD Cryptographic Coprocessor (CCP) AES crypto API support |
| 3 | * |
Gary R Hook | c7019c4 | 2016-03-01 13:49:15 -0600 | [diff] [blame] | 4 | * Copyright (C) 2013,2016 Advanced Micro Devices, Inc. |
Tom Lendacky | 2b78943 | 2013-11-12 11:46:28 -0600 | [diff] [blame] | 5 | * |
| 6 | * Author: Tom Lendacky <thomas.lendacky@amd.com> |
| 7 | * |
| 8 | * This program is free software; you can redistribute it and/or modify |
| 9 | * it under the terms of the GNU General Public License version 2 as |
| 10 | * published by the Free Software Foundation. |
| 11 | */ |
| 12 | |
| 13 | #include <linux/module.h> |
| 14 | #include <linux/sched.h> |
| 15 | #include <linux/delay.h> |
| 16 | #include <linux/scatterlist.h> |
| 17 | #include <linux/crypto.h> |
| 18 | #include <crypto/algapi.h> |
| 19 | #include <crypto/aes.h> |
| 20 | #include <crypto/ctr.h> |
| 21 | #include <crypto/scatterwalk.h> |
| 22 | |
| 23 | #include "ccp-crypto.h" |
| 24 | |
Tom Lendacky | 2b78943 | 2013-11-12 11:46:28 -0600 | [diff] [blame] | 25 | static int ccp_aes_complete(struct crypto_async_request *async_req, int ret) |
| 26 | { |
| 27 | struct ablkcipher_request *req = ablkcipher_request_cast(async_req); |
| 28 | struct ccp_ctx *ctx = crypto_tfm_ctx(req->base.tfm); |
| 29 | struct ccp_aes_req_ctx *rctx = ablkcipher_request_ctx(req); |
| 30 | |
| 31 | if (ret) |
| 32 | return ret; |
| 33 | |
| 34 | if (ctx->u.aes.mode != CCP_AES_MODE_ECB) |
| 35 | memcpy(req->info, rctx->iv, AES_BLOCK_SIZE); |
| 36 | |
| 37 | return 0; |
| 38 | } |
| 39 | |
| 40 | static int ccp_aes_setkey(struct crypto_ablkcipher *tfm, const u8 *key, |
| 41 | unsigned int key_len) |
| 42 | { |
| 43 | struct ccp_ctx *ctx = crypto_tfm_ctx(crypto_ablkcipher_tfm(tfm)); |
| 44 | struct ccp_crypto_ablkcipher_alg *alg = |
| 45 | ccp_crypto_ablkcipher_alg(crypto_ablkcipher_tfm(tfm)); |
| 46 | |
| 47 | switch (key_len) { |
| 48 | case AES_KEYSIZE_128: |
| 49 | ctx->u.aes.type = CCP_AES_TYPE_128; |
| 50 | break; |
| 51 | case AES_KEYSIZE_192: |
| 52 | ctx->u.aes.type = CCP_AES_TYPE_192; |
| 53 | break; |
| 54 | case AES_KEYSIZE_256: |
| 55 | ctx->u.aes.type = CCP_AES_TYPE_256; |
| 56 | break; |
| 57 | default: |
| 58 | crypto_ablkcipher_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); |
| 59 | return -EINVAL; |
| 60 | } |
| 61 | ctx->u.aes.mode = alg->mode; |
| 62 | ctx->u.aes.key_len = key_len; |
| 63 | |
| 64 | memcpy(ctx->u.aes.key, key, key_len); |
| 65 | sg_init_one(&ctx->u.aes.key_sg, ctx->u.aes.key, key_len); |
| 66 | |
| 67 | return 0; |
| 68 | } |
| 69 | |
| 70 | static int ccp_aes_crypt(struct ablkcipher_request *req, bool encrypt) |
| 71 | { |
| 72 | struct ccp_ctx *ctx = crypto_tfm_ctx(req->base.tfm); |
| 73 | struct ccp_aes_req_ctx *rctx = ablkcipher_request_ctx(req); |
| 74 | struct scatterlist *iv_sg = NULL; |
| 75 | unsigned int iv_len = 0; |
| 76 | int ret; |
| 77 | |
Tom Lendacky | 369f3da | 2013-12-10 10:38:44 -0600 | [diff] [blame] | 78 | if (!ctx->u.aes.key_len) |
Tom Lendacky | 2b78943 | 2013-11-12 11:46:28 -0600 | [diff] [blame] | 79 | return -EINVAL; |
Tom Lendacky | 2b78943 | 2013-11-12 11:46:28 -0600 | [diff] [blame] | 80 | |
| 81 | if (((ctx->u.aes.mode == CCP_AES_MODE_ECB) || |
| 82 | (ctx->u.aes.mode == CCP_AES_MODE_CBC) || |
| 83 | (ctx->u.aes.mode == CCP_AES_MODE_CFB)) && |
Tom Lendacky | 369f3da | 2013-12-10 10:38:44 -0600 | [diff] [blame] | 84 | (req->nbytes & (AES_BLOCK_SIZE - 1))) |
Tom Lendacky | 2b78943 | 2013-11-12 11:46:28 -0600 | [diff] [blame] | 85 | return -EINVAL; |
Tom Lendacky | 2b78943 | 2013-11-12 11:46:28 -0600 | [diff] [blame] | 86 | |
| 87 | if (ctx->u.aes.mode != CCP_AES_MODE_ECB) { |
Tom Lendacky | 369f3da | 2013-12-10 10:38:44 -0600 | [diff] [blame] | 88 | if (!req->info) |
Tom Lendacky | 2b78943 | 2013-11-12 11:46:28 -0600 | [diff] [blame] | 89 | return -EINVAL; |
Tom Lendacky | 2b78943 | 2013-11-12 11:46:28 -0600 | [diff] [blame] | 90 | |
| 91 | memcpy(rctx->iv, req->info, AES_BLOCK_SIZE); |
| 92 | iv_sg = &rctx->iv_sg; |
| 93 | iv_len = AES_BLOCK_SIZE; |
| 94 | sg_init_one(iv_sg, rctx->iv, iv_len); |
| 95 | } |
| 96 | |
| 97 | memset(&rctx->cmd, 0, sizeof(rctx->cmd)); |
| 98 | INIT_LIST_HEAD(&rctx->cmd.entry); |
| 99 | rctx->cmd.engine = CCP_ENGINE_AES; |
| 100 | rctx->cmd.u.aes.type = ctx->u.aes.type; |
| 101 | rctx->cmd.u.aes.mode = ctx->u.aes.mode; |
| 102 | rctx->cmd.u.aes.action = |
| 103 | (encrypt) ? CCP_AES_ACTION_ENCRYPT : CCP_AES_ACTION_DECRYPT; |
| 104 | rctx->cmd.u.aes.key = &ctx->u.aes.key_sg; |
| 105 | rctx->cmd.u.aes.key_len = ctx->u.aes.key_len; |
| 106 | rctx->cmd.u.aes.iv = iv_sg; |
| 107 | rctx->cmd.u.aes.iv_len = iv_len; |
| 108 | rctx->cmd.u.aes.src = req->src; |
| 109 | rctx->cmd.u.aes.src_len = req->nbytes; |
| 110 | rctx->cmd.u.aes.dst = req->dst; |
| 111 | |
| 112 | ret = ccp_crypto_enqueue_request(&req->base, &rctx->cmd); |
| 113 | |
| 114 | return ret; |
| 115 | } |
| 116 | |
| 117 | static int ccp_aes_encrypt(struct ablkcipher_request *req) |
| 118 | { |
| 119 | return ccp_aes_crypt(req, true); |
| 120 | } |
| 121 | |
| 122 | static int ccp_aes_decrypt(struct ablkcipher_request *req) |
| 123 | { |
| 124 | return ccp_aes_crypt(req, false); |
| 125 | } |
| 126 | |
| 127 | static int ccp_aes_cra_init(struct crypto_tfm *tfm) |
| 128 | { |
| 129 | struct ccp_ctx *ctx = crypto_tfm_ctx(tfm); |
| 130 | |
| 131 | ctx->complete = ccp_aes_complete; |
| 132 | ctx->u.aes.key_len = 0; |
| 133 | |
| 134 | tfm->crt_ablkcipher.reqsize = sizeof(struct ccp_aes_req_ctx); |
| 135 | |
| 136 | return 0; |
| 137 | } |
| 138 | |
| 139 | static void ccp_aes_cra_exit(struct crypto_tfm *tfm) |
| 140 | { |
| 141 | } |
| 142 | |
| 143 | static int ccp_aes_rfc3686_complete(struct crypto_async_request *async_req, |
| 144 | int ret) |
| 145 | { |
| 146 | struct ablkcipher_request *req = ablkcipher_request_cast(async_req); |
| 147 | struct ccp_aes_req_ctx *rctx = ablkcipher_request_ctx(req); |
| 148 | |
| 149 | /* Restore the original pointer */ |
| 150 | req->info = rctx->rfc3686_info; |
| 151 | |
| 152 | return ccp_aes_complete(async_req, ret); |
| 153 | } |
| 154 | |
| 155 | static int ccp_aes_rfc3686_setkey(struct crypto_ablkcipher *tfm, const u8 *key, |
| 156 | unsigned int key_len) |
| 157 | { |
| 158 | struct ccp_ctx *ctx = crypto_tfm_ctx(crypto_ablkcipher_tfm(tfm)); |
| 159 | |
| 160 | if (key_len < CTR_RFC3686_NONCE_SIZE) |
| 161 | return -EINVAL; |
| 162 | |
| 163 | key_len -= CTR_RFC3686_NONCE_SIZE; |
| 164 | memcpy(ctx->u.aes.nonce, key + key_len, CTR_RFC3686_NONCE_SIZE); |
| 165 | |
| 166 | return ccp_aes_setkey(tfm, key, key_len); |
| 167 | } |
| 168 | |
| 169 | static int ccp_aes_rfc3686_crypt(struct ablkcipher_request *req, bool encrypt) |
| 170 | { |
| 171 | struct ccp_ctx *ctx = crypto_tfm_ctx(req->base.tfm); |
| 172 | struct ccp_aes_req_ctx *rctx = ablkcipher_request_ctx(req); |
| 173 | u8 *iv; |
| 174 | |
| 175 | /* Initialize the CTR block */ |
| 176 | iv = rctx->rfc3686_iv; |
| 177 | memcpy(iv, ctx->u.aes.nonce, CTR_RFC3686_NONCE_SIZE); |
| 178 | |
| 179 | iv += CTR_RFC3686_NONCE_SIZE; |
| 180 | memcpy(iv, req->info, CTR_RFC3686_IV_SIZE); |
| 181 | |
| 182 | iv += CTR_RFC3686_IV_SIZE; |
| 183 | *(__be32 *)iv = cpu_to_be32(1); |
| 184 | |
| 185 | /* Point to the new IV */ |
| 186 | rctx->rfc3686_info = req->info; |
| 187 | req->info = rctx->rfc3686_iv; |
| 188 | |
| 189 | return ccp_aes_crypt(req, encrypt); |
| 190 | } |
| 191 | |
| 192 | static int ccp_aes_rfc3686_encrypt(struct ablkcipher_request *req) |
| 193 | { |
| 194 | return ccp_aes_rfc3686_crypt(req, true); |
| 195 | } |
| 196 | |
| 197 | static int ccp_aes_rfc3686_decrypt(struct ablkcipher_request *req) |
| 198 | { |
| 199 | return ccp_aes_rfc3686_crypt(req, false); |
| 200 | } |
| 201 | |
| 202 | static int ccp_aes_rfc3686_cra_init(struct crypto_tfm *tfm) |
| 203 | { |
| 204 | struct ccp_ctx *ctx = crypto_tfm_ctx(tfm); |
| 205 | |
| 206 | ctx->complete = ccp_aes_rfc3686_complete; |
| 207 | ctx->u.aes.key_len = 0; |
| 208 | |
| 209 | tfm->crt_ablkcipher.reqsize = sizeof(struct ccp_aes_req_ctx); |
| 210 | |
| 211 | return 0; |
| 212 | } |
| 213 | |
| 214 | static void ccp_aes_rfc3686_cra_exit(struct crypto_tfm *tfm) |
| 215 | { |
| 216 | } |
| 217 | |
| 218 | static struct crypto_alg ccp_aes_defaults = { |
| 219 | .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | |
| 220 | CRYPTO_ALG_ASYNC | |
| 221 | CRYPTO_ALG_KERN_DRIVER_ONLY | |
| 222 | CRYPTO_ALG_NEED_FALLBACK, |
| 223 | .cra_blocksize = AES_BLOCK_SIZE, |
| 224 | .cra_ctxsize = sizeof(struct ccp_ctx), |
| 225 | .cra_priority = CCP_CRA_PRIORITY, |
| 226 | .cra_type = &crypto_ablkcipher_type, |
| 227 | .cra_init = ccp_aes_cra_init, |
| 228 | .cra_exit = ccp_aes_cra_exit, |
| 229 | .cra_module = THIS_MODULE, |
| 230 | .cra_ablkcipher = { |
| 231 | .setkey = ccp_aes_setkey, |
| 232 | .encrypt = ccp_aes_encrypt, |
| 233 | .decrypt = ccp_aes_decrypt, |
| 234 | .min_keysize = AES_MIN_KEY_SIZE, |
| 235 | .max_keysize = AES_MAX_KEY_SIZE, |
| 236 | }, |
| 237 | }; |
| 238 | |
| 239 | static struct crypto_alg ccp_aes_rfc3686_defaults = { |
| 240 | .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | |
| 241 | CRYPTO_ALG_ASYNC | |
| 242 | CRYPTO_ALG_KERN_DRIVER_ONLY | |
| 243 | CRYPTO_ALG_NEED_FALLBACK, |
| 244 | .cra_blocksize = CTR_RFC3686_BLOCK_SIZE, |
| 245 | .cra_ctxsize = sizeof(struct ccp_ctx), |
| 246 | .cra_priority = CCP_CRA_PRIORITY, |
| 247 | .cra_type = &crypto_ablkcipher_type, |
| 248 | .cra_init = ccp_aes_rfc3686_cra_init, |
| 249 | .cra_exit = ccp_aes_rfc3686_cra_exit, |
| 250 | .cra_module = THIS_MODULE, |
| 251 | .cra_ablkcipher = { |
| 252 | .setkey = ccp_aes_rfc3686_setkey, |
| 253 | .encrypt = ccp_aes_rfc3686_encrypt, |
| 254 | .decrypt = ccp_aes_rfc3686_decrypt, |
| 255 | .min_keysize = AES_MIN_KEY_SIZE + CTR_RFC3686_NONCE_SIZE, |
| 256 | .max_keysize = AES_MAX_KEY_SIZE + CTR_RFC3686_NONCE_SIZE, |
| 257 | }, |
| 258 | }; |
| 259 | |
| 260 | struct ccp_aes_def { |
| 261 | enum ccp_aes_mode mode; |
Gary R Hook | c7019c4 | 2016-03-01 13:49:15 -0600 | [diff] [blame] | 262 | unsigned int version; |
Tom Lendacky | 2b78943 | 2013-11-12 11:46:28 -0600 | [diff] [blame] | 263 | const char *name; |
| 264 | const char *driver_name; |
| 265 | unsigned int blocksize; |
| 266 | unsigned int ivsize; |
| 267 | struct crypto_alg *alg_defaults; |
| 268 | }; |
| 269 | |
| 270 | static struct ccp_aes_def aes_algs[] = { |
| 271 | { |
| 272 | .mode = CCP_AES_MODE_ECB, |
Gary R Hook | c7019c4 | 2016-03-01 13:49:15 -0600 | [diff] [blame] | 273 | .version = CCP_VERSION(3, 0), |
Tom Lendacky | 2b78943 | 2013-11-12 11:46:28 -0600 | [diff] [blame] | 274 | .name = "ecb(aes)", |
| 275 | .driver_name = "ecb-aes-ccp", |
| 276 | .blocksize = AES_BLOCK_SIZE, |
| 277 | .ivsize = 0, |
| 278 | .alg_defaults = &ccp_aes_defaults, |
| 279 | }, |
| 280 | { |
| 281 | .mode = CCP_AES_MODE_CBC, |
Gary R Hook | c7019c4 | 2016-03-01 13:49:15 -0600 | [diff] [blame] | 282 | .version = CCP_VERSION(3, 0), |
Tom Lendacky | 2b78943 | 2013-11-12 11:46:28 -0600 | [diff] [blame] | 283 | .name = "cbc(aes)", |
| 284 | .driver_name = "cbc-aes-ccp", |
| 285 | .blocksize = AES_BLOCK_SIZE, |
| 286 | .ivsize = AES_BLOCK_SIZE, |
| 287 | .alg_defaults = &ccp_aes_defaults, |
| 288 | }, |
| 289 | { |
| 290 | .mode = CCP_AES_MODE_CFB, |
Gary R Hook | c7019c4 | 2016-03-01 13:49:15 -0600 | [diff] [blame] | 291 | .version = CCP_VERSION(3, 0), |
Tom Lendacky | 2b78943 | 2013-11-12 11:46:28 -0600 | [diff] [blame] | 292 | .name = "cfb(aes)", |
| 293 | .driver_name = "cfb-aes-ccp", |
| 294 | .blocksize = AES_BLOCK_SIZE, |
| 295 | .ivsize = AES_BLOCK_SIZE, |
| 296 | .alg_defaults = &ccp_aes_defaults, |
| 297 | }, |
| 298 | { |
| 299 | .mode = CCP_AES_MODE_OFB, |
Gary R Hook | c7019c4 | 2016-03-01 13:49:15 -0600 | [diff] [blame] | 300 | .version = CCP_VERSION(3, 0), |
Tom Lendacky | 2b78943 | 2013-11-12 11:46:28 -0600 | [diff] [blame] | 301 | .name = "ofb(aes)", |
| 302 | .driver_name = "ofb-aes-ccp", |
| 303 | .blocksize = 1, |
| 304 | .ivsize = AES_BLOCK_SIZE, |
| 305 | .alg_defaults = &ccp_aes_defaults, |
| 306 | }, |
| 307 | { |
| 308 | .mode = CCP_AES_MODE_CTR, |
Gary R Hook | c7019c4 | 2016-03-01 13:49:15 -0600 | [diff] [blame] | 309 | .version = CCP_VERSION(3, 0), |
Tom Lendacky | 2b78943 | 2013-11-12 11:46:28 -0600 | [diff] [blame] | 310 | .name = "ctr(aes)", |
| 311 | .driver_name = "ctr-aes-ccp", |
| 312 | .blocksize = 1, |
| 313 | .ivsize = AES_BLOCK_SIZE, |
| 314 | .alg_defaults = &ccp_aes_defaults, |
| 315 | }, |
| 316 | { |
| 317 | .mode = CCP_AES_MODE_CTR, |
Gary R Hook | c7019c4 | 2016-03-01 13:49:15 -0600 | [diff] [blame] | 318 | .version = CCP_VERSION(3, 0), |
Tom Lendacky | 2b78943 | 2013-11-12 11:46:28 -0600 | [diff] [blame] | 319 | .name = "rfc3686(ctr(aes))", |
| 320 | .driver_name = "rfc3686-ctr-aes-ccp", |
| 321 | .blocksize = 1, |
| 322 | .ivsize = CTR_RFC3686_IV_SIZE, |
| 323 | .alg_defaults = &ccp_aes_rfc3686_defaults, |
| 324 | }, |
| 325 | }; |
| 326 | |
| 327 | static int ccp_register_aes_alg(struct list_head *head, |
| 328 | const struct ccp_aes_def *def) |
| 329 | { |
| 330 | struct ccp_crypto_ablkcipher_alg *ccp_alg; |
| 331 | struct crypto_alg *alg; |
| 332 | int ret; |
| 333 | |
| 334 | ccp_alg = kzalloc(sizeof(*ccp_alg), GFP_KERNEL); |
| 335 | if (!ccp_alg) |
| 336 | return -ENOMEM; |
| 337 | |
| 338 | INIT_LIST_HEAD(&ccp_alg->entry); |
| 339 | |
| 340 | ccp_alg->mode = def->mode; |
| 341 | |
| 342 | /* Copy the defaults and override as necessary */ |
| 343 | alg = &ccp_alg->alg; |
Fengguang Wu | d1dd206 | 2013-12-09 20:08:19 +0800 | [diff] [blame] | 344 | *alg = *def->alg_defaults; |
Tom Lendacky | 2b78943 | 2013-11-12 11:46:28 -0600 | [diff] [blame] | 345 | snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s", def->name); |
| 346 | snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s", |
| 347 | def->driver_name); |
| 348 | alg->cra_blocksize = def->blocksize; |
| 349 | alg->cra_ablkcipher.ivsize = def->ivsize; |
| 350 | |
| 351 | ret = crypto_register_alg(alg); |
| 352 | if (ret) { |
| 353 | pr_err("%s ablkcipher algorithm registration error (%d)\n", |
Tom Lendacky | 8db8846 | 2015-02-03 13:07:05 -0600 | [diff] [blame] | 354 | alg->cra_name, ret); |
Tom Lendacky | 2b78943 | 2013-11-12 11:46:28 -0600 | [diff] [blame] | 355 | kfree(ccp_alg); |
| 356 | return ret; |
| 357 | } |
| 358 | |
| 359 | list_add(&ccp_alg->entry, head); |
| 360 | |
| 361 | return 0; |
| 362 | } |
| 363 | |
| 364 | int ccp_register_aes_algs(struct list_head *head) |
| 365 | { |
| 366 | int i, ret; |
Gary R Hook | c7019c4 | 2016-03-01 13:49:15 -0600 | [diff] [blame] | 367 | unsigned int ccpversion = ccp_version(); |
Tom Lendacky | 2b78943 | 2013-11-12 11:46:28 -0600 | [diff] [blame] | 368 | |
| 369 | for (i = 0; i < ARRAY_SIZE(aes_algs); i++) { |
Gary R Hook | c7019c4 | 2016-03-01 13:49:15 -0600 | [diff] [blame] | 370 | if (aes_algs[i].version > ccpversion) |
| 371 | continue; |
Tom Lendacky | 2b78943 | 2013-11-12 11:46:28 -0600 | [diff] [blame] | 372 | ret = ccp_register_aes_alg(head, &aes_algs[i]); |
| 373 | if (ret) |
| 374 | return ret; |
| 375 | } |
| 376 | |
| 377 | return 0; |
| 378 | } |