blob: ff149e176f649344276e8fb91679d78b8368be11 [file] [log] [blame]
Tadeusz Strukb3416fb2014-06-05 13:44:04 -07001/*
2 This file is provided under a dual BSD/GPLv2 license. When using or
3 redistributing this file, you may do so under either license.
4
5 GPL LICENSE SUMMARY
6 Copyright(c) 2014 Intel Corporation.
7 This program is free software; you can redistribute it and/or modify
8 it under the terms of version 2 of the GNU General Public License as
9 published by the Free Software Foundation.
10
11 This program is distributed in the hope that it will be useful, but
12 WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 General Public License for more details.
15
16 Contact Information:
17 qat-linux@intel.com
18
19 BSD LICENSE
20 Copyright(c) 2014 Intel Corporation.
21 Redistribution and use in source and binary forms, with or without
22 modification, are permitted provided that the following conditions
23 are met:
24
25 * Redistributions of source code must retain the above copyright
26 notice, this list of conditions and the following disclaimer.
27 * Redistributions in binary form must reproduce the above copyright
28 notice, this list of conditions and the following disclaimer in
29 the documentation and/or other materials provided with the
30 distribution.
31 * Neither the name of Intel Corporation nor the names of its
32 contributors may be used to endorse or promote products derived
33 from this software without specific prior written permission.
34
35 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
36 "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
37 LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
38 A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
39 OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
40 SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
41 LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
42 DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
43 THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
44 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
45 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
46*/
47#include <linux/slab.h>
Pingchao Yangb0272272015-12-04 16:56:23 -080048#include <linux/delay.h>
Tadeusz Strukb3416fb2014-06-05 13:44:04 -070049
50#include "adf_accel_devices.h"
51#include "adf_common_drv.h"
52#include "icp_qat_hal.h"
53#include "icp_qat_uclo.h"
54
Pingchao Yangb0272272015-12-04 16:56:23 -080055#define BAD_REGADDR 0xffff
56#define MAX_RETRY_TIMES 10000
57#define INIT_CTX_ARB_VALUE 0x0
Tadeusz Strukb3416fb2014-06-05 13:44:04 -070058#define INIT_CTX_ENABLE_VALUE 0x0
Pingchao Yangb0272272015-12-04 16:56:23 -080059#define INIT_PC_VALUE 0x0
Tadeusz Strukb3416fb2014-06-05 13:44:04 -070060#define INIT_WAKEUP_EVENTS_VALUE 0x1
61#define INIT_SIG_EVENTS_VALUE 0x1
62#define INIT_CCENABLE_VALUE 0x2000
Pingchao Yangb0272272015-12-04 16:56:23 -080063#define RST_CSR_QAT_LSB 20
Tadeusz Strukb3416fb2014-06-05 13:44:04 -070064#define RST_CSR_AE_LSB 0
65#define MC_TIMESTAMP_ENABLE (0x1 << 7)
66
67#define IGNORE_W1C_MASK ((~(1 << CE_BREAKPOINT_BITPOS)) & \
68 (~(1 << CE_CNTL_STORE_PARITY_ERROR_BITPOS)) & \
69 (~(1 << CE_REG_PAR_ERR_BITPOS)))
70#define INSERT_IMMED_GPRA_CONST(inst, const_val) \
71 (inst = ((inst & 0xFFFF00C03FFull) | \
72 ((((const_val) << 12) & 0x0FF00000ull) | \
73 (((const_val) << 10) & 0x0003FC00ull))))
74#define INSERT_IMMED_GPRB_CONST(inst, const_val) \
75 (inst = ((inst & 0xFFFF00FFF00ull) | \
76 ((((const_val) << 12) & 0x0FF00000ull) | \
77 (((const_val) << 0) & 0x000000FFull))))
78
79#define AE(handle, ae) handle->hal_handle->aes[ae]
80
81static const uint64_t inst_4b[] = {
82 0x0F0400C0000ull, 0x0F4400C0000ull, 0x0F040000300ull, 0x0F440000300ull,
83 0x0FC066C0000ull, 0x0F0000C0300ull, 0x0F0000C0300ull, 0x0F0000C0300ull,
84 0x0A021000000ull
85};
86
87static const uint64_t inst[] = {
88 0x0F0000C0000ull, 0x0F000000380ull, 0x0D805000011ull, 0x0FC082C0300ull,
89 0x0F0000C0300ull, 0x0F0000C0300ull, 0x0F0000C0300ull, 0x0F0000C0300ull,
90 0x0A0643C0000ull, 0x0BAC0000301ull, 0x0D802000101ull, 0x0F0000C0001ull,
91 0x0FC066C0001ull, 0x0F0000C0300ull, 0x0F0000C0300ull, 0x0F0000C0300ull,
92 0x0F000400300ull, 0x0A0610C0000ull, 0x0BAC0000301ull, 0x0D804400101ull,
93 0x0A0580C0000ull, 0x0A0581C0000ull, 0x0A0582C0000ull, 0x0A0583C0000ull,
94 0x0A0584C0000ull, 0x0A0585C0000ull, 0x0A0586C0000ull, 0x0A0587C0000ull,
95 0x0A0588C0000ull, 0x0A0589C0000ull, 0x0A058AC0000ull, 0x0A058BC0000ull,
96 0x0A058CC0000ull, 0x0A058DC0000ull, 0x0A058EC0000ull, 0x0A058FC0000ull,
97 0x0A05C0C0000ull, 0x0A05C1C0000ull, 0x0A05C2C0000ull, 0x0A05C3C0000ull,
98 0x0A05C4C0000ull, 0x0A05C5C0000ull, 0x0A05C6C0000ull, 0x0A05C7C0000ull,
99 0x0A05C8C0000ull, 0x0A05C9C0000ull, 0x0A05CAC0000ull, 0x0A05CBC0000ull,
100 0x0A05CCC0000ull, 0x0A05CDC0000ull, 0x0A05CEC0000ull, 0x0A05CFC0000ull,
101 0x0A0400C0000ull, 0x0B0400C0000ull, 0x0A0401C0000ull, 0x0B0401C0000ull,
102 0x0A0402C0000ull, 0x0B0402C0000ull, 0x0A0403C0000ull, 0x0B0403C0000ull,
103 0x0A0404C0000ull, 0x0B0404C0000ull, 0x0A0405C0000ull, 0x0B0405C0000ull,
104 0x0A0406C0000ull, 0x0B0406C0000ull, 0x0A0407C0000ull, 0x0B0407C0000ull,
105 0x0A0408C0000ull, 0x0B0408C0000ull, 0x0A0409C0000ull, 0x0B0409C0000ull,
106 0x0A040AC0000ull, 0x0B040AC0000ull, 0x0A040BC0000ull, 0x0B040BC0000ull,
107 0x0A040CC0000ull, 0x0B040CC0000ull, 0x0A040DC0000ull, 0x0B040DC0000ull,
108 0x0A040EC0000ull, 0x0B040EC0000ull, 0x0A040FC0000ull, 0x0B040FC0000ull,
109 0x0D81581C010ull, 0x0E000010000ull, 0x0E000010000ull,
110};
111
112void qat_hal_set_live_ctx(struct icp_qat_fw_loader_handle *handle,
113 unsigned char ae, unsigned int ctx_mask)
114{
115 AE(handle, ae).live_ctx_mask = ctx_mask;
116}
117
118#define CSR_RETRY_TIMES 500
119static int qat_hal_rd_ae_csr(struct icp_qat_fw_loader_handle *handle,
Arnd Bergmann8c9478a2017-12-11 13:07:00 +0100120 unsigned char ae, unsigned int csr)
Tadeusz Strukb3416fb2014-06-05 13:44:04 -0700121{
122 unsigned int iterations = CSR_RETRY_TIMES;
Arnd Bergmann8c9478a2017-12-11 13:07:00 +0100123 int value;
Tadeusz Strukb3416fb2014-06-05 13:44:04 -0700124
125 do {
Arnd Bergmann8c9478a2017-12-11 13:07:00 +0100126 value = GET_AE_CSR(handle, ae, csr);
Tadeusz Strukb3416fb2014-06-05 13:44:04 -0700127 if (!(GET_AE_CSR(handle, ae, LOCAL_CSR_STATUS) & LCS_STATUS))
Arnd Bergmann8c9478a2017-12-11 13:07:00 +0100128 return value;
Tadeusz Strukb3416fb2014-06-05 13:44:04 -0700129 } while (iterations--);
130
131 pr_err("QAT: Read CSR timeout\n");
Arnd Bergmann8c9478a2017-12-11 13:07:00 +0100132 return 0;
Tadeusz Strukb3416fb2014-06-05 13:44:04 -0700133}
134
135static int qat_hal_wr_ae_csr(struct icp_qat_fw_loader_handle *handle,
136 unsigned char ae, unsigned int csr,
137 unsigned int value)
138{
139 unsigned int iterations = CSR_RETRY_TIMES;
140
141 do {
142 SET_AE_CSR(handle, ae, csr, value);
143 if (!(GET_AE_CSR(handle, ae, LOCAL_CSR_STATUS) & LCS_STATUS))
144 return 0;
145 } while (iterations--);
146
147 pr_err("QAT: Write CSR Timeout\n");
148 return -EFAULT;
149}
150
151static void qat_hal_get_wakeup_event(struct icp_qat_fw_loader_handle *handle,
152 unsigned char ae, unsigned char ctx,
153 unsigned int *events)
154{
155 unsigned int cur_ctx;
156
Arnd Bergmann8c9478a2017-12-11 13:07:00 +0100157 cur_ctx = qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER);
Tadeusz Strukb3416fb2014-06-05 13:44:04 -0700158 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx);
Arnd Bergmann8c9478a2017-12-11 13:07:00 +0100159 *events = qat_hal_rd_ae_csr(handle, ae, CTX_WAKEUP_EVENTS_INDIRECT);
Tadeusz Strukb3416fb2014-06-05 13:44:04 -0700160 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx);
161}
162
163static int qat_hal_wait_cycles(struct icp_qat_fw_loader_handle *handle,
164 unsigned char ae, unsigned int cycles,
165 int chk_inactive)
166{
167 unsigned int base_cnt = 0, cur_cnt = 0;
168 unsigned int csr = (1 << ACS_ABO_BITPOS);
169 int times = MAX_RETRY_TIMES;
170 int elapsed_cycles = 0;
171
Arnd Bergmann8c9478a2017-12-11 13:07:00 +0100172 base_cnt = qat_hal_rd_ae_csr(handle, ae, PROFILE_COUNT);
Tadeusz Strukb3416fb2014-06-05 13:44:04 -0700173 base_cnt &= 0xffff;
174 while ((int)cycles > elapsed_cycles && times--) {
175 if (chk_inactive)
Arnd Bergmann8c9478a2017-12-11 13:07:00 +0100176 csr = qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS);
Tadeusz Strukb3416fb2014-06-05 13:44:04 -0700177
Arnd Bergmann8c9478a2017-12-11 13:07:00 +0100178 cur_cnt = qat_hal_rd_ae_csr(handle, ae, PROFILE_COUNT);
Tadeusz Strukb3416fb2014-06-05 13:44:04 -0700179 cur_cnt &= 0xffff;
180 elapsed_cycles = cur_cnt - base_cnt;
181
182 if (elapsed_cycles < 0)
183 elapsed_cycles += 0x10000;
184
185 /* ensure at least 8 time cycles elapsed in wait_cycles */
186 if (elapsed_cycles >= 8 && !(csr & (1 << ACS_ABO_BITPOS)))
187 return 0;
188 }
Pingchao Yang51d77dd2015-12-16 14:09:50 +0800189 if (times < 0) {
Tadeusz Strukb3416fb2014-06-05 13:44:04 -0700190 pr_err("QAT: wait_num_cycles time out\n");
191 return -EFAULT;
192 }
193 return 0;
194}
195
196#define CLR_BIT(wrd, bit) (wrd & ~(1 << bit))
197#define SET_BIT(wrd, bit) (wrd | 1 << bit)
198
199int qat_hal_set_ae_ctx_mode(struct icp_qat_fw_loader_handle *handle,
200 unsigned char ae, unsigned char mode)
201{
202 unsigned int csr, new_csr;
203
204 if ((mode != 4) && (mode != 8)) {
205 pr_err("QAT: bad ctx mode=%d\n", mode);
206 return -EINVAL;
207 }
208
209 /* Sets the accelaration engine context mode to either four or eight */
Arnd Bergmann8c9478a2017-12-11 13:07:00 +0100210 csr = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
Tadeusz Strukb3416fb2014-06-05 13:44:04 -0700211 csr = IGNORE_W1C_MASK & csr;
212 new_csr = (mode == 4) ?
213 SET_BIT(csr, CE_INUSE_CONTEXTS_BITPOS) :
214 CLR_BIT(csr, CE_INUSE_CONTEXTS_BITPOS);
215 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, new_csr);
216 return 0;
217}
218
219int qat_hal_set_ae_nn_mode(struct icp_qat_fw_loader_handle *handle,
220 unsigned char ae, unsigned char mode)
221{
222 unsigned int csr, new_csr;
223
Arnd Bergmann8c9478a2017-12-11 13:07:00 +0100224 csr = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
Tadeusz Strukb3416fb2014-06-05 13:44:04 -0700225 csr &= IGNORE_W1C_MASK;
226
227 new_csr = (mode) ?
228 SET_BIT(csr, CE_NN_MODE_BITPOS) :
229 CLR_BIT(csr, CE_NN_MODE_BITPOS);
230
231 if (new_csr != csr)
232 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, new_csr);
233
234 return 0;
235}
236
237int qat_hal_set_ae_lm_mode(struct icp_qat_fw_loader_handle *handle,
238 unsigned char ae, enum icp_qat_uof_regtype lm_type,
239 unsigned char mode)
240{
241 unsigned int csr, new_csr;
242
Arnd Bergmann8c9478a2017-12-11 13:07:00 +0100243 csr = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
Tadeusz Strukb3416fb2014-06-05 13:44:04 -0700244 csr &= IGNORE_W1C_MASK;
245 switch (lm_type) {
246 case ICP_LMEM0:
247 new_csr = (mode) ?
248 SET_BIT(csr, CE_LMADDR_0_GLOBAL_BITPOS) :
249 CLR_BIT(csr, CE_LMADDR_0_GLOBAL_BITPOS);
250 break;
251 case ICP_LMEM1:
252 new_csr = (mode) ?
253 SET_BIT(csr, CE_LMADDR_1_GLOBAL_BITPOS) :
254 CLR_BIT(csr, CE_LMADDR_1_GLOBAL_BITPOS);
255 break;
256 default:
257 pr_err("QAT: lmType = 0x%x\n", lm_type);
258 return -EINVAL;
259 }
260
261 if (new_csr != csr)
262 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, new_csr);
263 return 0;
264}
265
266static unsigned short qat_hal_get_reg_addr(unsigned int type,
267 unsigned short reg_num)
268{
269 unsigned short reg_addr;
Tadeusz Strukd65071e2014-06-24 15:19:34 -0700270
Tadeusz Strukb3416fb2014-06-05 13:44:04 -0700271 switch (type) {
272 case ICP_GPA_ABS:
273 case ICP_GPB_ABS:
274 reg_addr = 0x80 | (reg_num & 0x7f);
275 break;
276 case ICP_GPA_REL:
277 case ICP_GPB_REL:
278 reg_addr = reg_num & 0x1f;
279 break;
280 case ICP_SR_RD_REL:
281 case ICP_SR_WR_REL:
282 case ICP_SR_REL:
283 reg_addr = 0x180 | (reg_num & 0x1f);
284 break;
285 case ICP_SR_ABS:
286 reg_addr = 0x140 | ((reg_num & 0x3) << 1);
287 break;
288 case ICP_DR_RD_REL:
289 case ICP_DR_WR_REL:
290 case ICP_DR_REL:
291 reg_addr = 0x1c0 | (reg_num & 0x1f);
292 break;
293 case ICP_DR_ABS:
294 reg_addr = 0x100 | ((reg_num & 0x3) << 1);
295 break;
296 case ICP_NEIGH_REL:
297 reg_addr = 0x280 | (reg_num & 0x1f);
298 break;
299 case ICP_LMEM0:
300 reg_addr = 0x200;
301 break;
302 case ICP_LMEM1:
303 reg_addr = 0x220;
304 break;
305 case ICP_NO_DEST:
306 reg_addr = 0x300 | (reg_num & 0xff);
307 break;
308 default:
309 reg_addr = BAD_REGADDR;
310 break;
311 }
312 return reg_addr;
313}
314
315void qat_hal_reset(struct icp_qat_fw_loader_handle *handle)
316{
317 unsigned int ae_reset_csr;
318
319 ae_reset_csr = GET_GLB_CSR(handle, ICP_RESET);
320 ae_reset_csr |= handle->hal_handle->ae_mask << RST_CSR_AE_LSB;
321 ae_reset_csr |= handle->hal_handle->slice_mask << RST_CSR_QAT_LSB;
322 SET_GLB_CSR(handle, ICP_RESET, ae_reset_csr);
323}
324
325static void qat_hal_wr_indr_csr(struct icp_qat_fw_loader_handle *handle,
326 unsigned char ae, unsigned int ctx_mask,
327 unsigned int ae_csr, unsigned int csr_val)
328{
329 unsigned int ctx, cur_ctx;
330
Arnd Bergmann8c9478a2017-12-11 13:07:00 +0100331 cur_ctx = qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER);
Tadeusz Strukb3416fb2014-06-05 13:44:04 -0700332
333 for (ctx = 0; ctx < ICP_QAT_UCLO_MAX_CTX; ctx++) {
334 if (!(ctx_mask & (1 << ctx)))
335 continue;
336 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx);
337 qat_hal_wr_ae_csr(handle, ae, ae_csr, csr_val);
338 }
339
340 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx);
341}
342
Arnd Bergmann8c9478a2017-12-11 13:07:00 +0100343static unsigned int qat_hal_rd_indr_csr(struct icp_qat_fw_loader_handle *handle,
Tadeusz Strukb3416fb2014-06-05 13:44:04 -0700344 unsigned char ae, unsigned char ctx,
Arnd Bergmann8c9478a2017-12-11 13:07:00 +0100345 unsigned int ae_csr)
Tadeusz Strukb3416fb2014-06-05 13:44:04 -0700346{
Arnd Bergmann8c9478a2017-12-11 13:07:00 +0100347 unsigned int cur_ctx, csr_val;
Tadeusz Strukb3416fb2014-06-05 13:44:04 -0700348
Arnd Bergmann8c9478a2017-12-11 13:07:00 +0100349 cur_ctx = qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER);
Tadeusz Strukb3416fb2014-06-05 13:44:04 -0700350 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx);
Arnd Bergmann8c9478a2017-12-11 13:07:00 +0100351 csr_val = qat_hal_rd_ae_csr(handle, ae, ae_csr);
Tadeusz Strukb3416fb2014-06-05 13:44:04 -0700352 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx);
Arnd Bergmann8c9478a2017-12-11 13:07:00 +0100353
354 return csr_val;
Tadeusz Strukb3416fb2014-06-05 13:44:04 -0700355}
356
357static void qat_hal_put_sig_event(struct icp_qat_fw_loader_handle *handle,
358 unsigned char ae, unsigned int ctx_mask,
359 unsigned int events)
360{
361 unsigned int ctx, cur_ctx;
362
Arnd Bergmann8c9478a2017-12-11 13:07:00 +0100363 cur_ctx = qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER);
Tadeusz Strukb3416fb2014-06-05 13:44:04 -0700364 for (ctx = 0; ctx < ICP_QAT_UCLO_MAX_CTX; ctx++) {
365 if (!(ctx_mask & (1 << ctx)))
366 continue;
367 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx);
368 qat_hal_wr_ae_csr(handle, ae, CTX_SIG_EVENTS_INDIRECT, events);
369 }
370 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx);
371}
372
373static void qat_hal_put_wakeup_event(struct icp_qat_fw_loader_handle *handle,
374 unsigned char ae, unsigned int ctx_mask,
375 unsigned int events)
376{
377 unsigned int ctx, cur_ctx;
378
Arnd Bergmann8c9478a2017-12-11 13:07:00 +0100379 cur_ctx = qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER);
Tadeusz Strukb3416fb2014-06-05 13:44:04 -0700380 for (ctx = 0; ctx < ICP_QAT_UCLO_MAX_CTX; ctx++) {
381 if (!(ctx_mask & (1 << ctx)))
382 continue;
383 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx);
384 qat_hal_wr_ae_csr(handle, ae, CTX_WAKEUP_EVENTS_INDIRECT,
385 events);
386 }
387 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx);
388}
389
390static int qat_hal_check_ae_alive(struct icp_qat_fw_loader_handle *handle)
391{
392 unsigned int base_cnt, cur_cnt;
393 unsigned char ae;
Pingchao Yangc0e77a12016-01-07 09:39:46 +0800394 int times = MAX_RETRY_TIMES;
Tadeusz Strukb3416fb2014-06-05 13:44:04 -0700395
396 for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) {
Arnd Bergmann8c9478a2017-12-11 13:07:00 +0100397 base_cnt = qat_hal_rd_ae_csr(handle, ae, PROFILE_COUNT);
Tadeusz Strukb3416fb2014-06-05 13:44:04 -0700398 base_cnt &= 0xffff;
399
400 do {
Arnd Bergmann8c9478a2017-12-11 13:07:00 +0100401 cur_cnt = qat_hal_rd_ae_csr(handle, ae, PROFILE_COUNT);
Tadeusz Strukb3416fb2014-06-05 13:44:04 -0700402 cur_cnt &= 0xffff;
403 } while (times-- && (cur_cnt == base_cnt));
404
Pingchao Yangc0e77a12016-01-07 09:39:46 +0800405 if (times < 0) {
Tadeusz Strukb3416fb2014-06-05 13:44:04 -0700406 pr_err("QAT: AE%d is inactive!!\n", ae);
407 return -EFAULT;
408 }
409 }
410
411 return 0;
412}
413
Pingchao Yangb0272272015-12-04 16:56:23 -0800414int qat_hal_check_ae_active(struct icp_qat_fw_loader_handle *handle,
415 unsigned int ae)
416{
417 unsigned int enable = 0, active = 0;
418
Arnd Bergmann8c9478a2017-12-11 13:07:00 +0100419 enable = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
420 active = qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS);
Pingchao Yang46621e62015-12-16 10:39:40 +0800421 if ((enable & (0xff << CE_ENABLE_BITPOS)) ||
Pingchao Yangb0272272015-12-04 16:56:23 -0800422 (active & (1 << ACS_ABO_BITPOS)))
423 return 1;
424 else
425 return 0;
426}
427
Tadeusz Strukb3416fb2014-06-05 13:44:04 -0700428static void qat_hal_reset_timestamp(struct icp_qat_fw_loader_handle *handle)
429{
430 unsigned int misc_ctl;
431 unsigned char ae;
432
433 /* stop the timestamp timers */
434 misc_ctl = GET_GLB_CSR(handle, MISC_CONTROL);
435 if (misc_ctl & MC_TIMESTAMP_ENABLE)
436 SET_GLB_CSR(handle, MISC_CONTROL, misc_ctl &
437 (~MC_TIMESTAMP_ENABLE));
438
Tadeusz Struk9a147cb2014-07-25 15:55:46 -0700439 for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) {
Tadeusz Strukb3416fb2014-06-05 13:44:04 -0700440 qat_hal_wr_ae_csr(handle, ae, TIMESTAMP_LOW, 0);
441 qat_hal_wr_ae_csr(handle, ae, TIMESTAMP_HIGH, 0);
442 }
443 /* start timestamp timers */
444 SET_GLB_CSR(handle, MISC_CONTROL, misc_ctl | MC_TIMESTAMP_ENABLE);
445}
446
Allan, Bruce Waf6f2a72015-03-31 09:30:45 -0700447#define ESRAM_AUTO_TINIT BIT(2)
448#define ESRAM_AUTO_TINIT_DONE BIT(3)
Tadeusz Strukb3416fb2014-06-05 13:44:04 -0700449#define ESRAM_AUTO_INIT_USED_CYCLES (1640)
450#define ESRAM_AUTO_INIT_CSR_OFFSET 0xC1C
451static int qat_hal_init_esram(struct icp_qat_fw_loader_handle *handle)
452{
Pingchao Yangb0272272015-12-04 16:56:23 -0800453 void __iomem *csr_addr =
454 (void __iomem *)((uintptr_t)handle->hal_ep_csr_addr_v +
455 ESRAM_AUTO_INIT_CSR_OFFSET);
Pingchao Yangc0e77a12016-01-07 09:39:46 +0800456 unsigned int csr_val;
457 int times = 30;
Tadeusz Strukb3416fb2014-06-05 13:44:04 -0700458
Giovanni Cabiddu685ce062016-12-22 15:00:24 +0000459 if (handle->pci_dev->device != ADF_DH895XCC_PCI_DEVICE_ID)
Tadeusz Struk70401f42016-01-08 10:19:54 -0800460 return 0;
461
Tadeusz Strukb3416fb2014-06-05 13:44:04 -0700462 csr_val = ADF_CSR_RD(csr_addr, 0);
463 if ((csr_val & ESRAM_AUTO_TINIT) && (csr_val & ESRAM_AUTO_TINIT_DONE))
464 return 0;
465
466 csr_val = ADF_CSR_RD(csr_addr, 0);
467 csr_val |= ESRAM_AUTO_TINIT;
468 ADF_CSR_WR(csr_addr, 0, csr_val);
469
470 do {
471 qat_hal_wait_cycles(handle, 0, ESRAM_AUTO_INIT_USED_CYCLES, 0);
472 csr_val = ADF_CSR_RD(csr_addr, 0);
473 } while (!(csr_val & ESRAM_AUTO_TINIT_DONE) && times--);
Pingchao Yangc0e77a12016-01-07 09:39:46 +0800474 if ((times < 0)) {
Tadeusz Strukb3416fb2014-06-05 13:44:04 -0700475 pr_err("QAT: Fail to init eSram!\n");
476 return -EFAULT;
477 }
478 return 0;
479}
480
481#define SHRAM_INIT_CYCLES 2060
482int qat_hal_clr_reset(struct icp_qat_fw_loader_handle *handle)
483{
484 unsigned int ae_reset_csr;
485 unsigned char ae;
486 unsigned int clk_csr;
487 unsigned int times = 100;
488 unsigned int csr;
489
490 /* write to the reset csr */
491 ae_reset_csr = GET_GLB_CSR(handle, ICP_RESET);
492 ae_reset_csr &= ~(handle->hal_handle->ae_mask << RST_CSR_AE_LSB);
493 ae_reset_csr &= ~(handle->hal_handle->slice_mask << RST_CSR_QAT_LSB);
494 do {
495 SET_GLB_CSR(handle, ICP_RESET, ae_reset_csr);
496 if (!(times--))
497 goto out_err;
498 csr = GET_GLB_CSR(handle, ICP_RESET);
499 } while ((handle->hal_handle->ae_mask |
500 (handle->hal_handle->slice_mask << RST_CSR_QAT_LSB)) & csr);
501 /* enable clock */
502 clk_csr = GET_GLB_CSR(handle, ICP_GLOBAL_CLK_ENABLE);
503 clk_csr |= handle->hal_handle->ae_mask << 0;
504 clk_csr |= handle->hal_handle->slice_mask << 20;
505 SET_GLB_CSR(handle, ICP_GLOBAL_CLK_ENABLE, clk_csr);
506 if (qat_hal_check_ae_alive(handle))
507 goto out_err;
508
509 /* Set undefined power-up/reset states to reasonable default values */
Tadeusz Struk9a147cb2014-07-25 15:55:46 -0700510 for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) {
Tadeusz Strukb3416fb2014-06-05 13:44:04 -0700511 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES,
512 INIT_CTX_ENABLE_VALUE);
513 qat_hal_wr_indr_csr(handle, ae, ICP_QAT_UCLO_AE_ALL_CTX,
514 CTX_STS_INDIRECT,
515 handle->hal_handle->upc_mask &
516 INIT_PC_VALUE);
517 qat_hal_wr_ae_csr(handle, ae, CTX_ARB_CNTL, INIT_CTX_ARB_VALUE);
518 qat_hal_wr_ae_csr(handle, ae, CC_ENABLE, INIT_CCENABLE_VALUE);
519 qat_hal_put_wakeup_event(handle, ae,
520 ICP_QAT_UCLO_AE_ALL_CTX,
521 INIT_WAKEUP_EVENTS_VALUE);
522 qat_hal_put_sig_event(handle, ae,
523 ICP_QAT_UCLO_AE_ALL_CTX,
524 INIT_SIG_EVENTS_VALUE);
525 }
526 if (qat_hal_init_esram(handle))
527 goto out_err;
528 if (qat_hal_wait_cycles(handle, 0, SHRAM_INIT_CYCLES, 0))
529 goto out_err;
530 qat_hal_reset_timestamp(handle);
531
532 return 0;
533out_err:
534 pr_err("QAT: failed to get device out of reset\n");
535 return -EFAULT;
536}
537
538static void qat_hal_disable_ctx(struct icp_qat_fw_loader_handle *handle,
539 unsigned char ae, unsigned int ctx_mask)
540{
541 unsigned int ctx;
542
Arnd Bergmann8c9478a2017-12-11 13:07:00 +0100543 ctx = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
Tadeusz Strukb3416fb2014-06-05 13:44:04 -0700544 ctx &= IGNORE_W1C_MASK &
545 (~((ctx_mask & ICP_QAT_UCLO_AE_ALL_CTX) << CE_ENABLE_BITPOS));
546 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx);
547}
548
549static uint64_t qat_hal_parity_64bit(uint64_t word)
550{
551 word ^= word >> 1;
552 word ^= word >> 2;
553 word ^= word >> 4;
554 word ^= word >> 8;
555 word ^= word >> 16;
556 word ^= word >> 32;
557 return word & 1;
558}
559
560static uint64_t qat_hal_set_uword_ecc(uint64_t uword)
561{
562 uint64_t bit0_mask = 0xff800007fffULL, bit1_mask = 0x1f801ff801fULL,
563 bit2_mask = 0xe387e0781e1ULL, bit3_mask = 0x7cb8e388e22ULL,
564 bit4_mask = 0xaf5b2c93244ULL, bit5_mask = 0xf56d5525488ULL,
565 bit6_mask = 0xdaf69a46910ULL;
566
567 /* clear the ecc bits */
568 uword &= ~(0x7fULL << 0x2C);
569 uword |= qat_hal_parity_64bit(bit0_mask & uword) << 0x2C;
570 uword |= qat_hal_parity_64bit(bit1_mask & uword) << 0x2D;
571 uword |= qat_hal_parity_64bit(bit2_mask & uword) << 0x2E;
572 uword |= qat_hal_parity_64bit(bit3_mask & uword) << 0x2F;
573 uword |= qat_hal_parity_64bit(bit4_mask & uword) << 0x30;
574 uword |= qat_hal_parity_64bit(bit5_mask & uword) << 0x31;
575 uword |= qat_hal_parity_64bit(bit6_mask & uword) << 0x32;
576 return uword;
577}
578
579void qat_hal_wr_uwords(struct icp_qat_fw_loader_handle *handle,
580 unsigned char ae, unsigned int uaddr,
581 unsigned int words_num, uint64_t *uword)
582{
583 unsigned int ustore_addr;
584 unsigned int i;
585
Arnd Bergmann8c9478a2017-12-11 13:07:00 +0100586 ustore_addr = qat_hal_rd_ae_csr(handle, ae, USTORE_ADDRESS);
Tadeusz Strukb3416fb2014-06-05 13:44:04 -0700587 uaddr |= UA_ECS;
588 qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr);
589 for (i = 0; i < words_num; i++) {
590 unsigned int uwrd_lo, uwrd_hi;
591 uint64_t tmp;
592
593 tmp = qat_hal_set_uword_ecc(uword[i]);
594 uwrd_lo = (unsigned int)(tmp & 0xffffffff);
595 uwrd_hi = (unsigned int)(tmp >> 0x20);
596 qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_LOWER, uwrd_lo);
597 qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_UPPER, uwrd_hi);
598 }
599 qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, ustore_addr);
600}
601
602static void qat_hal_enable_ctx(struct icp_qat_fw_loader_handle *handle,
603 unsigned char ae, unsigned int ctx_mask)
604{
605 unsigned int ctx;
606
Arnd Bergmann8c9478a2017-12-11 13:07:00 +0100607 ctx = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
Tadeusz Strukb3416fb2014-06-05 13:44:04 -0700608 ctx &= IGNORE_W1C_MASK;
609 ctx_mask &= (ctx & CE_INUSE_CONTEXTS) ? 0x55 : 0xFF;
610 ctx |= (ctx_mask << CE_ENABLE_BITPOS);
611 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx);
612}
613
Pingchao Yangb0272272015-12-04 16:56:23 -0800614static void qat_hal_clear_xfer(struct icp_qat_fw_loader_handle *handle)
Tadeusz Strukb3416fb2014-06-05 13:44:04 -0700615{
616 unsigned char ae;
Tadeusz Strukb3416fb2014-06-05 13:44:04 -0700617 unsigned short reg;
Tadeusz Strukb3416fb2014-06-05 13:44:04 -0700618
Tadeusz Struk9a147cb2014-07-25 15:55:46 -0700619 for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) {
Tadeusz Strukb3416fb2014-06-05 13:44:04 -0700620 for (reg = 0; reg < ICP_QAT_UCLO_MAX_GPR_REG; reg++) {
621 qat_hal_init_rd_xfer(handle, ae, 0, ICP_SR_RD_ABS,
622 reg, 0);
623 qat_hal_init_rd_xfer(handle, ae, 0, ICP_DR_RD_ABS,
624 reg, 0);
625 }
Pingchao Yangb0272272015-12-04 16:56:23 -0800626 }
627}
628
629static int qat_hal_clear_gpr(struct icp_qat_fw_loader_handle *handle)
630{
631 unsigned char ae;
632 unsigned int ctx_mask = ICP_QAT_UCLO_AE_ALL_CTX;
633 int times = MAX_RETRY_TIMES;
634 unsigned int csr_val = 0;
635 unsigned int savctx = 0;
636 int ret = 0;
637
638 for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) {
Arnd Bergmann8c9478a2017-12-11 13:07:00 +0100639 csr_val = qat_hal_rd_ae_csr(handle, ae, AE_MISC_CONTROL);
Tadeusz Strukb3416fb2014-06-05 13:44:04 -0700640 csr_val &= ~(1 << MMC_SHARE_CS_BITPOS);
641 qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL, csr_val);
Arnd Bergmann8c9478a2017-12-11 13:07:00 +0100642 csr_val = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
Tadeusz Strukb3416fb2014-06-05 13:44:04 -0700643 csr_val &= IGNORE_W1C_MASK;
644 csr_val |= CE_NN_MODE;
645 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, csr_val);
646 qat_hal_wr_uwords(handle, ae, 0, ARRAY_SIZE(inst),
647 (uint64_t *)inst);
648 qat_hal_wr_indr_csr(handle, ae, ctx_mask, CTX_STS_INDIRECT,
649 handle->hal_handle->upc_mask &
650 INIT_PC_VALUE);
Arnd Bergmann8c9478a2017-12-11 13:07:00 +0100651 savctx = qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS);
Tadeusz Strukb3416fb2014-06-05 13:44:04 -0700652 qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS, 0);
653 qat_hal_put_wakeup_event(handle, ae, ctx_mask, XCWE_VOLUNTARY);
654 qat_hal_wr_indr_csr(handle, ae, ctx_mask,
655 CTX_SIG_EVENTS_INDIRECT, 0);
656 qat_hal_wr_ae_csr(handle, ae, CTX_SIG_EVENTS_ACTIVE, 0);
657 qat_hal_enable_ctx(handle, ae, ctx_mask);
658 }
Tadeusz Struk9a147cb2014-07-25 15:55:46 -0700659 for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) {
Tadeusz Strukb3416fb2014-06-05 13:44:04 -0700660 /* wait for AE to finish */
661 do {
662 ret = qat_hal_wait_cycles(handle, ae, 20, 1);
663 } while (ret && times--);
664
Pingchao Yangc0e77a12016-01-07 09:39:46 +0800665 if (times < 0) {
Tadeusz Strukb3416fb2014-06-05 13:44:04 -0700666 pr_err("QAT: clear GPR of AE %d failed", ae);
667 return -EINVAL;
668 }
669 qat_hal_disable_ctx(handle, ae, ctx_mask);
670 qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS,
671 savctx & ACS_ACNO);
672 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES,
673 INIT_CTX_ENABLE_VALUE);
674 qat_hal_wr_indr_csr(handle, ae, ctx_mask, CTX_STS_INDIRECT,
675 handle->hal_handle->upc_mask &
676 INIT_PC_VALUE);
677 qat_hal_wr_ae_csr(handle, ae, CTX_ARB_CNTL, INIT_CTX_ARB_VALUE);
678 qat_hal_wr_ae_csr(handle, ae, CC_ENABLE, INIT_CCENABLE_VALUE);
679 qat_hal_put_wakeup_event(handle, ae, ctx_mask,
680 INIT_WAKEUP_EVENTS_VALUE);
681 qat_hal_put_sig_event(handle, ae, ctx_mask,
682 INIT_SIG_EVENTS_VALUE);
683 }
684 return 0;
685}
686
Pingchao Yangb0272272015-12-04 16:56:23 -0800687#define ICP_QAT_AE_OFFSET 0x20000
688#define ICP_QAT_CAP_OFFSET (ICP_QAT_AE_OFFSET + 0x10000)
Tadeusz Strukb3416fb2014-06-05 13:44:04 -0700689#define LOCAL_TO_XFER_REG_OFFSET 0x800
Pingchao Yangb0272272015-12-04 16:56:23 -0800690#define ICP_QAT_EP_OFFSET 0x3a000
Tadeusz Strukb3416fb2014-06-05 13:44:04 -0700691int qat_hal_init(struct adf_accel_dev *accel_dev)
692{
Tadeusz Struk9a147cb2014-07-25 15:55:46 -0700693 unsigned char ae;
694 unsigned int max_en_ae_id = 0;
695 struct icp_qat_fw_loader_handle *handle;
Tadeusz Strukb3416fb2014-06-05 13:44:04 -0700696 struct adf_accel_pci *pci_info = &accel_dev->accel_pci_dev;
697 struct adf_hw_device_data *hw_data = accel_dev->hw_device;
Pingchao Yangf3dd7e62015-07-15 15:28:26 -0700698 struct adf_bar *misc_bar =
Tadeusz Struka727c4b62014-10-21 11:01:06 -0700699 &pci_info->pci_bars[hw_data->get_misc_bar_id(hw_data)];
Pingchao Yang91a93ea2016-01-06 17:56:34 +0800700 struct adf_bar *sram_bar;
Tadeusz Strukb3416fb2014-06-05 13:44:04 -0700701
702 handle = kzalloc(sizeof(*handle), GFP_KERNEL);
703 if (!handle)
Tadeusz Struk9a147cb2014-07-25 15:55:46 -0700704 return -ENOMEM;
Tadeusz Strukb3416fb2014-06-05 13:44:04 -0700705
Pingchao Yangb0272272015-12-04 16:56:23 -0800706 handle->hal_cap_g_ctl_csr_addr_v =
707 (void __iomem *)((uintptr_t)misc_bar->virt_addr +
708 ICP_QAT_CAP_OFFSET);
709 handle->hal_cap_ae_xfer_csr_addr_v =
710 (void __iomem *)((uintptr_t)misc_bar->virt_addr +
711 ICP_QAT_AE_OFFSET);
712 handle->hal_ep_csr_addr_v =
713 (void __iomem *)((uintptr_t)misc_bar->virt_addr +
714 ICP_QAT_EP_OFFSET);
715 handle->hal_cap_ae_local_csr_addr_v =
716 (void __iomem *)((uintptr_t)handle->hal_cap_ae_xfer_csr_addr_v +
717 LOCAL_TO_XFER_REG_OFFSET);
718 handle->pci_dev = pci_info->pci_dev;
Giovanni Cabiddu685ce062016-12-22 15:00:24 +0000719 if (handle->pci_dev->device == ADF_DH895XCC_PCI_DEVICE_ID) {
Pingchao Yang91a93ea2016-01-06 17:56:34 +0800720 sram_bar =
721 &pci_info->pci_bars[hw_data->get_sram_bar_id(hw_data)];
722 handle->hal_sram_addr_v = sram_bar->virt_addr;
723 }
Pingchao Yangb0272272015-12-04 16:56:23 -0800724 handle->fw_auth = (handle->pci_dev->device ==
725 ADF_DH895XCC_PCI_DEVICE_ID) ? false : true;
Tadeusz Strukb3416fb2014-06-05 13:44:04 -0700726 handle->hal_handle = kzalloc(sizeof(*handle->hal_handle), GFP_KERNEL);
727 if (!handle->hal_handle)
728 goto out_hal_handle;
729 handle->hal_handle->revision_id = accel_dev->accel_pci_dev.revid;
730 handle->hal_handle->ae_mask = hw_data->ae_mask;
731 handle->hal_handle->slice_mask = hw_data->accel_mask;
732 /* create AE objects */
733 handle->hal_handle->upc_mask = 0x1ffff;
734 handle->hal_handle->max_ustore = 0x4000;
735 for (ae = 0; ae < ICP_QAT_UCLO_MAX_AE; ae++) {
736 if (!(hw_data->ae_mask & (1 << ae)))
737 continue;
738 handle->hal_handle->aes[ae].free_addr = 0;
739 handle->hal_handle->aes[ae].free_size =
740 handle->hal_handle->max_ustore;
741 handle->hal_handle->aes[ae].ustore_size =
742 handle->hal_handle->max_ustore;
743 handle->hal_handle->aes[ae].live_ctx_mask =
744 ICP_QAT_UCLO_AE_ALL_CTX;
Tadeusz Struk9a147cb2014-07-25 15:55:46 -0700745 max_en_ae_id = ae;
Tadeusz Strukb3416fb2014-06-05 13:44:04 -0700746 }
Tadeusz Struk9a147cb2014-07-25 15:55:46 -0700747 handle->hal_handle->ae_max_num = max_en_ae_id + 1;
Tadeusz Strukb3416fb2014-06-05 13:44:04 -0700748 /* take all AEs out of reset */
749 if (qat_hal_clr_reset(handle)) {
Allan, Bruce W66550302015-03-19 16:03:44 -0700750 dev_err(&GET_DEV(accel_dev), "qat_hal_clr_reset error\n");
Tadeusz Strukb3416fb2014-06-05 13:44:04 -0700751 goto out_err;
752 }
Pingchao Yangb0272272015-12-04 16:56:23 -0800753 qat_hal_clear_xfer(handle);
754 if (!handle->fw_auth) {
755 if (qat_hal_clear_gpr(handle))
756 goto out_err;
757 }
758
Tadeusz Strukb3416fb2014-06-05 13:44:04 -0700759 /* Set SIGNATURE_ENABLE[0] to 0x1 in order to enable ALU_OUT csr */
Tadeusz Struk9a147cb2014-07-25 15:55:46 -0700760 for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) {
761 unsigned int csr_val = 0;
762
Arnd Bergmann8c9478a2017-12-11 13:07:00 +0100763 csr_val = qat_hal_rd_ae_csr(handle, ae, SIGNATURE_ENABLE);
Tadeusz Strukb3416fb2014-06-05 13:44:04 -0700764 csr_val |= 0x1;
765 qat_hal_wr_ae_csr(handle, ae, SIGNATURE_ENABLE, csr_val);
766 }
767 accel_dev->fw_loader->fw_loader = handle;
768 return 0;
769
770out_err:
771 kfree(handle->hal_handle);
772out_hal_handle:
773 kfree(handle);
Tadeusz Strukb3416fb2014-06-05 13:44:04 -0700774 return -EFAULT;
775}
776
777void qat_hal_deinit(struct icp_qat_fw_loader_handle *handle)
778{
779 if (!handle)
780 return;
781 kfree(handle->hal_handle);
782 kfree(handle);
783}
784
785void qat_hal_start(struct icp_qat_fw_loader_handle *handle, unsigned char ae,
786 unsigned int ctx_mask)
787{
Pingchao Yangb0272272015-12-04 16:56:23 -0800788 int retry = 0;
789 unsigned int fcu_sts = 0;
790
791 if (handle->fw_auth) {
792 SET_CAP_CSR(handle, FCU_CONTROL, FCU_CTRL_CMD_START);
793 do {
794 msleep(FW_AUTH_WAIT_PERIOD);
795 fcu_sts = GET_CAP_CSR(handle, FCU_STATUS);
796 if (((fcu_sts >> FCU_STS_DONE_POS) & 0x1))
797 return;
798 } while (retry++ < FW_AUTH_MAX_RETRY);
799 pr_err("QAT: start error (AE 0x%x FCU_STS = 0x%x)\n", ae,
800 fcu_sts);
801 } else {
802 qat_hal_put_wakeup_event(handle, ae, (~ctx_mask) &
Tadeusz Strukb3416fb2014-06-05 13:44:04 -0700803 ICP_QAT_UCLO_AE_ALL_CTX, 0x10000);
Pingchao Yangb0272272015-12-04 16:56:23 -0800804 qat_hal_enable_ctx(handle, ae, ctx_mask);
805 }
Tadeusz Strukb3416fb2014-06-05 13:44:04 -0700806}
807
808void qat_hal_stop(struct icp_qat_fw_loader_handle *handle, unsigned char ae,
809 unsigned int ctx_mask)
810{
Pingchao Yangb0272272015-12-04 16:56:23 -0800811 if (!handle->fw_auth)
812 qat_hal_disable_ctx(handle, ae, ctx_mask);
Tadeusz Strukb3416fb2014-06-05 13:44:04 -0700813}
814
815void qat_hal_set_pc(struct icp_qat_fw_loader_handle *handle,
816 unsigned char ae, unsigned int ctx_mask, unsigned int upc)
817{
818 qat_hal_wr_indr_csr(handle, ae, ctx_mask, CTX_STS_INDIRECT,
819 handle->hal_handle->upc_mask & upc);
820}
821
822static void qat_hal_get_uwords(struct icp_qat_fw_loader_handle *handle,
823 unsigned char ae, unsigned int uaddr,
824 unsigned int words_num, uint64_t *uword)
825{
826 unsigned int i, uwrd_lo, uwrd_hi;
827 unsigned int ustore_addr, misc_control;
828
Arnd Bergmann8c9478a2017-12-11 13:07:00 +0100829 misc_control = qat_hal_rd_ae_csr(handle, ae, AE_MISC_CONTROL);
Tadeusz Strukb3416fb2014-06-05 13:44:04 -0700830 qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL,
831 misc_control & 0xfffffffb);
Arnd Bergmann8c9478a2017-12-11 13:07:00 +0100832 ustore_addr = qat_hal_rd_ae_csr(handle, ae, USTORE_ADDRESS);
Tadeusz Strukb3416fb2014-06-05 13:44:04 -0700833 uaddr |= UA_ECS;
834 for (i = 0; i < words_num; i++) {
835 qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr);
836 uaddr++;
Arnd Bergmann8c9478a2017-12-11 13:07:00 +0100837 uwrd_lo = qat_hal_rd_ae_csr(handle, ae, USTORE_DATA_LOWER);
838 uwrd_hi = qat_hal_rd_ae_csr(handle, ae, USTORE_DATA_UPPER);
Tadeusz Strukb3416fb2014-06-05 13:44:04 -0700839 uword[i] = uwrd_hi;
840 uword[i] = (uword[i] << 0x20) | uwrd_lo;
841 }
842 qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL, misc_control);
843 qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, ustore_addr);
844}
845
Tadeusz Strukb3416fb2014-06-05 13:44:04 -0700846void qat_hal_wr_umem(struct icp_qat_fw_loader_handle *handle,
847 unsigned char ae, unsigned int uaddr,
848 unsigned int words_num, unsigned int *data)
849{
850 unsigned int i, ustore_addr;
851
Arnd Bergmann8c9478a2017-12-11 13:07:00 +0100852 ustore_addr = qat_hal_rd_ae_csr(handle, ae, USTORE_ADDRESS);
Tadeusz Strukb3416fb2014-06-05 13:44:04 -0700853 uaddr |= UA_ECS;
854 qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr);
855 for (i = 0; i < words_num; i++) {
856 unsigned int uwrd_lo, uwrd_hi, tmp;
Tadeusz Strukd65071e2014-06-24 15:19:34 -0700857
Tadeusz Strukb3416fb2014-06-05 13:44:04 -0700858 uwrd_lo = ((data[i] & 0xfff0000) << 4) | (0x3 << 18) |
859 ((data[i] & 0xff00) << 2) |
860 (0x3 << 8) | (data[i] & 0xff);
861 uwrd_hi = (0xf << 4) | ((data[i] & 0xf0000000) >> 28);
Tadeusz Strukd9a44ab2014-07-25 15:55:57 -0700862 uwrd_hi |= (hweight32(data[i] & 0xffff) & 0x1) << 8;
Tadeusz Strukb3416fb2014-06-05 13:44:04 -0700863 tmp = ((data[i] >> 0x10) & 0xffff);
Tadeusz Strukd9a44ab2014-07-25 15:55:57 -0700864 uwrd_hi |= (hweight32(tmp) & 0x1) << 9;
Tadeusz Strukb3416fb2014-06-05 13:44:04 -0700865 qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_LOWER, uwrd_lo);
866 qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_UPPER, uwrd_hi);
867 }
868 qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, ustore_addr);
869}
870
871#define MAX_EXEC_INST 100
872static int qat_hal_exec_micro_inst(struct icp_qat_fw_loader_handle *handle,
873 unsigned char ae, unsigned char ctx,
874 uint64_t *micro_inst, unsigned int inst_num,
875 int code_off, unsigned int max_cycle,
876 unsigned int *endpc)
877{
878 uint64_t savuwords[MAX_EXEC_INST];
879 unsigned int ind_lm_addr0, ind_lm_addr1;
880 unsigned int ind_lm_addr_byte0, ind_lm_addr_byte1;
881 unsigned int ind_cnt_sig;
882 unsigned int ind_sig, act_sig;
883 unsigned int csr_val = 0, newcsr_val;
884 unsigned int savctx;
885 unsigned int savcc, wakeup_events, savpc;
886 unsigned int ctxarb_ctl, ctx_enables;
887
888 if ((inst_num > handle->hal_handle->max_ustore) || !micro_inst) {
Tadeusz Struk68991722014-07-25 15:55:52 -0700889 pr_err("QAT: invalid instruction num %d\n", inst_num);
Tadeusz Strukb3416fb2014-06-05 13:44:04 -0700890 return -EINVAL;
891 }
892 /* save current context */
Arnd Bergmann8c9478a2017-12-11 13:07:00 +0100893 ind_lm_addr0 = qat_hal_rd_indr_csr(handle, ae, ctx, LM_ADDR_0_INDIRECT);
894 ind_lm_addr1 = qat_hal_rd_indr_csr(handle, ae, ctx, LM_ADDR_1_INDIRECT);
895 ind_lm_addr_byte0 = qat_hal_rd_indr_csr(handle, ae, ctx,
896 INDIRECT_LM_ADDR_0_BYTE_INDEX);
897 ind_lm_addr_byte1 = qat_hal_rd_indr_csr(handle, ae, ctx,
898 INDIRECT_LM_ADDR_1_BYTE_INDEX);
Tadeusz Strukb3416fb2014-06-05 13:44:04 -0700899 if (inst_num <= MAX_EXEC_INST)
900 qat_hal_get_uwords(handle, ae, 0, inst_num, savuwords);
901 qat_hal_get_wakeup_event(handle, ae, ctx, &wakeup_events);
Arnd Bergmann8c9478a2017-12-11 13:07:00 +0100902 savpc = qat_hal_rd_indr_csr(handle, ae, ctx, CTX_STS_INDIRECT);
Tadeusz Strukb3416fb2014-06-05 13:44:04 -0700903 savpc = (savpc & handle->hal_handle->upc_mask) >> 0;
Arnd Bergmann8c9478a2017-12-11 13:07:00 +0100904 ctx_enables = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
Tadeusz Strukb3416fb2014-06-05 13:44:04 -0700905 ctx_enables &= IGNORE_W1C_MASK;
Arnd Bergmann8c9478a2017-12-11 13:07:00 +0100906 savcc = qat_hal_rd_ae_csr(handle, ae, CC_ENABLE);
907 savctx = qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS);
908 ctxarb_ctl = qat_hal_rd_ae_csr(handle, ae, CTX_ARB_CNTL);
909 ind_cnt_sig = qat_hal_rd_indr_csr(handle, ae, ctx,
910 FUTURE_COUNT_SIGNAL_INDIRECT);
911 ind_sig = qat_hal_rd_indr_csr(handle, ae, ctx,
912 CTX_SIG_EVENTS_INDIRECT);
913 act_sig = qat_hal_rd_ae_csr(handle, ae, CTX_SIG_EVENTS_ACTIVE);
Tadeusz Strukb3416fb2014-06-05 13:44:04 -0700914 /* execute micro codes */
915 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables);
916 qat_hal_wr_uwords(handle, ae, 0, inst_num, micro_inst);
917 qat_hal_wr_indr_csr(handle, ae, (1 << ctx), CTX_STS_INDIRECT, 0);
918 qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS, ctx & ACS_ACNO);
919 if (code_off)
920 qat_hal_wr_ae_csr(handle, ae, CC_ENABLE, savcc & 0xffffdfff);
921 qat_hal_put_wakeup_event(handle, ae, (1 << ctx), XCWE_VOLUNTARY);
922 qat_hal_wr_indr_csr(handle, ae, (1 << ctx), CTX_SIG_EVENTS_INDIRECT, 0);
923 qat_hal_wr_ae_csr(handle, ae, CTX_SIG_EVENTS_ACTIVE, 0);
924 qat_hal_enable_ctx(handle, ae, (1 << ctx));
925 /* wait for micro codes to finish */
926 if (qat_hal_wait_cycles(handle, ae, max_cycle, 1) != 0)
927 return -EFAULT;
928 if (endpc) {
929 unsigned int ctx_status;
Tadeusz Strukd65071e2014-06-24 15:19:34 -0700930
Arnd Bergmann8c9478a2017-12-11 13:07:00 +0100931 ctx_status = qat_hal_rd_indr_csr(handle, ae, ctx,
932 CTX_STS_INDIRECT);
Tadeusz Strukb3416fb2014-06-05 13:44:04 -0700933 *endpc = ctx_status & handle->hal_handle->upc_mask;
934 }
935 /* retore to saved context */
936 qat_hal_disable_ctx(handle, ae, (1 << ctx));
937 if (inst_num <= MAX_EXEC_INST)
938 qat_hal_wr_uwords(handle, ae, 0, inst_num, savuwords);
939 qat_hal_put_wakeup_event(handle, ae, (1 << ctx), wakeup_events);
940 qat_hal_wr_indr_csr(handle, ae, (1 << ctx), CTX_STS_INDIRECT,
941 handle->hal_handle->upc_mask & savpc);
Arnd Bergmann8c9478a2017-12-11 13:07:00 +0100942 csr_val = qat_hal_rd_ae_csr(handle, ae, AE_MISC_CONTROL);
Tadeusz Strukb3416fb2014-06-05 13:44:04 -0700943 newcsr_val = CLR_BIT(csr_val, MMC_SHARE_CS_BITPOS);
944 qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL, newcsr_val);
945 qat_hal_wr_ae_csr(handle, ae, CC_ENABLE, savcc);
946 qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS, savctx & ACS_ACNO);
947 qat_hal_wr_ae_csr(handle, ae, CTX_ARB_CNTL, ctxarb_ctl);
948 qat_hal_wr_indr_csr(handle, ae, (1 << ctx),
949 LM_ADDR_0_INDIRECT, ind_lm_addr0);
950 qat_hal_wr_indr_csr(handle, ae, (1 << ctx),
951 LM_ADDR_1_INDIRECT, ind_lm_addr1);
952 qat_hal_wr_indr_csr(handle, ae, (1 << ctx),
953 INDIRECT_LM_ADDR_0_BYTE_INDEX, ind_lm_addr_byte0);
954 qat_hal_wr_indr_csr(handle, ae, (1 << ctx),
955 INDIRECT_LM_ADDR_1_BYTE_INDEX, ind_lm_addr_byte1);
956 qat_hal_wr_indr_csr(handle, ae, (1 << ctx),
957 FUTURE_COUNT_SIGNAL_INDIRECT, ind_cnt_sig);
958 qat_hal_wr_indr_csr(handle, ae, (1 << ctx),
959 CTX_SIG_EVENTS_INDIRECT, ind_sig);
960 qat_hal_wr_ae_csr(handle, ae, CTX_SIG_EVENTS_ACTIVE, act_sig);
961 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables);
962
963 return 0;
964}
965
966static int qat_hal_rd_rel_reg(struct icp_qat_fw_loader_handle *handle,
967 unsigned char ae, unsigned char ctx,
968 enum icp_qat_uof_regtype reg_type,
969 unsigned short reg_num, unsigned int *data)
970{
971 unsigned int savctx, uaddr, uwrd_lo, uwrd_hi;
972 unsigned int ctxarb_cntl, ustore_addr, ctx_enables;
973 unsigned short reg_addr;
974 int status = 0;
975 uint64_t insts, savuword;
976
977 reg_addr = qat_hal_get_reg_addr(reg_type, reg_num);
978 if (reg_addr == BAD_REGADDR) {
979 pr_err("QAT: bad regaddr=0x%x\n", reg_addr);
980 return -EINVAL;
981 }
982 switch (reg_type) {
983 case ICP_GPA_REL:
984 insts = 0xA070000000ull | (reg_addr & 0x3ff);
985 break;
986 default:
987 insts = (uint64_t)0xA030000000ull | ((reg_addr & 0x3ff) << 10);
988 break;
989 }
Arnd Bergmann8c9478a2017-12-11 13:07:00 +0100990 savctx = qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS);
991 ctxarb_cntl = qat_hal_rd_ae_csr(handle, ae, CTX_ARB_CNTL);
992 ctx_enables = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
Tadeusz Strukb3416fb2014-06-05 13:44:04 -0700993 ctx_enables &= IGNORE_W1C_MASK;
994 if (ctx != (savctx & ACS_ACNO))
995 qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS,
996 ctx & ACS_ACNO);
997 qat_hal_get_uwords(handle, ae, 0, 1, &savuword);
998 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables);
Arnd Bergmann8c9478a2017-12-11 13:07:00 +0100999 ustore_addr = qat_hal_rd_ae_csr(handle, ae, USTORE_ADDRESS);
Tadeusz Strukb3416fb2014-06-05 13:44:04 -07001000 uaddr = UA_ECS;
1001 qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr);
1002 insts = qat_hal_set_uword_ecc(insts);
1003 uwrd_lo = (unsigned int)(insts & 0xffffffff);
1004 uwrd_hi = (unsigned int)(insts >> 0x20);
1005 qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_LOWER, uwrd_lo);
1006 qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_UPPER, uwrd_hi);
1007 qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr);
1008 /* delay for at least 8 cycles */
1009 qat_hal_wait_cycles(handle, ae, 0x8, 0);
1010 /*
1011 * read ALU output
1012 * the instruction should have been executed
1013 * prior to clearing the ECS in putUwords
1014 */
Arnd Bergmann8c9478a2017-12-11 13:07:00 +01001015 *data = qat_hal_rd_ae_csr(handle, ae, ALU_OUT);
Tadeusz Strukb3416fb2014-06-05 13:44:04 -07001016 qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, ustore_addr);
1017 qat_hal_wr_uwords(handle, ae, 0, 1, &savuword);
1018 if (ctx != (savctx & ACS_ACNO))
1019 qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS,
1020 savctx & ACS_ACNO);
1021 qat_hal_wr_ae_csr(handle, ae, CTX_ARB_CNTL, ctxarb_cntl);
1022 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables);
1023
1024 return status;
1025}
1026
1027static int qat_hal_wr_rel_reg(struct icp_qat_fw_loader_handle *handle,
1028 unsigned char ae, unsigned char ctx,
1029 enum icp_qat_uof_regtype reg_type,
1030 unsigned short reg_num, unsigned int data)
1031{
1032 unsigned short src_hiaddr, src_lowaddr, dest_addr, data16hi, data16lo;
1033 uint64_t insts[] = {
1034 0x0F440000000ull,
1035 0x0F040000000ull,
1036 0x0F0000C0300ull,
1037 0x0E000010000ull
1038 };
1039 const int num_inst = ARRAY_SIZE(insts), code_off = 1;
1040 const int imm_w1 = 0, imm_w0 = 1;
1041
1042 dest_addr = qat_hal_get_reg_addr(reg_type, reg_num);
1043 if (dest_addr == BAD_REGADDR) {
1044 pr_err("QAT: bad destAddr=0x%x\n", dest_addr);
1045 return -EINVAL;
1046 }
1047
1048 data16lo = 0xffff & data;
1049 data16hi = 0xffff & (data >> 0x10);
1050 src_hiaddr = qat_hal_get_reg_addr(ICP_NO_DEST, (unsigned short)
1051 (0xff & data16hi));
1052 src_lowaddr = qat_hal_get_reg_addr(ICP_NO_DEST, (unsigned short)
1053 (0xff & data16lo));
1054 switch (reg_type) {
1055 case ICP_GPA_REL:
1056 insts[imm_w1] = insts[imm_w1] | ((data16hi >> 8) << 20) |
1057 ((src_hiaddr & 0x3ff) << 10) | (dest_addr & 0x3ff);
1058 insts[imm_w0] = insts[imm_w0] | ((data16lo >> 8) << 20) |
1059 ((src_lowaddr & 0x3ff) << 10) | (dest_addr & 0x3ff);
1060 break;
1061 default:
1062 insts[imm_w1] = insts[imm_w1] | ((data16hi >> 8) << 20) |
1063 ((dest_addr & 0x3ff) << 10) | (src_hiaddr & 0x3ff);
1064
1065 insts[imm_w0] = insts[imm_w0] | ((data16lo >> 8) << 20) |
1066 ((dest_addr & 0x3ff) << 10) | (src_lowaddr & 0x3ff);
1067 break;
1068 }
1069
1070 return qat_hal_exec_micro_inst(handle, ae, ctx, insts, num_inst,
1071 code_off, num_inst * 0x5, NULL);
1072}
1073
1074int qat_hal_get_ins_num(void)
1075{
1076 return ARRAY_SIZE(inst_4b);
1077}
1078
1079static int qat_hal_concat_micro_code(uint64_t *micro_inst,
1080 unsigned int inst_num, unsigned int size,
1081 unsigned int addr, unsigned int *value)
1082{
Tadeusz Struk9196d962015-09-30 05:40:00 -07001083 int i;
Tadeusz Strukb3416fb2014-06-05 13:44:04 -07001084 unsigned int cur_value;
1085 const uint64_t *inst_arr;
1086 int fixup_offset;
1087 int usize = 0;
1088 int orig_num;
1089
1090 orig_num = inst_num;
Tadeusz Struk9196d962015-09-30 05:40:00 -07001091 cur_value = value[0];
Tadeusz Strukb3416fb2014-06-05 13:44:04 -07001092 inst_arr = inst_4b;
1093 usize = ARRAY_SIZE(inst_4b);
1094 fixup_offset = inst_num;
1095 for (i = 0; i < usize; i++)
1096 micro_inst[inst_num++] = inst_arr[i];
1097 INSERT_IMMED_GPRA_CONST(micro_inst[fixup_offset], (addr));
1098 fixup_offset++;
1099 INSERT_IMMED_GPRA_CONST(micro_inst[fixup_offset], 0);
1100 fixup_offset++;
1101 INSERT_IMMED_GPRB_CONST(micro_inst[fixup_offset], (cur_value >> 0));
1102 fixup_offset++;
1103 INSERT_IMMED_GPRB_CONST(micro_inst[fixup_offset], (cur_value >> 0x10));
1104
1105 return inst_num - orig_num;
1106}
1107
1108static int qat_hal_exec_micro_init_lm(struct icp_qat_fw_loader_handle *handle,
1109 unsigned char ae, unsigned char ctx,
1110 int *pfirst_exec, uint64_t *micro_inst,
1111 unsigned int inst_num)
1112{
1113 int stat = 0;
1114 unsigned int gpra0 = 0, gpra1 = 0, gpra2 = 0;
1115 unsigned int gprb0 = 0, gprb1 = 0;
1116
1117 if (*pfirst_exec) {
1118 qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0, &gpra0);
1119 qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0x1, &gpra1);
1120 qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0x2, &gpra2);
1121 qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPB_REL, 0, &gprb0);
1122 qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPB_REL, 0x1, &gprb1);
1123 *pfirst_exec = 0;
1124 }
1125 stat = qat_hal_exec_micro_inst(handle, ae, ctx, micro_inst, inst_num, 1,
1126 inst_num * 0x5, NULL);
1127 if (stat != 0)
1128 return -EFAULT;
1129 qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0, gpra0);
1130 qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0x1, gpra1);
1131 qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0x2, gpra2);
1132 qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPB_REL, 0, gprb0);
1133 qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPB_REL, 0x1, gprb1);
1134
1135 return 0;
1136}
1137
1138int qat_hal_batch_wr_lm(struct icp_qat_fw_loader_handle *handle,
1139 unsigned char ae,
1140 struct icp_qat_uof_batch_init *lm_init_header)
1141{
1142 struct icp_qat_uof_batch_init *plm_init;
1143 uint64_t *micro_inst_arry;
1144 int micro_inst_num;
1145 int alloc_inst_size;
1146 int first_exec = 1;
1147 int stat = 0;
1148
1149 plm_init = lm_init_header->next;
1150 alloc_inst_size = lm_init_header->size;
1151 if ((unsigned int)alloc_inst_size > handle->hal_handle->max_ustore)
1152 alloc_inst_size = handle->hal_handle->max_ustore;
Tadeusz Strukd65071e2014-06-24 15:19:34 -07001153 micro_inst_arry = kmalloc_array(alloc_inst_size, sizeof(uint64_t),
1154 GFP_KERNEL);
Tadeusz Strukb3416fb2014-06-05 13:44:04 -07001155 if (!micro_inst_arry)
1156 return -ENOMEM;
1157 micro_inst_num = 0;
1158 while (plm_init) {
1159 unsigned int addr, *value, size;
1160
1161 ae = plm_init->ae;
1162 addr = plm_init->addr;
1163 value = plm_init->value;
1164 size = plm_init->size;
1165 micro_inst_num += qat_hal_concat_micro_code(micro_inst_arry,
1166 micro_inst_num,
1167 size, addr, value);
1168 plm_init = plm_init->next;
1169 }
1170 /* exec micro codes */
1171 if (micro_inst_arry && (micro_inst_num > 0)) {
1172 micro_inst_arry[micro_inst_num++] = 0x0E000010000ull;
1173 stat = qat_hal_exec_micro_init_lm(handle, ae, 0, &first_exec,
1174 micro_inst_arry,
1175 micro_inst_num);
1176 }
1177 kfree(micro_inst_arry);
1178 return stat;
1179}
1180
1181static int qat_hal_put_rel_rd_xfer(struct icp_qat_fw_loader_handle *handle,
1182 unsigned char ae, unsigned char ctx,
1183 enum icp_qat_uof_regtype reg_type,
1184 unsigned short reg_num, unsigned int val)
1185{
1186 int status = 0;
1187 unsigned int reg_addr;
1188 unsigned int ctx_enables;
1189 unsigned short mask;
1190 unsigned short dr_offset = 0x10;
1191
Arnd Bergmann8c9478a2017-12-11 13:07:00 +01001192 status = ctx_enables = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
Tadeusz Strukb3416fb2014-06-05 13:44:04 -07001193 if (CE_INUSE_CONTEXTS & ctx_enables) {
1194 if (ctx & 0x1) {
1195 pr_err("QAT: bad 4-ctx mode,ctx=0x%x\n", ctx);
1196 return -EINVAL;
1197 }
1198 mask = 0x1f;
1199 dr_offset = 0x20;
1200 } else {
1201 mask = 0x0f;
1202 }
1203 if (reg_num & ~mask)
1204 return -EINVAL;
1205 reg_addr = reg_num + (ctx << 0x5);
1206 switch (reg_type) {
1207 case ICP_SR_RD_REL:
1208 case ICP_SR_REL:
1209 SET_AE_XFER(handle, ae, reg_addr, val);
1210 break;
1211 case ICP_DR_RD_REL:
1212 case ICP_DR_REL:
1213 SET_AE_XFER(handle, ae, (reg_addr + dr_offset), val);
1214 break;
1215 default:
1216 status = -EINVAL;
1217 break;
1218 }
1219 return status;
1220}
1221
1222static int qat_hal_put_rel_wr_xfer(struct icp_qat_fw_loader_handle *handle,
1223 unsigned char ae, unsigned char ctx,
1224 enum icp_qat_uof_regtype reg_type,
1225 unsigned short reg_num, unsigned int data)
1226{
1227 unsigned int gprval, ctx_enables;
1228 unsigned short src_hiaddr, src_lowaddr, gpr_addr, xfr_addr, data16hi,
1229 data16low;
1230 unsigned short reg_mask;
1231 int status = 0;
1232 uint64_t micro_inst[] = {
1233 0x0F440000000ull,
1234 0x0F040000000ull,
1235 0x0A000000000ull,
1236 0x0F0000C0300ull,
1237 0x0E000010000ull
1238 };
1239 const int num_inst = ARRAY_SIZE(micro_inst), code_off = 1;
1240 const unsigned short gprnum = 0, dly = num_inst * 0x5;
1241
Arnd Bergmann8c9478a2017-12-11 13:07:00 +01001242 ctx_enables = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
Tadeusz Strukb3416fb2014-06-05 13:44:04 -07001243 if (CE_INUSE_CONTEXTS & ctx_enables) {
1244 if (ctx & 0x1) {
1245 pr_err("QAT: 4-ctx mode,ctx=0x%x\n", ctx);
1246 return -EINVAL;
1247 }
1248 reg_mask = (unsigned short)~0x1f;
1249 } else {
1250 reg_mask = (unsigned short)~0xf;
1251 }
1252 if (reg_num & reg_mask)
1253 return -EINVAL;
1254 xfr_addr = qat_hal_get_reg_addr(reg_type, reg_num);
1255 if (xfr_addr == BAD_REGADDR) {
1256 pr_err("QAT: bad xfrAddr=0x%x\n", xfr_addr);
1257 return -EINVAL;
1258 }
1259 qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPB_REL, gprnum, &gprval);
1260 gpr_addr = qat_hal_get_reg_addr(ICP_GPB_REL, gprnum);
1261 data16low = 0xffff & data;
1262 data16hi = 0xffff & (data >> 0x10);
1263 src_hiaddr = qat_hal_get_reg_addr(ICP_NO_DEST,
1264 (unsigned short)(0xff & data16hi));
1265 src_lowaddr = qat_hal_get_reg_addr(ICP_NO_DEST,
1266 (unsigned short)(0xff & data16low));
1267 micro_inst[0] = micro_inst[0x0] | ((data16hi >> 8) << 20) |
1268 ((gpr_addr & 0x3ff) << 10) | (src_hiaddr & 0x3ff);
1269 micro_inst[1] = micro_inst[0x1] | ((data16low >> 8) << 20) |
1270 ((gpr_addr & 0x3ff) << 10) | (src_lowaddr & 0x3ff);
1271 micro_inst[0x2] = micro_inst[0x2] |
1272 ((xfr_addr & 0x3ff) << 20) | ((gpr_addr & 0x3ff) << 10);
1273 status = qat_hal_exec_micro_inst(handle, ae, ctx, micro_inst, num_inst,
1274 code_off, dly, NULL);
1275 qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPB_REL, gprnum, gprval);
1276 return status;
1277}
1278
1279static int qat_hal_put_rel_nn(struct icp_qat_fw_loader_handle *handle,
1280 unsigned char ae, unsigned char ctx,
1281 unsigned short nn, unsigned int val)
1282{
1283 unsigned int ctx_enables;
1284 int stat = 0;
1285
Arnd Bergmann8c9478a2017-12-11 13:07:00 +01001286 ctx_enables = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
Tadeusz Strukb3416fb2014-06-05 13:44:04 -07001287 ctx_enables &= IGNORE_W1C_MASK;
1288 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables | CE_NN_MODE);
1289
1290 stat = qat_hal_put_rel_wr_xfer(handle, ae, ctx, ICP_NEIGH_REL, nn, val);
1291 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables);
1292 return stat;
1293}
1294
1295static int qat_hal_convert_abs_to_rel(struct icp_qat_fw_loader_handle
1296 *handle, unsigned char ae,
1297 unsigned short absreg_num,
1298 unsigned short *relreg,
1299 unsigned char *ctx)
1300{
1301 unsigned int ctx_enables;
1302
Arnd Bergmann8c9478a2017-12-11 13:07:00 +01001303 ctx_enables = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
Tadeusz Strukb3416fb2014-06-05 13:44:04 -07001304 if (ctx_enables & CE_INUSE_CONTEXTS) {
1305 /* 4-ctx mode */
1306 *relreg = absreg_num & 0x1F;
1307 *ctx = (absreg_num >> 0x4) & 0x6;
1308 } else {
1309 /* 8-ctx mode */
1310 *relreg = absreg_num & 0x0F;
1311 *ctx = (absreg_num >> 0x4) & 0x7;
1312 }
1313 return 0;
1314}
1315
1316int qat_hal_init_gpr(struct icp_qat_fw_loader_handle *handle,
1317 unsigned char ae, unsigned char ctx_mask,
1318 enum icp_qat_uof_regtype reg_type,
1319 unsigned short reg_num, unsigned int regdata)
1320{
1321 int stat = 0;
1322 unsigned short reg;
1323 unsigned char ctx = 0;
1324 enum icp_qat_uof_regtype type;
1325
1326 if (reg_num >= ICP_QAT_UCLO_MAX_GPR_REG)
1327 return -EINVAL;
1328
1329 do {
1330 if (ctx_mask == 0) {
1331 qat_hal_convert_abs_to_rel(handle, ae, reg_num, &reg,
1332 &ctx);
1333 type = reg_type - 1;
1334 } else {
1335 reg = reg_num;
1336 type = reg_type;
1337 if (!test_bit(ctx, (unsigned long *)&ctx_mask))
1338 continue;
1339 }
1340 stat = qat_hal_wr_rel_reg(handle, ae, ctx, type, reg, regdata);
1341 if (stat) {
1342 pr_err("QAT: write gpr fail\n");
1343 return -EINVAL;
1344 }
1345 } while (ctx_mask && (ctx++ < ICP_QAT_UCLO_MAX_CTX));
1346
1347 return 0;
1348}
1349
1350int qat_hal_init_wr_xfer(struct icp_qat_fw_loader_handle *handle,
1351 unsigned char ae, unsigned char ctx_mask,
1352 enum icp_qat_uof_regtype reg_type,
1353 unsigned short reg_num, unsigned int regdata)
1354{
1355 int stat = 0;
1356 unsigned short reg;
1357 unsigned char ctx = 0;
1358 enum icp_qat_uof_regtype type;
1359
1360 if (reg_num >= ICP_QAT_UCLO_MAX_XFER_REG)
1361 return -EINVAL;
1362
1363 do {
1364 if (ctx_mask == 0) {
1365 qat_hal_convert_abs_to_rel(handle, ae, reg_num, &reg,
1366 &ctx);
1367 type = reg_type - 3;
1368 } else {
1369 reg = reg_num;
1370 type = reg_type;
1371 if (!test_bit(ctx, (unsigned long *)&ctx_mask))
1372 continue;
1373 }
1374 stat = qat_hal_put_rel_wr_xfer(handle, ae, ctx, type, reg,
1375 regdata);
1376 if (stat) {
1377 pr_err("QAT: write wr xfer fail\n");
1378 return -EINVAL;
1379 }
1380 } while (ctx_mask && (ctx++ < ICP_QAT_UCLO_MAX_CTX));
1381
1382 return 0;
1383}
1384
1385int qat_hal_init_rd_xfer(struct icp_qat_fw_loader_handle *handle,
1386 unsigned char ae, unsigned char ctx_mask,
1387 enum icp_qat_uof_regtype reg_type,
1388 unsigned short reg_num, unsigned int regdata)
1389{
1390 int stat = 0;
1391 unsigned short reg;
1392 unsigned char ctx = 0;
1393 enum icp_qat_uof_regtype type;
1394
1395 if (reg_num >= ICP_QAT_UCLO_MAX_XFER_REG)
1396 return -EINVAL;
1397
1398 do {
1399 if (ctx_mask == 0) {
1400 qat_hal_convert_abs_to_rel(handle, ae, reg_num, &reg,
1401 &ctx);
1402 type = reg_type - 3;
1403 } else {
1404 reg = reg_num;
1405 type = reg_type;
1406 if (!test_bit(ctx, (unsigned long *)&ctx_mask))
1407 continue;
1408 }
1409 stat = qat_hal_put_rel_rd_xfer(handle, ae, ctx, type, reg,
1410 regdata);
1411 if (stat) {
1412 pr_err("QAT: write rd xfer fail\n");
1413 return -EINVAL;
1414 }
1415 } while (ctx_mask && (ctx++ < ICP_QAT_UCLO_MAX_CTX));
1416
1417 return 0;
1418}
1419
1420int qat_hal_init_nn(struct icp_qat_fw_loader_handle *handle,
1421 unsigned char ae, unsigned char ctx_mask,
1422 unsigned short reg_num, unsigned int regdata)
1423{
1424 int stat = 0;
1425 unsigned char ctx;
1426
1427 if (ctx_mask == 0)
1428 return -EINVAL;
1429
1430 for (ctx = 0; ctx < ICP_QAT_UCLO_MAX_CTX; ctx++) {
1431 if (!test_bit(ctx, (unsigned long *)&ctx_mask))
1432 continue;
1433 stat = qat_hal_put_rel_nn(handle, ae, ctx, reg_num, regdata);
1434 if (stat) {
1435 pr_err("QAT: write neigh error\n");
1436 return -EINVAL;
1437 }
1438 }
1439
1440 return 0;
1441}