Tadeusz Struk | b3416fb | 2014-06-05 13:44:04 -0700 | [diff] [blame] | 1 | /* |
| 2 | This file is provided under a dual BSD/GPLv2 license. When using or |
| 3 | redistributing this file, you may do so under either license. |
| 4 | |
| 5 | GPL LICENSE SUMMARY |
| 6 | Copyright(c) 2014 Intel Corporation. |
| 7 | This program is free software; you can redistribute it and/or modify |
| 8 | it under the terms of version 2 of the GNU General Public License as |
| 9 | published by the Free Software Foundation. |
| 10 | |
| 11 | This program is distributed in the hope that it will be useful, but |
| 12 | WITHOUT ANY WARRANTY; without even the implied warranty of |
| 13 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU |
| 14 | General Public License for more details. |
| 15 | |
| 16 | Contact Information: |
| 17 | qat-linux@intel.com |
| 18 | |
| 19 | BSD LICENSE |
| 20 | Copyright(c) 2014 Intel Corporation. |
| 21 | Redistribution and use in source and binary forms, with or without |
| 22 | modification, are permitted provided that the following conditions |
| 23 | are met: |
| 24 | |
| 25 | * Redistributions of source code must retain the above copyright |
| 26 | notice, this list of conditions and the following disclaimer. |
| 27 | * Redistributions in binary form must reproduce the above copyright |
| 28 | notice, this list of conditions and the following disclaimer in |
| 29 | the documentation and/or other materials provided with the |
| 30 | distribution. |
| 31 | * Neither the name of Intel Corporation nor the names of its |
| 32 | contributors may be used to endorse or promote products derived |
| 33 | from this software without specific prior written permission. |
| 34 | |
| 35 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
| 36 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
| 37 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
| 38 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
| 39 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
| 40 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
| 41 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
| 42 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
| 43 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| 44 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
| 45 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| 46 | */ |
| 47 | #include <linux/slab.h> |
Pingchao Yang | b027227 | 2015-12-04 16:56:23 -0800 | [diff] [blame] | 48 | #include <linux/delay.h> |
Tadeusz Struk | b3416fb | 2014-06-05 13:44:04 -0700 | [diff] [blame] | 49 | |
| 50 | #include "adf_accel_devices.h" |
| 51 | #include "adf_common_drv.h" |
| 52 | #include "icp_qat_hal.h" |
| 53 | #include "icp_qat_uclo.h" |
| 54 | |
Pingchao Yang | b027227 | 2015-12-04 16:56:23 -0800 | [diff] [blame] | 55 | #define BAD_REGADDR 0xffff |
| 56 | #define MAX_RETRY_TIMES 10000 |
| 57 | #define INIT_CTX_ARB_VALUE 0x0 |
Tadeusz Struk | b3416fb | 2014-06-05 13:44:04 -0700 | [diff] [blame] | 58 | #define INIT_CTX_ENABLE_VALUE 0x0 |
Pingchao Yang | b027227 | 2015-12-04 16:56:23 -0800 | [diff] [blame] | 59 | #define INIT_PC_VALUE 0x0 |
Tadeusz Struk | b3416fb | 2014-06-05 13:44:04 -0700 | [diff] [blame] | 60 | #define INIT_WAKEUP_EVENTS_VALUE 0x1 |
| 61 | #define INIT_SIG_EVENTS_VALUE 0x1 |
| 62 | #define INIT_CCENABLE_VALUE 0x2000 |
Pingchao Yang | b027227 | 2015-12-04 16:56:23 -0800 | [diff] [blame] | 63 | #define RST_CSR_QAT_LSB 20 |
Tadeusz Struk | b3416fb | 2014-06-05 13:44:04 -0700 | [diff] [blame] | 64 | #define RST_CSR_AE_LSB 0 |
| 65 | #define MC_TIMESTAMP_ENABLE (0x1 << 7) |
| 66 | |
| 67 | #define IGNORE_W1C_MASK ((~(1 << CE_BREAKPOINT_BITPOS)) & \ |
| 68 | (~(1 << CE_CNTL_STORE_PARITY_ERROR_BITPOS)) & \ |
| 69 | (~(1 << CE_REG_PAR_ERR_BITPOS))) |
| 70 | #define INSERT_IMMED_GPRA_CONST(inst, const_val) \ |
| 71 | (inst = ((inst & 0xFFFF00C03FFull) | \ |
| 72 | ((((const_val) << 12) & 0x0FF00000ull) | \ |
| 73 | (((const_val) << 10) & 0x0003FC00ull)))) |
| 74 | #define INSERT_IMMED_GPRB_CONST(inst, const_val) \ |
| 75 | (inst = ((inst & 0xFFFF00FFF00ull) | \ |
| 76 | ((((const_val) << 12) & 0x0FF00000ull) | \ |
| 77 | (((const_val) << 0) & 0x000000FFull)))) |
| 78 | |
| 79 | #define AE(handle, ae) handle->hal_handle->aes[ae] |
| 80 | |
| 81 | static const uint64_t inst_4b[] = { |
| 82 | 0x0F0400C0000ull, 0x0F4400C0000ull, 0x0F040000300ull, 0x0F440000300ull, |
| 83 | 0x0FC066C0000ull, 0x0F0000C0300ull, 0x0F0000C0300ull, 0x0F0000C0300ull, |
| 84 | 0x0A021000000ull |
| 85 | }; |
| 86 | |
| 87 | static const uint64_t inst[] = { |
| 88 | 0x0F0000C0000ull, 0x0F000000380ull, 0x0D805000011ull, 0x0FC082C0300ull, |
| 89 | 0x0F0000C0300ull, 0x0F0000C0300ull, 0x0F0000C0300ull, 0x0F0000C0300ull, |
| 90 | 0x0A0643C0000ull, 0x0BAC0000301ull, 0x0D802000101ull, 0x0F0000C0001ull, |
| 91 | 0x0FC066C0001ull, 0x0F0000C0300ull, 0x0F0000C0300ull, 0x0F0000C0300ull, |
| 92 | 0x0F000400300ull, 0x0A0610C0000ull, 0x0BAC0000301ull, 0x0D804400101ull, |
| 93 | 0x0A0580C0000ull, 0x0A0581C0000ull, 0x0A0582C0000ull, 0x0A0583C0000ull, |
| 94 | 0x0A0584C0000ull, 0x0A0585C0000ull, 0x0A0586C0000ull, 0x0A0587C0000ull, |
| 95 | 0x0A0588C0000ull, 0x0A0589C0000ull, 0x0A058AC0000ull, 0x0A058BC0000ull, |
| 96 | 0x0A058CC0000ull, 0x0A058DC0000ull, 0x0A058EC0000ull, 0x0A058FC0000ull, |
| 97 | 0x0A05C0C0000ull, 0x0A05C1C0000ull, 0x0A05C2C0000ull, 0x0A05C3C0000ull, |
| 98 | 0x0A05C4C0000ull, 0x0A05C5C0000ull, 0x0A05C6C0000ull, 0x0A05C7C0000ull, |
| 99 | 0x0A05C8C0000ull, 0x0A05C9C0000ull, 0x0A05CAC0000ull, 0x0A05CBC0000ull, |
| 100 | 0x0A05CCC0000ull, 0x0A05CDC0000ull, 0x0A05CEC0000ull, 0x0A05CFC0000ull, |
| 101 | 0x0A0400C0000ull, 0x0B0400C0000ull, 0x0A0401C0000ull, 0x0B0401C0000ull, |
| 102 | 0x0A0402C0000ull, 0x0B0402C0000ull, 0x0A0403C0000ull, 0x0B0403C0000ull, |
| 103 | 0x0A0404C0000ull, 0x0B0404C0000ull, 0x0A0405C0000ull, 0x0B0405C0000ull, |
| 104 | 0x0A0406C0000ull, 0x0B0406C0000ull, 0x0A0407C0000ull, 0x0B0407C0000ull, |
| 105 | 0x0A0408C0000ull, 0x0B0408C0000ull, 0x0A0409C0000ull, 0x0B0409C0000ull, |
| 106 | 0x0A040AC0000ull, 0x0B040AC0000ull, 0x0A040BC0000ull, 0x0B040BC0000ull, |
| 107 | 0x0A040CC0000ull, 0x0B040CC0000ull, 0x0A040DC0000ull, 0x0B040DC0000ull, |
| 108 | 0x0A040EC0000ull, 0x0B040EC0000ull, 0x0A040FC0000ull, 0x0B040FC0000ull, |
| 109 | 0x0D81581C010ull, 0x0E000010000ull, 0x0E000010000ull, |
| 110 | }; |
| 111 | |
| 112 | void qat_hal_set_live_ctx(struct icp_qat_fw_loader_handle *handle, |
| 113 | unsigned char ae, unsigned int ctx_mask) |
| 114 | { |
| 115 | AE(handle, ae).live_ctx_mask = ctx_mask; |
| 116 | } |
| 117 | |
| 118 | #define CSR_RETRY_TIMES 500 |
| 119 | static int qat_hal_rd_ae_csr(struct icp_qat_fw_loader_handle *handle, |
Arnd Bergmann | 8c9478a | 2017-12-11 13:07:00 +0100 | [diff] [blame] | 120 | unsigned char ae, unsigned int csr) |
Tadeusz Struk | b3416fb | 2014-06-05 13:44:04 -0700 | [diff] [blame] | 121 | { |
| 122 | unsigned int iterations = CSR_RETRY_TIMES; |
Arnd Bergmann | 8c9478a | 2017-12-11 13:07:00 +0100 | [diff] [blame] | 123 | int value; |
Tadeusz Struk | b3416fb | 2014-06-05 13:44:04 -0700 | [diff] [blame] | 124 | |
| 125 | do { |
Arnd Bergmann | 8c9478a | 2017-12-11 13:07:00 +0100 | [diff] [blame] | 126 | value = GET_AE_CSR(handle, ae, csr); |
Tadeusz Struk | b3416fb | 2014-06-05 13:44:04 -0700 | [diff] [blame] | 127 | if (!(GET_AE_CSR(handle, ae, LOCAL_CSR_STATUS) & LCS_STATUS)) |
Arnd Bergmann | 8c9478a | 2017-12-11 13:07:00 +0100 | [diff] [blame] | 128 | return value; |
Tadeusz Struk | b3416fb | 2014-06-05 13:44:04 -0700 | [diff] [blame] | 129 | } while (iterations--); |
| 130 | |
| 131 | pr_err("QAT: Read CSR timeout\n"); |
Arnd Bergmann | 8c9478a | 2017-12-11 13:07:00 +0100 | [diff] [blame] | 132 | return 0; |
Tadeusz Struk | b3416fb | 2014-06-05 13:44:04 -0700 | [diff] [blame] | 133 | } |
| 134 | |
| 135 | static int qat_hal_wr_ae_csr(struct icp_qat_fw_loader_handle *handle, |
| 136 | unsigned char ae, unsigned int csr, |
| 137 | unsigned int value) |
| 138 | { |
| 139 | unsigned int iterations = CSR_RETRY_TIMES; |
| 140 | |
| 141 | do { |
| 142 | SET_AE_CSR(handle, ae, csr, value); |
| 143 | if (!(GET_AE_CSR(handle, ae, LOCAL_CSR_STATUS) & LCS_STATUS)) |
| 144 | return 0; |
| 145 | } while (iterations--); |
| 146 | |
| 147 | pr_err("QAT: Write CSR Timeout\n"); |
| 148 | return -EFAULT; |
| 149 | } |
| 150 | |
| 151 | static void qat_hal_get_wakeup_event(struct icp_qat_fw_loader_handle *handle, |
| 152 | unsigned char ae, unsigned char ctx, |
| 153 | unsigned int *events) |
| 154 | { |
| 155 | unsigned int cur_ctx; |
| 156 | |
Arnd Bergmann | 8c9478a | 2017-12-11 13:07:00 +0100 | [diff] [blame] | 157 | cur_ctx = qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER); |
Tadeusz Struk | b3416fb | 2014-06-05 13:44:04 -0700 | [diff] [blame] | 158 | qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx); |
Arnd Bergmann | 8c9478a | 2017-12-11 13:07:00 +0100 | [diff] [blame] | 159 | *events = qat_hal_rd_ae_csr(handle, ae, CTX_WAKEUP_EVENTS_INDIRECT); |
Tadeusz Struk | b3416fb | 2014-06-05 13:44:04 -0700 | [diff] [blame] | 160 | qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx); |
| 161 | } |
| 162 | |
| 163 | static int qat_hal_wait_cycles(struct icp_qat_fw_loader_handle *handle, |
| 164 | unsigned char ae, unsigned int cycles, |
| 165 | int chk_inactive) |
| 166 | { |
| 167 | unsigned int base_cnt = 0, cur_cnt = 0; |
| 168 | unsigned int csr = (1 << ACS_ABO_BITPOS); |
| 169 | int times = MAX_RETRY_TIMES; |
| 170 | int elapsed_cycles = 0; |
| 171 | |
Arnd Bergmann | 8c9478a | 2017-12-11 13:07:00 +0100 | [diff] [blame] | 172 | base_cnt = qat_hal_rd_ae_csr(handle, ae, PROFILE_COUNT); |
Tadeusz Struk | b3416fb | 2014-06-05 13:44:04 -0700 | [diff] [blame] | 173 | base_cnt &= 0xffff; |
| 174 | while ((int)cycles > elapsed_cycles && times--) { |
| 175 | if (chk_inactive) |
Arnd Bergmann | 8c9478a | 2017-12-11 13:07:00 +0100 | [diff] [blame] | 176 | csr = qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS); |
Tadeusz Struk | b3416fb | 2014-06-05 13:44:04 -0700 | [diff] [blame] | 177 | |
Arnd Bergmann | 8c9478a | 2017-12-11 13:07:00 +0100 | [diff] [blame] | 178 | cur_cnt = qat_hal_rd_ae_csr(handle, ae, PROFILE_COUNT); |
Tadeusz Struk | b3416fb | 2014-06-05 13:44:04 -0700 | [diff] [blame] | 179 | cur_cnt &= 0xffff; |
| 180 | elapsed_cycles = cur_cnt - base_cnt; |
| 181 | |
| 182 | if (elapsed_cycles < 0) |
| 183 | elapsed_cycles += 0x10000; |
| 184 | |
| 185 | /* ensure at least 8 time cycles elapsed in wait_cycles */ |
| 186 | if (elapsed_cycles >= 8 && !(csr & (1 << ACS_ABO_BITPOS))) |
| 187 | return 0; |
| 188 | } |
Pingchao Yang | 51d77dd | 2015-12-16 14:09:50 +0800 | [diff] [blame] | 189 | if (times < 0) { |
Tadeusz Struk | b3416fb | 2014-06-05 13:44:04 -0700 | [diff] [blame] | 190 | pr_err("QAT: wait_num_cycles time out\n"); |
| 191 | return -EFAULT; |
| 192 | } |
| 193 | return 0; |
| 194 | } |
| 195 | |
| 196 | #define CLR_BIT(wrd, bit) (wrd & ~(1 << bit)) |
| 197 | #define SET_BIT(wrd, bit) (wrd | 1 << bit) |
| 198 | |
| 199 | int qat_hal_set_ae_ctx_mode(struct icp_qat_fw_loader_handle *handle, |
| 200 | unsigned char ae, unsigned char mode) |
| 201 | { |
| 202 | unsigned int csr, new_csr; |
| 203 | |
| 204 | if ((mode != 4) && (mode != 8)) { |
| 205 | pr_err("QAT: bad ctx mode=%d\n", mode); |
| 206 | return -EINVAL; |
| 207 | } |
| 208 | |
| 209 | /* Sets the accelaration engine context mode to either four or eight */ |
Arnd Bergmann | 8c9478a | 2017-12-11 13:07:00 +0100 | [diff] [blame] | 210 | csr = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES); |
Tadeusz Struk | b3416fb | 2014-06-05 13:44:04 -0700 | [diff] [blame] | 211 | csr = IGNORE_W1C_MASK & csr; |
| 212 | new_csr = (mode == 4) ? |
| 213 | SET_BIT(csr, CE_INUSE_CONTEXTS_BITPOS) : |
| 214 | CLR_BIT(csr, CE_INUSE_CONTEXTS_BITPOS); |
| 215 | qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, new_csr); |
| 216 | return 0; |
| 217 | } |
| 218 | |
| 219 | int qat_hal_set_ae_nn_mode(struct icp_qat_fw_loader_handle *handle, |
| 220 | unsigned char ae, unsigned char mode) |
| 221 | { |
| 222 | unsigned int csr, new_csr; |
| 223 | |
Arnd Bergmann | 8c9478a | 2017-12-11 13:07:00 +0100 | [diff] [blame] | 224 | csr = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES); |
Tadeusz Struk | b3416fb | 2014-06-05 13:44:04 -0700 | [diff] [blame] | 225 | csr &= IGNORE_W1C_MASK; |
| 226 | |
| 227 | new_csr = (mode) ? |
| 228 | SET_BIT(csr, CE_NN_MODE_BITPOS) : |
| 229 | CLR_BIT(csr, CE_NN_MODE_BITPOS); |
| 230 | |
| 231 | if (new_csr != csr) |
| 232 | qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, new_csr); |
| 233 | |
| 234 | return 0; |
| 235 | } |
| 236 | |
| 237 | int qat_hal_set_ae_lm_mode(struct icp_qat_fw_loader_handle *handle, |
| 238 | unsigned char ae, enum icp_qat_uof_regtype lm_type, |
| 239 | unsigned char mode) |
| 240 | { |
| 241 | unsigned int csr, new_csr; |
| 242 | |
Arnd Bergmann | 8c9478a | 2017-12-11 13:07:00 +0100 | [diff] [blame] | 243 | csr = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES); |
Tadeusz Struk | b3416fb | 2014-06-05 13:44:04 -0700 | [diff] [blame] | 244 | csr &= IGNORE_W1C_MASK; |
| 245 | switch (lm_type) { |
| 246 | case ICP_LMEM0: |
| 247 | new_csr = (mode) ? |
| 248 | SET_BIT(csr, CE_LMADDR_0_GLOBAL_BITPOS) : |
| 249 | CLR_BIT(csr, CE_LMADDR_0_GLOBAL_BITPOS); |
| 250 | break; |
| 251 | case ICP_LMEM1: |
| 252 | new_csr = (mode) ? |
| 253 | SET_BIT(csr, CE_LMADDR_1_GLOBAL_BITPOS) : |
| 254 | CLR_BIT(csr, CE_LMADDR_1_GLOBAL_BITPOS); |
| 255 | break; |
| 256 | default: |
| 257 | pr_err("QAT: lmType = 0x%x\n", lm_type); |
| 258 | return -EINVAL; |
| 259 | } |
| 260 | |
| 261 | if (new_csr != csr) |
| 262 | qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, new_csr); |
| 263 | return 0; |
| 264 | } |
| 265 | |
| 266 | static unsigned short qat_hal_get_reg_addr(unsigned int type, |
| 267 | unsigned short reg_num) |
| 268 | { |
| 269 | unsigned short reg_addr; |
Tadeusz Struk | d65071e | 2014-06-24 15:19:34 -0700 | [diff] [blame] | 270 | |
Tadeusz Struk | b3416fb | 2014-06-05 13:44:04 -0700 | [diff] [blame] | 271 | switch (type) { |
| 272 | case ICP_GPA_ABS: |
| 273 | case ICP_GPB_ABS: |
| 274 | reg_addr = 0x80 | (reg_num & 0x7f); |
| 275 | break; |
| 276 | case ICP_GPA_REL: |
| 277 | case ICP_GPB_REL: |
| 278 | reg_addr = reg_num & 0x1f; |
| 279 | break; |
| 280 | case ICP_SR_RD_REL: |
| 281 | case ICP_SR_WR_REL: |
| 282 | case ICP_SR_REL: |
| 283 | reg_addr = 0x180 | (reg_num & 0x1f); |
| 284 | break; |
| 285 | case ICP_SR_ABS: |
| 286 | reg_addr = 0x140 | ((reg_num & 0x3) << 1); |
| 287 | break; |
| 288 | case ICP_DR_RD_REL: |
| 289 | case ICP_DR_WR_REL: |
| 290 | case ICP_DR_REL: |
| 291 | reg_addr = 0x1c0 | (reg_num & 0x1f); |
| 292 | break; |
| 293 | case ICP_DR_ABS: |
| 294 | reg_addr = 0x100 | ((reg_num & 0x3) << 1); |
| 295 | break; |
| 296 | case ICP_NEIGH_REL: |
| 297 | reg_addr = 0x280 | (reg_num & 0x1f); |
| 298 | break; |
| 299 | case ICP_LMEM0: |
| 300 | reg_addr = 0x200; |
| 301 | break; |
| 302 | case ICP_LMEM1: |
| 303 | reg_addr = 0x220; |
| 304 | break; |
| 305 | case ICP_NO_DEST: |
| 306 | reg_addr = 0x300 | (reg_num & 0xff); |
| 307 | break; |
| 308 | default: |
| 309 | reg_addr = BAD_REGADDR; |
| 310 | break; |
| 311 | } |
| 312 | return reg_addr; |
| 313 | } |
| 314 | |
| 315 | void qat_hal_reset(struct icp_qat_fw_loader_handle *handle) |
| 316 | { |
| 317 | unsigned int ae_reset_csr; |
| 318 | |
| 319 | ae_reset_csr = GET_GLB_CSR(handle, ICP_RESET); |
| 320 | ae_reset_csr |= handle->hal_handle->ae_mask << RST_CSR_AE_LSB; |
| 321 | ae_reset_csr |= handle->hal_handle->slice_mask << RST_CSR_QAT_LSB; |
| 322 | SET_GLB_CSR(handle, ICP_RESET, ae_reset_csr); |
| 323 | } |
| 324 | |
| 325 | static void qat_hal_wr_indr_csr(struct icp_qat_fw_loader_handle *handle, |
| 326 | unsigned char ae, unsigned int ctx_mask, |
| 327 | unsigned int ae_csr, unsigned int csr_val) |
| 328 | { |
| 329 | unsigned int ctx, cur_ctx; |
| 330 | |
Arnd Bergmann | 8c9478a | 2017-12-11 13:07:00 +0100 | [diff] [blame] | 331 | cur_ctx = qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER); |
Tadeusz Struk | b3416fb | 2014-06-05 13:44:04 -0700 | [diff] [blame] | 332 | |
| 333 | for (ctx = 0; ctx < ICP_QAT_UCLO_MAX_CTX; ctx++) { |
| 334 | if (!(ctx_mask & (1 << ctx))) |
| 335 | continue; |
| 336 | qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx); |
| 337 | qat_hal_wr_ae_csr(handle, ae, ae_csr, csr_val); |
| 338 | } |
| 339 | |
| 340 | qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx); |
| 341 | } |
| 342 | |
Arnd Bergmann | 8c9478a | 2017-12-11 13:07:00 +0100 | [diff] [blame] | 343 | static unsigned int qat_hal_rd_indr_csr(struct icp_qat_fw_loader_handle *handle, |
Tadeusz Struk | b3416fb | 2014-06-05 13:44:04 -0700 | [diff] [blame] | 344 | unsigned char ae, unsigned char ctx, |
Arnd Bergmann | 8c9478a | 2017-12-11 13:07:00 +0100 | [diff] [blame] | 345 | unsigned int ae_csr) |
Tadeusz Struk | b3416fb | 2014-06-05 13:44:04 -0700 | [diff] [blame] | 346 | { |
Arnd Bergmann | 8c9478a | 2017-12-11 13:07:00 +0100 | [diff] [blame] | 347 | unsigned int cur_ctx, csr_val; |
Tadeusz Struk | b3416fb | 2014-06-05 13:44:04 -0700 | [diff] [blame] | 348 | |
Arnd Bergmann | 8c9478a | 2017-12-11 13:07:00 +0100 | [diff] [blame] | 349 | cur_ctx = qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER); |
Tadeusz Struk | b3416fb | 2014-06-05 13:44:04 -0700 | [diff] [blame] | 350 | qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx); |
Arnd Bergmann | 8c9478a | 2017-12-11 13:07:00 +0100 | [diff] [blame] | 351 | csr_val = qat_hal_rd_ae_csr(handle, ae, ae_csr); |
Tadeusz Struk | b3416fb | 2014-06-05 13:44:04 -0700 | [diff] [blame] | 352 | qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx); |
Arnd Bergmann | 8c9478a | 2017-12-11 13:07:00 +0100 | [diff] [blame] | 353 | |
| 354 | return csr_val; |
Tadeusz Struk | b3416fb | 2014-06-05 13:44:04 -0700 | [diff] [blame] | 355 | } |
| 356 | |
| 357 | static void qat_hal_put_sig_event(struct icp_qat_fw_loader_handle *handle, |
| 358 | unsigned char ae, unsigned int ctx_mask, |
| 359 | unsigned int events) |
| 360 | { |
| 361 | unsigned int ctx, cur_ctx; |
| 362 | |
Arnd Bergmann | 8c9478a | 2017-12-11 13:07:00 +0100 | [diff] [blame] | 363 | cur_ctx = qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER); |
Tadeusz Struk | b3416fb | 2014-06-05 13:44:04 -0700 | [diff] [blame] | 364 | for (ctx = 0; ctx < ICP_QAT_UCLO_MAX_CTX; ctx++) { |
| 365 | if (!(ctx_mask & (1 << ctx))) |
| 366 | continue; |
| 367 | qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx); |
| 368 | qat_hal_wr_ae_csr(handle, ae, CTX_SIG_EVENTS_INDIRECT, events); |
| 369 | } |
| 370 | qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx); |
| 371 | } |
| 372 | |
| 373 | static void qat_hal_put_wakeup_event(struct icp_qat_fw_loader_handle *handle, |
| 374 | unsigned char ae, unsigned int ctx_mask, |
| 375 | unsigned int events) |
| 376 | { |
| 377 | unsigned int ctx, cur_ctx; |
| 378 | |
Arnd Bergmann | 8c9478a | 2017-12-11 13:07:00 +0100 | [diff] [blame] | 379 | cur_ctx = qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER); |
Tadeusz Struk | b3416fb | 2014-06-05 13:44:04 -0700 | [diff] [blame] | 380 | for (ctx = 0; ctx < ICP_QAT_UCLO_MAX_CTX; ctx++) { |
| 381 | if (!(ctx_mask & (1 << ctx))) |
| 382 | continue; |
| 383 | qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx); |
| 384 | qat_hal_wr_ae_csr(handle, ae, CTX_WAKEUP_EVENTS_INDIRECT, |
| 385 | events); |
| 386 | } |
| 387 | qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx); |
| 388 | } |
| 389 | |
| 390 | static int qat_hal_check_ae_alive(struct icp_qat_fw_loader_handle *handle) |
| 391 | { |
| 392 | unsigned int base_cnt, cur_cnt; |
| 393 | unsigned char ae; |
Pingchao Yang | c0e77a1 | 2016-01-07 09:39:46 +0800 | [diff] [blame] | 394 | int times = MAX_RETRY_TIMES; |
Tadeusz Struk | b3416fb | 2014-06-05 13:44:04 -0700 | [diff] [blame] | 395 | |
| 396 | for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) { |
Arnd Bergmann | 8c9478a | 2017-12-11 13:07:00 +0100 | [diff] [blame] | 397 | base_cnt = qat_hal_rd_ae_csr(handle, ae, PROFILE_COUNT); |
Tadeusz Struk | b3416fb | 2014-06-05 13:44:04 -0700 | [diff] [blame] | 398 | base_cnt &= 0xffff; |
| 399 | |
| 400 | do { |
Arnd Bergmann | 8c9478a | 2017-12-11 13:07:00 +0100 | [diff] [blame] | 401 | cur_cnt = qat_hal_rd_ae_csr(handle, ae, PROFILE_COUNT); |
Tadeusz Struk | b3416fb | 2014-06-05 13:44:04 -0700 | [diff] [blame] | 402 | cur_cnt &= 0xffff; |
| 403 | } while (times-- && (cur_cnt == base_cnt)); |
| 404 | |
Pingchao Yang | c0e77a1 | 2016-01-07 09:39:46 +0800 | [diff] [blame] | 405 | if (times < 0) { |
Tadeusz Struk | b3416fb | 2014-06-05 13:44:04 -0700 | [diff] [blame] | 406 | pr_err("QAT: AE%d is inactive!!\n", ae); |
| 407 | return -EFAULT; |
| 408 | } |
| 409 | } |
| 410 | |
| 411 | return 0; |
| 412 | } |
| 413 | |
Pingchao Yang | b027227 | 2015-12-04 16:56:23 -0800 | [diff] [blame] | 414 | int qat_hal_check_ae_active(struct icp_qat_fw_loader_handle *handle, |
| 415 | unsigned int ae) |
| 416 | { |
| 417 | unsigned int enable = 0, active = 0; |
| 418 | |
Arnd Bergmann | 8c9478a | 2017-12-11 13:07:00 +0100 | [diff] [blame] | 419 | enable = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES); |
| 420 | active = qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS); |
Pingchao Yang | 46621e6 | 2015-12-16 10:39:40 +0800 | [diff] [blame] | 421 | if ((enable & (0xff << CE_ENABLE_BITPOS)) || |
Pingchao Yang | b027227 | 2015-12-04 16:56:23 -0800 | [diff] [blame] | 422 | (active & (1 << ACS_ABO_BITPOS))) |
| 423 | return 1; |
| 424 | else |
| 425 | return 0; |
| 426 | } |
| 427 | |
Tadeusz Struk | b3416fb | 2014-06-05 13:44:04 -0700 | [diff] [blame] | 428 | static void qat_hal_reset_timestamp(struct icp_qat_fw_loader_handle *handle) |
| 429 | { |
| 430 | unsigned int misc_ctl; |
| 431 | unsigned char ae; |
| 432 | |
| 433 | /* stop the timestamp timers */ |
| 434 | misc_ctl = GET_GLB_CSR(handle, MISC_CONTROL); |
| 435 | if (misc_ctl & MC_TIMESTAMP_ENABLE) |
| 436 | SET_GLB_CSR(handle, MISC_CONTROL, misc_ctl & |
| 437 | (~MC_TIMESTAMP_ENABLE)); |
| 438 | |
Tadeusz Struk | 9a147cb | 2014-07-25 15:55:46 -0700 | [diff] [blame] | 439 | for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) { |
Tadeusz Struk | b3416fb | 2014-06-05 13:44:04 -0700 | [diff] [blame] | 440 | qat_hal_wr_ae_csr(handle, ae, TIMESTAMP_LOW, 0); |
| 441 | qat_hal_wr_ae_csr(handle, ae, TIMESTAMP_HIGH, 0); |
| 442 | } |
| 443 | /* start timestamp timers */ |
| 444 | SET_GLB_CSR(handle, MISC_CONTROL, misc_ctl | MC_TIMESTAMP_ENABLE); |
| 445 | } |
| 446 | |
Allan, Bruce W | af6f2a7 | 2015-03-31 09:30:45 -0700 | [diff] [blame] | 447 | #define ESRAM_AUTO_TINIT BIT(2) |
| 448 | #define ESRAM_AUTO_TINIT_DONE BIT(3) |
Tadeusz Struk | b3416fb | 2014-06-05 13:44:04 -0700 | [diff] [blame] | 449 | #define ESRAM_AUTO_INIT_USED_CYCLES (1640) |
| 450 | #define ESRAM_AUTO_INIT_CSR_OFFSET 0xC1C |
| 451 | static int qat_hal_init_esram(struct icp_qat_fw_loader_handle *handle) |
| 452 | { |
Pingchao Yang | b027227 | 2015-12-04 16:56:23 -0800 | [diff] [blame] | 453 | void __iomem *csr_addr = |
| 454 | (void __iomem *)((uintptr_t)handle->hal_ep_csr_addr_v + |
| 455 | ESRAM_AUTO_INIT_CSR_OFFSET); |
Pingchao Yang | c0e77a1 | 2016-01-07 09:39:46 +0800 | [diff] [blame] | 456 | unsigned int csr_val; |
| 457 | int times = 30; |
Tadeusz Struk | b3416fb | 2014-06-05 13:44:04 -0700 | [diff] [blame] | 458 | |
Giovanni Cabiddu | 685ce06 | 2016-12-22 15:00:24 +0000 | [diff] [blame] | 459 | if (handle->pci_dev->device != ADF_DH895XCC_PCI_DEVICE_ID) |
Tadeusz Struk | 70401f4 | 2016-01-08 10:19:54 -0800 | [diff] [blame] | 460 | return 0; |
| 461 | |
Tadeusz Struk | b3416fb | 2014-06-05 13:44:04 -0700 | [diff] [blame] | 462 | csr_val = ADF_CSR_RD(csr_addr, 0); |
| 463 | if ((csr_val & ESRAM_AUTO_TINIT) && (csr_val & ESRAM_AUTO_TINIT_DONE)) |
| 464 | return 0; |
| 465 | |
| 466 | csr_val = ADF_CSR_RD(csr_addr, 0); |
| 467 | csr_val |= ESRAM_AUTO_TINIT; |
| 468 | ADF_CSR_WR(csr_addr, 0, csr_val); |
| 469 | |
| 470 | do { |
| 471 | qat_hal_wait_cycles(handle, 0, ESRAM_AUTO_INIT_USED_CYCLES, 0); |
| 472 | csr_val = ADF_CSR_RD(csr_addr, 0); |
| 473 | } while (!(csr_val & ESRAM_AUTO_TINIT_DONE) && times--); |
Pingchao Yang | c0e77a1 | 2016-01-07 09:39:46 +0800 | [diff] [blame] | 474 | if ((times < 0)) { |
Tadeusz Struk | b3416fb | 2014-06-05 13:44:04 -0700 | [diff] [blame] | 475 | pr_err("QAT: Fail to init eSram!\n"); |
| 476 | return -EFAULT; |
| 477 | } |
| 478 | return 0; |
| 479 | } |
| 480 | |
| 481 | #define SHRAM_INIT_CYCLES 2060 |
| 482 | int qat_hal_clr_reset(struct icp_qat_fw_loader_handle *handle) |
| 483 | { |
| 484 | unsigned int ae_reset_csr; |
| 485 | unsigned char ae; |
| 486 | unsigned int clk_csr; |
| 487 | unsigned int times = 100; |
| 488 | unsigned int csr; |
| 489 | |
| 490 | /* write to the reset csr */ |
| 491 | ae_reset_csr = GET_GLB_CSR(handle, ICP_RESET); |
| 492 | ae_reset_csr &= ~(handle->hal_handle->ae_mask << RST_CSR_AE_LSB); |
| 493 | ae_reset_csr &= ~(handle->hal_handle->slice_mask << RST_CSR_QAT_LSB); |
| 494 | do { |
| 495 | SET_GLB_CSR(handle, ICP_RESET, ae_reset_csr); |
| 496 | if (!(times--)) |
| 497 | goto out_err; |
| 498 | csr = GET_GLB_CSR(handle, ICP_RESET); |
| 499 | } while ((handle->hal_handle->ae_mask | |
| 500 | (handle->hal_handle->slice_mask << RST_CSR_QAT_LSB)) & csr); |
| 501 | /* enable clock */ |
| 502 | clk_csr = GET_GLB_CSR(handle, ICP_GLOBAL_CLK_ENABLE); |
| 503 | clk_csr |= handle->hal_handle->ae_mask << 0; |
| 504 | clk_csr |= handle->hal_handle->slice_mask << 20; |
| 505 | SET_GLB_CSR(handle, ICP_GLOBAL_CLK_ENABLE, clk_csr); |
| 506 | if (qat_hal_check_ae_alive(handle)) |
| 507 | goto out_err; |
| 508 | |
| 509 | /* Set undefined power-up/reset states to reasonable default values */ |
Tadeusz Struk | 9a147cb | 2014-07-25 15:55:46 -0700 | [diff] [blame] | 510 | for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) { |
Tadeusz Struk | b3416fb | 2014-06-05 13:44:04 -0700 | [diff] [blame] | 511 | qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, |
| 512 | INIT_CTX_ENABLE_VALUE); |
| 513 | qat_hal_wr_indr_csr(handle, ae, ICP_QAT_UCLO_AE_ALL_CTX, |
| 514 | CTX_STS_INDIRECT, |
| 515 | handle->hal_handle->upc_mask & |
| 516 | INIT_PC_VALUE); |
| 517 | qat_hal_wr_ae_csr(handle, ae, CTX_ARB_CNTL, INIT_CTX_ARB_VALUE); |
| 518 | qat_hal_wr_ae_csr(handle, ae, CC_ENABLE, INIT_CCENABLE_VALUE); |
| 519 | qat_hal_put_wakeup_event(handle, ae, |
| 520 | ICP_QAT_UCLO_AE_ALL_CTX, |
| 521 | INIT_WAKEUP_EVENTS_VALUE); |
| 522 | qat_hal_put_sig_event(handle, ae, |
| 523 | ICP_QAT_UCLO_AE_ALL_CTX, |
| 524 | INIT_SIG_EVENTS_VALUE); |
| 525 | } |
| 526 | if (qat_hal_init_esram(handle)) |
| 527 | goto out_err; |
| 528 | if (qat_hal_wait_cycles(handle, 0, SHRAM_INIT_CYCLES, 0)) |
| 529 | goto out_err; |
| 530 | qat_hal_reset_timestamp(handle); |
| 531 | |
| 532 | return 0; |
| 533 | out_err: |
| 534 | pr_err("QAT: failed to get device out of reset\n"); |
| 535 | return -EFAULT; |
| 536 | } |
| 537 | |
| 538 | static void qat_hal_disable_ctx(struct icp_qat_fw_loader_handle *handle, |
| 539 | unsigned char ae, unsigned int ctx_mask) |
| 540 | { |
| 541 | unsigned int ctx; |
| 542 | |
Arnd Bergmann | 8c9478a | 2017-12-11 13:07:00 +0100 | [diff] [blame] | 543 | ctx = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES); |
Tadeusz Struk | b3416fb | 2014-06-05 13:44:04 -0700 | [diff] [blame] | 544 | ctx &= IGNORE_W1C_MASK & |
| 545 | (~((ctx_mask & ICP_QAT_UCLO_AE_ALL_CTX) << CE_ENABLE_BITPOS)); |
| 546 | qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx); |
| 547 | } |
| 548 | |
| 549 | static uint64_t qat_hal_parity_64bit(uint64_t word) |
| 550 | { |
| 551 | word ^= word >> 1; |
| 552 | word ^= word >> 2; |
| 553 | word ^= word >> 4; |
| 554 | word ^= word >> 8; |
| 555 | word ^= word >> 16; |
| 556 | word ^= word >> 32; |
| 557 | return word & 1; |
| 558 | } |
| 559 | |
| 560 | static uint64_t qat_hal_set_uword_ecc(uint64_t uword) |
| 561 | { |
| 562 | uint64_t bit0_mask = 0xff800007fffULL, bit1_mask = 0x1f801ff801fULL, |
| 563 | bit2_mask = 0xe387e0781e1ULL, bit3_mask = 0x7cb8e388e22ULL, |
| 564 | bit4_mask = 0xaf5b2c93244ULL, bit5_mask = 0xf56d5525488ULL, |
| 565 | bit6_mask = 0xdaf69a46910ULL; |
| 566 | |
| 567 | /* clear the ecc bits */ |
| 568 | uword &= ~(0x7fULL << 0x2C); |
| 569 | uword |= qat_hal_parity_64bit(bit0_mask & uword) << 0x2C; |
| 570 | uword |= qat_hal_parity_64bit(bit1_mask & uword) << 0x2D; |
| 571 | uword |= qat_hal_parity_64bit(bit2_mask & uword) << 0x2E; |
| 572 | uword |= qat_hal_parity_64bit(bit3_mask & uword) << 0x2F; |
| 573 | uword |= qat_hal_parity_64bit(bit4_mask & uword) << 0x30; |
| 574 | uword |= qat_hal_parity_64bit(bit5_mask & uword) << 0x31; |
| 575 | uword |= qat_hal_parity_64bit(bit6_mask & uword) << 0x32; |
| 576 | return uword; |
| 577 | } |
| 578 | |
| 579 | void qat_hal_wr_uwords(struct icp_qat_fw_loader_handle *handle, |
| 580 | unsigned char ae, unsigned int uaddr, |
| 581 | unsigned int words_num, uint64_t *uword) |
| 582 | { |
| 583 | unsigned int ustore_addr; |
| 584 | unsigned int i; |
| 585 | |
Arnd Bergmann | 8c9478a | 2017-12-11 13:07:00 +0100 | [diff] [blame] | 586 | ustore_addr = qat_hal_rd_ae_csr(handle, ae, USTORE_ADDRESS); |
Tadeusz Struk | b3416fb | 2014-06-05 13:44:04 -0700 | [diff] [blame] | 587 | uaddr |= UA_ECS; |
| 588 | qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr); |
| 589 | for (i = 0; i < words_num; i++) { |
| 590 | unsigned int uwrd_lo, uwrd_hi; |
| 591 | uint64_t tmp; |
| 592 | |
| 593 | tmp = qat_hal_set_uword_ecc(uword[i]); |
| 594 | uwrd_lo = (unsigned int)(tmp & 0xffffffff); |
| 595 | uwrd_hi = (unsigned int)(tmp >> 0x20); |
| 596 | qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_LOWER, uwrd_lo); |
| 597 | qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_UPPER, uwrd_hi); |
| 598 | } |
| 599 | qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, ustore_addr); |
| 600 | } |
| 601 | |
| 602 | static void qat_hal_enable_ctx(struct icp_qat_fw_loader_handle *handle, |
| 603 | unsigned char ae, unsigned int ctx_mask) |
| 604 | { |
| 605 | unsigned int ctx; |
| 606 | |
Arnd Bergmann | 8c9478a | 2017-12-11 13:07:00 +0100 | [diff] [blame] | 607 | ctx = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES); |
Tadeusz Struk | b3416fb | 2014-06-05 13:44:04 -0700 | [diff] [blame] | 608 | ctx &= IGNORE_W1C_MASK; |
| 609 | ctx_mask &= (ctx & CE_INUSE_CONTEXTS) ? 0x55 : 0xFF; |
| 610 | ctx |= (ctx_mask << CE_ENABLE_BITPOS); |
| 611 | qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx); |
| 612 | } |
| 613 | |
Pingchao Yang | b027227 | 2015-12-04 16:56:23 -0800 | [diff] [blame] | 614 | static void qat_hal_clear_xfer(struct icp_qat_fw_loader_handle *handle) |
Tadeusz Struk | b3416fb | 2014-06-05 13:44:04 -0700 | [diff] [blame] | 615 | { |
| 616 | unsigned char ae; |
Tadeusz Struk | b3416fb | 2014-06-05 13:44:04 -0700 | [diff] [blame] | 617 | unsigned short reg; |
Tadeusz Struk | b3416fb | 2014-06-05 13:44:04 -0700 | [diff] [blame] | 618 | |
Tadeusz Struk | 9a147cb | 2014-07-25 15:55:46 -0700 | [diff] [blame] | 619 | for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) { |
Tadeusz Struk | b3416fb | 2014-06-05 13:44:04 -0700 | [diff] [blame] | 620 | for (reg = 0; reg < ICP_QAT_UCLO_MAX_GPR_REG; reg++) { |
| 621 | qat_hal_init_rd_xfer(handle, ae, 0, ICP_SR_RD_ABS, |
| 622 | reg, 0); |
| 623 | qat_hal_init_rd_xfer(handle, ae, 0, ICP_DR_RD_ABS, |
| 624 | reg, 0); |
| 625 | } |
Pingchao Yang | b027227 | 2015-12-04 16:56:23 -0800 | [diff] [blame] | 626 | } |
| 627 | } |
| 628 | |
| 629 | static int qat_hal_clear_gpr(struct icp_qat_fw_loader_handle *handle) |
| 630 | { |
| 631 | unsigned char ae; |
| 632 | unsigned int ctx_mask = ICP_QAT_UCLO_AE_ALL_CTX; |
| 633 | int times = MAX_RETRY_TIMES; |
| 634 | unsigned int csr_val = 0; |
| 635 | unsigned int savctx = 0; |
| 636 | int ret = 0; |
| 637 | |
| 638 | for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) { |
Arnd Bergmann | 8c9478a | 2017-12-11 13:07:00 +0100 | [diff] [blame] | 639 | csr_val = qat_hal_rd_ae_csr(handle, ae, AE_MISC_CONTROL); |
Tadeusz Struk | b3416fb | 2014-06-05 13:44:04 -0700 | [diff] [blame] | 640 | csr_val &= ~(1 << MMC_SHARE_CS_BITPOS); |
| 641 | qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL, csr_val); |
Arnd Bergmann | 8c9478a | 2017-12-11 13:07:00 +0100 | [diff] [blame] | 642 | csr_val = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES); |
Tadeusz Struk | b3416fb | 2014-06-05 13:44:04 -0700 | [diff] [blame] | 643 | csr_val &= IGNORE_W1C_MASK; |
| 644 | csr_val |= CE_NN_MODE; |
| 645 | qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, csr_val); |
| 646 | qat_hal_wr_uwords(handle, ae, 0, ARRAY_SIZE(inst), |
| 647 | (uint64_t *)inst); |
| 648 | qat_hal_wr_indr_csr(handle, ae, ctx_mask, CTX_STS_INDIRECT, |
| 649 | handle->hal_handle->upc_mask & |
| 650 | INIT_PC_VALUE); |
Arnd Bergmann | 8c9478a | 2017-12-11 13:07:00 +0100 | [diff] [blame] | 651 | savctx = qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS); |
Tadeusz Struk | b3416fb | 2014-06-05 13:44:04 -0700 | [diff] [blame] | 652 | qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS, 0); |
| 653 | qat_hal_put_wakeup_event(handle, ae, ctx_mask, XCWE_VOLUNTARY); |
| 654 | qat_hal_wr_indr_csr(handle, ae, ctx_mask, |
| 655 | CTX_SIG_EVENTS_INDIRECT, 0); |
| 656 | qat_hal_wr_ae_csr(handle, ae, CTX_SIG_EVENTS_ACTIVE, 0); |
| 657 | qat_hal_enable_ctx(handle, ae, ctx_mask); |
| 658 | } |
Tadeusz Struk | 9a147cb | 2014-07-25 15:55:46 -0700 | [diff] [blame] | 659 | for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) { |
Tadeusz Struk | b3416fb | 2014-06-05 13:44:04 -0700 | [diff] [blame] | 660 | /* wait for AE to finish */ |
| 661 | do { |
| 662 | ret = qat_hal_wait_cycles(handle, ae, 20, 1); |
| 663 | } while (ret && times--); |
| 664 | |
Pingchao Yang | c0e77a1 | 2016-01-07 09:39:46 +0800 | [diff] [blame] | 665 | if (times < 0) { |
Tadeusz Struk | b3416fb | 2014-06-05 13:44:04 -0700 | [diff] [blame] | 666 | pr_err("QAT: clear GPR of AE %d failed", ae); |
| 667 | return -EINVAL; |
| 668 | } |
| 669 | qat_hal_disable_ctx(handle, ae, ctx_mask); |
| 670 | qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS, |
| 671 | savctx & ACS_ACNO); |
| 672 | qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, |
| 673 | INIT_CTX_ENABLE_VALUE); |
| 674 | qat_hal_wr_indr_csr(handle, ae, ctx_mask, CTX_STS_INDIRECT, |
| 675 | handle->hal_handle->upc_mask & |
| 676 | INIT_PC_VALUE); |
| 677 | qat_hal_wr_ae_csr(handle, ae, CTX_ARB_CNTL, INIT_CTX_ARB_VALUE); |
| 678 | qat_hal_wr_ae_csr(handle, ae, CC_ENABLE, INIT_CCENABLE_VALUE); |
| 679 | qat_hal_put_wakeup_event(handle, ae, ctx_mask, |
| 680 | INIT_WAKEUP_EVENTS_VALUE); |
| 681 | qat_hal_put_sig_event(handle, ae, ctx_mask, |
| 682 | INIT_SIG_EVENTS_VALUE); |
| 683 | } |
| 684 | return 0; |
| 685 | } |
| 686 | |
Pingchao Yang | b027227 | 2015-12-04 16:56:23 -0800 | [diff] [blame] | 687 | #define ICP_QAT_AE_OFFSET 0x20000 |
| 688 | #define ICP_QAT_CAP_OFFSET (ICP_QAT_AE_OFFSET + 0x10000) |
Tadeusz Struk | b3416fb | 2014-06-05 13:44:04 -0700 | [diff] [blame] | 689 | #define LOCAL_TO_XFER_REG_OFFSET 0x800 |
Pingchao Yang | b027227 | 2015-12-04 16:56:23 -0800 | [diff] [blame] | 690 | #define ICP_QAT_EP_OFFSET 0x3a000 |
Tadeusz Struk | b3416fb | 2014-06-05 13:44:04 -0700 | [diff] [blame] | 691 | int qat_hal_init(struct adf_accel_dev *accel_dev) |
| 692 | { |
Tadeusz Struk | 9a147cb | 2014-07-25 15:55:46 -0700 | [diff] [blame] | 693 | unsigned char ae; |
| 694 | unsigned int max_en_ae_id = 0; |
| 695 | struct icp_qat_fw_loader_handle *handle; |
Tadeusz Struk | b3416fb | 2014-06-05 13:44:04 -0700 | [diff] [blame] | 696 | struct adf_accel_pci *pci_info = &accel_dev->accel_pci_dev; |
| 697 | struct adf_hw_device_data *hw_data = accel_dev->hw_device; |
Pingchao Yang | f3dd7e6 | 2015-07-15 15:28:26 -0700 | [diff] [blame] | 698 | struct adf_bar *misc_bar = |
Tadeusz Struk | a727c4b6 | 2014-10-21 11:01:06 -0700 | [diff] [blame] | 699 | &pci_info->pci_bars[hw_data->get_misc_bar_id(hw_data)]; |
Pingchao Yang | 91a93ea | 2016-01-06 17:56:34 +0800 | [diff] [blame] | 700 | struct adf_bar *sram_bar; |
Tadeusz Struk | b3416fb | 2014-06-05 13:44:04 -0700 | [diff] [blame] | 701 | |
| 702 | handle = kzalloc(sizeof(*handle), GFP_KERNEL); |
| 703 | if (!handle) |
Tadeusz Struk | 9a147cb | 2014-07-25 15:55:46 -0700 | [diff] [blame] | 704 | return -ENOMEM; |
Tadeusz Struk | b3416fb | 2014-06-05 13:44:04 -0700 | [diff] [blame] | 705 | |
Pingchao Yang | b027227 | 2015-12-04 16:56:23 -0800 | [diff] [blame] | 706 | handle->hal_cap_g_ctl_csr_addr_v = |
| 707 | (void __iomem *)((uintptr_t)misc_bar->virt_addr + |
| 708 | ICP_QAT_CAP_OFFSET); |
| 709 | handle->hal_cap_ae_xfer_csr_addr_v = |
| 710 | (void __iomem *)((uintptr_t)misc_bar->virt_addr + |
| 711 | ICP_QAT_AE_OFFSET); |
| 712 | handle->hal_ep_csr_addr_v = |
| 713 | (void __iomem *)((uintptr_t)misc_bar->virt_addr + |
| 714 | ICP_QAT_EP_OFFSET); |
| 715 | handle->hal_cap_ae_local_csr_addr_v = |
| 716 | (void __iomem *)((uintptr_t)handle->hal_cap_ae_xfer_csr_addr_v + |
| 717 | LOCAL_TO_XFER_REG_OFFSET); |
| 718 | handle->pci_dev = pci_info->pci_dev; |
Giovanni Cabiddu | 685ce06 | 2016-12-22 15:00:24 +0000 | [diff] [blame] | 719 | if (handle->pci_dev->device == ADF_DH895XCC_PCI_DEVICE_ID) { |
Pingchao Yang | 91a93ea | 2016-01-06 17:56:34 +0800 | [diff] [blame] | 720 | sram_bar = |
| 721 | &pci_info->pci_bars[hw_data->get_sram_bar_id(hw_data)]; |
| 722 | handle->hal_sram_addr_v = sram_bar->virt_addr; |
| 723 | } |
Pingchao Yang | b027227 | 2015-12-04 16:56:23 -0800 | [diff] [blame] | 724 | handle->fw_auth = (handle->pci_dev->device == |
| 725 | ADF_DH895XCC_PCI_DEVICE_ID) ? false : true; |
Tadeusz Struk | b3416fb | 2014-06-05 13:44:04 -0700 | [diff] [blame] | 726 | handle->hal_handle = kzalloc(sizeof(*handle->hal_handle), GFP_KERNEL); |
| 727 | if (!handle->hal_handle) |
| 728 | goto out_hal_handle; |
| 729 | handle->hal_handle->revision_id = accel_dev->accel_pci_dev.revid; |
| 730 | handle->hal_handle->ae_mask = hw_data->ae_mask; |
| 731 | handle->hal_handle->slice_mask = hw_data->accel_mask; |
| 732 | /* create AE objects */ |
| 733 | handle->hal_handle->upc_mask = 0x1ffff; |
| 734 | handle->hal_handle->max_ustore = 0x4000; |
| 735 | for (ae = 0; ae < ICP_QAT_UCLO_MAX_AE; ae++) { |
| 736 | if (!(hw_data->ae_mask & (1 << ae))) |
| 737 | continue; |
| 738 | handle->hal_handle->aes[ae].free_addr = 0; |
| 739 | handle->hal_handle->aes[ae].free_size = |
| 740 | handle->hal_handle->max_ustore; |
| 741 | handle->hal_handle->aes[ae].ustore_size = |
| 742 | handle->hal_handle->max_ustore; |
| 743 | handle->hal_handle->aes[ae].live_ctx_mask = |
| 744 | ICP_QAT_UCLO_AE_ALL_CTX; |
Tadeusz Struk | 9a147cb | 2014-07-25 15:55:46 -0700 | [diff] [blame] | 745 | max_en_ae_id = ae; |
Tadeusz Struk | b3416fb | 2014-06-05 13:44:04 -0700 | [diff] [blame] | 746 | } |
Tadeusz Struk | 9a147cb | 2014-07-25 15:55:46 -0700 | [diff] [blame] | 747 | handle->hal_handle->ae_max_num = max_en_ae_id + 1; |
Tadeusz Struk | b3416fb | 2014-06-05 13:44:04 -0700 | [diff] [blame] | 748 | /* take all AEs out of reset */ |
| 749 | if (qat_hal_clr_reset(handle)) { |
Allan, Bruce W | 6655030 | 2015-03-19 16:03:44 -0700 | [diff] [blame] | 750 | dev_err(&GET_DEV(accel_dev), "qat_hal_clr_reset error\n"); |
Tadeusz Struk | b3416fb | 2014-06-05 13:44:04 -0700 | [diff] [blame] | 751 | goto out_err; |
| 752 | } |
Pingchao Yang | b027227 | 2015-12-04 16:56:23 -0800 | [diff] [blame] | 753 | qat_hal_clear_xfer(handle); |
| 754 | if (!handle->fw_auth) { |
| 755 | if (qat_hal_clear_gpr(handle)) |
| 756 | goto out_err; |
| 757 | } |
| 758 | |
Tadeusz Struk | b3416fb | 2014-06-05 13:44:04 -0700 | [diff] [blame] | 759 | /* Set SIGNATURE_ENABLE[0] to 0x1 in order to enable ALU_OUT csr */ |
Tadeusz Struk | 9a147cb | 2014-07-25 15:55:46 -0700 | [diff] [blame] | 760 | for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) { |
| 761 | unsigned int csr_val = 0; |
| 762 | |
Arnd Bergmann | 8c9478a | 2017-12-11 13:07:00 +0100 | [diff] [blame] | 763 | csr_val = qat_hal_rd_ae_csr(handle, ae, SIGNATURE_ENABLE); |
Tadeusz Struk | b3416fb | 2014-06-05 13:44:04 -0700 | [diff] [blame] | 764 | csr_val |= 0x1; |
| 765 | qat_hal_wr_ae_csr(handle, ae, SIGNATURE_ENABLE, csr_val); |
| 766 | } |
| 767 | accel_dev->fw_loader->fw_loader = handle; |
| 768 | return 0; |
| 769 | |
| 770 | out_err: |
| 771 | kfree(handle->hal_handle); |
| 772 | out_hal_handle: |
| 773 | kfree(handle); |
Tadeusz Struk | b3416fb | 2014-06-05 13:44:04 -0700 | [diff] [blame] | 774 | return -EFAULT; |
| 775 | } |
| 776 | |
| 777 | void qat_hal_deinit(struct icp_qat_fw_loader_handle *handle) |
| 778 | { |
| 779 | if (!handle) |
| 780 | return; |
| 781 | kfree(handle->hal_handle); |
| 782 | kfree(handle); |
| 783 | } |
| 784 | |
| 785 | void qat_hal_start(struct icp_qat_fw_loader_handle *handle, unsigned char ae, |
| 786 | unsigned int ctx_mask) |
| 787 | { |
Pingchao Yang | b027227 | 2015-12-04 16:56:23 -0800 | [diff] [blame] | 788 | int retry = 0; |
| 789 | unsigned int fcu_sts = 0; |
| 790 | |
| 791 | if (handle->fw_auth) { |
| 792 | SET_CAP_CSR(handle, FCU_CONTROL, FCU_CTRL_CMD_START); |
| 793 | do { |
| 794 | msleep(FW_AUTH_WAIT_PERIOD); |
| 795 | fcu_sts = GET_CAP_CSR(handle, FCU_STATUS); |
| 796 | if (((fcu_sts >> FCU_STS_DONE_POS) & 0x1)) |
| 797 | return; |
| 798 | } while (retry++ < FW_AUTH_MAX_RETRY); |
| 799 | pr_err("QAT: start error (AE 0x%x FCU_STS = 0x%x)\n", ae, |
| 800 | fcu_sts); |
| 801 | } else { |
| 802 | qat_hal_put_wakeup_event(handle, ae, (~ctx_mask) & |
Tadeusz Struk | b3416fb | 2014-06-05 13:44:04 -0700 | [diff] [blame] | 803 | ICP_QAT_UCLO_AE_ALL_CTX, 0x10000); |
Pingchao Yang | b027227 | 2015-12-04 16:56:23 -0800 | [diff] [blame] | 804 | qat_hal_enable_ctx(handle, ae, ctx_mask); |
| 805 | } |
Tadeusz Struk | b3416fb | 2014-06-05 13:44:04 -0700 | [diff] [blame] | 806 | } |
| 807 | |
| 808 | void qat_hal_stop(struct icp_qat_fw_loader_handle *handle, unsigned char ae, |
| 809 | unsigned int ctx_mask) |
| 810 | { |
Pingchao Yang | b027227 | 2015-12-04 16:56:23 -0800 | [diff] [blame] | 811 | if (!handle->fw_auth) |
| 812 | qat_hal_disable_ctx(handle, ae, ctx_mask); |
Tadeusz Struk | b3416fb | 2014-06-05 13:44:04 -0700 | [diff] [blame] | 813 | } |
| 814 | |
| 815 | void qat_hal_set_pc(struct icp_qat_fw_loader_handle *handle, |
| 816 | unsigned char ae, unsigned int ctx_mask, unsigned int upc) |
| 817 | { |
| 818 | qat_hal_wr_indr_csr(handle, ae, ctx_mask, CTX_STS_INDIRECT, |
| 819 | handle->hal_handle->upc_mask & upc); |
| 820 | } |
| 821 | |
| 822 | static void qat_hal_get_uwords(struct icp_qat_fw_loader_handle *handle, |
| 823 | unsigned char ae, unsigned int uaddr, |
| 824 | unsigned int words_num, uint64_t *uword) |
| 825 | { |
| 826 | unsigned int i, uwrd_lo, uwrd_hi; |
| 827 | unsigned int ustore_addr, misc_control; |
| 828 | |
Arnd Bergmann | 8c9478a | 2017-12-11 13:07:00 +0100 | [diff] [blame] | 829 | misc_control = qat_hal_rd_ae_csr(handle, ae, AE_MISC_CONTROL); |
Tadeusz Struk | b3416fb | 2014-06-05 13:44:04 -0700 | [diff] [blame] | 830 | qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL, |
| 831 | misc_control & 0xfffffffb); |
Arnd Bergmann | 8c9478a | 2017-12-11 13:07:00 +0100 | [diff] [blame] | 832 | ustore_addr = qat_hal_rd_ae_csr(handle, ae, USTORE_ADDRESS); |
Tadeusz Struk | b3416fb | 2014-06-05 13:44:04 -0700 | [diff] [blame] | 833 | uaddr |= UA_ECS; |
| 834 | for (i = 0; i < words_num; i++) { |
| 835 | qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr); |
| 836 | uaddr++; |
Arnd Bergmann | 8c9478a | 2017-12-11 13:07:00 +0100 | [diff] [blame] | 837 | uwrd_lo = qat_hal_rd_ae_csr(handle, ae, USTORE_DATA_LOWER); |
| 838 | uwrd_hi = qat_hal_rd_ae_csr(handle, ae, USTORE_DATA_UPPER); |
Tadeusz Struk | b3416fb | 2014-06-05 13:44:04 -0700 | [diff] [blame] | 839 | uword[i] = uwrd_hi; |
| 840 | uword[i] = (uword[i] << 0x20) | uwrd_lo; |
| 841 | } |
| 842 | qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL, misc_control); |
| 843 | qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, ustore_addr); |
| 844 | } |
| 845 | |
Tadeusz Struk | b3416fb | 2014-06-05 13:44:04 -0700 | [diff] [blame] | 846 | void qat_hal_wr_umem(struct icp_qat_fw_loader_handle *handle, |
| 847 | unsigned char ae, unsigned int uaddr, |
| 848 | unsigned int words_num, unsigned int *data) |
| 849 | { |
| 850 | unsigned int i, ustore_addr; |
| 851 | |
Arnd Bergmann | 8c9478a | 2017-12-11 13:07:00 +0100 | [diff] [blame] | 852 | ustore_addr = qat_hal_rd_ae_csr(handle, ae, USTORE_ADDRESS); |
Tadeusz Struk | b3416fb | 2014-06-05 13:44:04 -0700 | [diff] [blame] | 853 | uaddr |= UA_ECS; |
| 854 | qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr); |
| 855 | for (i = 0; i < words_num; i++) { |
| 856 | unsigned int uwrd_lo, uwrd_hi, tmp; |
Tadeusz Struk | d65071e | 2014-06-24 15:19:34 -0700 | [diff] [blame] | 857 | |
Tadeusz Struk | b3416fb | 2014-06-05 13:44:04 -0700 | [diff] [blame] | 858 | uwrd_lo = ((data[i] & 0xfff0000) << 4) | (0x3 << 18) | |
| 859 | ((data[i] & 0xff00) << 2) | |
| 860 | (0x3 << 8) | (data[i] & 0xff); |
| 861 | uwrd_hi = (0xf << 4) | ((data[i] & 0xf0000000) >> 28); |
Tadeusz Struk | d9a44ab | 2014-07-25 15:55:57 -0700 | [diff] [blame] | 862 | uwrd_hi |= (hweight32(data[i] & 0xffff) & 0x1) << 8; |
Tadeusz Struk | b3416fb | 2014-06-05 13:44:04 -0700 | [diff] [blame] | 863 | tmp = ((data[i] >> 0x10) & 0xffff); |
Tadeusz Struk | d9a44ab | 2014-07-25 15:55:57 -0700 | [diff] [blame] | 864 | uwrd_hi |= (hweight32(tmp) & 0x1) << 9; |
Tadeusz Struk | b3416fb | 2014-06-05 13:44:04 -0700 | [diff] [blame] | 865 | qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_LOWER, uwrd_lo); |
| 866 | qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_UPPER, uwrd_hi); |
| 867 | } |
| 868 | qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, ustore_addr); |
| 869 | } |
| 870 | |
| 871 | #define MAX_EXEC_INST 100 |
| 872 | static int qat_hal_exec_micro_inst(struct icp_qat_fw_loader_handle *handle, |
| 873 | unsigned char ae, unsigned char ctx, |
| 874 | uint64_t *micro_inst, unsigned int inst_num, |
| 875 | int code_off, unsigned int max_cycle, |
| 876 | unsigned int *endpc) |
| 877 | { |
| 878 | uint64_t savuwords[MAX_EXEC_INST]; |
| 879 | unsigned int ind_lm_addr0, ind_lm_addr1; |
| 880 | unsigned int ind_lm_addr_byte0, ind_lm_addr_byte1; |
| 881 | unsigned int ind_cnt_sig; |
| 882 | unsigned int ind_sig, act_sig; |
| 883 | unsigned int csr_val = 0, newcsr_val; |
| 884 | unsigned int savctx; |
| 885 | unsigned int savcc, wakeup_events, savpc; |
| 886 | unsigned int ctxarb_ctl, ctx_enables; |
| 887 | |
| 888 | if ((inst_num > handle->hal_handle->max_ustore) || !micro_inst) { |
Tadeusz Struk | 6899172 | 2014-07-25 15:55:52 -0700 | [diff] [blame] | 889 | pr_err("QAT: invalid instruction num %d\n", inst_num); |
Tadeusz Struk | b3416fb | 2014-06-05 13:44:04 -0700 | [diff] [blame] | 890 | return -EINVAL; |
| 891 | } |
| 892 | /* save current context */ |
Arnd Bergmann | 8c9478a | 2017-12-11 13:07:00 +0100 | [diff] [blame] | 893 | ind_lm_addr0 = qat_hal_rd_indr_csr(handle, ae, ctx, LM_ADDR_0_INDIRECT); |
| 894 | ind_lm_addr1 = qat_hal_rd_indr_csr(handle, ae, ctx, LM_ADDR_1_INDIRECT); |
| 895 | ind_lm_addr_byte0 = qat_hal_rd_indr_csr(handle, ae, ctx, |
| 896 | INDIRECT_LM_ADDR_0_BYTE_INDEX); |
| 897 | ind_lm_addr_byte1 = qat_hal_rd_indr_csr(handle, ae, ctx, |
| 898 | INDIRECT_LM_ADDR_1_BYTE_INDEX); |
Tadeusz Struk | b3416fb | 2014-06-05 13:44:04 -0700 | [diff] [blame] | 899 | if (inst_num <= MAX_EXEC_INST) |
| 900 | qat_hal_get_uwords(handle, ae, 0, inst_num, savuwords); |
| 901 | qat_hal_get_wakeup_event(handle, ae, ctx, &wakeup_events); |
Arnd Bergmann | 8c9478a | 2017-12-11 13:07:00 +0100 | [diff] [blame] | 902 | savpc = qat_hal_rd_indr_csr(handle, ae, ctx, CTX_STS_INDIRECT); |
Tadeusz Struk | b3416fb | 2014-06-05 13:44:04 -0700 | [diff] [blame] | 903 | savpc = (savpc & handle->hal_handle->upc_mask) >> 0; |
Arnd Bergmann | 8c9478a | 2017-12-11 13:07:00 +0100 | [diff] [blame] | 904 | ctx_enables = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES); |
Tadeusz Struk | b3416fb | 2014-06-05 13:44:04 -0700 | [diff] [blame] | 905 | ctx_enables &= IGNORE_W1C_MASK; |
Arnd Bergmann | 8c9478a | 2017-12-11 13:07:00 +0100 | [diff] [blame] | 906 | savcc = qat_hal_rd_ae_csr(handle, ae, CC_ENABLE); |
| 907 | savctx = qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS); |
| 908 | ctxarb_ctl = qat_hal_rd_ae_csr(handle, ae, CTX_ARB_CNTL); |
| 909 | ind_cnt_sig = qat_hal_rd_indr_csr(handle, ae, ctx, |
| 910 | FUTURE_COUNT_SIGNAL_INDIRECT); |
| 911 | ind_sig = qat_hal_rd_indr_csr(handle, ae, ctx, |
| 912 | CTX_SIG_EVENTS_INDIRECT); |
| 913 | act_sig = qat_hal_rd_ae_csr(handle, ae, CTX_SIG_EVENTS_ACTIVE); |
Tadeusz Struk | b3416fb | 2014-06-05 13:44:04 -0700 | [diff] [blame] | 914 | /* execute micro codes */ |
| 915 | qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables); |
| 916 | qat_hal_wr_uwords(handle, ae, 0, inst_num, micro_inst); |
| 917 | qat_hal_wr_indr_csr(handle, ae, (1 << ctx), CTX_STS_INDIRECT, 0); |
| 918 | qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS, ctx & ACS_ACNO); |
| 919 | if (code_off) |
| 920 | qat_hal_wr_ae_csr(handle, ae, CC_ENABLE, savcc & 0xffffdfff); |
| 921 | qat_hal_put_wakeup_event(handle, ae, (1 << ctx), XCWE_VOLUNTARY); |
| 922 | qat_hal_wr_indr_csr(handle, ae, (1 << ctx), CTX_SIG_EVENTS_INDIRECT, 0); |
| 923 | qat_hal_wr_ae_csr(handle, ae, CTX_SIG_EVENTS_ACTIVE, 0); |
| 924 | qat_hal_enable_ctx(handle, ae, (1 << ctx)); |
| 925 | /* wait for micro codes to finish */ |
| 926 | if (qat_hal_wait_cycles(handle, ae, max_cycle, 1) != 0) |
| 927 | return -EFAULT; |
| 928 | if (endpc) { |
| 929 | unsigned int ctx_status; |
Tadeusz Struk | d65071e | 2014-06-24 15:19:34 -0700 | [diff] [blame] | 930 | |
Arnd Bergmann | 8c9478a | 2017-12-11 13:07:00 +0100 | [diff] [blame] | 931 | ctx_status = qat_hal_rd_indr_csr(handle, ae, ctx, |
| 932 | CTX_STS_INDIRECT); |
Tadeusz Struk | b3416fb | 2014-06-05 13:44:04 -0700 | [diff] [blame] | 933 | *endpc = ctx_status & handle->hal_handle->upc_mask; |
| 934 | } |
| 935 | /* retore to saved context */ |
| 936 | qat_hal_disable_ctx(handle, ae, (1 << ctx)); |
| 937 | if (inst_num <= MAX_EXEC_INST) |
| 938 | qat_hal_wr_uwords(handle, ae, 0, inst_num, savuwords); |
| 939 | qat_hal_put_wakeup_event(handle, ae, (1 << ctx), wakeup_events); |
| 940 | qat_hal_wr_indr_csr(handle, ae, (1 << ctx), CTX_STS_INDIRECT, |
| 941 | handle->hal_handle->upc_mask & savpc); |
Arnd Bergmann | 8c9478a | 2017-12-11 13:07:00 +0100 | [diff] [blame] | 942 | csr_val = qat_hal_rd_ae_csr(handle, ae, AE_MISC_CONTROL); |
Tadeusz Struk | b3416fb | 2014-06-05 13:44:04 -0700 | [diff] [blame] | 943 | newcsr_val = CLR_BIT(csr_val, MMC_SHARE_CS_BITPOS); |
| 944 | qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL, newcsr_val); |
| 945 | qat_hal_wr_ae_csr(handle, ae, CC_ENABLE, savcc); |
| 946 | qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS, savctx & ACS_ACNO); |
| 947 | qat_hal_wr_ae_csr(handle, ae, CTX_ARB_CNTL, ctxarb_ctl); |
| 948 | qat_hal_wr_indr_csr(handle, ae, (1 << ctx), |
| 949 | LM_ADDR_0_INDIRECT, ind_lm_addr0); |
| 950 | qat_hal_wr_indr_csr(handle, ae, (1 << ctx), |
| 951 | LM_ADDR_1_INDIRECT, ind_lm_addr1); |
| 952 | qat_hal_wr_indr_csr(handle, ae, (1 << ctx), |
| 953 | INDIRECT_LM_ADDR_0_BYTE_INDEX, ind_lm_addr_byte0); |
| 954 | qat_hal_wr_indr_csr(handle, ae, (1 << ctx), |
| 955 | INDIRECT_LM_ADDR_1_BYTE_INDEX, ind_lm_addr_byte1); |
| 956 | qat_hal_wr_indr_csr(handle, ae, (1 << ctx), |
| 957 | FUTURE_COUNT_SIGNAL_INDIRECT, ind_cnt_sig); |
| 958 | qat_hal_wr_indr_csr(handle, ae, (1 << ctx), |
| 959 | CTX_SIG_EVENTS_INDIRECT, ind_sig); |
| 960 | qat_hal_wr_ae_csr(handle, ae, CTX_SIG_EVENTS_ACTIVE, act_sig); |
| 961 | qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables); |
| 962 | |
| 963 | return 0; |
| 964 | } |
| 965 | |
| 966 | static int qat_hal_rd_rel_reg(struct icp_qat_fw_loader_handle *handle, |
| 967 | unsigned char ae, unsigned char ctx, |
| 968 | enum icp_qat_uof_regtype reg_type, |
| 969 | unsigned short reg_num, unsigned int *data) |
| 970 | { |
| 971 | unsigned int savctx, uaddr, uwrd_lo, uwrd_hi; |
| 972 | unsigned int ctxarb_cntl, ustore_addr, ctx_enables; |
| 973 | unsigned short reg_addr; |
| 974 | int status = 0; |
| 975 | uint64_t insts, savuword; |
| 976 | |
| 977 | reg_addr = qat_hal_get_reg_addr(reg_type, reg_num); |
| 978 | if (reg_addr == BAD_REGADDR) { |
| 979 | pr_err("QAT: bad regaddr=0x%x\n", reg_addr); |
| 980 | return -EINVAL; |
| 981 | } |
| 982 | switch (reg_type) { |
| 983 | case ICP_GPA_REL: |
| 984 | insts = 0xA070000000ull | (reg_addr & 0x3ff); |
| 985 | break; |
| 986 | default: |
| 987 | insts = (uint64_t)0xA030000000ull | ((reg_addr & 0x3ff) << 10); |
| 988 | break; |
| 989 | } |
Arnd Bergmann | 8c9478a | 2017-12-11 13:07:00 +0100 | [diff] [blame] | 990 | savctx = qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS); |
| 991 | ctxarb_cntl = qat_hal_rd_ae_csr(handle, ae, CTX_ARB_CNTL); |
| 992 | ctx_enables = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES); |
Tadeusz Struk | b3416fb | 2014-06-05 13:44:04 -0700 | [diff] [blame] | 993 | ctx_enables &= IGNORE_W1C_MASK; |
| 994 | if (ctx != (savctx & ACS_ACNO)) |
| 995 | qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS, |
| 996 | ctx & ACS_ACNO); |
| 997 | qat_hal_get_uwords(handle, ae, 0, 1, &savuword); |
| 998 | qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables); |
Arnd Bergmann | 8c9478a | 2017-12-11 13:07:00 +0100 | [diff] [blame] | 999 | ustore_addr = qat_hal_rd_ae_csr(handle, ae, USTORE_ADDRESS); |
Tadeusz Struk | b3416fb | 2014-06-05 13:44:04 -0700 | [diff] [blame] | 1000 | uaddr = UA_ECS; |
| 1001 | qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr); |
| 1002 | insts = qat_hal_set_uword_ecc(insts); |
| 1003 | uwrd_lo = (unsigned int)(insts & 0xffffffff); |
| 1004 | uwrd_hi = (unsigned int)(insts >> 0x20); |
| 1005 | qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_LOWER, uwrd_lo); |
| 1006 | qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_UPPER, uwrd_hi); |
| 1007 | qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr); |
| 1008 | /* delay for at least 8 cycles */ |
| 1009 | qat_hal_wait_cycles(handle, ae, 0x8, 0); |
| 1010 | /* |
| 1011 | * read ALU output |
| 1012 | * the instruction should have been executed |
| 1013 | * prior to clearing the ECS in putUwords |
| 1014 | */ |
Arnd Bergmann | 8c9478a | 2017-12-11 13:07:00 +0100 | [diff] [blame] | 1015 | *data = qat_hal_rd_ae_csr(handle, ae, ALU_OUT); |
Tadeusz Struk | b3416fb | 2014-06-05 13:44:04 -0700 | [diff] [blame] | 1016 | qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, ustore_addr); |
| 1017 | qat_hal_wr_uwords(handle, ae, 0, 1, &savuword); |
| 1018 | if (ctx != (savctx & ACS_ACNO)) |
| 1019 | qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS, |
| 1020 | savctx & ACS_ACNO); |
| 1021 | qat_hal_wr_ae_csr(handle, ae, CTX_ARB_CNTL, ctxarb_cntl); |
| 1022 | qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables); |
| 1023 | |
| 1024 | return status; |
| 1025 | } |
| 1026 | |
| 1027 | static int qat_hal_wr_rel_reg(struct icp_qat_fw_loader_handle *handle, |
| 1028 | unsigned char ae, unsigned char ctx, |
| 1029 | enum icp_qat_uof_regtype reg_type, |
| 1030 | unsigned short reg_num, unsigned int data) |
| 1031 | { |
| 1032 | unsigned short src_hiaddr, src_lowaddr, dest_addr, data16hi, data16lo; |
| 1033 | uint64_t insts[] = { |
| 1034 | 0x0F440000000ull, |
| 1035 | 0x0F040000000ull, |
| 1036 | 0x0F0000C0300ull, |
| 1037 | 0x0E000010000ull |
| 1038 | }; |
| 1039 | const int num_inst = ARRAY_SIZE(insts), code_off = 1; |
| 1040 | const int imm_w1 = 0, imm_w0 = 1; |
| 1041 | |
| 1042 | dest_addr = qat_hal_get_reg_addr(reg_type, reg_num); |
| 1043 | if (dest_addr == BAD_REGADDR) { |
| 1044 | pr_err("QAT: bad destAddr=0x%x\n", dest_addr); |
| 1045 | return -EINVAL; |
| 1046 | } |
| 1047 | |
| 1048 | data16lo = 0xffff & data; |
| 1049 | data16hi = 0xffff & (data >> 0x10); |
| 1050 | src_hiaddr = qat_hal_get_reg_addr(ICP_NO_DEST, (unsigned short) |
| 1051 | (0xff & data16hi)); |
| 1052 | src_lowaddr = qat_hal_get_reg_addr(ICP_NO_DEST, (unsigned short) |
| 1053 | (0xff & data16lo)); |
| 1054 | switch (reg_type) { |
| 1055 | case ICP_GPA_REL: |
| 1056 | insts[imm_w1] = insts[imm_w1] | ((data16hi >> 8) << 20) | |
| 1057 | ((src_hiaddr & 0x3ff) << 10) | (dest_addr & 0x3ff); |
| 1058 | insts[imm_w0] = insts[imm_w0] | ((data16lo >> 8) << 20) | |
| 1059 | ((src_lowaddr & 0x3ff) << 10) | (dest_addr & 0x3ff); |
| 1060 | break; |
| 1061 | default: |
| 1062 | insts[imm_w1] = insts[imm_w1] | ((data16hi >> 8) << 20) | |
| 1063 | ((dest_addr & 0x3ff) << 10) | (src_hiaddr & 0x3ff); |
| 1064 | |
| 1065 | insts[imm_w0] = insts[imm_w0] | ((data16lo >> 8) << 20) | |
| 1066 | ((dest_addr & 0x3ff) << 10) | (src_lowaddr & 0x3ff); |
| 1067 | break; |
| 1068 | } |
| 1069 | |
| 1070 | return qat_hal_exec_micro_inst(handle, ae, ctx, insts, num_inst, |
| 1071 | code_off, num_inst * 0x5, NULL); |
| 1072 | } |
| 1073 | |
| 1074 | int qat_hal_get_ins_num(void) |
| 1075 | { |
| 1076 | return ARRAY_SIZE(inst_4b); |
| 1077 | } |
| 1078 | |
| 1079 | static int qat_hal_concat_micro_code(uint64_t *micro_inst, |
| 1080 | unsigned int inst_num, unsigned int size, |
| 1081 | unsigned int addr, unsigned int *value) |
| 1082 | { |
Tadeusz Struk | 9196d96 | 2015-09-30 05:40:00 -0700 | [diff] [blame] | 1083 | int i; |
Tadeusz Struk | b3416fb | 2014-06-05 13:44:04 -0700 | [diff] [blame] | 1084 | unsigned int cur_value; |
| 1085 | const uint64_t *inst_arr; |
| 1086 | int fixup_offset; |
| 1087 | int usize = 0; |
| 1088 | int orig_num; |
| 1089 | |
| 1090 | orig_num = inst_num; |
Tadeusz Struk | 9196d96 | 2015-09-30 05:40:00 -0700 | [diff] [blame] | 1091 | cur_value = value[0]; |
Tadeusz Struk | b3416fb | 2014-06-05 13:44:04 -0700 | [diff] [blame] | 1092 | inst_arr = inst_4b; |
| 1093 | usize = ARRAY_SIZE(inst_4b); |
| 1094 | fixup_offset = inst_num; |
| 1095 | for (i = 0; i < usize; i++) |
| 1096 | micro_inst[inst_num++] = inst_arr[i]; |
| 1097 | INSERT_IMMED_GPRA_CONST(micro_inst[fixup_offset], (addr)); |
| 1098 | fixup_offset++; |
| 1099 | INSERT_IMMED_GPRA_CONST(micro_inst[fixup_offset], 0); |
| 1100 | fixup_offset++; |
| 1101 | INSERT_IMMED_GPRB_CONST(micro_inst[fixup_offset], (cur_value >> 0)); |
| 1102 | fixup_offset++; |
| 1103 | INSERT_IMMED_GPRB_CONST(micro_inst[fixup_offset], (cur_value >> 0x10)); |
| 1104 | |
| 1105 | return inst_num - orig_num; |
| 1106 | } |
| 1107 | |
| 1108 | static int qat_hal_exec_micro_init_lm(struct icp_qat_fw_loader_handle *handle, |
| 1109 | unsigned char ae, unsigned char ctx, |
| 1110 | int *pfirst_exec, uint64_t *micro_inst, |
| 1111 | unsigned int inst_num) |
| 1112 | { |
| 1113 | int stat = 0; |
| 1114 | unsigned int gpra0 = 0, gpra1 = 0, gpra2 = 0; |
| 1115 | unsigned int gprb0 = 0, gprb1 = 0; |
| 1116 | |
| 1117 | if (*pfirst_exec) { |
| 1118 | qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0, &gpra0); |
| 1119 | qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0x1, &gpra1); |
| 1120 | qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0x2, &gpra2); |
| 1121 | qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPB_REL, 0, &gprb0); |
| 1122 | qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPB_REL, 0x1, &gprb1); |
| 1123 | *pfirst_exec = 0; |
| 1124 | } |
| 1125 | stat = qat_hal_exec_micro_inst(handle, ae, ctx, micro_inst, inst_num, 1, |
| 1126 | inst_num * 0x5, NULL); |
| 1127 | if (stat != 0) |
| 1128 | return -EFAULT; |
| 1129 | qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0, gpra0); |
| 1130 | qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0x1, gpra1); |
| 1131 | qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0x2, gpra2); |
| 1132 | qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPB_REL, 0, gprb0); |
| 1133 | qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPB_REL, 0x1, gprb1); |
| 1134 | |
| 1135 | return 0; |
| 1136 | } |
| 1137 | |
| 1138 | int qat_hal_batch_wr_lm(struct icp_qat_fw_loader_handle *handle, |
| 1139 | unsigned char ae, |
| 1140 | struct icp_qat_uof_batch_init *lm_init_header) |
| 1141 | { |
| 1142 | struct icp_qat_uof_batch_init *plm_init; |
| 1143 | uint64_t *micro_inst_arry; |
| 1144 | int micro_inst_num; |
| 1145 | int alloc_inst_size; |
| 1146 | int first_exec = 1; |
| 1147 | int stat = 0; |
| 1148 | |
| 1149 | plm_init = lm_init_header->next; |
| 1150 | alloc_inst_size = lm_init_header->size; |
| 1151 | if ((unsigned int)alloc_inst_size > handle->hal_handle->max_ustore) |
| 1152 | alloc_inst_size = handle->hal_handle->max_ustore; |
Tadeusz Struk | d65071e | 2014-06-24 15:19:34 -0700 | [diff] [blame] | 1153 | micro_inst_arry = kmalloc_array(alloc_inst_size, sizeof(uint64_t), |
| 1154 | GFP_KERNEL); |
Tadeusz Struk | b3416fb | 2014-06-05 13:44:04 -0700 | [diff] [blame] | 1155 | if (!micro_inst_arry) |
| 1156 | return -ENOMEM; |
| 1157 | micro_inst_num = 0; |
| 1158 | while (plm_init) { |
| 1159 | unsigned int addr, *value, size; |
| 1160 | |
| 1161 | ae = plm_init->ae; |
| 1162 | addr = plm_init->addr; |
| 1163 | value = plm_init->value; |
| 1164 | size = plm_init->size; |
| 1165 | micro_inst_num += qat_hal_concat_micro_code(micro_inst_arry, |
| 1166 | micro_inst_num, |
| 1167 | size, addr, value); |
| 1168 | plm_init = plm_init->next; |
| 1169 | } |
| 1170 | /* exec micro codes */ |
| 1171 | if (micro_inst_arry && (micro_inst_num > 0)) { |
| 1172 | micro_inst_arry[micro_inst_num++] = 0x0E000010000ull; |
| 1173 | stat = qat_hal_exec_micro_init_lm(handle, ae, 0, &first_exec, |
| 1174 | micro_inst_arry, |
| 1175 | micro_inst_num); |
| 1176 | } |
| 1177 | kfree(micro_inst_arry); |
| 1178 | return stat; |
| 1179 | } |
| 1180 | |
| 1181 | static int qat_hal_put_rel_rd_xfer(struct icp_qat_fw_loader_handle *handle, |
| 1182 | unsigned char ae, unsigned char ctx, |
| 1183 | enum icp_qat_uof_regtype reg_type, |
| 1184 | unsigned short reg_num, unsigned int val) |
| 1185 | { |
| 1186 | int status = 0; |
| 1187 | unsigned int reg_addr; |
| 1188 | unsigned int ctx_enables; |
| 1189 | unsigned short mask; |
| 1190 | unsigned short dr_offset = 0x10; |
| 1191 | |
Arnd Bergmann | 8c9478a | 2017-12-11 13:07:00 +0100 | [diff] [blame] | 1192 | status = ctx_enables = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES); |
Tadeusz Struk | b3416fb | 2014-06-05 13:44:04 -0700 | [diff] [blame] | 1193 | if (CE_INUSE_CONTEXTS & ctx_enables) { |
| 1194 | if (ctx & 0x1) { |
| 1195 | pr_err("QAT: bad 4-ctx mode,ctx=0x%x\n", ctx); |
| 1196 | return -EINVAL; |
| 1197 | } |
| 1198 | mask = 0x1f; |
| 1199 | dr_offset = 0x20; |
| 1200 | } else { |
| 1201 | mask = 0x0f; |
| 1202 | } |
| 1203 | if (reg_num & ~mask) |
| 1204 | return -EINVAL; |
| 1205 | reg_addr = reg_num + (ctx << 0x5); |
| 1206 | switch (reg_type) { |
| 1207 | case ICP_SR_RD_REL: |
| 1208 | case ICP_SR_REL: |
| 1209 | SET_AE_XFER(handle, ae, reg_addr, val); |
| 1210 | break; |
| 1211 | case ICP_DR_RD_REL: |
| 1212 | case ICP_DR_REL: |
| 1213 | SET_AE_XFER(handle, ae, (reg_addr + dr_offset), val); |
| 1214 | break; |
| 1215 | default: |
| 1216 | status = -EINVAL; |
| 1217 | break; |
| 1218 | } |
| 1219 | return status; |
| 1220 | } |
| 1221 | |
| 1222 | static int qat_hal_put_rel_wr_xfer(struct icp_qat_fw_loader_handle *handle, |
| 1223 | unsigned char ae, unsigned char ctx, |
| 1224 | enum icp_qat_uof_regtype reg_type, |
| 1225 | unsigned short reg_num, unsigned int data) |
| 1226 | { |
| 1227 | unsigned int gprval, ctx_enables; |
| 1228 | unsigned short src_hiaddr, src_lowaddr, gpr_addr, xfr_addr, data16hi, |
| 1229 | data16low; |
| 1230 | unsigned short reg_mask; |
| 1231 | int status = 0; |
| 1232 | uint64_t micro_inst[] = { |
| 1233 | 0x0F440000000ull, |
| 1234 | 0x0F040000000ull, |
| 1235 | 0x0A000000000ull, |
| 1236 | 0x0F0000C0300ull, |
| 1237 | 0x0E000010000ull |
| 1238 | }; |
| 1239 | const int num_inst = ARRAY_SIZE(micro_inst), code_off = 1; |
| 1240 | const unsigned short gprnum = 0, dly = num_inst * 0x5; |
| 1241 | |
Arnd Bergmann | 8c9478a | 2017-12-11 13:07:00 +0100 | [diff] [blame] | 1242 | ctx_enables = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES); |
Tadeusz Struk | b3416fb | 2014-06-05 13:44:04 -0700 | [diff] [blame] | 1243 | if (CE_INUSE_CONTEXTS & ctx_enables) { |
| 1244 | if (ctx & 0x1) { |
| 1245 | pr_err("QAT: 4-ctx mode,ctx=0x%x\n", ctx); |
| 1246 | return -EINVAL; |
| 1247 | } |
| 1248 | reg_mask = (unsigned short)~0x1f; |
| 1249 | } else { |
| 1250 | reg_mask = (unsigned short)~0xf; |
| 1251 | } |
| 1252 | if (reg_num & reg_mask) |
| 1253 | return -EINVAL; |
| 1254 | xfr_addr = qat_hal_get_reg_addr(reg_type, reg_num); |
| 1255 | if (xfr_addr == BAD_REGADDR) { |
| 1256 | pr_err("QAT: bad xfrAddr=0x%x\n", xfr_addr); |
| 1257 | return -EINVAL; |
| 1258 | } |
| 1259 | qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPB_REL, gprnum, &gprval); |
| 1260 | gpr_addr = qat_hal_get_reg_addr(ICP_GPB_REL, gprnum); |
| 1261 | data16low = 0xffff & data; |
| 1262 | data16hi = 0xffff & (data >> 0x10); |
| 1263 | src_hiaddr = qat_hal_get_reg_addr(ICP_NO_DEST, |
| 1264 | (unsigned short)(0xff & data16hi)); |
| 1265 | src_lowaddr = qat_hal_get_reg_addr(ICP_NO_DEST, |
| 1266 | (unsigned short)(0xff & data16low)); |
| 1267 | micro_inst[0] = micro_inst[0x0] | ((data16hi >> 8) << 20) | |
| 1268 | ((gpr_addr & 0x3ff) << 10) | (src_hiaddr & 0x3ff); |
| 1269 | micro_inst[1] = micro_inst[0x1] | ((data16low >> 8) << 20) | |
| 1270 | ((gpr_addr & 0x3ff) << 10) | (src_lowaddr & 0x3ff); |
| 1271 | micro_inst[0x2] = micro_inst[0x2] | |
| 1272 | ((xfr_addr & 0x3ff) << 20) | ((gpr_addr & 0x3ff) << 10); |
| 1273 | status = qat_hal_exec_micro_inst(handle, ae, ctx, micro_inst, num_inst, |
| 1274 | code_off, dly, NULL); |
| 1275 | qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPB_REL, gprnum, gprval); |
| 1276 | return status; |
| 1277 | } |
| 1278 | |
| 1279 | static int qat_hal_put_rel_nn(struct icp_qat_fw_loader_handle *handle, |
| 1280 | unsigned char ae, unsigned char ctx, |
| 1281 | unsigned short nn, unsigned int val) |
| 1282 | { |
| 1283 | unsigned int ctx_enables; |
| 1284 | int stat = 0; |
| 1285 | |
Arnd Bergmann | 8c9478a | 2017-12-11 13:07:00 +0100 | [diff] [blame] | 1286 | ctx_enables = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES); |
Tadeusz Struk | b3416fb | 2014-06-05 13:44:04 -0700 | [diff] [blame] | 1287 | ctx_enables &= IGNORE_W1C_MASK; |
| 1288 | qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables | CE_NN_MODE); |
| 1289 | |
| 1290 | stat = qat_hal_put_rel_wr_xfer(handle, ae, ctx, ICP_NEIGH_REL, nn, val); |
| 1291 | qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables); |
| 1292 | return stat; |
| 1293 | } |
| 1294 | |
| 1295 | static int qat_hal_convert_abs_to_rel(struct icp_qat_fw_loader_handle |
| 1296 | *handle, unsigned char ae, |
| 1297 | unsigned short absreg_num, |
| 1298 | unsigned short *relreg, |
| 1299 | unsigned char *ctx) |
| 1300 | { |
| 1301 | unsigned int ctx_enables; |
| 1302 | |
Arnd Bergmann | 8c9478a | 2017-12-11 13:07:00 +0100 | [diff] [blame] | 1303 | ctx_enables = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES); |
Tadeusz Struk | b3416fb | 2014-06-05 13:44:04 -0700 | [diff] [blame] | 1304 | if (ctx_enables & CE_INUSE_CONTEXTS) { |
| 1305 | /* 4-ctx mode */ |
| 1306 | *relreg = absreg_num & 0x1F; |
| 1307 | *ctx = (absreg_num >> 0x4) & 0x6; |
| 1308 | } else { |
| 1309 | /* 8-ctx mode */ |
| 1310 | *relreg = absreg_num & 0x0F; |
| 1311 | *ctx = (absreg_num >> 0x4) & 0x7; |
| 1312 | } |
| 1313 | return 0; |
| 1314 | } |
| 1315 | |
| 1316 | int qat_hal_init_gpr(struct icp_qat_fw_loader_handle *handle, |
| 1317 | unsigned char ae, unsigned char ctx_mask, |
| 1318 | enum icp_qat_uof_regtype reg_type, |
| 1319 | unsigned short reg_num, unsigned int regdata) |
| 1320 | { |
| 1321 | int stat = 0; |
| 1322 | unsigned short reg; |
| 1323 | unsigned char ctx = 0; |
| 1324 | enum icp_qat_uof_regtype type; |
| 1325 | |
| 1326 | if (reg_num >= ICP_QAT_UCLO_MAX_GPR_REG) |
| 1327 | return -EINVAL; |
| 1328 | |
| 1329 | do { |
| 1330 | if (ctx_mask == 0) { |
| 1331 | qat_hal_convert_abs_to_rel(handle, ae, reg_num, ®, |
| 1332 | &ctx); |
| 1333 | type = reg_type - 1; |
| 1334 | } else { |
| 1335 | reg = reg_num; |
| 1336 | type = reg_type; |
| 1337 | if (!test_bit(ctx, (unsigned long *)&ctx_mask)) |
| 1338 | continue; |
| 1339 | } |
| 1340 | stat = qat_hal_wr_rel_reg(handle, ae, ctx, type, reg, regdata); |
| 1341 | if (stat) { |
| 1342 | pr_err("QAT: write gpr fail\n"); |
| 1343 | return -EINVAL; |
| 1344 | } |
| 1345 | } while (ctx_mask && (ctx++ < ICP_QAT_UCLO_MAX_CTX)); |
| 1346 | |
| 1347 | return 0; |
| 1348 | } |
| 1349 | |
| 1350 | int qat_hal_init_wr_xfer(struct icp_qat_fw_loader_handle *handle, |
| 1351 | unsigned char ae, unsigned char ctx_mask, |
| 1352 | enum icp_qat_uof_regtype reg_type, |
| 1353 | unsigned short reg_num, unsigned int regdata) |
| 1354 | { |
| 1355 | int stat = 0; |
| 1356 | unsigned short reg; |
| 1357 | unsigned char ctx = 0; |
| 1358 | enum icp_qat_uof_regtype type; |
| 1359 | |
| 1360 | if (reg_num >= ICP_QAT_UCLO_MAX_XFER_REG) |
| 1361 | return -EINVAL; |
| 1362 | |
| 1363 | do { |
| 1364 | if (ctx_mask == 0) { |
| 1365 | qat_hal_convert_abs_to_rel(handle, ae, reg_num, ®, |
| 1366 | &ctx); |
| 1367 | type = reg_type - 3; |
| 1368 | } else { |
| 1369 | reg = reg_num; |
| 1370 | type = reg_type; |
| 1371 | if (!test_bit(ctx, (unsigned long *)&ctx_mask)) |
| 1372 | continue; |
| 1373 | } |
| 1374 | stat = qat_hal_put_rel_wr_xfer(handle, ae, ctx, type, reg, |
| 1375 | regdata); |
| 1376 | if (stat) { |
| 1377 | pr_err("QAT: write wr xfer fail\n"); |
| 1378 | return -EINVAL; |
| 1379 | } |
| 1380 | } while (ctx_mask && (ctx++ < ICP_QAT_UCLO_MAX_CTX)); |
| 1381 | |
| 1382 | return 0; |
| 1383 | } |
| 1384 | |
| 1385 | int qat_hal_init_rd_xfer(struct icp_qat_fw_loader_handle *handle, |
| 1386 | unsigned char ae, unsigned char ctx_mask, |
| 1387 | enum icp_qat_uof_regtype reg_type, |
| 1388 | unsigned short reg_num, unsigned int regdata) |
| 1389 | { |
| 1390 | int stat = 0; |
| 1391 | unsigned short reg; |
| 1392 | unsigned char ctx = 0; |
| 1393 | enum icp_qat_uof_regtype type; |
| 1394 | |
| 1395 | if (reg_num >= ICP_QAT_UCLO_MAX_XFER_REG) |
| 1396 | return -EINVAL; |
| 1397 | |
| 1398 | do { |
| 1399 | if (ctx_mask == 0) { |
| 1400 | qat_hal_convert_abs_to_rel(handle, ae, reg_num, ®, |
| 1401 | &ctx); |
| 1402 | type = reg_type - 3; |
| 1403 | } else { |
| 1404 | reg = reg_num; |
| 1405 | type = reg_type; |
| 1406 | if (!test_bit(ctx, (unsigned long *)&ctx_mask)) |
| 1407 | continue; |
| 1408 | } |
| 1409 | stat = qat_hal_put_rel_rd_xfer(handle, ae, ctx, type, reg, |
| 1410 | regdata); |
| 1411 | if (stat) { |
| 1412 | pr_err("QAT: write rd xfer fail\n"); |
| 1413 | return -EINVAL; |
| 1414 | } |
| 1415 | } while (ctx_mask && (ctx++ < ICP_QAT_UCLO_MAX_CTX)); |
| 1416 | |
| 1417 | return 0; |
| 1418 | } |
| 1419 | |
| 1420 | int qat_hal_init_nn(struct icp_qat_fw_loader_handle *handle, |
| 1421 | unsigned char ae, unsigned char ctx_mask, |
| 1422 | unsigned short reg_num, unsigned int regdata) |
| 1423 | { |
| 1424 | int stat = 0; |
| 1425 | unsigned char ctx; |
| 1426 | |
| 1427 | if (ctx_mask == 0) |
| 1428 | return -EINVAL; |
| 1429 | |
| 1430 | for (ctx = 0; ctx < ICP_QAT_UCLO_MAX_CTX; ctx++) { |
| 1431 | if (!test_bit(ctx, (unsigned long *)&ctx_mask)) |
| 1432 | continue; |
| 1433 | stat = qat_hal_put_rel_nn(handle, ae, ctx, reg_num, regdata); |
| 1434 | if (stat) { |
| 1435 | pr_err("QAT: write neigh error\n"); |
| 1436 | return -EINVAL; |
| 1437 | } |
| 1438 | } |
| 1439 | |
| 1440 | return 0; |
| 1441 | } |