| /* |
| * Copyright (C) 2014-15 Synopsys, Inc. (www.synopsys.com) |
| * |
| * This program is free software; you can redistribute it and/or modify |
| * it under the terms of the GNU General Public License version 2 as |
| * published by the Free Software Foundation. |
| */ |
| |
| #include <linux/linkage.h> |
| |
| #ifdef __LITTLE_ENDIAN__ |
| # define SHIFT_1(RX,RY,IMM) asl RX, RY, IMM ; << |
| # define SHIFT_2(RX,RY,IMM) lsr RX, RY, IMM ; >> |
| # define MERGE_1(RX,RY,IMM) asl RX, RY, IMM |
| # define MERGE_2(RX,RY,IMM) |
| # define EXTRACT_1(RX,RY,IMM) and RX, RY, 0xFFFF |
| # define EXTRACT_2(RX,RY,IMM) lsr RX, RY, IMM |
| #else |
| # define SHIFT_1(RX,RY,IMM) lsr RX, RY, IMM ; >> |
| # define SHIFT_2(RX,RY,IMM) asl RX, RY, IMM ; << |
| # define MERGE_1(RX,RY,IMM) asl RX, RY, IMM ; << |
| # define MERGE_2(RX,RY,IMM) asl RX, RY, IMM ; << |
| # define EXTRACT_1(RX,RY,IMM) lsr RX, RY, IMM |
| # define EXTRACT_2(RX,RY,IMM) lsr RX, RY, 0x08 |
| #endif |
| |
| #ifdef CONFIG_ARC_HAS_LL64 |
| # define PREFETCH_READ(RX) prefetch [RX, 56] |
| # define PREFETCH_WRITE(RX) prefetchw [RX, 64] |
| # define LOADX(DST,RX) ldd.ab DST, [RX, 8] |
| # define STOREX(SRC,RX) std.ab SRC, [RX, 8] |
| # define ZOLSHFT 5 |
| # define ZOLAND 0x1F |
| #else |
| # define PREFETCH_READ(RX) prefetch [RX, 28] |
| # define PREFETCH_WRITE(RX) prefetchw [RX, 32] |
| # define LOADX(DST,RX) ld.ab DST, [RX, 4] |
| # define STOREX(SRC,RX) st.ab SRC, [RX, 4] |
| # define ZOLSHFT 4 |
| # define ZOLAND 0xF |
| #endif |
| |
| ENTRY_CFI(memcpy) |
| prefetch [r1] ; Prefetch the read location |
| prefetchw [r0] ; Prefetch the write location |
| mov.f 0, r2 |
| ;;; if size is zero |
| jz.d [blink] |
| mov r3, r0 ; don;t clobber ret val |
| |
| ;;; if size <= 8 |
| cmp r2, 8 |
| bls.d @.Lsmallchunk |
| mov.f lp_count, r2 |
| |
| and.f r4, r0, 0x03 |
| rsub lp_count, r4, 4 |
| lpnz @.Laligndestination |
| ;; LOOP BEGIN |
| ldb.ab r5, [r1,1] |
| sub r2, r2, 1 |
| stb.ab r5, [r3,1] |
| .Laligndestination: |
| |
| ;;; Check the alignment of the source |
| and.f r4, r1, 0x03 |
| bnz.d @.Lsourceunaligned |
| |
| ;;; CASE 0: Both source and destination are 32bit aligned |
| ;;; Convert len to Dwords, unfold x4 |
| lsr.f lp_count, r2, ZOLSHFT |
| lpnz @.Lcopy32_64bytes |
| ;; LOOP START |
| LOADX (r6, r1) |
| PREFETCH_READ (r1) |
| PREFETCH_WRITE (r3) |
| LOADX (r8, r1) |
| LOADX (r10, r1) |
| LOADX (r4, r1) |
| STOREX (r6, r3) |
| STOREX (r8, r3) |
| STOREX (r10, r3) |
| STOREX (r4, r3) |
| .Lcopy32_64bytes: |
| |
| and.f lp_count, r2, ZOLAND ;Last remaining 31 bytes |
| .Lsmallchunk: |
| lpnz @.Lcopyremainingbytes |
| ;; LOOP START |
| ldb.ab r5, [r1,1] |
| stb.ab r5, [r3,1] |
| .Lcopyremainingbytes: |
| |
| j [blink] |
| ;;; END CASE 0 |
| |
| .Lsourceunaligned: |
| cmp r4, 2 |
| beq.d @.LunalignedOffby2 |
| sub r2, r2, 1 |
| |
| bhi.d @.LunalignedOffby3 |
| ldb.ab r5, [r1, 1] |
| |
| ;;; CASE 1: The source is unaligned, off by 1 |
| ;; Hence I need to read 1 byte for a 16bit alignment |
| ;; and 2bytes to reach 32bit alignment |
| ldh.ab r6, [r1, 2] |
| sub r2, r2, 2 |
| ;; Convert to words, unfold x2 |
| lsr.f lp_count, r2, 3 |
| MERGE_1 (r6, r6, 8) |
| MERGE_2 (r5, r5, 24) |
| or r5, r5, r6 |
| |
| ;; Both src and dst are aligned |
| lpnz @.Lcopy8bytes_1 |
| ;; LOOP START |
| ld.ab r6, [r1, 4] |
| prefetch [r1, 28] ;Prefetch the next read location |
| ld.ab r8, [r1,4] |
| prefetchw [r3, 32] ;Prefetch the next write location |
| |
| SHIFT_1 (r7, r6, 24) |
| or r7, r7, r5 |
| SHIFT_2 (r5, r6, 8) |
| |
| SHIFT_1 (r9, r8, 24) |
| or r9, r9, r5 |
| SHIFT_2 (r5, r8, 8) |
| |
| st.ab r7, [r3, 4] |
| st.ab r9, [r3, 4] |
| .Lcopy8bytes_1: |
| |
| ;; Write back the remaining 16bits |
| EXTRACT_1 (r6, r5, 16) |
| sth.ab r6, [r3, 2] |
| ;; Write back the remaining 8bits |
| EXTRACT_2 (r5, r5, 16) |
| stb.ab r5, [r3, 1] |
| |
| and.f lp_count, r2, 0x07 ;Last 8bytes |
| lpnz @.Lcopybytewise_1 |
| ;; LOOP START |
| ldb.ab r6, [r1,1] |
| stb.ab r6, [r3,1] |
| .Lcopybytewise_1: |
| j [blink] |
| |
| .LunalignedOffby2: |
| ;;; CASE 2: The source is unaligned, off by 2 |
| ldh.ab r5, [r1, 2] |
| sub r2, r2, 1 |
| |
| ;; Both src and dst are aligned |
| ;; Convert to words, unfold x2 |
| lsr.f lp_count, r2, 3 |
| #ifdef __BIG_ENDIAN__ |
| asl.nz r5, r5, 16 |
| #endif |
| lpnz @.Lcopy8bytes_2 |
| ;; LOOP START |
| ld.ab r6, [r1, 4] |
| prefetch [r1, 28] ;Prefetch the next read location |
| ld.ab r8, [r1,4] |
| prefetchw [r3, 32] ;Prefetch the next write location |
| |
| SHIFT_1 (r7, r6, 16) |
| or r7, r7, r5 |
| SHIFT_2 (r5, r6, 16) |
| |
| SHIFT_1 (r9, r8, 16) |
| or r9, r9, r5 |
| SHIFT_2 (r5, r8, 16) |
| |
| st.ab r7, [r3, 4] |
| st.ab r9, [r3, 4] |
| .Lcopy8bytes_2: |
| |
| #ifdef __BIG_ENDIAN__ |
| lsr.nz r5, r5, 16 |
| #endif |
| sth.ab r5, [r3, 2] |
| |
| and.f lp_count, r2, 0x07 ;Last 8bytes |
| lpnz @.Lcopybytewise_2 |
| ;; LOOP START |
| ldb.ab r6, [r1,1] |
| stb.ab r6, [r3,1] |
| .Lcopybytewise_2: |
| j [blink] |
| |
| .LunalignedOffby3: |
| ;;; CASE 3: The source is unaligned, off by 3 |
| ;;; Hence, I need to read 1byte for achieve the 32bit alignment |
| |
| ;; Both src and dst are aligned |
| ;; Convert to words, unfold x2 |
| lsr.f lp_count, r2, 3 |
| #ifdef __BIG_ENDIAN__ |
| asl.ne r5, r5, 24 |
| #endif |
| lpnz @.Lcopy8bytes_3 |
| ;; LOOP START |
| ld.ab r6, [r1, 4] |
| prefetch [r1, 28] ;Prefetch the next read location |
| ld.ab r8, [r1,4] |
| prefetchw [r3, 32] ;Prefetch the next write location |
| |
| SHIFT_1 (r7, r6, 8) |
| or r7, r7, r5 |
| SHIFT_2 (r5, r6, 24) |
| |
| SHIFT_1 (r9, r8, 8) |
| or r9, r9, r5 |
| SHIFT_2 (r5, r8, 24) |
| |
| st.ab r7, [r3, 4] |
| st.ab r9, [r3, 4] |
| .Lcopy8bytes_3: |
| |
| #ifdef __BIG_ENDIAN__ |
| lsr.nz r5, r5, 24 |
| #endif |
| stb.ab r5, [r3, 1] |
| |
| and.f lp_count, r2, 0x07 ;Last 8bytes |
| lpnz @.Lcopybytewise_3 |
| ;; LOOP START |
| ldb.ab r6, [r1,1] |
| stb.ab r6, [r3,1] |
| .Lcopybytewise_3: |
| j [blink] |
| |
| END_CFI(memcpy) |
