← Back
Editing: memmove_ppc64x.s
// Copyright 2014 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. //go:build ppc64 || ppc64le #include "textflag.h" // See memmove Go doc for important implementation constraints. // func memmove(to, from unsafe.Pointer, n uintptr) // target address #define TGT R3 // source address #define SRC R4 // length to move #define LEN R5 // number of doublewords #define DWORDS R6 // number of bytes < 8 #define BYTES R7 // const 16 used as index #define IDX16 R8 // temp used for copies, etc. #define TMP R9 // number of 64 byte chunks #define QWORDS R10 // index values #define IDX32 R14 #define IDX48 R15 #define OCTWORDS R16 TEXT runtime·memmove<ABIInternal>(SB), NOSPLIT|NOFRAME, $0-24 // R3 = TGT = to // R4 = SRC = from // R5 = LEN = n // Determine if there are doublewords to // copy so a more efficient move can be done check: #ifdef GOPPC64_power10 CMP LEN, $16 BGT mcopy SLD $56, LEN, TMP LXVL SRC, TMP, V0 STXVL V0, TGT, TMP RET #endif mcopy: ANDCC $7, LEN, BYTES // R7: bytes to copy SRD $3, LEN, DWORDS // R6: double words to copy MOVFL CR0, CR3 // save CR from ANDCC CMP DWORDS, $0, CR1 // CR1[EQ] set if no double words to copy // Determine overlap by subtracting dest - src and comparing against the // length. This catches the cases where src and dest are in different types // of storage such as stack and static to avoid doing backward move when not // necessary. SUB SRC, TGT, TMP // dest - src CMPU TMP, LEN, CR2 // < len? BC 12, 8, backward // BLT CR2 backward // Copying forward if no overlap. BC 12, 6, checkbytes // BEQ CR1, checkbytes SRDCC $3, DWORDS, OCTWORDS // 64 byte chunks? MOVD $16, IDX16 BEQ lt64gt8 // < 64 bytes // Prepare for moves of 64 bytes at a time. forward64setup: DCBTST (TGT) // prepare data cache DCBT (SRC) MOVD OCTWORDS, CTR // Number of 64 byte chunks MOVD $32, IDX32 MOVD $48, IDX48 PCALIGN $16 forward64: LXVD2X (R0)(SRC), VS32 // load 64 bytes LXVD2X (IDX16)(SRC), VS33 LXVD2X (IDX32)(SRC), VS34 LXVD2X (IDX48)(SRC), VS35 ADD $64, SRC STXVD2X VS32, (R0)(TGT) // store 64 bytes STXVD2X VS33, (IDX16)(TGT) STXVD2X VS34, (IDX32)(TGT) STXVD2X VS35, (IDX48)(TGT) ADD $64,TGT // bump up for next set BC 16, 0, forward64 // continue ANDCC $7, DWORDS // remaining doublewords BEQ checkbytes // only bytes remain lt64gt8: CMP DWORDS, $4 BLT lt32gt8 LXVD2X (R0)(SRC), VS32 LXVD2X (IDX16)(SRC), VS33 ADD $-4, DWORDS STXVD2X VS32, (R0)(TGT) STXVD2X VS33, (IDX16)(TGT) ADD $32, SRC ADD $32, TGT lt32gt8: // At this point >= 8 and < 32 // Move 16 bytes if possible CMP DWORDS, $2 BLT lt16 LXVD2X (R0)(SRC), VS32 ADD $-2, DWORDS STXVD2X VS32, (R0)(TGT) ADD $16, SRC ADD $16, TGT lt16: // Move 8 bytes if possible CMP DWORDS, $1 BLT checkbytes #ifdef GOPPC64_power10 ADD $8, BYTES SLD $56, BYTES, TMP LXVL SRC, TMP, V0 STXVL V0, TGT, TMP RET #endif MOVD 0(SRC), TMP ADD $8, SRC MOVD TMP, 0(TGT) ADD $8, TGT checkbytes: BC 12, 14, LR // BEQ lr #ifdef GOPPC64_power10 SLD $56, BYTES, TMP LXVL SRC, TMP, V0 STXVL V0, TGT, TMP RET #endif lt8: // Move word if possible CMP BYTES, $4 BLT lt4 MOVWZ 0(SRC), TMP ADD $-4, BYTES MOVW TMP, 0(TGT) ADD $4, SRC ADD $4, TGT lt4: // Move halfword if possible CMP BYTES, $2 BLT lt2 MOVHZ 0(SRC), TMP ADD $-2, BYTES MOVH TMP, 0(TGT) ADD $2, SRC ADD $2, TGT lt2: // Move last byte if 1 left CMP BYTES, $1 BC 12, 0, LR // ble lr MOVBZ 0(SRC), TMP MOVBZ TMP, 0(TGT) RET backward: // Copying backwards proceeds by copying R7 bytes then copying R6 double words. // R3 and R4 are advanced to the end of the destination/source buffers // respectively and moved back as we copy. ADD LEN, SRC, SRC // end of source ADD TGT, LEN, TGT // end of dest BEQ nobackwardtail // earlier condition MOVD BYTES, CTR // bytes to move backwardtailloop: MOVBZ -1(SRC), TMP // point to last byte SUB $1,SRC MOVBZ TMP, -1(TGT) SUB $1,TGT BDNZ backwardtailloop nobackwardtail: BC 4, 5, LR // blelr cr1, return if DWORDS == 0 SRDCC $2,DWORDS,QWORDS // Compute number of 32B blocks and compare to 0 BNE backward32setup // If QWORDS != 0, start the 32B copy loop. backward24: // DWORDS is a value between 1-3. CMP DWORDS, $2 MOVD -8(SRC), TMP MOVD TMP, -8(TGT) BC 12, 0, LR // bltlr, return if DWORDS == 1 MOVD -16(SRC), TMP MOVD TMP, -16(TGT) BC 12, 2, LR // beqlr, return if DWORDS == 2 MOVD -24(SRC), TMP MOVD TMP, -24(TGT) RET backward32setup: ANDCC $3,DWORDS // Compute remaining DWORDS and compare to 0 MOVD QWORDS, CTR // set up loop ctr MOVD $16, IDX16 // 32 bytes at a time PCALIGN $16 backward32loop: SUB $32, TGT SUB $32, SRC LXVD2X (R0)(SRC), VS32 // load 16x2 bytes LXVD2X (IDX16)(SRC), VS33 STXVD2X VS32, (R0)(TGT) // store 16x2 bytes STXVD2X VS33, (IDX16)(TGT) BDNZ backward32loop BC 12, 2, LR // beqlr, return if DWORDS == 0 BR backward24
Save File
Cancel