Server : LiteSpeed System : Linux us-phx-web1202.main-hosting.eu 4.18.0-553.84.1.lve.el8.x86_64 #1 SMP Tue Nov 25 18:33:03 UTC 2025 x86_64 User : u615232177 ( 615232177) PHP Version : 8.1.33 Disable Function : NONE Directory : /opt/golang/1.22.0/src/crypto/aes/ |
// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:build ppc64 || ppc64le
// Portions based on CRYPTOGAMS code with the following comment:
// # ====================================================================
// # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
// # project. The module is, however, dual licensed under OpenSSL and
// # CRYPTOGAMS licenses depending on where you obtain it. For further
// # details see http://www.openssl.org/~appro/cryptogams/.
// # ====================================================================
// The implementations for gcmHash, gcmInit and gcmMul are based on the generated asm
// from the script https://github.com/dot-asm/cryptogams/blob/master/ppc/ghashp8-ppc.pl
// from commit d47afb3c.
// Changes were made due to differences in the ABI and some register usage.
// Some arguments were changed due to the way the Go code passes them.
// Portions that use the stitched AES-GCM approach in counterCryptASM
// are based on code found in
// https://github.com/IBM/ipcri/blob/main/aes/p10_aes_gcm.s
#include "textflag.h"
#define XIP R3
#define HTBL R4
#define INP R5
#define LEN R6
#define XL V0
#define XM V1
#define XH V2
#define IN V3
#define ZERO V4
#define T0 V5
#define T1 V6
#define T2 V7
#define XC2 V8
#define H V9
#define HH V10
#define HL V11
#define LEMASK V12
#define XL1 V13
#define XM1 V14
#define XH1 V15
#define IN1 V16
#define H2 V17
#define H2H V18
#define H2L V19
#define XL3 V20
#define XM2 V21
#define IN2 V22
#define H3L V23
#define H3 V24
#define H3H V25
#define XH3 V26
#define XM3 V27
#define IN3 V28
#define H4L V29
#define H4 V30
#define H4H V31
#define IN0 IN
#define H21L HL
#define H21H HH
#define LOPERM H2L
#define HIPERM H2H
#define VXL VS32
#define VIN VS35
#define VXC2 VS40
#define VH VS41
#define VHH VS42
#define VHL VS43
#define VIN1 VS48
#define VH2 VS49
#define VH2H VS50
#define VH2L VS51
#define VIN2 VS54
#define VH3L VS55
#define VH3 VS56
#define VH3H VS57
#define VIN3 VS60
#define VH4L VS61
#define VH4 VS62
#define VH4H VS63
#define VIN0 VIN
#define ESPERM V10
#define TMP2 V11
// The following macros provide appropriate
// implementations for endianness as well as
// ISA specific for power8 and power9.
#ifdef GOARCH_ppc64le
# ifdef GOPPC64_power9
#define P8_LXVB16X(RA,RB,VT) LXVB16X (RA)(RB), VT
#define P8_STXVB16X(VS,RA,RB) STXVB16X VS, (RA)(RB)
# else
#define NEEDS_ESPERM
#define P8_LXVB16X(RA,RB,VT) \
LXVD2X (RA+RB), VT \
VPERM VT, VT, ESPERM, VT
#define P8_STXVB16X(VS,RA,RB) \
VPERM VS, VS, ESPERM, TMP2; \
STXVD2X TMP2, (RA+RB)
# endif
#else
#define P8_LXVB16X(RA,RB,VT) \
LXVD2X (RA+RB), VT
#define P8_STXVB16X(VS,RA,RB) \
STXVD2X VS, (RA+RB)
#endif
#define MASK_PTR R8
#define MASKV V0
#define INV V1
// The following macros are used for
// the stitched implementation within
// counterCryptASM.
// Load the initial GCM counter value
// in V30 and set up the counter increment
// in V31
#define SETUP_COUNTER \
P8_LXVB16X(COUNTER, R0, V30); \
VSPLTISB $1, V28; \
VXOR V31, V31, V31; \
VSLDOI $1, V31, V28, V31
// These macros set up the initial value
// for a single encryption, or 4 or 8
// stitched encryptions implemented
// with interleaving vciphers.
//
// The input value for each encryption
// is generated by XORing the counter
// from V30 with the first key in VS0
// and incrementing the counter.
//
// Single encryption in V15
#define GEN_VCIPHER_INPUT \
XXLOR VS0, VS0, V29 \
VXOR V30, V29, V15; \
VADDUWM V30, V31, V30
// 4 encryptions in V15 - V18
#define GEN_VCIPHER_4_INPUTS \
XXLOR VS0, VS0, V29; \
VXOR V30, V29, V15; \
VADDUWM V30, V31, V30; \
VXOR V30, V29, V16; \
VADDUWM V30, V31, V30; \
VXOR V30, V29, V17; \
VADDUWM V30, V31, V30; \
VXOR V30, V29, V18; \
VADDUWM V30, V31, V30
// 8 encryptions in V15 - V22
#define GEN_VCIPHER_8_INPUTS \
XXLOR VS0, VS0, V29; \
VXOR V30, V29, V15; \
VADDUWM V30, V31, V30; \
VXOR V30, V29, V16; \
VADDUWM V30, V31, V30; \
VXOR V30, V29, V17; \
VADDUWM V30, V31, V30; \
VXOR V30, V29, V18; \
VADDUWM V30, V31, V30; \
VXOR V30, V29, V19; \
VADDUWM V30, V31, V30; \
VXOR V30, V29, V20; \
VADDUWM V30, V31, V30; \
VXOR V30, V29, V21; \
VADDUWM V30, V31, V30; \
VXOR V30, V29, V22; \
VADDUWM V30, V31, V30
// Load the keys to be used for
// encryption based on key_len.
// Keys are in VS0 - VS14
// depending on key_len.
// Valid keys sizes are verified
// here. CR2 is set and used
// throughout to check key_len.
#define LOAD_KEYS(blk_key, key_len) \
MOVD $16, R16; \
MOVD $32, R17; \
MOVD $48, R18; \
MOVD $64, R19; \
LXVD2X (blk_key)(R0), VS0; \
LXVD2X (blk_key)(R16), VS1; \
LXVD2X (blk_key)(R17), VS2; \
LXVD2X (blk_key)(R18), VS3; \
LXVD2X (blk_key)(R19), VS4; \
ADD $64, R16; \
ADD $64, R17; \
ADD $64, R18; \
ADD $64, R19; \
LXVD2X (blk_key)(R16), VS5; \
LXVD2X (blk_key)(R17), VS6; \
LXVD2X (blk_key)(R18), VS7; \
LXVD2X (blk_key)(R19), VS8; \
ADD $64, R16; \
ADD $64, R17; \
ADD $64, R18; \
ADD $64, R19; \
LXVD2X (blk_key)(R16), VS9; \
LXVD2X (blk_key)(R17), VS10; \
CMP key_len, $12, CR2; \
CMP key_len, $10; \
BEQ keysLoaded; \
LXVD2X (blk_key)(R18), VS11; \
LXVD2X (blk_key)(R19), VS12; \
BEQ CR2, keysLoaded; \
ADD $64, R16; \
ADD $64, R17; \
LXVD2X (blk_key)(R16), VS13; \
LXVD2X (blk_key)(R17), VS14; \
CMP key_len, $14; \
BEQ keysLoaded; \
MOVD R0,0(R0); \
keysLoaded:
// Encrypt 1 (vin) with first 9
// keys from VS1 - VS9.
#define VCIPHER_1X9_KEYS(vin) \
XXLOR VS1, VS1, V23; \
XXLOR VS2, VS2, V24; \
XXLOR VS3, VS3, V25; \
XXLOR VS4, VS4, V26; \
XXLOR VS5, VS5, V27; \
VCIPHER vin, V23, vin; \
VCIPHER vin, V24, vin; \
VCIPHER vin, V25, vin; \
VCIPHER vin, V26, vin; \
VCIPHER vin, V27, vin; \
XXLOR VS6, VS6, V23; \
XXLOR VS7, VS7, V24; \
XXLOR VS8, VS8, V25; \
XXLOR VS9, VS9, V26; \
VCIPHER vin, V23, vin; \
VCIPHER vin, V24, vin; \
VCIPHER vin, V25, vin; \
VCIPHER vin, V26, vin
// Encrypt 1 value (vin) with
// 2 specified keys
#define VCIPHER_1X2_KEYS(vin, key1, key2) \
XXLOR key1, key1, V25; \
XXLOR key2, key2, V26; \
VCIPHER vin, V25, vin; \
VCIPHER vin, V26, vin
// Encrypt 4 values in V15 - V18
// with the specified key from
// VS1 - VS9.
#define VCIPHER_4X1_KEY(key) \
XXLOR key, key, V23; \
VCIPHER V15, V23, V15; \
VCIPHER V16, V23, V16; \
VCIPHER V17, V23, V17; \
VCIPHER V18, V23, V18
// Encrypt 8 values in V15 - V22
// with the specified key,
// assuming it is a VSreg
#define VCIPHER_8X1_KEY(key) \
XXLOR key, key, V23; \
VCIPHER V15, V23, V15; \
VCIPHER V16, V23, V16; \
VCIPHER V17, V23, V17; \
VCIPHER V18, V23, V18; \
VCIPHER V19, V23, V19; \
VCIPHER V20, V23, V20; \
VCIPHER V21, V23, V21; \
VCIPHER V22, V23, V22
// Load input block into V1-V4
// in big endian order and
// update blk_inp by 64.
#define LOAD_INPUT_BLOCK64(blk_inp) \
MOVD $16, R16; \
MOVD $32, R17; \
MOVD $48, R18; \
P8_LXVB16X(blk_inp,R0,V1); \
P8_LXVB16X(blk_inp,R16,V2); \
P8_LXVB16X(blk_inp,R17,V3); \
P8_LXVB16X(blk_inp,R18,V4); \
ADD $64, blk_inp
// Load input block into V1-V8
// in big endian order and
// Update blk_inp by 128
#define LOAD_INPUT_BLOCK128(blk_inp) \
MOVD $16, R16; \
MOVD $32, R17; \
MOVD $48, R18; \
MOVD $64, R19; \
MOVD $80, R20; \
MOVD $96, R21; \
MOVD $112, R22; \
P8_LXVB16X(blk_inp,R0,V1); \
P8_LXVB16X(blk_inp,R16,V2); \
P8_LXVB16X(blk_inp,R17,V3); \
P8_LXVB16X(blk_inp,R18,V4); \
P8_LXVB16X(blk_inp,R19,V5); \
P8_LXVB16X(blk_inp,R20,V6); \
P8_LXVB16X(blk_inp,R21,V7); \
P8_LXVB16X(blk_inp,R22,V8); \
ADD $128, blk_inp
// Finish encryption on 8 streams and
// XOR with input block
#define VCIPHERLAST8_XOR_INPUT \
VCIPHERLAST V15, V23, V15; \
VCIPHERLAST V16, V23, V16; \
VCIPHERLAST V17, V23, V17; \
VCIPHERLAST V18, V23, V18; \
VCIPHERLAST V19, V23, V19; \
VCIPHERLAST V20, V23, V20; \
VCIPHERLAST V21, V23, V21; \
VCIPHERLAST V22, V23, V22; \
XXLXOR V1, V15, V1; \
XXLXOR V2, V16, V2; \
XXLXOR V3, V17, V3; \
XXLXOR V4, V18, V4; \
XXLXOR V5, V19, V5; \
XXLXOR V6, V20, V6; \
XXLXOR V7, V21, V7; \
XXLXOR V8, V22, V8
// Finish encryption on 4 streams and
// XOR with input block
#define VCIPHERLAST4_XOR_INPUT \
VCIPHERLAST V15, V23, V15; \
VCIPHERLAST V16, V23, V16; \
VCIPHERLAST V17, V23, V17; \
VCIPHERLAST V18, V23, V18; \
XXLXOR V1, V15, V1; \
XXLXOR V2, V16, V2; \
XXLXOR V3, V17, V3; \
XXLXOR V4, V18, V4
// Store output block from V1-V8
// in big endian order and
// Update blk_out by 128
#define STORE_OUTPUT_BLOCK128(blk_out) \
P8_STXVB16X(V1,blk_out,R0); \
P8_STXVB16X(V2,blk_out,R16); \
P8_STXVB16X(V3,blk_out,R17); \
P8_STXVB16X(V4,blk_out,R18); \
P8_STXVB16X(V5,blk_out,R19); \
P8_STXVB16X(V6,blk_out,R20); \
P8_STXVB16X(V7,blk_out,R21); \
P8_STXVB16X(V8,blk_out,R22); \
ADD $128, blk_out
// Store output block from V1-V4
// in big endian order and
// Update blk_out by 64
#define STORE_OUTPUT_BLOCK64(blk_out) \
P8_STXVB16X(V1,blk_out,R0); \
P8_STXVB16X(V2,blk_out,R16); \
P8_STXVB16X(V3,blk_out,R17); \
P8_STXVB16X(V4,blk_out,R18); \
ADD $64, blk_out
// func gcmInit(productTable *[256]byte, h []byte)
TEXT ·gcmInit(SB), NOSPLIT, $0-32
MOVD productTable+0(FP), XIP
MOVD h+8(FP), HTBL
MOVD $0x10, R8
MOVD $0x20, R9
MOVD $0x30, R10
LXVD2X (HTBL)(R0), VH // Load H
VSPLTISB $-16, XC2 // 0xf0
VSPLTISB $1, T0 // one
VADDUBM XC2, XC2, XC2 // 0xe0
VXOR ZERO, ZERO, ZERO
VOR XC2, T0, XC2 // 0xe1
VSLDOI $15, XC2, ZERO, XC2 // 0xe1...
VSLDOI $1, ZERO, T0, T1 // ...1
VADDUBM XC2, XC2, XC2 // 0xc2...
VSPLTISB $7, T2
VOR XC2, T1, XC2 // 0xc2....01
VSPLTB $0, H, T1 // most significant byte
VSL H, T0, H // H<<=1
VSRAB T1, T2, T1 // broadcast carry bit
VAND T1, XC2, T1
VXOR H, T1, IN // twisted H
VSLDOI $8, IN, IN, H // twist even more ...
VSLDOI $8, ZERO, XC2, XC2 // 0xc2.0
VSLDOI $8, ZERO, H, HL // ... and split
VSLDOI $8, H, ZERO, HH
STXVD2X VXC2, (XIP+R0) // save pre-computed table
STXVD2X VHL, (XIP+R8)
MOVD $0x40, R8
STXVD2X VH, (XIP+R9)
MOVD $0x50, R9
STXVD2X VHH, (XIP+R10)
MOVD $0x60, R10
VPMSUMD IN, HL, XL // H.lo·H.lo
VPMSUMD IN, H, XM // H.hi·H.lo+H.lo·H.hi
VPMSUMD IN, HH, XH // H.hi·H.hi
VPMSUMD XL, XC2, T2 // 1st reduction phase
VSLDOI $8, XM, ZERO, T0
VSLDOI $8, ZERO, XM, T1
VXOR XL, T0, XL
VXOR XH, T1, XH
VSLDOI $8, XL, XL, XL
VXOR XL, T2, XL
VSLDOI $8, XL, XL, T1 // 2nd reduction phase
VPMSUMD XL, XC2, XL
VXOR T1, XH, T1
VXOR XL, T1, IN1
VSLDOI $8, IN1, IN1, H2
VSLDOI $8, ZERO, H2, H2L
VSLDOI $8, H2, ZERO, H2H
STXVD2X VH2L, (XIP+R8) // save H^2
MOVD $0x70, R8
STXVD2X VH2, (XIP+R9)
MOVD $0x80, R9
STXVD2X VH2H, (XIP+R10)
MOVD $0x90, R10
VPMSUMD IN, H2L, XL // H.lo·H^2.lo
VPMSUMD IN1, H2L, XL1 // H^2.lo·H^2.lo
VPMSUMD IN, H2, XM // H.hi·H^2.lo+H.lo·H^2.hi
VPMSUMD IN1, H2, XM1 // H^2.hi·H^2.lo+H^2.lo·H^2.hi
VPMSUMD IN, H2H, XH // H.hi·H^2.hi
VPMSUMD IN1, H2H, XH1 // H^2.hi·H^2.hi
VPMSUMD XL, XC2, T2 // 1st reduction phase
VPMSUMD XL1, XC2, HH // 1st reduction phase
VSLDOI $8, XM, ZERO, T0
VSLDOI $8, ZERO, XM, T1
VSLDOI $8, XM1, ZERO, HL
VSLDOI $8, ZERO, XM1, H
VXOR XL, T0, XL
VXOR XH, T1, XH
VXOR XL1, HL, XL1
VXOR XH1, H, XH1
VSLDOI $8, XL, XL, XL
VSLDOI $8, XL1, XL1, XL1
VXOR XL, T2, XL
VXOR XL1, HH, XL1
VSLDOI $8, XL, XL, T1 // 2nd reduction phase
VSLDOI $8, XL1, XL1, H // 2nd reduction phase
VPMSUMD XL, XC2, XL
VPMSUMD XL1, XC2, XL1
VXOR T1, XH, T1
VXOR H, XH1, H
VXOR XL, T1, XL
VXOR XL1, H, XL1
VSLDOI $8, XL, XL, H
VSLDOI $8, XL1, XL1, H2
VSLDOI $8, ZERO, H, HL
VSLDOI $8, H, ZERO, HH
VSLDOI $8, ZERO, H2, H2L
VSLDOI $8, H2, ZERO, H2H
STXVD2X VHL, (XIP+R8) // save H^3
MOVD $0xa0, R8
STXVD2X VH, (XIP+R9)
MOVD $0xb0, R9
STXVD2X VHH, (XIP+R10)
MOVD $0xc0, R10
STXVD2X VH2L, (XIP+R8) // save H^4
STXVD2X VH2, (XIP+R9)
STXVD2X VH2H, (XIP+R10)
RET
// func gcmHash(output []byte, productTable *[256]byte, inp []byte, len int)
TEXT ·gcmHash(SB), NOSPLIT, $0-64
MOVD output+0(FP), XIP
MOVD productTable+24(FP), HTBL
MOVD inp+32(FP), INP
MOVD len+56(FP), LEN
MOVD $0x10, R8
MOVD $0x20, R9
MOVD $0x30, R10
LXVD2X (XIP)(R0), VXL // load Xi
LXVD2X (HTBL)(R8), VHL // load pre-computed table
MOVD $0x40, R8
LXVD2X (HTBL)(R9), VH
MOVD $0x50, R9
LXVD2X (HTBL)(R10), VHH
MOVD $0x60, R10
LXVD2X (HTBL)(R0), VXC2
#ifdef GOARCH_ppc64le
LVSL (R0)(R0), LEMASK
VSPLTISB $0x07, T0
VXOR LEMASK, T0, LEMASK
VPERM XL, XL, LEMASK, XL
#endif
VXOR ZERO, ZERO, ZERO
CMPU LEN, $64
BGE gcm_ghash_p8_4x
LXVD2X (INP)(R0), VIN
ADD $16, INP, INP
SUBCCC $16, LEN, LEN
#ifdef GOARCH_ppc64le
VPERM IN, IN, LEMASK, IN
#endif
VXOR IN, XL, IN
BEQ short
LXVD2X (HTBL)(R8), VH2L // load H^2
MOVD $16, R8
LXVD2X (HTBL)(R9), VH2
ADD LEN, INP, R9 // end of input
LXVD2X (HTBL)(R10), VH2H
loop_2x:
LXVD2X (INP)(R0), VIN1
#ifdef GOARCH_ppc64le
VPERM IN1, IN1, LEMASK, IN1
#endif
SUBC $32, LEN, LEN
VPMSUMD IN, H2L, XL // H^2.lo·Xi.lo
VPMSUMD IN1, HL, XL1 // H.lo·Xi+1.lo
SUBE R11, R11, R11 // borrow?-1:0
VPMSUMD IN, H2, XM // H^2.hi·Xi.lo+H^2.lo·Xi.hi
VPMSUMD IN1, H, XM1 // H.hi·Xi+1.lo+H.lo·Xi+1.hi
AND LEN, R11, R11
VPMSUMD IN, H2H, XH // H^2.hi·Xi.hi
VPMSUMD IN1, HH, XH1 // H.hi·Xi+1.hi
ADD R11, INP, INP
VXOR XL, XL1, XL
VXOR XM, XM1, XM
VPMSUMD XL, XC2, T2 // 1st reduction phase
VSLDOI $8, XM, ZERO, T0
VSLDOI $8, ZERO, XM, T1
VXOR XH, XH1, XH
VXOR XL, T0, XL
VXOR XH, T1, XH
VSLDOI $8, XL, XL, XL
VXOR XL, T2, XL
LXVD2X (INP)(R8), VIN
ADD $32, INP, INP
VSLDOI $8, XL, XL, T1 // 2nd reduction phase
VPMSUMD XL, XC2, XL
#ifdef GOARCH_ppc64le
VPERM IN, IN, LEMASK, IN
#endif
VXOR T1, XH, T1
VXOR IN, T1, IN
VXOR IN, XL, IN
CMP R9, INP
BGT loop_2x // done yet?
CMPWU LEN, $0
BNE even
short:
VPMSUMD IN, HL, XL // H.lo·Xi.lo
VPMSUMD IN, H, XM // H.hi·Xi.lo+H.lo·Xi.hi
VPMSUMD IN, HH, XH // H.hi·Xi.hi
VPMSUMD XL, XC2, T2 // 1st reduction phase
VSLDOI $8, XM, ZERO, T0
VSLDOI $8, ZERO, XM, T1
VXOR XL, T0, XL
VXOR XH, T1, XH
VSLDOI $8, XL, XL, XL
VXOR XL, T2, XL
VSLDOI $8, XL, XL, T1 // 2nd reduction phase
VPMSUMD XL, XC2, XL
VXOR T1, XH, T1
even:
VXOR XL, T1, XL
#ifdef GOARCH_ppc64le
VPERM XL, XL, LEMASK, XL
#endif
STXVD2X VXL, (XIP+R0)
OR R12, R12, R12 // write out Xi
RET
gcm_ghash_p8_4x:
LVSL (R8)(R0), T0 // 0x0001..0e0f
MOVD $0x70, R8
LXVD2X (HTBL)(R9), VH2
MOVD $0x80, R9
VSPLTISB $8, T1 // 0x0808..0808
MOVD $0x90, R10
LXVD2X (HTBL)(R8), VH3L // load H^3
MOVD $0xa0, R8
LXVD2X (HTBL)(R9), VH3
MOVD $0xb0, R9
LXVD2X (HTBL)(R10), VH3H
MOVD $0xc0, R10
LXVD2X (HTBL)(R8), VH4L // load H^4
MOVD $0x10, R8
LXVD2X (HTBL)(R9), VH4
MOVD $0x20, R9
LXVD2X (HTBL)(R10), VH4H
MOVD $0x30, R10
VSLDOI $8, ZERO, T1, T2 // 0x0000..0808
VADDUBM T0, T2, HIPERM // 0x0001..1617
VADDUBM T1, HIPERM, LOPERM // 0x0809..1e1f
SRD $4, LEN, LEN // this allows to use sign bit as carry
LXVD2X (INP)(R0), VIN0 // load input
LXVD2X (INP)(R8), VIN1
SUBCCC $8, LEN, LEN
LXVD2X (INP)(R9), VIN2
LXVD2X (INP)(R10), VIN3
ADD $0x40, INP, INP
#ifdef GOARCH_ppc64le
VPERM IN0, IN0, LEMASK, IN0
VPERM IN1, IN1, LEMASK, IN1
VPERM IN2, IN2, LEMASK, IN2
VPERM IN3, IN3, LEMASK, IN3
#endif
VXOR IN0, XL, XH
VPMSUMD IN1, H3L, XL1
VPMSUMD IN1, H3, XM1
VPMSUMD IN1, H3H, XH1
VPERM H2, H, HIPERM, H21L
VPERM IN2, IN3, LOPERM, T0
VPERM H2, H, LOPERM, H21H
VPERM IN2, IN3, HIPERM, T1
VPMSUMD IN2, H2, XM2 // H^2.lo·Xi+2.hi+H^2.hi·Xi+2.lo
VPMSUMD T0, H21L, XL3 // H^2.lo·Xi+2.lo+H.lo·Xi+3.lo
VPMSUMD IN3, H, XM3 // H.hi·Xi+3.lo +H.lo·Xi+3.hi
VPMSUMD T1, H21H, XH3 // H^2.hi·Xi+2.hi+H.hi·Xi+3.hi
VXOR XM2, XM1, XM2
VXOR XL3, XL1, XL3
VXOR XM3, XM2, XM3
VXOR XH3, XH1, XH3
BLT tail_4x
loop_4x:
LXVD2X (INP)(R0), VIN0
LXVD2X (INP)(R8), VIN1
SUBCCC $4, LEN, LEN
LXVD2X (INP)(R9), VIN2
LXVD2X (INP)(R10), VIN3
ADD $0x40, INP, INP
#ifdef GOARCH_ppc64le
VPERM IN1, IN1, LEMASK, IN1
VPERM IN2, IN2, LEMASK, IN2
VPERM IN3, IN3, LEMASK, IN3
VPERM IN0, IN0, LEMASK, IN0
#endif
VPMSUMD XH, H4L, XL // H^4.lo·Xi.lo
VPMSUMD XH, H4, XM // H^4.hi·Xi.lo+H^4.lo·Xi.hi
VPMSUMD XH, H4H, XH // H^4.hi·Xi.hi
VPMSUMD IN1, H3L, XL1
VPMSUMD IN1, H3, XM1
VPMSUMD IN1, H3H, XH1
VXOR XL, XL3, XL
VXOR XM, XM3, XM
VXOR XH, XH3, XH
VPERM IN2, IN3, LOPERM, T0
VPERM IN2, IN3, HIPERM, T1
VPMSUMD XL, XC2, T2 // 1st reduction phase
VPMSUMD T0, H21L, XL3 // H.lo·Xi+3.lo +H^2.lo·Xi+2.lo
VPMSUMD T1, H21H, XH3 // H.hi·Xi+3.hi +H^2.hi·Xi+2.hi
VSLDOI $8, XM, ZERO, T0
VSLDOI $8, ZERO, XM, T1
VXOR XL, T0, XL
VXOR XH, T1, XH
VSLDOI $8, XL, XL, XL
VXOR XL, T2, XL
VSLDOI $8, XL, XL, T1 // 2nd reduction phase
VPMSUMD IN2, H2, XM2 // H^2.hi·Xi+2.lo+H^2.lo·Xi+2.hi
VPMSUMD IN3, H, XM3 // H.hi·Xi+3.lo +H.lo·Xi+3.hi
VPMSUMD XL, XC2, XL
VXOR XL3, XL1, XL3
VXOR XH3, XH1, XH3
VXOR XH, IN0, XH
VXOR XM2, XM1, XM2
VXOR XH, T1, XH
VXOR XM3, XM2, XM3
VXOR XH, XL, XH
BGE loop_4x
tail_4x:
VPMSUMD XH, H4L, XL // H^4.lo·Xi.lo
VPMSUMD XH, H4, XM // H^4.hi·Xi.lo+H^4.lo·Xi.hi
VPMSUMD XH, H4H, XH // H^4.hi·Xi.hi
VXOR XL, XL3, XL
VXOR XM, XM3, XM
VPMSUMD XL, XC2, T2 // 1st reduction phase
VSLDOI $8, XM, ZERO, T0
VSLDOI $8, ZERO, XM, T1
VXOR XH, XH3, XH
VXOR XL, T0, XL
VXOR XH, T1, XH
VSLDOI $8, XL, XL, XL
VXOR XL, T2, XL
VSLDOI $8, XL, XL, T1 // 2nd reduction phase
VPMSUMD XL, XC2, XL
VXOR T1, XH, T1
VXOR XL, T1, XL
ADDCCC $4, LEN, LEN
BEQ done_4x
LXVD2X (INP)(R0), VIN0
CMPU LEN, $2
MOVD $-4, LEN
BLT one
LXVD2X (INP)(R8), VIN1
BEQ two
three:
LXVD2X (INP)(R9), VIN2
#ifdef GOARCH_ppc64le
VPERM IN0, IN0, LEMASK, IN0
VPERM IN1, IN1, LEMASK, IN1
VPERM IN2, IN2, LEMASK, IN2
#endif
VXOR IN0, XL, XH
VOR H3L, H3L, H4L
VOR H3, H3, H4
VOR H3H, H3H, H4H
VPERM IN1, IN2, LOPERM, T0
VPERM IN1, IN2, HIPERM, T1
VPMSUMD IN1, H2, XM2 // H^2.lo·Xi+1.hi+H^2.hi·Xi+1.lo
VPMSUMD IN2, H, XM3 // H.hi·Xi+2.lo +H.lo·Xi+2.hi
VPMSUMD T0, H21L, XL3 // H^2.lo·Xi+1.lo+H.lo·Xi+2.lo
VPMSUMD T1, H21H, XH3 // H^2.hi·Xi+1.hi+H.hi·Xi+2.hi
VXOR XM3, XM2, XM3
JMP tail_4x
two:
#ifdef GOARCH_ppc64le
VPERM IN0, IN0, LEMASK, IN0
VPERM IN1, IN1, LEMASK, IN1
#endif
VXOR IN, XL, XH
VPERM ZERO, IN1, LOPERM, T0
VPERM ZERO, IN1, HIPERM, T1
VSLDOI $8, ZERO, H2, H4L
VOR H2, H2, H4
VSLDOI $8, H2, ZERO, H4H
VPMSUMD T0, H21L, XL3 // H.lo·Xi+1.lo
VPMSUMD IN1, H, XM3 // H.hi·Xi+1.lo+H.lo·Xi+2.hi
VPMSUMD T1, H21H, XH3 // H.hi·Xi+1.hi
JMP tail_4x
one:
#ifdef GOARCH_ppc64le
VPERM IN0, IN0, LEMASK, IN0
#endif
VSLDOI $8, ZERO, H, H4L
VOR H, H, H4
VSLDOI $8, H, ZERO, H4H
VXOR IN0, XL, XH
VXOR XL3, XL3, XL3
VXOR XM3, XM3, XM3
VXOR XH3, XH3, XH3
JMP tail_4x
done_4x:
#ifdef GOARCH_ppc64le
VPERM XL, XL, LEMASK, XL
#endif
STXVD2X VXL, (XIP+R0) // write out Xi
RET
// func gcmMul(output []byte, productTable *[256]byte)
TEXT ·gcmMul(SB), NOSPLIT, $0-32
MOVD output+0(FP), XIP
MOVD productTable+24(FP), HTBL
MOVD $0x10, R8
MOVD $0x20, R9
MOVD $0x30, R10
LXVD2X (XIP)(R0), VIN // load Xi
LXVD2X (HTBL)(R8), VHL // Load pre-computed table
LXVD2X (HTBL)(R9), VH
LXVD2X (HTBL)(R10), VHH
LXVD2X (HTBL)(R0), VXC2
#ifdef GOARCH_ppc64le
VSPLTISB $0x07, T0
VXOR LEMASK, T0, LEMASK
VPERM IN, IN, LEMASK, IN
#endif
VXOR ZERO, ZERO, ZERO
VPMSUMD IN, HL, XL // H.lo·Xi.lo
VPMSUMD IN, H, XM // H.hi·Xi.lo+H.lo·Xi.hi
VPMSUMD IN, HH, XH // H.hi·Xi.hi
VPMSUMD XL, XC2, T2 // 1st reduction phase
VSLDOI $8, XM, ZERO, T0
VSLDOI $8, ZERO, XM, T1
VXOR XL, T0, XL
VXOR XH, T1, XH
VSLDOI $8, XL, XL, XL
VXOR XL, T2, XL
VSLDOI $8, XL, XL, T1 // 2nd reduction phase
VPMSUMD XL, XC2, XL
VXOR T1, XH, T1
VXOR XL, T1, XL
#ifdef GOARCH_ppc64le
VPERM XL, XL, LEMASK, XL
#endif
STXVD2X VXL, (XIP+R0) // write out Xi
RET
#define BLK_INP R3
#define BLK_OUT R4
#define BLK_KEY R5
#define KEY_LEN R6
#define BLK_IDX R7
#define IDX R8
#define IN_LEN R9
#define COUNTER R10
#define CONPTR R14
#define MASK V5
// Implementation of the counterCrypt function in assembler.
// Original loop is unrolled to allow for multiple encryption
// streams to be done in parallel, which is achieved by interleaving
// vcipher instructions from each stream. This is also referred to as
// stitching, and provides significant performance improvements.
// Some macros are defined which enable execution for big or little
// endian as well as different ISA targets.
//func (g *gcmAsm) counterCrypt(out, in []byte, counter *[gcmBlockSize]byte, key[gcmBlockSize]uint32)
//func counterCryptASM(xr, out, in, counter, key)
TEXT ·counterCryptASM(SB), NOSPLIT, $16-72
MOVD xr(FP), KEY_LEN
MOVD out+8(FP), BLK_OUT
MOVD out_len+16(FP), R8
MOVD in+32(FP), BLK_INP
MOVD in_len+40(FP), IN_LEN
MOVD counter+56(FP), COUNTER
MOVD key+64(FP), BLK_KEY
// Set up permute string when needed.
#ifdef NEEDS_ESPERM
MOVD $·rcon(SB), R14
LVX (R14), ESPERM // Permute value for P8_ macros.
#endif
SETUP_COUNTER // V30 Counter V31 BE {0, 0, 0, 1}
LOAD_KEYS(BLK_KEY, KEY_LEN) // VS1 - VS10/12/14 based on keysize
CMP IN_LEN, $128
BLT block64
block128_loop:
// Do 8 encryptions in parallel by setting
// input values in V15-V22 and executing
// vcipher on the updated value and the keys.
GEN_VCIPHER_8_INPUTS
VCIPHER_8X1_KEY(VS1)
VCIPHER_8X1_KEY(VS2)
VCIPHER_8X1_KEY(VS3)
VCIPHER_8X1_KEY(VS4)
VCIPHER_8X1_KEY(VS5)
VCIPHER_8X1_KEY(VS6)
VCIPHER_8X1_KEY(VS7)
VCIPHER_8X1_KEY(VS8)
VCIPHER_8X1_KEY(VS9)
// Additional encryptions are done based on
// the key length, with the last key moved
// to V23 for use with VCIPHERLAST.
// CR2 = CMP key_len, $12
XXLOR VS10, VS10, V23
BLT CR2, block128_last // key_len = 10
VCIPHER_8X1_KEY(VS10)
VCIPHER_8X1_KEY(VS11)
XXLOR VS12,VS12,V23
BEQ CR2, block128_last // ken_len = 12
VCIPHER_8X1_KEY(VS12)
VCIPHER_8X1_KEY(VS13)
XXLOR VS14,VS14,V23 // key_len = 14
block128_last:
// vcipher encryptions are in V15-V22 at this
// point with vcipherlast remaining to be done.
// Load input block into V1-V8, setting index offsets
// in R16-R22 to use with the STORE.
LOAD_INPUT_BLOCK128(BLK_INP)
// Do VCIPHERLAST on the last key for each encryption
// stream and XOR the result with the corresponding
// value from the input block.
VCIPHERLAST8_XOR_INPUT
// Store the results (8*16) and update BLK_OUT by 128.
STORE_OUTPUT_BLOCK128(BLK_OUT)
ADD $-128, IN_LEN // input size
CMP IN_LEN, $128 // check if >= blocksize
BGE block128_loop // next input block
CMP IN_LEN, $0
BEQ done
block64:
CMP IN_LEN, $64 // Check if >= 64
BLT block16_loop
// Do 4 encryptions in parallel by setting
// input values in V15-V18 and executing
// vcipher on the updated value and the keys.
GEN_VCIPHER_4_INPUTS
VCIPHER_4X1_KEY(VS1)
VCIPHER_4X1_KEY(VS2)
VCIPHER_4X1_KEY(VS3)
VCIPHER_4X1_KEY(VS4)
VCIPHER_4X1_KEY(VS5)
VCIPHER_4X1_KEY(VS6)
VCIPHER_4X1_KEY(VS7)
VCIPHER_4X1_KEY(VS8)
VCIPHER_4X1_KEY(VS9)
// Check key length based on CR2
// Move last key to V23 for use with later vcipherlast
XXLOR VS10, VS10, V23
BLT CR2, block64_last // size = 10
VCIPHER_4X1_KEY(VS10) // Encrypt next 2 keys
VCIPHER_4X1_KEY(VS11)
XXLOR VS12, VS12, V23
BEQ CR2, block64_last // size = 12
VCIPHER_4X1_KEY(VS12) // Encrypt last 2 keys
VCIPHER_4X1_KEY(VS13)
XXLOR VS14, VS14, V23 // size = 14
block64_last:
LOAD_INPUT_BLOCK64(BLK_INP) // Load 64 bytes of input
// Do VCIPHERLAST on the last for each encryption
// stream and XOR the result with the corresponding
// value from the input block.
VCIPHERLAST4_XOR_INPUT
// Store the results (4*16) and update BLK_OUT by 64.
STORE_OUTPUT_BLOCK64(BLK_OUT)
ADD $-64, IN_LEN // decrement input block length
CMP IN_LEN, $0 // check for remaining length
BEQ done
block16_loop:
CMP IN_LEN, $16 // More input
BLT final_block // If not, then handle partial block
// Single encryption, no stitching
GEN_VCIPHER_INPUT // Generate input value for single encryption
VCIPHER_1X9_KEYS(V15) // Encrypt V15 value with 9 keys
XXLOR VS10, VS10, V23 // Last key -> V23 for later vcipiherlast
// Key length based on CR2. (LT=10, EQ=12, GT=14)
BLT CR2, block16_last // Finish for key size 10
VCIPHER_1X2_KEYS(V15, VS10, VS11) // Encrypt V15 with 2 more keys
XXLOR VS12, VS12, V23 // Last key -> V23 for later vcipherlast
BEQ CR2, block16_last // Finish for key size 12
VCIPHER_1X2_KEYS(V15, VS12, VS13) // Encrypt V15 with last 2 keys
XXLOR VS14, VS14, V23 // Last key -> V23 for vcipherlast with key size 14
block16_last:
P8_LXVB16X(BLK_INP, R0, V1) // Load input
VCIPHERLAST V15, V23, V15 // Encrypt last value in V23
XXLXOR V15, V1, V1 // XOR with input
P8_STXVB16X(V1,R0,BLK_OUT) // Store final encryption value to output
ADD $16, BLK_INP // Increment input pointer
ADD $16, BLK_OUT // Increment output pointer
ADD $-16, IN_LEN // Decrement input length
BR block16_loop // Check for next
final_block:
CMP IN_LEN, $0
BEQ done
GEN_VCIPHER_INPUT // Generate input value for partial encryption
VCIPHER_1X9_KEYS(V15) // Encrypt V15 with 9 keys
XXLOR VS10, VS10, V23 // Save possible last key
BLT CR2, final_block_last
VCIPHER_1X2_KEYS(V15, VS10, VS11) // Encrypt V15 with next 2 keys
XXLOR VS12, VS12, V23 // Save possible last key
BEQ CR2, final_block_last
VCIPHER_1X2_KEYS(V15, VS12, VS13) // Encrypt V15 with last 2 keys
XXLOR VS14, VS14, V23 // Save last key
final_block_last:
VCIPHERLAST V15, V23, V15 // Finish encryption
#ifdef GOPPC64_power10
// set up length
SLD $56, IN_LEN, R17
LXVLL BLK_INP, R17, V25
VXOR V25, V15, V25
STXVLL V25, BLK_OUT, R17
#else
ADD $32, R1, MASK_PTR
MOVD $0, R16
P8_STXVB16X(V15, MASK_PTR, R0)
CMP IN_LEN, $8
BLT next4
MOVD 0(MASK_PTR), R14
MOVD 0(BLK_INP), R15
XOR R14, R15, R14
MOVD R14, 0(BLK_OUT)
ADD $8, R16
ADD $-8, IN_LEN
next4:
CMP IN_LEN, $4
BLT next2
MOVWZ (BLK_INP)(R16), R15
MOVWZ (MASK_PTR)(R16), R14
XOR R14, R15, R14
MOVW R14, (R16)(BLK_OUT)
ADD $4, R16
ADD $-4, IN_LEN
next2:
CMP IN_LEN, $2
BLT next1
MOVHZ (BLK_INP)(R16), R15
MOVHZ (MASK_PTR)(R16), R14
XOR R14, R15, R14
MOVH R14, (R16)(BLK_OUT)
ADD $2, R16
ADD $-2, IN_LEN
next1:
CMP IN_LEN, $1
BLT done
MOVBZ (MASK_PTR)(R16), R14
MOVBZ (BLK_INP)(R16), R15
XOR R14, R15, R14
MOVB R14, (R16)(BLK_OUT)
#endif
done:
// Save the updated counter value
P8_STXVB16X(V30, COUNTER, R0)
// Clear the keys
XXLXOR VS0, VS0, VS0
XXLXOR VS1, VS1, VS1
XXLXOR VS2, VS2, VS2
XXLXOR VS3, VS3, VS3
XXLXOR VS4, VS4, VS4
XXLXOR VS5, VS5, VS5
XXLXOR VS6, VS6, VS6
XXLXOR VS7, VS7, VS7
XXLXOR VS8, VS8, VS8
XXLXOR VS9, VS9, VS9
XXLXOR VS10, VS10, VS10
XXLXOR VS11, VS11, VS11
XXLXOR VS12, VS12, VS12
XXLXOR VS13, VS13, VS13
XXLXOR VS14, VS14, VS14
RET