/* The Keccak-p permutations, designed by Guido Bertoni, Joan Daemen, Michaël Peeters and Gilles Van Assche. Implementation by Ronny Van Keer, hereby denoted as "the implementer". For more information, feedback or questions, please refer to the Keccak Team website: https://keccak.team/ To the extent possible under law, the implementer has waived all copyright and related or neighboring rights to the source code in this file. http://creativecommons.org/publicdomain/zero/1.0/ --- This file implements Keccak-p[1600]×2 in a PlSnP-compatible way. Please refer to PlSnP-documentation.h for more details. This implementation comes with KeccakP-1600-times2-SnP.h in the same folder. Please refer to LowLevel.build for the exact list of other files it must be combined with. */ #include #include #include #include #include #include #include #include #include "align.h" #include "KeccakP-1600-times2-SnP.h" #include "SIMD512-2-config.h" #include "brg_endian.h" #if (PLATFORM_BYTE_ORDER != IS_LITTLE_ENDIAN) #error Expecting a little-endian platform #endif /* Comment the define hereunder when compiling for a CPU with AVX-512 SIMD */ /* * Warning: This code has only been tested on Haswell (AVX2) with SIMULATE_AVX512 defined, * errors will occur if we did a bad interpretation of the AVX-512 intrinsics' * API or functionality. */ /* #define SIMULATE_AVX512 */ #if defined(SIMULATE_AVX512) typedef struct { uint64_t x[8]; } __m512i; static __m512i _mm512_xor_si512( __m512i a, __m512i b) { __m512i r; unsigned int i; for ( i = 0; i < 8; ++i ) r.x[i] = a.x[i] ^ b.x[i]; return(r); } static __m128i _mm_ternarylogic_epi64(__m128i a, __m128i b, __m128i c, int imm) { if (imm == 0x96) return _mm_xor_si128( _mm_xor_si128( a, b ), c ); if (imm == 0xD2) return _mm_xor_si128( a, _mm_andnot_si128(b, c) ); printf( "_mm_ternarylogic_epi64( a, b, c, %02X) not implemented!\n", imm ); exit(1); } static __m128i _mm_rol_epi64(__m128i a, int offset) { return _mm_or_si128(_mm_slli_epi64(a, offset), _mm_srli_epi64(a, 64-offset)); } static __m512i _mm512_i32gather_epi64(__m256i idx, const void *p, int scale) { __m512i r; unsigned int i; uint32_t offset[8]; _mm256_store_si256( (__m256i*)offset, idx ); for ( i = 0; i < 8; ++i ) r.x[i] = *(const uint64_t*)((const char*)p + offset[i] * scale); return(r); } static void _mm_i32scatter_epi64( void *p, __m128i idx, __m128i value, int scale) { unsigned int i; uint64_t v[2]; uint32_t offset[4]; _mm_store_ps( (float*)offset, (__m128)idx ); _mm_store_pd( (double*)v, (__m128d)value ); for ( i = 0; i < 2; ++i ) *(uint64_t*)((char*)p + offset[i] * scale) = v[i]; } static void _mm512_i32scatter_epi64( void *p, __m256i idx, __m512i value, int scale) { unsigned int i; uint32_t offset[8]; _mm256_store_si256( (__m256i*)offset, idx ); for ( i = 0; i < 8; ++i ) *(uint64_t*)((char*)p + offset[i] * scale) = value.x[i]; } #endif typedef __m128i V128; typedef __m256i V256; typedef __m512i V512; #if defined(KeccakP1600times2_useAVX512) #define XOR(a,b) _mm_xor_si128(a,b) #define XOR3(a,b,c) _mm_ternarylogic_epi64(a,b,c,0x96) #define XOR5(a,b,c,d,e) XOR3(XOR3(a,b,c),d,e) #define XOR512(a,b) _mm512_xor_si512(a,b) #define ROL(a,offset) _mm_rol_epi64(a,offset) #define Chi(a,b,c) _mm_ternarylogic_epi64(a,b,c,0xD2) #define CONST128_64(a) _mm_set1_epi64x(a) #define LOAD4_32(a,b,c,d) _mm_set_epi32((uint64_t)(a), (uint32_t)(b), (uint32_t)(c), (uint32_t)(d)) #define LOAD8_32(a,b,c,d,e,f,g,h) _mm256_set_epi32((uint64_t)(a), (uint32_t)(b), (uint32_t)(c), (uint32_t)(d), (uint32_t)(e), (uint32_t)(f), (uint32_t)(g), (uint32_t)(h)) #define LOAD_GATHER2_64(idx,p) _mm_i32gather_epi64( (const long long int*)(p), idx, 8) #define LOAD_GATHER8_64(idx,p) _mm512_i32gather_epi64( idx, (const void*)(p), 8) #define STORE_SCATTER2_64(p,idx, v) _mm_i32scatter_epi64( (void*)(p), idx, v, 8) #define STORE_SCATTER8_64(p,idx, v) _mm512_i32scatter_epi64( (void*)(p), idx, v, 8) #endif #define laneIndex(instanceIndex, lanePosition) ((lanePosition)*2 + instanceIndex) #define SnP_laneLengthInBytes 8 void KeccakP1600times2_InitializeAll(KeccakP1600times2_align512SIMD128_states *states) { memset(states, 0, sizeof(KeccakP1600times2_align512SIMD128_states)); } void KeccakP1600times2_AddBytes(KeccakP1600times2_align512SIMD128_states *states, unsigned int instanceIndex, const unsigned char *data, unsigned int offset, unsigned int length) { unsigned int sizeLeft = length; unsigned int lanePosition = offset/SnP_laneLengthInBytes; unsigned int offsetInLane = offset%SnP_laneLengthInBytes; const unsigned char *curData = data; uint64_t *statesAsLanes = (uint64_t*)states->A; if ((sizeLeft > 0) && (offsetInLane != 0)) { unsigned int bytesInLane = SnP_laneLengthInBytes - offsetInLane; uint64_t lane = 0; if (bytesInLane > sizeLeft) bytesInLane = sizeLeft; memcpy((unsigned char*)&lane + offsetInLane, curData, bytesInLane); statesAsLanes[laneIndex(instanceIndex, lanePosition)] ^= lane; sizeLeft -= bytesInLane; lanePosition++; curData += bytesInLane; } while(sizeLeft >= SnP_laneLengthInBytes) { uint64_t lane = *((const uint64_t*)curData); statesAsLanes[laneIndex(instanceIndex, lanePosition)] ^= lane; sizeLeft -= SnP_laneLengthInBytes; lanePosition++; curData += SnP_laneLengthInBytes; } if (sizeLeft > 0) { uint64_t lane = 0; memcpy(&lane, curData, sizeLeft); statesAsLanes[laneIndex(instanceIndex, lanePosition)] ^= lane; } } void KeccakP1600times2_AddLanesAll(KeccakP1600times2_align512SIMD128_states *states, const unsigned char *data, unsigned int laneCount, unsigned int laneOffset) { V128 *stateAsLanes128 = states->A; V512 *stateAsLanes512 = (V512*)states->A; const uint64_t *dataAsLanes = (const uint64_t *)data; unsigned int i; V128 index128 = LOAD4_32(0, 0, 1*laneOffset, 0*laneOffset); V256 index512 = LOAD8_32(1*laneOffset+3, 0*laneOffset+3, 1*laneOffset+2, 0*laneOffset+2, 1*laneOffset+1, 0*laneOffset+1, 1*laneOffset, 0*laneOffset); #define Add_In1( argIndex ) stateAsLanes128[argIndex] = XOR(stateAsLanes128[argIndex], LOAD_GATHER2_64(index128, dataAsLanes+argIndex)) #define Add_In4( argIndex ) stateAsLanes512[argIndex/4] = XOR512(stateAsLanes512[argIndex/4], LOAD_GATHER8_64(index512, dataAsLanes+argIndex)) if ( laneCount >= 16 ) { Add_In4( 0 ); Add_In4( 4 ); Add_In4( 8 ); Add_In4( 12 ); if ( laneCount >= 20 ) { Add_In4( 16 ); for(i=20; iA; if ((sizeLeft > 0) && (offsetInLane != 0)) { unsigned int bytesInLane = SnP_laneLengthInBytes - offsetInLane; if (bytesInLane > sizeLeft) bytesInLane = sizeLeft; memcpy( ((unsigned char *)&statesAsLanes[laneIndex(instanceIndex, lanePosition)]) + offsetInLane, curData, bytesInLane); sizeLeft -= bytesInLane; lanePosition++; curData += bytesInLane; } while(sizeLeft >= SnP_laneLengthInBytes) { uint64_t lane = *((const uint64_t*)curData); statesAsLanes[laneIndex(instanceIndex, lanePosition)] = lane; sizeLeft -= SnP_laneLengthInBytes; lanePosition++; curData += SnP_laneLengthInBytes; } if (sizeLeft > 0) { memcpy(&statesAsLanes[laneIndex(instanceIndex, lanePosition)], curData, sizeLeft); } } void KeccakP1600times2_OverwriteLanesAll(KeccakP1600times2_align512SIMD128_states *states, const unsigned char *data, unsigned int laneCount, unsigned int laneOffset) { V128 *stateAsLanes128 = states->A; V512 *stateAsLanes512 = (V512*)states->A; const uint64_t *dataAsLanes = (const uint64_t *)data; unsigned int i; V128 index128 = LOAD4_32(0, 0, 1*laneOffset, 0*laneOffset); V256 index512 = LOAD8_32(1*laneOffset+3, 0*laneOffset+3, 1*laneOffset+2, 0*laneOffset+2, 1*laneOffset+1, 0*laneOffset+1, 1*laneOffset, 0*laneOffset); #define OverWr1( argIndex ) stateAsLanes128[argIndex] = LOAD_GATHER2_64(index128, dataAsLanes+argIndex) #define OverWr4( argIndex ) stateAsLanes512[argIndex/4] = LOAD_GATHER8_64(index512, dataAsLanes+argIndex) if ( laneCount >= 16 ) { OverWr4( 0 ); OverWr4( 4 ); OverWr4( 8 ); OverWr4( 12 ); if ( laneCount >= 20 ) { OverWr4( 16 ); for(i=20; iA; while(sizeLeft >= SnP_laneLengthInBytes) { statesAsLanes[laneIndex(instanceIndex, lanePosition)] = 0; sizeLeft -= SnP_laneLengthInBytes; lanePosition++; } if (sizeLeft > 0) { memset(&statesAsLanes[laneIndex(instanceIndex, lanePosition)], 0, sizeLeft); } } void KeccakP1600times2_ExtractBytes(const KeccakP1600times2_align512SIMD128_states *states, unsigned int instanceIndex, unsigned char *data, unsigned int offset, unsigned int length) { unsigned int sizeLeft = length; unsigned int lanePosition = offset/SnP_laneLengthInBytes; unsigned int offsetInLane = offset%SnP_laneLengthInBytes; unsigned char *curData = data; const uint64_t *statesAsLanes = (const uint64_t*)states->A; if ((sizeLeft > 0) && (offsetInLane != 0)) { unsigned int bytesInLane = SnP_laneLengthInBytes - offsetInLane; if (bytesInLane > sizeLeft) bytesInLane = sizeLeft; memcpy( curData, ((unsigned char *)&statesAsLanes[laneIndex(instanceIndex, lanePosition)]) + offsetInLane, bytesInLane); sizeLeft -= bytesInLane; lanePosition++; curData += bytesInLane; } while(sizeLeft >= SnP_laneLengthInBytes) { *(uint64_t*)curData = statesAsLanes[laneIndex(instanceIndex, lanePosition)]; sizeLeft -= SnP_laneLengthInBytes; lanePosition++; curData += SnP_laneLengthInBytes; } if (sizeLeft > 0) { memcpy( curData, &statesAsLanes[laneIndex(instanceIndex, lanePosition)], sizeLeft); } } void KeccakP1600times2_ExtractLanesAll(const KeccakP1600times2_align512SIMD128_states *states, unsigned char *data, unsigned int laneCount, unsigned int laneOffset) { const V128 *stateAsLanes128 = states->A; const V512 *stateAsLanes512 = (V512*)states->A; uint64_t *dataAsLanes = (uint64_t *)data; unsigned int i; V128 index128 = LOAD4_32(0, 0, 1*laneOffset, 0*laneOffset); V256 index512 = LOAD8_32(1*laneOffset+3, 0*laneOffset+3, 1*laneOffset+2, 0*laneOffset+2, 1*laneOffset+1, 0*laneOffset+1, 1*laneOffset, 0*laneOffset); #define Extr1( argIndex ) STORE_SCATTER2_64(dataAsLanes+argIndex, index128, stateAsLanes128[argIndex]) #define Extr4( argIndex ) STORE_SCATTER8_64(dataAsLanes+argIndex, index512, stateAsLanes512[argIndex/4]) if ( laneCount >= 16 ) { Extr4( 0 ); Extr4( 4 ); Extr4( 8 ); Extr4( 12 ); if ( laneCount >= 20 ) { Extr4( 16 ); for(i=20; iA; if ((sizeLeft > 0) && (offsetInLane != 0)) { unsigned int bytesInLane = SnP_laneLengthInBytes - offsetInLane; uint64_t lane = statesAsLanes[laneIndex(instanceIndex, lanePosition)] >> (8 * offsetInLane); if (bytesInLane > sizeLeft) bytesInLane = sizeLeft; sizeLeft -= bytesInLane; do { *(curOutput++) = *(curInput++) ^ (unsigned char)lane; lane >>= 8; } while ( --bytesInLane != 0); lanePosition++; } while(sizeLeft >= SnP_laneLengthInBytes) { *((uint64_t*)curOutput) = *((uint64_t*)curInput) ^ statesAsLanes[laneIndex(instanceIndex, lanePosition)]; sizeLeft -= SnP_laneLengthInBytes; lanePosition++; curInput += SnP_laneLengthInBytes; curOutput += SnP_laneLengthInBytes; } if (sizeLeft != 0) { uint64_t lane = statesAsLanes[laneIndex(instanceIndex, lanePosition)]; do { *(curOutput++) = *(curInput++) ^ (unsigned char)lane; lane >>= 8; } while ( --sizeLeft != 0); } } void KeccakP1600times2_ExtractAndAddLanesAll(const KeccakP1600times2_align512SIMD128_states *states, const unsigned char *input, unsigned char *output, unsigned int laneCount, unsigned int laneOffset) { const V128 *stateAsLanes128 = states->A; const V512 *stateAsLanes512 = (V512*)states->A; const uint64_t *inAsLanes = (const uint64_t *)input; uint64_t *outAsLanes = (uint64_t *)output; unsigned int i; V128 index128 = LOAD4_32(0, 0, 1*laneOffset, 0*laneOffset); V256 index512 = LOAD8_32(1*laneOffset+3, 0*laneOffset+3, 1*laneOffset+2, 0*laneOffset+2, 1*laneOffset+1, 0*laneOffset+1, 1*laneOffset, 0*laneOffset); #define ExtrAdd1( argIndex ) STORE_SCATTER2_64(outAsLanes+argIndex, index128, XOR(stateAsLanes128[argIndex], LOAD_GATHER2_64(index128, inAsLanes+argIndex))) #define ExtrAdd4( argIndex ) STORE_SCATTER8_64(outAsLanes+argIndex, index512, XOR512(stateAsLanes512[argIndex/4], LOAD_GATHER8_64(index512, inAsLanes+argIndex))) if ( laneCount >= 16 ) { ExtrAdd4( 0 ); ExtrAdd4( 4 ); ExtrAdd4( 8 ); ExtrAdd4( 12 ); if ( laneCount >= 20 ) { ExtrAdd4( 16 ); for(i=20; iA; KeccakP_DeclareVars; #ifndef KeccakP1600times2_fullUnrolling unsigned int i; #endif copyFromState(statesAsLanes); rounds24; copyToState(statesAsLanes); } void KeccakP1600times2_PermuteAll_12rounds(KeccakP1600times2_align512SIMD128_states *states) { V128 *statesAsLanes = states->A; KeccakP_DeclareVars; #if (KeccakP1600times2_unrolling < 12) unsigned int i; #endif copyFromState(statesAsLanes); rounds12; copyToState(statesAsLanes); } void KeccakP1600times2_PermuteAll_6rounds(KeccakP1600times2_align512SIMD128_states *states) { V128 *statesAsLanes = states->A; KeccakP_DeclareVars; copyFromState2rounds(statesAsLanes); KeccakP_2rounds( 18 ); KeccakP_4rounds( 20 ); copyToState(statesAsLanes); } void KeccakP1600times2_PermuteAll_4rounds(KeccakP1600times2_align512SIMD128_states *states) { V128 *statesAsLanes = states->A; KeccakP_DeclareVars; copyFromState(statesAsLanes); KeccakP_4rounds( 20 ); copyToState(statesAsLanes); } size_t KeccakF1600times2_FastLoop_Absorb(KeccakP1600times2_align512SIMD128_states *states, unsigned int laneCount, unsigned int laneOffsetParallel, unsigned int laneOffsetSerial, const unsigned char *data, size_t dataByteLen) { size_t dataMinimumSize = (laneOffsetParallel*1 + laneCount)*8; if (laneCount == 21) { #ifndef KeccakP1600times2_fullUnrolling unsigned int i; #endif const unsigned char *dataStart = data; V128 *statesAsLanes = states->A; const uint64_t *dataAsLanes = (const uint64_t *)data; KeccakP_DeclareVars; V128 index = LOAD4_32(0, 0, 1*laneOffsetParallel, 0*laneOffsetParallel); copyFromState(statesAsLanes); while(dataByteLen >= dataMinimumSize) { #define Add_In( argLane, argIndex ) argLane = XOR(argLane, LOAD_GATHER2_64(index, dataAsLanes+argIndex)) Add_In( _ba, 0 ); Add_In( _be, 1 ); Add_In( _bi, 2 ); Add_In( _bo, 3 ); Add_In( _bu, 4 ); Add_In( _ga, 5 ); Add_In( _ge, 6 ); Add_In( _gi, 7 ); Add_In( _go, 8 ); Add_In( _gu, 9 ); Add_In( _ka, 10 ); Add_In( _ke, 11 ); Add_In( _ki, 12 ); Add_In( _ko, 13 ); Add_In( _ku, 14 ); Add_In( _ma, 15 ); Add_In( _me, 16 ); Add_In( _mi, 17 ); Add_In( _mo, 18 ); Add_In( _mu, 19 ); Add_In( _sa, 20 ); #undef Add_In rounds24; dataAsLanes += laneOffsetSerial; dataByteLen -= laneOffsetSerial*8; } copyToState(statesAsLanes); return (const unsigned char *)dataAsLanes - dataStart; } else { const unsigned char *dataStart = data; while(dataByteLen >= dataMinimumSize) { KeccakP1600times2_AddLanesAll(states, data, laneCount, laneOffsetParallel); KeccakP1600times2_PermuteAll_24rounds(states); data += laneOffsetSerial*8; dataByteLen -= laneOffsetSerial*8; } return data - dataStart; } } size_t KeccakP1600times2_12rounds_FastLoop_Absorb(KeccakP1600times2_align512SIMD128_states *states, unsigned int laneCount, unsigned int laneOffsetParallel, unsigned int laneOffsetSerial, const unsigned char *data, size_t dataByteLen) { size_t dataMinimumSize = (laneOffsetParallel*1 + laneCount)*8; if (laneCount == 21) { #if (KeccakP1600times2_unrolling < 12) unsigned int i; #endif const unsigned char *dataStart = data; V128 *statesAsLanes = states->A; const uint64_t *dataAsLanes = (const uint64_t *)data; KeccakP_DeclareVars; V128 index = LOAD4_32(0, 0, 1*laneOffsetParallel, 0*laneOffsetParallel); copyFromState(statesAsLanes); while(dataByteLen >= dataMinimumSize) { #define Add_In( argLane, argIndex ) argLane = XOR(argLane, LOAD_GATHER2_64(index, dataAsLanes+argIndex)) Add_In( _ba, 0 ); Add_In( _be, 1 ); Add_In( _bi, 2 ); Add_In( _bo, 3 ); Add_In( _bu, 4 ); Add_In( _ga, 5 ); Add_In( _ge, 6 ); Add_In( _gi, 7 ); Add_In( _go, 8 ); Add_In( _gu, 9 ); Add_In( _ka, 10 ); Add_In( _ke, 11 ); Add_In( _ki, 12 ); Add_In( _ko, 13 ); Add_In( _ku, 14 ); Add_In( _ma, 15 ); Add_In( _me, 16 ); Add_In( _mi, 17 ); Add_In( _mo, 18 ); Add_In( _mu, 19 ); Add_In( _sa, 20 ); #undef Add_In rounds12; dataAsLanes += laneOffsetSerial; dataByteLen -= laneOffsetSerial*8; } copyToState(statesAsLanes); return (const unsigned char *)dataAsLanes - dataStart; } else { const unsigned char *dataStart = data; while(dataByteLen >= dataMinimumSize) { KeccakP1600times2_AddLanesAll(states, data, laneCount, laneOffsetParallel); KeccakP1600times2_PermuteAll_12rounds(states); data += laneOffsetSerial*8; dataByteLen -= laneOffsetSerial*8; } return data - dataStart; } }