Crypto++  8.5
Free C++ class library of cryptographic schemes
rijndael_simd.cpp
1 // rijndael_simd.cpp - written and placed in the public domain by
2 // Jeffrey Walton, Uri Blumenthal and Marcel Raad.
3 // AES-NI code originally written by Wei Dai.
4 //
5 // This source file uses intrinsics and built-ins to gain access to
6 // AES-NI, ARMv8a AES and Power8 AES instructions. A separate source
7 // file is needed because additional CXXFLAGS are required to enable
8 // the appropriate instructions sets in some build configurations.
9 //
10 // ARMv8a AES code based on CriticalBlue code from Johannes Schneiders,
11 // Skip Hovsmith and Barry O'Rourke for the mbedTLS project. Stepping
12 // mbedTLS under a debugger was helped for us to determine problems
13 // with our subkey generation and scheduling.
14 //
15 // AltiVec and Power8 code based on http://github.com/noloader/AES-Intrinsics and
16 // http://www.ibm.com/developerworks/library/se-power8-in-core-cryptography/
17 // For Power8 do not remove the casts, even when const-ness is cast away. It causes
18 // failed compiles and a 0.3 to 0.6 cpb drop in performance. The IBM documentation
19 // absolutely sucks. Thanks to Andy Polyakov, Paul R and Trudeaun for answering
20 // questions and filling the gaps in the IBM documentation.
21 //
22 
23 #include "pch.h"
24 #include "config.h"
25 #include "misc.h"
26 
27 #if (CRYPTOPP_AESNI_AVAILABLE)
28 # include "adv_simd.h"
29 # include <emmintrin.h>
30 # include <smmintrin.h>
31 # include <wmmintrin.h>
32 #endif
33 
34 #if (CRYPTOPP_ARM_NEON_HEADER)
35 # include "adv_simd.h"
36 # include <arm_neon.h>
37 #endif
38 
39 #if (CRYPTOPP_ARM_ACLE_HEADER)
40 # include "adv_simd.h"
41 # include <stdint.h>
42 # include <arm_acle.h>
43 #endif
44 
45 #if defined(_M_ARM64)
46 # include "adv_simd.h"
47 #endif
48 
49 #if defined(CRYPTOPP_POWER8_AES_AVAILABLE)
50 # include "adv_simd.h"
51 # include "ppc_simd.h"
52 #endif
53 
54 #ifdef CRYPTOPP_GNU_STYLE_INLINE_ASSEMBLY
55 # include <signal.h>
56 # include <setjmp.h>
57 #endif
58 
59 #ifndef EXCEPTION_EXECUTE_HANDLER
60 # define EXCEPTION_EXECUTE_HANDLER 1
61 #endif
62 
63 // Clang intrinsic casts, http://bugs.llvm.org/show_bug.cgi?id=20670
64 #define M128_CAST(x) ((__m128i *)(void *)(x))
65 #define CONST_M128_CAST(x) ((const __m128i *)(const void *)(x))
66 
67 // Squash MS LNK4221 and libtool warnings
68 extern const char RIJNDAEL_SIMD_FNAME[] = __FILE__;
69 
70 NAMESPACE_BEGIN(CryptoPP)
71 
72 // ************************* Feature Probes ************************* //
73 
74 #ifdef CRYPTOPP_GNU_STYLE_INLINE_ASSEMBLY
75 extern "C" {
76  typedef void (*SigHandler)(int);
77 
78  static jmp_buf s_jmpSIGILL;
79  static void SigIllHandler(int)
80  {
81  longjmp(s_jmpSIGILL, 1);
82  }
83 }
84 #endif // Not CRYPTOPP_MS_STYLE_INLINE_ASSEMBLY
85 
86 #if (CRYPTOPP_BOOL_ARM32 || CRYPTOPP_BOOL_ARMV8)
87 bool CPU_ProbeAES()
88 {
89 #if defined(CRYPTOPP_NO_CPU_FEATURE_PROBES)
90  return false;
91 #elif (CRYPTOPP_ARM_AES_AVAILABLE)
92 # if defined(CRYPTOPP_MS_STYLE_INLINE_ASSEMBLY)
93  volatile bool result = true;
94  __try
95  {
96  // AES encrypt and decrypt
97  uint8x16_t data = vdupq_n_u8(0), key = vdupq_n_u8(0);
98  uint8x16_t r1 = vaeseq_u8(data, key);
99  uint8x16_t r2 = vaesdq_u8(data, key);
100  r1 = vaesmcq_u8(r1);
101  r2 = vaesimcq_u8(r2);
102 
103  result = !!(vgetq_lane_u8(r1,0) | vgetq_lane_u8(r2,7));
104  }
105  __except (EXCEPTION_EXECUTE_HANDLER)
106  {
107  return false;
108  }
109  return result;
110 # else
111 
112  // longjmp and clobber warnings. Volatile is required.
113  // http://github.com/weidai11/cryptopp/issues/24 and http://stackoverflow.com/q/7721854
114  volatile bool result = true;
115 
116  volatile SigHandler oldHandler = signal(SIGILL, SigIllHandler);
117  if (oldHandler == SIG_ERR)
118  return false;
119 
120  volatile sigset_t oldMask;
121  if (sigprocmask(0, NULLPTR, (sigset_t*)&oldMask))
122  {
123  signal(SIGILL, oldHandler);
124  return false;
125  }
126 
127  if (setjmp(s_jmpSIGILL))
128  result = false;
129  else
130  {
131  uint8x16_t data = vdupq_n_u8(0), key = vdupq_n_u8(0);
132  uint8x16_t r1 = vaeseq_u8(data, key);
133  uint8x16_t r2 = vaesdq_u8(data, key);
134  r1 = vaesmcq_u8(r1);
135  r2 = vaesimcq_u8(r2);
136 
137  // Hack... GCC optimizes away the code and returns true
138  result = !!(vgetq_lane_u8(r1,0) | vgetq_lane_u8(r2,7));
139  }
140 
141  sigprocmask(SIG_SETMASK, (sigset_t*)&oldMask, NULLPTR);
142  signal(SIGILL, oldHandler);
143  return result;
144 # endif
145 #else
146  return false;
147 #endif // CRYPTOPP_ARM_AES_AVAILABLE
148 }
149 #endif // ARM32 or ARM64
150 
151 // ***************************** ARMv8 ***************************** //
152 
153 #if (CRYPTOPP_ARM_AES_AVAILABLE)
154 
155 ANONYMOUS_NAMESPACE_BEGIN
156 
157 inline void ARMV8_Enc_Block(uint64x2_t &data, const word32 *subkeys, unsigned int rounds)
158 {
159  CRYPTOPP_ASSERT(subkeys);
160  const byte *keys = reinterpret_cast<const byte*>(subkeys);
161  uint8x16_t block = vreinterpretq_u8_u64(data);
162 
163  // AES single round encryption
164  block = vaeseq_u8(block, vld1q_u8(keys+0*16));
165  // AES mix columns
166  block = vaesmcq_u8(block);
167 
168  for (unsigned int i=1; i<rounds-1; i+=2)
169  {
170  // AES single round encryption
171  block = vaeseq_u8(block, vld1q_u8(keys+i*16));
172  // AES mix columns
173  block = vaesmcq_u8(block);
174  // AES single round encryption
175  block = vaeseq_u8(block, vld1q_u8(keys+(i+1)*16));
176  // AES mix columns
177  block = vaesmcq_u8(block);
178  }
179 
180  // AES single round encryption
181  block = vaeseq_u8(block, vld1q_u8(keys+(rounds-1)*16));
182  // Final Add (bitwise Xor)
183  block = veorq_u8(block, vld1q_u8(keys+rounds*16));
184 
185  data = vreinterpretq_u64_u8(block);
186 }
187 
188 inline void ARMV8_Enc_6_Blocks(uint64x2_t &data0, uint64x2_t &data1,
189  uint64x2_t &data2, uint64x2_t &data3, uint64x2_t &data4, uint64x2_t &data5,
190  const word32 *subkeys, unsigned int rounds)
191 {
192  CRYPTOPP_ASSERT(subkeys);
193  const byte *keys = reinterpret_cast<const byte*>(subkeys);
194 
195  uint8x16_t block0 = vreinterpretq_u8_u64(data0);
196  uint8x16_t block1 = vreinterpretq_u8_u64(data1);
197  uint8x16_t block2 = vreinterpretq_u8_u64(data2);
198  uint8x16_t block3 = vreinterpretq_u8_u64(data3);
199  uint8x16_t block4 = vreinterpretq_u8_u64(data4);
200  uint8x16_t block5 = vreinterpretq_u8_u64(data5);
201 
202  uint8x16_t key;
203  for (unsigned int i=0; i<rounds-1; ++i)
204  {
205  key = vld1q_u8(keys+i*16);
206  // AES single round encryption
207  block0 = vaeseq_u8(block0, key);
208  // AES mix columns
209  block0 = vaesmcq_u8(block0);
210  // AES single round encryption
211  block1 = vaeseq_u8(block1, key);
212  // AES mix columns
213  block1 = vaesmcq_u8(block1);
214  // AES single round encryption
215  block2 = vaeseq_u8(block2, key);
216  // AES mix columns
217  block2 = vaesmcq_u8(block2);
218  // AES single round encryption
219  block3 = vaeseq_u8(block3, key);
220  // AES mix columns
221  block3 = vaesmcq_u8(block3);
222  // AES single round encryption
223  block4 = vaeseq_u8(block4, key);
224  // AES mix columns
225  block4 = vaesmcq_u8(block4);
226  // AES single round encryption
227  block5 = vaeseq_u8(block5, key);
228  // AES mix columns
229  block5 = vaesmcq_u8(block5);
230  }
231 
232  // AES single round encryption
233  key = vld1q_u8(keys+(rounds-1)*16);
234  block0 = vaeseq_u8(block0, key);
235  block1 = vaeseq_u8(block1, key);
236  block2 = vaeseq_u8(block2, key);
237  block3 = vaeseq_u8(block3, key);
238  block4 = vaeseq_u8(block4, key);
239  block5 = vaeseq_u8(block5, key);
240 
241  // Final Add (bitwise Xor)
242  key = vld1q_u8(keys+rounds*16);
243  data0 = vreinterpretq_u64_u8(veorq_u8(block0, key));
244  data1 = vreinterpretq_u64_u8(veorq_u8(block1, key));
245  data2 = vreinterpretq_u64_u8(veorq_u8(block2, key));
246  data3 = vreinterpretq_u64_u8(veorq_u8(block3, key));
247  data4 = vreinterpretq_u64_u8(veorq_u8(block4, key));
248  data5 = vreinterpretq_u64_u8(veorq_u8(block5, key));
249 }
250 
251 inline void ARMV8_Dec_Block(uint64x2_t &data, const word32 *subkeys, unsigned int rounds)
252 {
253  CRYPTOPP_ASSERT(subkeys);
254  const byte *keys = reinterpret_cast<const byte*>(subkeys);
255  uint8x16_t block = vreinterpretq_u8_u64(data);
256 
257  // AES single round decryption
258  block = vaesdq_u8(block, vld1q_u8(keys+0*16));
259  // AES inverse mix columns
260  block = vaesimcq_u8(block);
261 
262  for (unsigned int i=1; i<rounds-1; i+=2)
263  {
264  // AES single round decryption
265  block = vaesdq_u8(block, vld1q_u8(keys+i*16));
266  // AES inverse mix columns
267  block = vaesimcq_u8(block);
268  // AES single round decryption
269  block = vaesdq_u8(block, vld1q_u8(keys+(i+1)*16));
270  // AES inverse mix columns
271  block = vaesimcq_u8(block);
272  }
273 
274  // AES single round decryption
275  block = vaesdq_u8(block, vld1q_u8(keys+(rounds-1)*16));
276  // Final Add (bitwise Xor)
277  block = veorq_u8(block, vld1q_u8(keys+rounds*16));
278 
279  data = vreinterpretq_u64_u8(block);
280 }
281 
282 inline void ARMV8_Dec_6_Blocks(uint64x2_t &data0, uint64x2_t &data1,
283  uint64x2_t &data2, uint64x2_t &data3, uint64x2_t &data4, uint64x2_t &data5,
284  const word32 *subkeys, unsigned int rounds)
285 {
286  CRYPTOPP_ASSERT(subkeys);
287  const byte *keys = reinterpret_cast<const byte*>(subkeys);
288 
289  uint8x16_t block0 = vreinterpretq_u8_u64(data0);
290  uint8x16_t block1 = vreinterpretq_u8_u64(data1);
291  uint8x16_t block2 = vreinterpretq_u8_u64(data2);
292  uint8x16_t block3 = vreinterpretq_u8_u64(data3);
293  uint8x16_t block4 = vreinterpretq_u8_u64(data4);
294  uint8x16_t block5 = vreinterpretq_u8_u64(data5);
295 
296  uint8x16_t key;
297  for (unsigned int i=0; i<rounds-1; ++i)
298  {
299  key = vld1q_u8(keys+i*16);
300  // AES single round decryption
301  block0 = vaesdq_u8(block0, key);
302  // AES inverse mix columns
303  block0 = vaesimcq_u8(block0);
304  // AES single round decryption
305  block1 = vaesdq_u8(block1, key);
306  // AES inverse mix columns
307  block1 = vaesimcq_u8(block1);
308  // AES single round decryption
309  block2 = vaesdq_u8(block2, key);
310  // AES inverse mix columns
311  block2 = vaesimcq_u8(block2);
312  // AES single round decryption
313  block3 = vaesdq_u8(block3, key);
314  // AES inverse mix columns
315  block3 = vaesimcq_u8(block3);
316  // AES single round decryption
317  block4 = vaesdq_u8(block4, key);
318  // AES inverse mix columns
319  block4 = vaesimcq_u8(block4);
320  // AES single round decryption
321  block5 = vaesdq_u8(block5, key);
322  // AES inverse mix columns
323  block5 = vaesimcq_u8(block5);
324  }
325 
326  // AES single round decryption
327  key = vld1q_u8(keys+(rounds-1)*16);
328  block0 = vaesdq_u8(block0, key);
329  block1 = vaesdq_u8(block1, key);
330  block2 = vaesdq_u8(block2, key);
331  block3 = vaesdq_u8(block3, key);
332  block4 = vaesdq_u8(block4, key);
333  block5 = vaesdq_u8(block5, key);
334 
335  // Final Add (bitwise Xor)
336  key = vld1q_u8(keys+rounds*16);
337  data0 = vreinterpretq_u64_u8(veorq_u8(block0, key));
338  data1 = vreinterpretq_u64_u8(veorq_u8(block1, key));
339  data2 = vreinterpretq_u64_u8(veorq_u8(block2, key));
340  data3 = vreinterpretq_u64_u8(veorq_u8(block3, key));
341  data4 = vreinterpretq_u64_u8(veorq_u8(block4, key));
342  data5 = vreinterpretq_u64_u8(veorq_u8(block5, key));
343 }
344 
345 ANONYMOUS_NAMESPACE_END
346 
347 size_t Rijndael_Enc_AdvancedProcessBlocks_ARMV8(const word32 *subKeys, size_t rounds,
348  const byte *inBlocks, const byte *xorBlocks, byte *outBlocks, size_t length, word32 flags)
349 {
350  return AdvancedProcessBlocks128_6x1_NEON(ARMV8_Enc_Block, ARMV8_Enc_6_Blocks,
351  subKeys, rounds, inBlocks, xorBlocks, outBlocks, length, flags);
352 }
353 
354 size_t Rijndael_Dec_AdvancedProcessBlocks_ARMV8(const word32 *subKeys, size_t rounds,
355  const byte *inBlocks, const byte *xorBlocks, byte *outBlocks, size_t length, word32 flags)
356 {
357  return AdvancedProcessBlocks128_6x1_NEON(ARMV8_Dec_Block, ARMV8_Dec_6_Blocks,
358  subKeys, rounds, inBlocks, xorBlocks, outBlocks, length, flags);
359 }
360 
361 #endif // CRYPTOPP_ARM_AES_AVAILABLE
362 
363 // ***************************** AES-NI ***************************** //
364 
365 #if (CRYPTOPP_AESNI_AVAILABLE)
366 
367 ANONYMOUS_NAMESPACE_BEGIN
368 
369 /* for 128-bit blocks, Rijndael never uses more than 10 rcon values */
370 CRYPTOPP_ALIGN_DATA(16)
371 const word32 s_rconLE[] = {
372  0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x1B, 0x36
373 };
374 
375 inline void AESNI_Enc_Block(__m128i &block, MAYBE_CONST word32 *subkeys, unsigned int rounds)
376 {
377  const __m128i* skeys = reinterpret_cast<const __m128i*>(subkeys);
378 
379  block = _mm_xor_si128(block, skeys[0]);
380  for (unsigned int i=1; i<rounds-1; i+=2)
381  {
382  block = _mm_aesenc_si128(block, skeys[i]);
383  block = _mm_aesenc_si128(block, skeys[i+1]);
384  }
385  block = _mm_aesenc_si128(block, skeys[rounds-1]);
386  block = _mm_aesenclast_si128(block, skeys[rounds]);
387 }
388 
389 inline void AESNI_Enc_4_Blocks(__m128i &block0, __m128i &block1, __m128i &block2, __m128i &block3,
390  MAYBE_CONST word32 *subkeys, unsigned int rounds)
391 {
392  const __m128i* skeys = reinterpret_cast<const __m128i*>(subkeys);
393 
394  __m128i rk = skeys[0];
395  block0 = _mm_xor_si128(block0, rk);
396  block1 = _mm_xor_si128(block1, rk);
397  block2 = _mm_xor_si128(block2, rk);
398  block3 = _mm_xor_si128(block3, rk);
399  for (unsigned int i=1; i<rounds; i++)
400  {
401  rk = skeys[i];
402  block0 = _mm_aesenc_si128(block0, rk);
403  block1 = _mm_aesenc_si128(block1, rk);
404  block2 = _mm_aesenc_si128(block2, rk);
405  block3 = _mm_aesenc_si128(block3, rk);
406  }
407  rk = skeys[rounds];
408  block0 = _mm_aesenclast_si128(block0, rk);
409  block1 = _mm_aesenclast_si128(block1, rk);
410  block2 = _mm_aesenclast_si128(block2, rk);
411  block3 = _mm_aesenclast_si128(block3, rk);
412 }
413 
414 inline void AESNI_Dec_Block(__m128i &block, MAYBE_CONST word32 *subkeys, unsigned int rounds)
415 {
416  const __m128i* skeys = reinterpret_cast<const __m128i*>(subkeys);
417 
418  block = _mm_xor_si128(block, skeys[0]);
419  for (unsigned int i=1; i<rounds-1; i+=2)
420  {
421  block = _mm_aesdec_si128(block, skeys[i]);
422  block = _mm_aesdec_si128(block, skeys[i+1]);
423  }
424  block = _mm_aesdec_si128(block, skeys[rounds-1]);
425  block = _mm_aesdeclast_si128(block, skeys[rounds]);
426 }
427 
428 inline void AESNI_Dec_4_Blocks(__m128i &block0, __m128i &block1, __m128i &block2, __m128i &block3,
429  MAYBE_CONST word32 *subkeys, unsigned int rounds)
430 {
431  const __m128i* skeys = reinterpret_cast<const __m128i*>(subkeys);
432 
433  __m128i rk = skeys[0];
434  block0 = _mm_xor_si128(block0, rk);
435  block1 = _mm_xor_si128(block1, rk);
436  block2 = _mm_xor_si128(block2, rk);
437  block3 = _mm_xor_si128(block3, rk);
438  for (unsigned int i=1; i<rounds; i++)
439  {
440  rk = skeys[i];
441  block0 = _mm_aesdec_si128(block0, rk);
442  block1 = _mm_aesdec_si128(block1, rk);
443  block2 = _mm_aesdec_si128(block2, rk);
444  block3 = _mm_aesdec_si128(block3, rk);
445  }
446  rk = skeys[rounds];
447  block0 = _mm_aesdeclast_si128(block0, rk);
448  block1 = _mm_aesdeclast_si128(block1, rk);
449  block2 = _mm_aesdeclast_si128(block2, rk);
450  block3 = _mm_aesdeclast_si128(block3, rk);
451 }
452 
453 ANONYMOUS_NAMESPACE_END
454 
455 void Rijndael_UncheckedSetKey_SSE4_AESNI(const byte *userKey, size_t keyLen, word32 *rk)
456 {
457  const size_t rounds = keyLen / 4 + 6;
458  const word32 *rc = s_rconLE;
459 
460  __m128i temp = _mm_loadu_si128(M128_CAST(userKey+keyLen-16));
461  std::memcpy(rk, userKey, keyLen);
462 
463  // keySize: m_key allocates 4*(rounds+1) word32's.
464  const size_t keySize = 4*(rounds+1);
465  const word32* end = rk + keySize;
466 
467  while (true)
468  {
469  rk[keyLen/4] = rk[0] ^ _mm_extract_epi32(_mm_aeskeygenassist_si128(temp, 0), 3) ^ *(rc++);
470  rk[keyLen/4+1] = rk[1] ^ rk[keyLen/4];
471  rk[keyLen/4+2] = rk[2] ^ rk[keyLen/4+1];
472  rk[keyLen/4+3] = rk[3] ^ rk[keyLen/4+2];
473 
474  if (rk + keyLen/4 + 4 == end)
475  break;
476 
477  if (keyLen == 24)
478  {
479  rk[10] = rk[ 4] ^ rk[ 9];
480  rk[11] = rk[ 5] ^ rk[10];
481  temp = _mm_insert_epi32(temp, rk[11], 3);
482  }
483  else if (keyLen == 32)
484  {
485  temp = _mm_insert_epi32(temp, rk[11], 3);
486  rk[12] = rk[ 4] ^ _mm_extract_epi32(_mm_aeskeygenassist_si128(temp, 0), 2);
487  rk[13] = rk[ 5] ^ rk[12];
488  rk[14] = rk[ 6] ^ rk[13];
489  rk[15] = rk[ 7] ^ rk[14];
490  temp = _mm_insert_epi32(temp, rk[15], 3);
491  }
492  else
493  {
494  temp = _mm_insert_epi32(temp, rk[7], 3);
495  }
496 
497  rk += keyLen/4;
498  }
499 }
500 
501 void Rijndael_UncheckedSetKeyRev_AESNI(word32 *key, unsigned int rounds)
502 {
503  unsigned int i, j;
504  __m128i temp;
505 
506  vec_swap(*M128_CAST(key), *M128_CAST(key+4*rounds));
507 
508  for (i = 4, j = 4*rounds-4; i < j; i += 4, j -= 4)
509  {
510  temp = _mm_aesimc_si128(*M128_CAST(key+i));
511  *M128_CAST(key+i) = _mm_aesimc_si128(*M128_CAST(key+j));
512  *M128_CAST(key+j) = temp;
513  }
514 
515  *M128_CAST(key+i) = _mm_aesimc_si128(*M128_CAST(key+i));
516 }
517 
518 size_t Rijndael_Enc_AdvancedProcessBlocks_AESNI(const word32 *subKeys, size_t rounds,
519  const byte *inBlocks, const byte *xorBlocks, byte *outBlocks, size_t length, word32 flags)
520 {
521  // SunCC workaround
522  MAYBE_CONST word32* sk = MAYBE_UNCONST_CAST(word32*, subKeys);
523  MAYBE_CONST byte* ib = MAYBE_UNCONST_CAST(byte*, inBlocks);
524  MAYBE_CONST byte* xb = MAYBE_UNCONST_CAST(byte*, xorBlocks);
525 
526  return AdvancedProcessBlocks128_4x1_SSE(AESNI_Enc_Block, AESNI_Enc_4_Blocks,
527  sk, rounds, ib, xb, outBlocks, length, flags);
528 }
529 
530 size_t Rijndael_Dec_AdvancedProcessBlocks_AESNI(const word32 *subKeys, size_t rounds,
531  const byte *inBlocks, const byte *xorBlocks, byte *outBlocks, size_t length, word32 flags)
532 {
533  MAYBE_CONST word32* sk = MAYBE_UNCONST_CAST(word32*, subKeys);
534  MAYBE_CONST byte* ib = MAYBE_UNCONST_CAST(byte*, inBlocks);
535  MAYBE_CONST byte* xb = MAYBE_UNCONST_CAST(byte*, xorBlocks);
536 
537  return AdvancedProcessBlocks128_4x1_SSE(AESNI_Dec_Block, AESNI_Dec_4_Blocks,
538  sk, rounds, ib, xb, outBlocks, length, flags);
539 }
540 
541 #endif // CRYPTOPP_AESNI_AVAILABLE
542 
543 // ************************** Power 8 Crypto ************************** //
544 
545 #if (CRYPTOPP_POWER8_AES_AVAILABLE)
546 
547 ANONYMOUS_NAMESPACE_BEGIN
548 
549 /* for 128-bit blocks, Rijndael never uses more than 10 rcon values */
550 CRYPTOPP_ALIGN_DATA(16)
551 static const uint32_t s_rconBE[] = {
552  0x01000000, 0x02000000, 0x04000000, 0x08000000,
553  0x10000000, 0x20000000, 0x40000000, 0x80000000,
554  0x1B000000, 0x36000000
555 };
556 
557 inline void POWER8_Enc_Block(uint32x4_p &block, const word32 *subkeys, unsigned int rounds)
558 {
559  CRYPTOPP_ASSERT(IsAlignedOn(subkeys, 16));
560  const byte *keys = reinterpret_cast<const byte*>(subkeys);
561 
562  uint32x4_p k = VecLoadAligned(keys);
563  block = VecXor(block, k);
564 
565  for (size_t i=1; i<rounds-1; i+=2)
566  {
567  block = VecEncrypt(block, VecLoadAligned( i*16, keys));
568  block = VecEncrypt(block, VecLoadAligned((i+1)*16, keys));
569  }
570 
571  block = VecEncrypt(block, VecLoadAligned((rounds-1)*16, keys));
572  block = VecEncryptLast(block, VecLoadAligned(rounds*16, keys));
573 }
574 
575 inline void POWER8_Enc_6_Blocks(uint32x4_p &block0, uint32x4_p &block1,
576  uint32x4_p &block2, uint32x4_p &block3, uint32x4_p &block4,
577  uint32x4_p &block5, const word32 *subkeys, unsigned int rounds)
578 {
579  CRYPTOPP_ASSERT(IsAlignedOn(subkeys, 16));
580  const byte *keys = reinterpret_cast<const byte*>(subkeys);
581 
582  uint32x4_p k = VecLoadAligned(keys);
583  block0 = VecXor(block0, k);
584  block1 = VecXor(block1, k);
585  block2 = VecXor(block2, k);
586  block3 = VecXor(block3, k);
587  block4 = VecXor(block4, k);
588  block5 = VecXor(block5, k);
589 
590  for (size_t i=1; i<rounds; ++i)
591  {
592  k = VecLoadAligned(i*16, keys);
593  block0 = VecEncrypt(block0, k);
594  block1 = VecEncrypt(block1, k);
595  block2 = VecEncrypt(block2, k);
596  block3 = VecEncrypt(block3, k);
597  block4 = VecEncrypt(block4, k);
598  block5 = VecEncrypt(block5, k);
599  }
600 
601  k = VecLoadAligned(rounds*16, keys);
602  block0 = VecEncryptLast(block0, k);
603  block1 = VecEncryptLast(block1, k);
604  block2 = VecEncryptLast(block2, k);
605  block3 = VecEncryptLast(block3, k);
606  block4 = VecEncryptLast(block4, k);
607  block5 = VecEncryptLast(block5, k);
608 }
609 
610 inline void POWER8_Dec_Block(uint32x4_p &block, const word32 *subkeys, unsigned int rounds)
611 {
612  CRYPTOPP_ASSERT(IsAlignedOn(subkeys, 16));
613  const byte *keys = reinterpret_cast<const byte*>(subkeys);
614 
615  uint32x4_p k = VecLoadAligned(rounds*16, keys);
616  block = VecXor(block, k);
617 
618  for (size_t i=rounds-1; i>1; i-=2)
619  {
620  block = VecDecrypt(block, VecLoadAligned( i*16, keys));
621  block = VecDecrypt(block, VecLoadAligned((i-1)*16, keys));
622  }
623 
624  block = VecDecrypt(block, VecLoadAligned(16, keys));
625  block = VecDecryptLast(block, VecLoadAligned(0, keys));
626 }
627 
628 inline void POWER8_Dec_6_Blocks(uint32x4_p &block0, uint32x4_p &block1,
629  uint32x4_p &block2, uint32x4_p &block3, uint32x4_p &block4,
630  uint32x4_p &block5, const word32 *subkeys, unsigned int rounds)
631 {
632  CRYPTOPP_ASSERT(IsAlignedOn(subkeys, 16));
633  const byte *keys = reinterpret_cast<const byte*>(subkeys);
634 
635  uint32x4_p k = VecLoadAligned(rounds*16, keys);
636  block0 = VecXor(block0, k);
637  block1 = VecXor(block1, k);
638  block2 = VecXor(block2, k);
639  block3 = VecXor(block3, k);
640  block4 = VecXor(block4, k);
641  block5 = VecXor(block5, k);
642 
643  for (size_t i=rounds-1; i>0; --i)
644  {
645  k = VecLoadAligned(i*16, keys);
646  block0 = VecDecrypt(block0, k);
647  block1 = VecDecrypt(block1, k);
648  block2 = VecDecrypt(block2, k);
649  block3 = VecDecrypt(block3, k);
650  block4 = VecDecrypt(block4, k);
651  block5 = VecDecrypt(block5, k);
652  }
653 
654  k = VecLoadAligned(0, keys);
655  block0 = VecDecryptLast(block0, k);
656  block1 = VecDecryptLast(block1, k);
657  block2 = VecDecryptLast(block2, k);
658  block3 = VecDecryptLast(block3, k);
659  block4 = VecDecryptLast(block4, k);
660  block5 = VecDecryptLast(block5, k);
661 }
662 
663 ANONYMOUS_NAMESPACE_END
664 
665 void Rijndael_UncheckedSetKey_POWER8(const byte* userKey, size_t keyLen, word32* rk, const byte* Se)
666 {
667  const size_t rounds = keyLen / 4 + 6;
668  const word32 *rc = s_rconBE;
669  word32 *rkey = rk, temp;
670 
671  GetUserKey(BIG_ENDIAN_ORDER, rkey, keyLen/4, userKey, keyLen);
672 
673  // keySize: m_key allocates 4*(rounds+1) word32's.
674  const size_t keySize = 4*(rounds+1);
675  const word32* end = rkey + keySize;
676 
677  while (true)
678  {
679  temp = rkey[keyLen/4-1];
680  word32 x = (word32(Se[GETBYTE(temp, 2)]) << 24) ^ (word32(Se[GETBYTE(temp, 1)]) << 16) ^
681  (word32(Se[GETBYTE(temp, 0)]) << 8) ^ Se[GETBYTE(temp, 3)];
682  rkey[keyLen/4] = rkey[0] ^ x ^ *(rc++);
683  rkey[keyLen/4+1] = rkey[1] ^ rkey[keyLen/4];
684  rkey[keyLen/4+2] = rkey[2] ^ rkey[keyLen/4+1];
685  rkey[keyLen/4+3] = rkey[3] ^ rkey[keyLen/4+2];
686 
687  if (rkey + keyLen/4 + 4 == end)
688  break;
689 
690  if (keyLen == 24)
691  {
692  rkey[10] = rkey[ 4] ^ rkey[ 9];
693  rkey[11] = rkey[ 5] ^ rkey[10];
694  }
695  else if (keyLen == 32)
696  {
697  temp = rkey[11];
698  rkey[12] = rkey[ 4] ^ (word32(Se[GETBYTE(temp, 3)]) << 24) ^ (word32(Se[GETBYTE(temp, 2)]) << 16) ^ (word32(Se[GETBYTE(temp, 1)]) << 8) ^ Se[GETBYTE(temp, 0)];
699  rkey[13] = rkey[ 5] ^ rkey[12];
700  rkey[14] = rkey[ 6] ^ rkey[13];
701  rkey[15] = rkey[ 7] ^ rkey[14];
702  }
703  rkey += keyLen/4;
704  }
705 
706 #if (CRYPTOPP_LITTLE_ENDIAN)
707  rkey = rk;
708  const uint8x16_p mask = {12,13,14,15, 8,9,10,11, 4,5,6,7, 0,1,2,3};
709 
710  unsigned int i=0;
711  for (i=0; i<rounds; i+=2, rkey+=8)
712  {
713  VecStore(VecPermute(VecLoad(rkey+0), mask), rkey+0);
714  VecStore(VecPermute(VecLoad(rkey+4), mask), rkey+4);
715  }
716 
717  for ( ; i<rounds+1; i++, rkey+=4)
718  VecStore(VecPermute(VecLoad(rkey), mask), rkey);
719 #endif
720 }
721 
722 size_t Rijndael_Enc_AdvancedProcessBlocks128_6x1_ALTIVEC(const word32 *subKeys, size_t rounds,
723  const byte *inBlocks, const byte *xorBlocks, byte *outBlocks, size_t length, word32 flags)
724 {
725  return AdvancedProcessBlocks128_6x1_ALTIVEC(POWER8_Enc_Block, POWER8_Enc_6_Blocks,
726  subKeys, rounds, inBlocks, xorBlocks, outBlocks, length, flags);
727 }
728 
729 size_t Rijndael_Dec_AdvancedProcessBlocks128_6x1_ALTIVEC(const word32 *subKeys, size_t rounds,
730  const byte *inBlocks, const byte *xorBlocks, byte *outBlocks, size_t length, word32 flags)
731 {
732  return AdvancedProcessBlocks128_6x1_ALTIVEC(POWER8_Dec_Block, POWER8_Dec_6_Blocks,
733  subKeys, rounds, inBlocks, xorBlocks, outBlocks, length, flags);
734 }
735 
736 #endif // CRYPTOPP_POWER8_AES_AVAILABLE
737 NAMESPACE_END
Template for AdvancedProcessBlocks and SIMD processing.
#define MAYBE_UNCONST_CAST(T, x)
SunCC workaround.
Definition: adv_simd.h:595
size_t AdvancedProcessBlocks128_6x1_ALTIVEC(F1 func1, F6 func6, const W *subKeys, size_t rounds, const byte *inBlocks, const byte *xorBlocks, byte *outBlocks, size_t length, word32 flags)
AdvancedProcessBlocks for 1 and 6 blocks.
Definition: adv_simd.h:1116
#define M128_CAST(x)
Clang workaround.
Definition: adv_simd.h:609
size_t AdvancedProcessBlocks128_6x1_NEON(F1 func1, F6 func6, const W *subKeys, size_t rounds, const byte *inBlocks, const byte *xorBlocks, byte *outBlocks, size_t length, word32 flags)
AdvancedProcessBlocks for 1 and 6 blocks.
Definition: adv_simd.h:100
#define MAYBE_CONST
SunCC workaround.
Definition: adv_simd.h:590
size_t AdvancedProcessBlocks128_4x1_SSE(F1 func1, F4 func4, const W *subKeys, size_t rounds, const byte *inBlocks, const byte *xorBlocks, byte *outBlocks, size_t length, word32 flags)
AdvancedProcessBlocks for 1 and 4 blocks.
Definition: adv_simd.h:830
Library configuration file.
unsigned int word32
32-bit unsigned datatype
Definition: config_int.h:62
@ BIG_ENDIAN_ORDER
byte order is big-endian
Definition: cryptlib.h:147
Utility functions for the Crypto++ library.
bool IsAlignedOn(const void *ptr, unsigned int alignment)
Determines whether ptr is aligned to a minimum value.
Definition: misc.h:1206
void vec_swap(T &a, T &b)
Swaps two variables which are arrays.
Definition: misc.h:596
Crypto++ library namespace.
Precompiled header file.
Support functions for PowerPC and vector operations.
uint32x4_p VecLoadAligned(const byte src[16])
Loads a vector from an aligned byte array.
Definition: ppc_simd.h:560
__vector unsigned int uint32x4_p
Vector of 32-bit elements.
Definition: ppc_simd.h:202
T1 VecPermute(const T1 vec, const T2 mask)
Permutes a vector.
Definition: ppc_simd.h:1478
__vector unsigned char uint8x16_p
Vector of 8-bit elements.
Definition: ppc_simd.h:192
T1 VecXor(const T1 vec1, const T2 vec2)
XOR two vectors.
Definition: ppc_simd.h:1414
T1 VecEncryptLast(const T1 state, const T2 key)
Final round of AES encryption.
Definition: ppc_simd.h:2634
T1 VecEncrypt(const T1 state, const T2 key)
One round of AES encryption.
Definition: ppc_simd.h:2609
T1 VecDecryptLast(const T1 state, const T2 key)
Final round of AES decryption.
Definition: ppc_simd.h:2684
T1 VecDecrypt(const T1 state, const T2 key)
One round of AES decryption.
Definition: ppc_simd.h:2659
void VecStore(const T data, byte dest[16])
Stores a vector to a byte array.
Definition: ppc_simd.h:895
uint32x4_p VecLoad(const byte src[16])
Loads a vector from a byte array.
Definition: ppc_simd.h:369
#define CRYPTOPP_ASSERT(exp)
Debugging and diagnostic assertion.
Definition: trap.h:68