projects / GitHub / LineageOS / android_kernel_motorola_exynos9610.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
RDMA/nes: ACK MPA Reply frame
[GitHub/LineageOS/android_kernel_motorola_exynos9610.git] / drivers / crypto / vmx / aesp8-ppc.pl
1 #! /usr/bin/env perl
2 # Copyright 2014-2016 The OpenSSL Project Authors. All Rights Reserved.
3 #
4 # Licensed under the OpenSSL license (the "License"). You may not use
5 # this file except in compliance with the License. You can obtain a copy
6 # in the file LICENSE in the source distribution or at
7 # https://www.openssl.org/source/license.html
8
9 #
10 # ====================================================================
11 # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
12 # project. The module is, however, dual licensed under OpenSSL and
13 # CRYPTOGAMS licenses depending on where you obtain it. For further
14 # details see http://www.openssl.org/~appro/cryptogams/.
15 # ====================================================================
16 #
17 # This module implements support for AES instructions as per PowerISA
18 # specification version 2.07, first implemented by POWER8 processor.
19 # The module is endian-agnostic in sense that it supports both big-
20 # and little-endian cases. Data alignment in parallelizable modes is
21 # handled with VSX loads and stores, which implies MSR.VSX flag being
22 # set. It should also be noted that ISA specification doesn't prohibit
23 # alignment exceptions for these instructions on page boundaries.
24 # Initially alignment was handled in pure AltiVec/VMX way [when data
25 # is aligned programmatically, which in turn guarantees exception-
26 # free execution], but it turned to hamper performance when vcipher
27 # instructions are interleaved. It's reckoned that eventual
28 # misalignment penalties at page boundaries are in average lower
29 # than additional overhead in pure AltiVec approach.
30 #
31 # May 2016
32 #
33 # Add XTS subroutine, 9x on little- and 12x improvement on big-endian
34 # systems were measured.
35 #
36 ######################################################################
37 # Current large-block performance in cycles per byte processed with
38 # 128-bit key (less is better).
39 #
40 # CBC en-/decrypt CTR XTS
41 # POWER8[le] 3.96/0.72 0.74 1.1
42 # POWER8[be] 3.75/0.65 0.66 1.0
43
44 $flavour = shift;
45
46 if ($flavour =~ /64/) {
47 $SIZE_T =8;
48 $LRSAVE =2*$SIZE_T;
49 $STU ="stdu";
50 $POP ="ld";
51 $PUSH ="std";
52 $UCMP ="cmpld";
53 $SHL ="sldi";
54 } elsif ($flavour =~ /32/) {
55 $SIZE_T =4;
56 $LRSAVE =$SIZE_T;
57 $STU ="stwu";
58 $POP ="lwz";
59 $PUSH ="stw";
60 $UCMP ="cmplw";
61 $SHL ="slwi";
62 } else { die "nonsense $flavour"; }
63
64 $LITTLE_ENDIAN = ($flavour=~/le$/) ? $SIZE_T : 0;
65
66 $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
67 ( $xlate="${dir}ppc-xlate.pl" and -f $xlate ) or
68 ( $xlate="${dir}../../perlasm/ppc-xlate.pl" and -f $xlate) or
69 die "can't locate ppc-xlate.pl";
70
71 open STDOUT,"| $^X $xlate $flavour ".shift || die "can't call $xlate: $!";
72
73 $FRAME=8*$SIZE_T;
74 $prefix="aes_p8";
75
76 $sp="r1";
77 $vrsave="r12";
78
79 #########################################################################
80 {{{ # Key setup procedures #
81 my ($inp,$bits,$out,$ptr,$cnt,$rounds)=map("r$_",(3..8));
82 my ($zero,$in0,$in1,$key,$rcon,$mask,$tmp)=map("v$_",(0..6));
83 my ($stage,$outperm,$outmask,$outhead,$outtail)=map("v$_",(7..11));
84
85 $code.=<<___;
86 .machine "any"
87
88 .text
89
90 .align 7
91 rcon:
92 .long 0x01000000, 0x01000000, 0x01000000, 0x01000000 ?rev
93 .long 0x1b000000, 0x1b000000, 0x1b000000, 0x1b000000 ?rev
94 .long 0x0d0e0f0c, 0x0d0e0f0c, 0x0d0e0f0c, 0x0d0e0f0c ?rev
95 .long 0,0,0,0 ?asis
96 Lconsts:
97 mflr r0
98 bcl 20,31,\$+4
99 mflr $ptr #vvvvv "distance between . and rcon
100 addi $ptr,$ptr,-0x48
101 mtlr r0
102 blr
103 .long 0
104 .byte 0,12,0x14,0,0,0,0,0
105 .asciz "AES for PowerISA 2.07, CRYPTOGAMS by <appro\@openssl.org>"
106
107 .globl .${prefix}_set_encrypt_key
108 Lset_encrypt_key:
109 mflr r11
110 $PUSH r11,$LRSAVE($sp)
111
112 li $ptr,-1
113 ${UCMP}i $inp,0
114 beq- Lenc_key_abort # if ($inp==0) return -1;
115 ${UCMP}i $out,0
116 beq- Lenc_key_abort # if ($out==0) return -1;
117 li $ptr,-2
118 cmpwi $bits,128
119 blt- Lenc_key_abort
120 cmpwi $bits,256
121 bgt- Lenc_key_abort
122 andi. r0,$bits,0x3f
123 bne- Lenc_key_abort
124
125 lis r0,0xfff0
126 mfspr $vrsave,256
127 mtspr 256,r0
128
129 bl Lconsts
130 mtlr r11
131
132 neg r9,$inp
133 lvx $in0,0,$inp
134 addi $inp,$inp,15 # 15 is not typo
135 lvsr $key,0,r9 # borrow $key
136 li r8,0x20
137 cmpwi $bits,192
138 lvx $in1,0,$inp
139 le?vspltisb $mask,0x0f # borrow $mask
140 lvx $rcon,0,$ptr
141 le?vxor $key,$key,$mask # adjust for byte swap
142 lvx $mask,r8,$ptr
143 addi $ptr,$ptr,0x10
144 vperm $in0,$in0,$in1,$key # align [and byte swap in LE]
145 li $cnt,8
146 vxor $zero,$zero,$zero
147 mtctr $cnt
148
149 ?lvsr $outperm,0,$out
150 vspltisb $outmask,-1
151 lvx $outhead,0,$out
152 ?vperm $outmask,$zero,$outmask,$outperm
153
154 blt Loop128
155 addi $inp,$inp,8
156 beq L192
157 addi $inp,$inp,8
158 b L256
159
160 .align 4
161 Loop128:
162 vperm $key,$in0,$in0,$mask # rotate-n-splat
163 vsldoi $tmp,$zero,$in0,12 # >>32
164 vperm $outtail,$in0,$in0,$outperm # rotate
165 vsel $stage,$outhead,$outtail,$outmask
166 vmr $outhead,$outtail
167 vcipherlast $key,$key,$rcon
168 stvx $stage,0,$out
169 addi $out,$out,16
170
171 vxor $in0,$in0,$tmp
172 vsldoi $tmp,$zero,$tmp,12 # >>32
173 vxor $in0,$in0,$tmp
174 vsldoi $tmp,$zero,$tmp,12 # >>32
175 vxor $in0,$in0,$tmp
176 vadduwm $rcon,$rcon,$rcon
177 vxor $in0,$in0,$key
178 bdnz Loop128
179
180 lvx $rcon,0,$ptr # last two round keys
181
182 vperm $key,$in0,$in0,$mask # rotate-n-splat
183 vsldoi $tmp,$zero,$in0,12 # >>32
184 vperm $outtail,$in0,$in0,$outperm # rotate
185 vsel $stage,$outhead,$outtail,$outmask
186 vmr $outhead,$outtail
187 vcipherlast $key,$key,$rcon
188 stvx $stage,0,$out
189 addi $out,$out,16
190
191 vxor $in0,$in0,$tmp
192 vsldoi $tmp,$zero,$tmp,12 # >>32
193 vxor $in0,$in0,$tmp
194 vsldoi $tmp,$zero,$tmp,12 # >>32
195 vxor $in0,$in0,$tmp
196 vadduwm $rcon,$rcon,$rcon
197 vxor $in0,$in0,$key
198
199 vperm $key,$in0,$in0,$mask # rotate-n-splat
200 vsldoi $tmp,$zero,$in0,12 # >>32
201 vperm $outtail,$in0,$in0,$outperm # rotate
202 vsel $stage,$outhead,$outtail,$outmask
203 vmr $outhead,$outtail
204 vcipherlast $key,$key,$rcon
205 stvx $stage,0,$out
206 addi $out,$out,16
207
208 vxor $in0,$in0,$tmp
209 vsldoi $tmp,$zero,$tmp,12 # >>32
210 vxor $in0,$in0,$tmp
211 vsldoi $tmp,$zero,$tmp,12 # >>32
212 vxor $in0,$in0,$tmp
213 vxor $in0,$in0,$key
214 vperm $outtail,$in0,$in0,$outperm # rotate
215 vsel $stage,$outhead,$outtail,$outmask
216 vmr $outhead,$outtail
217 stvx $stage,0,$out
218
219 addi $inp,$out,15 # 15 is not typo
220 addi $out,$out,0x50
221
222 li $rounds,10
223 b Ldone
224
225 .align 4
226 L192:
227 lvx $tmp,0,$inp
228 li $cnt,4
229 vperm $outtail,$in0,$in0,$outperm # rotate
230 vsel $stage,$outhead,$outtail,$outmask
231 vmr $outhead,$outtail
232 stvx $stage,0,$out
233 addi $out,$out,16
234 vperm $in1,$in1,$tmp,$key # align [and byte swap in LE]
235 vspltisb $key,8 # borrow $key
236 mtctr $cnt
237 vsububm $mask,$mask,$key # adjust the mask
238
239 Loop192:
240 vperm $key,$in1,$in1,$mask # roate-n-splat
241 vsldoi $tmp,$zero,$in0,12 # >>32
242 vcipherlast $key,$key,$rcon
243
244 vxor $in0,$in0,$tmp
245 vsldoi $tmp,$zero,$tmp,12 # >>32
246 vxor $in0,$in0,$tmp
247 vsldoi $tmp,$zero,$tmp,12 # >>32
248 vxor $in0,$in0,$tmp
249
250 vsldoi $stage,$zero,$in1,8
251 vspltw $tmp,$in0,3
252 vxor $tmp,$tmp,$in1
253 vsldoi $in1,$zero,$in1,12 # >>32
254 vadduwm $rcon,$rcon,$rcon
255 vxor $in1,$in1,$tmp
256 vxor $in0,$in0,$key
257 vxor $in1,$in1,$key
258 vsldoi $stage,$stage,$in0,8
259
260 vperm $key,$in1,$in1,$mask # rotate-n-splat
261 vsldoi $tmp,$zero,$in0,12 # >>32
262 vperm $outtail,$stage,$stage,$outperm # rotate
263 vsel $stage,$outhead,$outtail,$outmask
264 vmr $outhead,$outtail
265 vcipherlast $key,$key,$rcon
266 stvx $stage,0,$out
267 addi $out,$out,16
268
269 vsldoi $stage,$in0,$in1,8
270 vxor $in0,$in0,$tmp
271 vsldoi $tmp,$zero,$tmp,12 # >>32
272 vperm $outtail,$stage,$stage,$outperm # rotate
273 vsel $stage,$outhead,$outtail,$outmask
274 vmr $outhead,$outtail
275 vxor $in0,$in0,$tmp
276 vsldoi $tmp,$zero,$tmp,12 # >>32
277 vxor $in0,$in0,$tmp
278 stvx $stage,0,$out
279 addi $out,$out,16
280
281 vspltw $tmp,$in0,3
282 vxor $tmp,$tmp,$in1
283 vsldoi $in1,$zero,$in1,12 # >>32
284 vadduwm $rcon,$rcon,$rcon
285 vxor $in1,$in1,$tmp
286 vxor $in0,$in0,$key
287 vxor $in1,$in1,$key
288 vperm $outtail,$in0,$in0,$outperm # rotate
289 vsel $stage,$outhead,$outtail,$outmask
290 vmr $outhead,$outtail
291 stvx $stage,0,$out
292 addi $inp,$out,15 # 15 is not typo
293 addi $out,$out,16
294 bdnz Loop192
295
296 li $rounds,12
297 addi $out,$out,0x20
298 b Ldone
299
300 .align 4
301 L256:
302 lvx $tmp,0,$inp
303 li $cnt,7
304 li $rounds,14
305 vperm $outtail,$in0,$in0,$outperm # rotate
306 vsel $stage,$outhead,$outtail,$outmask
307 vmr $outhead,$outtail
308 stvx $stage,0,$out
309 addi $out,$out,16
310 vperm $in1,$in1,$tmp,$key # align [and byte swap in LE]
311 mtctr $cnt
312
313 Loop256:
314 vperm $key,$in1,$in1,$mask # rotate-n-splat
315 vsldoi $tmp,$zero,$in0,12 # >>32
316 vperm $outtail,$in1,$in1,$outperm # rotate
317 vsel $stage,$outhead,$outtail,$outmask
318 vmr $outhead,$outtail
319 vcipherlast $key,$key,$rcon
320 stvx $stage,0,$out
321 addi $out,$out,16
322
323 vxor $in0,$in0,$tmp
324 vsldoi $tmp,$zero,$tmp,12 # >>32
325 vxor $in0,$in0,$tmp
326 vsldoi $tmp,$zero,$tmp,12 # >>32
327 vxor $in0,$in0,$tmp
328 vadduwm $rcon,$rcon,$rcon
329 vxor $in0,$in0,$key
330 vperm $outtail,$in0,$in0,$outperm # rotate
331 vsel $stage,$outhead,$outtail,$outmask
332 vmr $outhead,$outtail
333 stvx $stage,0,$out
334 addi $inp,$out,15 # 15 is not typo
335 addi $out,$out,16
336 bdz Ldone
337
338 vspltw $key,$in0,3 # just splat
339 vsldoi $tmp,$zero,$in1,12 # >>32
340 vsbox $key,$key
341
342 vxor $in1,$in1,$tmp
343 vsldoi $tmp,$zero,$tmp,12 # >>32
344 vxor $in1,$in1,$tmp
345 vsldoi $tmp,$zero,$tmp,12 # >>32
346 vxor $in1,$in1,$tmp
347
348 vxor $in1,$in1,$key
349 b Loop256
350
351 .align 4
352 Ldone:
353 lvx $in1,0,$inp # redundant in aligned case
354 vsel $in1,$outhead,$in1,$outmask
355 stvx $in1,0,$inp
356 li $ptr,0
357 mtspr 256,$vrsave
358 stw $rounds,0($out)
359
360 Lenc_key_abort:
361 mr r3,$ptr
362 blr
363 .long 0
364 .byte 0,12,0x14,1,0,0,3,0
365 .long 0
366 .size .${prefix}_set_encrypt_key,.-.${prefix}_set_encrypt_key
367
368 .globl .${prefix}_set_decrypt_key
369 $STU $sp,-$FRAME($sp)
370 mflr r10
371 $PUSH r10,$FRAME+$LRSAVE($sp)
372 bl Lset_encrypt_key
373 mtlr r10
374
375 cmpwi r3,0
376 bne- Ldec_key_abort
377
378 slwi $cnt,$rounds,4
379 subi $inp,$out,240 # first round key
380 srwi $rounds,$rounds,1
381 add $out,$inp,$cnt # last round key
382 mtctr $rounds
383
384 Ldeckey:
385 lwz r0, 0($inp)
386 lwz r6, 4($inp)
387 lwz r7, 8($inp)
388 lwz r8, 12($inp)
389 addi $inp,$inp,16
390 lwz r9, 0($out)
391 lwz r10,4($out)
392 lwz r11,8($out)
393 lwz r12,12($out)
394 stw r0, 0($out)
395 stw r6, 4($out)
396 stw r7, 8($out)
397 stw r8, 12($out)
398 subi $out,$out,16
399 stw r9, -16($inp)
400 stw r10,-12($inp)
401 stw r11,-8($inp)
402 stw r12,-4($inp)
403 bdnz Ldeckey
404
405 xor r3,r3,r3 # return value
406 Ldec_key_abort:
407 addi $sp,$sp,$FRAME
408 blr
409 .long 0
410 .byte 0,12,4,1,0x80,0,3,0
411 .long 0
412 .size .${prefix}_set_decrypt_key,.-.${prefix}_set_decrypt_key
413 ___
414 }}}
415 #########################################################################
416 {{{ # Single block en- and decrypt procedures #
417 sub gen_block () {
418 my $dir = shift;
419 my $n = $dir eq "de" ? "n" : "";
420 my ($inp,$out,$key,$rounds,$idx)=map("r$_",(3..7));
421
422 $code.=<<___;
423 .globl .${prefix}_${dir}crypt
424 lwz $rounds,240($key)
425 lis r0,0xfc00
426 mfspr $vrsave,256
427 li $idx,15 # 15 is not typo
428 mtspr 256,r0
429
430 lvx v0,0,$inp
431 neg r11,$out
432 lvx v1,$idx,$inp
433 lvsl v2,0,$inp # inpperm
434 le?vspltisb v4,0x0f
435 ?lvsl v3,0,r11 # outperm
436 le?vxor v2,v2,v4
437 li $idx,16
438 vperm v0,v0,v1,v2 # align [and byte swap in LE]
439 lvx v1,0,$key
440 ?lvsl v5,0,$key # keyperm
441 srwi $rounds,$rounds,1
442 lvx v2,$idx,$key
443 addi $idx,$idx,16
444 subi $rounds,$rounds,1
445 ?vperm v1,v1,v2,v5 # align round key
446
447 vxor v0,v0,v1
448 lvx v1,$idx,$key
449 addi $idx,$idx,16
450 mtctr $rounds
451
452 Loop_${dir}c:
453 ?vperm v2,v2,v1,v5
454 v${n}cipher v0,v0,v2
455 lvx v2,$idx,$key
456 addi $idx,$idx,16
457 ?vperm v1,v1,v2,v5
458 v${n}cipher v0,v0,v1
459 lvx v1,$idx,$key
460 addi $idx,$idx,16
461 bdnz Loop_${dir}c
462
463 ?vperm v2,v2,v1,v5
464 v${n}cipher v0,v0,v2
465 lvx v2,$idx,$key
466 ?vperm v1,v1,v2,v5
467 v${n}cipherlast v0,v0,v1
468
469 vspltisb v2,-1
470 vxor v1,v1,v1
471 li $idx,15 # 15 is not typo
472 ?vperm v2,v1,v2,v3 # outmask
473 le?vxor v3,v3,v4
474 lvx v1,0,$out # outhead
475 vperm v0,v0,v0,v3 # rotate [and byte swap in LE]
476 vsel v1,v1,v0,v2
477 lvx v4,$idx,$out
478 stvx v1,0,$out
479 vsel v0,v0,v4,v2
480 stvx v0,$idx,$out
481
482 mtspr 256,$vrsave
483 blr
484 .long 0
485 .byte 0,12,0x14,0,0,0,3,0
486 .long 0
487 .size .${prefix}_${dir}crypt,.-.${prefix}_${dir}crypt
488 ___
489 }
490 &gen_block("en");
491 &gen_block("de");
492 }}}
493 #########################################################################
494 {{{ # CBC en- and decrypt procedures #
495 my ($inp,$out,$len,$key,$ivp,$enc,$rounds,$idx)=map("r$_",(3..10));
496 my ($rndkey0,$rndkey1,$inout,$tmp)= map("v$_",(0..3));
497 my ($ivec,$inptail,$inpperm,$outhead,$outperm,$outmask,$keyperm)=
498 map("v$_",(4..10));
499 $code.=<<___;
500 .globl .${prefix}_cbc_encrypt
501 ${UCMP}i $len,16
502 bltlr-
503
504 cmpwi $enc,0 # test direction
505 lis r0,0xffe0
506 mfspr $vrsave,256
507 mtspr 256,r0
508
509 li $idx,15
510 vxor $rndkey0,$rndkey0,$rndkey0
511 le?vspltisb $tmp,0x0f
512
513 lvx $ivec,0,$ivp # load [unaligned] iv
514 lvsl $inpperm,0,$ivp
515 lvx $inptail,$idx,$ivp
516 le?vxor $inpperm,$inpperm,$tmp
517 vperm $ivec,$ivec,$inptail,$inpperm
518
519 neg r11,$inp
520 ?lvsl $keyperm,0,$key # prepare for unaligned key
521 lwz $rounds,240($key)
522
523 lvsr $inpperm,0,r11 # prepare for unaligned load
524 lvx $inptail,0,$inp
525 addi $inp,$inp,15 # 15 is not typo
526 le?vxor $inpperm,$inpperm,$tmp
527
528 ?lvsr $outperm,0,$out # prepare for unaligned store
529 vspltisb $outmask,-1
530 lvx $outhead,0,$out
531 ?vperm $outmask,$rndkey0,$outmask,$outperm
532 le?vxor $outperm,$outperm,$tmp
533
534 srwi $rounds,$rounds,1
535 li $idx,16
536 subi $rounds,$rounds,1
537 beq Lcbc_dec
538
539 Lcbc_enc:
540 vmr $inout,$inptail
541 lvx $inptail,0,$inp
542 addi $inp,$inp,16
543 mtctr $rounds
544 subi $len,$len,16 # len-=16
545
546 lvx $rndkey0,0,$key
547 vperm $inout,$inout,$inptail,$inpperm
548 lvx $rndkey1,$idx,$key
549 addi $idx,$idx,16
550 ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
551 vxor $inout,$inout,$rndkey0
552 lvx $rndkey0,$idx,$key
553 addi $idx,$idx,16
554 vxor $inout,$inout,$ivec
555
556 Loop_cbc_enc:
557 ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm
558 vcipher $inout,$inout,$rndkey1
559 lvx $rndkey1,$idx,$key
560 addi $idx,$idx,16
561 ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
562 vcipher $inout,$inout,$rndkey0
563 lvx $rndkey0,$idx,$key
564 addi $idx,$idx,16
565 bdnz Loop_cbc_enc
566
567 ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm
568 vcipher $inout,$inout,$rndkey1
569 lvx $rndkey1,$idx,$key
570 li $idx,16
571 ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
572 vcipherlast $ivec,$inout,$rndkey0
573 ${UCMP}i $len,16
574
575 vperm $tmp,$ivec,$ivec,$outperm
576 vsel $inout,$outhead,$tmp,$outmask
577 vmr $outhead,$tmp
578 stvx $inout,0,$out
579 addi $out,$out,16
580 bge Lcbc_enc
581
582 b Lcbc_done
583
584 .align 4
585 Lcbc_dec:
586 ${UCMP}i $len,128
587 bge _aesp8_cbc_decrypt8x
588 vmr $tmp,$inptail
589 lvx $inptail,0,$inp
590 addi $inp,$inp,16
591 mtctr $rounds
592 subi $len,$len,16 # len-=16
593
594 lvx $rndkey0,0,$key
595 vperm $tmp,$tmp,$inptail,$inpperm
596 lvx $rndkey1,$idx,$key
597 addi $idx,$idx,16
598 ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
599 vxor $inout,$tmp,$rndkey0
600 lvx $rndkey0,$idx,$key
601 addi $idx,$idx,16
602
603 Loop_cbc_dec:
604 ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm
605 vncipher $inout,$inout,$rndkey1
606 lvx $rndkey1,$idx,$key
607 addi $idx,$idx,16
608 ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
609 vncipher $inout,$inout,$rndkey0
610 lvx $rndkey0,$idx,$key
611 addi $idx,$idx,16
612 bdnz Loop_cbc_dec
613
614 ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm
615 vncipher $inout,$inout,$rndkey1
616 lvx $rndkey1,$idx,$key
617 li $idx,16
618 ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
619 vncipherlast $inout,$inout,$rndkey0
620 ${UCMP}i $len,16
621
622 vxor $inout,$inout,$ivec
623 vmr $ivec,$tmp
624 vperm $tmp,$inout,$inout,$outperm
625 vsel $inout,$outhead,$tmp,$outmask
626 vmr $outhead,$tmp
627 stvx $inout,0,$out
628 addi $out,$out,16
629 bge Lcbc_dec
630
631 Lcbc_done:
632 addi $out,$out,-1
633 lvx $inout,0,$out # redundant in aligned case
634 vsel $inout,$outhead,$inout,$outmask
635 stvx $inout,0,$out
636
637 neg $enc,$ivp # write [unaligned] iv
638 li $idx,15 # 15 is not typo
639 vxor $rndkey0,$rndkey0,$rndkey0
640 vspltisb $outmask,-1
641 le?vspltisb $tmp,0x0f
642 ?lvsl $outperm,0,$enc
643 ?vperm $outmask,$rndkey0,$outmask,$outperm
644 le?vxor $outperm,$outperm,$tmp
645 lvx $outhead,0,$ivp
646 vperm $ivec,$ivec,$ivec,$outperm
647 vsel $inout,$outhead,$ivec,$outmask
648 lvx $inptail,$idx,$ivp
649 stvx $inout,0,$ivp
650 vsel $inout,$ivec,$inptail,$outmask
651 stvx $inout,$idx,$ivp
652
653 mtspr 256,$vrsave
654 blr
655 .long 0
656 .byte 0,12,0x14,0,0,0,6,0
657 .long 0
658 ___
659 #########################################################################
660 {{ # Optimized CBC decrypt procedure #
661 my $key_="r11";
662 my ($x00,$x10,$x20,$x30,$x40,$x50,$x60,$x70)=map("r$_",(0,8,26..31));
663 my ($in0, $in1, $in2, $in3, $in4, $in5, $in6, $in7 )=map("v$_",(0..3,10..13));
664 my ($out0,$out1,$out2,$out3,$out4,$out5,$out6,$out7)=map("v$_",(14..21));
665 my $rndkey0="v23"; # v24-v25 rotating buffer for first found keys
666 # v26-v31 last 6 round keys
667 my ($tmp,$keyperm)=($in3,$in4); # aliases with "caller", redundant assignment
668
669 $code.=<<___;
670 .align 5
671 _aesp8_cbc_decrypt8x:
672 $STU $sp,-`($FRAME+21*16+6*$SIZE_T)`($sp)
673 li r10,`$FRAME+8*16+15`
674 li r11,`$FRAME+8*16+31`
675 stvx v20,r10,$sp # ABI says so
676 addi r10,r10,32
677 stvx v21,r11,$sp
678 addi r11,r11,32
679 stvx v22,r10,$sp
680 addi r10,r10,32
681 stvx v23,r11,$sp
682 addi r11,r11,32
683 stvx v24,r10,$sp
684 addi r10,r10,32
685 stvx v25,r11,$sp
686 addi r11,r11,32
687 stvx v26,r10,$sp
688 addi r10,r10,32
689 stvx v27,r11,$sp
690 addi r11,r11,32
691 stvx v28,r10,$sp
692 addi r10,r10,32
693 stvx v29,r11,$sp
694 addi r11,r11,32
695 stvx v30,r10,$sp
696 stvx v31,r11,$sp
697 li r0,-1
698 stw $vrsave,`$FRAME+21*16-4`($sp) # save vrsave
699 li $x10,0x10
700 $PUSH r26,`$FRAME+21*16+0*$SIZE_T`($sp)
701 li $x20,0x20
702 $PUSH r27,`$FRAME+21*16+1*$SIZE_T`($sp)
703 li $x30,0x30
704 $PUSH r28,`$FRAME+21*16+2*$SIZE_T`($sp)
705 li $x40,0x40
706 $PUSH r29,`$FRAME+21*16+3*$SIZE_T`($sp)
707 li $x50,0x50
708 $PUSH r30,`$FRAME+21*16+4*$SIZE_T`($sp)
709 li $x60,0x60
710 $PUSH r31,`$FRAME+21*16+5*$SIZE_T`($sp)
711 li $x70,0x70
712 mtspr 256,r0
713
714 subi $rounds,$rounds,3 # -4 in total
715 subi $len,$len,128 # bias
716
717 lvx $rndkey0,$x00,$key # load key schedule
718 lvx v30,$x10,$key
719 addi $key,$key,0x20
720 lvx v31,$x00,$key
721 ?vperm $rndkey0,$rndkey0,v30,$keyperm
722 addi $key_,$sp,$FRAME+15
723 mtctr $rounds
724
725 Load_cbc_dec_key:
726 ?vperm v24,v30,v31,$keyperm
727 lvx v30,$x10,$key
728 addi $key,$key,0x20
729 stvx v24,$x00,$key_ # off-load round[1]
730 ?vperm v25,v31,v30,$keyperm
731 lvx v31,$x00,$key
732 stvx v25,$x10,$key_ # off-load round[2]
733 addi $key_,$key_,0x20
734 bdnz Load_cbc_dec_key
735
736 lvx v26,$x10,$key
737 ?vperm v24,v30,v31,$keyperm
738 lvx v27,$x20,$key
739 stvx v24,$x00,$key_ # off-load round[3]
740 ?vperm v25,v31,v26,$keyperm
741 lvx v28,$x30,$key
742 stvx v25,$x10,$key_ # off-load round[4]
743 addi $key_,$sp,$FRAME+15 # rewind $key_
744 ?vperm v26,v26,v27,$keyperm
745 lvx v29,$x40,$key
746 ?vperm v27,v27,v28,$keyperm
747 lvx v30,$x50,$key
748 ?vperm v28,v28,v29,$keyperm
749 lvx v31,$x60,$key
750 ?vperm v29,v29,v30,$keyperm
751 lvx $out0,$x70,$key # borrow $out0
752 ?vperm v30,v30,v31,$keyperm
753 lvx v24,$x00,$key_ # pre-load round[1]
754 ?vperm v31,v31,$out0,$keyperm
755 lvx v25,$x10,$key_ # pre-load round[2]
756
757 #lvx $inptail,0,$inp # "caller" already did this
758 #addi $inp,$inp,15 # 15 is not typo
759 subi $inp,$inp,15 # undo "caller"
760
761 le?li $idx,8
762 lvx_u $in0,$x00,$inp # load first 8 "words"
763 le?lvsl $inpperm,0,$idx
764 le?vspltisb $tmp,0x0f
765 lvx_u $in1,$x10,$inp
766 le?vxor $inpperm,$inpperm,$tmp # transform for lvx_u/stvx_u
767 lvx_u $in2,$x20,$inp
768 le?vperm $in0,$in0,$in0,$inpperm
769 lvx_u $in3,$x30,$inp
770 le?vperm $in1,$in1,$in1,$inpperm
771 lvx_u $in4,$x40,$inp
772 le?vperm $in2,$in2,$in2,$inpperm
773 vxor $out0,$in0,$rndkey0
774 lvx_u $in5,$x50,$inp
775 le?vperm $in3,$in3,$in3,$inpperm
776 vxor $out1,$in1,$rndkey0
777 lvx_u $in6,$x60,$inp
778 le?vperm $in4,$in4,$in4,$inpperm
779 vxor $out2,$in2,$rndkey0
780 lvx_u $in7,$x70,$inp
781 addi $inp,$inp,0x80
782 le?vperm $in5,$in5,$in5,$inpperm
783 vxor $out3,$in3,$rndkey0
784 le?vperm $in6,$in6,$in6,$inpperm
785 vxor $out4,$in4,$rndkey0
786 le?vperm $in7,$in7,$in7,$inpperm
787 vxor $out5,$in5,$rndkey0
788 vxor $out6,$in6,$rndkey0
789 vxor $out7,$in7,$rndkey0
790
791 mtctr $rounds
792 b Loop_cbc_dec8x
793 .align 5
794 Loop_cbc_dec8x:
795 vncipher $out0,$out0,v24
796 vncipher $out1,$out1,v24
797 vncipher $out2,$out2,v24
798 vncipher $out3,$out3,v24
799 vncipher $out4,$out4,v24
800 vncipher $out5,$out5,v24
801 vncipher $out6,$out6,v24
802 vncipher $out7,$out7,v24
803 lvx v24,$x20,$key_ # round[3]
804 addi $key_,$key_,0x20
805
806 vncipher $out0,$out0,v25
807 vncipher $out1,$out1,v25
808 vncipher $out2,$out2,v25
809 vncipher $out3,$out3,v25
810 vncipher $out4,$out4,v25
811 vncipher $out5,$out5,v25
812 vncipher $out6,$out6,v25
813 vncipher $out7,$out7,v25
814 lvx v25,$x10,$key_ # round[4]
815 bdnz Loop_cbc_dec8x
816
817 subic $len,$len,128 # $len-=128
818 vncipher $out0,$out0,v24
819 vncipher $out1,$out1,v24
820 vncipher $out2,$out2,v24
821 vncipher $out3,$out3,v24
822 vncipher $out4,$out4,v24
823 vncipher $out5,$out5,v24
824 vncipher $out6,$out6,v24
825 vncipher $out7,$out7,v24
826
827 subfe. r0,r0,r0 # borrow?-1:0
828 vncipher $out0,$out0,v25
829 vncipher $out1,$out1,v25
830 vncipher $out2,$out2,v25
831 vncipher $out3,$out3,v25
832 vncipher $out4,$out4,v25
833 vncipher $out5,$out5,v25
834 vncipher $out6,$out6,v25
835 vncipher $out7,$out7,v25
836
837 and r0,r0,$len
838 vncipher $out0,$out0,v26
839 vncipher $out1,$out1,v26
840 vncipher $out2,$out2,v26
841 vncipher $out3,$out3,v26
842 vncipher $out4,$out4,v26
843 vncipher $out5,$out5,v26
844 vncipher $out6,$out6,v26
845 vncipher $out7,$out7,v26
846
847 add $inp,$inp,r0 # $inp is adjusted in such
848 # way that at exit from the
849 # loop inX-in7 are loaded
850 # with last "words"
851 vncipher $out0,$out0,v27
852 vncipher $out1,$out1,v27
853 vncipher $out2,$out2,v27
854 vncipher $out3,$out3,v27
855 vncipher $out4,$out4,v27
856 vncipher $out5,$out5,v27
857 vncipher $out6,$out6,v27
858 vncipher $out7,$out7,v27
859
860 addi $key_,$sp,$FRAME+15 # rewind $key_
861 vncipher $out0,$out0,v28
862 vncipher $out1,$out1,v28
863 vncipher $out2,$out2,v28
864 vncipher $out3,$out3,v28
865 vncipher $out4,$out4,v28
866 vncipher $out5,$out5,v28
867 vncipher $out6,$out6,v28
868 vncipher $out7,$out7,v28
869 lvx v24,$x00,$key_ # re-pre-load round[1]
870
871 vncipher $out0,$out0,v29
872 vncipher $out1,$out1,v29
873 vncipher $out2,$out2,v29
874 vncipher $out3,$out3,v29
875 vncipher $out4,$out4,v29
876 vncipher $out5,$out5,v29
877 vncipher $out6,$out6,v29
878 vncipher $out7,$out7,v29
879 lvx v25,$x10,$key_ # re-pre-load round[2]
880
881 vncipher $out0,$out0,v30
882 vxor $ivec,$ivec,v31 # xor with last round key
883 vncipher $out1,$out1,v30
884 vxor $in0,$in0,v31
885 vncipher $out2,$out2,v30
886 vxor $in1,$in1,v31
887 vncipher $out3,$out3,v30
888 vxor $in2,$in2,v31
889 vncipher $out4,$out4,v30
890 vxor $in3,$in3,v31
891 vncipher $out5,$out5,v30
892 vxor $in4,$in4,v31
893 vncipher $out6,$out6,v30
894 vxor $in5,$in5,v31
895 vncipher $out7,$out7,v30
896 vxor $in6,$in6,v31
897
898 vncipherlast $out0,$out0,$ivec
899 vncipherlast $out1,$out1,$in0
900 lvx_u $in0,$x00,$inp # load next input block
901 vncipherlast $out2,$out2,$in1
902 lvx_u $in1,$x10,$inp
903 vncipherlast $out3,$out3,$in2
904 le?vperm $in0,$in0,$in0,$inpperm
905 lvx_u $in2,$x20,$inp
906 vncipherlast $out4,$out4,$in3
907 le?vperm $in1,$in1,$in1,$inpperm
908 lvx_u $in3,$x30,$inp
909 vncipherlast $out5,$out5,$in4
910 le?vperm $in2,$in2,$in2,$inpperm
911 lvx_u $in4,$x40,$inp
912 vncipherlast $out6,$out6,$in5
913 le?vperm $in3,$in3,$in3,$inpperm
914 lvx_u $in5,$x50,$inp
915 vncipherlast $out7,$out7,$in6
916 le?vperm $in4,$in4,$in4,$inpperm
917 lvx_u $in6,$x60,$inp
918 vmr $ivec,$in7
919 le?vperm $in5,$in5,$in5,$inpperm
920 lvx_u $in7,$x70,$inp
921 addi $inp,$inp,0x80
922
923 le?vperm $out0,$out0,$out0,$inpperm
924 le?vperm $out1,$out1,$out1,$inpperm
925 stvx_u $out0,$x00,$out
926 le?vperm $in6,$in6,$in6,$inpperm
927 vxor $out0,$in0,$rndkey0
928 le?vperm $out2,$out2,$out2,$inpperm
929 stvx_u $out1,$x10,$out
930 le?vperm $in7,$in7,$in7,$inpperm
931 vxor $out1,$in1,$rndkey0
932 le?vperm $out3,$out3,$out3,$inpperm
933 stvx_u $out2,$x20,$out
934 vxor $out2,$in2,$rndkey0
935 le?vperm $out4,$out4,$out4,$inpperm
936 stvx_u $out3,$x30,$out
937 vxor $out3,$in3,$rndkey0
938 le?vperm $out5,$out5,$out5,$inpperm
939 stvx_u $out4,$x40,$out
940 vxor $out4,$in4,$rndkey0
941 le?vperm $out6,$out6,$out6,$inpperm
942 stvx_u $out5,$x50,$out
943 vxor $out5,$in5,$rndkey0
944 le?vperm $out7,$out7,$out7,$inpperm
945 stvx_u $out6,$x60,$out
946 vxor $out6,$in6,$rndkey0
947 stvx_u $out7,$x70,$out
948 addi $out,$out,0x80
949 vxor $out7,$in7,$rndkey0
950
951 mtctr $rounds
952 beq Loop_cbc_dec8x # did $len-=128 borrow?
953
954 addic. $len,$len,128
955 beq Lcbc_dec8x_done
956 nop
957 nop
958
959 Loop_cbc_dec8x_tail: # up to 7 "words" tail...
960 vncipher $out1,$out1,v24
961 vncipher $out2,$out2,v24
962 vncipher $out3,$out3,v24
963 vncipher $out4,$out4,v24
964 vncipher $out5,$out5,v24
965 vncipher $out6,$out6,v24
966 vncipher $out7,$out7,v24
967 lvx v24,$x20,$key_ # round[3]
968 addi $key_,$key_,0x20
969
970 vncipher $out1,$out1,v25
971 vncipher $out2,$out2,v25
972 vncipher $out3,$out3,v25
973 vncipher $out4,$out4,v25
974 vncipher $out5,$out5,v25
975 vncipher $out6,$out6,v25
976 vncipher $out7,$out7,v25
977 lvx v25,$x10,$key_ # round[4]
978 bdnz Loop_cbc_dec8x_tail
979
980 vncipher $out1,$out1,v24
981 vncipher $out2,$out2,v24
982 vncipher $out3,$out3,v24
983 vncipher $out4,$out4,v24
984 vncipher $out5,$out5,v24
985 vncipher $out6,$out6,v24
986 vncipher $out7,$out7,v24
987
988 vncipher $out1,$out1,v25
989 vncipher $out2,$out2,v25
990 vncipher $out3,$out3,v25
991 vncipher $out4,$out4,v25
992 vncipher $out5,$out5,v25
993 vncipher $out6,$out6,v25
994 vncipher $out7,$out7,v25
995
996 vncipher $out1,$out1,v26
997 vncipher $out2,$out2,v26
998 vncipher $out3,$out3,v26
999 vncipher $out4,$out4,v26
1000 vncipher $out5,$out5,v26
1001 vncipher $out6,$out6,v26
1002 vncipher $out7,$out7,v26
1003
1004 vncipher $out1,$out1,v27
1005 vncipher $out2,$out2,v27
1006 vncipher $out3,$out3,v27
1007 vncipher $out4,$out4,v27
1008 vncipher $out5,$out5,v27
1009 vncipher $out6,$out6,v27
1010 vncipher $out7,$out7,v27
1011
1012 vncipher $out1,$out1,v28
1013 vncipher $out2,$out2,v28
1014 vncipher $out3,$out3,v28
1015 vncipher $out4,$out4,v28
1016 vncipher $out5,$out5,v28
1017 vncipher $out6,$out6,v28
1018 vncipher $out7,$out7,v28
1019
1020 vncipher $out1,$out1,v29
1021 vncipher $out2,$out2,v29
1022 vncipher $out3,$out3,v29
1023 vncipher $out4,$out4,v29
1024 vncipher $out5,$out5,v29
1025 vncipher $out6,$out6,v29
1026 vncipher $out7,$out7,v29
1027
1028 vncipher $out1,$out1,v30
1029 vxor $ivec,$ivec,v31 # last round key
1030 vncipher $out2,$out2,v30
1031 vxor $in1,$in1,v31
1032 vncipher $out3,$out3,v30
1033 vxor $in2,$in2,v31
1034 vncipher $out4,$out4,v30
1035 vxor $in3,$in3,v31
1036 vncipher $out5,$out5,v30
1037 vxor $in4,$in4,v31
1038 vncipher $out6,$out6,v30
1039 vxor $in5,$in5,v31
1040 vncipher $out7,$out7,v30
1041 vxor $in6,$in6,v31
1042
1043 cmplwi $len,32 # switch($len)
1044 blt Lcbc_dec8x_one
1045 nop
1046 beq Lcbc_dec8x_two
1047 cmplwi $len,64
1048 blt Lcbc_dec8x_three
1049 nop
1050 beq Lcbc_dec8x_four
1051 cmplwi $len,96
1052 blt Lcbc_dec8x_five
1053 nop
1054 beq Lcbc_dec8x_six
1055
1056 Lcbc_dec8x_seven:
1057 vncipherlast $out1,$out1,$ivec
1058 vncipherlast $out2,$out2,$in1
1059 vncipherlast $out3,$out3,$in2
1060 vncipherlast $out4,$out4,$in3
1061 vncipherlast $out5,$out5,$in4
1062 vncipherlast $out6,$out6,$in5
1063 vncipherlast $out7,$out7,$in6
1064 vmr $ivec,$in7
1065
1066 le?vperm $out1,$out1,$out1,$inpperm
1067 le?vperm $out2,$out2,$out2,$inpperm
1068 stvx_u $out1,$x00,$out
1069 le?vperm $out3,$out3,$out3,$inpperm
1070 stvx_u $out2,$x10,$out
1071 le?vperm $out4,$out4,$out4,$inpperm
1072 stvx_u $out3,$x20,$out
1073 le?vperm $out5,$out5,$out5,$inpperm
1074 stvx_u $out4,$x30,$out
1075 le?vperm $out6,$out6,$out6,$inpperm
1076 stvx_u $out5,$x40,$out
1077 le?vperm $out7,$out7,$out7,$inpperm
1078 stvx_u $out6,$x50,$out
1079 stvx_u $out7,$x60,$out
1080 addi $out,$out,0x70
1081 b Lcbc_dec8x_done
1082
1083 .align 5
1084 Lcbc_dec8x_six:
1085 vncipherlast $out2,$out2,$ivec
1086 vncipherlast $out3,$out3,$in2
1087 vncipherlast $out4,$out4,$in3
1088 vncipherlast $out5,$out5,$in4
1089 vncipherlast $out6,$out6,$in5
1090 vncipherlast $out7,$out7,$in6
1091 vmr $ivec,$in7
1092
1093 le?vperm $out2,$out2,$out2,$inpperm
1094 le?vperm $out3,$out3,$out3,$inpperm
1095 stvx_u $out2,$x00,$out
1096 le?vperm $out4,$out4,$out4,$inpperm
1097 stvx_u $out3,$x10,$out
1098 le?vperm $out5,$out5,$out5,$inpperm
1099 stvx_u $out4,$x20,$out
1100 le?vperm $out6,$out6,$out6,$inpperm
1101 stvx_u $out5,$x30,$out
1102 le?vperm $out7,$out7,$out7,$inpperm
1103 stvx_u $out6,$x40,$out
1104 stvx_u $out7,$x50,$out
1105 addi $out,$out,0x60
1106 b Lcbc_dec8x_done
1107
1108 .align 5
1109 Lcbc_dec8x_five:
1110 vncipherlast $out3,$out3,$ivec
1111 vncipherlast $out4,$out4,$in3
1112 vncipherlast $out5,$out5,$in4
1113 vncipherlast $out6,$out6,$in5
1114 vncipherlast $out7,$out7,$in6
1115 vmr $ivec,$in7
1116
1117 le?vperm $out3,$out3,$out3,$inpperm
1118 le?vperm $out4,$out4,$out4,$inpperm
1119 stvx_u $out3,$x00,$out
1120 le?vperm $out5,$out5,$out5,$inpperm
1121 stvx_u $out4,$x10,$out
1122 le?vperm $out6,$out6,$out6,$inpperm
1123 stvx_u $out5,$x20,$out
1124 le?vperm $out7,$out7,$out7,$inpperm
1125 stvx_u $out6,$x30,$out
1126 stvx_u $out7,$x40,$out
1127 addi $out,$out,0x50
1128 b Lcbc_dec8x_done
1129
1130 .align 5
1131 Lcbc_dec8x_four:
1132 vncipherlast $out4,$out4,$ivec
1133 vncipherlast $out5,$out5,$in4
1134 vncipherlast $out6,$out6,$in5
1135 vncipherlast $out7,$out7,$in6
1136 vmr $ivec,$in7
1137
1138 le?vperm $out4,$out4,$out4,$inpperm
1139 le?vperm $out5,$out5,$out5,$inpperm
1140 stvx_u $out4,$x00,$out
1141 le?vperm $out6,$out6,$out6,$inpperm
1142 stvx_u $out5,$x10,$out
1143 le?vperm $out7,$out7,$out7,$inpperm
1144 stvx_u $out6,$x20,$out
1145 stvx_u $out7,$x30,$out
1146 addi $out,$out,0x40
1147 b Lcbc_dec8x_done
1148
1149 .align 5
1150 Lcbc_dec8x_three:
1151 vncipherlast $out5,$out5,$ivec
1152 vncipherlast $out6,$out6,$in5
1153 vncipherlast $out7,$out7,$in6
1154 vmr $ivec,$in7
1155
1156 le?vperm $out5,$out5,$out5,$inpperm
1157 le?vperm $out6,$out6,$out6,$inpperm
1158 stvx_u $out5,$x00,$out
1159 le?vperm $out7,$out7,$out7,$inpperm
1160 stvx_u $out6,$x10,$out
1161 stvx_u $out7,$x20,$out
1162 addi $out,$out,0x30
1163 b Lcbc_dec8x_done
1164
1165 .align 5
1166 Lcbc_dec8x_two:
1167 vncipherlast $out6,$out6,$ivec
1168 vncipherlast $out7,$out7,$in6
1169 vmr $ivec,$in7
1170
1171 le?vperm $out6,$out6,$out6,$inpperm
1172 le?vperm $out7,$out7,$out7,$inpperm
1173 stvx_u $out6,$x00,$out
1174 stvx_u $out7,$x10,$out
1175 addi $out,$out,0x20
1176 b Lcbc_dec8x_done
1177
1178 .align 5
1179 Lcbc_dec8x_one:
1180 vncipherlast $out7,$out7,$ivec
1181 vmr $ivec,$in7
1182
1183 le?vperm $out7,$out7,$out7,$inpperm
1184 stvx_u $out7,0,$out
1185 addi $out,$out,0x10
1186
1187 Lcbc_dec8x_done:
1188 le?vperm $ivec,$ivec,$ivec,$inpperm
1189 stvx_u $ivec,0,$ivp # write [unaligned] iv
1190
1191 li r10,`$FRAME+15`
1192 li r11,`$FRAME+31`
1193 stvx $inpperm,r10,$sp # wipe copies of round keys
1194 addi r10,r10,32
1195 stvx $inpperm,r11,$sp
1196 addi r11,r11,32
1197 stvx $inpperm,r10,$sp
1198 addi r10,r10,32
1199 stvx $inpperm,r11,$sp
1200 addi r11,r11,32
1201 stvx $inpperm,r10,$sp
1202 addi r10,r10,32
1203 stvx $inpperm,r11,$sp
1204 addi r11,r11,32
1205 stvx $inpperm,r10,$sp
1206 addi r10,r10,32
1207 stvx $inpperm,r11,$sp
1208 addi r11,r11,32
1209
1210 mtspr 256,$vrsave
1211 lvx v20,r10,$sp # ABI says so
1212 addi r10,r10,32
1213 lvx v21,r11,$sp
1214 addi r11,r11,32
1215 lvx v22,r10,$sp
1216 addi r10,r10,32
1217 lvx v23,r11,$sp
1218 addi r11,r11,32
1219 lvx v24,r10,$sp
1220 addi r10,r10,32
1221 lvx v25,r11,$sp
1222 addi r11,r11,32
1223 lvx v26,r10,$sp
1224 addi r10,r10,32
1225 lvx v27,r11,$sp
1226 addi r11,r11,32
1227 lvx v28,r10,$sp
1228 addi r10,r10,32
1229 lvx v29,r11,$sp
1230 addi r11,r11,32
1231 lvx v30,r10,$sp
1232 lvx v31,r11,$sp
1233 $POP r26,`$FRAME+21*16+0*$SIZE_T`($sp)
1234 $POP r27,`$FRAME+21*16+1*$SIZE_T`($sp)
1235 $POP r28,`$FRAME+21*16+2*$SIZE_T`($sp)
1236 $POP r29,`$FRAME+21*16+3*$SIZE_T`($sp)
1237 $POP r30,`$FRAME+21*16+4*$SIZE_T`($sp)
1238 $POP r31,`$FRAME+21*16+5*$SIZE_T`($sp)
1239 addi $sp,$sp,`$FRAME+21*16+6*$SIZE_T`
1240 blr
1241 .long 0
1242 .byte 0,12,0x14,0,0x80,6,6,0
1243 .long 0
1244 .size .${prefix}_cbc_encrypt,.-.${prefix}_cbc_encrypt
1245 ___
1246 }} }}}
1247
1248 #########################################################################
1249 {{{ # CTR procedure[s] #
1250 my ($inp,$out,$len,$key,$ivp,$x10,$rounds,$idx)=map("r$_",(3..10));
1251 my ($rndkey0,$rndkey1,$inout,$tmp)= map("v$_",(0..3));
1252 my ($ivec,$inptail,$inpperm,$outhead,$outperm,$outmask,$keyperm,$one)=
1253 map("v$_",(4..11));
1254 my $dat=$tmp;
1255
1256 $code.=<<___;
1257 .globl .${prefix}_ctr32_encrypt_blocks
1258 ${UCMP}i $len,1
1259 bltlr-
1260
1261 lis r0,0xfff0
1262 mfspr $vrsave,256
1263 mtspr 256,r0
1264
1265 li $idx,15
1266 vxor $rndkey0,$rndkey0,$rndkey0
1267 le?vspltisb $tmp,0x0f
1268
1269 lvx $ivec,0,$ivp # load [unaligned] iv
1270 lvsl $inpperm,0,$ivp
1271 lvx $inptail,$idx,$ivp
1272 vspltisb $one,1
1273 le?vxor $inpperm,$inpperm,$tmp
1274 vperm $ivec,$ivec,$inptail,$inpperm
1275 vsldoi $one,$rndkey0,$one,1
1276
1277 neg r11,$inp
1278 ?lvsl $keyperm,0,$key # prepare for unaligned key
1279 lwz $rounds,240($key)
1280
1281 lvsr $inpperm,0,r11 # prepare for unaligned load
1282 lvx $inptail,0,$inp
1283 addi $inp,$inp,15 # 15 is not typo
1284 le?vxor $inpperm,$inpperm,$tmp
1285
1286 srwi $rounds,$rounds,1
1287 li $idx,16
1288 subi $rounds,$rounds,1
1289
1290 ${UCMP}i $len,8
1291 bge _aesp8_ctr32_encrypt8x
1292
1293 ?lvsr $outperm,0,$out # prepare for unaligned store
1294 vspltisb $outmask,-1
1295 lvx $outhead,0,$out
1296 ?vperm $outmask,$rndkey0,$outmask,$outperm
1297 le?vxor $outperm,$outperm,$tmp
1298
1299 lvx $rndkey0,0,$key
1300 mtctr $rounds
1301 lvx $rndkey1,$idx,$key
1302 addi $idx,$idx,16
1303 ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
1304 vxor $inout,$ivec,$rndkey0
1305 lvx $rndkey0,$idx,$key
1306 addi $idx,$idx,16
1307 b Loop_ctr32_enc
1308
1309 .align 5
1310 Loop_ctr32_enc:
1311 ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm
1312 vcipher $inout,$inout,$rndkey1
1313 lvx $rndkey1,$idx,$key
1314 addi $idx,$idx,16
1315 ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
1316 vcipher $inout,$inout,$rndkey0
1317 lvx $rndkey0,$idx,$key
1318 addi $idx,$idx,16
1319 bdnz Loop_ctr32_enc
1320
1321 vadduwm $ivec,$ivec,$one
1322 vmr $dat,$inptail
1323 lvx $inptail,0,$inp
1324 addi $inp,$inp,16
1325 subic. $len,$len,1 # blocks--
1326
1327 ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm
1328 vcipher $inout,$inout,$rndkey1
1329 lvx $rndkey1,$idx,$key
1330 vperm $dat,$dat,$inptail,$inpperm
1331 li $idx,16
1332 ?vperm $rndkey1,$rndkey0,$rndkey1,$keyperm
1333 lvx $rndkey0,0,$key
1334 vxor $dat,$dat,$rndkey1 # last round key
1335 vcipherlast $inout,$inout,$dat
1336
1337 lvx $rndkey1,$idx,$key
1338 addi $idx,$idx,16
1339 vperm $inout,$inout,$inout,$outperm
1340 vsel $dat,$outhead,$inout,$outmask
1341 mtctr $rounds
1342 ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
1343 vmr $outhead,$inout
1344 vxor $inout,$ivec,$rndkey0
1345 lvx $rndkey0,$idx,$key
1346 addi $idx,$idx,16
1347 stvx $dat,0,$out
1348 addi $out,$out,16
1349 bne Loop_ctr32_enc
1350
1351 addi $out,$out,-1
1352 lvx $inout,0,$out # redundant in aligned case
1353 vsel $inout,$outhead,$inout,$outmask
1354 stvx $inout,0,$out
1355
1356 mtspr 256,$vrsave
1357 blr
1358 .long 0
1359 .byte 0,12,0x14,0,0,0,6,0
1360 .long 0
1361 ___
1362 #########################################################################
1363 {{ # Optimized CTR procedure #
1364 my $key_="r11";
1365 my ($x00,$x10,$x20,$x30,$x40,$x50,$x60,$x70)=map("r$_",(0,8,26..31));
1366 my ($in0, $in1, $in2, $in3, $in4, $in5, $in6, $in7 )=map("v$_",(0..3,10,12..14));
1367 my ($out0,$out1,$out2,$out3,$out4,$out5,$out6,$out7)=map("v$_",(15..22));
1368 my $rndkey0="v23"; # v24-v25 rotating buffer for first found keys
1369 # v26-v31 last 6 round keys
1370 my ($tmp,$keyperm)=($in3,$in4); # aliases with "caller", redundant assignment
1371 my ($two,$three,$four)=($outhead,$outperm,$outmask);
1372
1373 $code.=<<___;
1374 .align 5
1375 _aesp8_ctr32_encrypt8x:
1376 $STU $sp,-`($FRAME+21*16+6*$SIZE_T)`($sp)
1377 li r10,`$FRAME+8*16+15`
1378 li r11,`$FRAME+8*16+31`
1379 stvx v20,r10,$sp # ABI says so
1380 addi r10,r10,32
1381 stvx v21,r11,$sp
1382 addi r11,r11,32
1383 stvx v22,r10,$sp
1384 addi r10,r10,32
1385 stvx v23,r11,$sp
1386 addi r11,r11,32
1387 stvx v24,r10,$sp
1388 addi r10,r10,32
1389 stvx v25,r11,$sp
1390 addi r11,r11,32
1391 stvx v26,r10,$sp
1392 addi r10,r10,32
1393 stvx v27,r11,$sp
1394 addi r11,r11,32
1395 stvx v28,r10,$sp
1396 addi r10,r10,32
1397 stvx v29,r11,$sp
1398 addi r11,r11,32
1399 stvx v30,r10,$sp
1400 stvx v31,r11,$sp
1401 li r0,-1
1402 stw $vrsave,`$FRAME+21*16-4`($sp) # save vrsave
1403 li $x10,0x10
1404 $PUSH r26,`$FRAME+21*16+0*$SIZE_T`($sp)
1405 li $x20,0x20
1406 $PUSH r27,`$FRAME+21*16+1*$SIZE_T`($sp)
1407 li $x30,0x30
1408 $PUSH r28,`$FRAME+21*16+2*$SIZE_T`($sp)
1409 li $x40,0x40
1410 $PUSH r29,`$FRAME+21*16+3*$SIZE_T`($sp)
1411 li $x50,0x50
1412 $PUSH r30,`$FRAME+21*16+4*$SIZE_T`($sp)
1413 li $x60,0x60
1414 $PUSH r31,`$FRAME+21*16+5*$SIZE_T`($sp)
1415 li $x70,0x70
1416 mtspr 256,r0
1417
1418 subi $rounds,$rounds,3 # -4 in total
1419
1420 lvx $rndkey0,$x00,$key # load key schedule
1421 lvx v30,$x10,$key
1422 addi $key,$key,0x20
1423 lvx v31,$x00,$key
1424 ?vperm $rndkey0,$rndkey0,v30,$keyperm
1425 addi $key_,$sp,$FRAME+15
1426 mtctr $rounds
1427
1428 Load_ctr32_enc_key:
1429 ?vperm v24,v30,v31,$keyperm
1430 lvx v30,$x10,$key
1431 addi $key,$key,0x20
1432 stvx v24,$x00,$key_ # off-load round[1]
1433 ?vperm v25,v31,v30,$keyperm
1434 lvx v31,$x00,$key
1435 stvx v25,$x10,$key_ # off-load round[2]
1436 addi $key_,$key_,0x20
1437 bdnz Load_ctr32_enc_key
1438
1439 lvx v26,$x10,$key
1440 ?vperm v24,v30,v31,$keyperm
1441 lvx v27,$x20,$key
1442 stvx v24,$x00,$key_ # off-load round[3]
1443 ?vperm v25,v31,v26,$keyperm
1444 lvx v28,$x30,$key
1445 stvx v25,$x10,$key_ # off-load round[4]
1446 addi $key_,$sp,$FRAME+15 # rewind $key_
1447 ?vperm v26,v26,v27,$keyperm
1448 lvx v29,$x40,$key
1449 ?vperm v27,v27,v28,$keyperm
1450 lvx v30,$x50,$key
1451 ?vperm v28,v28,v29,$keyperm
1452 lvx v31,$x60,$key
1453 ?vperm v29,v29,v30,$keyperm
1454 lvx $out0,$x70,$key # borrow $out0
1455 ?vperm v30,v30,v31,$keyperm
1456 lvx v24,$x00,$key_ # pre-load round[1]
1457 ?vperm v31,v31,$out0,$keyperm
1458 lvx v25,$x10,$key_ # pre-load round[2]
1459
1460 vadduqm $two,$one,$one
1461 subi $inp,$inp,15 # undo "caller"
1462 $SHL $len,$len,4
1463
1464 vadduqm $out1,$ivec,$one # counter values ...
1465 vadduqm $out2,$ivec,$two
1466 vxor $out0,$ivec,$rndkey0 # ... xored with rndkey[0]
1467 le?li $idx,8
1468 vadduqm $out3,$out1,$two
1469 vxor $out1,$out1,$rndkey0
1470 le?lvsl $inpperm,0,$idx
1471 vadduqm $out4,$out2,$two
1472 vxor $out2,$out2,$rndkey0
1473 le?vspltisb $tmp,0x0f
1474 vadduqm $out5,$out3,$two
1475 vxor $out3,$out3,$rndkey0
1476 le?vxor $inpperm,$inpperm,$tmp # transform for lvx_u/stvx_u
1477 vadduqm $out6,$out4,$two
1478 vxor $out4,$out4,$rndkey0
1479 vadduqm $out7,$out5,$two
1480 vxor $out5,$out5,$rndkey0
1481 vadduqm $ivec,$out6,$two # next counter value
1482 vxor $out6,$out6,$rndkey0
1483 vxor $out7,$out7,$rndkey0
1484
1485 mtctr $rounds
1486 b Loop_ctr32_enc8x
1487 .align 5
1488 Loop_ctr32_enc8x:
1489 vcipher $out0,$out0,v24
1490 vcipher $out1,$out1,v24
1491 vcipher $out2,$out2,v24
1492 vcipher $out3,$out3,v24
1493 vcipher $out4,$out4,v24
1494 vcipher $out5,$out5,v24
1495 vcipher $out6,$out6,v24
1496 vcipher $out7,$out7,v24
1497 Loop_ctr32_enc8x_middle:
1498 lvx v24,$x20,$key_ # round[3]
1499 addi $key_,$key_,0x20
1500
1501 vcipher $out0,$out0,v25
1502 vcipher $out1,$out1,v25
1503 vcipher $out2,$out2,v25
1504 vcipher $out3,$out3,v25
1505 vcipher $out4,$out4,v25
1506 vcipher $out5,$out5,v25
1507 vcipher $out6,$out6,v25
1508 vcipher $out7,$out7,v25
1509 lvx v25,$x10,$key_ # round[4]
1510 bdnz Loop_ctr32_enc8x
1511
1512 subic r11,$len,256 # $len-256, borrow $key_
1513 vcipher $out0,$out0,v24
1514 vcipher $out1,$out1,v24
1515 vcipher $out2,$out2,v24
1516 vcipher $out3,$out3,v24
1517 vcipher $out4,$out4,v24
1518 vcipher $out5,$out5,v24
1519 vcipher $out6,$out6,v24
1520 vcipher $out7,$out7,v24
1521
1522 subfe r0,r0,r0 # borrow?-1:0
1523 vcipher $out0,$out0,v25
1524 vcipher $out1,$out1,v25
1525 vcipher $out2,$out2,v25
1526 vcipher $out3,$out3,v25
1527 vcipher $out4,$out4,v25
1528 vcipher $out5,$out5,v25
1529 vcipher $out6,$out6,v25
1530 vcipher $out7,$out7,v25
1531
1532 and r0,r0,r11
1533 addi $key_,$sp,$FRAME+15 # rewind $key_
1534 vcipher $out0,$out0,v26
1535 vcipher $out1,$out1,v26
1536 vcipher $out2,$out2,v26
1537 vcipher $out3,$out3,v26
1538 vcipher $out4,$out4,v26
1539 vcipher $out5,$out5,v26
1540 vcipher $out6,$out6,v26
1541 vcipher $out7,$out7,v26
1542 lvx v24,$x00,$key_ # re-pre-load round[1]
1543
1544 subic $len,$len,129 # $len-=129
1545 vcipher $out0,$out0,v27
1546 addi $len,$len,1 # $len-=128 really
1547 vcipher $out1,$out1,v27
1548 vcipher $out2,$out2,v27
1549 vcipher $out3,$out3,v27
1550 vcipher $out4,$out4,v27
1551 vcipher $out5,$out5,v27
1552 vcipher $out6,$out6,v27
1553 vcipher $out7,$out7,v27
1554 lvx v25,$x10,$key_ # re-pre-load round[2]
1555
1556 vcipher $out0,$out0,v28
1557 lvx_u $in0,$x00,$inp # load input
1558 vcipher $out1,$out1,v28
1559 lvx_u $in1,$x10,$inp
1560 vcipher $out2,$out2,v28
1561 lvx_u $in2,$x20,$inp
1562 vcipher $out3,$out3,v28
1563 lvx_u $in3,$x30,$inp
1564 vcipher $out4,$out4,v28
1565 lvx_u $in4,$x40,$inp
1566 vcipher $out5,$out5,v28
1567 lvx_u $in5,$x50,$inp
1568 vcipher $out6,$out6,v28
1569 lvx_u $in6,$x60,$inp
1570 vcipher $out7,$out7,v28
1571 lvx_u $in7,$x70,$inp
1572 addi $inp,$inp,0x80
1573
1574 vcipher $out0,$out0,v29
1575 le?vperm $in0,$in0,$in0,$inpperm
1576 vcipher $out1,$out1,v29
1577 le?vperm $in1,$in1,$in1,$inpperm
1578 vcipher $out2,$out2,v29
1579 le?vperm $in2,$in2,$in2,$inpperm
1580 vcipher $out3,$out3,v29
1581 le?vperm $in3,$in3,$in3,$inpperm
1582 vcipher $out4,$out4,v29
1583 le?vperm $in4,$in4,$in4,$inpperm
1584 vcipher $out5,$out5,v29
1585 le?vperm $in5,$in5,$in5,$inpperm
1586 vcipher $out6,$out6,v29
1587 le?vperm $in6,$in6,$in6,$inpperm
1588 vcipher $out7,$out7,v29
1589 le?vperm $in7,$in7,$in7,$inpperm
1590
1591 add $inp,$inp,r0 # $inp is adjusted in such
1592 # way that at exit from the
1593 # loop inX-in7 are loaded
1594 # with last "words"
1595 subfe. r0,r0,r0 # borrow?-1:0
1596 vcipher $out0,$out0,v30
1597 vxor $in0,$in0,v31 # xor with last round key
1598 vcipher $out1,$out1,v30
1599 vxor $in1,$in1,v31
1600 vcipher $out2,$out2,v30
1601 vxor $in2,$in2,v31
1602 vcipher $out3,$out3,v30
1603 vxor $in3,$in3,v31
1604 vcipher $out4,$out4,v30
1605 vxor $in4,$in4,v31
1606 vcipher $out5,$out5,v30
1607 vxor $in5,$in5,v31
1608 vcipher $out6,$out6,v30
1609 vxor $in6,$in6,v31
1610 vcipher $out7,$out7,v30
1611 vxor $in7,$in7,v31
1612
1613 bne Lctr32_enc8x_break # did $len-129 borrow?
1614
1615 vcipherlast $in0,$out0,$in0
1616 vcipherlast $in1,$out1,$in1
1617 vadduqm $out1,$ivec,$one # counter values ...
1618 vcipherlast $in2,$out2,$in2
1619 vadduqm $out2,$ivec,$two
1620 vxor $out0,$ivec,$rndkey0 # ... xored with rndkey[0]
1621 vcipherlast $in3,$out3,$in3
1622 vadduqm $out3,$out1,$two
1623 vxor $out1,$out1,$rndkey0
1624 vcipherlast $in4,$out4,$in4
1625 vadduqm $out4,$out2,$two
1626 vxor $out2,$out2,$rndkey0
1627 vcipherlast $in5,$out5,$in5
1628 vadduqm $out5,$out3,$two
1629 vxor $out3,$out3,$rndkey0
1630 vcipherlast $in6,$out6,$in6
1631 vadduqm $out6,$out4,$two
1632 vxor $out4,$out4,$rndkey0
1633 vcipherlast $in7,$out7,$in7
1634 vadduqm $out7,$out5,$two
1635 vxor $out5,$out5,$rndkey0
1636 le?vperm $in0,$in0,$in0,$inpperm
1637 vadduqm $ivec,$out6,$two # next counter value
1638 vxor $out6,$out6,$rndkey0
1639 le?vperm $in1,$in1,$in1,$inpperm
1640 vxor $out7,$out7,$rndkey0
1641 mtctr $rounds
1642
1643 vcipher $out0,$out0,v24
1644 stvx_u $in0,$x00,$out
1645 le?vperm $in2,$in2,$in2,$inpperm
1646 vcipher $out1,$out1,v24
1647 stvx_u $in1,$x10,$out
1648 le?vperm $in3,$in3,$in3,$inpperm
1649 vcipher $out2,$out2,v24
1650 stvx_u $in2,$x20,$out
1651 le?vperm $in4,$in4,$in4,$inpperm
1652 vcipher $out3,$out3,v24
1653 stvx_u $in3,$x30,$out
1654 le?vperm $in5,$in5,$in5,$inpperm
1655 vcipher $out4,$out4,v24
1656 stvx_u $in4,$x40,$out
1657 le?vperm $in6,$in6,$in6,$inpperm
1658 vcipher $out5,$out5,v24
1659 stvx_u $in5,$x50,$out
1660 le?vperm $in7,$in7,$in7,$inpperm
1661 vcipher $out6,$out6,v24
1662 stvx_u $in6,$x60,$out
1663 vcipher $out7,$out7,v24
1664 stvx_u $in7,$x70,$out
1665 addi $out,$out,0x80
1666
1667 b Loop_ctr32_enc8x_middle
1668
1669 .align 5
1670 Lctr32_enc8x_break:
1671 cmpwi $len,-0x60
1672 blt Lctr32_enc8x_one
1673 nop
1674 beq Lctr32_enc8x_two
1675 cmpwi $len,-0x40
1676 blt Lctr32_enc8x_three
1677 nop
1678 beq Lctr32_enc8x_four
1679 cmpwi $len,-0x20
1680 blt Lctr32_enc8x_five
1681 nop
1682 beq Lctr32_enc8x_six
1683 cmpwi $len,0x00
1684 blt Lctr32_enc8x_seven
1685
1686 Lctr32_enc8x_eight:
1687 vcipherlast $out0,$out0,$in0
1688 vcipherlast $out1,$out1,$in1
1689 vcipherlast $out2,$out2,$in2
1690 vcipherlast $out3,$out3,$in3
1691 vcipherlast $out4,$out4,$in4
1692 vcipherlast $out5,$out5,$in5
1693 vcipherlast $out6,$out6,$in6
1694 vcipherlast $out7,$out7,$in7
1695
1696 le?vperm $out0,$out0,$out0,$inpperm
1697 le?vperm $out1,$out1,$out1,$inpperm
1698 stvx_u $out0,$x00,$out
1699 le?vperm $out2,$out2,$out2,$inpperm
1700 stvx_u $out1,$x10,$out
1701 le?vperm $out3,$out3,$out3,$inpperm
1702 stvx_u $out2,$x20,$out
1703 le?vperm $out4,$out4,$out4,$inpperm
1704 stvx_u $out3,$x30,$out
1705 le?vperm $out5,$out5,$out5,$inpperm
1706 stvx_u $out4,$x40,$out
1707 le?vperm $out6,$out6,$out6,$inpperm
1708 stvx_u $out5,$x50,$out
1709 le?vperm $out7,$out7,$out7,$inpperm
1710 stvx_u $out6,$x60,$out
1711 stvx_u $out7,$x70,$out
1712 addi $out,$out,0x80
1713 b Lctr32_enc8x_done
1714
1715 .align 5
1716 Lctr32_enc8x_seven:
1717 vcipherlast $out0,$out0,$in1
1718 vcipherlast $out1,$out1,$in2
1719 vcipherlast $out2,$out2,$in3
1720 vcipherlast $out3,$out3,$in4
1721 vcipherlast $out4,$out4,$in5
1722 vcipherlast $out5,$out5,$in6
1723 vcipherlast $out6,$out6,$in7
1724
1725 le?vperm $out0,$out0,$out0,$inpperm
1726 le?vperm $out1,$out1,$out1,$inpperm
1727 stvx_u $out0,$x00,$out
1728 le?vperm $out2,$out2,$out2,$inpperm
1729 stvx_u $out1,$x10,$out
1730 le?vperm $out3,$out3,$out3,$inpperm
1731 stvx_u $out2,$x20,$out
1732 le?vperm $out4,$out4,$out4,$inpperm
1733 stvx_u $out3,$x30,$out
1734 le?vperm $out5,$out5,$out5,$inpperm
1735 stvx_u $out4,$x40,$out
1736 le?vperm $out6,$out6,$out6,$inpperm
1737 stvx_u $out5,$x50,$out
1738 stvx_u $out6,$x60,$out
1739 addi $out,$out,0x70
1740 b Lctr32_enc8x_done
1741
1742 .align 5
1743 Lctr32_enc8x_six:
1744 vcipherlast $out0,$out0,$in2
1745 vcipherlast $out1,$out1,$in3
1746 vcipherlast $out2,$out2,$in4
1747 vcipherlast $out3,$out3,$in5
1748 vcipherlast $out4,$out4,$in6
1749 vcipherlast $out5,$out5,$in7
1750
1751 le?vperm $out0,$out0,$out0,$inpperm
1752 le?vperm $out1,$out1,$out1,$inpperm
1753 stvx_u $out0,$x00,$out
1754 le?vperm $out2,$out2,$out2,$inpperm
1755 stvx_u $out1,$x10,$out
1756 le?vperm $out3,$out3,$out3,$inpperm
1757 stvx_u $out2,$x20,$out
1758 le?vperm $out4,$out4,$out4,$inpperm
1759 stvx_u $out3,$x30,$out
1760 le?vperm $out5,$out5,$out5,$inpperm
1761 stvx_u $out4,$x40,$out
1762 stvx_u $out5,$x50,$out
1763 addi $out,$out,0x60
1764 b Lctr32_enc8x_done
1765
1766 .align 5
1767 Lctr32_enc8x_five:
1768 vcipherlast $out0,$out0,$in3
1769 vcipherlast $out1,$out1,$in4
1770 vcipherlast $out2,$out2,$in5
1771 vcipherlast $out3,$out3,$in6
1772 vcipherlast $out4,$out4,$in7
1773
1774 le?vperm $out0,$out0,$out0,$inpperm
1775 le?vperm $out1,$out1,$out1,$inpperm
1776 stvx_u $out0,$x00,$out
1777 le?vperm $out2,$out2,$out2,$inpperm
1778 stvx_u $out1,$x10,$out
1779 le?vperm $out3,$out3,$out3,$inpperm
1780 stvx_u $out2,$x20,$out
1781 le?vperm $out4,$out4,$out4,$inpperm
1782 stvx_u $out3,$x30,$out
1783 stvx_u $out4,$x40,$out
1784 addi $out,$out,0x50
1785 b Lctr32_enc8x_done
1786
1787 .align 5
1788 Lctr32_enc8x_four:
1789 vcipherlast $out0,$out0,$in4
1790 vcipherlast $out1,$out1,$in5
1791 vcipherlast $out2,$out2,$in6
1792 vcipherlast $out3,$out3,$in7
1793
1794 le?vperm $out0,$out0,$out0,$inpperm
1795 le?vperm $out1,$out1,$out1,$inpperm
1796 stvx_u $out0,$x00,$out
1797 le?vperm $out2,$out2,$out2,$inpperm
1798 stvx_u $out1,$x10,$out
1799 le?vperm $out3,$out3,$out3,$inpperm
1800 stvx_u $out2,$x20,$out
1801 stvx_u $out3,$x30,$out
1802 addi $out,$out,0x40
1803 b Lctr32_enc8x_done
1804
1805 .align 5
1806 Lctr32_enc8x_three:
1807 vcipherlast $out0,$out0,$in5
1808 vcipherlast $out1,$out1,$in6
1809 vcipherlast $out2,$out2,$in7
1810
1811 le?vperm $out0,$out0,$out0,$inpperm
1812 le?vperm $out1,$out1,$out1,$inpperm
1813 stvx_u $out0,$x00,$out
1814 le?vperm $out2,$out2,$out2,$inpperm
1815 stvx_u $out1,$x10,$out
1816 stvx_u $out2,$x20,$out
1817 addi $out,$out,0x30
1818 b Lcbc_dec8x_done
1819
1820 .align 5
1821 Lctr32_enc8x_two:
1822 vcipherlast $out0,$out0,$in6
1823 vcipherlast $out1,$out1,$in7
1824
1825 le?vperm $out0,$out0,$out0,$inpperm
1826 le?vperm $out1,$out1,$out1,$inpperm
1827 stvx_u $out0,$x00,$out
1828 stvx_u $out1,$x10,$out
1829 addi $out,$out,0x20
1830 b Lcbc_dec8x_done
1831
1832 .align 5
1833 Lctr32_enc8x_one:
1834 vcipherlast $out0,$out0,$in7
1835
1836 le?vperm $out0,$out0,$out0,$inpperm
1837 stvx_u $out0,0,$out
1838 addi $out,$out,0x10
1839
1840 Lctr32_enc8x_done:
1841 li r10,`$FRAME+15`
1842 li r11,`$FRAME+31`
1843 stvx $inpperm,r10,$sp # wipe copies of round keys
1844 addi r10,r10,32
1845 stvx $inpperm,r11,$sp
1846 addi r11,r11,32
1847 stvx $inpperm,r10,$sp
1848 addi r10,r10,32
1849 stvx $inpperm,r11,$sp
1850 addi r11,r11,32
1851 stvx $inpperm,r10,$sp
1852 addi r10,r10,32
1853 stvx $inpperm,r11,$sp
1854 addi r11,r11,32
1855 stvx $inpperm,r10,$sp
1856 addi r10,r10,32
1857 stvx $inpperm,r11,$sp
1858 addi r11,r11,32
1859
1860 mtspr 256,$vrsave
1861 lvx v20,r10,$sp # ABI says so
1862 addi r10,r10,32
1863 lvx v21,r11,$sp
1864 addi r11,r11,32
1865 lvx v22,r10,$sp
1866 addi r10,r10,32
1867 lvx v23,r11,$sp
1868 addi r11,r11,32
1869 lvx v24,r10,$sp
1870 addi r10,r10,32
1871 lvx v25,r11,$sp
1872 addi r11,r11,32
1873 lvx v26,r10,$sp
1874 addi r10,r10,32
1875 lvx v27,r11,$sp
1876 addi r11,r11,32
1877 lvx v28,r10,$sp
1878 addi r10,r10,32
1879 lvx v29,r11,$sp
1880 addi r11,r11,32
1881 lvx v30,r10,$sp
1882 lvx v31,r11,$sp
1883 $POP r26,`$FRAME+21*16+0*$SIZE_T`($sp)
1884 $POP r27,`$FRAME+21*16+1*$SIZE_T`($sp)
1885 $POP r28,`$FRAME+21*16+2*$SIZE_T`($sp)
1886 $POP r29,`$FRAME+21*16+3*$SIZE_T`($sp)
1887 $POP r30,`$FRAME+21*16+4*$SIZE_T`($sp)
1888 $POP r31,`$FRAME+21*16+5*$SIZE_T`($sp)
1889 addi $sp,$sp,`$FRAME+21*16+6*$SIZE_T`
1890 blr
1891 .long 0
1892 .byte 0,12,0x14,0,0x80,6,6,0
1893 .long 0
1894 .size .${prefix}_ctr32_encrypt_blocks,.-.${prefix}_ctr32_encrypt_blocks
1895 ___
1896 }} }}}
1897
1898 #########################################################################
1899 {{{ # XTS procedures #
1900 # int aes_p8_xts_[en|de]crypt(const char *inp, char *out, size_t len, #
1901 # const AES_KEY *key1, const AES_KEY *key2, #
1902 # [const] unsigned char iv[16]); #
1903 # If $key2 is NULL, then a "tweak chaining" mode is engaged, in which #
1904 # input tweak value is assumed to be encrypted already, and last tweak #
1905 # value, one suitable for consecutive call on same chunk of data, is #
1906 # written back to original buffer. In addition, in "tweak chaining" #
1907 # mode only complete input blocks are processed. #
1908
1909 my ($inp,$out,$len,$key1,$key2,$ivp,$rounds,$idx) = map("r$_",(3..10));
1910 my ($rndkey0,$rndkey1,$inout) = map("v$_",(0..2));
1911 my ($output,$inptail,$inpperm,$leperm,$keyperm) = map("v$_",(3..7));
1912 my ($tweak,$seven,$eighty7,$tmp,$tweak1) = map("v$_",(8..12));
1913 my $taillen = $key2;
1914
1915 ($inp,$idx) = ($idx,$inp); # reassign
1916
1917 $code.=<<___;
1918 .globl .${prefix}_xts_encrypt
1919 mr $inp,r3 # reassign
1920 li r3,-1
1921 ${UCMP}i $len,16
1922 bltlr-
1923
1924 lis r0,0xfff0
1925 mfspr r12,256 # save vrsave
1926 li r11,0
1927 mtspr 256,r0
1928
1929 vspltisb $seven,0x07 # 0x070707..07
1930 le?lvsl $leperm,r11,r11
1931 le?vspltisb $tmp,0x0f
1932 le?vxor $leperm,$leperm,$seven
1933
1934 li $idx,15
1935 lvx $tweak,0,$ivp # load [unaligned] iv
1936 lvsl $inpperm,0,$ivp
1937 lvx $inptail,$idx,$ivp
1938 le?vxor $inpperm,$inpperm,$tmp
1939 vperm $tweak,$tweak,$inptail,$inpperm
1940
1941 neg r11,$inp
1942 lvsr $inpperm,0,r11 # prepare for unaligned load
1943 lvx $inout,0,$inp
1944 addi $inp,$inp,15 # 15 is not typo
1945 le?vxor $inpperm,$inpperm,$tmp
1946
1947 ${UCMP}i $key2,0 # key2==NULL?
1948 beq Lxts_enc_no_key2
1949
1950 ?lvsl $keyperm,0,$key2 # prepare for unaligned key
1951 lwz $rounds,240($key2)
1952 srwi $rounds,$rounds,1
1953 subi $rounds,$rounds,1
1954 li $idx,16
1955
1956 lvx $rndkey0,0,$key2
1957 lvx $rndkey1,$idx,$key2
1958 addi $idx,$idx,16
1959 ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
1960 vxor $tweak,$tweak,$rndkey0
1961 lvx $rndkey0,$idx,$key2
1962 addi $idx,$idx,16
1963 mtctr $rounds
1964
1965 Ltweak_xts_enc:
1966 ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm
1967 vcipher $tweak,$tweak,$rndkey1
1968 lvx $rndkey1,$idx,$key2
1969 addi $idx,$idx,16
1970 ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
1971 vcipher $tweak,$tweak,$rndkey0
1972 lvx $rndkey0,$idx,$key2
1973 addi $idx,$idx,16
1974 bdnz Ltweak_xts_enc
1975
1976 ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm
1977 vcipher $tweak,$tweak,$rndkey1
1978 lvx $rndkey1,$idx,$key2
1979 ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
1980 vcipherlast $tweak,$tweak,$rndkey0
1981
1982 li $ivp,0 # don't chain the tweak
1983 b Lxts_enc
1984
1985 Lxts_enc_no_key2:
1986 li $idx,-16
1987 and $len,$len,$idx # in "tweak chaining"
1988 # mode only complete
1989 # blocks are processed
1990 Lxts_enc:
1991 lvx $inptail,0,$inp
1992 addi $inp,$inp,16
1993
1994 ?lvsl $keyperm,0,$key1 # prepare for unaligned key
1995 lwz $rounds,240($key1)
1996 srwi $rounds,$rounds,1
1997 subi $rounds,$rounds,1
1998 li $idx,16
1999
2000 vslb $eighty7,$seven,$seven # 0x808080..80
2001 vor $eighty7,$eighty7,$seven # 0x878787..87
2002 vspltisb $tmp,1 # 0x010101..01
2003 vsldoi $eighty7,$eighty7,$tmp,15 # 0x870101..01
2004
2005 ${UCMP}i $len,96
2006 bge _aesp8_xts_encrypt6x
2007
2008 andi. $taillen,$len,15
2009 subic r0,$len,32
2010 subi $taillen,$taillen,16
2011 subfe r0,r0,r0
2012 and r0,r0,$taillen
2013 add $inp,$inp,r0
2014
2015 lvx $rndkey0,0,$key1
2016 lvx $rndkey1,$idx,$key1
2017 addi $idx,$idx,16
2018 vperm $inout,$inout,$inptail,$inpperm
2019 ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
2020 vxor $inout,$inout,$tweak
2021 vxor $inout,$inout,$rndkey0
2022 lvx $rndkey0,$idx,$key1
2023 addi $idx,$idx,16
2024 mtctr $rounds
2025 b Loop_xts_enc
2026
2027 .align 5
2028 Loop_xts_enc:
2029 ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm
2030 vcipher $inout,$inout,$rndkey1
2031 lvx $rndkey1,$idx,$key1
2032 addi $idx,$idx,16
2033 ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
2034 vcipher $inout,$inout,$rndkey0
2035 lvx $rndkey0,$idx,$key1
2036 addi $idx,$idx,16
2037 bdnz Loop_xts_enc
2038
2039 ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm
2040 vcipher $inout,$inout,$rndkey1
2041 lvx $rndkey1,$idx,$key1
2042 li $idx,16
2043 ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
2044 vxor $rndkey0,$rndkey0,$tweak
2045 vcipherlast $output,$inout,$rndkey0
2046
2047 le?vperm $tmp,$output,$output,$leperm
2048 be?nop
2049 le?stvx_u $tmp,0,$out
2050 be?stvx_u $output,0,$out
2051 addi $out,$out,16
2052
2053 subic. $len,$len,16
2054 beq Lxts_enc_done
2055
2056 vmr $inout,$inptail
2057 lvx $inptail,0,$inp
2058 addi $inp,$inp,16
2059 lvx $rndkey0,0,$key1
2060 lvx $rndkey1,$idx,$key1
2061 addi $idx,$idx,16
2062
2063 subic r0,$len,32
2064 subfe r0,r0,r0
2065 and r0,r0,$taillen
2066 add $inp,$inp,r0
2067
2068 vsrab $tmp,$tweak,$seven # next tweak value
2069 vaddubm $tweak,$tweak,$tweak
2070 vsldoi $tmp,$tmp,$tmp,15
2071 vand $tmp,$tmp,$eighty7
2072 vxor $tweak,$tweak,$tmp
2073
2074 vperm $inout,$inout,$inptail,$inpperm
2075 ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
2076 vxor $inout,$inout,$tweak
2077 vxor $output,$output,$rndkey0 # just in case $len<16
2078 vxor $inout,$inout,$rndkey0
2079 lvx $rndkey0,$idx,$key1
2080 addi $idx,$idx,16
2081
2082 mtctr $rounds
2083 ${UCMP}i $len,16
2084 bge Loop_xts_enc
2085
2086 vxor $output,$output,$tweak
2087 lvsr $inpperm,0,$len # $inpperm is no longer needed
2088 vxor $inptail,$inptail,$inptail # $inptail is no longer needed
2089 vspltisb $tmp,-1
2090 vperm $inptail,$inptail,$tmp,$inpperm
2091 vsel $inout,$inout,$output,$inptail
2092
2093 subi r11,$out,17
2094 subi $out,$out,16
2095 mtctr $len
2096 li $len,16
2097 Loop_xts_enc_steal:
2098 lbzu r0,1(r11)
2099 stb r0,16(r11)
2100 bdnz Loop_xts_enc_steal
2101
2102 mtctr $rounds
2103 b Loop_xts_enc # one more time...
2104
2105 Lxts_enc_done:
2106 ${UCMP}i $ivp,0
2107 beq Lxts_enc_ret
2108
2109 vsrab $tmp,$tweak,$seven # next tweak value
2110 vaddubm $tweak,$tweak,$tweak
2111 vsldoi $tmp,$tmp,$tmp,15
2112 vand $tmp,$tmp,$eighty7
2113 vxor $tweak,$tweak,$tmp
2114
2115 le?vperm $tweak,$tweak,$tweak,$leperm
2116 stvx_u $tweak,0,$ivp
2117
2118 Lxts_enc_ret:
2119 mtspr 256,r12 # restore vrsave
2120 li r3,0
2121 blr
2122 .long 0
2123 .byte 0,12,0x04,0,0x80,6,6,0
2124 .long 0
2125 .size .${prefix}_xts_encrypt,.-.${prefix}_xts_encrypt
2126
2127 .globl .${prefix}_xts_decrypt
2128 mr $inp,r3 # reassign
2129 li r3,-1
2130 ${UCMP}i $len,16
2131 bltlr-
2132
2133 lis r0,0xfff8
2134 mfspr r12,256 # save vrsave
2135 li r11,0
2136 mtspr 256,r0
2137
2138 andi. r0,$len,15
2139 neg r0,r0
2140 andi. r0,r0,16
2141 sub $len,$len,r0
2142
2143 vspltisb $seven,0x07 # 0x070707..07
2144 le?lvsl $leperm,r11,r11
2145 le?vspltisb $tmp,0x0f
2146 le?vxor $leperm,$leperm,$seven
2147
2148 li $idx,15
2149 lvx $tweak,0,$ivp # load [unaligned] iv
2150 lvsl $inpperm,0,$ivp
2151 lvx $inptail,$idx,$ivp
2152 le?vxor $inpperm,$inpperm,$tmp
2153 vperm $tweak,$tweak,$inptail,$inpperm
2154
2155 neg r11,$inp
2156 lvsr $inpperm,0,r11 # prepare for unaligned load
2157 lvx $inout,0,$inp
2158 addi $inp,$inp,15 # 15 is not typo
2159 le?vxor $inpperm,$inpperm,$tmp
2160
2161 ${UCMP}i $key2,0 # key2==NULL?
2162 beq Lxts_dec_no_key2
2163
2164 ?lvsl $keyperm,0,$key2 # prepare for unaligned key
2165 lwz $rounds,240($key2)
2166 srwi $rounds,$rounds,1
2167 subi $rounds,$rounds,1
2168 li $idx,16
2169
2170 lvx $rndkey0,0,$key2
2171 lvx $rndkey1,$idx,$key2
2172 addi $idx,$idx,16
2173 ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
2174 vxor $tweak,$tweak,$rndkey0
2175 lvx $rndkey0,$idx,$key2
2176 addi $idx,$idx,16
2177 mtctr $rounds
2178
2179 Ltweak_xts_dec:
2180 ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm
2181 vcipher $tweak,$tweak,$rndkey1
2182 lvx $rndkey1,$idx,$key2
2183 addi $idx,$idx,16
2184 ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
2185 vcipher $tweak,$tweak,$rndkey0
2186 lvx $rndkey0,$idx,$key2
2187 addi $idx,$idx,16
2188 bdnz Ltweak_xts_dec
2189
2190 ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm
2191 vcipher $tweak,$tweak,$rndkey1
2192 lvx $rndkey1,$idx,$key2
2193 ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
2194 vcipherlast $tweak,$tweak,$rndkey0
2195
2196 li $ivp,0 # don't chain the tweak
2197 b Lxts_dec
2198
2199 Lxts_dec_no_key2:
2200 neg $idx,$len
2201 andi. $idx,$idx,15
2202 add $len,$len,$idx # in "tweak chaining"
2203 # mode only complete
2204 # blocks are processed
2205 Lxts_dec:
2206 lvx $inptail,0,$inp
2207 addi $inp,$inp,16
2208
2209 ?lvsl $keyperm,0,$key1 # prepare for unaligned key
2210 lwz $rounds,240($key1)
2211 srwi $rounds,$rounds,1
2212 subi $rounds,$rounds,1
2213 li $idx,16
2214
2215 vslb $eighty7,$seven,$seven # 0x808080..80
2216 vor $eighty7,$eighty7,$seven # 0x878787..87
2217 vspltisb $tmp,1 # 0x010101..01
2218 vsldoi $eighty7,$eighty7,$tmp,15 # 0x870101..01
2219
2220 ${UCMP}i $len,96
2221 bge _aesp8_xts_decrypt6x
2222
2223 lvx $rndkey0,0,$key1
2224 lvx $rndkey1,$idx,$key1
2225 addi $idx,$idx,16
2226 vperm $inout,$inout,$inptail,$inpperm
2227 ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
2228 vxor $inout,$inout,$tweak
2229 vxor $inout,$inout,$rndkey0
2230 lvx $rndkey0,$idx,$key1
2231 addi $idx,$idx,16
2232 mtctr $rounds
2233
2234 ${UCMP}i $len,16
2235 blt Ltail_xts_dec
2236 be?b Loop_xts_dec
2237
2238 .align 5
2239 Loop_xts_dec:
2240 ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm
2241 vncipher $inout,$inout,$rndkey1
2242 lvx $rndkey1,$idx,$key1
2243 addi $idx,$idx,16
2244 ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
2245 vncipher $inout,$inout,$rndkey0
2246 lvx $rndkey0,$idx,$key1
2247 addi $idx,$idx,16
2248 bdnz Loop_xts_dec
2249
2250 ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm
2251 vncipher $inout,$inout,$rndkey1
2252 lvx $rndkey1,$idx,$key1
2253 li $idx,16
2254 ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
2255 vxor $rndkey0,$rndkey0,$tweak
2256 vncipherlast $output,$inout,$rndkey0
2257
2258 le?vperm $tmp,$output,$output,$leperm
2259 be?nop
2260 le?stvx_u $tmp,0,$out
2261 be?stvx_u $output,0,$out
2262 addi $out,$out,16
2263
2264 subic. $len,$len,16
2265 beq Lxts_dec_done
2266
2267 vmr $inout,$inptail
2268 lvx $inptail,0,$inp
2269 addi $inp,$inp,16
2270 lvx $rndkey0,0,$key1
2271 lvx $rndkey1,$idx,$key1
2272 addi $idx,$idx,16
2273
2274 vsrab $tmp,$tweak,$seven # next tweak value
2275 vaddubm $tweak,$tweak,$tweak
2276 vsldoi $tmp,$tmp,$tmp,15
2277 vand $tmp,$tmp,$eighty7
2278 vxor $tweak,$tweak,$tmp
2279
2280 vperm $inout,$inout,$inptail,$inpperm
2281 ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
2282 vxor $inout,$inout,$tweak
2283 vxor $inout,$inout,$rndkey0
2284 lvx $rndkey0,$idx,$key1
2285 addi $idx,$idx,16
2286
2287 mtctr $rounds
2288 ${UCMP}i $len,16
2289 bge Loop_xts_dec
2290
2291 Ltail_xts_dec:
2292 vsrab $tmp,$tweak,$seven # next tweak value
2293 vaddubm $tweak1,$tweak,$tweak
2294 vsldoi $tmp,$tmp,$tmp,15
2295 vand $tmp,$tmp,$eighty7
2296 vxor $tweak1,$tweak1,$tmp
2297
2298 subi $inp,$inp,16
2299 add $inp,$inp,$len
2300
2301 vxor $inout,$inout,$tweak # :-(
2302 vxor $inout,$inout,$tweak1 # :-)
2303
2304 Loop_xts_dec_short:
2305 ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm
2306 vncipher $inout,$inout,$rndkey1
2307 lvx $rndkey1,$idx,$key1
2308 addi $idx,$idx,16
2309 ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
2310 vncipher $inout,$inout,$rndkey0
2311 lvx $rndkey0,$idx,$key1
2312 addi $idx,$idx,16
2313 bdnz Loop_xts_dec_short
2314
2315 ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm
2316 vncipher $inout,$inout,$rndkey1
2317 lvx $rndkey1,$idx,$key1
2318 li $idx,16
2319 ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
2320 vxor $rndkey0,$rndkey0,$tweak1
2321 vncipherlast $output,$inout,$rndkey0
2322
2323 le?vperm $tmp,$output,$output,$leperm
2324 be?nop
2325 le?stvx_u $tmp,0,$out
2326 be?stvx_u $output,0,$out
2327
2328 vmr $inout,$inptail
2329 lvx $inptail,0,$inp
2330 #addi $inp,$inp,16
2331 lvx $rndkey0,0,$key1
2332 lvx $rndkey1,$idx,$key1
2333 addi $idx,$idx,16
2334 vperm $inout,$inout,$inptail,$inpperm
2335 ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
2336
2337 lvsr $inpperm,0,$len # $inpperm is no longer needed
2338 vxor $inptail,$inptail,$inptail # $inptail is no longer needed
2339 vspltisb $tmp,-1
2340 vperm $inptail,$inptail,$tmp,$inpperm
2341 vsel $inout,$inout,$output,$inptail
2342
2343 vxor $rndkey0,$rndkey0,$tweak
2344 vxor $inout,$inout,$rndkey0
2345 lvx $rndkey0,$idx,$key1
2346 addi $idx,$idx,16
2347
2348 subi r11,$out,1
2349 mtctr $len
2350 li $len,16
2351 Loop_xts_dec_steal:
2352 lbzu r0,1(r11)
2353 stb r0,16(r11)
2354 bdnz Loop_xts_dec_steal
2355
2356 mtctr $rounds
2357 b Loop_xts_dec # one more time...
2358
2359 Lxts_dec_done:
2360 ${UCMP}i $ivp,0
2361 beq Lxts_dec_ret
2362
2363 vsrab $tmp,$tweak,$seven # next tweak value
2364 vaddubm $tweak,$tweak,$tweak
2365 vsldoi $tmp,$tmp,$tmp,15
2366 vand $tmp,$tmp,$eighty7
2367 vxor $tweak,$tweak,$tmp
2368
2369 le?vperm $tweak,$tweak,$tweak,$leperm
2370 stvx_u $tweak,0,$ivp
2371
2372 Lxts_dec_ret:
2373 mtspr 256,r12 # restore vrsave
2374 li r3,0
2375 blr
2376 .long 0
2377 .byte 0,12,0x04,0,0x80,6,6,0
2378 .long 0
2379 .size .${prefix}_xts_decrypt,.-.${prefix}_xts_decrypt
2380 ___
2381 #########################################################################
2382 {{ # Optimized XTS procedures #
2383 my $key_=$key2;
2384 my ($x00,$x10,$x20,$x30,$x40,$x50,$x60,$x70)=map("r$_",(0,3,26..31));
2385 $x00=0 if ($flavour =~ /osx/);
2386 my ($in0, $in1, $in2, $in3, $in4, $in5 )=map("v$_",(0..5));
2387 my ($out0, $out1, $out2, $out3, $out4, $out5)=map("v$_",(7,12..16));
2388 my ($twk0, $twk1, $twk2, $twk3, $twk4, $twk5)=map("v$_",(17..22));
2389 my $rndkey0="v23"; # v24-v25 rotating buffer for first found keys
2390 # v26-v31 last 6 round keys
2391 my ($keyperm)=($out0); # aliases with "caller", redundant assignment
2392 my $taillen=$x70;
2393
2394 $code.=<<___;
2395 .align 5
2396 _aesp8_xts_encrypt6x:
2397 $STU $sp,-`($FRAME+21*16+6*$SIZE_T)`($sp)
2398 mflr r11
2399 li r7,`$FRAME+8*16+15`
2400 li r3,`$FRAME+8*16+31`
2401 $PUSH r11,`$FRAME+21*16+6*$SIZE_T+$LRSAVE`($sp)
2402 stvx v20,r7,$sp # ABI says so
2403 addi r7,r7,32
2404 stvx v21,r3,$sp
2405 addi r3,r3,32
2406 stvx v22,r7,$sp
2407 addi r7,r7,32
2408 stvx v23,r3,$sp
2409 addi r3,r3,32
2410 stvx v24,r7,$sp
2411 addi r7,r7,32
2412 stvx v25,r3,$sp
2413 addi r3,r3,32
2414 stvx v26,r7,$sp
2415 addi r7,r7,32
2416 stvx v27,r3,$sp
2417 addi r3,r3,32
2418 stvx v28,r7,$sp
2419 addi r7,r7,32
2420 stvx v29,r3,$sp
2421 addi r3,r3,32
2422 stvx v30,r7,$sp
2423 stvx v31,r3,$sp
2424 li r0,-1
2425 stw $vrsave,`$FRAME+21*16-4`($sp) # save vrsave
2426 li $x10,0x10
2427 $PUSH r26,`$FRAME+21*16+0*$SIZE_T`($sp)
2428 li $x20,0x20
2429 $PUSH r27,`$FRAME+21*16+1*$SIZE_T`($sp)
2430 li $x30,0x30
2431 $PUSH r28,`$FRAME+21*16+2*$SIZE_T`($sp)
2432 li $x40,0x40
2433 $PUSH r29,`$FRAME+21*16+3*$SIZE_T`($sp)
2434 li $x50,0x50
2435 $PUSH r30,`$FRAME+21*16+4*$SIZE_T`($sp)
2436 li $x60,0x60
2437 $PUSH r31,`$FRAME+21*16+5*$SIZE_T`($sp)
2438 li $x70,0x70
2439 mtspr 256,r0
2440
2441 subi $rounds,$rounds,3 # -4 in total
2442
2443 lvx $rndkey0,$x00,$key1 # load key schedule
2444 lvx v30,$x10,$key1
2445 addi $key1,$key1,0x20
2446 lvx v31,$x00,$key1
2447 ?vperm $rndkey0,$rndkey0,v30,$keyperm
2448 addi $key_,$sp,$FRAME+15
2449 mtctr $rounds
2450
2451 Load_xts_enc_key:
2452 ?vperm v24,v30,v31,$keyperm
2453 lvx v30,$x10,$key1
2454 addi $key1,$key1,0x20
2455 stvx v24,$x00,$key_ # off-load round[1]
2456 ?vperm v25,v31,v30,$keyperm
2457 lvx v31,$x00,$key1
2458 stvx v25,$x10,$key_ # off-load round[2]
2459 addi $key_,$key_,0x20
2460 bdnz Load_xts_enc_key
2461
2462 lvx v26,$x10,$key1
2463 ?vperm v24,v30,v31,$keyperm
2464 lvx v27,$x20,$key1
2465 stvx v24,$x00,$key_ # off-load round[3]
2466 ?vperm v25,v31,v26,$keyperm
2467 lvx v28,$x30,$key1
2468 stvx v25,$x10,$key_ # off-load round[4]
2469 addi $key_,$sp,$FRAME+15 # rewind $key_
2470 ?vperm v26,v26,v27,$keyperm
2471 lvx v29,$x40,$key1
2472 ?vperm v27,v27,v28,$keyperm
2473 lvx v30,$x50,$key1
2474 ?vperm v28,v28,v29,$keyperm
2475 lvx v31,$x60,$key1
2476 ?vperm v29,v29,v30,$keyperm
2477 lvx $twk5,$x70,$key1 # borrow $twk5
2478 ?vperm v30,v30,v31,$keyperm
2479 lvx v24,$x00,$key_ # pre-load round[1]
2480 ?vperm v31,v31,$twk5,$keyperm
2481 lvx v25,$x10,$key_ # pre-load round[2]
2482
2483 vperm $in0,$inout,$inptail,$inpperm
2484 subi $inp,$inp,31 # undo "caller"
2485 vxor $twk0,$tweak,$rndkey0
2486 vsrab $tmp,$tweak,$seven # next tweak value
2487 vaddubm $tweak,$tweak,$tweak
2488 vsldoi $tmp,$tmp,$tmp,15
2489 vand $tmp,$tmp,$eighty7
2490 vxor $out0,$in0,$twk0
2491 vxor $tweak,$tweak,$tmp
2492
2493 lvx_u $in1,$x10,$inp
2494 vxor $twk1,$tweak,$rndkey0
2495 vsrab $tmp,$tweak,$seven # next tweak value
2496 vaddubm $tweak,$tweak,$tweak
2497 vsldoi $tmp,$tmp,$tmp,15
2498 le?vperm $in1,$in1,$in1,$leperm
2499 vand $tmp,$tmp,$eighty7
2500 vxor $out1,$in1,$twk1
2501 vxor $tweak,$tweak,$tmp
2502
2503 lvx_u $in2,$x20,$inp
2504 andi. $taillen,$len,15
2505 vxor $twk2,$tweak,$rndkey0
2506 vsrab $tmp,$tweak,$seven # next tweak value
2507 vaddubm $tweak,$tweak,$tweak
2508 vsldoi $tmp,$tmp,$tmp,15
2509 le?vperm $in2,$in2,$in2,$leperm
2510 vand $tmp,$tmp,$eighty7
2511 vxor $out2,$in2,$twk2
2512 vxor $tweak,$tweak,$tmp
2513
2514 lvx_u $in3,$x30,$inp
2515 sub $len,$len,$taillen
2516 vxor $twk3,$tweak,$rndkey0
2517 vsrab $tmp,$tweak,$seven # next tweak value
2518 vaddubm $tweak,$tweak,$tweak
2519 vsldoi $tmp,$tmp,$tmp,15
2520 le?vperm $in3,$in3,$in3,$leperm
2521 vand $tmp,$tmp,$eighty7
2522 vxor $out3,$in3,$twk3
2523 vxor $tweak,$tweak,$tmp
2524
2525 lvx_u $in4,$x40,$inp
2526 subi $len,$len,0x60
2527 vxor $twk4,$tweak,$rndkey0
2528 vsrab $tmp,$tweak,$seven # next tweak value
2529 vaddubm $tweak,$tweak,$tweak
2530 vsldoi $tmp,$tmp,$tmp,15
2531 le?vperm $in4,$in4,$in4,$leperm
2532 vand $tmp,$tmp,$eighty7
2533 vxor $out4,$in4,$twk4
2534 vxor $tweak,$tweak,$tmp
2535
2536 lvx_u $in5,$x50,$inp
2537 addi $inp,$inp,0x60
2538 vxor $twk5,$tweak,$rndkey0
2539 vsrab $tmp,$tweak,$seven # next tweak value
2540 vaddubm $tweak,$tweak,$tweak
2541 vsldoi $tmp,$tmp,$tmp,15
2542 le?vperm $in5,$in5,$in5,$leperm
2543 vand $tmp,$tmp,$eighty7
2544 vxor $out5,$in5,$twk5
2545 vxor $tweak,$tweak,$tmp
2546
2547 vxor v31,v31,$rndkey0
2548 mtctr $rounds
2549 b Loop_xts_enc6x
2550
2551 .align 5
2552 Loop_xts_enc6x:
2553 vcipher $out0,$out0,v24
2554 vcipher $out1,$out1,v24
2555 vcipher $out2,$out2,v24
2556 vcipher $out3,$out3,v24
2557 vcipher $out4,$out4,v24
2558 vcipher $out5,$out5,v24
2559 lvx v24,$x20,$key_ # round[3]
2560 addi $key_,$key_,0x20
2561
2562 vcipher $out0,$out0,v25
2563 vcipher $out1,$out1,v25
2564 vcipher $out2,$out2,v25
2565 vcipher $out3,$out3,v25
2566 vcipher $out4,$out4,v25
2567 vcipher $out5,$out5,v25
2568 lvx v25,$x10,$key_ # round[4]
2569 bdnz Loop_xts_enc6x
2570
2571 subic $len,$len,96 # $len-=96
2572 vxor $in0,$twk0,v31 # xor with last round key
2573 vcipher $out0,$out0,v24
2574 vcipher $out1,$out1,v24
2575 vsrab $tmp,$tweak,$seven # next tweak value
2576 vxor $twk0,$tweak,$rndkey0
2577 vaddubm $tweak,$tweak,$tweak
2578 vcipher $out2,$out2,v24
2579 vcipher $out3,$out3,v24
2580 vsldoi $tmp,$tmp,$tmp,15
2581 vcipher $out4,$out4,v24
2582 vcipher $out5,$out5,v24
2583
2584 subfe. r0,r0,r0 # borrow?-1:0
2585 vand $tmp,$tmp,$eighty7
2586 vcipher $out0,$out0,v25
2587 vcipher $out1,$out1,v25
2588 vxor $tweak,$tweak,$tmp
2589 vcipher $out2,$out2,v25
2590 vcipher $out3,$out3,v25
2591 vxor $in1,$twk1,v31
2592 vsrab $tmp,$tweak,$seven # next tweak value
2593 vxor $twk1,$tweak,$rndkey0
2594 vcipher $out4,$out4,v25
2595 vcipher $out5,$out5,v25
2596
2597 and r0,r0,$len
2598 vaddubm $tweak,$tweak,$tweak
2599 vsldoi $tmp,$tmp,$tmp,15
2600 vcipher $out0,$out0,v26
2601 vcipher $out1,$out1,v26
2602 vand $tmp,$tmp,$eighty7
2603 vcipher $out2,$out2,v26
2604 vcipher $out3,$out3,v26
2605 vxor $tweak,$tweak,$tmp
2606 vcipher $out4,$out4,v26
2607 vcipher $out5,$out5,v26
2608
2609 add $inp,$inp,r0 # $inp is adjusted in such
2610 # way that at exit from the
2611 # loop inX-in5 are loaded
2612 # with last "words"
2613 vxor $in2,$twk2,v31
2614 vsrab $tmp,$tweak,$seven # next tweak value
2615 vxor $twk2,$tweak,$rndkey0
2616 vaddubm $tweak,$tweak,$tweak
2617 vcipher $out0,$out0,v27
2618 vcipher $out1,$out1,v27
2619 vsldoi $tmp,$tmp,$tmp,15
2620 vcipher $out2,$out2,v27
2621 vcipher $out3,$out3,v27
2622 vand $tmp,$tmp,$eighty7
2623 vcipher $out4,$out4,v27
2624 vcipher $out5,$out5,v27
2625
2626 addi $key_,$sp,$FRAME+15 # rewind $key_
2627 vxor $tweak,$tweak,$tmp
2628 vcipher $out0,$out0,v28
2629 vcipher $out1,$out1,v28
2630 vxor $in3,$twk3,v31
2631 vsrab $tmp,$tweak,$seven # next tweak value
2632 vxor $twk3,$tweak,$rndkey0
2633 vcipher $out2,$out2,v28
2634 vcipher $out3,$out3,v28
2635 vaddubm $tweak,$tweak,$tweak
2636 vsldoi $tmp,$tmp,$tmp,15
2637 vcipher $out4,$out4,v28
2638 vcipher $out5,$out5,v28
2639 lvx v24,$x00,$key_ # re-pre-load round[1]
2640 vand $tmp,$tmp,$eighty7
2641
2642 vcipher $out0,$out0,v29
2643 vcipher $out1,$out1,v29
2644 vxor $tweak,$tweak,$tmp
2645 vcipher $out2,$out2,v29
2646 vcipher $out3,$out3,v29
2647 vxor $in4,$twk4,v31
2648 vsrab $tmp,$tweak,$seven # next tweak value
2649 vxor $twk4,$tweak,$rndkey0
2650 vcipher $out4,$out4,v29
2651 vcipher $out5,$out5,v29
2652 lvx v25,$x10,$key_ # re-pre-load round[2]
2653 vaddubm $tweak,$tweak,$tweak
2654 vsldoi $tmp,$tmp,$tmp,15
2655
2656 vcipher $out0,$out0,v30
2657 vcipher $out1,$out1,v30
2658 vand $tmp,$tmp,$eighty7
2659 vcipher $out2,$out2,v30
2660 vcipher $out3,$out3,v30
2661 vxor $tweak,$tweak,$tmp
2662 vcipher $out4,$out4,v30
2663 vcipher $out5,$out5,v30
2664 vxor $in5,$twk5,v31
2665 vsrab $tmp,$tweak,$seven # next tweak value
2666 vxor $twk5,$tweak,$rndkey0
2667
2668 vcipherlast $out0,$out0,$in0
2669 lvx_u $in0,$x00,$inp # load next input block
2670 vaddubm $tweak,$tweak,$tweak
2671 vsldoi $tmp,$tmp,$tmp,15
2672 vcipherlast $out1,$out1,$in1
2673 lvx_u $in1,$x10,$inp
2674 vcipherlast $out2,$out2,$in2
2675 le?vperm $in0,$in0,$in0,$leperm
2676 lvx_u $in2,$x20,$inp
2677 vand $tmp,$tmp,$eighty7
2678 vcipherlast $out3,$out3,$in3
2679 le?vperm $in1,$in1,$in1,$leperm
2680 lvx_u $in3,$x30,$inp
2681 vcipherlast $out4,$out4,$in4
2682 le?vperm $in2,$in2,$in2,$leperm
2683 lvx_u $in4,$x40,$inp
2684 vxor $tweak,$tweak,$tmp
2685 vcipherlast $tmp,$out5,$in5 # last block might be needed
2686 # in stealing mode
2687 le?vperm $in3,$in3,$in3,$leperm
2688 lvx_u $in5,$x50,$inp
2689 addi $inp,$inp,0x60
2690 le?vperm $in4,$in4,$in4,$leperm
2691 le?vperm $in5,$in5,$in5,$leperm
2692
2693 le?vperm $out0,$out0,$out0,$leperm
2694 le?vperm $out1,$out1,$out1,$leperm
2695 stvx_u $out0,$x00,$out # store output
2696 vxor $out0,$in0,$twk0
2697 le?vperm $out2,$out2,$out2,$leperm
2698 stvx_u $out1,$x10,$out
2699 vxor $out1,$in1,$twk1
2700 le?vperm $out3,$out3,$out3,$leperm
2701 stvx_u $out2,$x20,$out
2702 vxor $out2,$in2,$twk2
2703 le?vperm $out4,$out4,$out4,$leperm
2704 stvx_u $out3,$x30,$out
2705 vxor $out3,$in3,$twk3
2706 le?vperm $out5,$tmp,$tmp,$leperm
2707 stvx_u $out4,$x40,$out
2708 vxor $out4,$in4,$twk4
2709 le?stvx_u $out5,$x50,$out
2710 be?stvx_u $tmp, $x50,$out
2711 vxor $out5,$in5,$twk5
2712 addi $out,$out,0x60
2713
2714 mtctr $rounds
2715 beq Loop_xts_enc6x # did $len-=96 borrow?
2716
2717 addic. $len,$len,0x60
2718 beq Lxts_enc6x_zero
2719 cmpwi $len,0x20
2720 blt Lxts_enc6x_one
2721 nop
2722 beq Lxts_enc6x_two
2723 cmpwi $len,0x40
2724 blt Lxts_enc6x_three
2725 nop
2726 beq Lxts_enc6x_four
2727
2728 Lxts_enc6x_five:
2729 vxor $out0,$in1,$twk0
2730 vxor $out1,$in2,$twk1
2731 vxor $out2,$in3,$twk2
2732 vxor $out3,$in4,$twk3
2733 vxor $out4,$in5,$twk4
2734
2735 bl _aesp8_xts_enc5x
2736
2737 le?vperm $out0,$out0,$out0,$leperm
2738 vmr $twk0,$twk5 # unused tweak
2739 le?vperm $out1,$out1,$out1,$leperm
2740 stvx_u $out0,$x00,$out # store output
2741 le?vperm $out2,$out2,$out2,$leperm
2742 stvx_u $out1,$x10,$out
2743 le?vperm $out3,$out3,$out3,$leperm
2744 stvx_u $out2,$x20,$out
2745 vxor $tmp,$out4,$twk5 # last block prep for stealing
2746 le?vperm $out4,$out4,$out4,$leperm
2747 stvx_u $out3,$x30,$out
2748 stvx_u $out4,$x40,$out
2749 addi $out,$out,0x50
2750 bne Lxts_enc6x_steal
2751 b Lxts_enc6x_done
2752
2753 .align 4
2754 Lxts_enc6x_four:
2755 vxor $out0,$in2,$twk0
2756 vxor $out1,$in3,$twk1
2757 vxor $out2,$in4,$twk2
2758 vxor $out3,$in5,$twk3
2759 vxor $out4,$out4,$out4
2760
2761 bl _aesp8_xts_enc5x
2762
2763 le?vperm $out0,$out0,$out0,$leperm
2764 vmr $twk0,$twk4 # unused tweak
2765 le?vperm $out1,$out1,$out1,$leperm
2766 stvx_u $out0,$x00,$out # store output
2767 le?vperm $out2,$out2,$out2,$leperm
2768 stvx_u $out1,$x10,$out
2769 vxor $tmp,$out3,$twk4 # last block prep for stealing
2770 le?vperm $out3,$out3,$out3,$leperm
2771 stvx_u $out2,$x20,$out
2772 stvx_u $out3,$x30,$out
2773 addi $out,$out,0x40
2774 bne Lxts_enc6x_steal
2775 b Lxts_enc6x_done
2776
2777 .align 4
2778 Lxts_enc6x_three:
2779 vxor $out0,$in3,$twk0
2780 vxor $out1,$in4,$twk1
2781 vxor $out2,$in5,$twk2
2782 vxor $out3,$out3,$out3
2783 vxor $out4,$out4,$out4
2784
2785 bl _aesp8_xts_enc5x
2786
2787 le?vperm $out0,$out0,$out0,$leperm
2788 vmr $twk0,$twk3 # unused tweak
2789 le?vperm $out1,$out1,$out1,$leperm
2790 stvx_u $out0,$x00,$out # store output
2791 vxor $tmp,$out2,$twk3 # last block prep for stealing
2792 le?vperm $out2,$out2,$out2,$leperm
2793 stvx_u $out1,$x10,$out
2794 stvx_u $out2,$x20,$out
2795 addi $out,$out,0x30
2796 bne Lxts_enc6x_steal
2797 b Lxts_enc6x_done
2798
2799 .align 4
2800 Lxts_enc6x_two:
2801 vxor $out0,$in4,$twk0
2802 vxor $out1,$in5,$twk1
2803 vxor $out2,$out2,$out2
2804 vxor $out3,$out3,$out3
2805 vxor $out4,$out4,$out4
2806
2807 bl _aesp8_xts_enc5x
2808
2809 le?vperm $out0,$out0,$out0,$leperm
2810 vmr $twk0,$twk2 # unused tweak
2811 vxor $tmp,$out1,$twk2 # last block prep for stealing
2812 le?vperm $out1,$out1,$out1,$leperm
2813 stvx_u $out0,$x00,$out # store output
2814 stvx_u $out1,$x10,$out
2815 addi $out,$out,0x20
2816 bne Lxts_enc6x_steal
2817 b Lxts_enc6x_done
2818
2819 .align 4
2820 Lxts_enc6x_one:
2821 vxor $out0,$in5,$twk0
2822 nop
2823 Loop_xts_enc1x:
2824 vcipher $out0,$out0,v24
2825 lvx v24,$x20,$key_ # round[3]
2826 addi $key_,$key_,0x20
2827
2828 vcipher $out0,$out0,v25
2829 lvx v25,$x10,$key_ # round[4]
2830 bdnz Loop_xts_enc1x
2831
2832 add $inp,$inp,$taillen
2833 cmpwi $taillen,0
2834 vcipher $out0,$out0,v24
2835
2836 subi $inp,$inp,16
2837 vcipher $out0,$out0,v25
2838
2839 lvsr $inpperm,0,$taillen
2840 vcipher $out0,$out0,v26
2841
2842 lvx_u $in0,0,$inp
2843 vcipher $out0,$out0,v27
2844
2845 addi $key_,$sp,$FRAME+15 # rewind $key_
2846 vcipher $out0,$out0,v28
2847 lvx v24,$x00,$key_ # re-pre-load round[1]
2848
2849 vcipher $out0,$out0,v29
2850 lvx v25,$x10,$key_ # re-pre-load round[2]
2851 vxor $twk0,$twk0,v31
2852
2853 le?vperm $in0,$in0,$in0,$leperm
2854 vcipher $out0,$out0,v30
2855
2856 vperm $in0,$in0,$in0,$inpperm
2857 vcipherlast $out0,$out0,$twk0
2858
2859 vmr $twk0,$twk1 # unused tweak
2860 vxor $tmp,$out0,$twk1 # last block prep for stealing
2861 le?vperm $out0,$out0,$out0,$leperm
2862 stvx_u $out0,$x00,$out # store output
2863 addi $out,$out,0x10
2864 bne Lxts_enc6x_steal
2865 b Lxts_enc6x_done
2866
2867 .align 4
2868 Lxts_enc6x_zero:
2869 cmpwi $taillen,0
2870 beq Lxts_enc6x_done
2871
2872 add $inp,$inp,$taillen
2873 subi $inp,$inp,16
2874 lvx_u $in0,0,$inp
2875 lvsr $inpperm,0,$taillen # $in5 is no more
2876 le?vperm $in0,$in0,$in0,$leperm
2877 vperm $in0,$in0,$in0,$inpperm
2878 vxor $tmp,$tmp,$twk0
2879 Lxts_enc6x_steal:
2880 vxor $in0,$in0,$twk0
2881 vxor $out0,$out0,$out0
2882 vspltisb $out1,-1
2883 vperm $out0,$out0,$out1,$inpperm
2884 vsel $out0,$in0,$tmp,$out0 # $tmp is last block, remember?
2885
2886 subi r30,$out,17
2887 subi $out,$out,16
2888 mtctr $taillen
2889 Loop_xts_enc6x_steal:
2890 lbzu r0,1(r30)
2891 stb r0,16(r30)
2892 bdnz Loop_xts_enc6x_steal
2893
2894 li $taillen,0
2895 mtctr $rounds
2896 b Loop_xts_enc1x # one more time...
2897
2898 .align 4
2899 Lxts_enc6x_done:
2900 ${UCMP}i $ivp,0
2901 beq Lxts_enc6x_ret
2902
2903 vxor $tweak,$twk0,$rndkey0
2904 le?vperm $tweak,$tweak,$tweak,$leperm
2905 stvx_u $tweak,0,$ivp
2906
2907 Lxts_enc6x_ret:
2908 mtlr r11
2909 li r10,`$FRAME+15`
2910 li r11,`$FRAME+31`
2911 stvx $seven,r10,$sp # wipe copies of round keys
2912 addi r10,r10,32
2913 stvx $seven,r11,$sp
2914 addi r11,r11,32
2915 stvx $seven,r10,$sp
2916 addi r10,r10,32
2917 stvx $seven,r11,$sp
2918 addi r11,r11,32
2919 stvx $seven,r10,$sp
2920 addi r10,r10,32
2921 stvx $seven,r11,$sp
2922 addi r11,r11,32
2923 stvx $seven,r10,$sp
2924 addi r10,r10,32
2925 stvx $seven,r11,$sp
2926 addi r11,r11,32
2927
2928 mtspr 256,$vrsave
2929 lvx v20,r10,$sp # ABI says so
2930 addi r10,r10,32
2931 lvx v21,r11,$sp
2932 addi r11,r11,32
2933 lvx v22,r10,$sp
2934 addi r10,r10,32
2935 lvx v23,r11,$sp
2936 addi r11,r11,32
2937 lvx v24,r10,$sp
2938 addi r10,r10,32
2939 lvx v25,r11,$sp
2940 addi r11,r11,32
2941 lvx v26,r10,$sp
2942 addi r10,r10,32
2943 lvx v27,r11,$sp
2944 addi r11,r11,32
2945 lvx v28,r10,$sp
2946 addi r10,r10,32
2947 lvx v29,r11,$sp
2948 addi r11,r11,32
2949 lvx v30,r10,$sp
2950 lvx v31,r11,$sp
2951 $POP r26,`$FRAME+21*16+0*$SIZE_T`($sp)
2952 $POP r27,`$FRAME+21*16+1*$SIZE_T`($sp)
2953 $POP r28,`$FRAME+21*16+2*$SIZE_T`($sp)
2954 $POP r29,`$FRAME+21*16+3*$SIZE_T`($sp)
2955 $POP r30,`$FRAME+21*16+4*$SIZE_T`($sp)
2956 $POP r31,`$FRAME+21*16+5*$SIZE_T`($sp)
2957 addi $sp,$sp,`$FRAME+21*16+6*$SIZE_T`
2958 blr
2959 .long 0
2960 .byte 0,12,0x04,1,0x80,6,6,0
2961 .long 0
2962
2963 .align 5
2964 _aesp8_xts_enc5x:
2965 vcipher $out0,$out0,v24
2966 vcipher $out1,$out1,v24
2967 vcipher $out2,$out2,v24
2968 vcipher $out3,$out3,v24
2969 vcipher $out4,$out4,v24
2970 lvx v24,$x20,$key_ # round[3]
2971 addi $key_,$key_,0x20
2972
2973 vcipher $out0,$out0,v25
2974 vcipher $out1,$out1,v25
2975 vcipher $out2,$out2,v25
2976 vcipher $out3,$out3,v25
2977 vcipher $out4,$out4,v25
2978 lvx v25,$x10,$key_ # round[4]
2979 bdnz _aesp8_xts_enc5x
2980
2981 add $inp,$inp,$taillen
2982 cmpwi $taillen,0
2983 vcipher $out0,$out0,v24
2984 vcipher $out1,$out1,v24
2985 vcipher $out2,$out2,v24
2986 vcipher $out3,$out3,v24
2987 vcipher $out4,$out4,v24
2988
2989 subi $inp,$inp,16
2990 vcipher $out0,$out0,v25
2991 vcipher $out1,$out1,v25
2992 vcipher $out2,$out2,v25
2993 vcipher $out3,$out3,v25
2994 vcipher $out4,$out4,v25
2995 vxor $twk0,$twk0,v31
2996
2997 vcipher $out0,$out0,v26
2998 lvsr $inpperm,r0,$taillen # $in5 is no more
2999 vcipher $out1,$out1,v26
3000 vcipher $out2,$out2,v26
3001 vcipher $out3,$out3,v26
3002 vcipher $out4,$out4,v26
3003 vxor $in1,$twk1,v31
3004
3005 vcipher $out0,$out0,v27
3006 lvx_u $in0,0,$inp
3007 vcipher $out1,$out1,v27
3008 vcipher $out2,$out2,v27
3009 vcipher $out3,$out3,v27
3010 vcipher $out4,$out4,v27
3011 vxor $in2,$twk2,v31
3012
3013 addi $key_,$sp,$FRAME+15 # rewind $key_
3014 vcipher $out0,$out0,v28
3015 vcipher $out1,$out1,v28
3016 vcipher $out2,$out2,v28
3017 vcipher $out3,$out3,v28
3018 vcipher $out4,$out4,v28
3019 lvx v24,$x00,$key_ # re-pre-load round[1]
3020 vxor $in3,$twk3,v31
3021
3022 vcipher $out0,$out0,v29
3023 le?vperm $in0,$in0,$in0,$leperm
3024 vcipher $out1,$out1,v29
3025 vcipher $out2,$out2,v29
3026 vcipher $out3,$out3,v29
3027 vcipher $out4,$out4,v29
3028 lvx v25,$x10,$key_ # re-pre-load round[2]
3029 vxor $in4,$twk4,v31
3030
3031 vcipher $out0,$out0,v30
3032 vperm $in0,$in0,$in0,$inpperm
3033 vcipher $out1,$out1,v30
3034 vcipher $out2,$out2,v30
3035 vcipher $out3,$out3,v30
3036 vcipher $out4,$out4,v30
3037
3038 vcipherlast $out0,$out0,$twk0
3039 vcipherlast $out1,$out1,$in1
3040 vcipherlast $out2,$out2,$in2
3041 vcipherlast $out3,$out3,$in3
3042 vcipherlast $out4,$out4,$in4
3043 blr
3044 .long 0
3045 .byte 0,12,0x14,0,0,0,0,0
3046
3047 .align 5
3048 _aesp8_xts_decrypt6x:
3049 $STU $sp,-`($FRAME+21*16+6*$SIZE_T)`($sp)
3050 mflr r11
3051 li r7,`$FRAME+8*16+15`
3052 li r3,`$FRAME+8*16+31`
3053 $PUSH r11,`$FRAME+21*16+6*$SIZE_T+$LRSAVE`($sp)
3054 stvx v20,r7,$sp # ABI says so
3055 addi r7,r7,32
3056 stvx v21,r3,$sp
3057 addi r3,r3,32
3058 stvx v22,r7,$sp
3059 addi r7,r7,32
3060 stvx v23,r3,$sp
3061 addi r3,r3,32
3062 stvx v24,r7,$sp
3063 addi r7,r7,32
3064 stvx v25,r3,$sp
3065 addi r3,r3,32
3066 stvx v26,r7,$sp
3067 addi r7,r7,32
3068 stvx v27,r3,$sp
3069 addi r3,r3,32
3070 stvx v28,r7,$sp
3071 addi r7,r7,32
3072 stvx v29,r3,$sp
3073 addi r3,r3,32
3074 stvx v30,r7,$sp
3075 stvx v31,r3,$sp
3076 li r0,-1
3077 stw $vrsave,`$FRAME+21*16-4`($sp) # save vrsave
3078 li $x10,0x10
3079 $PUSH r26,`$FRAME+21*16+0*$SIZE_T`($sp)
3080 li $x20,0x20
3081 $PUSH r27,`$FRAME+21*16+1*$SIZE_T`($sp)
3082 li $x30,0x30
3083 $PUSH r28,`$FRAME+21*16+2*$SIZE_T`($sp)
3084 li $x40,0x40
3085 $PUSH r29,`$FRAME+21*16+3*$SIZE_T`($sp)
3086 li $x50,0x50
3087 $PUSH r30,`$FRAME+21*16+4*$SIZE_T`($sp)
3088 li $x60,0x60
3089 $PUSH r31,`$FRAME+21*16+5*$SIZE_T`($sp)
3090 li $x70,0x70
3091 mtspr 256,r0
3092
3093 subi $rounds,$rounds,3 # -4 in total
3094
3095 lvx $rndkey0,$x00,$key1 # load key schedule
3096 lvx v30,$x10,$key1
3097 addi $key1,$key1,0x20
3098 lvx v31,$x00,$key1
3099 ?vperm $rndkey0,$rndkey0,v30,$keyperm
3100 addi $key_,$sp,$FRAME+15
3101 mtctr $rounds
3102
3103 Load_xts_dec_key:
3104 ?vperm v24,v30,v31,$keyperm
3105 lvx v30,$x10,$key1
3106 addi $key1,$key1,0x20
3107 stvx v24,$x00,$key_ # off-load round[1]
3108 ?vperm v25,v31,v30,$keyperm
3109 lvx v31,$x00,$key1
3110 stvx v25,$x10,$key_ # off-load round[2]
3111 addi $key_,$key_,0x20
3112 bdnz Load_xts_dec_key
3113
3114 lvx v26,$x10,$key1
3115 ?vperm v24,v30,v31,$keyperm
3116 lvx v27,$x20,$key1
3117 stvx v24,$x00,$key_ # off-load round[3]
3118 ?vperm v25,v31,v26,$keyperm
3119 lvx v28,$x30,$key1
3120 stvx v25,$x10,$key_ # off-load round[4]
3121 addi $key_,$sp,$FRAME+15 # rewind $key_
3122 ?vperm v26,v26,v27,$keyperm
3123 lvx v29,$x40,$key1
3124 ?vperm v27,v27,v28,$keyperm
3125 lvx v30,$x50,$key1
3126 ?vperm v28,v28,v29,$keyperm
3127 lvx v31,$x60,$key1
3128 ?vperm v29,v29,v30,$keyperm
3129 lvx $twk5,$x70,$key1 # borrow $twk5
3130 ?vperm v30,v30,v31,$keyperm
3131 lvx v24,$x00,$key_ # pre-load round[1]
3132 ?vperm v31,v31,$twk5,$keyperm
3133 lvx v25,$x10,$key_ # pre-load round[2]
3134
3135 vperm $in0,$inout,$inptail,$inpperm
3136 subi $inp,$inp,31 # undo "caller"
3137 vxor $twk0,$tweak,$rndkey0
3138 vsrab $tmp,$tweak,$seven # next tweak value
3139 vaddubm $tweak,$tweak,$tweak
3140 vsldoi $tmp,$tmp,$tmp,15
3141 vand $tmp,$tmp,$eighty7
3142 vxor $out0,$in0,$twk0
3143 vxor $tweak,$tweak,$tmp
3144
3145 lvx_u $in1,$x10,$inp
3146 vxor $twk1,$tweak,$rndkey0
3147 vsrab $tmp,$tweak,$seven # next tweak value
3148 vaddubm $tweak,$tweak,$tweak
3149 vsldoi $tmp,$tmp,$tmp,15
3150 le?vperm $in1,$in1,$in1,$leperm
3151 vand $tmp,$tmp,$eighty7
3152 vxor $out1,$in1,$twk1
3153 vxor $tweak,$tweak,$tmp
3154
3155 lvx_u $in2,$x20,$inp
3156 andi. $taillen,$len,15
3157 vxor $twk2,$tweak,$rndkey0
3158 vsrab $tmp,$tweak,$seven # next tweak value
3159 vaddubm $tweak,$tweak,$tweak
3160 vsldoi $tmp,$tmp,$tmp,15
3161 le?vperm $in2,$in2,$in2,$leperm
3162 vand $tmp,$tmp,$eighty7
3163 vxor $out2,$in2,$twk2
3164 vxor $tweak,$tweak,$tmp
3165
3166 lvx_u $in3,$x30,$inp
3167 sub $len,$len,$taillen
3168 vxor $twk3,$tweak,$rndkey0
3169 vsrab $tmp,$tweak,$seven # next tweak value
3170 vaddubm $tweak,$tweak,$tweak
3171 vsldoi $tmp,$tmp,$tmp,15
3172 le?vperm $in3,$in3,$in3,$leperm
3173 vand $tmp,$tmp,$eighty7
3174 vxor $out3,$in3,$twk3
3175 vxor $tweak,$tweak,$tmp
3176
3177 lvx_u $in4,$x40,$inp
3178 subi $len,$len,0x60
3179 vxor $twk4,$tweak,$rndkey0
3180 vsrab $tmp,$tweak,$seven # next tweak value
3181 vaddubm $tweak,$tweak,$tweak
3182 vsldoi $tmp,$tmp,$tmp,15
3183 le?vperm $in4,$in4,$in4,$leperm
3184 vand $tmp,$tmp,$eighty7
3185 vxor $out4,$in4,$twk4
3186 vxor $tweak,$tweak,$tmp
3187
3188 lvx_u $in5,$x50,$inp
3189 addi $inp,$inp,0x60
3190 vxor $twk5,$tweak,$rndkey0
3191 vsrab $tmp,$tweak,$seven # next tweak value
3192 vaddubm $tweak,$tweak,$tweak
3193 vsldoi $tmp,$tmp,$tmp,15
3194 le?vperm $in5,$in5,$in5,$leperm
3195 vand $tmp,$tmp,$eighty7
3196 vxor $out5,$in5,$twk5
3197 vxor $tweak,$tweak,$tmp
3198
3199 vxor v31,v31,$rndkey0
3200 mtctr $rounds
3201 b Loop_xts_dec6x
3202
3203 .align 5
3204 Loop_xts_dec6x:
3205 vncipher $out0,$out0,v24
3206 vncipher $out1,$out1,v24
3207 vncipher $out2,$out2,v24
3208 vncipher $out3,$out3,v24
3209 vncipher $out4,$out4,v24
3210 vncipher $out5,$out5,v24
3211 lvx v24,$x20,$key_ # round[3]
3212 addi $key_,$key_,0x20
3213
3214 vncipher $out0,$out0,v25
3215 vncipher $out1,$out1,v25
3216 vncipher $out2,$out2,v25
3217 vncipher $out3,$out3,v25
3218 vncipher $out4,$out4,v25
3219 vncipher $out5,$out5,v25
3220 lvx v25,$x10,$key_ # round[4]
3221 bdnz Loop_xts_dec6x
3222
3223 subic $len,$len,96 # $len-=96
3224 vxor $in0,$twk0,v31 # xor with last round key
3225 vncipher $out0,$out0,v24
3226 vncipher $out1,$out1,v24
3227 vsrab $tmp,$tweak,$seven # next tweak value
3228 vxor $twk0,$tweak,$rndkey0
3229 vaddubm $tweak,$tweak,$tweak
3230 vncipher $out2,$out2,v24
3231 vncipher $out3,$out3,v24
3232 vsldoi $tmp,$tmp,$tmp,15
3233 vncipher $out4,$out4,v24
3234 vncipher $out5,$out5,v24
3235
3236 subfe. r0,r0,r0 # borrow?-1:0
3237 vand $tmp,$tmp,$eighty7
3238 vncipher $out0,$out0,v25
3239 vncipher $out1,$out1,v25
3240 vxor $tweak,$tweak,$tmp
3241 vncipher $out2,$out2,v25
3242 vncipher $out3,$out3,v25
3243 vxor $in1,$twk1,v31
3244 vsrab $tmp,$tweak,$seven # next tweak value
3245 vxor $twk1,$tweak,$rndkey0
3246 vncipher $out4,$out4,v25
3247 vncipher $out5,$out5,v25
3248
3249 and r0,r0,$len
3250 vaddubm $tweak,$tweak,$tweak
3251 vsldoi $tmp,$tmp,$tmp,15
3252 vncipher $out0,$out0,v26
3253 vncipher $out1,$out1,v26
3254 vand $tmp,$tmp,$eighty7
3255 vncipher $out2,$out2,v26
3256 vncipher $out3,$out3,v26
3257 vxor $tweak,$tweak,$tmp
3258 vncipher $out4,$out4,v26
3259 vncipher $out5,$out5,v26
3260
3261 add $inp,$inp,r0 # $inp is adjusted in such
3262 # way that at exit from the
3263 # loop inX-in5 are loaded
3264 # with last "words"
3265 vxor $in2,$twk2,v31
3266 vsrab $tmp,$tweak,$seven # next tweak value
3267 vxor $twk2,$tweak,$rndkey0
3268 vaddubm $tweak,$tweak,$tweak
3269 vncipher $out0,$out0,v27
3270 vncipher $out1,$out1,v27
3271 vsldoi $tmp,$tmp,$tmp,15
3272 vncipher $out2,$out2,v27
3273 vncipher $out3,$out3,v27
3274 vand $tmp,$tmp,$eighty7
3275 vncipher $out4,$out4,v27
3276 vncipher $out5,$out5,v27
3277
3278 addi $key_,$sp,$FRAME+15 # rewind $key_
3279 vxor $tweak,$tweak,$tmp
3280 vncipher $out0,$out0,v28
3281 vncipher $out1,$out1,v28
3282 vxor $in3,$twk3,v31
3283 vsrab $tmp,$tweak,$seven # next tweak value
3284 vxor $twk3,$tweak,$rndkey0
3285 vncipher $out2,$out2,v28
3286 vncipher $out3,$out3,v28
3287 vaddubm $tweak,$tweak,$tweak
3288 vsldoi $tmp,$tmp,$tmp,15
3289 vncipher $out4,$out4,v28
3290 vncipher $out5,$out5,v28
3291 lvx v24,$x00,$key_ # re-pre-load round[1]
3292 vand $tmp,$tmp,$eighty7
3293
3294 vncipher $out0,$out0,v29
3295 vncipher $out1,$out1,v29
3296 vxor $tweak,$tweak,$tmp
3297 vncipher $out2,$out2,v29
3298 vncipher $out3,$out3,v29
3299 vxor $in4,$twk4,v31
3300 vsrab $tmp,$tweak,$seven # next tweak value
3301 vxor $twk4,$tweak,$rndkey0
3302 vncipher $out4,$out4,v29
3303 vncipher $out5,$out5,v29
3304 lvx v25,$x10,$key_ # re-pre-load round[2]
3305 vaddubm $tweak,$tweak,$tweak
3306 vsldoi $tmp,$tmp,$tmp,15
3307
3308 vncipher $out0,$out0,v30
3309 vncipher $out1,$out1,v30
3310 vand $tmp,$tmp,$eighty7
3311 vncipher $out2,$out2,v30
3312 vncipher $out3,$out3,v30
3313 vxor $tweak,$tweak,$tmp
3314 vncipher $out4,$out4,v30
3315 vncipher $out5,$out5,v30
3316 vxor $in5,$twk5,v31
3317 vsrab $tmp,$tweak,$seven # next tweak value
3318 vxor $twk5,$tweak,$rndkey0
3319
3320 vncipherlast $out0,$out0,$in0
3321 lvx_u $in0,$x00,$inp # load next input block
3322 vaddubm $tweak,$tweak,$tweak
3323 vsldoi $tmp,$tmp,$tmp,15
3324 vncipherlast $out1,$out1,$in1
3325 lvx_u $in1,$x10,$inp
3326 vncipherlast $out2,$out2,$in2
3327 le?vperm $in0,$in0,$in0,$leperm
3328 lvx_u $in2,$x20,$inp
3329 vand $tmp,$tmp,$eighty7
3330 vncipherlast $out3,$out3,$in3
3331 le?vperm $in1,$in1,$in1,$leperm
3332 lvx_u $in3,$x30,$inp
3333 vncipherlast $out4,$out4,$in4
3334 le?vperm $in2,$in2,$in2,$leperm
3335 lvx_u $in4,$x40,$inp
3336 vxor $tweak,$tweak,$tmp
3337 vncipherlast $out5,$out5,$in5
3338 le?vperm $in3,$in3,$in3,$leperm
3339 lvx_u $in5,$x50,$inp
3340 addi $inp,$inp,0x60
3341 le?vperm $in4,$in4,$in4,$leperm
3342 le?vperm $in5,$in5,$in5,$leperm
3343
3344 le?vperm $out0,$out0,$out0,$leperm
3345 le?vperm $out1,$out1,$out1,$leperm
3346 stvx_u $out0,$x00,$out # store output
3347 vxor $out0,$in0,$twk0
3348 le?vperm $out2,$out2,$out2,$leperm
3349 stvx_u $out1,$x10,$out
3350 vxor $out1,$in1,$twk1
3351 le?vperm $out3,$out3,$out3,$leperm
3352 stvx_u $out2,$x20,$out
3353 vxor $out2,$in2,$twk2
3354 le?vperm $out4,$out4,$out4,$leperm
3355 stvx_u $out3,$x30,$out
3356 vxor $out3,$in3,$twk3
3357 le?vperm $out5,$out5,$out5,$leperm
3358 stvx_u $out4,$x40,$out
3359 vxor $out4,$in4,$twk4
3360 stvx_u $out5,$x50,$out
3361 vxor $out5,$in5,$twk5
3362 addi $out,$out,0x60
3363
3364 mtctr $rounds
3365 beq Loop_xts_dec6x # did $len-=96 borrow?
3366
3367 addic. $len,$len,0x60
3368 beq Lxts_dec6x_zero
3369 cmpwi $len,0x20
3370 blt Lxts_dec6x_one
3371 nop
3372 beq Lxts_dec6x_two
3373 cmpwi $len,0x40
3374 blt Lxts_dec6x_three
3375 nop
3376 beq Lxts_dec6x_four
3377
3378 Lxts_dec6x_five:
3379 vxor $out0,$in1,$twk0
3380 vxor $out1,$in2,$twk1
3381 vxor $out2,$in3,$twk2
3382 vxor $out3,$in4,$twk3
3383 vxor $out4,$in5,$twk4
3384
3385 bl _aesp8_xts_dec5x
3386
3387 le?vperm $out0,$out0,$out0,$leperm
3388 vmr $twk0,$twk5 # unused tweak
3389 vxor $twk1,$tweak,$rndkey0
3390 le?vperm $out1,$out1,$out1,$leperm
3391 stvx_u $out0,$x00,$out # store output
3392 vxor $out0,$in0,$twk1
3393 le?vperm $out2,$out2,$out2,$leperm
3394 stvx_u $out1,$x10,$out
3395 le?vperm $out3,$out3,$out3,$leperm
3396 stvx_u $out2,$x20,$out
3397 le?vperm $out4,$out4,$out4,$leperm
3398 stvx_u $out3,$x30,$out
3399 stvx_u $out4,$x40,$out
3400 addi $out,$out,0x50
3401 bne Lxts_dec6x_steal
3402 b Lxts_dec6x_done
3403
3404 .align 4
3405 Lxts_dec6x_four:
3406 vxor $out0,$in2,$twk0
3407 vxor $out1,$in3,$twk1
3408 vxor $out2,$in4,$twk2
3409 vxor $out3,$in5,$twk3
3410 vxor $out4,$out4,$out4
3411
3412 bl _aesp8_xts_dec5x
3413
3414 le?vperm $out0,$out0,$out0,$leperm
3415 vmr $twk0,$twk4 # unused tweak
3416 vmr $twk1,$twk5
3417 le?vperm $out1,$out1,$out1,$leperm
3418 stvx_u $out0,$x00,$out # store output
3419 vxor $out0,$in0,$twk5
3420 le?vperm $out2,$out2,$out2,$leperm
3421 stvx_u $out1,$x10,$out
3422 le?vperm $out3,$out3,$out3,$leperm
3423 stvx_u $out2,$x20,$out
3424 stvx_u $out3,$x30,$out
3425 addi $out,$out,0x40
3426 bne Lxts_dec6x_steal
3427 b Lxts_dec6x_done
3428
3429 .align 4
3430 Lxts_dec6x_three:
3431 vxor $out0,$in3,$twk0
3432 vxor $out1,$in4,$twk1
3433 vxor $out2,$in5,$twk2
3434 vxor $out3,$out3,$out3
3435 vxor $out4,$out4,$out4
3436
3437 bl _aesp8_xts_dec5x
3438
3439 le?vperm $out0,$out0,$out0,$leperm
3440 vmr $twk0,$twk3 # unused tweak
3441 vmr $twk1,$twk4
3442 le?vperm $out1,$out1,$out1,$leperm
3443 stvx_u $out0,$x00,$out # store output
3444 vxor $out0,$in0,$twk4
3445 le?vperm $out2,$out2,$out2,$leperm
3446 stvx_u $out1,$x10,$out
3447 stvx_u $out2,$x20,$out
3448 addi $out,$out,0x30
3449 bne Lxts_dec6x_steal
3450 b Lxts_dec6x_done
3451
3452 .align 4
3453 Lxts_dec6x_two:
3454 vxor $out0,$in4,$twk0
3455 vxor $out1,$in5,$twk1
3456 vxor $out2,$out2,$out2
3457 vxor $out3,$out3,$out3
3458 vxor $out4,$out4,$out4
3459
3460 bl _aesp8_xts_dec5x
3461
3462 le?vperm $out0,$out0,$out0,$leperm
3463 vmr $twk0,$twk2 # unused tweak
3464 vmr $twk1,$twk3
3465 le?vperm $out1,$out1,$out1,$leperm
3466 stvx_u $out0,$x00,$out # store output
3467 vxor $out0,$in0,$twk3
3468 stvx_u $out1,$x10,$out
3469 addi $out,$out,0x20
3470 bne Lxts_dec6x_steal
3471 b Lxts_dec6x_done
3472
3473 .align 4
3474 Lxts_dec6x_one:
3475 vxor $out0,$in5,$twk0
3476 nop
3477 Loop_xts_dec1x:
3478 vncipher $out0,$out0,v24
3479 lvx v24,$x20,$key_ # round[3]
3480 addi $key_,$key_,0x20
3481
3482 vncipher $out0,$out0,v25
3483 lvx v25,$x10,$key_ # round[4]
3484 bdnz Loop_xts_dec1x
3485
3486 subi r0,$taillen,1
3487 vncipher $out0,$out0,v24
3488
3489 andi. r0,r0,16
3490 cmpwi $taillen,0
3491 vncipher $out0,$out0,v25
3492
3493 sub $inp,$inp,r0
3494 vncipher $out0,$out0,v26
3495
3496 lvx_u $in0,0,$inp
3497 vncipher $out0,$out0,v27
3498
3499 addi $key_,$sp,$FRAME+15 # rewind $key_
3500 vncipher $out0,$out0,v28
3501 lvx v24,$x00,$key_ # re-pre-load round[1]
3502
3503 vncipher $out0,$out0,v29
3504 lvx v25,$x10,$key_ # re-pre-load round[2]
3505 vxor $twk0,$twk0,v31
3506
3507 le?vperm $in0,$in0,$in0,$leperm
3508 vncipher $out0,$out0,v30
3509
3510 mtctr $rounds
3511 vncipherlast $out0,$out0,$twk0
3512
3513 vmr $twk0,$twk1 # unused tweak
3514 vmr $twk1,$twk2
3515 le?vperm $out0,$out0,$out0,$leperm
3516 stvx_u $out0,$x00,$out # store output
3517 addi $out,$out,0x10
3518 vxor $out0,$in0,$twk2
3519 bne Lxts_dec6x_steal
3520 b Lxts_dec6x_done
3521
3522 .align 4
3523 Lxts_dec6x_zero:
3524 cmpwi $taillen,0
3525 beq Lxts_dec6x_done
3526
3527 lvx_u $in0,0,$inp
3528 le?vperm $in0,$in0,$in0,$leperm
3529 vxor $out0,$in0,$twk1
3530 Lxts_dec6x_steal:
3531 vncipher $out0,$out0,v24
3532 lvx v24,$x20,$key_ # round[3]
3533 addi $key_,$key_,0x20
3534
3535 vncipher $out0,$out0,v25
3536 lvx v25,$x10,$key_ # round[4]
3537 bdnz Lxts_dec6x_steal
3538
3539 add $inp,$inp,$taillen
3540 vncipher $out0,$out0,v24
3541
3542 cmpwi $taillen,0
3543 vncipher $out0,$out0,v25
3544
3545 lvx_u $in0,0,$inp
3546 vncipher $out0,$out0,v26
3547
3548 lvsr $inpperm,0,$taillen # $in5 is no more
3549 vncipher $out0,$out0,v27
3550
3551 addi $key_,$sp,$FRAME+15 # rewind $key_
3552 vncipher $out0,$out0,v28
3553 lvx v24,$x00,$key_ # re-pre-load round[1]
3554
3555 vncipher $out0,$out0,v29
3556 lvx v25,$x10,$key_ # re-pre-load round[2]
3557 vxor $twk1,$twk1,v31
3558
3559 le?vperm $in0,$in0,$in0,$leperm
3560 vncipher $out0,$out0,v30
3561
3562 vperm $in0,$in0,$in0,$inpperm
3563 vncipherlast $tmp,$out0,$twk1
3564
3565 le?vperm $out0,$tmp,$tmp,$leperm
3566 le?stvx_u $out0,0,$out
3567 be?stvx_u $tmp,0,$out
3568
3569 vxor $out0,$out0,$out0
3570 vspltisb $out1,-1
3571 vperm $out0,$out0,$out1,$inpperm
3572 vsel $out0,$in0,$tmp,$out0
3573 vxor $out0,$out0,$twk0
3574
3575 subi r30,$out,1
3576 mtctr $taillen
3577 Loop_xts_dec6x_steal:
3578 lbzu r0,1(r30)
3579 stb r0,16(r30)
3580 bdnz Loop_xts_dec6x_steal
3581
3582 li $taillen,0
3583 mtctr $rounds
3584 b Loop_xts_dec1x # one more time...
3585
3586 .align 4
3587 Lxts_dec6x_done:
3588 ${UCMP}i $ivp,0
3589 beq Lxts_dec6x_ret
3590
3591 vxor $tweak,$twk0,$rndkey0
3592 le?vperm $tweak,$tweak,$tweak,$leperm
3593 stvx_u $tweak,0,$ivp
3594
3595 Lxts_dec6x_ret:
3596 mtlr r11
3597 li r10,`$FRAME+15`
3598 li r11,`$FRAME+31`
3599 stvx $seven,r10,$sp # wipe copies of round keys
3600 addi r10,r10,32
3601 stvx $seven,r11,$sp
3602 addi r11,r11,32
3603 stvx $seven,r10,$sp
3604 addi r10,r10,32
3605 stvx $seven,r11,$sp
3606 addi r11,r11,32
3607 stvx $seven,r10,$sp
3608 addi r10,r10,32
3609 stvx $seven,r11,$sp
3610 addi r11,r11,32
3611 stvx $seven,r10,$sp
3612 addi r10,r10,32
3613 stvx $seven,r11,$sp
3614 addi r11,r11,32
3615
3616 mtspr 256,$vrsave
3617 lvx v20,r10,$sp # ABI says so
3618 addi r10,r10,32
3619 lvx v21,r11,$sp
3620 addi r11,r11,32
3621 lvx v22,r10,$sp
3622 addi r10,r10,32
3623 lvx v23,r11,$sp
3624 addi r11,r11,32
3625 lvx v24,r10,$sp
3626 addi r10,r10,32
3627 lvx v25,r11,$sp
3628 addi r11,r11,32
3629 lvx v26,r10,$sp
3630 addi r10,r10,32
3631 lvx v27,r11,$sp
3632 addi r11,r11,32
3633 lvx v28,r10,$sp
3634 addi r10,r10,32
3635 lvx v29,r11,$sp
3636 addi r11,r11,32
3637 lvx v30,r10,$sp
3638 lvx v31,r11,$sp
3639 $POP r26,`$FRAME+21*16+0*$SIZE_T`($sp)
3640 $POP r27,`$FRAME+21*16+1*$SIZE_T`($sp)
3641 $POP r28,`$FRAME+21*16+2*$SIZE_T`($sp)
3642 $POP r29,`$FRAME+21*16+3*$SIZE_T`($sp)
3643 $POP r30,`$FRAME+21*16+4*$SIZE_T`($sp)
3644 $POP r31,`$FRAME+21*16+5*$SIZE_T`($sp)
3645 addi $sp,$sp,`$FRAME+21*16+6*$SIZE_T`
3646 blr
3647 .long 0
3648 .byte 0,12,0x04,1,0x80,6,6,0
3649 .long 0
3650
3651 .align 5
3652 _aesp8_xts_dec5x:
3653 vncipher $out0,$out0,v24
3654 vncipher $out1,$out1,v24
3655 vncipher $out2,$out2,v24
3656 vncipher $out3,$out3,v24
3657 vncipher $out4,$out4,v24
3658 lvx v24,$x20,$key_ # round[3]
3659 addi $key_,$key_,0x20
3660
3661 vncipher $out0,$out0,v25
3662 vncipher $out1,$out1,v25
3663 vncipher $out2,$out2,v25
3664 vncipher $out3,$out3,v25
3665 vncipher $out4,$out4,v25
3666 lvx v25,$x10,$key_ # round[4]
3667 bdnz _aesp8_xts_dec5x
3668
3669 subi r0,$taillen,1
3670 vncipher $out0,$out0,v24
3671 vncipher $out1,$out1,v24
3672 vncipher $out2,$out2,v24
3673 vncipher $out3,$out3,v24
3674 vncipher $out4,$out4,v24
3675
3676 andi. r0,r0,16
3677 cmpwi $taillen,0
3678 vncipher $out0,$out0,v25
3679 vncipher $out1,$out1,v25
3680 vncipher $out2,$out2,v25
3681 vncipher $out3,$out3,v25
3682 vncipher $out4,$out4,v25
3683 vxor $twk0,$twk0,v31
3684
3685 sub $inp,$inp,r0
3686 vncipher $out0,$out0,v26
3687 vncipher $out1,$out1,v26
3688 vncipher $out2,$out2,v26
3689 vncipher $out3,$out3,v26
3690 vncipher $out4,$out4,v26
3691 vxor $in1,$twk1,v31
3692
3693 vncipher $out0,$out0,v27
3694 lvx_u $in0,0,$inp
3695 vncipher $out1,$out1,v27
3696 vncipher $out2,$out2,v27
3697 vncipher $out3,$out3,v27
3698 vncipher $out4,$out4,v27
3699 vxor $in2,$twk2,v31
3700
3701 addi $key_,$sp,$FRAME+15 # rewind $key_
3702 vncipher $out0,$out0,v28
3703 vncipher $out1,$out1,v28
3704 vncipher $out2,$out2,v28
3705 vncipher $out3,$out3,v28
3706 vncipher $out4,$out4,v28
3707 lvx v24,$x00,$key_ # re-pre-load round[1]
3708 vxor $in3,$twk3,v31
3709
3710 vncipher $out0,$out0,v29
3711 le?vperm $in0,$in0,$in0,$leperm
3712 vncipher $out1,$out1,v29
3713 vncipher $out2,$out2,v29
3714 vncipher $out3,$out3,v29
3715 vncipher $out4,$out4,v29
3716 lvx v25,$x10,$key_ # re-pre-load round[2]
3717 vxor $in4,$twk4,v31
3718
3719 vncipher $out0,$out0,v30
3720 vncipher $out1,$out1,v30
3721 vncipher $out2,$out2,v30
3722 vncipher $out3,$out3,v30
3723 vncipher $out4,$out4,v30
3724
3725 vncipherlast $out0,$out0,$twk0
3726 vncipherlast $out1,$out1,$in1
3727 vncipherlast $out2,$out2,$in2
3728 vncipherlast $out3,$out3,$in3
3729 vncipherlast $out4,$out4,$in4
3730 mtctr $rounds
3731 blr
3732 .long 0
3733 .byte 0,12,0x14,0,0,0,0,0
3734 ___
3735 }} }}}
3736
3737 my $consts=1;
3738 foreach(split("\n",$code)) {
3739 s/\`([^\`]*)\`/eval($1)/geo;
3740
3741 # constants table endian-specific conversion
3742 if ($consts && m/\.(long|byte)\s+(.+)\s+(\?[a-z]*)$/o) {
3743 my $conv=$3;
3744 my @bytes=();
3745
3746 # convert to endian-agnostic format
3747 if ($1 eq "long") {
3748 foreach (split(/,\s*/,$2)) {
3749 my $l = /^0/?oct:int;
3750 push @bytes,($l>>24)&0xff,($l>>16)&0xff,($l>>8)&0xff,$l&0xff;
3751 }
3752 } else {
3753 @bytes = map(/^0/?oct:int,split(/,\s*/,$2));
3754 }
3755
3756 # little-endian conversion
3757 if ($flavour =~ /le$/o) {
3758 SWITCH: for($conv) {
3759 /\?inv/ && do { @bytes=map($_^0xf,@bytes); last; };
3760 /\?rev/ && do { @bytes=reverse(@bytes); last; };
3761 }
3762 }
3763
3764 #emit
3765 print ".byte\t",join(',',map (sprintf("0x%02x",$_),@bytes)),"\n";
3766 next;
3767 }
3768 $consts=0 if (m/Lconsts:/o); # end of table
3769
3770 # instructions prefixed with '?' are endian-specific and need
3771 # to be adjusted accordingly...
3772 if ($flavour =~ /le$/o) { # little-endian
3773 s/le\?//o or
3774 s/be\?/#be#/o or
3775 s/\?lvsr/lvsl/o or
3776 s/\?lvsl/lvsr/o or
3777 s/\?(vperm\s+v[0-9]+,\s*)(v[0-9]+,\s*)(v[0-9]+,\s*)(v[0-9]+)/$1$3$2$4/o or
3778 s/\?(vsldoi\s+v[0-9]+,\s*)(v[0-9]+,)\s*(v[0-9]+,\s*)([0-9]+)/$1$3$2 16-$4/o or
3779 s/\?(vspltw\s+v[0-9]+,\s*)(v[0-9]+,)\s*([0-9])/$1$2 3-$3/o;
3780 } else { # big-endian
3781 s/le\?/#le#/o or
3782 s/be\?//o or
3783 s/\?([a-z]+)/$1/o;
3784 }
3785
3786 print $_,"\n";
3787 }
3788
3789 close STDOUT;