projects / GitHub / MotorolaMobilityLLC / kernel-slsi.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
License cleanup: add SPDX GPL-2.0 license identifier to files with no license
[GitHub/MotorolaMobilityLLC/kernel-slsi.git] / arch / sparc / net / bpf_jit_comp_64.c
1 // SPDX-License-Identifier: GPL-2.0
2 #include <linux/moduleloader.h>
3 #include <linux/workqueue.h>
4 #include <linux/netdevice.h>
5 #include <linux/filter.h>
6 #include <linux/bpf.h>
7 #include <linux/cache.h>
8 #include <linux/if_vlan.h>
9
10 #include <asm/cacheflush.h>
11 #include <asm/ptrace.h>
12
13 #include "bpf_jit_64.h"
14
15 int bpf_jit_enable __read_mostly;
16
17 static inline bool is_simm13(unsigned int value)
18 {
19 return value + 0x1000 < 0x2000;
20 }
21
22 static inline bool is_simm10(unsigned int value)
23 {
24 return value + 0x200 < 0x400;
25 }
26
27 static inline bool is_simm5(unsigned int value)
28 {
29 return value + 0x10 < 0x20;
30 }
31
32 static inline bool is_sethi(unsigned int value)
33 {
34 return (value & ~0x3fffff) == 0;
35 }
36
37 static void bpf_flush_icache(void *start_, void *end_)
38 {
39 /* Cheetah's I-cache is fully coherent. */
40 if (tlb_type == spitfire) {
41 unsigned long start = (unsigned long) start_;
42 unsigned long end = (unsigned long) end_;
43
44 start &= ~7UL;
45 end = (end + 7UL) & ~7UL;
46 while (start < end) {
47 flushi(start);
48 start += 32;
49 }
50 }
51 }
52
53 #define SEEN_DATAREF 1 /* might call external helpers */
54 #define SEEN_XREG 2 /* ebx is used */
55 #define SEEN_MEM 4 /* use mem[] for temporary storage */
56
57 #define S13(X) ((X) & 0x1fff)
58 #define S5(X) ((X) & 0x1f)
59 #define IMMED 0x00002000
60 #define RD(X) ((X) << 25)
61 #define RS1(X) ((X) << 14)
62 #define RS2(X) ((X))
63 #define OP(X) ((X) << 30)
64 #define OP2(X) ((X) << 22)
65 #define OP3(X) ((X) << 19)
66 #define COND(X) (((X) & 0xf) << 25)
67 #define CBCOND(X) (((X) & 0x1f) << 25)
68 #define F1(X) OP(X)
69 #define F2(X, Y) (OP(X) | OP2(Y))
70 #define F3(X, Y) (OP(X) | OP3(Y))
71 #define ASI(X) (((X) & 0xff) << 5)
72
73 #define CONDN COND(0x0)
74 #define CONDE COND(0x1)
75 #define CONDLE COND(0x2)
76 #define CONDL COND(0x3)
77 #define CONDLEU COND(0x4)
78 #define CONDCS COND(0x5)
79 #define CONDNEG COND(0x6)
80 #define CONDVC COND(0x7)
81 #define CONDA COND(0x8)
82 #define CONDNE COND(0x9)
83 #define CONDG COND(0xa)
84 #define CONDGE COND(0xb)
85 #define CONDGU COND(0xc)
86 #define CONDCC COND(0xd)
87 #define CONDPOS COND(0xe)
88 #define CONDVS COND(0xf)
89
90 #define CONDGEU CONDCC
91 #define CONDLU CONDCS
92
93 #define WDISP22(X) (((X) >> 2) & 0x3fffff)
94 #define WDISP19(X) (((X) >> 2) & 0x7ffff)
95
96 /* The 10-bit branch displacement for CBCOND is split into two fields */
97 static u32 WDISP10(u32 off)
98 {
99 u32 ret = ((off >> 2) & 0xff) << 5;
100
101 ret |= ((off >> (2 + 8)) & 0x03) << 19;
102
103 return ret;
104 }
105
106 #define CBCONDE CBCOND(0x09)
107 #define CBCONDLE CBCOND(0x0a)
108 #define CBCONDL CBCOND(0x0b)
109 #define CBCONDLEU CBCOND(0x0c)
110 #define CBCONDCS CBCOND(0x0d)
111 #define CBCONDN CBCOND(0x0e)
112 #define CBCONDVS CBCOND(0x0f)
113 #define CBCONDNE CBCOND(0x19)
114 #define CBCONDG CBCOND(0x1a)
115 #define CBCONDGE CBCOND(0x1b)
116 #define CBCONDGU CBCOND(0x1c)
117 #define CBCONDCC CBCOND(0x1d)
118 #define CBCONDPOS CBCOND(0x1e)
119 #define CBCONDVC CBCOND(0x1f)
120
121 #define CBCONDGEU CBCONDCC
122 #define CBCONDLU CBCONDCS
123
124 #define ANNUL (1 << 29)
125 #define XCC (1 << 21)
126
127 #define BRANCH (F2(0, 1) | XCC)
128 #define CBCOND_OP (F2(0, 3) | XCC)
129
130 #define BA (BRANCH | CONDA)
131 #define BG (BRANCH | CONDG)
132 #define BL (BRANCH | CONDL)
133 #define BLE (BRANCH | CONDLE)
134 #define BGU (BRANCH | CONDGU)
135 #define BLEU (BRANCH | CONDLEU)
136 #define BGE (BRANCH | CONDGE)
137 #define BGEU (BRANCH | CONDGEU)
138 #define BLU (BRANCH | CONDLU)
139 #define BE (BRANCH | CONDE)
140 #define BNE (BRANCH | CONDNE)
141
142 #define SETHI(K, REG) \
143 (F2(0, 0x4) | RD(REG) | (((K) >> 10) & 0x3fffff))
144 #define OR_LO(K, REG) \
145 (F3(2, 0x02) | IMMED | RS1(REG) | ((K) & 0x3ff) | RD(REG))
146
147 #define ADD F3(2, 0x00)
148 #define AND F3(2, 0x01)
149 #define ANDCC F3(2, 0x11)
150 #define OR F3(2, 0x02)
151 #define XOR F3(2, 0x03)
152 #define SUB F3(2, 0x04)
153 #define SUBCC F3(2, 0x14)
154 #define MUL F3(2, 0x0a)
155 #define MULX F3(2, 0x09)
156 #define UDIVX F3(2, 0x0d)
157 #define DIV F3(2, 0x0e)
158 #define SLL F3(2, 0x25)
159 #define SLLX (F3(2, 0x25)|(1<<12))
160 #define SRA F3(2, 0x27)
161 #define SRAX (F3(2, 0x27)|(1<<12))
162 #define SRL F3(2, 0x26)
163 #define SRLX (F3(2, 0x26)|(1<<12))
164 #define JMPL F3(2, 0x38)
165 #define SAVE F3(2, 0x3c)
166 #define RESTORE F3(2, 0x3d)
167 #define CALL F1(1)
168 #define BR F2(0, 0x01)
169 #define RD_Y F3(2, 0x28)
170 #define WR_Y F3(2, 0x30)
171
172 #define LD32 F3(3, 0x00)
173 #define LD8 F3(3, 0x01)
174 #define LD16 F3(3, 0x02)
175 #define LD64 F3(3, 0x0b)
176 #define LD64A F3(3, 0x1b)
177 #define ST8 F3(3, 0x05)
178 #define ST16 F3(3, 0x06)
179 #define ST32 F3(3, 0x04)
180 #define ST64 F3(3, 0x0e)
181
182 #define CAS F3(3, 0x3c)
183 #define CASX F3(3, 0x3e)
184
185 #define LDPTR LD64
186 #define BASE_STACKFRAME 176
187
188 #define LD32I (LD32 | IMMED)
189 #define LD8I (LD8 | IMMED)
190 #define LD16I (LD16 | IMMED)
191 #define LD64I (LD64 | IMMED)
192 #define LDPTRI (LDPTR | IMMED)
193 #define ST32I (ST32 | IMMED)
194
195 struct jit_ctx {
196 struct bpf_prog *prog;
197 unsigned int *offset;
198 int idx;
199 int epilogue_offset;
200 bool tmp_1_used;
201 bool tmp_2_used;
202 bool tmp_3_used;
203 bool saw_ld_abs_ind;
204 bool saw_frame_pointer;
205 bool saw_call;
206 bool saw_tail_call;
207 u32 *image;
208 };
209
210 #define TMP_REG_1 (MAX_BPF_JIT_REG + 0)
211 #define TMP_REG_2 (MAX_BPF_JIT_REG + 1)
212 #define SKB_HLEN_REG (MAX_BPF_JIT_REG + 2)
213 #define SKB_DATA_REG (MAX_BPF_JIT_REG + 3)
214 #define TMP_REG_3 (MAX_BPF_JIT_REG + 4)
215
216 /* Map BPF registers to SPARC registers */
217 static const int bpf2sparc[] = {
218 /* return value from in-kernel function, and exit value from eBPF */
219 [BPF_REG_0] = O5,
220
221 /* arguments from eBPF program to in-kernel function */
222 [BPF_REG_1] = O0,
223 [BPF_REG_2] = O1,
224 [BPF_REG_3] = O2,
225 [BPF_REG_4] = O3,
226 [BPF_REG_5] = O4,
227
228 /* callee saved registers that in-kernel function will preserve */
229 [BPF_REG_6] = L0,
230 [BPF_REG_7] = L1,
231 [BPF_REG_8] = L2,
232 [BPF_REG_9] = L3,
233
234 /* read-only frame pointer to access stack */
235 [BPF_REG_FP] = L6,
236
237 [BPF_REG_AX] = G7,
238
239 /* temporary register for internal BPF JIT */
240 [TMP_REG_1] = G1,
241 [TMP_REG_2] = G2,
242 [TMP_REG_3] = G3,
243
244 [SKB_HLEN_REG] = L4,
245 [SKB_DATA_REG] = L5,
246 };
247
248 static void emit(const u32 insn, struct jit_ctx *ctx)
249 {
250 if (ctx->image != NULL)
251 ctx->image[ctx->idx] = insn;
252
253 ctx->idx++;
254 }
255
256 static void emit_call(u32 *func, struct jit_ctx *ctx)
257 {
258 if (ctx->image != NULL) {
259 void *here = &ctx->image[ctx->idx];
260 unsigned int off;
261
262 off = (void *)func - here;
263 ctx->image[ctx->idx] = CALL | ((off >> 2) & 0x3fffffff);
264 }
265 ctx->idx++;
266 }
267
268 static void emit_nop(struct jit_ctx *ctx)
269 {
270 emit(SETHI(0, G0), ctx);
271 }
272
273 static void emit_reg_move(u32 from, u32 to, struct jit_ctx *ctx)
274 {
275 emit(OR | RS1(G0) | RS2(from) | RD(to), ctx);
276 }
277
278 /* Emit 32-bit constant, zero extended. */
279 static void emit_set_const(s32 K, u32 reg, struct jit_ctx *ctx)
280 {
281 emit(SETHI(K, reg), ctx);
282 emit(OR_LO(K, reg), ctx);
283 }
284
285 /* Emit 32-bit constant, sign extended. */
286 static void emit_set_const_sext(s32 K, u32 reg, struct jit_ctx *ctx)
287 {
288 if (K >= 0) {
289 emit(SETHI(K, reg), ctx);
290 emit(OR_LO(K, reg), ctx);
291 } else {
292 u32 hbits = ~(u32) K;
293 u32 lbits = -0x400 | (u32) K;
294
295 emit(SETHI(hbits, reg), ctx);
296 emit(XOR | IMMED | RS1(reg) | S13(lbits) | RD(reg), ctx);
297 }
298 }
299
300 static void emit_alu(u32 opcode, u32 src, u32 dst, struct jit_ctx *ctx)
301 {
302 emit(opcode | RS1(dst) | RS2(src) | RD(dst), ctx);
303 }
304
305 static void emit_alu3(u32 opcode, u32 a, u32 b, u32 c, struct jit_ctx *ctx)
306 {
307 emit(opcode | RS1(a) | RS2(b) | RD(c), ctx);
308 }
309
310 static void emit_alu_K(unsigned int opcode, unsigned int dst, unsigned int imm,
311 struct jit_ctx *ctx)
312 {
313 bool small_immed = is_simm13(imm);
314 unsigned int insn = opcode;
315
316 insn |= RS1(dst) | RD(dst);
317 if (small_immed) {
318 emit(insn | IMMED | S13(imm), ctx);
319 } else {
320 unsigned int tmp = bpf2sparc[TMP_REG_1];
321
322 ctx->tmp_1_used = true;
323
324 emit_set_const_sext(imm, tmp, ctx);
325 emit(insn | RS2(tmp), ctx);
326 }
327 }
328
329 static void emit_alu3_K(unsigned int opcode, unsigned int src, unsigned int imm,
330 unsigned int dst, struct jit_ctx *ctx)
331 {
332 bool small_immed = is_simm13(imm);
333 unsigned int insn = opcode;
334
335 insn |= RS1(src) | RD(dst);
336 if (small_immed) {
337 emit(insn | IMMED | S13(imm), ctx);
338 } else {
339 unsigned int tmp = bpf2sparc[TMP_REG_1];
340
341 ctx->tmp_1_used = true;
342
343 emit_set_const_sext(imm, tmp, ctx);
344 emit(insn | RS2(tmp), ctx);
345 }
346 }
347
348 static void emit_loadimm32(s32 K, unsigned int dest, struct jit_ctx *ctx)
349 {
350 if (K >= 0 && is_simm13(K)) {
351 /* or %g0, K, DEST */
352 emit(OR | IMMED | RS1(G0) | S13(K) | RD(dest), ctx);
353 } else {
354 emit_set_const(K, dest, ctx);
355 }
356 }
357
358 static void emit_loadimm(s32 K, unsigned int dest, struct jit_ctx *ctx)
359 {
360 if (is_simm13(K)) {
361 /* or %g0, K, DEST */
362 emit(OR | IMMED | RS1(G0) | S13(K) | RD(dest), ctx);
363 } else {
364 emit_set_const(K, dest, ctx);
365 }
366 }
367
368 static void emit_loadimm_sext(s32 K, unsigned int dest, struct jit_ctx *ctx)
369 {
370 if (is_simm13(K)) {
371 /* or %g0, K, DEST */
372 emit(OR | IMMED | RS1(G0) | S13(K) | RD(dest), ctx);
373 } else {
374 emit_set_const_sext(K, dest, ctx);
375 }
376 }
377
378 static void analyze_64bit_constant(u32 high_bits, u32 low_bits,
379 int *hbsp, int *lbsp, int *abbasp)
380 {
381 int lowest_bit_set, highest_bit_set, all_bits_between_are_set;
382 int i;
383
384 lowest_bit_set = highest_bit_set = -1;
385 i = 0;
386 do {
387 if ((lowest_bit_set == -1) && ((low_bits >> i) & 1))
388 lowest_bit_set = i;
389 if ((highest_bit_set == -1) && ((high_bits >> (32 - i - 1)) & 1))
390 highest_bit_set = (64 - i - 1);
391 } while (++i < 32 && (highest_bit_set == -1 ||
392 lowest_bit_set == -1));
393 if (i == 32) {
394 i = 0;
395 do {
396 if (lowest_bit_set == -1 && ((high_bits >> i) & 1))
397 lowest_bit_set = i + 32;
398 if (highest_bit_set == -1 &&
399 ((low_bits >> (32 - i - 1)) & 1))
400 highest_bit_set = 32 - i - 1;
401 } while (++i < 32 && (highest_bit_set == -1 ||
402 lowest_bit_set == -1));
403 }
404
405 all_bits_between_are_set = 1;
406 for (i = lowest_bit_set; i <= highest_bit_set; i++) {
407 if (i < 32) {
408 if ((low_bits & (1 << i)) != 0)
409 continue;
410 } else {
411 if ((high_bits & (1 << (i - 32))) != 0)
412 continue;
413 }
414 all_bits_between_are_set = 0;
415 break;
416 }
417 *hbsp = highest_bit_set;
418 *lbsp = lowest_bit_set;
419 *abbasp = all_bits_between_are_set;
420 }
421
422 static unsigned long create_simple_focus_bits(unsigned long high_bits,
423 unsigned long low_bits,
424 int lowest_bit_set, int shift)
425 {
426 long hi, lo;
427
428 if (lowest_bit_set < 32) {
429 lo = (low_bits >> lowest_bit_set) << shift;
430 hi = ((high_bits << (32 - lowest_bit_set)) << shift);
431 } else {
432 lo = 0;
433 hi = ((high_bits >> (lowest_bit_set - 32)) << shift);
434 }
435 return hi | lo;
436 }
437
438 static bool const64_is_2insns(unsigned long high_bits,
439 unsigned long low_bits)
440 {
441 int highest_bit_set, lowest_bit_set, all_bits_between_are_set;
442
443 if (high_bits == 0 || high_bits == 0xffffffff)
444 return true;
445
446 analyze_64bit_constant(high_bits, low_bits,
447 &highest_bit_set, &lowest_bit_set,
448 &all_bits_between_are_set);
449
450 if ((highest_bit_set == 63 || lowest_bit_set == 0) &&
451 all_bits_between_are_set != 0)
452 return true;
453
454 if (highest_bit_set - lowest_bit_set < 21)
455 return true;
456
457 return false;
458 }
459
460 static void sparc_emit_set_const64_quick2(unsigned long high_bits,
461 unsigned long low_imm,
462 unsigned int dest,
463 int shift_count, struct jit_ctx *ctx)
464 {
465 emit_loadimm32(high_bits, dest, ctx);
466
467 /* Now shift it up into place. */
468 emit_alu_K(SLLX, dest, shift_count, ctx);
469
470 /* If there is a low immediate part piece, finish up by
471 * putting that in as well.
472 */
473 if (low_imm != 0)
474 emit(OR | IMMED | RS1(dest) | S13(low_imm) | RD(dest), ctx);
475 }
476
477 static void emit_loadimm64(u64 K, unsigned int dest, struct jit_ctx *ctx)
478 {
479 int all_bits_between_are_set, lowest_bit_set, highest_bit_set;
480 unsigned int tmp = bpf2sparc[TMP_REG_1];
481 u32 low_bits = (K & 0xffffffff);
482 u32 high_bits = (K >> 32);
483
484 /* These two tests also take care of all of the one
485 * instruction cases.
486 */
487 if (high_bits == 0xffffffff && (low_bits & 0x80000000))
488 return emit_loadimm_sext(K, dest, ctx);
489 if (high_bits == 0x00000000)
490 return emit_loadimm32(K, dest, ctx);
491
492 analyze_64bit_constant(high_bits, low_bits, &highest_bit_set,
493 &lowest_bit_set, &all_bits_between_are_set);
494
495 /* 1) mov -1, %reg
496 * sllx %reg, shift, %reg
497 * 2) mov -1, %reg
498 * srlx %reg, shift, %reg
499 * 3) mov some_small_const, %reg
500 * sllx %reg, shift, %reg
501 */
502 if (((highest_bit_set == 63 || lowest_bit_set == 0) &&
503 all_bits_between_are_set != 0) ||
504 ((highest_bit_set - lowest_bit_set) < 12)) {
505 int shift = lowest_bit_set;
506 long the_const = -1;
507
508 if ((highest_bit_set != 63 && lowest_bit_set != 0) ||
509 all_bits_between_are_set == 0) {
510 the_const =
511 create_simple_focus_bits(high_bits, low_bits,
512 lowest_bit_set, 0);
513 } else if (lowest_bit_set == 0)
514 shift = -(63 - highest_bit_set);
515
516 emit(OR | IMMED | RS1(G0) | S13(the_const) | RD(dest), ctx);
517 if (shift > 0)
518 emit_alu_K(SLLX, dest, shift, ctx);
519 else if (shift < 0)
520 emit_alu_K(SRLX, dest, -shift, ctx);
521
522 return;
523 }
524
525 /* Now a range of 22 or less bits set somewhere.
526 * 1) sethi %hi(focus_bits), %reg
527 * sllx %reg, shift, %reg
528 * 2) sethi %hi(focus_bits), %reg
529 * srlx %reg, shift, %reg
530 */
531 if ((highest_bit_set - lowest_bit_set) < 21) {
532 unsigned long focus_bits =
533 create_simple_focus_bits(high_bits, low_bits,
534 lowest_bit_set, 10);
535
536 emit(SETHI(focus_bits, dest), ctx);
537
538 /* If lowest_bit_set == 10 then a sethi alone could
539 * have done it.
540 */
541 if (lowest_bit_set < 10)
542 emit_alu_K(SRLX, dest, 10 - lowest_bit_set, ctx);
543 else if (lowest_bit_set > 10)
544 emit_alu_K(SLLX, dest, lowest_bit_set - 10, ctx);
545 return;
546 }
547
548 /* Ok, now 3 instruction sequences. */
549 if (low_bits == 0) {
550 emit_loadimm32(high_bits, dest, ctx);
551 emit_alu_K(SLLX, dest, 32, ctx);
552 return;
553 }
554
555 /* We may be able to do something quick
556 * when the constant is negated, so try that.
557 */
558 if (const64_is_2insns((~high_bits) & 0xffffffff,
559 (~low_bits) & 0xfffffc00)) {
560 /* NOTE: The trailing bits get XOR'd so we need the
561 * non-negated bits, not the negated ones.
562 */
563 unsigned long trailing_bits = low_bits & 0x3ff;
564
565 if ((((~high_bits) & 0xffffffff) == 0 &&
566 ((~low_bits) & 0x80000000) == 0) ||
567 (((~high_bits) & 0xffffffff) == 0xffffffff &&
568 ((~low_bits) & 0x80000000) != 0)) {
569 unsigned long fast_int = (~low_bits & 0xffffffff);
570
571 if ((is_sethi(fast_int) &&
572 (~high_bits & 0xffffffff) == 0)) {
573 emit(SETHI(fast_int, dest), ctx);
574 } else if (is_simm13(fast_int)) {
575 emit(OR | IMMED | RS1(G0) | S13(fast_int) | RD(dest), ctx);
576 } else {
577 emit_loadimm64(fast_int, dest, ctx);
578 }
579 } else {
580 u64 n = ((~low_bits) & 0xfffffc00) |
581 (((unsigned long)((~high_bits) & 0xffffffff))<<32);
582 emit_loadimm64(n, dest, ctx);
583 }
584
585 low_bits = -0x400 | trailing_bits;
586
587 emit(XOR | IMMED | RS1(dest) | S13(low_bits) | RD(dest), ctx);
588 return;
589 }
590
591 /* 1) sethi %hi(xxx), %reg
592 * or %reg, %lo(xxx), %reg
593 * sllx %reg, yyy, %reg
594 */
595 if ((highest_bit_set - lowest_bit_set) < 32) {
596 unsigned long focus_bits =
597 create_simple_focus_bits(high_bits, low_bits,
598 lowest_bit_set, 0);
599
600 /* So what we know is that the set bits straddle the
601 * middle of the 64-bit word.
602 */
603 sparc_emit_set_const64_quick2(focus_bits, 0, dest,
604 lowest_bit_set, ctx);
605 return;
606 }
607
608 /* 1) sethi %hi(high_bits), %reg
609 * or %reg, %lo(high_bits), %reg
610 * sllx %reg, 32, %reg
611 * or %reg, low_bits, %reg
612 */
613 if (is_simm13(low_bits) && ((int)low_bits > 0)) {
614 sparc_emit_set_const64_quick2(high_bits, low_bits,
615 dest, 32, ctx);
616 return;
617 }
618
619 /* Oh well, we tried... Do a full 64-bit decomposition. */
620 ctx->tmp_1_used = true;
621
622 emit_loadimm32(high_bits, tmp, ctx);
623 emit_loadimm32(low_bits, dest, ctx);
624 emit_alu_K(SLLX, tmp, 32, ctx);
625 emit(OR | RS1(dest) | RS2(tmp) | RD(dest), ctx);
626 }
627
628 static void emit_branch(unsigned int br_opc, unsigned int from_idx, unsigned int to_idx,
629 struct jit_ctx *ctx)
630 {
631 unsigned int off = to_idx - from_idx;
632
633 if (br_opc & XCC)
634 emit(br_opc | WDISP19(off << 2), ctx);
635 else
636 emit(br_opc | WDISP22(off << 2), ctx);
637 }
638
639 static void emit_cbcond(unsigned int cb_opc, unsigned int from_idx, unsigned int to_idx,
640 const u8 dst, const u8 src, struct jit_ctx *ctx)
641 {
642 unsigned int off = to_idx - from_idx;
643
644 emit(cb_opc | WDISP10(off << 2) | RS1(dst) | RS2(src), ctx);
645 }
646
647 static void emit_cbcondi(unsigned int cb_opc, unsigned int from_idx, unsigned int to_idx,
648 const u8 dst, s32 imm, struct jit_ctx *ctx)
649 {
650 unsigned int off = to_idx - from_idx;
651
652 emit(cb_opc | IMMED | WDISP10(off << 2) | RS1(dst) | S5(imm), ctx);
653 }
654
655 #define emit_read_y(REG, CTX) emit(RD_Y | RD(REG), CTX)
656 #define emit_write_y(REG, CTX) emit(WR_Y | IMMED | RS1(REG) | S13(0), CTX)
657
658 #define emit_cmp(R1, R2, CTX) \
659 emit(SUBCC | RS1(R1) | RS2(R2) | RD(G0), CTX)
660
661 #define emit_cmpi(R1, IMM, CTX) \
662 emit(SUBCC | IMMED | RS1(R1) | S13(IMM) | RD(G0), CTX)
663
664 #define emit_btst(R1, R2, CTX) \
665 emit(ANDCC | RS1(R1) | RS2(R2) | RD(G0), CTX)
666
667 #define emit_btsti(R1, IMM, CTX) \
668 emit(ANDCC | IMMED | RS1(R1) | S13(IMM) | RD(G0), CTX)
669
670 static int emit_compare_and_branch(const u8 code, const u8 dst, u8 src,
671 const s32 imm, bool is_imm, int branch_dst,
672 struct jit_ctx *ctx)
673 {
674 bool use_cbcond = (sparc64_elf_hwcap & AV_SPARC_CBCOND) != 0;
675 const u8 tmp = bpf2sparc[TMP_REG_1];
676
677 branch_dst = ctx->offset[branch_dst];
678
679 if (!is_simm10(branch_dst - ctx->idx) ||
680 BPF_OP(code) == BPF_JSET)
681 use_cbcond = false;
682
683 if (is_imm) {
684 bool fits = true;
685
686 if (use_cbcond) {
687 if (!is_simm5(imm))
688 fits = false;
689 } else if (!is_simm13(imm)) {
690 fits = false;
691 }
692 if (!fits) {
693 ctx->tmp_1_used = true;
694 emit_loadimm_sext(imm, tmp, ctx);
695 src = tmp;
696 is_imm = false;
697 }
698 }
699
700 if (!use_cbcond) {
701 u32 br_opcode;
702
703 if (BPF_OP(code) == BPF_JSET) {
704 if (is_imm)
705 emit_btsti(dst, imm, ctx);
706 else
707 emit_btst(dst, src, ctx);
708 } else {
709 if (is_imm)
710 emit_cmpi(dst, imm, ctx);
711 else
712 emit_cmp(dst, src, ctx);
713 }
714 switch (BPF_OP(code)) {
715 case BPF_JEQ:
716 br_opcode = BE;
717 break;
718 case BPF_JGT:
719 br_opcode = BGU;
720 break;
721 case BPF_JLT:
722 br_opcode = BLU;
723 break;
724 case BPF_JGE:
725 br_opcode = BGEU;
726 break;
727 case BPF_JLE:
728 br_opcode = BLEU;
729 break;
730 case BPF_JSET:
731 case BPF_JNE:
732 br_opcode = BNE;
733 break;
734 case BPF_JSGT:
735 br_opcode = BG;
736 break;
737 case BPF_JSLT:
738 br_opcode = BL;
739 break;
740 case BPF_JSGE:
741 br_opcode = BGE;
742 break;
743 case BPF_JSLE:
744 br_opcode = BLE;
745 break;
746 default:
747 /* Make sure we dont leak kernel information to the
748 * user.
749 */
750 return -EFAULT;
751 }
752 emit_branch(br_opcode, ctx->idx, branch_dst, ctx);
753 emit_nop(ctx);
754 } else {
755 u32 cbcond_opcode;
756
757 switch (BPF_OP(code)) {
758 case BPF_JEQ:
759 cbcond_opcode = CBCONDE;
760 break;
761 case BPF_JGT:
762 cbcond_opcode = CBCONDGU;
763 break;
764 case BPF_JLT:
765 cbcond_opcode = CBCONDLU;
766 break;
767 case BPF_JGE:
768 cbcond_opcode = CBCONDGEU;
769 break;
770 case BPF_JLE:
771 cbcond_opcode = CBCONDLEU;
772 break;
773 case BPF_JNE:
774 cbcond_opcode = CBCONDNE;
775 break;
776 case BPF_JSGT:
777 cbcond_opcode = CBCONDG;
778 break;
779 case BPF_JSLT:
780 cbcond_opcode = CBCONDL;
781 break;
782 case BPF_JSGE:
783 cbcond_opcode = CBCONDGE;
784 break;
785 case BPF_JSLE:
786 cbcond_opcode = CBCONDLE;
787 break;
788 default:
789 /* Make sure we dont leak kernel information to the
790 * user.
791 */
792 return -EFAULT;
793 }
794 cbcond_opcode |= CBCOND_OP;
795 if (is_imm)
796 emit_cbcondi(cbcond_opcode, ctx->idx, branch_dst,
797 dst, imm, ctx);
798 else
799 emit_cbcond(cbcond_opcode, ctx->idx, branch_dst,
800 dst, src, ctx);
801 }
802 return 0;
803 }
804
805 static void load_skb_regs(struct jit_ctx *ctx, u8 r_skb)
806 {
807 const u8 r_headlen = bpf2sparc[SKB_HLEN_REG];
808 const u8 r_data = bpf2sparc[SKB_DATA_REG];
809 const u8 r_tmp = bpf2sparc[TMP_REG_1];
810 unsigned int off;
811
812 off = offsetof(struct sk_buff, len);
813 emit(LD32I | RS1(r_skb) | S13(off) | RD(r_headlen), ctx);
814
815 off = offsetof(struct sk_buff, data_len);
816 emit(LD32I | RS1(r_skb) | S13(off) | RD(r_tmp), ctx);
817
818 emit(SUB | RS1(r_headlen) | RS2(r_tmp) | RD(r_headlen), ctx);
819
820 off = offsetof(struct sk_buff, data);
821 emit(LDPTRI | RS1(r_skb) | S13(off) | RD(r_data), ctx);
822 }
823
824 /* Just skip the save instruction and the ctx register move. */
825 #define BPF_TAILCALL_PROLOGUE_SKIP 16
826 #define BPF_TAILCALL_CNT_SP_OFF (STACK_BIAS + 128)
827
828 static void build_prologue(struct jit_ctx *ctx)
829 {
830 s32 stack_needed = BASE_STACKFRAME;
831
832 if (ctx->saw_frame_pointer || ctx->saw_tail_call) {
833 struct bpf_prog *prog = ctx->prog;
834 u32 stack_depth;
835
836 stack_depth = prog->aux->stack_depth;
837 stack_needed += round_up(stack_depth, 16);
838 }
839
840 if (ctx->saw_tail_call)
841 stack_needed += 8;
842
843 /* save %sp, -176, %sp */
844 emit(SAVE | IMMED | RS1(SP) | S13(-stack_needed) | RD(SP), ctx);
845
846 /* tail_call_cnt = 0 */
847 if (ctx->saw_tail_call) {
848 u32 off = BPF_TAILCALL_CNT_SP_OFF;
849
850 emit(ST32 | IMMED | RS1(SP) | S13(off) | RD(G0), ctx);
851 } else {
852 emit_nop(ctx);
853 }
854 if (ctx->saw_frame_pointer) {
855 const u8 vfp = bpf2sparc[BPF_REG_FP];
856
857 emit(ADD | IMMED | RS1(FP) | S13(STACK_BIAS) | RD(vfp), ctx);
858 }
859
860 emit_reg_move(I0, O0, ctx);
861 /* If you add anything here, adjust BPF_TAILCALL_PROLOGUE_SKIP above. */
862
863 if (ctx->saw_ld_abs_ind)
864 load_skb_regs(ctx, bpf2sparc[BPF_REG_1]);
865 }
866
867 static void build_epilogue(struct jit_ctx *ctx)
868 {
869 ctx->epilogue_offset = ctx->idx;
870
871 /* ret (jmpl %i7 + 8, %g0) */
872 emit(JMPL | IMMED | RS1(I7) | S13(8) | RD(G0), ctx);
873
874 /* restore %i5, %g0, %o0 */
875 emit(RESTORE | RS1(bpf2sparc[BPF_REG_0]) | RS2(G0) | RD(O0), ctx);
876 }
877
878 static void emit_tail_call(struct jit_ctx *ctx)
879 {
880 const u8 bpf_array = bpf2sparc[BPF_REG_2];
881 const u8 bpf_index = bpf2sparc[BPF_REG_3];
882 const u8 tmp = bpf2sparc[TMP_REG_1];
883 u32 off;
884
885 ctx->saw_tail_call = true;
886
887 off = offsetof(struct bpf_array, map.max_entries);
888 emit(LD32 | IMMED | RS1(bpf_array) | S13(off) | RD(tmp), ctx);
889 emit_cmp(bpf_index, tmp, ctx);
890 #define OFFSET1 17
891 emit_branch(BGEU, ctx->idx, ctx->idx + OFFSET1, ctx);
892 emit_nop(ctx);
893
894 off = BPF_TAILCALL_CNT_SP_OFF;
895 emit(LD32 | IMMED | RS1(SP) | S13(off) | RD(tmp), ctx);
896 emit_cmpi(tmp, MAX_TAIL_CALL_CNT, ctx);
897 #define OFFSET2 13
898 emit_branch(BGU, ctx->idx, ctx->idx + OFFSET2, ctx);
899 emit_nop(ctx);
900
901 emit_alu_K(ADD, tmp, 1, ctx);
902 off = BPF_TAILCALL_CNT_SP_OFF;
903 emit(ST32 | IMMED | RS1(SP) | S13(off) | RD(tmp), ctx);
904
905 emit_alu3_K(SLL, bpf_index, 3, tmp, ctx);
906 emit_alu(ADD, bpf_array, tmp, ctx);
907 off = offsetof(struct bpf_array, ptrs);
908 emit(LD64 | IMMED | RS1(tmp) | S13(off) | RD(tmp), ctx);
909
910 emit_cmpi(tmp, 0, ctx);
911 #define OFFSET3 5
912 emit_branch(BE, ctx->idx, ctx->idx + OFFSET3, ctx);
913 emit_nop(ctx);
914
915 off = offsetof(struct bpf_prog, bpf_func);
916 emit(LD64 | IMMED | RS1(tmp) | S13(off) | RD(tmp), ctx);
917
918 off = BPF_TAILCALL_PROLOGUE_SKIP;
919 emit(JMPL | IMMED | RS1(tmp) | S13(off) | RD(G0), ctx);
920 emit_nop(ctx);
921 }
922
923 static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx)
924 {
925 const u8 code = insn->code;
926 const u8 dst = bpf2sparc[insn->dst_reg];
927 const u8 src = bpf2sparc[insn->src_reg];
928 const int i = insn - ctx->prog->insnsi;
929 const s16 off = insn->off;
930 const s32 imm = insn->imm;
931 u32 *func;
932
933 if (insn->src_reg == BPF_REG_FP)
934 ctx->saw_frame_pointer = true;
935
936 switch (code) {
937 /* dst = src */
938 case BPF_ALU | BPF_MOV | BPF_X:
939 emit_alu3_K(SRL, src, 0, dst, ctx);
940 break;
941 case BPF_ALU64 | BPF_MOV | BPF_X:
942 emit_reg_move(src, dst, ctx);
943 break;
944 /* dst = dst OP src */
945 case BPF_ALU | BPF_ADD | BPF_X:
946 case BPF_ALU64 | BPF_ADD | BPF_X:
947 emit_alu(ADD, src, dst, ctx);
948 goto do_alu32_trunc;
949 case BPF_ALU | BPF_SUB | BPF_X:
950 case BPF_ALU64 | BPF_SUB | BPF_X:
951 emit_alu(SUB, src, dst, ctx);
952 goto do_alu32_trunc;
953 case BPF_ALU | BPF_AND | BPF_X:
954 case BPF_ALU64 | BPF_AND | BPF_X:
955 emit_alu(AND, src, dst, ctx);
956 goto do_alu32_trunc;
957 case BPF_ALU | BPF_OR | BPF_X:
958 case BPF_ALU64 | BPF_OR | BPF_X:
959 emit_alu(OR, src, dst, ctx);
960 goto do_alu32_trunc;
961 case BPF_ALU | BPF_XOR | BPF_X:
962 case BPF_ALU64 | BPF_XOR | BPF_X:
963 emit_alu(XOR, src, dst, ctx);
964 goto do_alu32_trunc;
965 case BPF_ALU | BPF_MUL | BPF_X:
966 emit_alu(MUL, src, dst, ctx);
967 goto do_alu32_trunc;
968 case BPF_ALU64 | BPF_MUL | BPF_X:
969 emit_alu(MULX, src, dst, ctx);
970 break;
971 case BPF_ALU | BPF_DIV | BPF_X:
972 emit_cmp(src, G0, ctx);
973 emit_branch(BE|ANNUL, ctx->idx, ctx->epilogue_offset, ctx);
974 emit_loadimm(0, bpf2sparc[BPF_REG_0], ctx);
975
976 emit_write_y(G0, ctx);
977 emit_alu(DIV, src, dst, ctx);
978 break;
979
980 case BPF_ALU64 | BPF_DIV | BPF_X:
981 emit_cmp(src, G0, ctx);
982 emit_branch(BE|ANNUL, ctx->idx, ctx->epilogue_offset, ctx);
983 emit_loadimm(0, bpf2sparc[BPF_REG_0], ctx);
984
985 emit_alu(UDIVX, src, dst, ctx);
986 break;
987
988 case BPF_ALU | BPF_MOD | BPF_X: {
989 const u8 tmp = bpf2sparc[TMP_REG_1];
990
991 ctx->tmp_1_used = true;
992
993 emit_cmp(src, G0, ctx);
994 emit_branch(BE|ANNUL, ctx->idx, ctx->epilogue_offset, ctx);
995 emit_loadimm(0, bpf2sparc[BPF_REG_0], ctx);
996
997 emit_write_y(G0, ctx);
998 emit_alu3(DIV, dst, src, tmp, ctx);
999 emit_alu3(MULX, tmp, src, tmp, ctx);
1000 emit_alu3(SUB, dst, tmp, dst, ctx);
1001 goto do_alu32_trunc;
1002 }
1003 case BPF_ALU64 | BPF_MOD | BPF_X: {
1004 const u8 tmp = bpf2sparc[TMP_REG_1];
1005
1006 ctx->tmp_1_used = true;
1007
1008 emit_cmp(src, G0, ctx);
1009 emit_branch(BE|ANNUL, ctx->idx, ctx->epilogue_offset, ctx);
1010 emit_loadimm(0, bpf2sparc[BPF_REG_0], ctx);
1011
1012 emit_alu3(UDIVX, dst, src, tmp, ctx);
1013 emit_alu3(MULX, tmp, src, tmp, ctx);
1014 emit_alu3(SUB, dst, tmp, dst, ctx);
1015 break;
1016 }
1017 case BPF_ALU | BPF_LSH | BPF_X:
1018 emit_alu(SLL, src, dst, ctx);
1019 goto do_alu32_trunc;
1020 case BPF_ALU64 | BPF_LSH | BPF_X:
1021 emit_alu(SLLX, src, dst, ctx);
1022 break;
1023 case BPF_ALU | BPF_RSH | BPF_X:
1024 emit_alu(SRL, src, dst, ctx);
1025 break;
1026 case BPF_ALU64 | BPF_RSH | BPF_X:
1027 emit_alu(SRLX, src, dst, ctx);
1028 break;
1029 case BPF_ALU | BPF_ARSH | BPF_X:
1030 emit_alu(SRA, src, dst, ctx);
1031 goto do_alu32_trunc;
1032 case BPF_ALU64 | BPF_ARSH | BPF_X:
1033 emit_alu(SRAX, src, dst, ctx);
1034 break;
1035
1036 /* dst = -dst */
1037 case BPF_ALU | BPF_NEG:
1038 case BPF_ALU64 | BPF_NEG:
1039 emit(SUB | RS1(0) | RS2(dst) | RD(dst), ctx);
1040 goto do_alu32_trunc;
1041
1042 case BPF_ALU | BPF_END | BPF_FROM_BE:
1043 switch (imm) {
1044 case 16:
1045 emit_alu_K(SLL, dst, 16, ctx);
1046 emit_alu_K(SRL, dst, 16, ctx);
1047 break;
1048 case 32:
1049 emit_alu_K(SRL, dst, 0, ctx);
1050 break;
1051 case 64:
1052 /* nop */
1053 break;
1054
1055 }
1056 break;
1057
1058 /* dst = BSWAP##imm(dst) */
1059 case BPF_ALU | BPF_END | BPF_FROM_LE: {
1060 const u8 tmp = bpf2sparc[TMP_REG_1];
1061 const u8 tmp2 = bpf2sparc[TMP_REG_2];
1062
1063 ctx->tmp_1_used = true;
1064 switch (imm) {
1065 case 16:
1066 emit_alu3_K(AND, dst, 0xff, tmp, ctx);
1067 emit_alu3_K(SRL, dst, 8, dst, ctx);
1068 emit_alu3_K(AND, dst, 0xff, dst, ctx);
1069 emit_alu3_K(SLL, tmp, 8, tmp, ctx);
1070 emit_alu(OR, tmp, dst, ctx);
1071 break;
1072
1073 case 32:
1074 ctx->tmp_2_used = true;
1075 emit_alu3_K(SRL, dst, 24, tmp, ctx); /* tmp = dst >> 24 */
1076 emit_alu3_K(SRL, dst, 16, tmp2, ctx); /* tmp2 = dst >> 16 */
1077 emit_alu3_K(AND, tmp2, 0xff, tmp2, ctx);/* tmp2 = tmp2 & 0xff */
1078 emit_alu3_K(SLL, tmp2, 8, tmp2, ctx); /* tmp2 = tmp2 << 8 */
1079 emit_alu(OR, tmp2, tmp, ctx); /* tmp = tmp | tmp2 */
1080 emit_alu3_K(SRL, dst, 8, tmp2, ctx); /* tmp2 = dst >> 8 */
1081 emit_alu3_K(AND, tmp2, 0xff, tmp2, ctx);/* tmp2 = tmp2 & 0xff */
1082 emit_alu3_K(SLL, tmp2, 16, tmp2, ctx); /* tmp2 = tmp2 << 16 */
1083 emit_alu(OR, tmp2, tmp, ctx); /* tmp = tmp | tmp2 */
1084 emit_alu3_K(AND, dst, 0xff, dst, ctx); /* dst = dst & 0xff */
1085 emit_alu3_K(SLL, dst, 24, dst, ctx); /* dst = dst << 24 */
1086 emit_alu(OR, tmp, dst, ctx); /* dst = dst | tmp */
1087 break;
1088
1089 case 64:
1090 emit_alu3_K(ADD, SP, STACK_BIAS + 128, tmp, ctx);
1091 emit(ST64 | RS1(tmp) | RS2(G0) | RD(dst), ctx);
1092 emit(LD64A | ASI(ASI_PL) | RS1(tmp) | RS2(G0) | RD(dst), ctx);
1093 break;
1094 }
1095 break;
1096 }
1097 /* dst = imm */
1098 case BPF_ALU | BPF_MOV | BPF_K:
1099 emit_loadimm32(imm, dst, ctx);
1100 break;
1101 case BPF_ALU64 | BPF_MOV | BPF_K:
1102 emit_loadimm_sext(imm, dst, ctx);
1103 break;
1104 /* dst = dst OP imm */
1105 case BPF_ALU | BPF_ADD | BPF_K:
1106 case BPF_ALU64 | BPF_ADD | BPF_K:
1107 emit_alu_K(ADD, dst, imm, ctx);
1108 goto do_alu32_trunc;
1109 case BPF_ALU | BPF_SUB | BPF_K:
1110 case BPF_ALU64 | BPF_SUB | BPF_K:
1111 emit_alu_K(SUB, dst, imm, ctx);
1112 goto do_alu32_trunc;
1113 case BPF_ALU | BPF_AND | BPF_K:
1114 case BPF_ALU64 | BPF_AND | BPF_K:
1115 emit_alu_K(AND, dst, imm, ctx);
1116 goto do_alu32_trunc;
1117 case BPF_ALU | BPF_OR | BPF_K:
1118 case BPF_ALU64 | BPF_OR | BPF_K:
1119 emit_alu_K(OR, dst, imm, ctx);
1120 goto do_alu32_trunc;
1121 case BPF_ALU | BPF_XOR | BPF_K:
1122 case BPF_ALU64 | BPF_XOR | BPF_K:
1123 emit_alu_K(XOR, dst, imm, ctx);
1124 goto do_alu32_trunc;
1125 case BPF_ALU | BPF_MUL | BPF_K:
1126 emit_alu_K(MUL, dst, imm, ctx);
1127 goto do_alu32_trunc;
1128 case BPF_ALU64 | BPF_MUL | BPF_K:
1129 emit_alu_K(MULX, dst, imm, ctx);
1130 break;
1131 case BPF_ALU | BPF_DIV | BPF_K:
1132 if (imm == 0)
1133 return -EINVAL;
1134
1135 emit_write_y(G0, ctx);
1136 emit_alu_K(DIV, dst, imm, ctx);
1137 goto do_alu32_trunc;
1138 case BPF_ALU64 | BPF_DIV | BPF_K:
1139 if (imm == 0)
1140 return -EINVAL;
1141
1142 emit_alu_K(UDIVX, dst, imm, ctx);
1143 break;
1144 case BPF_ALU64 | BPF_MOD | BPF_K:
1145 case BPF_ALU | BPF_MOD | BPF_K: {
1146 const u8 tmp = bpf2sparc[TMP_REG_2];
1147 unsigned int div;
1148
1149 if (imm == 0)
1150 return -EINVAL;
1151
1152 div = (BPF_CLASS(code) == BPF_ALU64) ? UDIVX : DIV;
1153
1154 ctx->tmp_2_used = true;
1155
1156 if (BPF_CLASS(code) != BPF_ALU64)
1157 emit_write_y(G0, ctx);
1158 if (is_simm13(imm)) {
1159 emit(div | IMMED | RS1(dst) | S13(imm) | RD(tmp), ctx);
1160 emit(MULX | IMMED | RS1(tmp) | S13(imm) | RD(tmp), ctx);
1161 emit(SUB | RS1(dst) | RS2(tmp) | RD(dst), ctx);
1162 } else {
1163 const u8 tmp1 = bpf2sparc[TMP_REG_1];
1164
1165 ctx->tmp_1_used = true;
1166
1167 emit_set_const_sext(imm, tmp1, ctx);
1168 emit(div | RS1(dst) | RS2(tmp1) | RD(tmp), ctx);
1169 emit(MULX | RS1(tmp) | RS2(tmp1) | RD(tmp), ctx);
1170 emit(SUB | RS1(dst) | RS2(tmp) | RD(dst), ctx);
1171 }
1172 goto do_alu32_trunc;
1173 }
1174 case BPF_ALU | BPF_LSH | BPF_K:
1175 emit_alu_K(SLL, dst, imm, ctx);
1176 goto do_alu32_trunc;
1177 case BPF_ALU64 | BPF_LSH | BPF_K:
1178 emit_alu_K(SLLX, dst, imm, ctx);
1179 break;
1180 case BPF_ALU | BPF_RSH | BPF_K:
1181 emit_alu_K(SRL, dst, imm, ctx);
1182 break;
1183 case BPF_ALU64 | BPF_RSH | BPF_K:
1184 emit_alu_K(SRLX, dst, imm, ctx);
1185 break;
1186 case BPF_ALU | BPF_ARSH | BPF_K:
1187 emit_alu_K(SRA, dst, imm, ctx);
1188 goto do_alu32_trunc;
1189 case BPF_ALU64 | BPF_ARSH | BPF_K:
1190 emit_alu_K(SRAX, dst, imm, ctx);
1191 break;
1192
1193 do_alu32_trunc:
1194 if (BPF_CLASS(code) == BPF_ALU)
1195 emit_alu_K(SRL, dst, 0, ctx);
1196 break;
1197
1198 /* JUMP off */
1199 case BPF_JMP | BPF_JA:
1200 emit_branch(BA, ctx->idx, ctx->offset[i + off], ctx);
1201 emit_nop(ctx);
1202 break;
1203 /* IF (dst COND src) JUMP off */
1204 case BPF_JMP | BPF_JEQ | BPF_X:
1205 case BPF_JMP | BPF_JGT | BPF_X:
1206 case BPF_JMP | BPF_JLT | BPF_X:
1207 case BPF_JMP | BPF_JGE | BPF_X:
1208 case BPF_JMP | BPF_JLE | BPF_X:
1209 case BPF_JMP | BPF_JNE | BPF_X:
1210 case BPF_JMP | BPF_JSGT | BPF_X:
1211 case BPF_JMP | BPF_JSLT | BPF_X:
1212 case BPF_JMP | BPF_JSGE | BPF_X:
1213 case BPF_JMP | BPF_JSLE | BPF_X:
1214 case BPF_JMP | BPF_JSET | BPF_X: {
1215 int err;
1216
1217 err = emit_compare_and_branch(code, dst, src, 0, false, i + off, ctx);
1218 if (err)
1219 return err;
1220 break;
1221 }
1222 /* IF (dst COND imm) JUMP off */
1223 case BPF_JMP | BPF_JEQ | BPF_K:
1224 case BPF_JMP | BPF_JGT | BPF_K:
1225 case BPF_JMP | BPF_JLT | BPF_K:
1226 case BPF_JMP | BPF_JGE | BPF_K:
1227 case BPF_JMP | BPF_JLE | BPF_K:
1228 case BPF_JMP | BPF_JNE | BPF_K:
1229 case BPF_JMP | BPF_JSGT | BPF_K:
1230 case BPF_JMP | BPF_JSLT | BPF_K:
1231 case BPF_JMP | BPF_JSGE | BPF_K:
1232 case BPF_JMP | BPF_JSLE | BPF_K:
1233 case BPF_JMP | BPF_JSET | BPF_K: {
1234 int err;
1235
1236 err = emit_compare_and_branch(code, dst, 0, imm, true, i + off, ctx);
1237 if (err)
1238 return err;
1239 break;
1240 }
1241
1242 /* function call */
1243 case BPF_JMP | BPF_CALL:
1244 {
1245 u8 *func = ((u8 *)__bpf_call_base) + imm;
1246
1247 ctx->saw_call = true;
1248
1249 emit_call((u32 *)func, ctx);
1250 emit_nop(ctx);
1251
1252 emit_reg_move(O0, bpf2sparc[BPF_REG_0], ctx);
1253
1254 if (bpf_helper_changes_pkt_data(func) && ctx->saw_ld_abs_ind)
1255 load_skb_regs(ctx, bpf2sparc[BPF_REG_6]);
1256 break;
1257 }
1258
1259 /* tail call */
1260 case BPF_JMP | BPF_TAIL_CALL:
1261 emit_tail_call(ctx);
1262 break;
1263
1264 /* function return */
1265 case BPF_JMP | BPF_EXIT:
1266 /* Optimization: when last instruction is EXIT,
1267 simply fallthrough to epilogue. */
1268 if (i == ctx->prog->len - 1)
1269 break;
1270 emit_branch(BA, ctx->idx, ctx->epilogue_offset, ctx);
1271 emit_nop(ctx);
1272 break;
1273
1274 /* dst = imm64 */
1275 case BPF_LD | BPF_IMM | BPF_DW:
1276 {
1277 const struct bpf_insn insn1 = insn[1];
1278 u64 imm64;
1279
1280 imm64 = (u64)insn1.imm << 32 | (u32)imm;
1281 emit_loadimm64(imm64, dst, ctx);
1282
1283 return 1;
1284 }
1285
1286 /* LDX: dst = *(size *)(src + off) */
1287 case BPF_LDX | BPF_MEM | BPF_W:
1288 case BPF_LDX | BPF_MEM | BPF_H:
1289 case BPF_LDX | BPF_MEM | BPF_B:
1290 case BPF_LDX | BPF_MEM | BPF_DW: {
1291 const u8 tmp = bpf2sparc[TMP_REG_1];
1292 u32 opcode = 0, rs2;
1293
1294 ctx->tmp_1_used = true;
1295 switch (BPF_SIZE(code)) {
1296 case BPF_W:
1297 opcode = LD32;
1298 break;
1299 case BPF_H:
1300 opcode = LD16;
1301 break;
1302 case BPF_B:
1303 opcode = LD8;
1304 break;
1305 case BPF_DW:
1306 opcode = LD64;
1307 break;
1308 }
1309
1310 if (is_simm13(off)) {
1311 opcode |= IMMED;
1312 rs2 = S13(off);
1313 } else {
1314 emit_loadimm(off, tmp, ctx);
1315 rs2 = RS2(tmp);
1316 }
1317 emit(opcode | RS1(src) | rs2 | RD(dst), ctx);
1318 break;
1319 }
1320 /* ST: *(size *)(dst + off) = imm */
1321 case BPF_ST | BPF_MEM | BPF_W:
1322 case BPF_ST | BPF_MEM | BPF_H:
1323 case BPF_ST | BPF_MEM | BPF_B:
1324 case BPF_ST | BPF_MEM | BPF_DW: {
1325 const u8 tmp = bpf2sparc[TMP_REG_1];
1326 const u8 tmp2 = bpf2sparc[TMP_REG_2];
1327 u32 opcode = 0, rs2;
1328
1329 ctx->tmp_2_used = true;
1330 emit_loadimm(imm, tmp2, ctx);
1331
1332 switch (BPF_SIZE(code)) {
1333 case BPF_W:
1334 opcode = ST32;
1335 break;
1336 case BPF_H:
1337 opcode = ST16;
1338 break;
1339 case BPF_B:
1340 opcode = ST8;
1341 break;
1342 case BPF_DW:
1343 opcode = ST64;
1344 break;
1345 }
1346
1347 if (is_simm13(off)) {
1348 opcode |= IMMED;
1349 rs2 = S13(off);
1350 } else {
1351 ctx->tmp_1_used = true;
1352 emit_loadimm(off, tmp, ctx);
1353 rs2 = RS2(tmp);
1354 }
1355 emit(opcode | RS1(dst) | rs2 | RD(tmp2), ctx);
1356 break;
1357 }
1358
1359 /* STX: *(size *)(dst + off) = src */
1360 case BPF_STX | BPF_MEM | BPF_W:
1361 case BPF_STX | BPF_MEM | BPF_H:
1362 case BPF_STX | BPF_MEM | BPF_B:
1363 case BPF_STX | BPF_MEM | BPF_DW: {
1364 const u8 tmp = bpf2sparc[TMP_REG_1];
1365 u32 opcode = 0, rs2;
1366
1367 switch (BPF_SIZE(code)) {
1368 case BPF_W:
1369 opcode = ST32;
1370 break;
1371 case BPF_H:
1372 opcode = ST16;
1373 break;
1374 case BPF_B:
1375 opcode = ST8;
1376 break;
1377 case BPF_DW:
1378 opcode = ST64;
1379 break;
1380 }
1381 if (is_simm13(off)) {
1382 opcode |= IMMED;
1383 rs2 = S13(off);
1384 } else {
1385 ctx->tmp_1_used = true;
1386 emit_loadimm(off, tmp, ctx);
1387 rs2 = RS2(tmp);
1388 }
1389 emit(opcode | RS1(dst) | rs2 | RD(src), ctx);
1390 break;
1391 }
1392
1393 /* STX XADD: lock *(u32 *)(dst + off) += src */
1394 case BPF_STX | BPF_XADD | BPF_W: {
1395 const u8 tmp = bpf2sparc[TMP_REG_1];
1396 const u8 tmp2 = bpf2sparc[TMP_REG_2];
1397 const u8 tmp3 = bpf2sparc[TMP_REG_3];
1398
1399 ctx->tmp_1_used = true;
1400 ctx->tmp_2_used = true;
1401 ctx->tmp_3_used = true;
1402 emit_loadimm(off, tmp, ctx);
1403 emit_alu3(ADD, dst, tmp, tmp, ctx);
1404
1405 emit(LD32 | RS1(tmp) | RS2(G0) | RD(tmp2), ctx);
1406 emit_alu3(ADD, tmp2, src, tmp3, ctx);
1407 emit(CAS | ASI(ASI_P) | RS1(tmp) | RS2(tmp2) | RD(tmp3), ctx);
1408 emit_cmp(tmp2, tmp3, ctx);
1409 emit_branch(BNE, 4, 0, ctx);
1410 emit_nop(ctx);
1411 break;
1412 }
1413 /* STX XADD: lock *(u64 *)(dst + off) += src */
1414 case BPF_STX | BPF_XADD | BPF_DW: {
1415 const u8 tmp = bpf2sparc[TMP_REG_1];
1416 const u8 tmp2 = bpf2sparc[TMP_REG_2];
1417 const u8 tmp3 = bpf2sparc[TMP_REG_3];
1418
1419 ctx->tmp_1_used = true;
1420 ctx->tmp_2_used = true;
1421 ctx->tmp_3_used = true;
1422 emit_loadimm(off, tmp, ctx);
1423 emit_alu3(ADD, dst, tmp, tmp, ctx);
1424
1425 emit(LD64 | RS1(tmp) | RS2(G0) | RD(tmp2), ctx);
1426 emit_alu3(ADD, tmp2, src, tmp3, ctx);
1427 emit(CASX | ASI(ASI_P) | RS1(tmp) | RS2(tmp2) | RD(tmp3), ctx);
1428 emit_cmp(tmp2, tmp3, ctx);
1429 emit_branch(BNE, 4, 0, ctx);
1430 emit_nop(ctx);
1431 break;
1432 }
1433 #define CHOOSE_LOAD_FUNC(K, func) \
1434 ((int)K < 0 ? ((int)K >= SKF_LL_OFF ? func##_negative_offset : func) : func##_positive_offset)
1435
1436 /* R0 = ntohx(*(size *)(((struct sk_buff *)R6)->data + imm)) */
1437 case BPF_LD | BPF_ABS | BPF_W:
1438 func = CHOOSE_LOAD_FUNC(imm, bpf_jit_load_word);
1439 goto common_load;
1440 case BPF_LD | BPF_ABS | BPF_H:
1441 func = CHOOSE_LOAD_FUNC(imm, bpf_jit_load_half);
1442 goto common_load;
1443 case BPF_LD | BPF_ABS | BPF_B:
1444 func = CHOOSE_LOAD_FUNC(imm, bpf_jit_load_byte);
1445 goto common_load;
1446 /* R0 = ntohx(*(size *)(((struct sk_buff *)R6)->data + src + imm)) */
1447 case BPF_LD | BPF_IND | BPF_W:
1448 func = bpf_jit_load_word;
1449 goto common_load;
1450 case BPF_LD | BPF_IND | BPF_H:
1451 func = bpf_jit_load_half;
1452 goto common_load;
1453
1454 case BPF_LD | BPF_IND | BPF_B:
1455 func = bpf_jit_load_byte;
1456 common_load:
1457 ctx->saw_ld_abs_ind = true;
1458
1459 emit_reg_move(bpf2sparc[BPF_REG_6], O0, ctx);
1460 emit_loadimm(imm, O1, ctx);
1461
1462 if (BPF_MODE(code) == BPF_IND)
1463 emit_alu(ADD, src, O1, ctx);
1464
1465 emit_call(func, ctx);
1466 emit_alu_K(SRA, O1, 0, ctx);
1467
1468 emit_reg_move(O0, bpf2sparc[BPF_REG_0], ctx);
1469 break;
1470
1471 default:
1472 pr_err_once("unknown opcode %02x\n", code);
1473 return -EINVAL;
1474 }
1475
1476 return 0;
1477 }
1478
1479 static int build_body(struct jit_ctx *ctx)
1480 {
1481 const struct bpf_prog *prog = ctx->prog;
1482 int i;
1483
1484 for (i = 0; i < prog->len; i++) {
1485 const struct bpf_insn *insn = &prog->insnsi[i];
1486 int ret;
1487
1488 ret = build_insn(insn, ctx);
1489
1490 if (ret > 0) {
1491 i++;
1492 ctx->offset[i] = ctx->idx;
1493 continue;
1494 }
1495 ctx->offset[i] = ctx->idx;
1496 if (ret)
1497 return ret;
1498 }
1499 return 0;
1500 }
1501
1502 static void jit_fill_hole(void *area, unsigned int size)
1503 {
1504 u32 *ptr;
1505 /* We are guaranteed to have aligned memory. */
1506 for (ptr = area; size >= sizeof(u32); size -= sizeof(u32))
1507 *ptr++ = 0x91d02005; /* ta 5 */
1508 }
1509
1510 struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
1511 {
1512 struct bpf_prog *tmp, *orig_prog = prog;
1513 struct bpf_binary_header *header;
1514 bool tmp_blinded = false;
1515 struct jit_ctx ctx;
1516 u32 image_size;
1517 u8 *image_ptr;
1518 int pass;
1519
1520 if (!bpf_jit_enable)
1521 return orig_prog;
1522
1523 tmp = bpf_jit_blind_constants(prog);
1524 /* If blinding was requested and we failed during blinding,
1525 * we must fall back to the interpreter.
1526 */
1527 if (IS_ERR(tmp))
1528 return orig_prog;
1529 if (tmp != prog) {
1530 tmp_blinded = true;
1531 prog = tmp;
1532 }
1533
1534 memset(&ctx, 0, sizeof(ctx));
1535 ctx.prog = prog;
1536
1537 ctx.offset = kcalloc(prog->len, sizeof(unsigned int), GFP_KERNEL);
1538 if (ctx.offset == NULL) {
1539 prog = orig_prog;
1540 goto out;
1541 }
1542
1543 /* Fake pass to detect features used, and get an accurate assessment
1544 * of what the final image size will be.
1545 */
1546 if (build_body(&ctx)) {
1547 prog = orig_prog;
1548 goto out_off;
1549 }
1550 build_prologue(&ctx);
1551 build_epilogue(&ctx);
1552
1553 /* Now we know the actual image size. */
1554 image_size = sizeof(u32) * ctx.idx;
1555 header = bpf_jit_binary_alloc(image_size, &image_ptr,
1556 sizeof(u32), jit_fill_hole);
1557 if (header == NULL) {
1558 prog = orig_prog;
1559 goto out_off;
1560 }
1561
1562 ctx.image = (u32 *)image_ptr;
1563
1564 for (pass = 1; pass < 3; pass++) {
1565 ctx.idx = 0;
1566
1567 build_prologue(&ctx);
1568
1569 if (build_body(&ctx)) {
1570 bpf_jit_binary_free(header);
1571 prog = orig_prog;
1572 goto out_off;
1573 }
1574
1575 build_epilogue(&ctx);
1576
1577 if (bpf_jit_enable > 1)
1578 pr_info("Pass %d: shrink = %d, seen = [%c%c%c%c%c%c%c]\n", pass,
1579 image_size - (ctx.idx * 4),
1580 ctx.tmp_1_used ? '1' : ' ',
1581 ctx.tmp_2_used ? '2' : ' ',
1582 ctx.tmp_3_used ? '3' : ' ',
1583 ctx.saw_ld_abs_ind ? 'L' : ' ',
1584 ctx.saw_frame_pointer ? 'F' : ' ',
1585 ctx.saw_call ? 'C' : ' ',
1586 ctx.saw_tail_call ? 'T' : ' ');
1587 }
1588
1589 if (bpf_jit_enable > 1)
1590 bpf_jit_dump(prog->len, image_size, pass, ctx.image);
1591
1592 bpf_flush_icache(header, (u8 *)header + (header->pages * PAGE_SIZE));
1593
1594 bpf_jit_binary_lock_ro(header);
1595
1596 prog->bpf_func = (void *)ctx.image;
1597 prog->jited = 1;
1598 prog->jited_len = image_size;
1599
1600 out_off:
1601 kfree(ctx.offset);
1602 out:
1603 if (tmp_blinded)
1604 bpf_jit_prog_release_other(prog, prog == orig_prog ?
1605 tmp : orig_prog);
1606 return prog;
1607 }
Motorola kernel-slsi