bpf: add bpf_patch_insn_single helper

Move the functionality to patch instructions out of the verifier
code and into the core as the new bpf_patch_insn_single() helper
will be needed later on for blinding as well. No changes in
functionality.

Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
Acked-by: Alexei Starovoitov <ast@kernel.org>
Signed-off-by: David S. Miller <davem@davemloft.net>

diff --git a/include/linux/filter.h b/include/linux/filter.h
index 4ff0e647598f05bd9b943087734f82262861bd81..c4aae496f376637106ccdebfe304f5df4cdbf2ad 100644 (file)
--- a/include/linux/filter.h
+++ b/include/linux/filter.h
@@ -495,6 +495,9 @@ u64 __bpf_call_base(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5);
 void bpf_int_jit_compile(struct bpf_prog *fp);
 bool bpf_helper_changes_skb_data(void *func);
 
+struct bpf_prog *bpf_patch_insn_single(struct bpf_prog *prog, u32 off,
+                                      const struct bpf_insn *patch, u32 len);
+
 #ifdef CONFIG_BPF_JIT
 extern int bpf_jit_enable;
 

diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c
index 5313d09d4b62f2e940ab4b2e05c2880ad9d92b7c..49b5538a5301243b8a9559ee7a1acf9994c9cdcd 100644 (file)
--- a/kernel/bpf/core.c
+++ b/kernel/bpf/core.c
@@ -136,6 +136,77 @@ void __bpf_prog_free(struct bpf_prog *fp)
        vfree(fp);
 }
 
+static bool bpf_is_jmp_and_has_target(const struct bpf_insn *insn)
+{
+       return BPF_CLASS(insn->code) == BPF_JMP  &&
+              /* Call and Exit are both special jumps with no
+               * target inside the BPF instruction image.
+               */
+              BPF_OP(insn->code) != BPF_CALL &&
+              BPF_OP(insn->code) != BPF_EXIT;
+}
+
+static void bpf_adj_branches(struct bpf_prog *prog, u32 pos, u32 delta)
+{
+       struct bpf_insn *insn = prog->insnsi;
+       u32 i, insn_cnt = prog->len;
+
+       for (i = 0; i < insn_cnt; i++, insn++) {
+               if (!bpf_is_jmp_and_has_target(insn))
+                       continue;
+
+               /* Adjust offset of jmps if we cross boundaries. */
+               if (i < pos && i + insn->off + 1 > pos)
+                       insn->off += delta;
+               else if (i > pos + delta && i + insn->off + 1 <= pos + delta)
+                       insn->off -= delta;
+       }
+}
+
+struct bpf_prog *bpf_patch_insn_single(struct bpf_prog *prog, u32 off,
+                                      const struct bpf_insn *patch, u32 len)
+{
+       u32 insn_adj_cnt, insn_rest, insn_delta = len - 1;
+       struct bpf_prog *prog_adj;
+
+       /* Since our patchlet doesn't expand the image, we're done. */
+       if (insn_delta == 0) {
+               memcpy(prog->insnsi + off, patch, sizeof(*patch));
+               return prog;
+       }
+
+       insn_adj_cnt = prog->len + insn_delta;
+
+       /* Several new instructions need to be inserted. Make room
+        * for them. Likely, there's no need for a new allocation as
+        * last page could have large enough tailroom.
+        */
+       prog_adj = bpf_prog_realloc(prog, bpf_prog_size(insn_adj_cnt),
+                                   GFP_USER);
+       if (!prog_adj)
+               return NULL;
+
+       prog_adj->len = insn_adj_cnt;
+
+       /* Patching happens in 3 steps:
+        *
+        * 1) Move over tail of insnsi from next instruction onwards,
+        *    so we can patch the single target insn with one or more
+        *    new ones (patching is always from 1 to n insns, n > 0).
+        * 2) Inject new instructions at the target location.
+        * 3) Adjust branch offsets if necessary.
+        */
+       insn_rest = insn_adj_cnt - off - len;
+
+       memmove(prog_adj->insnsi + off + len, prog_adj->insnsi + off + 1,
+               sizeof(*patch) * insn_rest);
+       memcpy(prog_adj->insnsi + off, patch, sizeof(*patch) * len);
+
+       bpf_adj_branches(prog_adj, off, insn_delta);
+
+       return prog_adj;
+}
+
 #ifdef CONFIG_BPF_JIT
 struct bpf_binary_header *
 bpf_jit_binary_alloc(unsigned int proglen, u8 **image_ptr,

diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index 84bff68cf80e0c20baf44df53b203c4e3ddd6b8c..a08d66215245712b5c08808ae6b00058906c99f7 100644 (file)
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -2587,26 +2587,6 @@ static void convert_pseudo_ld_imm64(struct verifier_env *env)
                        insn->src_reg = 0;
 }
 
-static void adjust_branches(struct bpf_prog *prog, int pos, int delta)
-{
-       struct bpf_insn *insn = prog->insnsi;
-       int insn_cnt = prog->len;
-       int i;
-
-       for (i = 0; i < insn_cnt; i++, insn++) {
-               if (BPF_CLASS(insn->code) != BPF_JMP ||
-                   BPF_OP(insn->code) == BPF_CALL ||
-                   BPF_OP(insn->code) == BPF_EXIT)
-                       continue;
-
-               /* adjust offset of jmps if necessary */
-               if (i < pos && i + insn->off + 1 > pos)
-                       insn->off += delta;
-               else if (i > pos + delta && i + insn->off + 1 <= pos + delta)
-                       insn->off -= delta;
-       }
-}
-
 /* convert load instructions that access fields of 'struct __sk_buff'
  * into sequence of instructions that access fields of 'struct sk_buff'
  */
@@ -2616,14 +2596,15 @@ static int convert_ctx_accesses(struct verifier_env *env)
        int insn_cnt = env->prog->len;
        struct bpf_insn insn_buf[16];
        struct bpf_prog *new_prog;
-       u32 cnt;
-       int i;
        enum bpf_access_type type;
+       int i;
 
        if (!env->prog->aux->ops->convert_ctx_access)
                return 0;
 
        for (i = 0; i < insn_cnt; i++, insn++) {
+               u32 insn_delta, cnt;
+
                if (insn->code == (BPF_LDX | BPF_MEM | BPF_W))
                        type = BPF_READ;
                else if (insn->code == (BPF_STX | BPF_MEM | BPF_W))
@@ -2645,34 +2626,18 @@ static int convert_ctx_accesses(struct verifier_env *env)
                        return -EINVAL;
                }
 
-               if (cnt == 1) {
-                       memcpy(insn, insn_buf, sizeof(*insn));
-                       continue;
-               }
-
-               /* several new insns need to be inserted. Make room for them */
-               insn_cnt += cnt - 1;
-               new_prog = bpf_prog_realloc(env->prog,
-                                           bpf_prog_size(insn_cnt),
-                                           GFP_USER);
+               new_prog = bpf_patch_insn_single(env->prog, i, insn_buf, cnt);
                if (!new_prog)
                        return -ENOMEM;
 
-               new_prog->len = insn_cnt;
-
-               memmove(new_prog->insnsi + i + cnt, new_prog->insns + i + 1,
-                       sizeof(*insn) * (insn_cnt - i - cnt));
-
-               /* copy substitute insns in place of load instruction */
-               memcpy(new_prog->insnsi + i, insn_buf, sizeof(*insn) * cnt);
-
-               /* adjust branches in the whole program */
-               adjust_branches(new_prog, i, cnt - 1);
+               insn_delta = cnt - 1;
 
                /* keep walking new program and skip insns we just inserted */
                env->prog = new_prog;
-               insn = new_prog->insnsi + i + cnt - 1;
-               i += cnt - 1;
+               insn      = new_prog->insnsi + i + insn_delta;
+
+               insn_cnt += insn_delta;
+               i        += insn_delta;
        }
 
        return 0;