nfp: add support for offload of XDP programs

Most infrastructure can be reused, provide separate handling
of context offsets and exit codes.

Signed-off-by: Jakub Kicinski <jakub.kicinski@netronome.com>
Signed-off-by: David S. Miller <davem@davemloft.net>

diff --git a/drivers/net/ethernet/netronome/nfp/nfp_bpf.h b/drivers/net/ethernet/netronome/nfp/nfp_bpf.h
index 87aa8a3e9112f613da68e741fe9d791470b9c708..76a19f1796af71a338e0f47636ef70dcaa6ae8ba 100644 (file)
--- a/drivers/net/ethernet/netronome/nfp/nfp_bpf.h
+++ b/drivers/net/ethernet/netronome/nfp/nfp_bpf.h
@@ -62,6 +62,7 @@ enum nfp_bpf_action_type {
        NN_ACT_TC_DROP,
        NN_ACT_TC_REDIR,
        NN_ACT_DIRECT,
+       NN_ACT_XDP,
 };
 
 /* Software register representation, hardware encoding in asm.h */

diff --git a/drivers/net/ethernet/netronome/nfp/nfp_bpf_jit.c b/drivers/net/ethernet/netronome/nfp/nfp_bpf_jit.c
index f8df5300f49c1182e271d444c3b42714f30951e6..335beb8b8b45c75193a604550257be821fd2a1b5 100644 (file)
--- a/drivers/net/ethernet/netronome/nfp/nfp_bpf_jit.c
+++ b/drivers/net/ethernet/netronome/nfp/nfp_bpf_jit.c
@@ -1126,7 +1126,7 @@ static int data_ind_ld4(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
                                     meta->insn.src_reg * 2, true, 4);
 }
 
-static int mem_ldx4(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
+static int mem_ldx4_skb(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
 {
        if (meta->insn.off == offsetof(struct sk_buff, len))
                emit_alu(nfp_prog, reg_both(meta->insn.dst_reg * 2),
@@ -1134,12 +1134,42 @@ static int mem_ldx4(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
        else
                return -ENOTSUPP;
 
-       wrp_immed(nfp_prog, reg_both(meta->insn.dst_reg * 2 + 1), 0);
+       return 0;
+}
+
+static int mem_ldx4_xdp(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
+{
+       u32 dst = reg_both(meta->insn.dst_reg * 2);
+
+       if (meta->insn.off != offsetof(struct xdp_md, data) &&
+           meta->insn.off != offsetof(struct xdp_md, data_end))
+               return -ENOTSUPP;
+
+       emit_alu(nfp_prog, dst, reg_none(), ALU_OP_NONE, NFP_BPF_ABI_PKT);
+
+       if (meta->insn.off == offsetof(struct xdp_md, data))
+               return 0;
+
+       emit_alu(nfp_prog, dst, dst, ALU_OP_ADD, NFP_BPF_ABI_LEN);
 
        return 0;
 }
 
-static int mem_stx4(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
+static int mem_ldx4(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
+{
+       int ret;
+
+       if (nfp_prog->act == NN_ACT_XDP)
+               ret = mem_ldx4_xdp(nfp_prog, meta);
+       else
+               ret = mem_ldx4_skb(nfp_prog, meta);
+
+       wrp_immed(nfp_prog, reg_both(meta->insn.dst_reg * 2 + 1), 0);
+
+       return ret;
+}
+
+static int mem_stx4_skb(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
 {
        if (meta->insn.off == offsetof(struct sk_buff, mark))
                return wrp_set_mark(nfp_prog, meta->insn.src_reg * 2);
@@ -1147,6 +1177,18 @@ static int mem_stx4(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
        return -ENOTSUPP;
 }
 
+static int mem_stx4_xdp(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
+{
+       return -ENOTSUPP;
+}
+
+static int mem_stx4(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
+{
+       if (nfp_prog->act == NN_ACT_XDP)
+               return mem_stx4_xdp(nfp_prog, meta);
+       return mem_stx4_skb(nfp_prog, meta);
+}
+
 static int jump(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
 {
        if (meta->insn.off < 0) /* TODO */
@@ -1530,6 +1572,47 @@ static void nfp_outro_tc_da(struct nfp_prog *nfp_prog)
        emit_ld_field(nfp_prog, reg_a(0), 0xc, reg_b(2), SHF_SC_L_SHF, 16);
 }
 
+static void nfp_outro_xdp(struct nfp_prog *nfp_prog)
+{
+       /* XDP return codes:
+        *   0 aborted  0x82 -> drop,  count as stat3
+        *   1    drop  0x22 -> drop,  count as stat1
+        *   2    pass  0x11 -> pass,  count as stat0
+        *   3      tx  0x44 -> redir, count as stat2
+        *   * unknown  0x82 -> drop,  count as stat3
+        */
+       /* Target for aborts */
+       nfp_prog->tgt_abort = nfp_prog_current_offset(nfp_prog);
+
+       emit_br_def(nfp_prog, nfp_prog->tgt_done, 2);
+
+       emit_alu(nfp_prog, reg_a(0),
+                reg_none(), ALU_OP_NONE, NFP_BPF_ABI_FLAGS);
+       emit_ld_field(nfp_prog, reg_a(0), 0xc, reg_imm(0x82), SHF_SC_L_SHF, 16);
+
+       /* Target for normal exits */
+       nfp_prog->tgt_out = nfp_prog_current_offset(nfp_prog);
+
+       /* if R0 > 3 jump to abort */
+       emit_alu(nfp_prog, reg_none(), reg_imm(3), ALU_OP_SUB, reg_b(0));
+       emit_br(nfp_prog, BR_BLO, nfp_prog->tgt_abort, 0);
+
+       wrp_immed(nfp_prog, reg_b(2), 0x44112282);
+
+       emit_shf(nfp_prog, reg_a(1),
+                reg_none(), SHF_OP_NONE, reg_b(0), SHF_SC_L_SHF, 3);
+
+       emit_alu(nfp_prog, reg_none(), reg_a(1), ALU_OP_OR, reg_imm(0));
+       emit_shf(nfp_prog, reg_b(2),
+                reg_imm(0xff), SHF_OP_AND, reg_b(2), SHF_SC_R_SHF, 0);
+
+       emit_br_def(nfp_prog, nfp_prog->tgt_done, 2);
+
+       emit_alu(nfp_prog, reg_a(0),
+                reg_none(), ALU_OP_NONE, NFP_BPF_ABI_FLAGS);
+       emit_ld_field(nfp_prog, reg_a(0), 0xc, reg_b(2), SHF_SC_L_SHF, 16);
+}
+
 static void nfp_outro(struct nfp_prog *nfp_prog)
 {
        switch (nfp_prog->act) {
@@ -1540,6 +1623,9 @@ static void nfp_outro(struct nfp_prog *nfp_prog)
        case NN_ACT_TC_REDIR:
                nfp_outro_tc_legacy(nfp_prog);
                break;
+       case NN_ACT_XDP:
+               nfp_outro_xdp(nfp_prog);
+               break;
        }
 }
 

diff --git a/drivers/net/ethernet/netronome/nfp/nfp_bpf_verifier.c b/drivers/net/ethernet/netronome/nfp/nfp_bpf_verifier.c
index 144cae87f63a19b441f385661f2705b37d826a71..b3361f9b8e5c156c9ce58b156126993e5258fa68 100644 (file)
--- a/drivers/net/ethernet/netronome/nfp/nfp_bpf_verifier.c
+++ b/drivers/net/ethernet/netronome/nfp/nfp_bpf_verifier.c
@@ -80,6 +80,9 @@ nfp_bpf_check_exit(struct nfp_prog *nfp_prog,
 {
        const struct bpf_reg_state *reg0 = &env->cur_state.regs[0];
 
+       if (nfp_prog->act == NN_ACT_XDP)
+               return 0;
+
        if (reg0->type != CONST_IMM) {
                pr_info("unsupported exit state: %d, imm: %llx\n",
                        reg0->type, reg0->imm);

diff --git a/drivers/net/ethernet/netronome/nfp/nfp_net.h b/drivers/net/ethernet/netronome/nfp/nfp_net.h
index fd29a6306991d054eb2fc1fd1e938f7cdb740a50..2115f446031ef46c3e2c6c586d58f31e064efa38 100644 (file)
--- a/drivers/net/ethernet/netronome/nfp/nfp_net.h
+++ b/drivers/net/ethernet/netronome/nfp/nfp_net.h
@@ -435,6 +435,7 @@ struct nfp_stat_pair {
  * @is_vf:              Is the driver attached to a VF?
  * @fw_loaded:          Is the firmware loaded?
  * @bpf_offload_skip_sw:  Offloaded BPF program will not be rerun by cls_bpf
+ * @bpf_offload_xdp:   Offloaded BPF program is XDP
  * @ctrl:               Local copy of the control register/word.
  * @fl_bufsz:           Currently configured size of the freelist buffers
  * @rx_offset:         Offset in the RX buffers where packet data starts
@@ -502,6 +503,7 @@ struct nfp_net {
        unsigned is_vf:1;
        unsigned fw_loaded:1;
        unsigned bpf_offload_skip_sw:1;
+       unsigned bpf_offload_xdp:1;
 
        u32 ctrl;
        u32 fl_bufsz;

diff --git a/drivers/net/ethernet/netronome/nfp/nfp_net_common.c b/drivers/net/ethernet/netronome/nfp/nfp_net_common.c
index 1e8e00d25c51b4c665159beeed7343107210bbff..99edb9fd84bf9a302a91520d5781b7a9d78fff97 100644 (file)
--- a/drivers/net/ethernet/netronome/nfp/nfp_net_common.c
+++ b/drivers/net/ethernet/netronome/nfp/nfp_net_common.c
@@ -1600,7 +1600,8 @@ static int nfp_net_rx(struct nfp_net_rx_ring *rx_ring, int budget)
                r_vec->rx_bytes += pkt_len;
                u64_stats_update_end(&r_vec->rx_sync);
 
-               if (xdp_prog) {
+               if (xdp_prog && !(rxd->rxd.flags & PCIE_DESC_RX_BPF &&
+                                 nn->bpf_offload_xdp)) {
                        int act;
 
                        dma_sync_single_for_cpu(&nn->pdev->dev,
@@ -2693,8 +2694,12 @@ nfp_net_setup_tc(struct net_device *netdev, u32 handle, __be16 proto,
        if (proto != htons(ETH_P_ALL))
                return -ENOTSUPP;
 
-       if (tc->type == TC_SETUP_CLSBPF && nfp_net_ebpf_capable(nn))
-               return nfp_net_bpf_offload(nn, tc->cls_bpf);
+       if (tc->type == TC_SETUP_CLSBPF && nfp_net_ebpf_capable(nn)) {
+               if (!nn->bpf_offload_xdp)
+                       return nfp_net_bpf_offload(nn, tc->cls_bpf);
+               else
+                       return -EBUSY;
+       }
 
        return -EINVAL;
 }
@@ -2902,6 +2907,34 @@ static void nfp_net_del_vxlan_port(struct net_device *netdev,
                nfp_net_set_vxlan_port(nn, idx, 0);
 }
 
+static int nfp_net_xdp_offload(struct nfp_net *nn, struct bpf_prog *prog)
+{
+       struct tc_cls_bpf_offload cmd = {
+               .prog = prog,
+       };
+       int ret;
+
+       if (!nfp_net_ebpf_capable(nn))
+               return -EINVAL;
+
+       if (nn->ctrl & NFP_NET_CFG_CTRL_BPF) {
+               if (!nn->bpf_offload_xdp)
+                       return prog ? -EBUSY : 0;
+               cmd.command = prog ? TC_CLSBPF_REPLACE : TC_CLSBPF_DESTROY;
+       } else {
+               if (!prog)
+                       return 0;
+               cmd.command = TC_CLSBPF_ADD;
+       }
+
+       ret = nfp_net_bpf_offload(nn, &cmd);
+       /* Stop offload if replace not possible */
+       if (ret && cmd.command == TC_CLSBPF_REPLACE)
+               nfp_net_xdp_offload(nn, NULL);
+       nn->bpf_offload_xdp = prog && !ret;
+       return ret;
+}
+
 static int nfp_net_xdp_setup(struct nfp_net *nn, struct bpf_prog *prog)
 {
        struct nfp_net_ring_set rx = {
@@ -2920,6 +2953,7 @@ static int nfp_net_xdp_setup(struct nfp_net *nn, struct bpf_prog *prog)
        if (prog && nn->xdp_prog) {
                prog = xchg(&nn->xdp_prog, prog);
                bpf_prog_put(prog);
+               nfp_net_xdp_offload(nn, nn->xdp_prog);
                return 0;
        }
 
@@ -2934,6 +2968,8 @@ static int nfp_net_xdp_setup(struct nfp_net *nn, struct bpf_prog *prog)
        if (prog)
                bpf_prog_put(prog);
 
+       nfp_net_xdp_offload(nn, nn->xdp_prog);
+
        return 0;
 }
 
@@ -3233,5 +3269,7 @@ void nfp_net_netdev_clean(struct net_device *netdev)
 
        if (nn->xdp_prog)
                bpf_prog_put(nn->xdp_prog);
+       if (nn->bpf_offload_xdp)
+               nfp_net_xdp_offload(nn, NULL);
        unregister_netdev(nn->netdev);
 }

diff --git a/drivers/net/ethernet/netronome/nfp/nfp_net_offload.c b/drivers/net/ethernet/netronome/nfp/nfp_net_offload.c
index 4bb6f16e2a7ac8670feb26b3fce233889b4ad813..18a851eb35084397dd6fa003b3def76ed42a6960 100644 (file)
--- a/drivers/net/ethernet/netronome/nfp/nfp_net_offload.c
+++ b/drivers/net/ethernet/netronome/nfp/nfp_net_offload.c
@@ -111,6 +111,9 @@ nfp_net_bpf_get_act(struct nfp_net *nn, struct tc_cls_bpf_offload *cls_bpf)
        const struct tc_action *a;
        LIST_HEAD(actions);
 
+       if (!cls_bpf->exts)
+               return NN_ACT_XDP;
+
        /* TC direct action */
        if (cls_bpf->exts_integrated) {
                if (tc_no_actions(cls_bpf->exts))