netvsc: optimize receive path
authorstephen hemminger <stephen@networkplumber.org>
Tue, 24 Jan 2017 21:06:08 +0000 (13:06 -0800)
committerDavid S. Miller <davem@davemloft.net>
Tue, 24 Jan 2017 21:29:00 +0000 (16:29 -0500)
Do manual optimizations of receive path:
  - remove checks for impossible conditions (but keep checks
    for bad data from host)
  - pass argument down, rather than having callee recompute what
    is already known
  - remove indirection about receive buffer datalength
  - remove dependence on VLAN_TAG_PRESENCE
  - use _hot/_cold and likely/unlikely

Signed-off-by: Stephen Hemminger <sthemmin@microsoft.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
drivers/net/hyperv/hyperv_net.h
drivers/net/hyperv/netvsc.c
drivers/net/hyperv/netvsc_drv.c
drivers/net/hyperv/rndis_filter.c

index fb73caad096564d247263dabd8ad88a41148a997..4b91e37c85ca151a3f26a56088c08b008a099af1 100644 (file)
@@ -119,6 +119,7 @@ struct ndis_recv_scale_param { /* NDIS_RECEIVE_SCALE_PARAMETERS */
 
 /* Fwd declaration */
 struct ndis_tcp_ip_checksum_info;
+struct ndis_pkt_8021q_info;
 
 /*
  * Represent netvsc packet which contains 1 RNDIS and 1 ethernet frame
@@ -186,12 +187,11 @@ int netvsc_send(struct hv_device *device,
                struct sk_buff *skb);
 void netvsc_linkstatus_callback(struct hv_device *device_obj,
                                struct rndis_message *resp);
-int netvsc_recv_callback(struct hv_device *device_obj,
-                       struct hv_netvsc_packet *packet,
-                       void **data,
-                       struct ndis_tcp_ip_checksum_info *csum_info,
-                       struct vmbus_channel *channel,
-                       u16 vlan_tci);
+int netvsc_recv_callback(struct net_device *net,
+                        struct vmbus_channel *channel,
+                        void  *data, u32 len,
+                        const struct ndis_tcp_ip_checksum_info *csum_info,
+                        const struct ndis_pkt_8021q_info *vlan);
 void netvsc_channel_cb(void *context);
 int rndis_filter_open(struct netvsc_device *nvdev);
 int rndis_filter_close(struct netvsc_device *nvdev);
@@ -200,10 +200,11 @@ int rndis_filter_device_add(struct hv_device *dev,
 void rndis_filter_device_remove(struct hv_device *dev);
 int rndis_filter_set_rss_param(struct rndis_device *rdev,
                               const u8 *key, int num_queue);
-int rndis_filter_receive(struct hv_device *dev,
-                       struct hv_netvsc_packet *pkt,
-                       void **data,
-                       struct vmbus_channel *channel);
+int rndis_filter_receive(struct net_device *ndev,
+                        struct netvsc_device *net_dev,
+                        struct hv_device *dev,
+                        struct vmbus_channel *channel,
+                        void *data, u32 buflen);
 
 int rndis_filter_set_packet_filter(struct rndis_device *dev, u32 new_filter);
 int rndis_filter_set_device_mac(struct net_device *ndev, char *mac);
index 4dd2a1f2da115d9470f3cff87283f35f8e3750db..80eecb4f9456c8a9853382683a67d4fdaa6ffa71 100644 (file)
@@ -1095,51 +1095,35 @@ static inline struct recv_comp_data *get_recv_comp_slot(
        return rcd;
 }
 
-static void netvsc_receive(struct netvsc_device *net_device,
-                       struct vmbus_channel *channel,
-                       struct hv_device *device,
-                       struct vmpacket_descriptor *packet)
+static void netvsc_receive(struct net_device *ndev,
+                  struct netvsc_device *net_device,
+                  struct net_device_context *net_device_ctx,
+                  struct hv_device *device,
+                  struct vmbus_channel *channel,
+                  struct vmtransfer_page_packet_header *vmxferpage_packet,
+                  struct nvsp_message *nvsp)
 {
-       struct vmtransfer_page_packet_header *vmxferpage_packet;
-       struct nvsp_message *nvsp_packet;
-       struct hv_netvsc_packet nv_pkt;
-       struct hv_netvsc_packet *netvsc_packet = &nv_pkt;
+       char *recv_buf = net_device->recv_buf;
        u32 status = NVSP_STAT_SUCCESS;
        int i;
        int count = 0;
-       struct net_device *ndev = hv_get_drvdata(device);
-       void *data;
        int ret;
        struct recv_comp_data *rcd;
        u16 q_idx = channel->offermsg.offer.sub_channel_index;
 
-       /*
-        * All inbound packets other than send completion should be xfer page
-        * packet
-        */
-       if (packet->type != VM_PKT_DATA_USING_XFER_PAGES) {
-               netdev_err(ndev, "Unknown packet type received - %d\n",
-                          packet->type);
-               return;
-       }
-
-       nvsp_packet = (struct nvsp_message *)((unsigned long)packet +
-                       (packet->offset8 << 3));
-
        /* Make sure this is a valid nvsp packet */
-       if (nvsp_packet->hdr.msg_type !=
-           NVSP_MSG1_TYPE_SEND_RNDIS_PKT) {
-               netdev_err(ndev, "Unknown nvsp packet type received-"
-                       " %d\n", nvsp_packet->hdr.msg_type);
+       if (unlikely(nvsp->hdr.msg_type != NVSP_MSG1_TYPE_SEND_RNDIS_PKT)) {
+               netif_err(net_device_ctx, rx_err, ndev,
+                         "Unknown nvsp packet type received %u\n",
+                         nvsp->hdr.msg_type);
                return;
        }
 
-       vmxferpage_packet = (struct vmtransfer_page_packet_header *)packet;
-
-       if (vmxferpage_packet->xfer_pageset_id != NETVSC_RECEIVE_BUFFER_ID) {
-               netdev_err(ndev, "Invalid xfer page set id - "
-                          "expecting %x got %x\n", NETVSC_RECEIVE_BUFFER_ID,
-                          vmxferpage_packet->xfer_pageset_id);
+       if (unlikely(vmxferpage_packet->xfer_pageset_id != NETVSC_RECEIVE_BUFFER_ID)) {
+               netif_err(net_device_ctx, rx_err, ndev,
+                         "Invalid xfer page set id - expecting %x got %x\n",
+                         NETVSC_RECEIVE_BUFFER_ID,
+                         vmxferpage_packet->xfer_pageset_id);
                return;
        }
 
@@ -1147,15 +1131,13 @@ static void netvsc_receive(struct netvsc_device *net_device,
 
        /* Each range represents 1 RNDIS pkt that contains 1 ethernet frame */
        for (i = 0; i < count; i++) {
-               /* Initialize the netvsc packet */
-               data = (void *)((unsigned long)net_device->
-                       recv_buf + vmxferpage_packet->ranges[i].byte_offset);
-               netvsc_packet->total_data_buflen =
-                                       vmxferpage_packet->ranges[i].byte_count;
+               void *data = recv_buf
+                       + vmxferpage_packet->ranges[i].byte_offset;
+               u32 buflen = vmxferpage_packet->ranges[i].byte_count;
 
                /* Pass it to the upper layer */
-               status = rndis_filter_receive(device, netvsc_packet, &data,
-                                             channel);
+               status = rndis_filter_receive(ndev, net_device, device,
+                                             channel, data, buflen);
        }
 
        if (!net_device->chan_table[q_idx].mrc.buf) {
@@ -1234,11 +1216,10 @@ static void netvsc_process_raw_pkt(struct hv_device *device,
                                   u64 request_id,
                                   struct vmpacket_descriptor *desc)
 {
-       struct nvsp_message *nvmsg;
        struct net_device_context *net_device_ctx = netdev_priv(ndev);
-
-       nvmsg = (struct nvsp_message *)((unsigned long)
-               desc + (desc->offset8 << 3));
+       struct nvsp_message *nvmsg
+               = (struct nvsp_message *)((unsigned long)desc
+                                         + (desc->offset8 << 3));
 
        switch (desc->type) {
        case VM_PKT_COMP:
@@ -1246,7 +1227,10 @@ static void netvsc_process_raw_pkt(struct hv_device *device,
                break;
 
        case VM_PKT_DATA_USING_XFER_PAGES:
-               netvsc_receive(net_device, channel, device, desc);
+               netvsc_receive(ndev, net_device, net_device_ctx,
+                              device, channel,
+                              (struct vmtransfer_page_packet_header *)desc,
+                              nvmsg);
                break;
 
        case VM_PKT_DATA_INBAND:
index da5863641703358ce7ec727c73b81a5f0034f1d7..4bc1fdbc8cd7b054c6ba56443731674208edd79f 100644 (file)
@@ -596,13 +596,13 @@ void netvsc_linkstatus_callback(struct hv_device *device_obj,
 }
 
 static struct sk_buff *netvsc_alloc_recv_skb(struct net_device *net,
-                               struct hv_netvsc_packet *packet,
-                               struct ndis_tcp_ip_checksum_info *csum_info,
-                               void *data, u16 vlan_tci)
+                                            const struct ndis_tcp_ip_checksum_info *csum_info,
+                                            const struct ndis_pkt_8021q_info *vlan,
+                                            void *data, u32 buflen)
 {
        struct sk_buff *skb;
 
-       skb = netdev_alloc_skb_ip_align(net, packet->total_data_buflen);
+       skb = netdev_alloc_skb_ip_align(net, buflen);
        if (!skb)
                return skb;
 
@@ -610,8 +610,7 @@ static struct sk_buff *netvsc_alloc_recv_skb(struct net_device *net,
         * Copy to skb. This copy is needed here since the memory pointed by
         * hv_netvsc_packet cannot be deallocated
         */
-       memcpy(skb_put(skb, packet->total_data_buflen), data,
-              packet->total_data_buflen);
+       memcpy(skb_put(skb, buflen), data, buflen);
 
        skb->protocol = eth_type_trans(skb, net);
 
@@ -628,9 +627,12 @@ static struct sk_buff *netvsc_alloc_recv_skb(struct net_device *net,
                        skb->ip_summed = CHECKSUM_UNNECESSARY;
        }
 
-       if (vlan_tci & VLAN_TAG_PRESENT)
+       if (vlan) {
+               u16 vlan_tci = vlan->vlanid | (vlan->pri << VLAN_PRIO_SHIFT);
+
                __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),
                                       vlan_tci);
+       }
 
        return skb;
 }
@@ -639,14 +641,12 @@ static struct sk_buff *netvsc_alloc_recv_skb(struct net_device *net,
  * netvsc_recv_callback -  Callback when we receive a packet from the
  * "wire" on the specified device.
  */
-int netvsc_recv_callback(struct hv_device *device_obj,
-                               struct hv_netvsc_packet *packet,
-                               void **data,
-                               struct ndis_tcp_ip_checksum_info *csum_info,
-                               struct vmbus_channel *channel,
-                               u16 vlan_tci)
+int netvsc_recv_callback(struct net_device *net,
+                        struct vmbus_channel *channel,
+                        void  *data, u32 len,
+                        const struct ndis_tcp_ip_checksum_info *csum_info,
+                        const struct ndis_pkt_8021q_info *vlan)
 {
-       struct net_device *net = hv_get_drvdata(device_obj);
        struct net_device_context *net_device_ctx = netdev_priv(net);
        struct net_device *vf_netdev;
        struct sk_buff *skb;
@@ -668,7 +668,7 @@ int netvsc_recv_callback(struct hv_device *device_obj,
                net = vf_netdev;
 
        /* Allocate a skb - TODO direct I/O to pages? */
-       skb = netvsc_alloc_recv_skb(net, packet, csum_info, *data, vlan_tci);
+       skb = netvsc_alloc_recv_skb(net, csum_info, vlan, data, len);
        if (unlikely(!skb)) {
                ++net->stats.rx_dropped;
                rcu_read_unlock();
@@ -687,7 +687,7 @@ int netvsc_recv_callback(struct hv_device *device_obj,
        rx_stats = this_cpu_ptr(net_device_ctx->rx_stats);
        u64_stats_update_begin(&rx_stats->syncp);
        rx_stats->packets++;
-       rx_stats->bytes += packet->total_data_buflen;
+       rx_stats->bytes += len;
 
        if (skb->pkt_type == PACKET_BROADCAST)
                ++rx_stats->broadcast;
index 0df02c9808e8fb34aec7e71da2f3b5dacc8444c6..decce4f8fda8da06a5a8aa42d0705d61c4293f3e 100644 (file)
@@ -132,7 +132,7 @@ static void put_rndis_request(struct rndis_device *dev,
 }
 
 static void dump_rndis_message(struct hv_device *hv_dev,
-                       struct rndis_message *rndis_msg)
+                              const struct rndis_message *rndis_msg)
 {
        struct net_device *netdev = hv_get_drvdata(hv_dev);
 
@@ -347,102 +347,78 @@ static inline void *rndis_get_ppi(struct rndis_packet *rpkt, u32 type)
        return NULL;
 }
 
-static int rndis_filter_receive_data(struct rndis_device *dev,
-                                  struct rndis_message *msg,
-                                  struct hv_netvsc_packet *pkt,
-                                  void **data,
-                                  struct vmbus_channel *channel)
+static int rndis_filter_receive_data(struct net_device *ndev,
+                                    struct rndis_device *dev,
+                                    struct rndis_message *msg,
+                                    struct vmbus_channel *channel,
+                                    void *data, u32 data_buflen)
 {
-       struct rndis_packet *rndis_pkt;
+       struct rndis_packet *rndis_pkt = &msg->msg.pkt;
+       const struct ndis_tcp_ip_checksum_info *csum_info;
+       const struct ndis_pkt_8021q_info *vlan;
        u32 data_offset;
-       struct ndis_pkt_8021q_info *vlan;
-       struct ndis_tcp_ip_checksum_info *csum_info;
-       u16 vlan_tci = 0;
-       struct net_device_context *net_device_ctx = netdev_priv(dev->ndev);
-
-       rndis_pkt = &msg->msg.pkt;
 
        /* Remove the rndis header and pass it back up the stack */
        data_offset = RNDIS_HEADER_SIZE + rndis_pkt->data_offset;
 
-       pkt->total_data_buflen -= data_offset;
+       data_buflen -= data_offset;
 
        /*
         * Make sure we got a valid RNDIS message, now total_data_buflen
         * should be the data packet size plus the trailer padding size
         */
-       if (pkt->total_data_buflen < rndis_pkt->data_len) {
+       if (unlikely(data_buflen < rndis_pkt->data_len)) {
                netdev_err(dev->ndev, "rndis message buffer "
                           "overflow detected (got %u, min %u)"
                           "...dropping this message!\n",
-                          pkt->total_data_buflen, rndis_pkt->data_len);
+                          data_buflen, rndis_pkt->data_len);
                return NVSP_STAT_FAIL;
        }
 
+       vlan = rndis_get_ppi(rndis_pkt, IEEE_8021Q_INFO);
+
        /*
         * Remove the rndis trailer padding from rndis packet message
         * rndis_pkt->data_len tell us the real data length, we only copy
         * the data packet to the stack, without the rndis trailer padding
         */
-       pkt->total_data_buflen = rndis_pkt->data_len;
-       *data = (void *)((unsigned long)(*data) + data_offset);
-
-       vlan = rndis_get_ppi(rndis_pkt, IEEE_8021Q_INFO);
-       if (vlan) {
-               vlan_tci = VLAN_TAG_PRESENT | vlan->vlanid |
-                       (vlan->pri << VLAN_PRIO_SHIFT);
-       }
-
+       data = (void *)((unsigned long)data + data_offset);
        csum_info = rndis_get_ppi(rndis_pkt, TCPIP_CHKSUM_PKTINFO);
-       return netvsc_recv_callback(net_device_ctx->device_ctx, pkt, data,
-                                   csum_info, channel, vlan_tci);
+       return netvsc_recv_callback(ndev, channel,
+                                   data, rndis_pkt->data_len,
+                                   csum_info, vlan);
 }
 
-int rndis_filter_receive(struct hv_device *dev,
-                               struct hv_netvsc_packet *pkt,
-                               void **data,
-                               struct vmbus_channel *channel)
+int rndis_filter_receive(struct net_device *ndev,
+                        struct netvsc_device *net_dev,
+                        struct hv_device *dev,
+                        struct vmbus_channel *channel,
+                        void *data, u32 buflen)
 {
-       struct net_device *ndev = hv_get_drvdata(dev);
        struct net_device_context *net_device_ctx = netdev_priv(ndev);
-       struct netvsc_device *net_dev = net_device_ctx->nvdev;
-       struct rndis_device *rndis_dev;
-       struct rndis_message *rndis_msg;
-       int ret = 0;
-
-       if (!net_dev) {
-               ret = NVSP_STAT_FAIL;
-               goto exit;
-       }
+       struct rndis_device *rndis_dev = net_dev->extension;
+       struct rndis_message *rndis_msg = data;
 
        /* Make sure the rndis device state is initialized */
-       if (!net_dev->extension) {
-               netdev_err(ndev, "got rndis message but no rndis device - "
-                         "dropping this message!\n");
-               ret = NVSP_STAT_FAIL;
-               goto exit;
+       if (unlikely(!rndis_dev)) {
+               netif_err(net_device_ctx, rx_err, ndev,
+                         "got rndis message but no rndis device!\n");
+               return NVSP_STAT_FAIL;
        }
 
-       rndis_dev = (struct rndis_device *)net_dev->extension;
-       if (rndis_dev->state == RNDIS_DEV_UNINITIALIZED) {
-               netdev_err(ndev, "got rndis message but rndis device "
-                          "uninitialized...dropping this message!\n");
-               ret = NVSP_STAT_FAIL;
-               goto exit;
+       if (unlikely(rndis_dev->state == RNDIS_DEV_UNINITIALIZED)) {
+               netif_err(net_device_ctx, rx_err, ndev,
+                         "got rndis message uninitialized\n");
+               return NVSP_STAT_FAIL;
        }
 
-       rndis_msg = *data;
-
-       if (netif_msg_rx_err(net_device_ctx))
+       if (netif_msg_rx_status(net_device_ctx))
                dump_rndis_message(dev, rndis_msg);
 
        switch (rndis_msg->ndis_msg_type) {
        case RNDIS_MSG_PACKET:
-               /* data msg */
-               ret = rndis_filter_receive_data(rndis_dev, rndis_msg, pkt,
-                                               data, channel);
-               break;
-
+               return rndis_filter_receive_data(ndev, rndis_dev, rndis_msg,
+                                                channel, data, buflen);
        case RNDIS_MSG_INIT_C:
        case RNDIS_MSG_QUERY_C:
        case RNDIS_MSG_SET_C:
@@ -462,8 +438,7 @@ int rndis_filter_receive(struct hv_device *dev,
                break;
        }
 
-exit:
-       return ret;
+       return 0;
 }
 
 static int rndis_filter_query_device(struct rndis_device *dev, u32 oid,