drm/i915/gvt: Introduce basic vGPU life cycle management
authorZhi Wang <zhi.a.wang@intel.com>
Tue, 5 Jul 2016 16:40:49 +0000 (12:40 -0400)
committerZhenyu Wang <zhenyuw@linux.intel.com>
Fri, 14 Oct 2016 10:11:59 +0000 (18:11 +0800)
A vGPU represents a virtual Intel GEN hardware, which consists following
virtual resources:

- Configuration space (virtualized)
- HW registers (virtualized)
- GGTT memory space (partitioned)
- GPU page table (shadowed)
- Fence registers (partitioned)

* virtualized: fully emulated by GVT-g.
* partitioned: Only a part of the HW resource is allowed to be accessed
by VM.
* shadowed: Resource needs to be translated and shadowed before getting
applied into HW.

This patch introduces vGPU life cycle management framework, which is
responsible for creating/destroying a vGPU and preparing/free resources
related to a vGPU.

Signed-off-by: Zhi Wang <zhi.a.wang@intel.com>
Signed-off-by: Zhenyu Wang <zhenyuw@linux.intel.com>
drivers/gpu/drm/i915/gvt/Makefile
drivers/gpu/drm/i915/gvt/gvt.h
drivers/gpu/drm/i915/gvt/hypercall.h
drivers/gpu/drm/i915/gvt/mpt.h
drivers/gpu/drm/i915/gvt/reg.h [new file with mode: 0644]
drivers/gpu/drm/i915/gvt/vgpu.c [new file with mode: 0644]

index ebbb90a6dca57a61515346f4d1e8b78d85051664..18216bcbf1342b9c20bc394ef32e4ca0a3cd9292 100644 (file)
@@ -1,5 +1,5 @@
 GVT_DIR := gvt
-GVT_SOURCE := gvt.o aperture_gm.o handlers.o firmware.o
+GVT_SOURCE := gvt.o aperture_gm.o handlers.o firmware.o vgpu.o
 
 ccflags-y                      += -I$(src) -I$(src)/$(GVT_DIR) -Wall
 i915-y                        += $(addprefix $(GVT_DIR)/, $(GVT_SOURCE))
index e8759680e9526fb61ec1db7023b4fa624ed7e7d9..e00c2d66f6580c60db03798af2ee055f2c6d299c 100644 (file)
@@ -36,6 +36,7 @@
 #include "debug.h"
 #include "hypercall.h"
 #include "mmio.h"
+#include "reg.h"
 
 #define GVT_MAX_VGPU 8
 
@@ -77,13 +78,37 @@ struct intel_vgpu_fence {
        u32 size;
 };
 
+struct intel_vgpu_mmio {
+       void *vreg;
+       void *sreg;
+};
+
+#define INTEL_GVT_MAX_CFG_SPACE_SZ 256
+#define INTEL_GVT_MAX_BAR_NUM 4
+
+struct intel_vgpu_pci_bar {
+       u64 size;
+       bool tracked;
+};
+
+struct intel_vgpu_cfg_space {
+       unsigned char virtual_cfg_space[INTEL_GVT_MAX_CFG_SPACE_SZ];
+       struct intel_vgpu_pci_bar bar[INTEL_GVT_MAX_BAR_NUM];
+};
+
+#define vgpu_cfg_space(vgpu) ((vgpu)->cfg_space.virtual_cfg_space)
+
 struct intel_vgpu {
        struct intel_gvt *gvt;
        int id;
        unsigned long handle; /* vGPU handle used by hypervisor MPT modules */
+       bool active;
+       bool resetting;
 
        struct intel_vgpu_fence fence;
        struct intel_vgpu_gm gm;
+       struct intel_vgpu_cfg_space cfg_space;
+       struct intel_vgpu_mmio mmio;
 };
 
 struct intel_gvt_gm {
@@ -183,6 +208,52 @@ void intel_vgpu_free_resource(struct intel_vgpu *vgpu);
 void intel_vgpu_write_fence(struct intel_vgpu *vgpu,
        u32 fence, u64 value);
 
+/* Macros for easily accessing vGPU virtual/shadow register */
+#define vgpu_vreg(vgpu, reg) \
+       (*(u32 *)(vgpu->mmio.vreg + INTEL_GVT_MMIO_OFFSET(reg)))
+#define vgpu_vreg8(vgpu, reg) \
+       (*(u8 *)(vgpu->mmio.vreg + INTEL_GVT_MMIO_OFFSET(reg)))
+#define vgpu_vreg16(vgpu, reg) \
+       (*(u16 *)(vgpu->mmio.vreg + INTEL_GVT_MMIO_OFFSET(reg)))
+#define vgpu_vreg64(vgpu, reg) \
+       (*(u64 *)(vgpu->mmio.vreg + INTEL_GVT_MMIO_OFFSET(reg)))
+#define vgpu_sreg(vgpu, reg) \
+       (*(u32 *)(vgpu->mmio.sreg + INTEL_GVT_MMIO_OFFSET(reg)))
+#define vgpu_sreg8(vgpu, reg) \
+       (*(u8 *)(vgpu->mmio.sreg + INTEL_GVT_MMIO_OFFSET(reg)))
+#define vgpu_sreg16(vgpu, reg) \
+       (*(u16 *)(vgpu->mmio.sreg + INTEL_GVT_MMIO_OFFSET(reg)))
+#define vgpu_sreg64(vgpu, reg) \
+       (*(u64 *)(vgpu->mmio.sreg + INTEL_GVT_MMIO_OFFSET(reg)))
+
+#define for_each_active_vgpu(gvt, vgpu, id) \
+       idr_for_each_entry((&(gvt)->vgpu_idr), (vgpu), (id)) \
+               for_each_if(vgpu->active)
+
+static inline void intel_vgpu_write_pci_bar(struct intel_vgpu *vgpu,
+                                           u32 offset, u32 val, bool low)
+{
+       u32 *pval;
+
+       /* BAR offset should be 32 bits algiend */
+       offset = rounddown(offset, 4);
+       pval = (u32 *)(vgpu_cfg_space(vgpu) + offset);
+
+       if (low) {
+               /*
+                * only update bit 31 - bit 4,
+                * leave the bit 3 - bit 0 unchanged.
+                */
+               *pval = (val & GENMASK(31, 4)) | (*pval & GENMASK(3, 0));
+       }
+}
+
+struct intel_vgpu *intel_gvt_create_vgpu(struct intel_gvt *gvt,
+                                        struct intel_vgpu_creation_params *
+                                        param);
+
+void intel_gvt_destroy_vgpu(struct intel_vgpu *vgpu);
+
 #include "mpt.h"
 
 #endif
index 254df8bf1f355cda4ee54cc823ec5a0559cfd77c..eff8af56e01282756bb4a0bb2965fea879c2239d 100644 (file)
  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  * SOFTWARE.
+ *
+ * Authors:
+ *    Eddie Dong <eddie.dong@intel.com>
+ *    Dexuan Cui
+ *    Jike Song <jike.song@intel.com>
+ *
+ * Contributors:
+ *    Zhi Wang <zhi.a.wang@intel.com>
+ *
  */
 
 #ifndef _GVT_HYPERCALL_H_
@@ -30,6 +39,8 @@
  */
 struct intel_gvt_mpt {
        int (*detect_host)(void);
+       int (*attach_vgpu)(void *vgpu, unsigned long *handle);
+       void (*detach_vgpu)(unsigned long handle);
 };
 
 extern struct intel_gvt_mpt xengt_mpt;
index 03601e3ffa7c80cf0f8e8b884141e74db7f7b821..f78186884a5c0f1d47b8b6cbf79ab781048998ed 100644 (file)
  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  * SOFTWARE.
+ *
+ * Authors:
+ *    Eddie Dong <eddie.dong@intel.com>
+ *    Dexuan Cui
+ *    Jike Song <jike.song@intel.com>
+ *
+ * Contributors:
+ *    Zhi Wang <zhi.a.wang@intel.com>
+ *
  */
 
 #ifndef _GVT_MPT_H_
@@ -46,4 +55,28 @@ static inline int intel_gvt_hypervisor_detect_host(void)
        return intel_gvt_host.mpt->detect_host();
 }
 
+/**
+ * intel_gvt_hypervisor_attach_vgpu - call hypervisor to initialize vGPU
+ * related stuffs inside hypervisor.
+ *
+ * Returns:
+ * Zero on success, negative error code if failed.
+ */
+static inline int intel_gvt_hypervisor_attach_vgpu(struct intel_vgpu *vgpu)
+{
+       return intel_gvt_host.mpt->attach_vgpu(vgpu, &vgpu->handle);
+}
+
+/**
+ * intel_gvt_hypervisor_detach_vgpu - call hypervisor to release vGPU
+ * related stuffs inside hypervisor.
+ *
+ * Returns:
+ * Zero on success, negative error code if failed.
+ */
+static inline void intel_gvt_hypervisor_detach_vgpu(struct intel_vgpu *vgpu)
+{
+       intel_gvt_host.mpt->detach_vgpu(vgpu->handle);
+}
+
 #endif /* _GVT_MPT_H_ */
diff --git a/drivers/gpu/drm/i915/gvt/reg.h b/drivers/gpu/drm/i915/gvt/reg.h
new file mode 100644 (file)
index 0000000..3172fb3
--- /dev/null
@@ -0,0 +1,33 @@
+/*
+ * Copyright(c) 2011-2016 Intel Corporation. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#ifndef _GVT_REG_H
+#define _GVT_REG_H
+
+#define INTEL_GVT_PCI_CLASS_VGA_OTHER   0x80
+
+#define INTEL_GVT_PCI_GMCH_CONTROL     0x50
+#define   BDW_GMCH_GMS_SHIFT           8
+#define   BDW_GMCH_GMS_MASK            0xff
+
+#endif
diff --git a/drivers/gpu/drm/i915/gvt/vgpu.c b/drivers/gpu/drm/i915/gvt/vgpu.c
new file mode 100644 (file)
index 0000000..0f1a59b
--- /dev/null
@@ -0,0 +1,215 @@
+/*
+ * Copyright(c) 2011-2016 Intel Corporation. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * Authors:
+ *    Eddie Dong <eddie.dong@intel.com>
+ *    Kevin Tian <kevin.tian@intel.com>
+ *
+ * Contributors:
+ *    Ping Gao <ping.a.gao@intel.com>
+ *    Zhi Wang <zhi.a.wang@intel.com>
+ *    Bing Niu <bing.niu@intel.com>
+ *
+ */
+
+#include "i915_drv.h"
+
+static void clean_vgpu_mmio(struct intel_vgpu *vgpu)
+{
+       vfree(vgpu->mmio.vreg);
+       vgpu->mmio.vreg = vgpu->mmio.sreg = NULL;
+}
+
+static int setup_vgpu_mmio(struct intel_vgpu *vgpu)
+{
+       struct intel_gvt *gvt = vgpu->gvt;
+       const struct intel_gvt_device_info *info = &gvt->device_info;
+
+       vgpu->mmio.vreg = vzalloc(info->mmio_size * 2);
+       if (!vgpu->mmio.vreg)
+               return -ENOMEM;
+
+       vgpu->mmio.sreg = vgpu->mmio.vreg + info->mmio_size;
+
+       memcpy(vgpu->mmio.vreg, gvt->firmware.mmio, info->mmio_size);
+       memcpy(vgpu->mmio.sreg, gvt->firmware.mmio, info->mmio_size);
+       return 0;
+}
+
+static void setup_vgpu_cfg_space(struct intel_vgpu *vgpu,
+       struct intel_vgpu_creation_params *param)
+{
+       struct intel_gvt *gvt = vgpu->gvt;
+       const struct intel_gvt_device_info *info = &gvt->device_info;
+       u16 *gmch_ctl;
+       int i;
+
+       memcpy(vgpu_cfg_space(vgpu), gvt->firmware.cfg_space,
+              info->cfg_space_size);
+
+       if (!param->primary) {
+               vgpu_cfg_space(vgpu)[PCI_CLASS_DEVICE] =
+                       INTEL_GVT_PCI_CLASS_VGA_OTHER;
+               vgpu_cfg_space(vgpu)[PCI_CLASS_PROG] =
+                       INTEL_GVT_PCI_CLASS_VGA_OTHER;
+       }
+
+       /* Show guest that there isn't any stolen memory.*/
+       gmch_ctl = (u16 *)(vgpu_cfg_space(vgpu) + INTEL_GVT_PCI_GMCH_CONTROL);
+       *gmch_ctl &= ~(BDW_GMCH_GMS_MASK << BDW_GMCH_GMS_SHIFT);
+
+       intel_vgpu_write_pci_bar(vgpu, PCI_BASE_ADDRESS_2,
+                                gvt_aperture_pa_base(gvt), true);
+
+       vgpu_cfg_space(vgpu)[PCI_COMMAND] &= ~(PCI_COMMAND_IO
+                                            | PCI_COMMAND_MEMORY
+                                            | PCI_COMMAND_MASTER);
+       /*
+        * Clear the bar upper 32bit and let guest to assign the new value
+        */
+       memset(vgpu_cfg_space(vgpu) + PCI_BASE_ADDRESS_1, 0, 4);
+       memset(vgpu_cfg_space(vgpu) + PCI_BASE_ADDRESS_3, 0, 4);
+
+       for (i = 0; i < INTEL_GVT_MAX_BAR_NUM; i++) {
+               vgpu->cfg_space.bar[i].size = pci_resource_len(
+                                             gvt->dev_priv->drm.pdev, i * 2);
+               vgpu->cfg_space.bar[i].tracked = false;
+       }
+}
+
+static void populate_pvinfo_page(struct intel_vgpu *vgpu)
+{
+       /* setup the ballooning information */
+       vgpu_vreg64(vgpu, vgtif_reg(magic)) = VGT_MAGIC;
+       vgpu_vreg(vgpu, vgtif_reg(version_major)) = 1;
+       vgpu_vreg(vgpu, vgtif_reg(version_minor)) = 0;
+       vgpu_vreg(vgpu, vgtif_reg(display_ready)) = 0;
+       vgpu_vreg(vgpu, vgtif_reg(vgt_id)) = vgpu->id;
+       vgpu_vreg(vgpu, vgtif_reg(avail_rs.mappable_gmadr.base)) =
+               vgpu_aperture_gmadr_base(vgpu);
+       vgpu_vreg(vgpu, vgtif_reg(avail_rs.mappable_gmadr.size)) =
+               vgpu_aperture_sz(vgpu);
+       vgpu_vreg(vgpu, vgtif_reg(avail_rs.nonmappable_gmadr.base)) =
+               vgpu_hidden_gmadr_base(vgpu);
+       vgpu_vreg(vgpu, vgtif_reg(avail_rs.nonmappable_gmadr.size)) =
+               vgpu_hidden_sz(vgpu);
+
+       vgpu_vreg(vgpu, vgtif_reg(avail_rs.fence_num)) = vgpu_fence_sz(vgpu);
+
+       gvt_dbg_core("Populate PVINFO PAGE for vGPU %d\n", vgpu->id);
+       gvt_dbg_core("aperture base [GMADR] 0x%llx size 0x%llx\n",
+               vgpu_aperture_gmadr_base(vgpu), vgpu_aperture_sz(vgpu));
+       gvt_dbg_core("hidden base [GMADR] 0x%llx size=0x%llx\n",
+               vgpu_hidden_gmadr_base(vgpu), vgpu_hidden_sz(vgpu));
+       gvt_dbg_core("fence size %d\n", vgpu_fence_sz(vgpu));
+
+       WARN_ON(sizeof(struct vgt_if) != VGT_PVINFO_SIZE);
+}
+
+/**
+ * intel_gvt_destroy_vgpu - destroy a virtual GPU
+ * @vgpu: virtual GPU
+ *
+ * This function is called when user wants to destroy a virtual GPU.
+ *
+ */
+void intel_gvt_destroy_vgpu(struct intel_vgpu *vgpu)
+{
+       struct intel_gvt *gvt = vgpu->gvt;
+
+       mutex_lock(&gvt->lock);
+
+       vgpu->active = false;
+       idr_remove(&gvt->vgpu_idr, vgpu->id);
+
+       intel_gvt_hypervisor_detach_vgpu(vgpu);
+       intel_vgpu_free_resource(vgpu);
+       clean_vgpu_mmio(vgpu);
+       vfree(vgpu);
+
+       mutex_unlock(&gvt->lock);
+}
+
+/**
+ * intel_gvt_create_vgpu - create a virtual GPU
+ * @gvt: GVT device
+ * @param: vGPU creation parameters
+ *
+ * This function is called when user wants to create a virtual GPU.
+ *
+ * Returns:
+ * pointer to intel_vgpu, error pointer if failed.
+ */
+struct intel_vgpu *intel_gvt_create_vgpu(struct intel_gvt *gvt,
+               struct intel_vgpu_creation_params *param)
+{
+       struct intel_vgpu *vgpu;
+       int ret;
+
+       gvt_dbg_core("handle %llu low %llu MB high %llu MB fence %llu\n",
+                       param->handle, param->low_gm_sz, param->high_gm_sz,
+                       param->fence_sz);
+
+       vgpu = vzalloc(sizeof(*vgpu));
+       if (!vgpu)
+               return ERR_PTR(-ENOMEM);
+
+       mutex_lock(&gvt->lock);
+
+       ret = idr_alloc(&gvt->vgpu_idr, vgpu, 1, GVT_MAX_VGPU, GFP_KERNEL);
+       if (ret < 0)
+               goto out_free_vgpu;
+
+       vgpu->id = ret;
+       vgpu->handle = param->handle;
+       vgpu->gvt = gvt;
+
+       setup_vgpu_cfg_space(vgpu, param);
+
+       ret = setup_vgpu_mmio(vgpu);
+       if (ret)
+               goto out_free_vgpu;
+
+       ret = intel_vgpu_alloc_resource(vgpu, param);
+       if (ret)
+               goto out_clean_vgpu_mmio;
+
+       populate_pvinfo_page(vgpu);
+
+       ret = intel_gvt_hypervisor_attach_vgpu(vgpu);
+       if (ret)
+               goto out_clean_vgpu_resource;
+
+       vgpu->active = true;
+       mutex_unlock(&gvt->lock);
+
+       return vgpu;
+
+out_clean_vgpu_resource:
+       intel_vgpu_free_resource(vgpu);
+out_clean_vgpu_mmio:
+       clean_vgpu_mmio(vgpu);
+out_free_vgpu:
+       vfree(vgpu);
+       mutex_unlock(&gvt->lock);
+       return ERR_PTR(ret);
+}