KVM, pkeys: add pkeys support for permission_fault

Protection keys define a new 4-bit protection key field (PKEY) in bits
62:59 of leaf entries of the page tables, the PKEY is an index to PKRU
register(16 domains), every domain has 2 bits(write disable bit, access
disable bit).

Static logic has been produced in update_pkru_bitmask, dynamic logic need
read pkey from page table entries, get pkru value, and deduce the correct
result.

[ Huaitong: Xiao helps to modify many sections. ]

Signed-off-by: Huaitong Han <huaitong.han@intel.com>
Signed-off-by: Xiao Guangrong <guangrong.xiao@linux.intel.com>
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>

diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
index 8968165963e4c818345a3601e216e706fe601474..0d75ecdfa077a0f3ce55ed5e74a0f40374ce69c5 100644 (file)
--- a/arch/x86/include/asm/kvm_host.h
+++ b/arch/x86/include/asm/kvm_host.h
@@ -187,12 +187,14 @@ enum {
 #define PFERR_USER_BIT 2
 #define PFERR_RSVD_BIT 3
 #define PFERR_FETCH_BIT 4
+#define PFERR_PK_BIT 5
 
 #define PFERR_PRESENT_MASK (1U << PFERR_PRESENT_BIT)
 #define PFERR_WRITE_MASK (1U << PFERR_WRITE_BIT)
 #define PFERR_USER_MASK (1U << PFERR_USER_BIT)
 #define PFERR_RSVD_MASK (1U << PFERR_RSVD_BIT)
 #define PFERR_FETCH_MASK (1U << PFERR_FETCH_BIT)
+#define PFERR_PK_MASK (1U << PFERR_PK_BIT)
 
 /* apic attention bits */
 #define KVM_APIC_CHECK_VAPIC   0
@@ -882,6 +884,7 @@ struct kvm_x86_ops {
        void (*cache_reg)(struct kvm_vcpu *vcpu, enum kvm_reg reg);
        unsigned long (*get_rflags)(struct kvm_vcpu *vcpu);
        void (*set_rflags)(struct kvm_vcpu *vcpu, unsigned long rflags);
+       u32 (*get_pkru)(struct kvm_vcpu *vcpu);
        void (*fpu_activate)(struct kvm_vcpu *vcpu);
        void (*fpu_deactivate)(struct kvm_vcpu *vcpu);
 

diff --git a/arch/x86/kvm/kvm_cache_regs.h b/arch/x86/kvm/kvm_cache_regs.h
index e1e89ee4af750dc51f78cfbf8aa71e22d77b4cd1..762cdf2595f992fd4ac8bb1e4c2c8914b344db04 100644 (file)
--- a/arch/x86/kvm/kvm_cache_regs.h
+++ b/arch/x86/kvm/kvm_cache_regs.h
@@ -84,6 +84,11 @@ static inline u64 kvm_read_edx_eax(struct kvm_vcpu *vcpu)
                | ((u64)(kvm_register_read(vcpu, VCPU_REGS_RDX) & -1u) << 32);
 }
 
+static inline u32 kvm_read_pkru(struct kvm_vcpu *vcpu)
+{
+       return kvm_x86_ops->get_pkru(vcpu);
+}
+
 static inline void enter_guest_mode(struct kvm_vcpu *vcpu)
 {
        vcpu->arch.hflags |= HF_GUEST_MASK;

diff --git a/arch/x86/kvm/mmu.h b/arch/x86/kvm/mmu.h
index 8b2b3dfca1ae292742251dc434924c9ffe69517b..b70df72e2b33d417307d01f57bca57388650d34f 100644 (file)
--- a/arch/x86/kvm/mmu.h
+++ b/arch/x86/kvm/mmu.h
@@ -10,10 +10,11 @@
 #define PT32_ENT_PER_PAGE (1 << PT32_PT_BITS)
 
 #define PT_WRITABLE_SHIFT 1
+#define PT_USER_SHIFT 2
 
 #define PT_PRESENT_MASK (1ULL << 0)
 #define PT_WRITABLE_MASK (1ULL << PT_WRITABLE_SHIFT)
-#define PT_USER_MASK (1ULL << 2)
+#define PT_USER_MASK (1ULL << PT_USER_SHIFT)
 #define PT_PWT_MASK (1ULL << 3)
 #define PT_PCD_MASK (1ULL << 4)
 #define PT_ACCESSED_SHIFT 5
@@ -149,7 +150,8 @@ static inline bool is_write_protection(struct kvm_vcpu *vcpu)
  * if the access faults.
  */
 static inline u8 permission_fault(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
-                                 unsigned pte_access, unsigned pfec)
+                                 unsigned pte_access, unsigned pte_pkey,
+                                 unsigned pfec)
 {
        int cpl = kvm_x86_ops->get_cpl(vcpu);
        unsigned long rflags = kvm_x86_ops->get_rflags(vcpu);
@@ -170,11 +172,32 @@ static inline u8 permission_fault(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
        unsigned long smap = (cpl - 3) & (rflags & X86_EFLAGS_AC);
        int index = (pfec >> 1) +
                    (smap >> (X86_EFLAGS_AC_BIT - PFERR_RSVD_BIT + 1));
+       bool fault = (mmu->permissions[index] >> pte_access) & 1;
 
-       WARN_ON(pfec & PFERR_RSVD_MASK);
-
+       WARN_ON(pfec & (PFERR_PK_MASK | PFERR_RSVD_MASK));
        pfec |= PFERR_PRESENT_MASK;
-       return -((mmu->permissions[index] >> pte_access) & 1) & pfec;
+
+       if (unlikely(mmu->pkru_mask)) {
+               u32 pkru_bits, offset;
+
+               /*
+               * PKRU defines 32 bits, there are 16 domains and 2
+               * attribute bits per domain in pkru.  pte_pkey is the
+               * index of the protection domain, so pte_pkey * 2 is
+               * is the index of the first bit for the domain.
+               */
+               pkru_bits = (kvm_read_pkru(vcpu) >> (pte_pkey * 2)) & 3;
+
+               /* clear present bit, replace PFEC.RSVD with ACC_USER_MASK. */
+               offset = pfec - 1 +
+                       ((pte_access & PT_USER_MASK) << (PFERR_RSVD_BIT - PT_USER_SHIFT));
+
+               pkru_bits &= mmu->pkru_mask >> offset;
+               pfec |= -pkru_bits & PFERR_PK_MASK;
+               fault |= (pkru_bits != 0);
+       }
+
+       return -(uint32_t)fault & pfec;
 }
 
 void kvm_mmu_invalidate_zap_all_pages(struct kvm *kvm);

diff --git a/arch/x86/kvm/paging_tmpl.h b/arch/x86/kvm/paging_tmpl.h
index 45ed3c38d20e6ce0a9f2924664ae86a46755ae38..e0c2254211574c29427be353aa68d4c856606ac4 100644 (file)
--- a/arch/x86/kvm/paging_tmpl.h
+++ b/arch/x86/kvm/paging_tmpl.h
@@ -257,6 +257,17 @@ static int FNAME(update_accessed_dirty_bits)(struct kvm_vcpu *vcpu,
        return 0;
 }
 
+static inline unsigned FNAME(gpte_pkeys)(struct kvm_vcpu *vcpu, u64 gpte)
+{
+       unsigned pkeys = 0;
+#if PTTYPE == 64
+       pte_t pte = {.pte = gpte};
+
+       pkeys = pte_flags_pkey(pte_flags(pte));
+#endif
+       return pkeys;
+}
+
 /*
  * Fetch a guest pte for a guest virtual address
  */
@@ -268,7 +279,7 @@ static int FNAME(walk_addr_generic)(struct guest_walker *walker,
        pt_element_t pte;
        pt_element_t __user *uninitialized_var(ptep_user);
        gfn_t table_gfn;
-       unsigned index, pt_access, pte_access, accessed_dirty;
+       unsigned index, pt_access, pte_access, accessed_dirty, pte_pkey;
        gpa_t pte_gpa;
        int offset;
        const int write_fault = access & PFERR_WRITE_MASK;
@@ -359,7 +370,8 @@ retry_walk:
                walker->ptes[walker->level - 1] = pte;
        } while (!is_last_gpte(mmu, walker->level, pte));
 
-       errcode = permission_fault(vcpu, mmu, pte_access, access);
+       pte_pkey = FNAME(gpte_pkeys)(vcpu, pte);
+       errcode = permission_fault(vcpu, mmu, pte_access, pte_pkey, access);
        if (unlikely(errcode))
                goto error;
 

diff --git a/arch/x86/kvm/svm.c b/arch/x86/kvm/svm.c
index 95070386d5991baef11e2934349727af714cfce3..31346a3f20a5c8b5384e6fda2a81029059cdf621 100644 (file)
--- a/arch/x86/kvm/svm.c
+++ b/arch/x86/kvm/svm.c
@@ -1280,6 +1280,11 @@ static void svm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
        to_svm(vcpu)->vmcb->save.rflags = rflags;
 }
 
+static u32 svm_get_pkru(struct kvm_vcpu *vcpu)
+{
+       return 0;
+}
+
 static void svm_cache_reg(struct kvm_vcpu *vcpu, enum kvm_reg reg)
 {
        switch (reg) {
@@ -4347,6 +4352,9 @@ static struct kvm_x86_ops svm_x86_ops = {
        .cache_reg = svm_cache_reg,
        .get_rflags = svm_get_rflags,
        .set_rflags = svm_set_rflags,
+
+       .get_pkru = svm_get_pkru,
+
        .fpu_activate = svm_fpu_activate,
        .fpu_deactivate = svm_fpu_deactivate,
 

diff --git a/arch/x86/kvm/vmx.c b/arch/x86/kvm/vmx.c
index 87664775b51ed736496aa6a25795b9dd7a11dc47..efc243e4dabfc75d44240fab88052598c0d3da26 100644 (file)
--- a/arch/x86/kvm/vmx.c
+++ b/arch/x86/kvm/vmx.c
@@ -2292,6 +2292,11 @@ static void vmx_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
        vmcs_writel(GUEST_RFLAGS, rflags);
 }
 
+static u32 vmx_get_pkru(struct kvm_vcpu *vcpu)
+{
+       return to_vmx(vcpu)->guest_pkru;
+}
+
 static u32 vmx_get_interrupt_shadow(struct kvm_vcpu *vcpu)
 {
        u32 interruptibility = vmcs_read32(GUEST_INTERRUPTIBILITY_INFO);
@@ -10925,6 +10930,9 @@ static struct kvm_x86_ops vmx_x86_ops = {
        .cache_reg = vmx_cache_reg,
        .get_rflags = vmx_get_rflags,
        .set_rflags = vmx_set_rflags,
+
+       .get_pkru = vmx_get_pkru,
+
        .fpu_activate = vmx_fpu_activate,
        .fpu_deactivate = vmx_fpu_deactivate,
 

diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index 7236bd3a4c3d7a0c5a6148decc6fad276eb18bb7..d0a5918da6544959a2d4807c0520a4cb43e65851 100644 (file)
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -4326,9 +4326,14 @@ static int vcpu_mmio_gva_to_gpa(struct kvm_vcpu *vcpu, unsigned long gva,
        u32 access = ((kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0)
                | (write ? PFERR_WRITE_MASK : 0);
 
+       /*
+        * currently PKRU is only applied to ept enabled guest so
+        * there is no pkey in EPT page table for L1 guest or EPT
+        * shadow page table for L2 guest.
+        */
        if (vcpu_match_mmio_gva(vcpu, gva)
            && !permission_fault(vcpu, vcpu->arch.walk_mmu,
-                                vcpu->arch.access, access)) {
+                                vcpu->arch.access, 0, access)) {
                *gpa = vcpu->arch.mmio_gfn << PAGE_SHIFT |
                                        (gva & (PAGE_SIZE - 1));
                trace_vcpu_match_mmio(gva, *gpa, write, false);