drm/i915/gen8+: Add RC6 CTX corruption WA

commit 7e34f4e4aad3fd34c02b294a3cf2321adf5b4438 upstream.

In some circumstances the RC6 context can get corrupted. We can detect
this and take the required action, that is disable RC6 and runtime PM.
The HW recovers from the corrupted state after a system suspend/resume
cycle, so detect the recovery and re-enable RC6 and runtime PM.

v2: rebase (Mika)
v3:
- Move intel_suspend_gt_powersave() to the end of the GEM suspend
  sequence.
- Add commit message.
v4:
- Rebased on intel_uncore_forcewake_put(i915->uncore, ...) API
  change.
v5:
- Rebased on latest upstream gt_pm refactoring.

Signed-off-by: Imre Deak <imre.deak@intel.com>
Signed-off-by: Mika Kuoppala <mika.kuoppala@linux.intel.com>
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>

diff --git a/drivers/gpu/drm/i915/i915_drv.c b/drivers/gpu/drm/i915/i915_drv.c
index 9b642dd040cfd7faa0cd462b8bd20a779ab0cde4..02a2af7c8166fbc7cf814517f40ba83670db2647 100644 (file)
--- a/drivers/gpu/drm/i915/i915_drv.c
+++ b/drivers/gpu/drm/i915/i915_drv.c
@@ -1564,6 +1564,7 @@ static int i915_drm_suspend_late(struct drm_device *dev, bool hibernation)
        disable_rpm_wakeref_asserts(dev_priv);
 
        intel_display_set_init_power(dev_priv, false);
+       i915_rc6_ctx_wa_suspend(dev_priv);
 
        fw_csr = !IS_GEN9_LP(dev_priv) &&
                suspend_to_idle(dev_priv) && dev_priv->csr.dmc_payload;
@@ -1800,6 +1801,7 @@ static int i915_drm_resume_early(struct drm_device *dev)
                intel_display_set_init_power(dev_priv, true);
 
        i915_gem_sanitize(dev_priv);
+       i915_rc6_ctx_wa_resume(dev_priv);
 
        enable_rpm_wakeref_asserts(dev_priv);
 

diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h
index e7e5f94e97f75bd07d8dfa37e614cf52b22a21fd..a5fb7404b29ea121854ca85816e6e37e989d4b21 100644 (file)
--- a/drivers/gpu/drm/i915/i915_drv.h
+++ b/drivers/gpu/drm/i915/i915_drv.h
@@ -1320,6 +1320,7 @@ struct intel_gen6_power_mgmt {
        enum { LOW_POWER, BETWEEN, HIGH_POWER } power;
 
        bool enabled;
+       bool ctx_corrupted;
        struct delayed_work autoenable_work;
        atomic_t num_waiters;
        atomic_t boosts;
@@ -3025,9 +3026,12 @@ intel_info(const struct drm_i915_private *dev_priv)
 /* Early gen2 have a totally busted CS tlb and require pinned batches. */
 #define HAS_BROKEN_CS_TLB(dev_priv)    (IS_I830(dev_priv) || IS_I845G(dev_priv))
 
+#define NEEDS_RC6_CTX_CORRUPTION_WA(dev_priv)  \
+       (IS_BROADWELL(dev_priv) || INTEL_GEN(dev_priv) == 9)
+
 /* WaRsDisableCoarsePowerGating:skl,bxt */
 #define NEEDS_WaRsDisableCoarsePowerGating(dev_priv) \
-       (IS_SKL_GT3(dev_priv) || IS_SKL_GT4(dev_priv))
+       (INTEL_GEN(dev_priv) == 9)
 
 /*
  * dp aux and gmbus irq on gen4 seems to be able to generate legacy interrupts

diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c
index 785d192b27c804478a6031419ea21cc2399fd4dd..9263b65720bc6197eba3fa5f8b2e51b647b8b79a 100644 (file)
--- a/drivers/gpu/drm/i915/i915_gem.c
+++ b/drivers/gpu/drm/i915/i915_gem.c
@@ -3243,6 +3243,12 @@ i915_gem_idle_work_handler(struct work_struct *work)
 
        if (INTEL_GEN(dev_priv) >= 6)
                gen6_rps_idle(dev_priv);
+
+       if (NEEDS_RC6_CTX_CORRUPTION_WA(dev_priv)) {
+               i915_rc6_ctx_wa_check(dev_priv);
+               intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
+       }
+
        intel_runtime_pm_put(dev_priv);
 out_unlock:
        mutex_unlock(&dev->struct_mutex);

diff --git a/drivers/gpu/drm/i915/i915_gem_request.c b/drivers/gpu/drm/i915/i915_gem_request.c
index 813a3b546d6e2d5338676f8e7e3c342000d78729..1d556dcbd6562654ecd13fa5a905e82d37d257b7 100644 (file)
--- a/drivers/gpu/drm/i915/i915_gem_request.c
+++ b/drivers/gpu/drm/i915/i915_gem_request.c
@@ -252,6 +252,10 @@ static void mark_busy(struct drm_i915_private *i915)
        GEM_BUG_ON(!i915->gt.active_requests);
 
        intel_runtime_pm_get_noresume(i915);
+
+       if (NEEDS_RC6_CTX_CORRUPTION_WA(i915))
+               intel_uncore_forcewake_get(i915, FORCEWAKE_ALL);
+
        i915->gt.awake = true;
 
        intel_enable_gt_powersave(i915);

diff --git a/drivers/gpu/drm/i915/i915_reg.h b/drivers/gpu/drm/i915/i915_reg.h
index e95547aad312477869f415be8c7bc8d4f68f63cc..1db70350af0bf18b5de08cd6b0be9e9fea796c4d 100644 (file)
--- a/drivers/gpu/drm/i915/i915_reg.h
+++ b/drivers/gpu/drm/i915/i915_reg.h
@@ -358,6 +358,8 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg)
 #define GEN8_CONFIG0                   _MMIO(0xD00)
 #define  GEN9_DEFAULT_FIXES            (1 << 3 | 1 << 2 | 1 << 1)
 
+#define GEN8_RC6_CTX_INFO              _MMIO(0x8504)
+
 #define GAC_ECO_BITS                   _MMIO(0x14090)
 #define   ECOBITS_SNB_BIT              (1<<13)
 #define   ECOBITS_PPGTT_CACHE64B       (3<<8)

diff --git a/drivers/gpu/drm/i915/intel_drv.h b/drivers/gpu/drm/i915/intel_drv.h
index 3adb9c3b412e7a0bcdad5ada9d8a4c7ec343dbbe..cba10cdab2a962b52fcc8eb1551344271dba1384 100644 (file)
--- a/drivers/gpu/drm/i915/intel_drv.h
+++ b/drivers/gpu/drm/i915/intel_drv.h
@@ -1838,6 +1838,9 @@ void intel_enable_gt_powersave(struct drm_i915_private *dev_priv);
 void intel_autoenable_gt_powersave(struct drm_i915_private *dev_priv);
 void intel_disable_gt_powersave(struct drm_i915_private *dev_priv);
 void intel_suspend_gt_powersave(struct drm_i915_private *dev_priv);
+bool i915_rc6_ctx_wa_check(struct drm_i915_private *i915);
+void i915_rc6_ctx_wa_suspend(struct drm_i915_private *i915);
+void i915_rc6_ctx_wa_resume(struct drm_i915_private *i915);
 void gen6_rps_busy(struct drm_i915_private *dev_priv);
 void gen6_rps_reset_ei(struct drm_i915_private *dev_priv);
 void gen6_rps_idle(struct drm_i915_private *dev_priv);

diff --git a/drivers/gpu/drm/i915/intel_pm.c b/drivers/gpu/drm/i915/intel_pm.c
index 391c248a57553b9118a4ec377e93a3ce12fbc841..674410682ccc2cd96e02532b234b1dd12638e72d 100644 (file)
--- a/drivers/gpu/drm/i915/intel_pm.c
+++ b/drivers/gpu/drm/i915/intel_pm.c
@@ -6282,19 +6282,23 @@ static void gen9_disable_rps(struct drm_i915_private *dev_priv)
        I915_WRITE(GEN6_RP_CONTROL, 0);
 }
 
-static void gen6_disable_rps(struct drm_i915_private *dev_priv)
+static void gen6_disable_rc6(struct drm_i915_private *dev_priv)
 {
        I915_WRITE(GEN6_RC_CONTROL, 0);
+}
+
+static void gen6_disable_rps(struct drm_i915_private *dev_priv)
+{
        I915_WRITE(GEN6_RPNSWREQ, 1 << 31);
        I915_WRITE(GEN6_RP_CONTROL, 0);
 }
 
-static void cherryview_disable_rps(struct drm_i915_private *dev_priv)
+static void cherryview_disable_rc6(struct drm_i915_private *dev_priv)
 {
        I915_WRITE(GEN6_RC_CONTROL, 0);
 }
 
-static void valleyview_disable_rps(struct drm_i915_private *dev_priv)
+static void valleyview_disable_rc6(struct drm_i915_private *dev_priv)
 {
        /* we're doing forcewake before Disabling RC6,
         * This what the BIOS expects when going into suspend */
@@ -6545,7 +6549,8 @@ static void gen9_enable_rc6(struct drm_i915_private *dev_priv)
        I915_WRITE(GEN9_RENDER_PG_IDLE_HYSTERESIS, 25);
 
        /* 3a: Enable RC6 */
-       if (intel_enable_rc6() & INTEL_RC6_ENABLE)
+       if (!dev_priv->rps.ctx_corrupted &&
+           intel_enable_rc6() & INTEL_RC6_ENABLE)
                rc6_mask = GEN6_RC_CTL_RC6_ENABLE;
        DRM_INFO("RC6 %s\n", onoff(rc6_mask & GEN6_RC_CTL_RC6_ENABLE));
        I915_WRITE(GEN6_RC6_THRESHOLD, 37500); /* 37.5/125ms per EI */
@@ -6594,7 +6599,8 @@ static void gen8_enable_rps(struct drm_i915_private *dev_priv)
                I915_WRITE(GEN6_RC6_THRESHOLD, 50000); /* 50/125ms per EI */
 
        /* 3: Enable RC6 */
-       if (intel_enable_rc6() & INTEL_RC6_ENABLE)
+       if (!dev_priv->rps.ctx_corrupted &&
+           intel_enable_rc6() & INTEL_RC6_ENABLE)
                rc6_mask = GEN6_RC_CTL_RC6_ENABLE;
        intel_print_rc6_info(dev_priv, rc6_mask);
        if (IS_BROADWELL(dev_priv))
@@ -7775,6 +7781,95 @@ static void intel_init_emon(struct drm_i915_private *dev_priv)
        dev_priv->ips.corr = (lcfuse & LCFUSE_HIV_MASK);
 }
 
+static bool i915_rc6_ctx_corrupted(struct drm_i915_private *dev_priv)
+{
+       return !I915_READ(GEN8_RC6_CTX_INFO);
+}
+
+static void i915_rc6_ctx_wa_init(struct drm_i915_private *i915)
+{
+       if (!NEEDS_RC6_CTX_CORRUPTION_WA(i915))
+               return;
+
+       if (i915_rc6_ctx_corrupted(i915)) {
+               DRM_INFO("RC6 context corrupted, disabling runtime power management\n");
+               i915->rps.ctx_corrupted = true;
+               intel_runtime_pm_get(i915);
+       }
+}
+
+static void i915_rc6_ctx_wa_cleanup(struct drm_i915_private *i915)
+{
+       if (i915->rps.ctx_corrupted) {
+               intel_runtime_pm_put(i915);
+               i915->rps.ctx_corrupted = false;
+       }
+}
+
+/**
+ * i915_rc6_ctx_wa_suspend - system suspend sequence for the RC6 CTX WA
+ * @i915: i915 device
+ *
+ * Perform any steps needed to clean up the RC6 CTX WA before system suspend.
+ */
+void i915_rc6_ctx_wa_suspend(struct drm_i915_private *i915)
+{
+       if (i915->rps.ctx_corrupted)
+               intel_runtime_pm_put(i915);
+}
+
+/**
+ * i915_rc6_ctx_wa_resume - system resume sequence for the RC6 CTX WA
+ * @i915: i915 device
+ *
+ * Perform any steps needed to re-init the RC6 CTX WA after system resume.
+ */
+void i915_rc6_ctx_wa_resume(struct drm_i915_private *i915)
+{
+       if (!i915->rps.ctx_corrupted)
+               return;
+
+       if (i915_rc6_ctx_corrupted(i915)) {
+               intel_runtime_pm_get(i915);
+               return;
+       }
+
+       DRM_INFO("RC6 context restored, re-enabling runtime power management\n");
+       i915->rps.ctx_corrupted = false;
+}
+
+static void intel_disable_rc6(struct drm_i915_private *dev_priv);
+
+/**
+ * i915_rc6_ctx_wa_check - check for a new RC6 CTX corruption
+ * @i915: i915 device
+ *
+ * Check if an RC6 CTX corruption has happened since the last check and if so
+ * disable RC6 and runtime power management.
+ *
+ * Return false if no context corruption has happened since the last call of
+ * this function, true otherwise.
+*/
+bool i915_rc6_ctx_wa_check(struct drm_i915_private *i915)
+{
+       if (!NEEDS_RC6_CTX_CORRUPTION_WA(i915))
+               return false;
+
+       if (i915->rps.ctx_corrupted)
+               return false;
+
+       if (!i915_rc6_ctx_corrupted(i915))
+               return false;
+
+       DRM_NOTE("RC6 context corruption, disabling runtime power management\n");
+
+       intel_disable_rc6(i915);
+       i915->rps.ctx_corrupted = true;
+       intel_runtime_pm_get_noresume(i915);
+
+       return true;
+}
+
 void intel_init_gt_powersave(struct drm_i915_private *dev_priv)
 {
        /*
@@ -7789,6 +7884,8 @@ void intel_init_gt_powersave(struct drm_i915_private *dev_priv)
        mutex_lock(&dev_priv->drm.struct_mutex);
        mutex_lock(&dev_priv->rps.hw_lock);
 
+       i915_rc6_ctx_wa_init(dev_priv);
+
        /* Initialize RPS limits (for userspace) */
        if (IS_CHERRYVIEW(dev_priv))
                cherryview_init_gt_powersave(dev_priv);
@@ -7838,6 +7935,8 @@ void intel_cleanup_gt_powersave(struct drm_i915_private *dev_priv)
        if (IS_VALLEYVIEW(dev_priv))
                valleyview_cleanup_gt_powersave(dev_priv);
 
+       i915_rc6_ctx_wa_cleanup(dev_priv);
+
        if (!i915.enable_rc6)
                intel_runtime_pm_put(dev_priv);
 }
@@ -7869,27 +7968,47 @@ void intel_sanitize_gt_powersave(struct drm_i915_private *dev_priv)
        gen6_reset_rps_interrupts(dev_priv);
 }
 
-void intel_disable_gt_powersave(struct drm_i915_private *dev_priv)
+static void __intel_disable_rc6(struct drm_i915_private *dev_priv)
 {
-       if (!READ_ONCE(dev_priv->rps.enabled))
-               return;
+       if (INTEL_GEN(dev_priv) >= 9)
+               gen9_disable_rc6(dev_priv);
+       else if (IS_CHERRYVIEW(dev_priv))
+               cherryview_disable_rc6(dev_priv);
+       else if (IS_VALLEYVIEW(dev_priv))
+               valleyview_disable_rc6(dev_priv);
+       else if (INTEL_GEN(dev_priv) >= 6)
+               gen6_disable_rc6(dev_priv);
+}
 
+static void intel_disable_rc6(struct drm_i915_private *dev_priv)
+{
        mutex_lock(&dev_priv->rps.hw_lock);
+       __intel_disable_rc6(dev_priv);
+       mutex_unlock(&dev_priv->rps.hw_lock);
+}
 
-       if (INTEL_GEN(dev_priv) >= 9) {
-               gen9_disable_rc6(dev_priv);
+static void intel_disable_rps(struct drm_i915_private *dev_priv)
+{
+       if (INTEL_GEN(dev_priv) >= 9)
                gen9_disable_rps(dev_priv);
-       } else if (IS_CHERRYVIEW(dev_priv)) {
-               cherryview_disable_rps(dev_priv);
-       } else if (IS_VALLEYVIEW(dev_priv)) {
-               valleyview_disable_rps(dev_priv);
-       } else if (INTEL_GEN(dev_priv) >= 6) {
+       else if (INTEL_GEN(dev_priv) >= 6)
                gen6_disable_rps(dev_priv);
-       }  else if (IS_IRONLAKE_M(dev_priv)) {
+       else if (IS_IRONLAKE_M(dev_priv))
                ironlake_disable_drps(dev_priv);
-       }
+}
+
+void intel_disable_gt_powersave(struct drm_i915_private *dev_priv)
+{
+       if (!READ_ONCE(dev_priv->rps.enabled))
+               return;
+
+       mutex_lock(&dev_priv->rps.hw_lock);
+
+       __intel_disable_rc6(dev_priv);
+       intel_disable_rps(dev_priv);
 
        dev_priv->rps.enabled = false;
+
        mutex_unlock(&dev_priv->rps.hw_lock);
 }