From: Chris Wilson Date: Fri, 16 Jan 2015 09:34:37 +0000 (+0200) Subject: drm/i915: Reduce duplicated forcewake logic X-Git-Url: https://git.stricted.de/?a=commitdiff_plain;h=b2cff0dbbb7ef03e08865b906b236a06c6cf2e2f;p=GitHub%2Fmoto-9609%2Fandroid_kernel_motorola_exynos9610.git drm/i915: Reduce duplicated forcewake logic Introduce a structure to track the individual forcewake domains and use that to eliminate duplicate logic. v2: - Rebase on latest dinq (Mika) - for_each_fw_domain macro (Mika) - Handle reset atomically, keeping the timer running (Mika) - for_each_fw_domain parameter ordering (Chris) - defer timer on new register access (Mika) v3: - Fix forcewake_reset/get race by waiting pending timers v4: - cond_resched and verbose warning on timer deletion (Chris) - need to run pending timers manually on reset Signed-off-by: Chris Wilson (v1) Signed-off-by: Mika Kuoppala Acked-by: Deepak S (v2) Signed-off-by: Daniel Vetter --- diff --git a/drivers/gpu/drm/i915/i915_debugfs.c b/drivers/gpu/drm/i915/i915_debugfs.c index ac517eb07c00..4d156e65837f 100644 --- a/drivers/gpu/drm/i915/i915_debugfs.c +++ b/drivers/gpu/drm/i915/i915_debugfs.c @@ -1288,14 +1288,36 @@ static int ironlake_drpc_info(struct seq_file *m) return 0; } -static int vlv_drpc_info(struct seq_file *m) +static int i915_gen6_forcewake_count_info(struct seq_file *m, void *data) { + struct drm_info_node *node = m->private; + struct drm_device *dev = node->minor->dev; + struct drm_i915_private *dev_priv = dev->dev_private; + struct intel_uncore_forcewake_domain *fw_domain; + const char *domain_names[] = { + "render", + "blitter", + "media", + }; + int i; + + spin_lock_irq(&dev_priv->uncore.lock); + for_each_fw_domain(fw_domain, dev_priv, i) { + seq_printf(m, "%s.wake_count = %u\n", + domain_names[i], + fw_domain->wake_count); + } + spin_unlock_irq(&dev_priv->uncore.lock); + return 0; +} + +static int vlv_drpc_info(struct seq_file *m) +{ struct drm_info_node *node = m->private; struct drm_device *dev = node->minor->dev; struct drm_i915_private *dev_priv = dev->dev_private; u32 rpmodectl1, rcctl1, pw_status; - unsigned fw_rendercount = 0, fw_mediacount = 0; intel_runtime_pm_get(dev_priv); @@ -1327,22 +1349,11 @@ static int vlv_drpc_info(struct seq_file *m) seq_printf(m, "Media RC6 residency since boot: %u\n", I915_READ(VLV_GT_MEDIA_RC6)); - spin_lock_irq(&dev_priv->uncore.lock); - fw_rendercount = dev_priv->uncore.fw_rendercount; - fw_mediacount = dev_priv->uncore.fw_mediacount; - spin_unlock_irq(&dev_priv->uncore.lock); - - seq_printf(m, "Forcewake Render Count = %u\n", fw_rendercount); - seq_printf(m, "Forcewake Media Count = %u\n", fw_mediacount); - - - return 0; + return i915_gen6_forcewake_count_info(m, NULL); } - static int gen6_drpc_info(struct seq_file *m) { - struct drm_info_node *node = m->private; struct drm_device *dev = node->minor->dev; struct drm_i915_private *dev_priv = dev->dev_private; @@ -1356,7 +1367,7 @@ static int gen6_drpc_info(struct seq_file *m) intel_runtime_pm_get(dev_priv); spin_lock_irq(&dev_priv->uncore.lock); - forcewake_count = dev_priv->uncore.forcewake_count; + forcewake_count = dev_priv->uncore.fw_domain[FW_DOMAIN_ID_RENDER].wake_count; spin_unlock_irq(&dev_priv->uncore.lock); if (forcewake_count) { @@ -1984,30 +1995,6 @@ static int i915_execlists(struct seq_file *m, void *data) return 0; } -static int i915_gen6_forcewake_count_info(struct seq_file *m, void *data) -{ - struct drm_info_node *node = m->private; - struct drm_device *dev = node->minor->dev; - struct drm_i915_private *dev_priv = dev->dev_private; - unsigned forcewake_count = 0, fw_rendercount = 0, fw_mediacount = 0; - - spin_lock_irq(&dev_priv->uncore.lock); - if (IS_VALLEYVIEW(dev)) { - fw_rendercount = dev_priv->uncore.fw_rendercount; - fw_mediacount = dev_priv->uncore.fw_mediacount; - } else - forcewake_count = dev_priv->uncore.forcewake_count; - spin_unlock_irq(&dev_priv->uncore.lock); - - if (IS_VALLEYVIEW(dev)) { - seq_printf(m, "fw_rendercount = %u\n", fw_rendercount); - seq_printf(m, "fw_mediacount = %u\n", fw_mediacount); - } else - seq_printf(m, "forcewake count = %u\n", forcewake_count); - - return 0; -} - static const char *swizzle_string(unsigned swizzle) { switch (swizzle) { diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h index 3a32e0d81a9f..ac2082b022f3 100644 --- a/drivers/gpu/drm/i915/i915_drv.h +++ b/drivers/gpu/drm/i915/i915_drv.h @@ -596,20 +596,45 @@ struct intel_uncore_funcs { uint64_t val, bool trace); }; +enum { + FW_DOMAIN_ID_RENDER = 0, + FW_DOMAIN_ID_BLITTER, + FW_DOMAIN_ID_MEDIA, + + FW_DOMAIN_ID_COUNT +}; + struct intel_uncore { spinlock_t lock; /** lock is also taken in irq contexts. */ struct intel_uncore_funcs funcs; unsigned fifo_count; - unsigned forcewake_count; - - unsigned fw_rendercount; - unsigned fw_mediacount; - unsigned fw_blittercount; - - struct timer_list force_wake_timer; -}; + unsigned fw_domains; + + struct intel_uncore_forcewake_domain { + struct drm_i915_private *i915; + int id; + unsigned wake_count; + struct timer_list timer; + } fw_domain[FW_DOMAIN_ID_COUNT]; +#define FORCEWAKE_RENDER (1 << FW_DOMAIN_ID_RENDER) +#define FORCEWAKE_BLITTER (1 << FW_DOMAIN_ID_BLITTER) +#define FORCEWAKE_MEDIA (1 << FW_DOMAIN_ID_MEDIA) +#define FORCEWAKE_ALL (FORCEWAKE_RENDER | \ + FORCEWAKE_BLITTER | \ + FORCEWAKE_MEDIA) +}; + +/* Iterate over initialised fw domains */ +#define for_each_fw_domain_mask(domain__, mask__, dev_priv__, i__) \ + for ((i__) = 0, (domain__) = &(dev_priv__)->uncore.fw_domain[0]; \ + (i__) < FW_DOMAIN_ID_COUNT; \ + (i__)++, (domain__) = &(dev_priv__)->uncore.fw_domain[i__]) \ + if (((mask__) & (dev_priv__)->uncore.fw_domains) & (1 << (i__))) + +#define for_each_fw_domain(domain__, dev_priv__, i__) \ + for_each_fw_domain_mask(domain__, FORCEWAKE_ALL, dev_priv__, i__) #define DEV_INFO_FOR_EACH_FLAG(func, sep) \ func(is_mobile) sep \ @@ -3167,8 +3192,10 @@ extern void intel_display_print_error_state(struct drm_i915_error_state_buf *e, * must be set to prevent GT core from power down and stale values being * returned. */ -void gen6_gt_force_wake_get(struct drm_i915_private *dev_priv, int fw_engine); -void gen6_gt_force_wake_put(struct drm_i915_private *dev_priv, int fw_engine); +void gen6_gt_force_wake_get(struct drm_i915_private *dev_priv, + unsigned fw_domains); +void gen6_gt_force_wake_put(struct drm_i915_private *dev_priv, + unsigned fw_domains); void assert_force_wake_inactive(struct drm_i915_private *dev_priv); int sandybridge_pcode_read(struct drm_i915_private *dev_priv, u32 mbox, u32 *val); @@ -3200,13 +3227,6 @@ void vlv_flisdsi_write(struct drm_i915_private *dev_priv, u32 reg, u32 val); int vlv_gpu_freq(struct drm_i915_private *dev_priv, int val); int vlv_freq_opcode(struct drm_i915_private *dev_priv, int val); -#define FORCEWAKE_RENDER (1 << 0) -#define FORCEWAKE_MEDIA (1 << 1) -#define FORCEWAKE_BLITTER (1 << 2) -#define FORCEWAKE_ALL (FORCEWAKE_RENDER | FORCEWAKE_MEDIA | \ - FORCEWAKE_BLITTER) - - #define I915_READ8(reg) dev_priv->uncore.funcs.mmio_readb(dev_priv, (reg), true) #define I915_WRITE8(reg, val) dev_priv->uncore.funcs.mmio_writeb(dev_priv, (reg), (val), true) diff --git a/drivers/gpu/drm/i915/intel_uncore.c b/drivers/gpu/drm/i915/intel_uncore.c index 3d1ffacb7612..b35f3a9e6869 100644 --- a/drivers/gpu/drm/i915/intel_uncore.c +++ b/drivers/gpu/drm/i915/intel_uncore.c @@ -67,7 +67,7 @@ static void __gen6_gt_force_wake_reset(struct drm_i915_private *dev_priv) } static void __gen6_gt_force_wake_get(struct drm_i915_private *dev_priv, - int fw_engine) + int fw_engine) { if (wait_for_atomic((__raw_i915_read32(dev_priv, FORCEWAKE_ACK) & 1) == 0, FORCEWAKE_ACK_TIMEOUT_MS)) @@ -93,7 +93,7 @@ static void __gen7_gt_force_wake_mt_reset(struct drm_i915_private *dev_priv) } static void __gen7_gt_force_wake_mt_get(struct drm_i915_private *dev_priv, - int fw_engine) + int fw_engine) { u32 forcewake_ack; @@ -129,7 +129,7 @@ static void gen6_gt_check_fifodbg(struct drm_i915_private *dev_priv) } static void __gen6_gt_force_wake_put(struct drm_i915_private *dev_priv, - int fw_engine) + int fw_engine) { __raw_i915_write32(dev_priv, FORCEWAKE, 0); /* something from same cacheline, but !FORCEWAKE */ @@ -138,7 +138,7 @@ static void __gen6_gt_force_wake_put(struct drm_i915_private *dev_priv, } static void __gen7_gt_force_wake_mt_put(struct drm_i915_private *dev_priv, - int fw_engine) + int fw_engine) { __raw_i915_write32(dev_priv, FORCEWAKE_MT, _MASKED_BIT_DISABLE(FORCEWAKE_KERNEL)); @@ -187,7 +187,7 @@ static void vlv_force_wake_reset(struct drm_i915_private *dev_priv) } static void __vlv_force_wake_get(struct drm_i915_private *dev_priv, - int fw_engine) + int fw_engine) { /* Check for Render Engine */ if (FORCEWAKE_RENDER & fw_engine) { @@ -227,9 +227,8 @@ static void __vlv_force_wake_get(struct drm_i915_private *dev_priv, } static void __vlv_force_wake_put(struct drm_i915_private *dev_priv, - int fw_engine) + int fw_engine) { - /* Check for Render Engine */ if (FORCEWAKE_RENDER & fw_engine) __raw_i915_write32(dev_priv, FORCEWAKE_VLV, @@ -247,37 +246,6 @@ static void __vlv_force_wake_put(struct drm_i915_private *dev_priv, gen6_gt_check_fifodbg(dev_priv); } -static void vlv_force_wake_get(struct drm_i915_private *dev_priv, int fw_engine) -{ - if (fw_engine & FORCEWAKE_RENDER && - dev_priv->uncore.fw_rendercount++ != 0) - fw_engine &= ~FORCEWAKE_RENDER; - if (fw_engine & FORCEWAKE_MEDIA && - dev_priv->uncore.fw_mediacount++ != 0) - fw_engine &= ~FORCEWAKE_MEDIA; - - if (fw_engine) - dev_priv->uncore.funcs.force_wake_get(dev_priv, fw_engine); -} - -static void vlv_force_wake_put(struct drm_i915_private *dev_priv, int fw_engine) -{ - if (fw_engine & FORCEWAKE_RENDER) { - WARN_ON(!dev_priv->uncore.fw_rendercount); - if (--dev_priv->uncore.fw_rendercount != 0) - fw_engine &= ~FORCEWAKE_RENDER; - } - - if (fw_engine & FORCEWAKE_MEDIA) { - WARN_ON(!dev_priv->uncore.fw_mediacount); - if (--dev_priv->uncore.fw_mediacount != 0) - fw_engine &= ~FORCEWAKE_MEDIA; - } - - if (fw_engine) - dev_priv->uncore.funcs.force_wake_put(dev_priv, fw_engine); -} - static void __gen9_gt_force_wake_mt_reset(struct drm_i915_private *dev_priv) { __raw_i915_write32(dev_priv, FORCEWAKE_RENDER_GEN9, @@ -367,80 +335,72 @@ __gen9_force_wake_put(struct drm_i915_private *dev_priv, int fw_engine) _MASKED_BIT_DISABLE(FORCEWAKE_KERNEL)); } -static void -gen9_force_wake_get(struct drm_i915_private *dev_priv, int fw_engine) +static void gen6_force_wake_timer(unsigned long arg) { - if (FORCEWAKE_RENDER & fw_engine) { - if (dev_priv->uncore.fw_rendercount++ == 0) - dev_priv->uncore.funcs.force_wake_get(dev_priv, - FORCEWAKE_RENDER); - } + struct intel_uncore_forcewake_domain *domain = (void *)arg; + unsigned long irqflags; - if (FORCEWAKE_MEDIA & fw_engine) { - if (dev_priv->uncore.fw_mediacount++ == 0) - dev_priv->uncore.funcs.force_wake_get(dev_priv, - FORCEWAKE_MEDIA); - } + assert_device_not_suspended(domain->i915); - if (FORCEWAKE_BLITTER & fw_engine) { - if (dev_priv->uncore.fw_blittercount++ == 0) - dev_priv->uncore.funcs.force_wake_get(dev_priv, - FORCEWAKE_BLITTER); - } + spin_lock_irqsave(&domain->i915->uncore.lock, irqflags); + if (WARN_ON(domain->wake_count == 0)) + domain->wake_count++; + + if (--domain->wake_count == 0) + domain->i915->uncore.funcs.force_wake_put(domain->i915, + 1 << domain->id); + + spin_unlock_irqrestore(&domain->i915->uncore.lock, irqflags); } -static void -gen9_force_wake_put(struct drm_i915_private *dev_priv, int fw_engine) +void intel_uncore_forcewake_reset(struct drm_device *dev, bool restore) { - if (FORCEWAKE_RENDER & fw_engine) { - WARN_ON(dev_priv->uncore.fw_rendercount == 0); - if (--dev_priv->uncore.fw_rendercount == 0) - dev_priv->uncore.funcs.force_wake_put(dev_priv, - FORCEWAKE_RENDER); - } + struct drm_i915_private *dev_priv = dev->dev_private; + unsigned long irqflags, fw = 0; + struct intel_uncore_forcewake_domain *domain; + int id, active_domains, retry_count = 100; - if (FORCEWAKE_MEDIA & fw_engine) { - WARN_ON(dev_priv->uncore.fw_mediacount == 0); - if (--dev_priv->uncore.fw_mediacount == 0) - dev_priv->uncore.funcs.force_wake_put(dev_priv, - FORCEWAKE_MEDIA); - } + /* Hold uncore.lock across reset to prevent any register access + * with forcewake not set correctly. Wait until all pending + * timers are run before holding. + */ + while (1) { + active_domains = 0; - if (FORCEWAKE_BLITTER & fw_engine) { - WARN_ON(dev_priv->uncore.fw_blittercount == 0); - if (--dev_priv->uncore.fw_blittercount == 0) - dev_priv->uncore.funcs.force_wake_put(dev_priv, - FORCEWAKE_BLITTER); - } -} + for_each_fw_domain(domain, dev_priv, id) { + if (del_timer_sync(&domain->timer) == 0) + continue; -static void gen6_force_wake_timer(unsigned long arg) -{ - struct drm_i915_private *dev_priv = (void *)arg; - unsigned long irqflags; + gen6_force_wake_timer((unsigned long)domain); + } - assert_device_not_suspended(dev_priv); + spin_lock_irqsave(&dev_priv->uncore.lock, irqflags); - spin_lock_irqsave(&dev_priv->uncore.lock, irqflags); - WARN_ON(!dev_priv->uncore.forcewake_count); + for_each_fw_domain(domain, dev_priv, id) { + if (timer_pending(&domain->timer)) + active_domains |= (1 << id); + } - if (--dev_priv->uncore.forcewake_count == 0) - dev_priv->uncore.funcs.force_wake_put(dev_priv, FORCEWAKE_ALL); - spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); -} + if (active_domains == 0) + break; -void intel_uncore_forcewake_reset(struct drm_device *dev, bool restore) -{ - struct drm_i915_private *dev_priv = dev->dev_private; - unsigned long irqflags; + if (--retry_count == 0) { + DRM_ERROR("Timed out waiting for forcewake timers to finish\n"); + break; + } - if (del_timer_sync(&dev_priv->uncore.force_wake_timer)) - gen6_force_wake_timer((unsigned long)dev_priv); + spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); + cond_resched(); + } - /* Hold uncore.lock across reset to prevent any register access - * with forcewake not set correctly - */ - spin_lock_irqsave(&dev_priv->uncore.lock, irqflags); + WARN_ON(active_domains); + + for_each_fw_domain(domain, dev_priv, id) + if (domain->wake_count) + fw |= 1 << id; + + if (fw) + dev_priv->uncore.funcs.force_wake_put(dev_priv, fw); if (IS_VALLEYVIEW(dev)) vlv_force_wake_reset(dev_priv); @@ -454,28 +414,6 @@ void intel_uncore_forcewake_reset(struct drm_device *dev, bool restore) __gen9_gt_force_wake_mt_reset(dev_priv); if (restore) { /* If reset with a user forcewake, try to restore */ - unsigned fw = 0; - - if (IS_VALLEYVIEW(dev)) { - if (dev_priv->uncore.fw_rendercount) - fw |= FORCEWAKE_RENDER; - - if (dev_priv->uncore.fw_mediacount) - fw |= FORCEWAKE_MEDIA; - } else if (IS_GEN9(dev)) { - if (dev_priv->uncore.fw_rendercount) - fw |= FORCEWAKE_RENDER; - - if (dev_priv->uncore.fw_mediacount) - fw |= FORCEWAKE_MEDIA; - - if (dev_priv->uncore.fw_blittercount) - fw |= FORCEWAKE_BLITTER; - } else { - if (dev_priv->uncore.forcewake_count) - fw = FORCEWAKE_ALL; - } - if (fw) dev_priv->uncore.funcs.force_wake_get(dev_priv, fw); @@ -485,6 +423,9 @@ void intel_uncore_forcewake_reset(struct drm_device *dev, bool restore) GT_FIFO_FREE_ENTRIES_MASK; } + if (!restore) + assert_force_wake_inactive(dev_priv); + spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); } @@ -533,53 +474,59 @@ void intel_uncore_sanitize(struct drm_device *dev) * be called at the beginning of the sequence followed by a call to * gen6_gt_force_wake_put() at the end of the sequence. */ -void gen6_gt_force_wake_get(struct drm_i915_private *dev_priv, int fw_engine) +void gen6_gt_force_wake_get(struct drm_i915_private *dev_priv, + unsigned fw_domains) { unsigned long irqflags; + struct intel_uncore_forcewake_domain *domain; + int id; if (!dev_priv->uncore.funcs.force_wake_get) return; WARN_ON(dev_priv->pm.suspended); + fw_domains &= dev_priv->uncore.fw_domains; + spin_lock_irqsave(&dev_priv->uncore.lock, irqflags); - if (IS_GEN9(dev_priv->dev)) { - gen9_force_wake_get(dev_priv, fw_engine); - } else if (IS_VALLEYVIEW(dev_priv->dev)) { - vlv_force_wake_get(dev_priv, fw_engine); - } else { - if (dev_priv->uncore.forcewake_count++ == 0) - dev_priv->uncore.funcs.force_wake_get(dev_priv, - FORCEWAKE_ALL); + for_each_fw_domain_mask(domain, fw_domains, dev_priv, id) { + if (domain->wake_count++) + fw_domains &= ~(1 << id); } + if (fw_domains) + dev_priv->uncore.funcs.force_wake_get(dev_priv, fw_domains); + spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); } /* * see gen6_gt_force_wake_get() */ -void gen6_gt_force_wake_put(struct drm_i915_private *dev_priv, int fw_engine) +void gen6_gt_force_wake_put(struct drm_i915_private *dev_priv, + unsigned fw_domains) { unsigned long irqflags; + struct intel_uncore_forcewake_domain *domain; + int id; if (!dev_priv->uncore.funcs.force_wake_put) return; + fw_domains &= dev_priv->uncore.fw_domains; + spin_lock_irqsave(&dev_priv->uncore.lock, irqflags); - if (IS_GEN9(dev_priv->dev)) { - gen9_force_wake_put(dev_priv, fw_engine); - } else if (IS_VALLEYVIEW(dev_priv->dev)) { - vlv_force_wake_put(dev_priv, fw_engine); - } else { - WARN_ON(!dev_priv->uncore.forcewake_count); - if (--dev_priv->uncore.forcewake_count == 0) { - dev_priv->uncore.forcewake_count++; - mod_timer_pinned(&dev_priv->uncore.force_wake_timer, - jiffies + 1); - } + for_each_fw_domain_mask(domain, fw_domains, dev_priv, id) { + if (WARN_ON(domain->wake_count == 0)) + continue; + + if (--domain->wake_count) + continue; + + domain->wake_count++; + mod_timer_pinned(&domain->timer, jiffies + 1); } spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); @@ -587,10 +534,14 @@ void gen6_gt_force_wake_put(struct drm_i915_private *dev_priv, int fw_engine) void assert_force_wake_inactive(struct drm_i915_private *dev_priv) { + struct intel_uncore_forcewake_domain *domain; + int i; + if (!dev_priv->uncore.funcs.force_wake_get) return; - WARN_ON(dev_priv->uncore.forcewake_count > 0); + for_each_fw_domain(domain, dev_priv, i) + WARN_ON(domain->wake_count); } /* We give fast paths for the really cool registers */ @@ -753,19 +704,37 @@ __gen2_read(64) trace_i915_reg_rw(false, reg, val, sizeof(val), trace); \ return val +static inline void __force_wake_get(struct drm_i915_private *dev_priv, + unsigned fw_domains) +{ + struct intel_uncore_forcewake_domain *domain; + int i; + + if (WARN_ON(!fw_domains)) + return; + + /* Ideally GCC would be constant-fold and eliminate this loop */ + for_each_fw_domain_mask(domain, fw_domains, dev_priv, i) { + if (domain->wake_count) { + fw_domains &= ~(1 << i); + continue; + } + + domain->wake_count++; + mod_timer_pinned(&domain->timer, jiffies + 1); + } + + if (fw_domains) + dev_priv->uncore.funcs.force_wake_get(dev_priv, fw_domains); +} + #define __gen6_read(x) \ static u##x \ gen6_read##x(struct drm_i915_private *dev_priv, off_t reg, bool trace) { \ GEN6_READ_HEADER(x); \ hsw_unclaimed_reg_debug(dev_priv, reg, true, true); \ - if (dev_priv->uncore.forcewake_count == 0 && \ - NEEDS_FORCE_WAKE((dev_priv), (reg))) { \ - dev_priv->uncore.funcs.force_wake_get(dev_priv, \ - FORCEWAKE_ALL); \ - dev_priv->uncore.forcewake_count++; \ - mod_timer_pinned(&dev_priv->uncore.force_wake_timer, \ - jiffies + 1); \ - } \ + if (NEEDS_FORCE_WAKE((dev_priv), (reg))) \ + __force_wake_get(dev_priv, FORCEWAKE_RENDER); \ val = __raw_i915_read##x(dev_priv, reg); \ hsw_unclaimed_reg_debug(dev_priv, reg, true, false); \ GEN6_READ_FOOTER; \ @@ -774,45 +743,27 @@ gen6_read##x(struct drm_i915_private *dev_priv, off_t reg, bool trace) { \ #define __vlv_read(x) \ static u##x \ vlv_read##x(struct drm_i915_private *dev_priv, off_t reg, bool trace) { \ - unsigned fwengine = 0; \ GEN6_READ_HEADER(x); \ - if (FORCEWAKE_VLV_RENDER_RANGE_OFFSET(reg)) { \ - if (dev_priv->uncore.fw_rendercount == 0) \ - fwengine = FORCEWAKE_RENDER; \ - } else if (FORCEWAKE_VLV_MEDIA_RANGE_OFFSET(reg)) { \ - if (dev_priv->uncore.fw_mediacount == 0) \ - fwengine = FORCEWAKE_MEDIA; \ - } \ - if (fwengine) \ - dev_priv->uncore.funcs.force_wake_get(dev_priv, fwengine); \ + if (FORCEWAKE_VLV_RENDER_RANGE_OFFSET(reg)) \ + __force_wake_get(dev_priv, FORCEWAKE_RENDER); \ + else if (FORCEWAKE_VLV_MEDIA_RANGE_OFFSET(reg)) \ + __force_wake_get(dev_priv, FORCEWAKE_MEDIA); \ val = __raw_i915_read##x(dev_priv, reg); \ - if (fwengine) \ - dev_priv->uncore.funcs.force_wake_put(dev_priv, fwengine); \ GEN6_READ_FOOTER; \ } #define __chv_read(x) \ static u##x \ chv_read##x(struct drm_i915_private *dev_priv, off_t reg, bool trace) { \ - unsigned fwengine = 0; \ GEN6_READ_HEADER(x); \ - if (FORCEWAKE_CHV_RENDER_RANGE_OFFSET(reg)) { \ - if (dev_priv->uncore.fw_rendercount == 0) \ - fwengine = FORCEWAKE_RENDER; \ - } else if (FORCEWAKE_CHV_MEDIA_RANGE_OFFSET(reg)) { \ - if (dev_priv->uncore.fw_mediacount == 0) \ - fwengine = FORCEWAKE_MEDIA; \ - } else if (FORCEWAKE_CHV_COMMON_RANGE_OFFSET(reg)) { \ - if (dev_priv->uncore.fw_rendercount == 0) \ - fwengine |= FORCEWAKE_RENDER; \ - if (dev_priv->uncore.fw_mediacount == 0) \ - fwengine |= FORCEWAKE_MEDIA; \ - } \ - if (fwengine) \ - dev_priv->uncore.funcs.force_wake_get(dev_priv, fwengine); \ + if (FORCEWAKE_CHV_RENDER_RANGE_OFFSET(reg)) \ + __force_wake_get(dev_priv, FORCEWAKE_RENDER); \ + else if (FORCEWAKE_CHV_MEDIA_RANGE_OFFSET(reg)) \ + __force_wake_get(dev_priv, FORCEWAKE_MEDIA); \ + else if (FORCEWAKE_CHV_COMMON_RANGE_OFFSET(reg)) \ + __force_wake_get(dev_priv, \ + FORCEWAKE_RENDER | FORCEWAKE_MEDIA); \ val = __raw_i915_read##x(dev_priv, reg); \ - if (fwengine) \ - dev_priv->uncore.funcs.force_wake_put(dev_priv, fwengine); \ GEN6_READ_FOOTER; \ } @@ -822,32 +773,21 @@ chv_read##x(struct drm_i915_private *dev_priv, off_t reg, bool trace) { \ #define __gen9_read(x) \ static u##x \ gen9_read##x(struct drm_i915_private *dev_priv, off_t reg, bool trace) { \ + unsigned fw_engine; \ GEN6_READ_HEADER(x); \ - if (!SKL_NEEDS_FORCE_WAKE((dev_priv), (reg))) { \ - val = __raw_i915_read##x(dev_priv, reg); \ - } else { \ - unsigned fwengine = 0; \ - if (FORCEWAKE_GEN9_RENDER_RANGE_OFFSET(reg)) { \ - if (dev_priv->uncore.fw_rendercount == 0) \ - fwengine = FORCEWAKE_RENDER; \ - } else if (FORCEWAKE_GEN9_MEDIA_RANGE_OFFSET(reg)) { \ - if (dev_priv->uncore.fw_mediacount == 0) \ - fwengine = FORCEWAKE_MEDIA; \ - } else if (FORCEWAKE_GEN9_COMMON_RANGE_OFFSET(reg)) { \ - if (dev_priv->uncore.fw_rendercount == 0) \ - fwengine |= FORCEWAKE_RENDER; \ - if (dev_priv->uncore.fw_mediacount == 0) \ - fwengine |= FORCEWAKE_MEDIA; \ - } else { \ - if (dev_priv->uncore.fw_blittercount == 0) \ - fwengine = FORCEWAKE_BLITTER; \ - } \ - if (fwengine) \ - dev_priv->uncore.funcs.force_wake_get(dev_priv, fwengine); \ - val = __raw_i915_read##x(dev_priv, reg); \ - if (fwengine) \ - dev_priv->uncore.funcs.force_wake_put(dev_priv, fwengine); \ - } \ + if (!SKL_NEEDS_FORCE_WAKE((dev_priv), (reg))) \ + fw_engine = 0; \ + else if (FORCEWAKE_GEN9_RENDER_RANGE_OFFSET(reg)) \ + fw_engine = FORCEWAKE_RENDER; \ + else if (FORCEWAKE_GEN9_MEDIA_RANGE_OFFSET(reg)) \ + fw_engine = FORCEWAKE_MEDIA; \ + else if (FORCEWAKE_GEN9_COMMON_RANGE_OFFSET(reg)) \ + fw_engine = FORCEWAKE_RENDER | FORCEWAKE_MEDIA; \ + else \ + fw_engine = FORCEWAKE_BLITTER; \ + if (fw_engine) \ + __force_wake_get(dev_priv, fw_engine); \ + val = __raw_i915_read##x(dev_priv, reg); \ GEN6_READ_FOOTER; \ } @@ -981,17 +921,9 @@ static void \ gen8_write##x(struct drm_i915_private *dev_priv, off_t reg, u##x val, bool trace) { \ GEN6_WRITE_HEADER; \ hsw_unclaimed_reg_debug(dev_priv, reg, false, true); \ - if (reg < 0x40000 && !is_gen8_shadowed(dev_priv, reg)) { \ - if (dev_priv->uncore.forcewake_count == 0) \ - dev_priv->uncore.funcs.force_wake_get(dev_priv, \ - FORCEWAKE_ALL); \ - __raw_i915_write##x(dev_priv, reg, val); \ - if (dev_priv->uncore.forcewake_count == 0) \ - dev_priv->uncore.funcs.force_wake_put(dev_priv, \ - FORCEWAKE_ALL); \ - } else { \ - __raw_i915_write##x(dev_priv, reg, val); \ - } \ + if (reg < 0x40000 && !is_gen8_shadowed(dev_priv, reg)) \ + __force_wake_get(dev_priv, FORCEWAKE_RENDER); \ + __raw_i915_write##x(dev_priv, reg, val); \ hsw_unclaimed_reg_debug(dev_priv, reg, false, false); \ hsw_unclaimed_reg_detect(dev_priv); \ GEN6_WRITE_FOOTER; \ @@ -1000,28 +932,17 @@ gen8_write##x(struct drm_i915_private *dev_priv, off_t reg, u##x val, bool trace #define __chv_write(x) \ static void \ chv_write##x(struct drm_i915_private *dev_priv, off_t reg, u##x val, bool trace) { \ - unsigned fwengine = 0; \ bool shadowed = is_gen8_shadowed(dev_priv, reg); \ GEN6_WRITE_HEADER; \ if (!shadowed) { \ - if (FORCEWAKE_CHV_RENDER_RANGE_OFFSET(reg)) { \ - if (dev_priv->uncore.fw_rendercount == 0) \ - fwengine = FORCEWAKE_RENDER; \ - } else if (FORCEWAKE_CHV_MEDIA_RANGE_OFFSET(reg)) { \ - if (dev_priv->uncore.fw_mediacount == 0) \ - fwengine = FORCEWAKE_MEDIA; \ - } else if (FORCEWAKE_CHV_COMMON_RANGE_OFFSET(reg)) { \ - if (dev_priv->uncore.fw_rendercount == 0) \ - fwengine |= FORCEWAKE_RENDER; \ - if (dev_priv->uncore.fw_mediacount == 0) \ - fwengine |= FORCEWAKE_MEDIA; \ - } \ + if (FORCEWAKE_CHV_RENDER_RANGE_OFFSET(reg)) \ + __force_wake_get(dev_priv, FORCEWAKE_RENDER); \ + else if (FORCEWAKE_CHV_MEDIA_RANGE_OFFSET(reg)) \ + __force_wake_get(dev_priv, FORCEWAKE_MEDIA); \ + else if (FORCEWAKE_CHV_COMMON_RANGE_OFFSET(reg)) \ + __force_wake_get(dev_priv, FORCEWAKE_RENDER | FORCEWAKE_MEDIA); \ } \ - if (fwengine) \ - dev_priv->uncore.funcs.force_wake_get(dev_priv, fwengine); \ __raw_i915_write##x(dev_priv, reg, val); \ - if (fwengine) \ - dev_priv->uncore.funcs.force_wake_put(dev_priv, fwengine); \ GEN6_WRITE_FOOTER; \ } @@ -1052,35 +973,22 @@ static bool is_gen9_shadowed(struct drm_i915_private *dev_priv, u32 reg) static void \ gen9_write##x(struct drm_i915_private *dev_priv, off_t reg, u##x val, \ bool trace) { \ + unsigned fw_engine; \ GEN6_WRITE_HEADER; \ - if (!SKL_NEEDS_FORCE_WAKE((dev_priv), (reg)) || \ - is_gen9_shadowed(dev_priv, reg)) { \ - __raw_i915_write##x(dev_priv, reg, val); \ - } else { \ - unsigned fwengine = 0; \ - if (FORCEWAKE_GEN9_RENDER_RANGE_OFFSET(reg)) { \ - if (dev_priv->uncore.fw_rendercount == 0) \ - fwengine = FORCEWAKE_RENDER; \ - } else if (FORCEWAKE_GEN9_MEDIA_RANGE_OFFSET(reg)) { \ - if (dev_priv->uncore.fw_mediacount == 0) \ - fwengine = FORCEWAKE_MEDIA; \ - } else if (FORCEWAKE_GEN9_COMMON_RANGE_OFFSET(reg)) { \ - if (dev_priv->uncore.fw_rendercount == 0) \ - fwengine |= FORCEWAKE_RENDER; \ - if (dev_priv->uncore.fw_mediacount == 0) \ - fwengine |= FORCEWAKE_MEDIA; \ - } else { \ - if (dev_priv->uncore.fw_blittercount == 0) \ - fwengine = FORCEWAKE_BLITTER; \ - } \ - if (fwengine) \ - dev_priv->uncore.funcs.force_wake_get(dev_priv, \ - fwengine); \ - __raw_i915_write##x(dev_priv, reg, val); \ - if (fwengine) \ - dev_priv->uncore.funcs.force_wake_put(dev_priv, \ - fwengine); \ - } \ + if (!SKL_NEEDS_FORCE_WAKE((dev_priv), (reg)) || \ + is_gen9_shadowed(dev_priv, reg)) \ + fw_engine = 0; \ + else if (FORCEWAKE_GEN9_RENDER_RANGE_OFFSET(reg)) \ + fw_engine = FORCEWAKE_RENDER; \ + else if (FORCEWAKE_GEN9_MEDIA_RANGE_OFFSET(reg)) \ + fw_engine = FORCEWAKE_MEDIA; \ + else if (FORCEWAKE_GEN9_COMMON_RANGE_OFFSET(reg)) \ + fw_engine = FORCEWAKE_RENDER | FORCEWAKE_MEDIA; \ + else \ + fw_engine = FORCEWAKE_BLITTER; \ + if (fw_engine) \ + __force_wake_get(dev_priv, fw_engine); \ + __raw_i915_write##x(dev_priv, reg, val); \ GEN6_WRITE_FOOTER; \ } @@ -1132,21 +1040,24 @@ do { \ void intel_uncore_init(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; - - setup_timer(&dev_priv->uncore.force_wake_timer, - gen6_force_wake_timer, (unsigned long)dev_priv); + struct intel_uncore_forcewake_domain *domain; + int i; __intel_uncore_early_sanitize(dev, false); if (IS_GEN9(dev)) { dev_priv->uncore.funcs.force_wake_get = __gen9_force_wake_get; dev_priv->uncore.funcs.force_wake_put = __gen9_force_wake_put; + dev_priv->uncore.fw_domains = FORCEWAKE_RENDER | + FORCEWAKE_BLITTER | FORCEWAKE_MEDIA; } else if (IS_VALLEYVIEW(dev)) { dev_priv->uncore.funcs.force_wake_get = __vlv_force_wake_get; dev_priv->uncore.funcs.force_wake_put = __vlv_force_wake_put; + dev_priv->uncore.fw_domains = FORCEWAKE_RENDER | FORCEWAKE_MEDIA; } else if (IS_HASWELL(dev) || IS_BROADWELL(dev)) { dev_priv->uncore.funcs.force_wake_get = __gen7_gt_force_wake_mt_get; dev_priv->uncore.funcs.force_wake_put = __gen7_gt_force_wake_mt_put; + dev_priv->uncore.fw_domains = FORCEWAKE_RENDER; } else if (IS_IVYBRIDGE(dev)) { u32 ecobus; @@ -1178,11 +1089,21 @@ void intel_uncore_init(struct drm_device *dev) dev_priv->uncore.funcs.force_wake_put = __gen6_gt_force_wake_put; } + dev_priv->uncore.fw_domains = FORCEWAKE_RENDER; } else if (IS_GEN6(dev)) { dev_priv->uncore.funcs.force_wake_get = __gen6_gt_force_wake_get; dev_priv->uncore.funcs.force_wake_put = __gen6_gt_force_wake_put; + dev_priv->uncore.fw_domains = FORCEWAKE_RENDER; + } + + for_each_fw_domain(domain, dev_priv, i) { + domain->i915 = dev_priv; + domain->id = i; + + setup_timer(&domain->timer, gen6_force_wake_timer, + (unsigned long)domain); } switch (INTEL_INFO(dev)->gen) {