projects / GitHub / MotorolaMobilityLLC / kernel-slsi.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
drm/i915: HSW FBC WaFbcAsynchFlipDisableFbcQueue
[GitHub/MotorolaMobilityLLC/kernel-slsi.git] / drivers / gpu / drm / i915 / intel_pm.c
CommitLineData
85208be0
ED		 1/*
2 * Copyright © 2012 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 * Authors:
24 * Eugeni Dodonov <eugeni.dodonov@intel.com>
25 *
26 */
27
2b4e57bd 28#include <linux/cpufreq.h>
85208be0
ED		 29#include "i915_drv.h"
30#include "intel_drv.h"
eb48eb00
DV		 31#include "../../../platform/x86/intel_ips.h"
32#include <linux/module.h>
85208be0 33
057d3860 34#define FORCEWAKE_ACK_TIMEOUT_MS 2
b67a4376 35
f6750b3c
ED		 36/* FBC, or Frame Buffer Compression, is a technique employed to compress the
37 * framebuffer contents in-memory, aiming at reducing the required bandwidth
38 * during in-memory transfers and, therefore, reduce the power packet.
85208be0 39 *
f6750b3c
ED		 40 * The benefits of FBC are mostly visible with solid backgrounds and
41 * variation-less patterns.
85208be0 42 *
f6750b3c
ED		 43 * FBC-related functionality can be enabled by the means of the
44 * i915.i915_enable_fbc parameter
85208be0
ED		 45 */
46
3490ea5d
CW		 47static bool intel_crtc_active(struct drm_crtc *crtc)
48{
49 /* Be paranoid as we can arrive here with only partial
50 * state retrieved from the hardware during setup.
51 */
52 return to_intel_crtc(crtc)->active && crtc->fb && crtc->mode.clock;
53}
54
1fa61106 55static void i8xx_disable_fbc(struct drm_device *dev)
85208be0
ED		 56{
57 struct drm_i915_private *dev_priv = dev->dev_private;
58 u32 fbc_ctl;
59
60 /* Disable compression */
61 fbc_ctl = I915_READ(FBC_CONTROL);
62 if ((fbc_ctl & FBC_CTL_EN) == 0)
63 return;
64
65 fbc_ctl &= ~FBC_CTL_EN;
66 I915_WRITE(FBC_CONTROL, fbc_ctl);
67
68 /* Wait for compressing bit to clear */
69 if (wait_for((I915_READ(FBC_STATUS) & FBC_STAT_COMPRESSING) == 0, 10)) {
70 DRM_DEBUG_KMS("FBC idle timed out\n");
71 return;
72 }
73
74 DRM_DEBUG_KMS("disabled FBC\n");
75}
76
1fa61106 77static void i8xx_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
85208be0
ED		 78{
79 struct drm_device *dev = crtc->dev;
80 struct drm_i915_private *dev_priv = dev->dev_private;
81 struct drm_framebuffer *fb = crtc->fb;
82 struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
83 struct drm_i915_gem_object *obj = intel_fb->obj;
84 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
85 int cfb_pitch;
86 int plane, i;
87 u32 fbc_ctl, fbc_ctl2;
88
89 cfb_pitch = dev_priv->cfb_size / FBC_LL_SIZE;
90 if (fb->pitches[0] < cfb_pitch)
91 cfb_pitch = fb->pitches[0];
92
93 /* FBC_CTL wants 64B units */
94 cfb_pitch = (cfb_pitch / 64) - 1;
95 plane = intel_crtc->plane == 0 ? FBC_CTL_PLANEA : FBC_CTL_PLANEB;
96
97 /* Clear old tags */
98 for (i = 0; i < (FBC_LL_SIZE / 32) + 1; i++)
99 I915_WRITE(FBC_TAG + (i * 4), 0);
100
101 /* Set it up... */
102 fbc_ctl2 = FBC_CTL_FENCE_DBL | FBC_CTL_IDLE_IMM | FBC_CTL_CPU_FENCE;
103 fbc_ctl2 |= plane;
104 I915_WRITE(FBC_CONTROL2, fbc_ctl2);
105 I915_WRITE(FBC_FENCE_OFF, crtc->y);
106
107 /* enable it... */
108 fbc_ctl = FBC_CTL_EN | FBC_CTL_PERIODIC;
109 if (IS_I945GM(dev))
110 fbc_ctl |= FBC_CTL_C3_IDLE; /* 945 needs special SR handling */
111 fbc_ctl |= (cfb_pitch & 0xff) << FBC_CTL_STRIDE_SHIFT;
112 fbc_ctl |= (interval & 0x2fff) << FBC_CTL_INTERVAL_SHIFT;
113 fbc_ctl |= obj->fence_reg;
114 I915_WRITE(FBC_CONTROL, fbc_ctl);
115
84f44ce7
VS		 116 DRM_DEBUG_KMS("enabled FBC, pitch %d, yoff %d, plane %c, ",
117 cfb_pitch, crtc->y, plane_name(intel_crtc->plane));
85208be0
ED		 118}
119
1fa61106 120static bool i8xx_fbc_enabled(struct drm_device *dev)
85208be0
ED		 121{
122 struct drm_i915_private *dev_priv = dev->dev_private;
123
124 return I915_READ(FBC_CONTROL) & FBC_CTL_EN;
125}
126
1fa61106 127static void g4x_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
85208be0
ED		 128{
129 struct drm_device *dev = crtc->dev;
130 struct drm_i915_private *dev_priv = dev->dev_private;
131 struct drm_framebuffer *fb = crtc->fb;
132 struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
133 struct drm_i915_gem_object *obj = intel_fb->obj;
134 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
135 int plane = intel_crtc->plane == 0 ? DPFC_CTL_PLANEA : DPFC_CTL_PLANEB;
136 unsigned long stall_watermark = 200;
137 u32 dpfc_ctl;
138
139 dpfc_ctl = plane | DPFC_SR_EN | DPFC_CTL_LIMIT_1X;
140 dpfc_ctl |= DPFC_CTL_FENCE_EN | obj->fence_reg;
141 I915_WRITE(DPFC_CHICKEN, DPFC_HT_MODIFY);
142
143 I915_WRITE(DPFC_RECOMP_CTL, DPFC_RECOMP_STALL_EN |
144 (stall_watermark << DPFC_RECOMP_STALL_WM_SHIFT) |
145 (interval << DPFC_RECOMP_TIMER_COUNT_SHIFT));
146 I915_WRITE(DPFC_FENCE_YOFF, crtc->y);
147
148 /* enable it... */
149 I915_WRITE(DPFC_CONTROL, I915_READ(DPFC_CONTROL) | DPFC_CTL_EN);
150
84f44ce7 151 DRM_DEBUG_KMS("enabled fbc on plane %c\n", plane_name(intel_crtc->plane));
85208be0
ED		 152}
153
1fa61106 154static void g4x_disable_fbc(struct drm_device *dev)
85208be0
ED		 155{
156 struct drm_i915_private *dev_priv = dev->dev_private;
157 u32 dpfc_ctl;
158
159 /* Disable compression */
160 dpfc_ctl = I915_READ(DPFC_CONTROL);
161 if (dpfc_ctl & DPFC_CTL_EN) {
162 dpfc_ctl &= ~DPFC_CTL_EN;
163 I915_WRITE(DPFC_CONTROL, dpfc_ctl);
164
165 DRM_DEBUG_KMS("disabled FBC\n");
166 }
167}
168
1fa61106 169static bool g4x_fbc_enabled(struct drm_device *dev)
85208be0
ED		 170{
171 struct drm_i915_private *dev_priv = dev->dev_private;
172
173 return I915_READ(DPFC_CONTROL) & DPFC_CTL_EN;
174}
175
176static void sandybridge_blit_fbc_update(struct drm_device *dev)
177{
178 struct drm_i915_private *dev_priv = dev->dev_private;
179 u32 blt_ecoskpd;
180
181 /* Make sure blitter notifies FBC of writes */
182 gen6_gt_force_wake_get(dev_priv);
183 blt_ecoskpd = I915_READ(GEN6_BLITTER_ECOSKPD);
184 blt_ecoskpd |= GEN6_BLITTER_FBC_NOTIFY <<
185 GEN6_BLITTER_LOCK_SHIFT;
186 I915_WRITE(GEN6_BLITTER_ECOSKPD, blt_ecoskpd);
187 blt_ecoskpd |= GEN6_BLITTER_FBC_NOTIFY;
188 I915_WRITE(GEN6_BLITTER_ECOSKPD, blt_ecoskpd);
189 blt_ecoskpd &= ~(GEN6_BLITTER_FBC_NOTIFY <<
190 GEN6_BLITTER_LOCK_SHIFT);
191 I915_WRITE(GEN6_BLITTER_ECOSKPD, blt_ecoskpd);
192 POSTING_READ(GEN6_BLITTER_ECOSKPD);
193 gen6_gt_force_wake_put(dev_priv);
194}
195
1fa61106 196static void ironlake_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
85208be0
ED		 197{
198 struct drm_device *dev = crtc->dev;
199 struct drm_i915_private *dev_priv = dev->dev_private;
200 struct drm_framebuffer *fb = crtc->fb;
201 struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
202 struct drm_i915_gem_object *obj = intel_fb->obj;
203 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
204 int plane = intel_crtc->plane == 0 ? DPFC_CTL_PLANEA : DPFC_CTL_PLANEB;
205 unsigned long stall_watermark = 200;
206 u32 dpfc_ctl;
207
208 dpfc_ctl = I915_READ(ILK_DPFC_CONTROL);
209 dpfc_ctl &= DPFC_RESERVED;
210 dpfc_ctl |= (plane | DPFC_CTL_LIMIT_1X);
211 /* Set persistent mode for front-buffer rendering, ala X. */
212 dpfc_ctl |= DPFC_CTL_PERSISTENT_MODE;
213 dpfc_ctl |= (DPFC_CTL_FENCE_EN | obj->fence_reg);
214 I915_WRITE(ILK_DPFC_CHICKEN, DPFC_HT_MODIFY);
215
216 I915_WRITE(ILK_DPFC_RECOMP_CTL, DPFC_RECOMP_STALL_EN |
217 (stall_watermark << DPFC_RECOMP_STALL_WM_SHIFT) |
218 (interval << DPFC_RECOMP_TIMER_COUNT_SHIFT));
219 I915_WRITE(ILK_DPFC_FENCE_YOFF, crtc->y);
220 I915_WRITE(ILK_FBC_RT_BASE, obj->gtt_offset | ILK_FBC_RT_VALID);
221 /* enable it... */
222 I915_WRITE(ILK_DPFC_CONTROL, dpfc_ctl | DPFC_CTL_EN);
223
224 if (IS_GEN6(dev)) {
225 I915_WRITE(SNB_DPFC_CTL_SA,
226 SNB_CPU_FENCE_ENABLE | obj->fence_reg);
227 I915_WRITE(DPFC_CPU_FENCE_OFFSET, crtc->y);
228 sandybridge_blit_fbc_update(dev);
229 }
230
84f44ce7 231 DRM_DEBUG_KMS("enabled fbc on plane %c\n", plane_name(intel_crtc->plane));
85208be0
ED		 232}
233
1fa61106 234static void ironlake_disable_fbc(struct drm_device *dev)
85208be0
ED		 235{
236 struct drm_i915_private *dev_priv = dev->dev_private;
237 u32 dpfc_ctl;
238
239 /* Disable compression */
240 dpfc_ctl = I915_READ(ILK_DPFC_CONTROL);
241 if (dpfc_ctl & DPFC_CTL_EN) {
242 dpfc_ctl &= ~DPFC_CTL_EN;
243 I915_WRITE(ILK_DPFC_CONTROL, dpfc_ctl);
244
b74ea102
RV		 245 if (IS_IVYBRIDGE(dev))
246 /* WaFbcDisableDpfcClockGating */
247 I915_WRITE(ILK_DSPCLK_GATE_D,
248 I915_READ(ILK_DSPCLK_GATE_D) &
249 ~ILK_DPFCUNIT_CLOCK_GATE_DISABLE);
250
85208be0
ED		 251 DRM_DEBUG_KMS("disabled FBC\n");
252 }
253}
254
1fa61106 255static bool ironlake_fbc_enabled(struct drm_device *dev)
85208be0
ED		 256{
257 struct drm_i915_private *dev_priv = dev->dev_private;
258
259 return I915_READ(ILK_DPFC_CONTROL) & DPFC_CTL_EN;
260}
261
abe959c7
RV		 262static void gen7_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
263{
264 struct drm_device *dev = crtc->dev;
265 struct drm_i915_private *dev_priv = dev->dev_private;
266 struct drm_framebuffer *fb = crtc->fb;
267 struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
268 struct drm_i915_gem_object *obj = intel_fb->obj;
269 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
270
271 I915_WRITE(IVB_FBC_RT_BASE, obj->gtt_offset | ILK_FBC_RT_VALID);
272
273 I915_WRITE(ILK_DPFC_CONTROL, DPFC_CTL_EN | DPFC_CTL_LIMIT_1X |
274 IVB_DPFC_CTL_FENCE_EN |
275 intel_crtc->plane << IVB_DPFC_CTL_PLANE_SHIFT);
276
891348b2
RV		 277 if (IS_IVYBRIDGE(dev)) {
278 /* WaFbcAsynchFlipDisableFbcQueue */
279 I915_WRITE(ILK_DISPLAY_CHICKEN1, ILK_FBCQ_DIS);
280 /* WaFbcDisableDpfcClockGating */
281 I915_WRITE(ILK_DSPCLK_GATE_D,
282 I915_READ(ILK_DSPCLK_GATE_D) |
283 ILK_DPFCUNIT_CLOCK_GATE_DISABLE);
28554164
RV		 284 } else {
285 /* WaFbcAsynchFlipDisableFbcQueue */
286 I915_WRITE(HSW_PIPE_SLICE_CHICKEN_1(intel_crtc->pipe),
287 HSW_BYPASS_FBC_QUEUE);
891348b2 288 }
b74ea102 289
abe959c7
RV		 290 I915_WRITE(SNB_DPFC_CTL_SA,
291 SNB_CPU_FENCE_ENABLE | obj->fence_reg);
292 I915_WRITE(DPFC_CPU_FENCE_OFFSET, crtc->y);
293
294 sandybridge_blit_fbc_update(dev);
295
296 DRM_DEBUG_KMS("enabled fbc on plane %d\n", intel_crtc->plane);
297}
298
85208be0
ED		 299bool intel_fbc_enabled(struct drm_device *dev)
300{
301 struct drm_i915_private *dev_priv = dev->dev_private;
302
303 if (!dev_priv->display.fbc_enabled)
304 return false;
305
306 return dev_priv->display.fbc_enabled(dev);
307}
308
309static void intel_fbc_work_fn(struct work_struct *__work)
310{
311 struct intel_fbc_work *work =
312 container_of(to_delayed_work(__work),
313 struct intel_fbc_work, work);
314 struct drm_device *dev = work->crtc->dev;
315 struct drm_i915_private *dev_priv = dev->dev_private;
316
317 mutex_lock(&dev->struct_mutex);
318 if (work == dev_priv->fbc_work) {
319 /* Double check that we haven't switched fb without cancelling
320 * the prior work.
321 */
322 if (work->crtc->fb == work->fb) {
323 dev_priv->display.enable_fbc(work->crtc,
324 work->interval);
325
326 dev_priv->cfb_plane = to_intel_crtc(work->crtc)->plane;
327 dev_priv->cfb_fb = work->crtc->fb->base.id;
328 dev_priv->cfb_y = work->crtc->y;
329 }
330
331 dev_priv->fbc_work = NULL;
332 }
333 mutex_unlock(&dev->struct_mutex);
334
335 kfree(work);
336}
337
338static void intel_cancel_fbc_work(struct drm_i915_private *dev_priv)
339{
340 if (dev_priv->fbc_work == NULL)
341 return;
342
343 DRM_DEBUG_KMS("cancelling pending FBC enable\n");
344
345 /* Synchronisation is provided by struct_mutex and checking of
346 * dev_priv->fbc_work, so we can perform the cancellation
347 * entirely asynchronously.
348 */
349 if (cancel_delayed_work(&dev_priv->fbc_work->work))
350 /* tasklet was killed before being run, clean up */
351 kfree(dev_priv->fbc_work);
352
353 /* Mark the work as no longer wanted so that if it does
354 * wake-up (because the work was already running and waiting
355 * for our mutex), it will discover that is no longer
356 * necessary to run.
357 */
358 dev_priv->fbc_work = NULL;
359}
360
361void intel_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
362{
363 struct intel_fbc_work *work;
364 struct drm_device *dev = crtc->dev;
365 struct drm_i915_private *dev_priv = dev->dev_private;
366
367 if (!dev_priv->display.enable_fbc)
368 return;
369
370 intel_cancel_fbc_work(dev_priv);
371
372 work = kzalloc(sizeof *work, GFP_KERNEL);
373 if (work == NULL) {
374 dev_priv->display.enable_fbc(crtc, interval);
375 return;
376 }
377
378 work->crtc = crtc;
379 work->fb = crtc->fb;
380 work->interval = interval;
381 INIT_DELAYED_WORK(&work->work, intel_fbc_work_fn);
382
383 dev_priv->fbc_work = work;
384
385 DRM_DEBUG_KMS("scheduling delayed FBC enable\n");
386
387 /* Delay the actual enabling to let pageflipping cease and the
388 * display to settle before starting the compression. Note that
389 * this delay also serves a second purpose: it allows for a
390 * vblank to pass after disabling the FBC before we attempt
391 * to modify the control registers.
392 *
393 * A more complicated solution would involve tracking vblanks
394 * following the termination of the page-flipping sequence
395 * and indeed performing the enable as a co-routine and not
396 * waiting synchronously upon the vblank.
397 */
398 schedule_delayed_work(&work->work, msecs_to_jiffies(50));
399}
400
401void intel_disable_fbc(struct drm_device *dev)
402{
403 struct drm_i915_private *dev_priv = dev->dev_private;
404
405 intel_cancel_fbc_work(dev_priv);
406
407 if (!dev_priv->display.disable_fbc)
408 return;
409
410 dev_priv->display.disable_fbc(dev);
411 dev_priv->cfb_plane = -1;
412}
413
414/**
415 * intel_update_fbc - enable/disable FBC as needed
416 * @dev: the drm_device
417 *
418 * Set up the framebuffer compression hardware at mode set time. We
419 * enable it if possible:
420 * - plane A only (on pre-965)
421 * - no pixel mulitply/line duplication
422 * - no alpha buffer discard
423 * - no dual wide
424 * - framebuffer <= 2048 in width, 1536 in height
425 *
426 * We can't assume that any compression will take place (worst case),
427 * so the compressed buffer has to be the same size as the uncompressed
428 * one. It also must reside (along with the line length buffer) in
429 * stolen memory.
430 *
431 * We need to enable/disable FBC on a global basis.
432 */
433void intel_update_fbc(struct drm_device *dev)
434{
435 struct drm_i915_private *dev_priv = dev->dev_private;
436 struct drm_crtc *crtc = NULL, *tmp_crtc;
437 struct intel_crtc *intel_crtc;
438 struct drm_framebuffer *fb;
439 struct intel_framebuffer *intel_fb;
440 struct drm_i915_gem_object *obj;
441 int enable_fbc;
442
85208be0
ED		 443 if (!i915_powersave)
444 return;
445
446 if (!I915_HAS_FBC(dev))
447 return;
448
449 /*
450 * If FBC is already on, we just have to verify that we can
451 * keep it that way...
452 * Need to disable if:
453 * - more than one pipe is active
454 * - changing FBC params (stride, fence, mode)
455 * - new fb is too large to fit in compressed buffer
456 * - going to an unsupported config (interlace, pixel multiply, etc.)
457 */
458 list_for_each_entry(tmp_crtc, &dev->mode_config.crtc_list, head) {
3490ea5d
CW		 459 if (intel_crtc_active(tmp_crtc) &&
460 !to_intel_crtc(tmp_crtc)->primary_disabled) {
85208be0
ED		 461 if (crtc) {
462 DRM_DEBUG_KMS("more than one pipe active, disabling compression\n");
463 dev_priv->no_fbc_reason = FBC_MULTIPLE_PIPES;
464 goto out_disable;
465 }
466 crtc = tmp_crtc;
467 }
468 }
469
470 if (!crtc || crtc->fb == NULL) {
471 DRM_DEBUG_KMS("no output, disabling\n");
472 dev_priv->no_fbc_reason = FBC_NO_OUTPUT;
473 goto out_disable;
474 }
475
476 intel_crtc = to_intel_crtc(crtc);
477 fb = crtc->fb;
478 intel_fb = to_intel_framebuffer(fb);
479 obj = intel_fb->obj;
480
481 enable_fbc = i915_enable_fbc;
482 if (enable_fbc < 0) {
483 DRM_DEBUG_KMS("fbc set to per-chip default\n");
484 enable_fbc = 1;
891348b2 485 if (INTEL_INFO(dev)->gen <= 7 && !IS_HASWELL(dev))
85208be0
ED		 486 enable_fbc = 0;
487 }
488 if (!enable_fbc) {
489 DRM_DEBUG_KMS("fbc disabled per module param\n");
490 dev_priv->no_fbc_reason = FBC_MODULE_PARAM;
491 goto out_disable;
492 }
85208be0
ED		 493 if ((crtc->mode.flags & DRM_MODE_FLAG_INTERLACE) ||
494 (crtc->mode.flags & DRM_MODE_FLAG_DBLSCAN)) {
495 DRM_DEBUG_KMS("mode incompatible with compression, "
496 "disabling\n");
497 dev_priv->no_fbc_reason = FBC_UNSUPPORTED_MODE;
498 goto out_disable;
499 }
500 if ((crtc->mode.hdisplay > 2048) ||
501 (crtc->mode.vdisplay > 1536)) {
502 DRM_DEBUG_KMS("mode too large for compression, disabling\n");
503 dev_priv->no_fbc_reason = FBC_MODE_TOO_LARGE;
504 goto out_disable;
505 }
891348b2
RV		 506 if ((IS_I915GM(dev) || IS_I945GM(dev) || IS_HASWELL(dev)) &&
507 intel_crtc->plane != 0) {
85208be0
ED		 508 DRM_DEBUG_KMS("plane not 0, disabling compression\n");
509 dev_priv->no_fbc_reason = FBC_BAD_PLANE;
510 goto out_disable;
511 }
512
513 /* The use of a CPU fence is mandatory in order to detect writes
514 * by the CPU to the scanout and trigger updates to the FBC.
515 */
516 if (obj->tiling_mode != I915_TILING_X ||
517 obj->fence_reg == I915_FENCE_REG_NONE) {
518 DRM_DEBUG_KMS("framebuffer not tiled or fenced, disabling compression\n");
519 dev_priv->no_fbc_reason = FBC_NOT_TILED;
520 goto out_disable;
521 }
522
523 /* If the kernel debugger is active, always disable compression */
524 if (in_dbg_master())
525 goto out_disable;
526
11be49eb 527 if (i915_gem_stolen_setup_compression(dev, intel_fb->obj->base.size)) {
11be49eb
CW		 528 DRM_DEBUG_KMS("framebuffer too large, disabling compression\n");
529 dev_priv->no_fbc_reason = FBC_STOLEN_TOO_SMALL;
530 goto out_disable;
531 }
532
85208be0
ED		 533 /* If the scanout has not changed, don't modify the FBC settings.
534 * Note that we make the fundamental assumption that the fb->obj
535 * cannot be unpinned (and have its GTT offset and fence revoked)
536 * without first being decoupled from the scanout and FBC disabled.
537 */
538 if (dev_priv->cfb_plane == intel_crtc->plane &&
539 dev_priv->cfb_fb == fb->base.id &&
540 dev_priv->cfb_y == crtc->y)
541 return;
542
543 if (intel_fbc_enabled(dev)) {
544 /* We update FBC along two paths, after changing fb/crtc
545 * configuration (modeswitching) and after page-flipping
546 * finishes. For the latter, we know that not only did
547 * we disable the FBC at the start of the page-flip
548 * sequence, but also more than one vblank has passed.
549 *
550 * For the former case of modeswitching, it is possible
551 * to switch between two FBC valid configurations
552 * instantaneously so we do need to disable the FBC
553 * before we can modify its control registers. We also
554 * have to wait for the next vblank for that to take
555 * effect. However, since we delay enabling FBC we can
556 * assume that a vblank has passed since disabling and
557 * that we can safely alter the registers in the deferred
558 * callback.
559 *
560 * In the scenario that we go from a valid to invalid
561 * and then back to valid FBC configuration we have
562 * no strict enforcement that a vblank occurred since
563 * disabling the FBC. However, along all current pipe
564 * disabling paths we do need to wait for a vblank at
565 * some point. And we wait before enabling FBC anyway.
566 */
567 DRM_DEBUG_KMS("disabling active FBC for update\n");
568 intel_disable_fbc(dev);
569 }
570
571 intel_enable_fbc(crtc, 500);
572 return;
573
574out_disable:
575 /* Multiple disables should be harmless */
576 if (intel_fbc_enabled(dev)) {
577 DRM_DEBUG_KMS("unsupported config, disabling FBC\n");
578 intel_disable_fbc(dev);
579 }
11be49eb 580 i915_gem_stolen_cleanup_compression(dev);
85208be0
ED		 581}
582
c921aba8
DV		 583static void i915_pineview_get_mem_freq(struct drm_device *dev)
584{
585 drm_i915_private_t *dev_priv = dev->dev_private;
586 u32 tmp;
587
588 tmp = I915_READ(CLKCFG);
589
590 switch (tmp & CLKCFG_FSB_MASK) {
591 case CLKCFG_FSB_533:
592 dev_priv->fsb_freq = 533; /* 133*4 */
593 break;
594 case CLKCFG_FSB_800:
595 dev_priv->fsb_freq = 800; /* 200*4 */
596 break;
597 case CLKCFG_FSB_667:
598 dev_priv->fsb_freq = 667; /* 167*4 */
599 break;
600 case CLKCFG_FSB_400:
601 dev_priv->fsb_freq = 400; /* 100*4 */
602 break;
603 }
604
605 switch (tmp & CLKCFG_MEM_MASK) {
606 case CLKCFG_MEM_533:
607 dev_priv->mem_freq = 533;
608 break;
609 case CLKCFG_MEM_667:
610 dev_priv->mem_freq = 667;
611 break;
612 case CLKCFG_MEM_800:
613 dev_priv->mem_freq = 800;
614 break;
615 }
616
617 /* detect pineview DDR3 setting */
618 tmp = I915_READ(CSHRDDR3CTL);
619 dev_priv->is_ddr3 = (tmp & CSHRDDR3CTL_DDR3) ? 1 : 0;
620}
621
622static void i915_ironlake_get_mem_freq(struct drm_device *dev)
623{
624 drm_i915_private_t *dev_priv = dev->dev_private;
625 u16 ddrpll, csipll;
626
627 ddrpll = I915_READ16(DDRMPLL1);
628 csipll = I915_READ16(CSIPLL0);
629
630 switch (ddrpll & 0xff) {
631 case 0xc:
632 dev_priv->mem_freq = 800;
633 break;
634 case 0x10:
635 dev_priv->mem_freq = 1066;
636 break;
637 case 0x14:
638 dev_priv->mem_freq = 1333;
639 break;
640 case 0x18:
641 dev_priv->mem_freq = 1600;
642 break;
643 default:
644 DRM_DEBUG_DRIVER("unknown memory frequency 0x%02x\n",
645 ddrpll & 0xff);
646 dev_priv->mem_freq = 0;
647 break;
648 }
649
20e4d407 650 dev_priv->ips.r_t = dev_priv->mem_freq;
c921aba8
DV		 651
652 switch (csipll & 0x3ff) {
653 case 0x00c:
654 dev_priv->fsb_freq = 3200;
655 break;
656 case 0x00e:
657 dev_priv->fsb_freq = 3733;
658 break;
659 case 0x010:
660 dev_priv->fsb_freq = 4266;
661 break;
662 case 0x012:
663 dev_priv->fsb_freq = 4800;
664 break;
665 case 0x014:
666 dev_priv->fsb_freq = 5333;
667 break;
668 case 0x016:
669 dev_priv->fsb_freq = 5866;
670 break;
671 case 0x018:
672 dev_priv->fsb_freq = 6400;
673 break;
674 default:
675 DRM_DEBUG_DRIVER("unknown fsb frequency 0x%04x\n",
676 csipll & 0x3ff);
677 dev_priv->fsb_freq = 0;
678 break;
679 }
680
681 if (dev_priv->fsb_freq == 3200) {
20e4d407 682 dev_priv->ips.c_m = 0;
c921aba8 683 } else if (dev_priv->fsb_freq > 3200 && dev_priv->fsb_freq <= 4800) {
20e4d407 684 dev_priv->ips.c_m = 1;
c921aba8 685 } else {
20e4d407 686 dev_priv->ips.c_m = 2;
c921aba8
DV		 687 }
688}
689
b445e3b0
ED		 690static const struct cxsr_latency cxsr_latency_table[] = {
691 {1, 0, 800, 400, 3382, 33382, 3983, 33983}, /* DDR2-400 SC */
692 {1, 0, 800, 667, 3354, 33354, 3807, 33807}, /* DDR2-667 SC */
693 {1, 0, 800, 800, 3347, 33347, 3763, 33763}, /* DDR2-800 SC */
694 {1, 1, 800, 667, 6420, 36420, 6873, 36873}, /* DDR3-667 SC */
695 {1, 1, 800, 800, 5902, 35902, 6318, 36318}, /* DDR3-800 SC */
696
697 {1, 0, 667, 400, 3400, 33400, 4021, 34021}, /* DDR2-400 SC */
698 {1, 0, 667, 667, 3372, 33372, 3845, 33845}, /* DDR2-667 SC */
699 {1, 0, 667, 800, 3386, 33386, 3822, 33822}, /* DDR2-800 SC */
700 {1, 1, 667, 667, 6438, 36438, 6911, 36911}, /* DDR3-667 SC */
701 {1, 1, 667, 800, 5941, 35941, 6377, 36377}, /* DDR3-800 SC */
702
703 {1, 0, 400, 400, 3472, 33472, 4173, 34173}, /* DDR2-400 SC */
704 {1, 0, 400, 667, 3443, 33443, 3996, 33996}, /* DDR2-667 SC */
705 {1, 0, 400, 800, 3430, 33430, 3946, 33946}, /* DDR2-800 SC */
706 {1, 1, 400, 667, 6509, 36509, 7062, 37062}, /* DDR3-667 SC */
707 {1, 1, 400, 800, 5985, 35985, 6501, 36501}, /* DDR3-800 SC */
708
709 {0, 0, 800, 400, 3438, 33438, 4065, 34065}, /* DDR2-400 SC */
710 {0, 0, 800, 667, 3410, 33410, 3889, 33889}, /* DDR2-667 SC */
711 {0, 0, 800, 800, 3403, 33403, 3845, 33845}, /* DDR2-800 SC */
712 {0, 1, 800, 667, 6476, 36476, 6955, 36955}, /* DDR3-667 SC */
713 {0, 1, 800, 800, 5958, 35958, 6400, 36400}, /* DDR3-800 SC */
714
715 {0, 0, 667, 400, 3456, 33456, 4103, 34106}, /* DDR2-400 SC */
716 {0, 0, 667, 667, 3428, 33428, 3927, 33927}, /* DDR2-667 SC */
717 {0, 0, 667, 800, 3443, 33443, 3905, 33905}, /* DDR2-800 SC */
718 {0, 1, 667, 667, 6494, 36494, 6993, 36993}, /* DDR3-667 SC */
719 {0, 1, 667, 800, 5998, 35998, 6460, 36460}, /* DDR3-800 SC */
720
721 {0, 0, 400, 400, 3528, 33528, 4255, 34255}, /* DDR2-400 SC */
722 {0, 0, 400, 667, 3500, 33500, 4079, 34079}, /* DDR2-667 SC */
723 {0, 0, 400, 800, 3487, 33487, 4029, 34029}, /* DDR2-800 SC */
724 {0, 1, 400, 667, 6566, 36566, 7145, 37145}, /* DDR3-667 SC */
725 {0, 1, 400, 800, 6042, 36042, 6584, 36584}, /* DDR3-800 SC */
726};
727
63c62275 728static const struct cxsr_latency *intel_get_cxsr_latency(int is_desktop,
b445e3b0
ED		 729 int is_ddr3,
730 int fsb,
731 int mem)
732{
733 const struct cxsr_latency *latency;
734 int i;
735
736 if (fsb == 0 || mem == 0)
737 return NULL;
738
739 for (i = 0; i < ARRAY_SIZE(cxsr_latency_table); i++) {
740 latency = &cxsr_latency_table[i];
741 if (is_desktop == latency->is_desktop &&
742 is_ddr3 == latency->is_ddr3 &&
743 fsb == latency->fsb_freq && mem == latency->mem_freq)
744 return latency;
745 }
746
747 DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");
748
749 return NULL;
750}
751
1fa61106 752static void pineview_disable_cxsr(struct drm_device *dev)
b445e3b0
ED		 753{
754 struct drm_i915_private *dev_priv = dev->dev_private;
755
756 /* deactivate cxsr */
757 I915_WRITE(DSPFW3, I915_READ(DSPFW3) & ~PINEVIEW_SELF_REFRESH_EN);
758}
759
760/*
761 * Latency for FIFO fetches is dependent on several factors:
762 * - memory configuration (speed, channels)
763 * - chipset
764 * - current MCH state
765 * It can be fairly high in some situations, so here we assume a fairly
766 * pessimal value. It's a tradeoff between extra memory fetches (if we
767 * set this value too high, the FIFO will fetch frequently to stay full)
768 * and power consumption (set it too low to save power and we might see
769 * FIFO underruns and display "flicker").
770 *
771 * A value of 5us seems to be a good balance; safe for very low end
772 * platforms but not overly aggressive on lower latency configs.
773 */
774static const int latency_ns = 5000;
775
1fa61106 776static int i9xx_get_fifo_size(struct drm_device *dev, int plane)
b445e3b0
ED		 777{
778 struct drm_i915_private *dev_priv = dev->dev_private;
779 uint32_t dsparb = I915_READ(DSPARB);
780 int size;
781
782 size = dsparb & 0x7f;
783 if (plane)
784 size = ((dsparb >> DSPARB_CSTART_SHIFT) & 0x7f) - size;
785
786 DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
787 plane ? "B" : "A", size);
788
789 return size;
790}
791
1fa61106 792static int i85x_get_fifo_size(struct drm_device *dev, int plane)
b445e3b0
ED		 793{
794 struct drm_i915_private *dev_priv = dev->dev_private;
795 uint32_t dsparb = I915_READ(DSPARB);
796 int size;
797
798 size = dsparb & 0x1ff;
799 if (plane)
800 size = ((dsparb >> DSPARB_BEND_SHIFT) & 0x1ff) - size;
801 size >>= 1; /* Convert to cachelines */
802
803 DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
804 plane ? "B" : "A", size);
805
806 return size;
807}
808
1fa61106 809static int i845_get_fifo_size(struct drm_device *dev, int plane)
b445e3b0
ED		 810{
811 struct drm_i915_private *dev_priv = dev->dev_private;
812 uint32_t dsparb = I915_READ(DSPARB);
813 int size;
814
815 size = dsparb & 0x7f;
816 size >>= 2; /* Convert to cachelines */
817
818 DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
819 plane ? "B" : "A",
820 size);
821
822 return size;
823}
824
1fa61106 825static int i830_get_fifo_size(struct drm_device *dev, int plane)
b445e3b0
ED		 826{
827 struct drm_i915_private *dev_priv = dev->dev_private;
828 uint32_t dsparb = I915_READ(DSPARB);
829 int size;
830
831 size = dsparb & 0x7f;
832 size >>= 1; /* Convert to cachelines */
833
834 DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
835 plane ? "B" : "A", size);
836
837 return size;
838}
839
840/* Pineview has different values for various configs */
841static const struct intel_watermark_params pineview_display_wm = {
842 PINEVIEW_DISPLAY_FIFO,
843 PINEVIEW_MAX_WM,
844 PINEVIEW_DFT_WM,
845 PINEVIEW_GUARD_WM,
846 PINEVIEW_FIFO_LINE_SIZE
847};
848static const struct intel_watermark_params pineview_display_hplloff_wm = {
849 PINEVIEW_DISPLAY_FIFO,
850 PINEVIEW_MAX_WM,
851 PINEVIEW_DFT_HPLLOFF_WM,
852 PINEVIEW_GUARD_WM,
853 PINEVIEW_FIFO_LINE_SIZE
854};
855static const struct intel_watermark_params pineview_cursor_wm = {
856 PINEVIEW_CURSOR_FIFO,
857 PINEVIEW_CURSOR_MAX_WM,
858 PINEVIEW_CURSOR_DFT_WM,
859 PINEVIEW_CURSOR_GUARD_WM,
860 PINEVIEW_FIFO_LINE_SIZE,
861};
862static const struct intel_watermark_params pineview_cursor_hplloff_wm = {
863 PINEVIEW_CURSOR_FIFO,
864 PINEVIEW_CURSOR_MAX_WM,
865 PINEVIEW_CURSOR_DFT_WM,
866 PINEVIEW_CURSOR_GUARD_WM,
867 PINEVIEW_FIFO_LINE_SIZE
868};
869static const struct intel_watermark_params g4x_wm_info = {
870 G4X_FIFO_SIZE,
871 G4X_MAX_WM,
872 G4X_MAX_WM,
873 2,
874 G4X_FIFO_LINE_SIZE,
875};
876static const struct intel_watermark_params g4x_cursor_wm_info = {
877 I965_CURSOR_FIFO,
878 I965_CURSOR_MAX_WM,
879 I965_CURSOR_DFT_WM,
880 2,
881 G4X_FIFO_LINE_SIZE,
882};
883static const struct intel_watermark_params valleyview_wm_info = {
884 VALLEYVIEW_FIFO_SIZE,
885 VALLEYVIEW_MAX_WM,
886 VALLEYVIEW_MAX_WM,
887 2,
888 G4X_FIFO_LINE_SIZE,
889};
890static const struct intel_watermark_params valleyview_cursor_wm_info = {
891 I965_CURSOR_FIFO,
892 VALLEYVIEW_CURSOR_MAX_WM,
893 I965_CURSOR_DFT_WM,
894 2,
895 G4X_FIFO_LINE_SIZE,
896};
897static const struct intel_watermark_params i965_cursor_wm_info = {
898 I965_CURSOR_FIFO,
899 I965_CURSOR_MAX_WM,
900 I965_CURSOR_DFT_WM,
901 2,
902 I915_FIFO_LINE_SIZE,
903};
904static const struct intel_watermark_params i945_wm_info = {
905 I945_FIFO_SIZE,
906 I915_MAX_WM,
907 1,
908 2,
909 I915_FIFO_LINE_SIZE
910};
911static const struct intel_watermark_params i915_wm_info = {
912 I915_FIFO_SIZE,
913 I915_MAX_WM,
914 1,
915 2,
916 I915_FIFO_LINE_SIZE
917};
918static const struct intel_watermark_params i855_wm_info = {
919 I855GM_FIFO_SIZE,
920 I915_MAX_WM,
921 1,
922 2,
923 I830_FIFO_LINE_SIZE
924};
925static const struct intel_watermark_params i830_wm_info = {
926 I830_FIFO_SIZE,
927 I915_MAX_WM,
928 1,
929 2,
930 I830_FIFO_LINE_SIZE
931};
932
933static const struct intel_watermark_params ironlake_display_wm_info = {
934 ILK_DISPLAY_FIFO,
935 ILK_DISPLAY_MAXWM,
936 ILK_DISPLAY_DFTWM,
937 2,
938 ILK_FIFO_LINE_SIZE
939};
940static const struct intel_watermark_params ironlake_cursor_wm_info = {
941 ILK_CURSOR_FIFO,
942 ILK_CURSOR_MAXWM,
943 ILK_CURSOR_DFTWM,
944 2,
945 ILK_FIFO_LINE_SIZE
946};
947static const struct intel_watermark_params ironlake_display_srwm_info = {
948 ILK_DISPLAY_SR_FIFO,
949 ILK_DISPLAY_MAX_SRWM,
950 ILK_DISPLAY_DFT_SRWM,
951 2,
952 ILK_FIFO_LINE_SIZE
953};
954static const struct intel_watermark_params ironlake_cursor_srwm_info = {
955 ILK_CURSOR_SR_FIFO,
956 ILK_CURSOR_MAX_SRWM,
957 ILK_CURSOR_DFT_SRWM,
958 2,
959 ILK_FIFO_LINE_SIZE
960};
961
962static const struct intel_watermark_params sandybridge_display_wm_info = {
963 SNB_DISPLAY_FIFO,
964 SNB_DISPLAY_MAXWM,
965 SNB_DISPLAY_DFTWM,
966 2,
967 SNB_FIFO_LINE_SIZE
968};
969static const struct intel_watermark_params sandybridge_cursor_wm_info = {
970 SNB_CURSOR_FIFO,
971 SNB_CURSOR_MAXWM,
972 SNB_CURSOR_DFTWM,
973 2,
974 SNB_FIFO_LINE_SIZE
975};
976static const struct intel_watermark_params sandybridge_display_srwm_info = {
977 SNB_DISPLAY_SR_FIFO,
978 SNB_DISPLAY_MAX_SRWM,
979 SNB_DISPLAY_DFT_SRWM,
980 2,
981 SNB_FIFO_LINE_SIZE
982};
983static const struct intel_watermark_params sandybridge_cursor_srwm_info = {
984 SNB_CURSOR_SR_FIFO,
985 SNB_CURSOR_MAX_SRWM,
986 SNB_CURSOR_DFT_SRWM,
987 2,
988 SNB_FIFO_LINE_SIZE
989};
990
991
992/**
993 * intel_calculate_wm - calculate watermark level
994 * @clock_in_khz: pixel clock
995 * @wm: chip FIFO params
996 * @pixel_size: display pixel size
997 * @latency_ns: memory latency for the platform
998 *
999 * Calculate the watermark level (the level at which the display plane will
1000 * start fetching from memory again). Each chip has a different display
1001 * FIFO size and allocation, so the caller needs to figure that out and pass
1002 * in the correct intel_watermark_params structure.
1003 *
1004 * As the pixel clock runs, the FIFO will be drained at a rate that depends
1005 * on the pixel size. When it reaches the watermark level, it'll start
1006 * fetching FIFO line sized based chunks from memory until the FIFO fills
1007 * past the watermark point. If the FIFO drains completely, a FIFO underrun
1008 * will occur, and a display engine hang could result.
1009 */
1010static unsigned long intel_calculate_wm(unsigned long clock_in_khz,
1011 const struct intel_watermark_params *wm,
1012 int fifo_size,
1013 int pixel_size,
1014 unsigned long latency_ns)
1015{
1016 long entries_required, wm_size;
1017
1018 /*
1019 * Note: we need to make sure we don't overflow for various clock &
1020 * latency values.
1021 * clocks go from a few thousand to several hundred thousand.
1022 * latency is usually a few thousand
1023 */
1024 entries_required = ((clock_in_khz / 1000) * pixel_size * latency_ns) /
1025 1000;
1026 entries_required = DIV_ROUND_UP(entries_required, wm->cacheline_size);
1027
1028 DRM_DEBUG_KMS("FIFO entries required for mode: %ld\n", entries_required);
1029
1030 wm_size = fifo_size - (entries_required + wm->guard_size);
1031
1032 DRM_DEBUG_KMS("FIFO watermark level: %ld\n", wm_size);
1033
1034 /* Don't promote wm_size to unsigned... */
1035 if (wm_size > (long)wm->max_wm)
1036 wm_size = wm->max_wm;
1037 if (wm_size <= 0)
1038 wm_size = wm->default_wm;
1039 return wm_size;
1040}
1041
1042static struct drm_crtc *single_enabled_crtc(struct drm_device *dev)
1043{
1044 struct drm_crtc *crtc, *enabled = NULL;
1045
1046 list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
3490ea5d 1047 if (intel_crtc_active(crtc)) {
b445e3b0
ED		 1048 if (enabled)
1049 return NULL;
1050 enabled = crtc;
1051 }
1052 }
1053
1054 return enabled;
1055}
1056
1fa61106 1057static void pineview_update_wm(struct drm_device *dev)
b445e3b0
ED		 1058{
1059 struct drm_i915_private *dev_priv = dev->dev_private;
1060 struct drm_crtc *crtc;
1061 const struct cxsr_latency *latency;
1062 u32 reg;
1063 unsigned long wm;
1064
1065 latency = intel_get_cxsr_latency(IS_PINEVIEW_G(dev), dev_priv->is_ddr3,
1066 dev_priv->fsb_freq, dev_priv->mem_freq);
1067 if (!latency) {
1068 DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");
1069 pineview_disable_cxsr(dev);
1070 return;
1071 }
1072
1073 crtc = single_enabled_crtc(dev);
1074 if (crtc) {
1075 int clock = crtc->mode.clock;
1076 int pixel_size = crtc->fb->bits_per_pixel / 8;
1077
1078 /* Display SR */
1079 wm = intel_calculate_wm(clock, &pineview_display_wm,
1080 pineview_display_wm.fifo_size,
1081 pixel_size, latency->display_sr);
1082 reg = I915_READ(DSPFW1);
1083 reg &= ~DSPFW_SR_MASK;
1084 reg |= wm << DSPFW_SR_SHIFT;
1085 I915_WRITE(DSPFW1, reg);
1086 DRM_DEBUG_KMS("DSPFW1 register is %x\n", reg);
1087
1088 /* cursor SR */
1089 wm = intel_calculate_wm(clock, &pineview_cursor_wm,
1090 pineview_display_wm.fifo_size,
1091 pixel_size, latency->cursor_sr);
1092 reg = I915_READ(DSPFW3);
1093 reg &= ~DSPFW_CURSOR_SR_MASK;
1094 reg |= (wm & 0x3f) << DSPFW_CURSOR_SR_SHIFT;
1095 I915_WRITE(DSPFW3, reg);
1096
1097 /* Display HPLL off SR */
1098 wm = intel_calculate_wm(clock, &pineview_display_hplloff_wm,
1099 pineview_display_hplloff_wm.fifo_size,
1100 pixel_size, latency->display_hpll_disable);
1101 reg = I915_READ(DSPFW3);
1102 reg &= ~DSPFW_HPLL_SR_MASK;
1103 reg |= wm & DSPFW_HPLL_SR_MASK;
1104 I915_WRITE(DSPFW3, reg);
1105
1106 /* cursor HPLL off SR */
1107 wm = intel_calculate_wm(clock, &pineview_cursor_hplloff_wm,
1108 pineview_display_hplloff_wm.fifo_size,
1109 pixel_size, latency->cursor_hpll_disable);
1110 reg = I915_READ(DSPFW3);
1111 reg &= ~DSPFW_HPLL_CURSOR_MASK;
1112 reg |= (wm & 0x3f) << DSPFW_HPLL_CURSOR_SHIFT;
1113 I915_WRITE(DSPFW3, reg);
1114 DRM_DEBUG_KMS("DSPFW3 register is %x\n", reg);
1115
1116 /* activate cxsr */
1117 I915_WRITE(DSPFW3,
1118 I915_READ(DSPFW3) | PINEVIEW_SELF_REFRESH_EN);
1119 DRM_DEBUG_KMS("Self-refresh is enabled\n");
1120 } else {
1121 pineview_disable_cxsr(dev);
1122 DRM_DEBUG_KMS("Self-refresh is disabled\n");
1123 }
1124}
1125
1126static bool g4x_compute_wm0(struct drm_device *dev,
1127 int plane,
1128 const struct intel_watermark_params *display,
1129 int display_latency_ns,
1130 const struct intel_watermark_params *cursor,
1131 int cursor_latency_ns,
1132 int *plane_wm,
1133 int *cursor_wm)
1134{
1135 struct drm_crtc *crtc;
1136 int htotal, hdisplay, clock, pixel_size;
1137 int line_time_us, line_count;
1138 int entries, tlb_miss;
1139
1140 crtc = intel_get_crtc_for_plane(dev, plane);
3490ea5d 1141 if (!intel_crtc_active(crtc)) {
b445e3b0
ED		 1142 *cursor_wm = cursor->guard_size;
1143 *plane_wm = display->guard_size;
1144 return false;
1145 }
1146
1147 htotal = crtc->mode.htotal;
1148 hdisplay = crtc->mode.hdisplay;
1149 clock = crtc->mode.clock;
1150 pixel_size = crtc->fb->bits_per_pixel / 8;
1151
1152 /* Use the small buffer method to calculate plane watermark */
1153 entries = ((clock * pixel_size / 1000) * display_latency_ns) / 1000;
1154 tlb_miss = display->fifo_size*display->cacheline_size - hdisplay * 8;
1155 if (tlb_miss > 0)
1156 entries += tlb_miss;
1157 entries = DIV_ROUND_UP(entries, display->cacheline_size);
1158 *plane_wm = entries + display->guard_size;
1159 if (*plane_wm > (int)display->max_wm)
1160 *plane_wm = display->max_wm;
1161
1162 /* Use the large buffer method to calculate cursor watermark */
1163 line_time_us = ((htotal * 1000) / clock);
1164 line_count = (cursor_latency_ns / line_time_us + 1000) / 1000;
1165 entries = line_count * 64 * pixel_size;
1166 tlb_miss = cursor->fifo_size*cursor->cacheline_size - hdisplay * 8;
1167 if (tlb_miss > 0)
1168 entries += tlb_miss;
1169 entries = DIV_ROUND_UP(entries, cursor->cacheline_size);
1170 *cursor_wm = entries + cursor->guard_size;
1171 if (*cursor_wm > (int)cursor->max_wm)
1172 *cursor_wm = (int)cursor->max_wm;
1173
1174 return true;
1175}
1176
1177/*
1178 * Check the wm result.
1179 *
1180 * If any calculated watermark values is larger than the maximum value that
1181 * can be programmed into the associated watermark register, that watermark
1182 * must be disabled.
1183 */
1184static bool g4x_check_srwm(struct drm_device *dev,
1185 int display_wm, int cursor_wm,
1186 const struct intel_watermark_params *display,
1187 const struct intel_watermark_params *cursor)
1188{
1189 DRM_DEBUG_KMS("SR watermark: display plane %d, cursor %d\n",
1190 display_wm, cursor_wm);
1191
1192 if (display_wm > display->max_wm) {
1193 DRM_DEBUG_KMS("display watermark is too large(%d/%ld), disabling\n",
1194 display_wm, display->max_wm);
1195 return false;
1196 }
1197
1198 if (cursor_wm > cursor->max_wm) {
1199 DRM_DEBUG_KMS("cursor watermark is too large(%d/%ld), disabling\n",
1200 cursor_wm, cursor->max_wm);
1201 return false;
1202 }
1203
1204 if (!(display_wm || cursor_wm)) {
1205 DRM_DEBUG_KMS("SR latency is 0, disabling\n");
1206 return false;
1207 }
1208
1209 return true;
1210}
1211
1212static bool g4x_compute_srwm(struct drm_device *dev,
1213 int plane,
1214 int latency_ns,
1215 const struct intel_watermark_params *display,
1216 const struct intel_watermark_params *cursor,
1217 int *display_wm, int *cursor_wm)
1218{
1219 struct drm_crtc *crtc;
1220 int hdisplay, htotal, pixel_size, clock;
1221 unsigned long line_time_us;
1222 int line_count, line_size;
1223 int small, large;
1224 int entries;
1225
1226 if (!latency_ns) {
1227 *display_wm = *cursor_wm = 0;
1228 return false;
1229 }
1230
1231 crtc = intel_get_crtc_for_plane(dev, plane);
1232 hdisplay = crtc->mode.hdisplay;
1233 htotal = crtc->mode.htotal;
1234 clock = crtc->mode.clock;
1235 pixel_size = crtc->fb->bits_per_pixel / 8;
1236
1237 line_time_us = (htotal * 1000) / clock;
1238 line_count = (latency_ns / line_time_us + 1000) / 1000;
1239 line_size = hdisplay * pixel_size;
1240
1241 /* Use the minimum of the small and large buffer method for primary */
1242 small = ((clock * pixel_size / 1000) * latency_ns) / 1000;
1243 large = line_count * line_size;
1244
1245 entries = DIV_ROUND_UP(min(small, large), display->cacheline_size);
1246 *display_wm = entries + display->guard_size;
1247
1248 /* calculate the self-refresh watermark for display cursor */
1249 entries = line_count * pixel_size * 64;
1250 entries = DIV_ROUND_UP(entries, cursor->cacheline_size);
1251 *cursor_wm = entries + cursor->guard_size;
1252
1253 return g4x_check_srwm(dev,
1254 *display_wm, *cursor_wm,
1255 display, cursor);
1256}
1257
1258static bool vlv_compute_drain_latency(struct drm_device *dev,
1259 int plane,
1260 int *plane_prec_mult,
1261 int *plane_dl,
1262 int *cursor_prec_mult,
1263 int *cursor_dl)
1264{
1265 struct drm_crtc *crtc;
1266 int clock, pixel_size;
1267 int entries;
1268
1269 crtc = intel_get_crtc_for_plane(dev, plane);
3490ea5d 1270 if (!intel_crtc_active(crtc))
b445e3b0
ED		 1271 return false;
1272
1273 clock = crtc->mode.clock; /* VESA DOT Clock */
1274 pixel_size = crtc->fb->bits_per_pixel / 8; /* BPP */
1275
1276 entries = (clock / 1000) * pixel_size;
1277 *plane_prec_mult = (entries > 256) ?
1278 DRAIN_LATENCY_PRECISION_32 : DRAIN_LATENCY_PRECISION_16;
1279 *plane_dl = (64 * (*plane_prec_mult) * 4) / ((clock / 1000) *
1280 pixel_size);
1281
1282 entries = (clock / 1000) * 4; /* BPP is always 4 for cursor */
1283 *cursor_prec_mult = (entries > 256) ?
1284 DRAIN_LATENCY_PRECISION_32 : DRAIN_LATENCY_PRECISION_16;
1285 *cursor_dl = (64 * (*cursor_prec_mult) * 4) / ((clock / 1000) * 4);
1286
1287 return true;
1288}
1289
1290/*
1291 * Update drain latency registers of memory arbiter
1292 *
1293 * Valleyview SoC has a new memory arbiter and needs drain latency registers
1294 * to be programmed. Each plane has a drain latency multiplier and a drain
1295 * latency value.
1296 */
1297
1298static void vlv_update_drain_latency(struct drm_device *dev)
1299{
1300 struct drm_i915_private *dev_priv = dev->dev_private;
1301 int planea_prec, planea_dl, planeb_prec, planeb_dl;
1302 int cursora_prec, cursora_dl, cursorb_prec, cursorb_dl;
1303 int plane_prec_mult, cursor_prec_mult; /* Precision multiplier is
1304 either 16 or 32 */
1305
1306 /* For plane A, Cursor A */
1307 if (vlv_compute_drain_latency(dev, 0, &plane_prec_mult, &planea_dl,
1308 &cursor_prec_mult, &cursora_dl)) {
1309 cursora_prec = (cursor_prec_mult == DRAIN_LATENCY_PRECISION_32) ?
1310 DDL_CURSORA_PRECISION_32 : DDL_CURSORA_PRECISION_16;
1311 planea_prec = (plane_prec_mult == DRAIN_LATENCY_PRECISION_32) ?
1312 DDL_PLANEA_PRECISION_32 : DDL_PLANEA_PRECISION_16;
1313
1314 I915_WRITE(VLV_DDL1, cursora_prec |
1315 (cursora_dl << DDL_CURSORA_SHIFT) |
1316 planea_prec | planea_dl);
1317 }
1318
1319 /* For plane B, Cursor B */
1320 if (vlv_compute_drain_latency(dev, 1, &plane_prec_mult, &planeb_dl,
1321 &cursor_prec_mult, &cursorb_dl)) {
1322 cursorb_prec = (cursor_prec_mult == DRAIN_LATENCY_PRECISION_32) ?
1323 DDL_CURSORB_PRECISION_32 : DDL_CURSORB_PRECISION_16;
1324 planeb_prec = (plane_prec_mult == DRAIN_LATENCY_PRECISION_32) ?
1325 DDL_PLANEB_PRECISION_32 : DDL_PLANEB_PRECISION_16;
1326
1327 I915_WRITE(VLV_DDL2, cursorb_prec |
1328 (cursorb_dl << DDL_CURSORB_SHIFT) |
1329 planeb_prec | planeb_dl);
1330 }
1331}
1332
1333#define single_plane_enabled(mask) is_power_of_2(mask)
1334
1fa61106 1335static void valleyview_update_wm(struct drm_device *dev)
b445e3b0
ED		 1336{
1337 static const int sr_latency_ns = 12000;
1338 struct drm_i915_private *dev_priv = dev->dev_private;
1339 int planea_wm, planeb_wm, cursora_wm, cursorb_wm;
1340 int plane_sr, cursor_sr;
af6c4575 1341 int ignore_plane_sr, ignore_cursor_sr;
b445e3b0
ED		 1342 unsigned int enabled = 0;
1343
1344 vlv_update_drain_latency(dev);
1345
1346 if (g4x_compute_wm0(dev, 0,
1347 &valleyview_wm_info, latency_ns,
1348 &valleyview_cursor_wm_info, latency_ns,
1349 &planea_wm, &cursora_wm))
1350 enabled |= 1;
1351
1352 if (g4x_compute_wm0(dev, 1,
1353 &valleyview_wm_info, latency_ns,
1354 &valleyview_cursor_wm_info, latency_ns,
1355 &planeb_wm, &cursorb_wm))
1356 enabled |= 2;
1357
b445e3b0
ED		 1358 if (single_plane_enabled(enabled) &&
1359 g4x_compute_srwm(dev, ffs(enabled) - 1,
1360 sr_latency_ns,
1361 &valleyview_wm_info,
1362 &valleyview_cursor_wm_info,
af6c4575
CW		 1363 &plane_sr, &ignore_cursor_sr) &&
1364 g4x_compute_srwm(dev, ffs(enabled) - 1,
1365 2*sr_latency_ns,
1366 &valleyview_wm_info,
1367 &valleyview_cursor_wm_info,
52bd02d8 1368 &ignore_plane_sr, &cursor_sr)) {
b445e3b0 1369 I915_WRITE(FW_BLC_SELF_VLV, FW_CSPWRDWNEN);
52bd02d8 1370 } else {
b445e3b0
ED		 1371 I915_WRITE(FW_BLC_SELF_VLV,
1372 I915_READ(FW_BLC_SELF_VLV) & ~FW_CSPWRDWNEN);
52bd02d8
CW		 1373 plane_sr = cursor_sr = 0;
1374 }
b445e3b0
ED		 1375
1376 DRM_DEBUG_KMS("Setting FIFO watermarks - A: plane=%d, cursor=%d, B: plane=%d, cursor=%d, SR: plane=%d, cursor=%d\n",
1377 planea_wm, cursora_wm,
1378 planeb_wm, cursorb_wm,
1379 plane_sr, cursor_sr);
1380
1381 I915_WRITE(DSPFW1,
1382 (plane_sr << DSPFW_SR_SHIFT) |
1383 (cursorb_wm << DSPFW_CURSORB_SHIFT) |
1384 (planeb_wm << DSPFW_PLANEB_SHIFT) |
1385 planea_wm);
1386 I915_WRITE(DSPFW2,
8c919b28 1387 (I915_READ(DSPFW2) & ~DSPFW_CURSORA_MASK) |
b445e3b0
ED		 1388 (cursora_wm << DSPFW_CURSORA_SHIFT));
1389 I915_WRITE(DSPFW3,
8c919b28
CW		 1390 (I915_READ(DSPFW3) & ~DSPFW_CURSOR_SR_MASK) |
1391 (cursor_sr << DSPFW_CURSOR_SR_SHIFT));
b445e3b0
ED		 1392}
1393
1fa61106 1394static void g4x_update_wm(struct drm_device *dev)
b445e3b0
ED		 1395{
1396 static const int sr_latency_ns = 12000;
1397 struct drm_i915_private *dev_priv = dev->dev_private;
1398 int planea_wm, planeb_wm, cursora_wm, cursorb_wm;
1399 int plane_sr, cursor_sr;
1400 unsigned int enabled = 0;
1401
1402 if (g4x_compute_wm0(dev, 0,
1403 &g4x_wm_info, latency_ns,
1404 &g4x_cursor_wm_info, latency_ns,
1405 &planea_wm, &cursora_wm))
1406 enabled |= 1;
1407
1408 if (g4x_compute_wm0(dev, 1,
1409 &g4x_wm_info, latency_ns,
1410 &g4x_cursor_wm_info, latency_ns,
1411 &planeb_wm, &cursorb_wm))
1412 enabled |= 2;
1413
b445e3b0
ED		 1414 if (single_plane_enabled(enabled) &&
1415 g4x_compute_srwm(dev, ffs(enabled) - 1,
1416 sr_latency_ns,
1417 &g4x_wm_info,
1418 &g4x_cursor_wm_info,
52bd02d8 1419 &plane_sr, &cursor_sr)) {
b445e3b0 1420 I915_WRITE(FW_BLC_SELF, FW_BLC_SELF_EN);
52bd02d8 1421 } else {
b445e3b0
ED		 1422 I915_WRITE(FW_BLC_SELF,
1423 I915_READ(FW_BLC_SELF) & ~FW_BLC_SELF_EN);
52bd02d8
CW		 1424 plane_sr = cursor_sr = 0;
1425 }
b445e3b0
ED		 1426
1427 DRM_DEBUG_KMS("Setting FIFO watermarks - A: plane=%d, cursor=%d, B: plane=%d, cursor=%d, SR: plane=%d, cursor=%d\n",
1428 planea_wm, cursora_wm,
1429 planeb_wm, cursorb_wm,
1430 plane_sr, cursor_sr);
1431
1432 I915_WRITE(DSPFW1,
1433 (plane_sr << DSPFW_SR_SHIFT) |
1434 (cursorb_wm << DSPFW_CURSORB_SHIFT) |
1435 (planeb_wm << DSPFW_PLANEB_SHIFT) |
1436 planea_wm);
1437 I915_WRITE(DSPFW2,
8c919b28 1438 (I915_READ(DSPFW2) & ~DSPFW_CURSORA_MASK) |
b445e3b0
ED		 1439 (cursora_wm << DSPFW_CURSORA_SHIFT));
1440 /* HPLL off in SR has some issues on G4x... disable it */
1441 I915_WRITE(DSPFW3,
8c919b28 1442 (I915_READ(DSPFW3) & ~(DSPFW_HPLL_SR_EN | DSPFW_CURSOR_SR_MASK)) |
b445e3b0
ED		 1443 (cursor_sr << DSPFW_CURSOR_SR_SHIFT));
1444}
1445
1fa61106 1446static void i965_update_wm(struct drm_device *dev)
b445e3b0
ED		 1447{
1448 struct drm_i915_private *dev_priv = dev->dev_private;
1449 struct drm_crtc *crtc;
1450 int srwm = 1;
1451 int cursor_sr = 16;
1452
1453 /* Calc sr entries for one plane configs */
1454 crtc = single_enabled_crtc(dev);
1455 if (crtc) {
1456 /* self-refresh has much higher latency */
1457 static const int sr_latency_ns = 12000;
1458 int clock = crtc->mode.clock;
1459 int htotal = crtc->mode.htotal;
1460 int hdisplay = crtc->mode.hdisplay;
1461 int pixel_size = crtc->fb->bits_per_pixel / 8;
1462 unsigned long line_time_us;
1463 int entries;
1464
1465 line_time_us = ((htotal * 1000) / clock);
1466
1467 /* Use ns/us then divide to preserve precision */
1468 entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) *
1469 pixel_size * hdisplay;
1470 entries = DIV_ROUND_UP(entries, I915_FIFO_LINE_SIZE);
1471 srwm = I965_FIFO_SIZE - entries;
1472 if (srwm < 0)
1473 srwm = 1;
1474 srwm &= 0x1ff;
1475 DRM_DEBUG_KMS("self-refresh entries: %d, wm: %d\n",
1476 entries, srwm);
1477
1478 entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) *
1479 pixel_size * 64;
1480 entries = DIV_ROUND_UP(entries,
1481 i965_cursor_wm_info.cacheline_size);
1482 cursor_sr = i965_cursor_wm_info.fifo_size -
1483 (entries + i965_cursor_wm_info.guard_size);
1484
1485 if (cursor_sr > i965_cursor_wm_info.max_wm)
1486 cursor_sr = i965_cursor_wm_info.max_wm;
1487
1488 DRM_DEBUG_KMS("self-refresh watermark: display plane %d "
1489 "cursor %d\n", srwm, cursor_sr);
1490
1491 if (IS_CRESTLINE(dev))
1492 I915_WRITE(FW_BLC_SELF, FW_BLC_SELF_EN);
1493 } else {
1494 /* Turn off self refresh if both pipes are enabled */
1495 if (IS_CRESTLINE(dev))
1496 I915_WRITE(FW_BLC_SELF, I915_READ(FW_BLC_SELF)
1497 & ~FW_BLC_SELF_EN);
1498 }
1499
1500 DRM_DEBUG_KMS("Setting FIFO watermarks - A: 8, B: 8, C: 8, SR %d\n",
1501 srwm);
1502
1503 /* 965 has limitations... */
1504 I915_WRITE(DSPFW1, (srwm << DSPFW_SR_SHIFT) |
1505 (8 << 16) | (8 << 8) | (8 << 0));
1506 I915_WRITE(DSPFW2, (8 << 8) | (8 << 0));
1507 /* update cursor SR watermark */
1508 I915_WRITE(DSPFW3, (cursor_sr << DSPFW_CURSOR_SR_SHIFT));
1509}
1510
1fa61106 1511static void i9xx_update_wm(struct drm_device *dev)
b445e3b0
ED		 1512{
1513 struct drm_i915_private *dev_priv = dev->dev_private;
1514 const struct intel_watermark_params *wm_info;
1515 uint32_t fwater_lo;
1516 uint32_t fwater_hi;
1517 int cwm, srwm = 1;
1518 int fifo_size;
1519 int planea_wm, planeb_wm;
1520 struct drm_crtc *crtc, *enabled = NULL;
1521
1522 if (IS_I945GM(dev))
1523 wm_info = &i945_wm_info;
1524 else if (!IS_GEN2(dev))
1525 wm_info = &i915_wm_info;
1526 else
1527 wm_info = &i855_wm_info;
1528
1529 fifo_size = dev_priv->display.get_fifo_size(dev, 0);
1530 crtc = intel_get_crtc_for_plane(dev, 0);
3490ea5d 1531 if (intel_crtc_active(crtc)) {
b9e0bda3
CW		 1532 int cpp = crtc->fb->bits_per_pixel / 8;
1533 if (IS_GEN2(dev))
1534 cpp = 4;
1535
b445e3b0 1536 planea_wm = intel_calculate_wm(crtc->mode.clock,
b9e0bda3 1537 wm_info, fifo_size, cpp,
b445e3b0
ED		 1538 latency_ns);
1539 enabled = crtc;
1540 } else
1541 planea_wm = fifo_size - wm_info->guard_size;
1542
1543 fifo_size = dev_priv->display.get_fifo_size(dev, 1);
1544 crtc = intel_get_crtc_for_plane(dev, 1);
3490ea5d 1545 if (intel_crtc_active(crtc)) {
b9e0bda3
CW		 1546 int cpp = crtc->fb->bits_per_pixel / 8;
1547 if (IS_GEN2(dev))
1548 cpp = 4;
1549
b445e3b0 1550 planeb_wm = intel_calculate_wm(crtc->mode.clock,
b9e0bda3 1551 wm_info, fifo_size, cpp,
b445e3b0
ED		 1552 latency_ns);
1553 if (enabled == NULL)
1554 enabled = crtc;
1555 else
1556 enabled = NULL;
1557 } else
1558 planeb_wm = fifo_size - wm_info->guard_size;
1559
1560 DRM_DEBUG_KMS("FIFO watermarks - A: %d, B: %d\n", planea_wm, planeb_wm);
1561
1562 /*
1563 * Overlay gets an aggressive default since video jitter is bad.
1564 */
1565 cwm = 2;
1566
1567 /* Play safe and disable self-refresh before adjusting watermarks. */
1568 if (IS_I945G(dev) || IS_I945GM(dev))
1569 I915_WRITE(FW_BLC_SELF, FW_BLC_SELF_EN_MASK | 0);
1570 else if (IS_I915GM(dev))
1571 I915_WRITE(INSTPM, I915_READ(INSTPM) & ~INSTPM_SELF_EN);
1572
1573 /* Calc sr entries for one plane configs */
1574 if (HAS_FW_BLC(dev) && enabled) {
1575 /* self-refresh has much higher latency */
1576 static const int sr_latency_ns = 6000;
1577 int clock = enabled->mode.clock;
1578 int htotal = enabled->mode.htotal;
1579 int hdisplay = enabled->mode.hdisplay;
1580 int pixel_size = enabled->fb->bits_per_pixel / 8;
1581 unsigned long line_time_us;
1582 int entries;
1583
1584 line_time_us = (htotal * 1000) / clock;
1585
1586 /* Use ns/us then divide to preserve precision */
1587 entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) *
1588 pixel_size * hdisplay;
1589 entries = DIV_ROUND_UP(entries, wm_info->cacheline_size);
1590 DRM_DEBUG_KMS("self-refresh entries: %d\n", entries);
1591 srwm = wm_info->fifo_size - entries;
1592 if (srwm < 0)
1593 srwm = 1;
1594
1595 if (IS_I945G(dev) || IS_I945GM(dev))
1596 I915_WRITE(FW_BLC_SELF,
1597 FW_BLC_SELF_FIFO_MASK | (srwm & 0xff));
1598 else if (IS_I915GM(dev))
1599 I915_WRITE(FW_BLC_SELF, srwm & 0x3f);
1600 }
1601
1602 DRM_DEBUG_KMS("Setting FIFO watermarks - A: %d, B: %d, C: %d, SR %d\n",
1603 planea_wm, planeb_wm, cwm, srwm);
1604
1605 fwater_lo = ((planeb_wm & 0x3f) << 16) | (planea_wm & 0x3f);
1606 fwater_hi = (cwm & 0x1f);
1607
1608 /* Set request length to 8 cachelines per fetch */
1609 fwater_lo = fwater_lo | (1 << 24) | (1 << 8);
1610 fwater_hi = fwater_hi | (1 << 8);
1611
1612 I915_WRITE(FW_BLC, fwater_lo);
1613 I915_WRITE(FW_BLC2, fwater_hi);
1614
1615 if (HAS_FW_BLC(dev)) {
1616 if (enabled) {
1617 if (IS_I945G(dev) || IS_I945GM(dev))
1618 I915_WRITE(FW_BLC_SELF,
1619 FW_BLC_SELF_EN_MASK | FW_BLC_SELF_EN);
1620 else if (IS_I915GM(dev))
1621 I915_WRITE(INSTPM, I915_READ(INSTPM) | INSTPM_SELF_EN);
1622 DRM_DEBUG_KMS("memory self refresh enabled\n");
1623 } else
1624 DRM_DEBUG_KMS("memory self refresh disabled\n");
1625 }
1626}
1627
1fa61106 1628static void i830_update_wm(struct drm_device *dev)
b445e3b0
ED		 1629{
1630 struct drm_i915_private *dev_priv = dev->dev_private;
1631 struct drm_crtc *crtc;
1632 uint32_t fwater_lo;
1633 int planea_wm;
1634
1635 crtc = single_enabled_crtc(dev);
1636 if (crtc == NULL)
1637 return;
1638
1639 planea_wm = intel_calculate_wm(crtc->mode.clock, &i830_wm_info,
1640 dev_priv->display.get_fifo_size(dev, 0),
b9e0bda3 1641 4, latency_ns);
b445e3b0
ED		 1642 fwater_lo = I915_READ(FW_BLC) & ~0xfff;
1643 fwater_lo |= (3<<8) | planea_wm;
1644
1645 DRM_DEBUG_KMS("Setting FIFO watermarks - A: %d\n", planea_wm);
1646
1647 I915_WRITE(FW_BLC, fwater_lo);
1648}
1649
1650#define ILK_LP0_PLANE_LATENCY 700
1651#define ILK_LP0_CURSOR_LATENCY 1300
1652
1653/*
1654 * Check the wm result.
1655 *
1656 * If any calculated watermark values is larger than the maximum value that
1657 * can be programmed into the associated watermark register, that watermark
1658 * must be disabled.
1659 */
1660static bool ironlake_check_srwm(struct drm_device *dev, int level,
1661 int fbc_wm, int display_wm, int cursor_wm,
1662 const struct intel_watermark_params *display,
1663 const struct intel_watermark_params *cursor)
1664{
1665 struct drm_i915_private *dev_priv = dev->dev_private;
1666
1667 DRM_DEBUG_KMS("watermark %d: display plane %d, fbc lines %d,"
1668 " cursor %d\n", level, display_wm, fbc_wm, cursor_wm);
1669
1670 if (fbc_wm > SNB_FBC_MAX_SRWM) {
1671 DRM_DEBUG_KMS("fbc watermark(%d) is too large(%d), disabling wm%d+\n",
1672 fbc_wm, SNB_FBC_MAX_SRWM, level);
1673
1674 /* fbc has it's own way to disable FBC WM */
1675 I915_WRITE(DISP_ARB_CTL,
1676 I915_READ(DISP_ARB_CTL) | DISP_FBC_WM_DIS);
1677 return false;
615aaa5f
VS		 1678 } else if (INTEL_INFO(dev)->gen >= 6) {
1679 /* enable FBC WM (except on ILK, where it must remain off) */
1680 I915_WRITE(DISP_ARB_CTL,
1681 I915_READ(DISP_ARB_CTL) & ~DISP_FBC_WM_DIS);
b445e3b0
ED		 1682 }
1683
1684 if (display_wm > display->max_wm) {
1685 DRM_DEBUG_KMS("display watermark(%d) is too large(%d), disabling wm%d+\n",
1686 display_wm, SNB_DISPLAY_MAX_SRWM, level);
1687 return false;
1688 }
1689
1690 if (cursor_wm > cursor->max_wm) {
1691 DRM_DEBUG_KMS("cursor watermark(%d) is too large(%d), disabling wm%d+\n",
1692 cursor_wm, SNB_CURSOR_MAX_SRWM, level);
1693 return false;
1694 }
1695
1696 if (!(fbc_wm || display_wm || cursor_wm)) {
1697 DRM_DEBUG_KMS("latency %d is 0, disabling wm%d+\n", level, level);
1698 return false;
1699 }
1700
1701 return true;
1702}
1703
1704/*
1705 * Compute watermark values of WM[1-3],
1706 */
1707static bool ironlake_compute_srwm(struct drm_device *dev, int level, int plane,
1708 int latency_ns,
1709 const struct intel_watermark_params *display,
1710 const struct intel_watermark_params *cursor,
1711 int *fbc_wm, int *display_wm, int *cursor_wm)
1712{
1713 struct drm_crtc *crtc;
1714 unsigned long line_time_us;
1715 int hdisplay, htotal, pixel_size, clock;
1716 int line_count, line_size;
1717 int small, large;
1718 int entries;
1719
1720 if (!latency_ns) {
1721 *fbc_wm = *display_wm = *cursor_wm = 0;
1722 return false;
1723 }
1724
1725 crtc = intel_get_crtc_for_plane(dev, plane);
1726 hdisplay = crtc->mode.hdisplay;
1727 htotal = crtc->mode.htotal;
1728 clock = crtc->mode.clock;
1729 pixel_size = crtc->fb->bits_per_pixel / 8;
1730
1731 line_time_us = (htotal * 1000) / clock;
1732 line_count = (latency_ns / line_time_us + 1000) / 1000;
1733 line_size = hdisplay * pixel_size;
1734
1735 /* Use the minimum of the small and large buffer method for primary */
1736 small = ((clock * pixel_size / 1000) * latency_ns) / 1000;
1737 large = line_count * line_size;
1738
1739 entries = DIV_ROUND_UP(min(small, large), display->cacheline_size);
1740 *display_wm = entries + display->guard_size;
1741
1742 /*
1743 * Spec says:
1744 * FBC WM = ((Final Primary WM * 64) / number of bytes per line) + 2
1745 */
1746 *fbc_wm = DIV_ROUND_UP(*display_wm * 64, line_size) + 2;
1747
1748 /* calculate the self-refresh watermark for display cursor */
1749 entries = line_count * pixel_size * 64;
1750 entries = DIV_ROUND_UP(entries, cursor->cacheline_size);
1751 *cursor_wm = entries + cursor->guard_size;
1752
1753 return ironlake_check_srwm(dev, level,
1754 *fbc_wm, *display_wm, *cursor_wm,
1755 display, cursor);
1756}
1757
1fa61106 1758static void ironlake_update_wm(struct drm_device *dev)
b445e3b0
ED		 1759{
1760 struct drm_i915_private *dev_priv = dev->dev_private;
1761 int fbc_wm, plane_wm, cursor_wm;
1762 unsigned int enabled;
1763
1764 enabled = 0;
1765 if (g4x_compute_wm0(dev, 0,
1766 &ironlake_display_wm_info,
1767 ILK_LP0_PLANE_LATENCY,
1768 &ironlake_cursor_wm_info,
1769 ILK_LP0_CURSOR_LATENCY,
1770 &plane_wm, &cursor_wm)) {
1771 I915_WRITE(WM0_PIPEA_ILK,
1772 (plane_wm << WM0_PIPE_PLANE_SHIFT) | cursor_wm);
1773 DRM_DEBUG_KMS("FIFO watermarks For pipe A -"
1774 " plane %d, " "cursor: %d\n",
1775 plane_wm, cursor_wm);
1776 enabled |= 1;
1777 }
1778
1779 if (g4x_compute_wm0(dev, 1,
1780 &ironlake_display_wm_info,
1781 ILK_LP0_PLANE_LATENCY,
1782 &ironlake_cursor_wm_info,
1783 ILK_LP0_CURSOR_LATENCY,
1784 &plane_wm, &cursor_wm)) {
1785 I915_WRITE(WM0_PIPEB_ILK,
1786 (plane_wm << WM0_PIPE_PLANE_SHIFT) | cursor_wm);
1787 DRM_DEBUG_KMS("FIFO watermarks For pipe B -"
1788 " plane %d, cursor: %d\n",
1789 plane_wm, cursor_wm);
1790 enabled |= 2;
1791 }
1792
1793 /*
1794 * Calculate and update the self-refresh watermark only when one
1795 * display plane is used.
1796 */
1797 I915_WRITE(WM3_LP_ILK, 0);
1798 I915_WRITE(WM2_LP_ILK, 0);
1799 I915_WRITE(WM1_LP_ILK, 0);
1800
1801 if (!single_plane_enabled(enabled))
1802 return;
1803 enabled = ffs(enabled) - 1;
1804
1805 /* WM1 */
1806 if (!ironlake_compute_srwm(dev, 1, enabled,
1807 ILK_READ_WM1_LATENCY() * 500,
1808 &ironlake_display_srwm_info,
1809 &ironlake_cursor_srwm_info,
1810 &fbc_wm, &plane_wm, &cursor_wm))
1811 return;
1812
1813 I915_WRITE(WM1_LP_ILK,
1814 WM1_LP_SR_EN |
1815 (ILK_READ_WM1_LATENCY() << WM1_LP_LATENCY_SHIFT) |
1816 (fbc_wm << WM1_LP_FBC_SHIFT) |
1817 (plane_wm << WM1_LP_SR_SHIFT) |
1818 cursor_wm);
1819
1820 /* WM2 */
1821 if (!ironlake_compute_srwm(dev, 2, enabled,
1822 ILK_READ_WM2_LATENCY() * 500,
1823 &ironlake_display_srwm_info,
1824 &ironlake_cursor_srwm_info,
1825 &fbc_wm, &plane_wm, &cursor_wm))
1826 return;
1827
1828 I915_WRITE(WM2_LP_ILK,
1829 WM2_LP_EN |
1830 (ILK_READ_WM2_LATENCY() << WM1_LP_LATENCY_SHIFT) |
1831 (fbc_wm << WM1_LP_FBC_SHIFT) |
1832 (plane_wm << WM1_LP_SR_SHIFT) |
1833 cursor_wm);
1834
1835 /*
1836 * WM3 is unsupported on ILK, probably because we don't have latency
1837 * data for that power state
1838 */
1839}
1840
1fa61106 1841static void sandybridge_update_wm(struct drm_device *dev)
b445e3b0
ED		 1842{
1843 struct drm_i915_private *dev_priv = dev->dev_private;
1844 int latency = SNB_READ_WM0_LATENCY() * 100; /* In unit 0.1us */
1845 u32 val;
1846 int fbc_wm, plane_wm, cursor_wm;
1847 unsigned int enabled;
1848
1849 enabled = 0;
1850 if (g4x_compute_wm0(dev, 0,
1851 &sandybridge_display_wm_info, latency,
1852 &sandybridge_cursor_wm_info, latency,
1853 &plane_wm, &cursor_wm)) {
1854 val = I915_READ(WM0_PIPEA_ILK);
1855 val &= ~(WM0_PIPE_PLANE_MASK | WM0_PIPE_CURSOR_MASK);
1856 I915_WRITE(WM0_PIPEA_ILK, val |
1857 ((plane_wm << WM0_PIPE_PLANE_SHIFT) | cursor_wm));
1858 DRM_DEBUG_KMS("FIFO watermarks For pipe A -"
1859 " plane %d, " "cursor: %d\n",
1860 plane_wm, cursor_wm);
1861 enabled |= 1;
1862 }
1863
1864 if (g4x_compute_wm0(dev, 1,
1865 &sandybridge_display_wm_info, latency,
1866 &sandybridge_cursor_wm_info, latency,
1867 &plane_wm, &cursor_wm)) {
1868 val = I915_READ(WM0_PIPEB_ILK);
1869 val &= ~(WM0_PIPE_PLANE_MASK | WM0_PIPE_CURSOR_MASK);
1870 I915_WRITE(WM0_PIPEB_ILK, val |
1871 ((plane_wm << WM0_PIPE_PLANE_SHIFT) | cursor_wm));
1872 DRM_DEBUG_KMS("FIFO watermarks For pipe B -"
1873 " plane %d, cursor: %d\n",
1874 plane_wm, cursor_wm);
1875 enabled |= 2;
1876 }
1877
c43d0188
CW		 1878 /*
1879 * Calculate and update the self-refresh watermark only when one
1880 * display plane is used.
1881 *
1882 * SNB support 3 levels of watermark.
1883 *
1884 * WM1/WM2/WM2 watermarks have to be enabled in the ascending order,
1885 * and disabled in the descending order
1886 *
1887 */
1888 I915_WRITE(WM3_LP_ILK, 0);
1889 I915_WRITE(WM2_LP_ILK, 0);
1890 I915_WRITE(WM1_LP_ILK, 0);
1891
1892 if (!single_plane_enabled(enabled) ||
1893 dev_priv->sprite_scaling_enabled)
1894 return;
1895 enabled = ffs(enabled) - 1;
1896
1897 /* WM1 */
1898 if (!ironlake_compute_srwm(dev, 1, enabled,
1899 SNB_READ_WM1_LATENCY() * 500,
1900 &sandybridge_display_srwm_info,
1901 &sandybridge_cursor_srwm_info,
1902 &fbc_wm, &plane_wm, &cursor_wm))
1903 return;
1904
1905 I915_WRITE(WM1_LP_ILK,
1906 WM1_LP_SR_EN |
1907 (SNB_READ_WM1_LATENCY() << WM1_LP_LATENCY_SHIFT) |
1908 (fbc_wm << WM1_LP_FBC_SHIFT) |
1909 (plane_wm << WM1_LP_SR_SHIFT) |
1910 cursor_wm);
1911
1912 /* WM2 */
1913 if (!ironlake_compute_srwm(dev, 2, enabled,
1914 SNB_READ_WM2_LATENCY() * 500,
1915 &sandybridge_display_srwm_info,
1916 &sandybridge_cursor_srwm_info,
1917 &fbc_wm, &plane_wm, &cursor_wm))
1918 return;
1919
1920 I915_WRITE(WM2_LP_ILK,
1921 WM2_LP_EN |
1922 (SNB_READ_WM2_LATENCY() << WM1_LP_LATENCY_SHIFT) |
1923 (fbc_wm << WM1_LP_FBC_SHIFT) |
1924 (plane_wm << WM1_LP_SR_SHIFT) |
1925 cursor_wm);
1926
1927 /* WM3 */
1928 if (!ironlake_compute_srwm(dev, 3, enabled,
1929 SNB_READ_WM3_LATENCY() * 500,
1930 &sandybridge_display_srwm_info,
1931 &sandybridge_cursor_srwm_info,
1932 &fbc_wm, &plane_wm, &cursor_wm))
1933 return;
1934
1935 I915_WRITE(WM3_LP_ILK,
1936 WM3_LP_EN |
1937 (SNB_READ_WM3_LATENCY() << WM1_LP_LATENCY_SHIFT) |
1938 (fbc_wm << WM1_LP_FBC_SHIFT) |
1939 (plane_wm << WM1_LP_SR_SHIFT) |
1940 cursor_wm);
1941}
1942
1943static void ivybridge_update_wm(struct drm_device *dev)
1944{
1945 struct drm_i915_private *dev_priv = dev->dev_private;
1946 int latency = SNB_READ_WM0_LATENCY() * 100; /* In unit 0.1us */
1947 u32 val;
1948 int fbc_wm, plane_wm, cursor_wm;
1949 int ignore_fbc_wm, ignore_plane_wm, ignore_cursor_wm;
1950 unsigned int enabled;
1951
1952 enabled = 0;
1953 if (g4x_compute_wm0(dev, 0,
1954 &sandybridge_display_wm_info, latency,
1955 &sandybridge_cursor_wm_info, latency,
1956 &plane_wm, &cursor_wm)) {
1957 val = I915_READ(WM0_PIPEA_ILK);
1958 val &= ~(WM0_PIPE_PLANE_MASK | WM0_PIPE_CURSOR_MASK);
1959 I915_WRITE(WM0_PIPEA_ILK, val |
1960 ((plane_wm << WM0_PIPE_PLANE_SHIFT) | cursor_wm));
1961 DRM_DEBUG_KMS("FIFO watermarks For pipe A -"
1962 " plane %d, " "cursor: %d\n",
1963 plane_wm, cursor_wm);
1964 enabled |= 1;
1965 }
1966
1967 if (g4x_compute_wm0(dev, 1,
1968 &sandybridge_display_wm_info, latency,
1969 &sandybridge_cursor_wm_info, latency,
1970 &plane_wm, &cursor_wm)) {
1971 val = I915_READ(WM0_PIPEB_ILK);
1972 val &= ~(WM0_PIPE_PLANE_MASK | WM0_PIPE_CURSOR_MASK);
1973 I915_WRITE(WM0_PIPEB_ILK, val |
1974 ((plane_wm << WM0_PIPE_PLANE_SHIFT) | cursor_wm));
1975 DRM_DEBUG_KMS("FIFO watermarks For pipe B -"
1976 " plane %d, cursor: %d\n",
1977 plane_wm, cursor_wm);
1978 enabled |= 2;
1979 }
1980
1981 if (g4x_compute_wm0(dev, 2,
b445e3b0
ED		 1982 &sandybridge_display_wm_info, latency,
1983 &sandybridge_cursor_wm_info, latency,
1984 &plane_wm, &cursor_wm)) {
1985 val = I915_READ(WM0_PIPEC_IVB);
1986 val &= ~(WM0_PIPE_PLANE_MASK | WM0_PIPE_CURSOR_MASK);
1987 I915_WRITE(WM0_PIPEC_IVB, val |
1988 ((plane_wm << WM0_PIPE_PLANE_SHIFT) | cursor_wm));
1989 DRM_DEBUG_KMS("FIFO watermarks For pipe C -"
1990 " plane %d, cursor: %d\n",
1991 plane_wm, cursor_wm);
1992 enabled |= 3;
1993 }
1994
1995 /*
1996 * Calculate and update the self-refresh watermark only when one
1997 * display plane is used.
1998 *
1999 * SNB support 3 levels of watermark.
2000 *
2001 * WM1/WM2/WM2 watermarks have to be enabled in the ascending order,
2002 * and disabled in the descending order
2003 *
2004 */
2005 I915_WRITE(WM3_LP_ILK, 0);
2006 I915_WRITE(WM2_LP_ILK, 0);
2007 I915_WRITE(WM1_LP_ILK, 0);
2008
2009 if (!single_plane_enabled(enabled) ||
2010 dev_priv->sprite_scaling_enabled)
2011 return;
2012 enabled = ffs(enabled) - 1;
2013
2014 /* WM1 */
2015 if (!ironlake_compute_srwm(dev, 1, enabled,
2016 SNB_READ_WM1_LATENCY() * 500,
2017 &sandybridge_display_srwm_info,
2018 &sandybridge_cursor_srwm_info,
2019 &fbc_wm, &plane_wm, &cursor_wm))
2020 return;
2021
2022 I915_WRITE(WM1_LP_ILK,
2023 WM1_LP_SR_EN |
2024 (SNB_READ_WM1_LATENCY() << WM1_LP_LATENCY_SHIFT) |
2025 (fbc_wm << WM1_LP_FBC_SHIFT) |
2026 (plane_wm << WM1_LP_SR_SHIFT) |
2027 cursor_wm);
2028
2029 /* WM2 */
2030 if (!ironlake_compute_srwm(dev, 2, enabled,
2031 SNB_READ_WM2_LATENCY() * 500,
2032 &sandybridge_display_srwm_info,
2033 &sandybridge_cursor_srwm_info,
2034 &fbc_wm, &plane_wm, &cursor_wm))
2035 return;
2036
2037 I915_WRITE(WM2_LP_ILK,
2038 WM2_LP_EN |
2039 (SNB_READ_WM2_LATENCY() << WM1_LP_LATENCY_SHIFT) |
2040 (fbc_wm << WM1_LP_FBC_SHIFT) |
2041 (plane_wm << WM1_LP_SR_SHIFT) |
2042 cursor_wm);
2043
c43d0188 2044 /* WM3, note we have to correct the cursor latency */
b445e3b0
ED		 2045 if (!ironlake_compute_srwm(dev, 3, enabled,
2046 SNB_READ_WM3_LATENCY() * 500,
2047 &sandybridge_display_srwm_info,
2048 &sandybridge_cursor_srwm_info,
c43d0188
CW		 2049 &fbc_wm, &plane_wm, &ignore_cursor_wm) ||
2050 !ironlake_compute_srwm(dev, 3, enabled,
2051 2 * SNB_READ_WM3_LATENCY() * 500,
2052 &sandybridge_display_srwm_info,
2053 &sandybridge_cursor_srwm_info,
2054 &ignore_fbc_wm, &ignore_plane_wm, &cursor_wm))
b445e3b0
ED		 2055 return;
2056
2057 I915_WRITE(WM3_LP_ILK,
2058 WM3_LP_EN |
2059 (SNB_READ_WM3_LATENCY() << WM1_LP_LATENCY_SHIFT) |
2060 (fbc_wm << WM1_LP_FBC_SHIFT) |
2061 (plane_wm << WM1_LP_SR_SHIFT) |
2062 cursor_wm);
2063}
2064
1f8eeabf
ED		 2065static void
2066haswell_update_linetime_wm(struct drm_device *dev, int pipe,
2067 struct drm_display_mode *mode)
2068{
2069 struct drm_i915_private *dev_priv = dev->dev_private;
2070 u32 temp;
2071
2072 temp = I915_READ(PIPE_WM_LINETIME(pipe));
2073 temp &= ~PIPE_WM_LINETIME_MASK;
2074
2075 /* The WM are computed with base on how long it takes to fill a single
2076 * row at the given clock rate, multiplied by 8.
2077 * */
2078 temp |= PIPE_WM_LINETIME_TIME(
2079 ((mode->crtc_hdisplay * 1000) / mode->clock) * 8);
2080
2081 /* IPS watermarks are only used by pipe A, and are ignored by
2082 * pipes B and C. They are calculated similarly to the common
2083 * linetime values, except that we are using CD clock frequency
2084 * in MHz instead of pixel rate for the division.
2085 *
2086 * This is a placeholder for the IPS watermark calculation code.
2087 */
2088
2089 I915_WRITE(PIPE_WM_LINETIME(pipe), temp);
2090}
2091
b445e3b0
ED		 2092static bool
2093sandybridge_compute_sprite_wm(struct drm_device *dev, int plane,
2094 uint32_t sprite_width, int pixel_size,
2095 const struct intel_watermark_params *display,
2096 int display_latency_ns, int *sprite_wm)
2097{
2098 struct drm_crtc *crtc;
2099 int clock;
2100 int entries, tlb_miss;
2101
2102 crtc = intel_get_crtc_for_plane(dev, plane);
3490ea5d 2103 if (!intel_crtc_active(crtc)) {
b445e3b0
ED		 2104 *sprite_wm = display->guard_size;
2105 return false;
2106 }
2107
2108 clock = crtc->mode.clock;
2109
2110 /* Use the small buffer method to calculate the sprite watermark */
2111 entries = ((clock * pixel_size / 1000) * display_latency_ns) / 1000;
2112 tlb_miss = display->fifo_size*display->cacheline_size -
2113 sprite_width * 8;
2114 if (tlb_miss > 0)
2115 entries += tlb_miss;
2116 entries = DIV_ROUND_UP(entries, display->cacheline_size);
2117 *sprite_wm = entries + display->guard_size;
2118 if (*sprite_wm > (int)display->max_wm)
2119 *sprite_wm = display->max_wm;
2120
2121 return true;
2122}
2123
2124static bool
2125sandybridge_compute_sprite_srwm(struct drm_device *dev, int plane,
2126 uint32_t sprite_width, int pixel_size,
2127 const struct intel_watermark_params *display,
2128 int latency_ns, int *sprite_wm)
2129{
2130 struct drm_crtc *crtc;
2131 unsigned long line_time_us;
2132 int clock;
2133 int line_count, line_size;
2134 int small, large;
2135 int entries;
2136
2137 if (!latency_ns) {
2138 *sprite_wm = 0;
2139 return false;
2140 }
2141
2142 crtc = intel_get_crtc_for_plane(dev, plane);
2143 clock = crtc->mode.clock;
2144 if (!clock) {
2145 *sprite_wm = 0;
2146 return false;
2147 }
2148
2149 line_time_us = (sprite_width * 1000) / clock;
2150 if (!line_time_us) {
2151 *sprite_wm = 0;
2152 return false;
2153 }
2154
2155 line_count = (latency_ns / line_time_us + 1000) / 1000;
2156 line_size = sprite_width * pixel_size;
2157
2158 /* Use the minimum of the small and large buffer method for primary */
2159 small = ((clock * pixel_size / 1000) * latency_ns) / 1000;
2160 large = line_count * line_size;
2161
2162 entries = DIV_ROUND_UP(min(small, large), display->cacheline_size);
2163 *sprite_wm = entries + display->guard_size;
2164
2165 return *sprite_wm > 0x3ff ? false : true;
2166}
2167
1fa61106 2168static void sandybridge_update_sprite_wm(struct drm_device *dev, int pipe,
b445e3b0
ED		 2169 uint32_t sprite_width, int pixel_size)
2170{
2171 struct drm_i915_private *dev_priv = dev->dev_private;
2172 int latency = SNB_READ_WM0_LATENCY() * 100; /* In unit 0.1us */
2173 u32 val;
2174 int sprite_wm, reg;
2175 int ret;
2176
2177 switch (pipe) {
2178 case 0:
2179 reg = WM0_PIPEA_ILK;
2180 break;
2181 case 1:
2182 reg = WM0_PIPEB_ILK;
2183 break;
2184 case 2:
2185 reg = WM0_PIPEC_IVB;
2186 break;
2187 default:
2188 return; /* bad pipe */
2189 }
2190
2191 ret = sandybridge_compute_sprite_wm(dev, pipe, sprite_width, pixel_size,
2192 &sandybridge_display_wm_info,
2193 latency, &sprite_wm);
2194 if (!ret) {
84f44ce7
VS		 2195 DRM_DEBUG_KMS("failed to compute sprite wm for pipe %c\n",
2196 pipe_name(pipe));
b445e3b0
ED		 2197 return;
2198 }
2199
2200 val = I915_READ(reg);
2201 val &= ~WM0_PIPE_SPRITE_MASK;
2202 I915_WRITE(reg, val | (sprite_wm << WM0_PIPE_SPRITE_SHIFT));
84f44ce7 2203 DRM_DEBUG_KMS("sprite watermarks For pipe %c - %d\n", pipe_name(pipe), sprite_wm);
b445e3b0
ED		 2204
2205
2206 ret = sandybridge_compute_sprite_srwm(dev, pipe, sprite_width,
2207 pixel_size,
2208 &sandybridge_display_srwm_info,
2209 SNB_READ_WM1_LATENCY() * 500,
2210 &sprite_wm);
2211 if (!ret) {
84f44ce7
VS		 2212 DRM_DEBUG_KMS("failed to compute sprite lp1 wm on pipe %c\n",
2213 pipe_name(pipe));
b445e3b0
ED		 2214 return;
2215 }
2216 I915_WRITE(WM1S_LP_ILK, sprite_wm);
2217
2218 /* Only IVB has two more LP watermarks for sprite */
2219 if (!IS_IVYBRIDGE(dev))
2220 return;
2221
2222 ret = sandybridge_compute_sprite_srwm(dev, pipe, sprite_width,
2223 pixel_size,
2224 &sandybridge_display_srwm_info,
2225 SNB_READ_WM2_LATENCY() * 500,
2226 &sprite_wm);
2227 if (!ret) {
84f44ce7
VS		 2228 DRM_DEBUG_KMS("failed to compute sprite lp2 wm on pipe %c\n",
2229 pipe_name(pipe));
b445e3b0
ED		 2230 return;
2231 }
2232 I915_WRITE(WM2S_LP_IVB, sprite_wm);
2233
2234 ret = sandybridge_compute_sprite_srwm(dev, pipe, sprite_width,
2235 pixel_size,
2236 &sandybridge_display_srwm_info,
2237 SNB_READ_WM3_LATENCY() * 500,
2238 &sprite_wm);
2239 if (!ret) {
84f44ce7
VS		 2240 DRM_DEBUG_KMS("failed to compute sprite lp3 wm on pipe %c\n",
2241 pipe_name(pipe));
b445e3b0
ED		 2242 return;
2243 }
2244 I915_WRITE(WM3S_LP_IVB, sprite_wm);
2245}
2246
2247/**
2248 * intel_update_watermarks - update FIFO watermark values based on current modes
2249 *
2250 * Calculate watermark values for the various WM regs based on current mode
2251 * and plane configuration.
2252 *
2253 * There are several cases to deal with here:
2254 * - normal (i.e. non-self-refresh)
2255 * - self-refresh (SR) mode
2256 * - lines are large relative to FIFO size (buffer can hold up to 2)
2257 * - lines are small relative to FIFO size (buffer can hold more than 2
2258 * lines), so need to account for TLB latency
2259 *
2260 * The normal calculation is:
2261 * watermark = dotclock * bytes per pixel * latency
2262 * where latency is platform & configuration dependent (we assume pessimal
2263 * values here).
2264 *
2265 * The SR calculation is:
2266 * watermark = (trunc(latency/line time)+1) * surface width *
2267 * bytes per pixel
2268 * where
2269 * line time = htotal / dotclock
2270 * surface width = hdisplay for normal plane and 64 for cursor
2271 * and latency is assumed to be high, as above.
2272 *
2273 * The final value programmed to the register should always be rounded up,
2274 * and include an extra 2 entries to account for clock crossings.
2275 *
2276 * We don't use the sprite, so we can ignore that. And on Crestline we have
2277 * to set the non-SR watermarks to 8.
2278 */
2279void intel_update_watermarks(struct drm_device *dev)
2280{
2281 struct drm_i915_private *dev_priv = dev->dev_private;
2282
2283 if (dev_priv->display.update_wm)
2284 dev_priv->display.update_wm(dev);
2285}
2286
1f8eeabf
ED		 2287void intel_update_linetime_watermarks(struct drm_device *dev,
2288 int pipe, struct drm_display_mode *mode)
2289{
2290 struct drm_i915_private *dev_priv = dev->dev_private;
2291
2292 if (dev_priv->display.update_linetime_wm)
2293 dev_priv->display.update_linetime_wm(dev, pipe, mode);
2294}
2295
b445e3b0
ED		 2296void intel_update_sprite_watermarks(struct drm_device *dev, int pipe,
2297 uint32_t sprite_width, int pixel_size)
2298{
2299 struct drm_i915_private *dev_priv = dev->dev_private;
2300
2301 if (dev_priv->display.update_sprite_wm)
2302 dev_priv->display.update_sprite_wm(dev, pipe, sprite_width,
2303 pixel_size);
2304}
2305
2b4e57bd
ED		 2306static struct drm_i915_gem_object *
2307intel_alloc_context_page(struct drm_device *dev)
2308{
2309 struct drm_i915_gem_object *ctx;
2310 int ret;
2311
2312 WARN_ON(!mutex_is_locked(&dev->struct_mutex));
2313
2314 ctx = i915_gem_alloc_object(dev, 4096);
2315 if (!ctx) {
2316 DRM_DEBUG("failed to alloc power context, RC6 disabled\n");
2317 return NULL;
2318 }
2319
86a1ee26 2320 ret = i915_gem_object_pin(ctx, 4096, true, false);
2b4e57bd
ED		 2321 if (ret) {
2322 DRM_ERROR("failed to pin power context: %d\n", ret);
2323 goto err_unref;
2324 }
2325
2326 ret = i915_gem_object_set_to_gtt_domain(ctx, 1);
2327 if (ret) {
2328 DRM_ERROR("failed to set-domain on power context: %d\n", ret);
2329 goto err_unpin;
2330 }
2331
2332 return ctx;
2333
2334err_unpin:
2335 i915_gem_object_unpin(ctx);
2336err_unref:
2337 drm_gem_object_unreference(&ctx->base);
2b4e57bd
ED		 2338 return NULL;
2339}
2340
9270388e
DV		 2341/**
2342 * Lock protecting IPS related data structures
9270388e
DV		 2343 */
2344DEFINE_SPINLOCK(mchdev_lock);
2345
2346/* Global for IPS driver to get at the current i915 device. Protected by
2347 * mchdev_lock. */
2348static struct drm_i915_private *i915_mch_dev;
2349
2b4e57bd
ED		 2350bool ironlake_set_drps(struct drm_device *dev, u8 val)
2351{
2352 struct drm_i915_private *dev_priv = dev->dev_private;
2353 u16 rgvswctl;
2354
9270388e
DV		 2355 assert_spin_locked(&mchdev_lock);
2356
2b4e57bd
ED		 2357 rgvswctl = I915_READ16(MEMSWCTL);
2358 if (rgvswctl & MEMCTL_CMD_STS) {
2359 DRM_DEBUG("gpu busy, RCS change rejected\n");
2360 return false; /* still busy with another command */
2361 }
2362
2363 rgvswctl = (MEMCTL_CMD_CHFREQ << MEMCTL_CMD_SHIFT) |
2364 (val << MEMCTL_FREQ_SHIFT) | MEMCTL_SFCAVM;
2365 I915_WRITE16(MEMSWCTL, rgvswctl);
2366 POSTING_READ16(MEMSWCTL);
2367
2368 rgvswctl |= MEMCTL_CMD_STS;
2369 I915_WRITE16(MEMSWCTL, rgvswctl);
2370
2371 return true;
2372}
2373
8090c6b9 2374static void ironlake_enable_drps(struct drm_device *dev)
2b4e57bd
ED		 2375{
2376 struct drm_i915_private *dev_priv = dev->dev_private;
2377 u32 rgvmodectl = I915_READ(MEMMODECTL);
2378 u8 fmax, fmin, fstart, vstart;
2379
9270388e
DV		 2380 spin_lock_irq(&mchdev_lock);
2381
2b4e57bd
ED		 2382 /* Enable temp reporting */
2383 I915_WRITE16(PMMISC, I915_READ(PMMISC) | MCPPCE_EN);
2384 I915_WRITE16(TSC1, I915_READ(TSC1) | TSE);
2385
2386 /* 100ms RC evaluation intervals */
2387 I915_WRITE(RCUPEI, 100000);
2388 I915_WRITE(RCDNEI, 100000);
2389
2390 /* Set max/min thresholds to 90ms and 80ms respectively */
2391 I915_WRITE(RCBMAXAVG, 90000);
2392 I915_WRITE(RCBMINAVG, 80000);
2393
2394 I915_WRITE(MEMIHYST, 1);
2395
2396 /* Set up min, max, and cur for interrupt handling */
2397 fmax = (rgvmodectl & MEMMODE_FMAX_MASK) >> MEMMODE_FMAX_SHIFT;
2398 fmin = (rgvmodectl & MEMMODE_FMIN_MASK);
2399 fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >>
2400 MEMMODE_FSTART_SHIFT;
2401
2402 vstart = (I915_READ(PXVFREQ_BASE + (fstart * 4)) & PXVFREQ_PX_MASK) >>
2403 PXVFREQ_PX_SHIFT;
2404
20e4d407
DV		 2405 dev_priv->ips.fmax = fmax; /* IPS callback will increase this */
2406 dev_priv->ips.fstart = fstart;
2b4e57bd 2407
20e4d407
DV		 2408 dev_priv->ips.max_delay = fstart;
2409 dev_priv->ips.min_delay = fmin;
2410 dev_priv->ips.cur_delay = fstart;
2b4e57bd
ED		 2411
2412 DRM_DEBUG_DRIVER("fmax: %d, fmin: %d, fstart: %d\n",
2413 fmax, fmin, fstart);
2414
2415 I915_WRITE(MEMINTREN, MEMINT_CX_SUPR_EN | MEMINT_EVAL_CHG_EN);
2416
2417 /*
2418 * Interrupts will be enabled in ironlake_irq_postinstall
2419 */
2420
2421 I915_WRITE(VIDSTART, vstart);
2422 POSTING_READ(VIDSTART);
2423
2424 rgvmodectl |= MEMMODE_SWMODE_EN;
2425 I915_WRITE(MEMMODECTL, rgvmodectl);
2426
9270388e 2427 if (wait_for_atomic((I915_READ(MEMSWCTL) & MEMCTL_CMD_STS) == 0, 10))
2b4e57bd 2428 DRM_ERROR("stuck trying to change perf mode\n");
9270388e 2429 mdelay(1);
2b4e57bd
ED		 2430
2431 ironlake_set_drps(dev, fstart);
2432
20e4d407 2433 dev_priv->ips.last_count1 = I915_READ(0x112e4) + I915_READ(0x112e8) +
2b4e57bd 2434 I915_READ(0x112e0);
20e4d407
DV		 2435 dev_priv->ips.last_time1 = jiffies_to_msecs(jiffies);
2436 dev_priv->ips.last_count2 = I915_READ(0x112f4);
2437 getrawmonotonic(&dev_priv->ips.last_time2);
9270388e
DV		 2438
2439 spin_unlock_irq(&mchdev_lock);
2b4e57bd
ED		 2440}
2441
8090c6b9 2442static void ironlake_disable_drps(struct drm_device *dev)
2b4e57bd
ED		 2443{
2444 struct drm_i915_private *dev_priv = dev->dev_private;
9270388e
DV		 2445 u16 rgvswctl;
2446
2447 spin_lock_irq(&mchdev_lock);
2448
2449 rgvswctl = I915_READ16(MEMSWCTL);
2b4e57bd
ED		 2450
2451 /* Ack interrupts, disable EFC interrupt */
2452 I915_WRITE(MEMINTREN, I915_READ(MEMINTREN) & ~MEMINT_EVAL_CHG_EN);
2453 I915_WRITE(MEMINTRSTS, MEMINT_EVAL_CHG);
2454 I915_WRITE(DEIER, I915_READ(DEIER) & ~DE_PCU_EVENT);
2455 I915_WRITE(DEIIR, DE_PCU_EVENT);
2456 I915_WRITE(DEIMR, I915_READ(DEIMR) | DE_PCU_EVENT);
2457
2458 /* Go back to the starting frequency */
20e4d407 2459 ironlake_set_drps(dev, dev_priv->ips.fstart);
9270388e 2460 mdelay(1);
2b4e57bd
ED		 2461 rgvswctl |= MEMCTL_CMD_STS;
2462 I915_WRITE(MEMSWCTL, rgvswctl);
9270388e 2463 mdelay(1);
2b4e57bd 2464
9270388e 2465 spin_unlock_irq(&mchdev_lock);
2b4e57bd
ED		 2466}
2467
acbe9475
DV		 2468/* There's a funny hw issue where the hw returns all 0 when reading from
2469 * GEN6_RP_INTERRUPT_LIMITS. Hence we always need to compute the desired value
2470 * ourselves, instead of doing a rmw cycle (which might result in us clearing
2471 * all limits and the gpu stuck at whatever frequency it is at atm).
2472 */
65bccb5c 2473static u32 gen6_rps_limits(struct drm_i915_private *dev_priv, u8 *val)
2b4e57bd 2474{
7b9e0ae6 2475 u32 limits;
2b4e57bd 2476
7b9e0ae6 2477 limits = 0;
c6a828d3
DV		 2478
2479 if (*val >= dev_priv->rps.max_delay)
2480 *val = dev_priv->rps.max_delay;
2481 limits |= dev_priv->rps.max_delay << 24;
20b46e59
DV		 2482
2483 /* Only set the down limit when we've reached the lowest level to avoid
2484 * getting more interrupts, otherwise leave this clear. This prevents a
2485 * race in the hw when coming out of rc6: There's a tiny window where
2486 * the hw runs at the minimal clock before selecting the desired
2487 * frequency, if the down threshold expires in that window we will not
2488 * receive a down interrupt. */
c6a828d3
DV		 2489 if (*val <= dev_priv->rps.min_delay) {
2490 *val = dev_priv->rps.min_delay;
2491 limits |= dev_priv->rps.min_delay << 16;
20b46e59
DV		 2492 }
2493
2494 return limits;
2495}
2496
2497void gen6_set_rps(struct drm_device *dev, u8 val)
2498{
2499 struct drm_i915_private *dev_priv = dev->dev_private;
65bccb5c 2500 u32 limits = gen6_rps_limits(dev_priv, &val);
7b9e0ae6 2501
4fc688ce 2502 WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
79249636
BW		 2503 WARN_ON(val > dev_priv->rps.max_delay);
2504 WARN_ON(val < dev_priv->rps.min_delay);
004777cb 2505
c6a828d3 2506 if (val == dev_priv->rps.cur_delay)
7b9e0ae6
CW		 2507 return;
2508
92bd1bf0
RV		 2509 if (IS_HASWELL(dev))
2510 I915_WRITE(GEN6_RPNSWREQ,
2511 HSW_FREQUENCY(val));
2512 else
2513 I915_WRITE(GEN6_RPNSWREQ,
2514 GEN6_FREQUENCY(val) |
2515 GEN6_OFFSET(0) |
2516 GEN6_AGGRESSIVE_TURBO);
7b9e0ae6
CW		 2517
2518 /* Make sure we continue to get interrupts
2519 * until we hit the minimum or maximum frequencies.
2520 */
2521 I915_WRITE(GEN6_RP_INTERRUPT_LIMITS, limits);
2522
d5570a72
BW		 2523 POSTING_READ(GEN6_RPNSWREQ);
2524
c6a828d3 2525 dev_priv->rps.cur_delay = val;
be2cde9a
DV		 2526
2527 trace_intel_gpu_freq_change(val * 50);
2b4e57bd
ED		 2528}
2529
0a073b84
JB		 2530void valleyview_set_rps(struct drm_device *dev, u8 val)
2531{
2532 struct drm_i915_private *dev_priv = dev->dev_private;
2533 unsigned long timeout = jiffies + msecs_to_jiffies(10);
2534 u32 limits = gen6_rps_limits(dev_priv, &val);
2535 u32 pval;
2536
2537 WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
2538 WARN_ON(val > dev_priv->rps.max_delay);
2539 WARN_ON(val < dev_priv->rps.min_delay);
2540
2541 DRM_DEBUG_DRIVER("gpu freq request from %d to %d\n",
2542 vlv_gpu_freq(dev_priv->mem_freq,
2543 dev_priv->rps.cur_delay),
2544 vlv_gpu_freq(dev_priv->mem_freq, val));
2545
2546 if (val == dev_priv->rps.cur_delay)
2547 return;
2548
2549 valleyview_punit_write(dev_priv, PUNIT_REG_GPU_FREQ_REQ, val);
2550
2551 do {
2552 valleyview_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS, &pval);
2553 if (time_after(jiffies, timeout)) {
2554 DRM_DEBUG_DRIVER("timed out waiting for Punit\n");
2555 break;
2556 }
2557 udelay(10);
2558 } while (pval & 1);
2559
2560 valleyview_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS, &pval);
2561 if ((pval >> 8) != val)
2562 DRM_DEBUG_DRIVER("punit overrode freq: %d requested, but got %d\n",
2563 val, pval >> 8);
2564
2565 /* Make sure we continue to get interrupts
2566 * until we hit the minimum or maximum frequencies.
2567 */
2568 I915_WRITE(GEN6_RP_INTERRUPT_LIMITS, limits);
2569
2570 dev_priv->rps.cur_delay = pval >> 8;
2571
2572 trace_intel_gpu_freq_change(vlv_gpu_freq(dev_priv->mem_freq, val));
2573}
2574
2575
8090c6b9 2576static void gen6_disable_rps(struct drm_device *dev)
2b4e57bd
ED		 2577{
2578 struct drm_i915_private *dev_priv = dev->dev_private;
2579
88509484 2580 I915_WRITE(GEN6_RC_CONTROL, 0);
2b4e57bd
ED		 2581 I915_WRITE(GEN6_RPNSWREQ, 1 << 31);
2582 I915_WRITE(GEN6_PMINTRMSK, 0xffffffff);
2583 I915_WRITE(GEN6_PMIER, 0);
2584 /* Complete PM interrupt masking here doesn't race with the rps work
2585 * item again unmasking PM interrupts because that is using a different
2586 * register (PMIMR) to mask PM interrupts. The only risk is in leaving
2587 * stale bits in PMIIR and PMIMR which gen6_enable_rps will clean up. */
2588
c6a828d3
DV		 2589 spin_lock_irq(&dev_priv->rps.lock);
2590 dev_priv->rps.pm_iir = 0;
2591 spin_unlock_irq(&dev_priv->rps.lock);
2b4e57bd
ED		 2592
2593 I915_WRITE(GEN6_PMIIR, I915_READ(GEN6_PMIIR));
2594}
2595
d20d4f0c
JB		 2596static void valleyview_disable_rps(struct drm_device *dev)
2597{
2598 struct drm_i915_private *dev_priv = dev->dev_private;
2599
2600 I915_WRITE(GEN6_RC_CONTROL, 0);
2601 I915_WRITE(GEN6_PMINTRMSK, 0xffffffff);
2602 I915_WRITE(GEN6_PMIER, 0);
2603 /* Complete PM interrupt masking here doesn't race with the rps work
2604 * item again unmasking PM interrupts because that is using a different
2605 * register (PMIMR) to mask PM interrupts. The only risk is in leaving
2606 * stale bits in PMIIR and PMIMR which gen6_enable_rps will clean up. */
2607
2608 spin_lock_irq(&dev_priv->rps.lock);
2609 dev_priv->rps.pm_iir = 0;
2610 spin_unlock_irq(&dev_priv->rps.lock);
2611
2612 I915_WRITE(GEN6_PMIIR, I915_READ(GEN6_PMIIR));
c9cddffc
JB		 2613
2614 if (dev_priv->vlv_pctx) {
2615 drm_gem_object_unreference(&dev_priv->vlv_pctx->base);
2616 dev_priv->vlv_pctx = NULL;
2617 }
d20d4f0c
JB		 2618}
2619
2b4e57bd
ED		 2620int intel_enable_rc6(const struct drm_device *dev)
2621{
456470eb 2622 /* Respect the kernel parameter if it is set */
2b4e57bd
ED		 2623 if (i915_enable_rc6 >= 0)
2624 return i915_enable_rc6;
2625
6567d748
CW		 2626 /* Disable RC6 on Ironlake */
2627 if (INTEL_INFO(dev)->gen == 5)
2628 return 0;
2b4e57bd 2629
456470eb
DV		 2630 if (IS_HASWELL(dev)) {
2631 DRM_DEBUG_DRIVER("Haswell: only RC6 available\n");
4a637c2c 2632 return INTEL_RC6_ENABLE;
456470eb 2633 }
2b4e57bd 2634
456470eb 2635 /* snb/ivb have more than one rc6 state. */
2b4e57bd
ED		 2636 if (INTEL_INFO(dev)->gen == 6) {
2637 DRM_DEBUG_DRIVER("Sandybridge: deep RC6 disabled\n");
2638 return INTEL_RC6_ENABLE;
2639 }
456470eb 2640
2b4e57bd
ED		 2641 DRM_DEBUG_DRIVER("RC6 and deep RC6 enabled\n");
2642 return (INTEL_RC6_ENABLE | INTEL_RC6p_ENABLE);
2643}
2644
79f5b2c7 2645static void gen6_enable_rps(struct drm_device *dev)
2b4e57bd 2646{
79f5b2c7 2647 struct drm_i915_private *dev_priv = dev->dev_private;
b4519513 2648 struct intel_ring_buffer *ring;
7b9e0ae6
CW		 2649 u32 rp_state_cap;
2650 u32 gt_perf_status;
31643d54 2651 u32 rc6vids, pcu_mbox, rc6_mask = 0;
2b4e57bd 2652 u32 gtfifodbg;
2b4e57bd 2653 int rc6_mode;
42c0526c 2654 int i, ret;
2b4e57bd 2655
4fc688ce 2656 WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
79f5b2c7 2657
2b4e57bd
ED		 2658 /* Here begins a magic sequence of register writes to enable
2659 * auto-downclocking.
2660 *
2661 * Perhaps there might be some value in exposing these to
2662 * userspace...
2663 */
2664 I915_WRITE(GEN6_RC_STATE, 0);
2b4e57bd
ED		 2665
2666 /* Clear the DBG now so we don't confuse earlier errors */
2667 if ((gtfifodbg = I915_READ(GTFIFODBG))) {
2668 DRM_ERROR("GT fifo had a previous error %x\n", gtfifodbg);
2669 I915_WRITE(GTFIFODBG, gtfifodbg);
2670 }
2671
2672 gen6_gt_force_wake_get(dev_priv);
2673
7b9e0ae6
CW		 2674 rp_state_cap = I915_READ(GEN6_RP_STATE_CAP);
2675 gt_perf_status = I915_READ(GEN6_GT_PERF_STATUS);
2676
31c77388
BW		 2677 /* In units of 50MHz */
2678 dev_priv->rps.hw_max = dev_priv->rps.max_delay = rp_state_cap & 0xff;
c6a828d3
DV		 2679 dev_priv->rps.min_delay = (rp_state_cap & 0xff0000) >> 16;
2680 dev_priv->rps.cur_delay = 0;
7b9e0ae6 2681
2b4e57bd
ED		 2682 /* disable the counters and set deterministic thresholds */
2683 I915_WRITE(GEN6_RC_CONTROL, 0);
2684
2685 I915_WRITE(GEN6_RC1_WAKE_RATE_LIMIT, 1000 << 16);
2686 I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16 | 30);
2687 I915_WRITE(GEN6_RC6pp_WAKE_RATE_LIMIT, 30);
2688 I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000);
2689 I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25);
2690
b4519513
CW		 2691 for_each_ring(ring, dev_priv, i)
2692 I915_WRITE(RING_MAX_IDLE(ring->mmio_base), 10);
2b4e57bd
ED		 2693
2694 I915_WRITE(GEN6_RC_SLEEP, 0);
2695 I915_WRITE(GEN6_RC1e_THRESHOLD, 1000);
2696 I915_WRITE(GEN6_RC6_THRESHOLD, 50000);
0920a487 2697 I915_WRITE(GEN6_RC6p_THRESHOLD, 150000);
2b4e57bd
ED		 2698 I915_WRITE(GEN6_RC6pp_THRESHOLD, 64000); /* unused */
2699
5a7dc92a 2700 /* Check if we are enabling RC6 */
2b4e57bd
ED		 2701 rc6_mode = intel_enable_rc6(dev_priv->dev);
2702 if (rc6_mode & INTEL_RC6_ENABLE)
2703 rc6_mask |= GEN6_RC_CTL_RC6_ENABLE;
2704
5a7dc92a
ED		 2705 /* We don't use those on Haswell */
2706 if (!IS_HASWELL(dev)) {
2707 if (rc6_mode & INTEL_RC6p_ENABLE)
2708 rc6_mask |= GEN6_RC_CTL_RC6p_ENABLE;
2b4e57bd 2709
5a7dc92a
ED		 2710 if (rc6_mode & INTEL_RC6pp_ENABLE)
2711 rc6_mask |= GEN6_RC_CTL_RC6pp_ENABLE;
2712 }
2b4e57bd
ED		 2713
2714 DRM_INFO("Enabling RC6 states: RC6 %s, RC6p %s, RC6pp %s\n",
5a7dc92a
ED		 2715 (rc6_mask & GEN6_RC_CTL_RC6_ENABLE) ? "on" : "off",
2716 (rc6_mask & GEN6_RC_CTL_RC6p_ENABLE) ? "on" : "off",
2717 (rc6_mask & GEN6_RC_CTL_RC6pp_ENABLE) ? "on" : "off");
2b4e57bd
ED		 2718
2719 I915_WRITE(GEN6_RC_CONTROL,
2720 rc6_mask |
2721 GEN6_RC_CTL_EI_MODE(1) |
2722 GEN6_RC_CTL_HW_ENABLE);
2723
92bd1bf0
RV		 2724 if (IS_HASWELL(dev)) {
2725 I915_WRITE(GEN6_RPNSWREQ,
2726 HSW_FREQUENCY(10));
2727 I915_WRITE(GEN6_RC_VIDEO_FREQ,
2728 HSW_FREQUENCY(12));
2729 } else {
2730 I915_WRITE(GEN6_RPNSWREQ,
2731 GEN6_FREQUENCY(10) |
2732 GEN6_OFFSET(0) |
2733 GEN6_AGGRESSIVE_TURBO);
2734 I915_WRITE(GEN6_RC_VIDEO_FREQ,
2735 GEN6_FREQUENCY(12));
2736 }
2b4e57bd
ED		 2737
2738 I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 1000000);
2739 I915_WRITE(GEN6_RP_INTERRUPT_LIMITS,
c6a828d3
DV		 2740 dev_priv->rps.max_delay << 24 |
2741 dev_priv->rps.min_delay << 16);
5a7dc92a 2742
1ee9ae32
DV		 2743 I915_WRITE(GEN6_RP_UP_THRESHOLD, 59400);
2744 I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 245000);
2745 I915_WRITE(GEN6_RP_UP_EI, 66000);
2746 I915_WRITE(GEN6_RP_DOWN_EI, 350000);
5a7dc92a 2747
2b4e57bd
ED		 2748 I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
2749 I915_WRITE(GEN6_RP_CONTROL,
2750 GEN6_RP_MEDIA_TURBO |
89ba829e 2751 GEN6_RP_MEDIA_HW_NORMAL_MODE |
2b4e57bd
ED		 2752 GEN6_RP_MEDIA_IS_GFX |
2753 GEN6_RP_ENABLE |
2754 GEN6_RP_UP_BUSY_AVG |
5a7dc92a 2755 (IS_HASWELL(dev) ? GEN7_RP_DOWN_IDLE_AVG : GEN6_RP_DOWN_IDLE_CONT));
2b4e57bd 2756
42c0526c 2757 ret = sandybridge_pcode_write(dev_priv, GEN6_PCODE_WRITE_MIN_FREQ_TABLE, 0);
fec46b5e 2758 if (!ret && (IS_GEN6(dev) || IS_IVYBRIDGE(dev))) {
42c0526c
BW		 2759 pcu_mbox = 0;
2760 ret = sandybridge_pcode_read(dev_priv, GEN6_READ_OC_PARAMS, &pcu_mbox);
a2b3fc01 2761 if (!ret && (pcu_mbox & (1<<31))) { /* OC supported */
10e08497 2762 DRM_DEBUG_DRIVER("Overclocking supported. Max: %dMHz, Overclock max: %dMHz\n",
a2b3fc01
BW		 2763 (dev_priv->rps.max_delay & 0xff) * 50,
2764 (pcu_mbox & 0xff) * 50);
31c77388 2765 dev_priv->rps.hw_max = pcu_mbox & 0xff;
42c0526c
BW		 2766 }
2767 } else {
2768 DRM_DEBUG_DRIVER("Failed to set the min frequency\n");
2b4e57bd
ED		 2769 }
2770
7b9e0ae6 2771 gen6_set_rps(dev_priv->dev, (gt_perf_status & 0xff00) >> 8);
2b4e57bd
ED		 2772
2773 /* requires MSI enabled */
ff928261 2774 I915_WRITE(GEN6_PMIER, GEN6_PM_DEFERRED_EVENTS);
c6a828d3
DV		 2775 spin_lock_irq(&dev_priv->rps.lock);
2776 WARN_ON(dev_priv->rps.pm_iir != 0);
2b4e57bd 2777 I915_WRITE(GEN6_PMIMR, 0);
c6a828d3 2778 spin_unlock_irq(&dev_priv->rps.lock);
2b4e57bd
ED		 2779 /* enable all PM interrupts */
2780 I915_WRITE(GEN6_PMINTRMSK, 0);
2781
31643d54
BW		 2782 rc6vids = 0;
2783 ret = sandybridge_pcode_read(dev_priv, GEN6_PCODE_READ_RC6VIDS, &rc6vids);
2784 if (IS_GEN6(dev) && ret) {
2785 DRM_DEBUG_DRIVER("Couldn't check for BIOS workaround\n");
2786 } else if (IS_GEN6(dev) && (GEN6_DECODE_RC6_VID(rc6vids & 0xff) < 450)) {
2787 DRM_DEBUG_DRIVER("You should update your BIOS. Correcting minimum rc6 voltage (%dmV->%dmV)\n",
2788 GEN6_DECODE_RC6_VID(rc6vids & 0xff), 450);
2789 rc6vids &= 0xffff00;
2790 rc6vids |= GEN6_ENCODE_RC6_VID(450);
2791 ret = sandybridge_pcode_write(dev_priv, GEN6_PCODE_WRITE_RC6VIDS, rc6vids);
2792 if (ret)
2793 DRM_ERROR("Couldn't fix incorrect rc6 voltage\n");
2794 }
2795
2b4e57bd 2796 gen6_gt_force_wake_put(dev_priv);
2b4e57bd
ED		 2797}
2798
79f5b2c7 2799static void gen6_update_ring_freq(struct drm_device *dev)
2b4e57bd 2800{
79f5b2c7 2801 struct drm_i915_private *dev_priv = dev->dev_private;
2b4e57bd 2802 int min_freq = 15;
3ebecd07
CW		 2803 unsigned int gpu_freq;
2804 unsigned int max_ia_freq, min_ring_freq;
2b4e57bd
ED		 2805 int scaling_factor = 180;
2806
4fc688ce 2807 WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
79f5b2c7 2808
2b4e57bd
ED		 2809 max_ia_freq = cpufreq_quick_get_max(0);
2810 /*
2811 * Default to measured freq if none found, PCU will ensure we don't go
2812 * over
2813 */
2814 if (!max_ia_freq)
2815 max_ia_freq = tsc_khz;
2816
2817 /* Convert from kHz to MHz */
2818 max_ia_freq /= 1000;
2819
3ebecd07
CW		 2820 min_ring_freq = I915_READ(MCHBAR_MIRROR_BASE_SNB + DCLK);
2821 /* convert DDR frequency from units of 133.3MHz to bandwidth */
2822 min_ring_freq = (2 * 4 * min_ring_freq + 2) / 3;
2823
2b4e57bd
ED		 2824 /*
2825 * For each potential GPU frequency, load a ring frequency we'd like
2826 * to use for memory access. We do this by specifying the IA frequency
2827 * the PCU should use as a reference to determine the ring frequency.
2828 */
c6a828d3 2829 for (gpu_freq = dev_priv->rps.max_delay; gpu_freq >= dev_priv->rps.min_delay;
2b4e57bd 2830 gpu_freq--) {
c6a828d3 2831 int diff = dev_priv->rps.max_delay - gpu_freq;
3ebecd07
CW		 2832 unsigned int ia_freq = 0, ring_freq = 0;
2833
2834 if (IS_HASWELL(dev)) {
2835 ring_freq = (gpu_freq * 5 + 3) / 4;
2836 ring_freq = max(min_ring_freq, ring_freq);
2837 /* leave ia_freq as the default, chosen by cpufreq */
2838 } else {
2839 /* On older processors, there is no separate ring
2840 * clock domain, so in order to boost the bandwidth
2841 * of the ring, we need to upclock the CPU (ia_freq).
2842 *
2843 * For GPU frequencies less than 750MHz,
2844 * just use the lowest ring freq.
2845 */
2846 if (gpu_freq < min_freq)
2847 ia_freq = 800;
2848 else
2849 ia_freq = max_ia_freq - ((diff * scaling_factor) / 2);
2850 ia_freq = DIV_ROUND_CLOSEST(ia_freq, 100);
2851 }
2b4e57bd 2852
42c0526c
BW		 2853 sandybridge_pcode_write(dev_priv,
2854 GEN6_PCODE_WRITE_MIN_FREQ_TABLE,
3ebecd07
CW		 2855 ia_freq << GEN6_PCODE_FREQ_IA_RATIO_SHIFT |
2856 ring_freq << GEN6_PCODE_FREQ_RING_RATIO_SHIFT |
2857 gpu_freq);
2b4e57bd 2858 }
2b4e57bd
ED		 2859}
2860
0a073b84
JB		 2861int valleyview_rps_max_freq(struct drm_i915_private *dev_priv)
2862{
2863 u32 val, rp0;
2864
2865 valleyview_nc_read(dev_priv, IOSF_NC_FB_GFX_FREQ_FUSE, &val);
2866
2867 rp0 = (val & FB_GFX_MAX_FREQ_FUSE_MASK) >> FB_GFX_MAX_FREQ_FUSE_SHIFT;
2868 /* Clamp to max */
2869 rp0 = min_t(u32, rp0, 0xea);
2870
2871 return rp0;
2872}
2873
2874static int valleyview_rps_rpe_freq(struct drm_i915_private *dev_priv)
2875{
2876 u32 val, rpe;
2877
2878 valleyview_nc_read(dev_priv, IOSF_NC_FB_GFX_FMAX_FUSE_LO, &val);
2879 rpe = (val & FB_FMAX_VMIN_FREQ_LO_MASK) >> FB_FMAX_VMIN_FREQ_LO_SHIFT;
2880 valleyview_nc_read(dev_priv, IOSF_NC_FB_GFX_FMAX_FUSE_HI, &val);
2881 rpe |= (val & FB_FMAX_VMIN_FREQ_HI_MASK) << 5;
2882
2883 return rpe;
2884}
2885
2886int valleyview_rps_min_freq(struct drm_i915_private *dev_priv)
2887{
2888 u32 val;
2889
2890 valleyview_punit_read(dev_priv, PUNIT_REG_GPU_LFM, &val);
2891
2892 return val & 0xff;
2893}
2894
52ceb908
JB		 2895static void vlv_rps_timer_work(struct work_struct *work)
2896{
2897 drm_i915_private_t *dev_priv = container_of(work, drm_i915_private_t,
2898 rps.vlv_work.work);
2899
2900 /*
2901 * Timer fired, we must be idle. Drop to min voltage state.
2902 * Note: we use RPe here since it should match the
2903 * Vmin we were shooting for. That should give us better
2904 * perf when we come back out of RC6 than if we used the
2905 * min freq available.
2906 */
2907 mutex_lock(&dev_priv->rps.hw_lock);
2908 valleyview_set_rps(dev_priv->dev, dev_priv->rps.rpe_delay);
2909 mutex_unlock(&dev_priv->rps.hw_lock);
2910}
2911
c9cddffc
JB		 2912static void valleyview_setup_pctx(struct drm_device *dev)
2913{
2914 struct drm_i915_private *dev_priv = dev->dev_private;
2915 struct drm_i915_gem_object *pctx;
2916 unsigned long pctx_paddr;
2917 u32 pcbr;
2918 int pctx_size = 24*1024;
2919
2920 pcbr = I915_READ(VLV_PCBR);
2921 if (pcbr) {
2922 /* BIOS set it up already, grab the pre-alloc'd space */
2923 int pcbr_offset;
2924
2925 pcbr_offset = (pcbr & (~4095)) - dev_priv->mm.stolen_base;
2926 pctx = i915_gem_object_create_stolen_for_preallocated(dev_priv->dev,
2927 pcbr_offset,
3727d55e 2928 -1,
c9cddffc
JB		 2929 pctx_size);
2930 goto out;
2931 }
2932
2933 /*
2934 * From the Gunit register HAS:
2935 * The Gfx driver is expected to program this register and ensure
2936 * proper allocation within Gfx stolen memory. For example, this
2937 * register should be programmed such than the PCBR range does not
2938 * overlap with other ranges, such as the frame buffer, protected
2939 * memory, or any other relevant ranges.
2940 */
2941 pctx = i915_gem_object_create_stolen(dev, pctx_size);
2942 if (!pctx) {
2943 DRM_DEBUG("not enough stolen space for PCTX, disabling\n");
2944 return;
2945 }
2946
2947 pctx_paddr = dev_priv->mm.stolen_base + pctx->stolen->start;
2948 I915_WRITE(VLV_PCBR, pctx_paddr);
2949
2950out:
2951 dev_priv->vlv_pctx = pctx;
2952}
2953
0a073b84
JB		 2954static void valleyview_enable_rps(struct drm_device *dev)
2955{
2956 struct drm_i915_private *dev_priv = dev->dev_private;
2957 struct intel_ring_buffer *ring;
2958 u32 gtfifodbg, val, rpe;
2959 int i;
2960
2961 WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
2962
2963 if ((gtfifodbg = I915_READ(GTFIFODBG))) {
2964 DRM_ERROR("GT fifo had a previous error %x\n", gtfifodbg);
2965 I915_WRITE(GTFIFODBG, gtfifodbg);
2966 }
2967
c9cddffc
JB		 2968 valleyview_setup_pctx(dev);
2969
0a073b84
JB		 2970 gen6_gt_force_wake_get(dev_priv);
2971
2972 I915_WRITE(GEN6_RP_UP_THRESHOLD, 59400);
2973 I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 245000);
2974 I915_WRITE(GEN6_RP_UP_EI, 66000);
2975 I915_WRITE(GEN6_RP_DOWN_EI, 350000);
2976
2977 I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
2978
2979 I915_WRITE(GEN6_RP_CONTROL,
2980 GEN6_RP_MEDIA_TURBO |
2981 GEN6_RP_MEDIA_HW_NORMAL_MODE |
2982 GEN6_RP_MEDIA_IS_GFX |
2983 GEN6_RP_ENABLE |
2984 GEN6_RP_UP_BUSY_AVG |
2985 GEN6_RP_DOWN_IDLE_CONT);
2986
2987 I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 0x00280000);
2988 I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000);
2989 I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25);
2990
2991 for_each_ring(ring, dev_priv, i)
2992 I915_WRITE(RING_MAX_IDLE(ring->mmio_base), 10);
2993
2994 I915_WRITE(GEN6_RC6_THRESHOLD, 0xc350);
2995
2996 /* allows RC6 residency counter to work */
2997 I915_WRITE(0x138104, _MASKED_BIT_ENABLE(0x3));
2998 I915_WRITE(GEN6_RC_CONTROL,
2999 GEN7_RC_CTL_TO_MODE);
3000
3001 valleyview_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS, &val);
2445966e
JB		 3002 switch ((val >> 6) & 3) {
3003 case 0:
3004 case 1:
3005 dev_priv->mem_freq = 800;
3006 break;
3007 case 2:
3008 dev_priv->mem_freq = 1066;
3009 break;
3010 case 3:
3011 dev_priv->mem_freq = 1333;
3012 break;
3013 }
0a073b84
JB		 3014 DRM_DEBUG_DRIVER("DDR speed: %d MHz", dev_priv->mem_freq);
3015
3016 DRM_DEBUG_DRIVER("GPLL enabled? %s\n", val & 0x10 ? "yes" : "no");
3017 DRM_DEBUG_DRIVER("GPU status: 0x%08x\n", val);
3018
3019 DRM_DEBUG_DRIVER("current GPU freq: %d\n",
3020 vlv_gpu_freq(dev_priv->mem_freq, (val >> 8) & 0xff));
3021 dev_priv->rps.cur_delay = (val >> 8) & 0xff;
3022
3023 dev_priv->rps.max_delay = valleyview_rps_max_freq(dev_priv);
3024 dev_priv->rps.hw_max = dev_priv->rps.max_delay;
3025 DRM_DEBUG_DRIVER("max GPU freq: %d\n", vlv_gpu_freq(dev_priv->mem_freq,
3026 dev_priv->rps.max_delay));
3027
3028 rpe = valleyview_rps_rpe_freq(dev_priv);
3029 DRM_DEBUG_DRIVER("RPe GPU freq: %d\n",
3030 vlv_gpu_freq(dev_priv->mem_freq, rpe));
52ceb908 3031 dev_priv->rps.rpe_delay = rpe;
0a073b84
JB		 3032
3033 val = valleyview_rps_min_freq(dev_priv);
3034 DRM_DEBUG_DRIVER("min GPU freq: %d\n", vlv_gpu_freq(dev_priv->mem_freq,
3035 val));
3036 dev_priv->rps.min_delay = val;
3037
3038 DRM_DEBUG_DRIVER("setting GPU freq to %d\n",
3039 vlv_gpu_freq(dev_priv->mem_freq, rpe));
3040
52ceb908
JB		 3041 INIT_DELAYED_WORK(&dev_priv->rps.vlv_work, vlv_rps_timer_work);
3042
0a073b84
JB		 3043 valleyview_set_rps(dev_priv->dev, rpe);
3044
3045 /* requires MSI enabled */
3046 I915_WRITE(GEN6_PMIER, GEN6_PM_DEFERRED_EVENTS);
3047 spin_lock_irq(&dev_priv->rps.lock);
3048 WARN_ON(dev_priv->rps.pm_iir != 0);
3049 I915_WRITE(GEN6_PMIMR, 0);
3050 spin_unlock_irq(&dev_priv->rps.lock);
3051 /* enable all PM interrupts */
3052 I915_WRITE(GEN6_PMINTRMSK, 0);
3053
3054 gen6_gt_force_wake_put(dev_priv);
3055}
3056
930ebb46 3057void ironlake_teardown_rc6(struct drm_device *dev)
2b4e57bd
ED		 3058{
3059 struct drm_i915_private *dev_priv = dev->dev_private;
3060
3e373948
DV		 3061 if (dev_priv->ips.renderctx) {
3062 i915_gem_object_unpin(dev_priv->ips.renderctx);
3063 drm_gem_object_unreference(&dev_priv->ips.renderctx->base);
3064 dev_priv->ips.renderctx = NULL;
2b4e57bd
ED		 3065 }
3066
3e373948
DV		 3067 if (dev_priv->ips.pwrctx) {
3068 i915_gem_object_unpin(dev_priv->ips.pwrctx);
3069 drm_gem_object_unreference(&dev_priv->ips.pwrctx->base);
3070 dev_priv->ips.pwrctx = NULL;
2b4e57bd
ED		 3071 }
3072}
3073
930ebb46 3074static void ironlake_disable_rc6(struct drm_device *dev)
2b4e57bd
ED		 3075{
3076 struct drm_i915_private *dev_priv = dev->dev_private;
3077
3078 if (I915_READ(PWRCTXA)) {
3079 /* Wake the GPU, prevent RC6, then restore RSTDBYCTL */
3080 I915_WRITE(RSTDBYCTL, I915_READ(RSTDBYCTL) | RCX_SW_EXIT);
3081 wait_for(((I915_READ(RSTDBYCTL) & RSX_STATUS_MASK) == RSX_STATUS_ON),
3082 50);
3083
3084 I915_WRITE(PWRCTXA, 0);
3085 POSTING_READ(PWRCTXA);
3086
3087 I915_WRITE(RSTDBYCTL, I915_READ(RSTDBYCTL) & ~RCX_SW_EXIT);
3088 POSTING_READ(RSTDBYCTL);
3089 }
2b4e57bd
ED		 3090}
3091
3092static int ironlake_setup_rc6(struct drm_device *dev)
3093{
3094 struct drm_i915_private *dev_priv = dev->dev_private;
3095
3e373948
DV		 3096 if (dev_priv->ips.renderctx == NULL)
3097 dev_priv->ips.renderctx = intel_alloc_context_page(dev);
3098 if (!dev_priv->ips.renderctx)
2b4e57bd
ED		 3099 return -ENOMEM;
3100
3e373948
DV		 3101 if (dev_priv->ips.pwrctx == NULL)
3102 dev_priv->ips.pwrctx = intel_alloc_context_page(dev);
3103 if (!dev_priv->ips.pwrctx) {
2b4e57bd
ED		 3104 ironlake_teardown_rc6(dev);
3105 return -ENOMEM;
3106 }
3107
3108 return 0;
3109}
3110
930ebb46 3111static void ironlake_enable_rc6(struct drm_device *dev)
2b4e57bd
ED		 3112{
3113 struct drm_i915_private *dev_priv = dev->dev_private;
6d90c952 3114 struct intel_ring_buffer *ring = &dev_priv->ring[RCS];
3e960501 3115 bool was_interruptible;
2b4e57bd
ED		 3116 int ret;
3117
3118 /* rc6 disabled by default due to repeated reports of hanging during
3119 * boot and resume.
3120 */
3121 if (!intel_enable_rc6(dev))
3122 return;
3123
79f5b2c7
DV		 3124 WARN_ON(!mutex_is_locked(&dev->struct_mutex));
3125
2b4e57bd 3126 ret = ironlake_setup_rc6(dev);
79f5b2c7 3127 if (ret)
2b4e57bd 3128 return;
2b4e57bd 3129
3e960501
CW		 3130 was_interruptible = dev_priv->mm.interruptible;
3131 dev_priv->mm.interruptible = false;
3132
2b4e57bd
ED		 3133 /*
3134 * GPU can automatically power down the render unit if given a page
3135 * to save state.
3136 */
6d90c952 3137 ret = intel_ring_begin(ring, 6);
2b4e57bd
ED		 3138 if (ret) {
3139 ironlake_teardown_rc6(dev);
3e960501 3140 dev_priv->mm.interruptible = was_interruptible;
2b4e57bd
ED		 3141 return;
3142 }
3143
6d90c952
DV		 3144 intel_ring_emit(ring, MI_SUSPEND_FLUSH | MI_SUSPEND_FLUSH_EN);
3145 intel_ring_emit(ring, MI_SET_CONTEXT);
3e373948 3146 intel_ring_emit(ring, dev_priv->ips.renderctx->gtt_offset |
6d90c952
DV		 3147 MI_MM_SPACE_GTT |
3148 MI_SAVE_EXT_STATE_EN |
3149 MI_RESTORE_EXT_STATE_EN |
3150 MI_RESTORE_INHIBIT);
3151 intel_ring_emit(ring, MI_SUSPEND_FLUSH);
3152 intel_ring_emit(ring, MI_NOOP);
3153 intel_ring_emit(ring, MI_FLUSH);
3154 intel_ring_advance(ring);
2b4e57bd
ED		 3155
3156 /*
3157 * Wait for the command parser to advance past MI_SET_CONTEXT. The HW
3158 * does an implicit flush, combined with MI_FLUSH above, it should be
3159 * safe to assume that renderctx is valid
3160 */
3e960501
CW		 3161 ret = intel_ring_idle(ring);
3162 dev_priv->mm.interruptible = was_interruptible;
2b4e57bd 3163 if (ret) {
def27a58 3164 DRM_ERROR("failed to enable ironlake power savings\n");
2b4e57bd 3165 ironlake_teardown_rc6(dev);
2b4e57bd
ED		 3166 return;
3167 }
3168
3e373948 3169 I915_WRITE(PWRCTXA, dev_priv->ips.pwrctx->gtt_offset | PWRCTX_EN);
2b4e57bd 3170 I915_WRITE(RSTDBYCTL, I915_READ(RSTDBYCTL) & ~RCX_SW_EXIT);
2b4e57bd
ED		 3171}
3172
dde18883
ED		 3173static unsigned long intel_pxfreq(u32 vidfreq)
3174{
3175 unsigned long freq;
3176 int div = (vidfreq & 0x3f0000) >> 16;
3177 int post = (vidfreq & 0x3000) >> 12;
3178 int pre = (vidfreq & 0x7);
3179
3180 if (!pre)
3181 return 0;
3182
3183 freq = ((div * 133333) / ((1<<post) * pre));
3184
3185 return freq;
3186}
3187
eb48eb00
DV		 3188static const struct cparams {
3189 u16 i;
3190 u16 t;
3191 u16 m;
3192 u16 c;
3193} cparams[] = {
3194 { 1, 1333, 301, 28664 },
3195 { 1, 1066, 294, 24460 },
3196 { 1, 800, 294, 25192 },
3197 { 0, 1333, 276, 27605 },
3198 { 0, 1066, 276, 27605 },
3199 { 0, 800, 231, 23784 },
3200};
3201
f531dcb2 3202static unsigned long __i915_chipset_val(struct drm_i915_private *dev_priv)
eb48eb00
DV		 3203{
3204 u64 total_count, diff, ret;
3205 u32 count1, count2, count3, m = 0, c = 0;
3206 unsigned long now = jiffies_to_msecs(jiffies), diff1;
3207 int i;
3208
02d71956
DV		 3209 assert_spin_locked(&mchdev_lock);
3210
20e4d407 3211 diff1 = now - dev_priv->ips.last_time1;
eb48eb00
DV		 3212
3213 /* Prevent division-by-zero if we are asking too fast.
3214 * Also, we don't get interesting results if we are polling
3215 * faster than once in 10ms, so just return the saved value
3216 * in such cases.
3217 */
3218 if (diff1 <= 10)
20e4d407 3219 return dev_priv->ips.chipset_power;
eb48eb00
DV		 3220
3221 count1 = I915_READ(DMIEC);
3222 count2 = I915_READ(DDREC);
3223 count3 = I915_READ(CSIEC);
3224
3225 total_count = count1 + count2 + count3;
3226
3227 /* FIXME: handle per-counter overflow */
20e4d407
DV		 3228 if (total_count < dev_priv->ips.last_count1) {
3229 diff = ~0UL - dev_priv->ips.last_count1;
eb48eb00
DV		 3230 diff += total_count;
3231 } else {
20e4d407 3232 diff = total_count - dev_priv->ips.last_count1;
eb48eb00
DV		 3233 }
3234
3235 for (i = 0; i < ARRAY_SIZE(cparams); i++) {
20e4d407
DV		 3236 if (cparams[i].i == dev_priv->ips.c_m &&
3237 cparams[i].t == dev_priv->ips.r_t) {
eb48eb00
DV		 3238 m = cparams[i].m;
3239 c = cparams[i].c;
3240 break;
3241 }
3242 }
3243
3244 diff = div_u64(diff, diff1);
3245 ret = ((m * diff) + c);
3246 ret = div_u64(ret, 10);
3247
20e4d407
DV		 3248 dev_priv->ips.last_count1 = total_count;
3249 dev_priv->ips.last_time1 = now;
eb48eb00 3250
20e4d407 3251 dev_priv->ips.chipset_power = ret;
eb48eb00
DV		 3252
3253 return ret;
3254}
3255
f531dcb2
CW		 3256unsigned long i915_chipset_val(struct drm_i915_private *dev_priv)
3257{
3258 unsigned long val;
3259
3260 if (dev_priv->info->gen != 5)
3261 return 0;
3262
3263 spin_lock_irq(&mchdev_lock);
3264
3265 val = __i915_chipset_val(dev_priv);
3266
3267 spin_unlock_irq(&mchdev_lock);
3268
3269 return val;
3270}
3271
eb48eb00
DV		 3272unsigned long i915_mch_val(struct drm_i915_private *dev_priv)
3273{
3274 unsigned long m, x, b;
3275 u32 tsfs;
3276
3277 tsfs = I915_READ(TSFS);
3278
3279 m = ((tsfs & TSFS_SLOPE_MASK) >> TSFS_SLOPE_SHIFT);
3280 x = I915_READ8(TR1);
3281
3282 b = tsfs & TSFS_INTR_MASK;
3283
3284 return ((m * x) / 127) - b;
3285}
3286
3287static u16 pvid_to_extvid(struct drm_i915_private *dev_priv, u8 pxvid)
3288{
3289 static const struct v_table {
3290 u16 vd; /* in .1 mil */
3291 u16 vm; /* in .1 mil */
3292 } v_table[] = {
3293 { 0, 0, },
3294 { 375, 0, },
3295 { 500, 0, },
3296 { 625, 0, },
3297 { 750, 0, },
3298 { 875, 0, },
3299 { 1000, 0, },
3300 { 1125, 0, },
3301 { 4125, 3000, },
3302 { 4125, 3000, },
3303 { 4125, 3000, },
3304 { 4125, 3000, },
3305 { 4125, 3000, },
3306 { 4125, 3000, },
3307 { 4125, 3000, },
3308 { 4125, 3000, },
3309 { 4125, 3000, },
3310 { 4125, 3000, },
3311 { 4125, 3000, },
3312 { 4125, 3000, },
3313 { 4125, 3000, },
3314 { 4125, 3000, },
3315 { 4125, 3000, },
3316 { 4125, 3000, },
3317 { 4125, 3000, },
3318 { 4125, 3000, },
3319 { 4125, 3000, },
3320 { 4125, 3000, },
3321 { 4125, 3000, },
3322 { 4125, 3000, },
3323 { 4125, 3000, },
3324 { 4125, 3000, },
3325 { 4250, 3125, },
3326 { 4375, 3250, },
3327 { 4500, 3375, },
3328 { 4625, 3500, },
3329 { 4750, 3625, },
3330 { 4875, 3750, },
3331 { 5000, 3875, },
3332 { 5125, 4000, },
3333 { 5250, 4125, },
3334 { 5375, 4250, },
3335 { 5500, 4375, },
3336 { 5625, 4500, },
3337 { 5750, 4625, },
3338 { 5875, 4750, },
3339 { 6000, 4875, },
3340 { 6125, 5000, },
3341 { 6250, 5125, },
3342 { 6375, 5250, },
3343 { 6500, 5375, },
3344 { 6625, 5500, },
3345 { 6750, 5625, },
3346 { 6875, 5750, },
3347 { 7000, 5875, },
3348 { 7125, 6000, },
3349 { 7250, 6125, },
3350 { 7375, 6250, },
3351 { 7500, 6375, },
3352 { 7625, 6500, },
3353 { 7750, 6625, },
3354 { 7875, 6750, },
3355 { 8000, 6875, },
3356 { 8125, 7000, },
3357 { 8250, 7125, },
3358 { 8375, 7250, },
3359 { 8500, 7375, },
3360 { 8625, 7500, },
3361 { 8750, 7625, },
3362 { 8875, 7750, },
3363 { 9000, 7875, },
3364 { 9125, 8000, },
3365 { 9250, 8125, },
3366 { 9375, 8250, },
3367 { 9500, 8375, },
3368 { 9625, 8500, },
3369 { 9750, 8625, },
3370 { 9875, 8750, },
3371 { 10000, 8875, },
3372 { 10125, 9000, },
3373 { 10250, 9125, },
3374 { 10375, 9250, },
3375 { 10500, 9375, },
3376 { 10625, 9500, },
3377 { 10750, 9625, },
3378 { 10875, 9750, },
3379 { 11000, 9875, },
3380 { 11125, 10000, },
3381 { 11250, 10125, },
3382 { 11375, 10250, },
3383 { 11500, 10375, },
3384 { 11625, 10500, },
3385 { 11750, 10625, },
3386 { 11875, 10750, },
3387 { 12000, 10875, },
3388 { 12125, 11000, },
3389 { 12250, 11125, },
3390 { 12375, 11250, },
3391 { 12500, 11375, },
3392 { 12625, 11500, },
3393 { 12750, 11625, },
3394 { 12875, 11750, },
3395 { 13000, 11875, },
3396 { 13125, 12000, },
3397 { 13250, 12125, },
3398 { 13375, 12250, },
3399 { 13500, 12375, },
3400 { 13625, 12500, },
3401 { 13750, 12625, },
3402 { 13875, 12750, },
3403 { 14000, 12875, },
3404 { 14125, 13000, },
3405 { 14250, 13125, },
3406 { 14375, 13250, },
3407 { 14500, 13375, },
3408 { 14625, 13500, },
3409 { 14750, 13625, },
3410 { 14875, 13750, },
3411 { 15000, 13875, },
3412 { 15125, 14000, },
3413 { 15250, 14125, },
3414 { 15375, 14250, },
3415 { 15500, 14375, },
3416 { 15625, 14500, },
3417 { 15750, 14625, },
3418 { 15875, 14750, },
3419 { 16000, 14875, },
3420 { 16125, 15000, },
3421 };
3422 if (dev_priv->info->is_mobile)
3423 return v_table[pxvid].vm;
3424 else
3425 return v_table[pxvid].vd;
3426}
3427
02d71956 3428static void __i915_update_gfx_val(struct drm_i915_private *dev_priv)
eb48eb00
DV		 3429{
3430 struct timespec now, diff1;
3431 u64 diff;
3432 unsigned long diffms;
3433 u32 count;
3434
02d71956 3435 assert_spin_locked(&mchdev_lock);
eb48eb00
DV		 3436
3437 getrawmonotonic(&now);
20e4d407 3438 diff1 = timespec_sub(now, dev_priv->ips.last_time2);
eb48eb00
DV		 3439
3440 /* Don't divide by 0 */
3441 diffms = diff1.tv_sec * 1000 + diff1.tv_nsec / 1000000;
3442 if (!diffms)
3443 return;
3444
3445 count = I915_READ(GFXEC);
3446
20e4d407
DV		 3447 if (count < dev_priv->ips.last_count2) {
3448 diff = ~0UL - dev_priv->ips.last_count2;
eb48eb00
DV		 3449 diff += count;
3450 } else {
20e4d407 3451 diff = count - dev_priv->ips.last_count2;
eb48eb00
DV		 3452 }
3453
20e4d407
DV		 3454 dev_priv->ips.last_count2 = count;
3455 dev_priv->ips.last_time2 = now;
eb48eb00
DV		 3456
3457 /* More magic constants... */
3458 diff = diff * 1181;
3459 diff = div_u64(diff, diffms * 10);
20e4d407 3460 dev_priv->ips.gfx_power = diff;
eb48eb00
DV		 3461}
3462
02d71956
DV		 3463void i915_update_gfx_val(struct drm_i915_private *dev_priv)
3464{
3465 if (dev_priv->info->gen != 5)
3466 return;
3467
9270388e 3468 spin_lock_irq(&mchdev_lock);
02d71956
DV		 3469
3470 __i915_update_gfx_val(dev_priv);
3471
9270388e 3472 spin_unlock_irq(&mchdev_lock);
02d71956
DV		 3473}
3474
f531dcb2 3475static unsigned long __i915_gfx_val(struct drm_i915_private *dev_priv)
eb48eb00
DV		 3476{
3477 unsigned long t, corr, state1, corr2, state2;
3478 u32 pxvid, ext_v;
3479
02d71956
DV		 3480 assert_spin_locked(&mchdev_lock);
3481
c6a828d3 3482 pxvid = I915_READ(PXVFREQ_BASE + (dev_priv->rps.cur_delay * 4));
eb48eb00
DV		 3483 pxvid = (pxvid >> 24) & 0x7f;
3484 ext_v = pvid_to_extvid(dev_priv, pxvid);
3485
3486 state1 = ext_v;
3487
3488 t = i915_mch_val(dev_priv);
3489
3490 /* Revel in the empirically derived constants */
3491
3492 /* Correction factor in 1/100000 units */
3493 if (t > 80)
3494 corr = ((t * 2349) + 135940);
3495 else if (t >= 50)
3496 corr = ((t * 964) + 29317);
3497 else /* < 50 */
3498 corr = ((t * 301) + 1004);
3499
3500 corr = corr * ((150142 * state1) / 10000 - 78642);
3501 corr /= 100000;
20e4d407 3502 corr2 = (corr * dev_priv->ips.corr);
eb48eb00
DV		 3503
3504 state2 = (corr2 * state1) / 10000;
3505 state2 /= 100; /* convert to mW */
3506
02d71956 3507 __i915_update_gfx_val(dev_priv);
eb48eb00 3508
20e4d407 3509 return dev_priv->ips.gfx_power + state2;
eb48eb00
DV		 3510}
3511
f531dcb2
CW		 3512unsigned long i915_gfx_val(struct drm_i915_private *dev_priv)
3513{
3514 unsigned long val;
3515
3516 if (dev_priv->info->gen != 5)
3517 return 0;
3518
3519 spin_lock_irq(&mchdev_lock);
3520
3521 val = __i915_gfx_val(dev_priv);
3522
3523 spin_unlock_irq(&mchdev_lock);
3524
3525 return val;
3526}
3527
eb48eb00
DV		 3528/**
3529 * i915_read_mch_val - return value for IPS use
3530 *
3531 * Calculate and return a value for the IPS driver to use when deciding whether
3532 * we have thermal and power headroom to increase CPU or GPU power budget.
3533 */
3534unsigned long i915_read_mch_val(void)
3535{
3536 struct drm_i915_private *dev_priv;
3537 unsigned long chipset_val, graphics_val, ret = 0;
3538
9270388e 3539 spin_lock_irq(&mchdev_lock);
eb48eb00
DV		 3540 if (!i915_mch_dev)
3541 goto out_unlock;
3542 dev_priv = i915_mch_dev;
3543
f531dcb2
CW		 3544 chipset_val = __i915_chipset_val(dev_priv);
3545 graphics_val = __i915_gfx_val(dev_priv);
eb48eb00
DV		 3546
3547 ret = chipset_val + graphics_val;
3548
3549out_unlock:
9270388e 3550 spin_unlock_irq(&mchdev_lock);
eb48eb00
DV		 3551
3552 return ret;
3553}
3554EXPORT_SYMBOL_GPL(i915_read_mch_val);
3555
3556/**
3557 * i915_gpu_raise - raise GPU frequency limit
3558 *
3559 * Raise the limit; IPS indicates we have thermal headroom.
3560 */
3561bool i915_gpu_raise(void)
3562{
3563 struct drm_i915_private *dev_priv;
3564 bool ret = true;
3565
9270388e 3566 spin_lock_irq(&mchdev_lock);
eb48eb00
DV		 3567 if (!i915_mch_dev) {
3568 ret = false;
3569 goto out_unlock;
3570 }
3571 dev_priv = i915_mch_dev;
3572
20e4d407
DV		 3573 if (dev_priv->ips.max_delay > dev_priv->ips.fmax)
3574 dev_priv->ips.max_delay--;
eb48eb00
DV		 3575
3576out_unlock:
9270388e 3577 spin_unlock_irq(&mchdev_lock);
eb48eb00
DV		 3578
3579 return ret;
3580}
3581EXPORT_SYMBOL_GPL(i915_gpu_raise);
3582
3583/**
3584 * i915_gpu_lower - lower GPU frequency limit
3585 *
3586 * IPS indicates we're close to a thermal limit, so throttle back the GPU
3587 * frequency maximum.
3588 */
3589bool i915_gpu_lower(void)
3590{
3591 struct drm_i915_private *dev_priv;
3592 bool ret = true;
3593
9270388e 3594 spin_lock_irq(&mchdev_lock);
eb48eb00
DV		 3595 if (!i915_mch_dev) {
3596 ret = false;
3597 goto out_unlock;
3598 }
3599 dev_priv = i915_mch_dev;
3600
20e4d407
DV		 3601 if (dev_priv->ips.max_delay < dev_priv->ips.min_delay)
3602 dev_priv->ips.max_delay++;
eb48eb00
DV		 3603
3604out_unlock:
9270388e 3605 spin_unlock_irq(&mchdev_lock);
eb48eb00
DV		 3606
3607 return ret;
3608}
3609EXPORT_SYMBOL_GPL(i915_gpu_lower);
3610
3611/**
3612 * i915_gpu_busy - indicate GPU business to IPS
3613 *
3614 * Tell the IPS driver whether or not the GPU is busy.
3615 */
3616bool i915_gpu_busy(void)
3617{
3618 struct drm_i915_private *dev_priv;
f047e395 3619 struct intel_ring_buffer *ring;
eb48eb00 3620 bool ret = false;
f047e395 3621 int i;
eb48eb00 3622
9270388e 3623 spin_lock_irq(&mchdev_lock);
eb48eb00
DV		 3624 if (!i915_mch_dev)
3625 goto out_unlock;
3626 dev_priv = i915_mch_dev;
3627
f047e395
CW		 3628 for_each_ring(ring, dev_priv, i)
3629 ret |= !list_empty(&ring->request_list);
eb48eb00
DV		 3630
3631out_unlock:
9270388e 3632 spin_unlock_irq(&mchdev_lock);
eb48eb00
DV		 3633
3634 return ret;
3635}
3636EXPORT_SYMBOL_GPL(i915_gpu_busy);
3637
3638/**
3639 * i915_gpu_turbo_disable - disable graphics turbo
3640 *
3641 * Disable graphics turbo by resetting the max frequency and setting the
3642 * current frequency to the default.
3643 */
3644bool i915_gpu_turbo_disable(void)
3645{
3646 struct drm_i915_private *dev_priv;
3647 bool ret = true;
3648
9270388e 3649 spin_lock_irq(&mchdev_lock);
eb48eb00
DV		 3650 if (!i915_mch_dev) {
3651 ret = false;
3652 goto out_unlock;
3653 }
3654 dev_priv = i915_mch_dev;
3655
20e4d407 3656 dev_priv->ips.max_delay = dev_priv->ips.fstart;
eb48eb00 3657
20e4d407 3658 if (!ironlake_set_drps(dev_priv->dev, dev_priv->ips.fstart))
eb48eb00
DV		 3659 ret = false;
3660
3661out_unlock:
9270388e 3662 spin_unlock_irq(&mchdev_lock);
eb48eb00
DV		 3663
3664 return ret;
3665}
3666EXPORT_SYMBOL_GPL(i915_gpu_turbo_disable);
3667
3668/**
3669 * Tells the intel_ips driver that the i915 driver is now loaded, if
3670 * IPS got loaded first.
3671 *
3672 * This awkward dance is so that neither module has to depend on the
3673 * other in order for IPS to do the appropriate communication of
3674 * GPU turbo limits to i915.
3675 */
3676static void
3677ips_ping_for_i915_load(void)
3678{
3679 void (*link)(void);
3680
3681 link = symbol_get(ips_link_to_i915_driver);
3682 if (link) {
3683 link();
3684 symbol_put(ips_link_to_i915_driver);
3685 }
3686}
3687
3688void intel_gpu_ips_init(struct drm_i915_private *dev_priv)
3689{
02d71956
DV		 3690 /* We only register the i915 ips part with intel-ips once everything is
3691 * set up, to avoid intel-ips sneaking in and reading bogus values. */
9270388e 3692 spin_lock_irq(&mchdev_lock);
eb48eb00 3693 i915_mch_dev = dev_priv;
9270388e 3694 spin_unlock_irq(&mchdev_lock);
eb48eb00
DV		 3695
3696 ips_ping_for_i915_load();
3697}
3698
3699void intel_gpu_ips_teardown(void)
3700{
9270388e 3701 spin_lock_irq(&mchdev_lock);
eb48eb00 3702 i915_mch_dev = NULL;
9270388e 3703 spin_unlock_irq(&mchdev_lock);
eb48eb00 3704}
8090c6b9 3705static void intel_init_emon(struct drm_device *dev)
dde18883
ED		 3706{
3707 struct drm_i915_private *dev_priv = dev->dev_private;
3708 u32 lcfuse;
3709 u8 pxw[16];
3710 int i;
3711
3712 /* Disable to program */
3713 I915_WRITE(ECR, 0);
3714 POSTING_READ(ECR);
3715
3716 /* Program energy weights for various events */
3717 I915_WRITE(SDEW, 0x15040d00);
3718 I915_WRITE(CSIEW0, 0x007f0000);
3719 I915_WRITE(CSIEW1, 0x1e220004);
3720 I915_WRITE(CSIEW2, 0x04000004);
3721
3722 for (i = 0; i < 5; i++)
3723 I915_WRITE(PEW + (i * 4), 0);
3724 for (i = 0; i < 3; i++)
3725 I915_WRITE(DEW + (i * 4), 0);
3726
3727 /* Program P-state weights to account for frequency power adjustment */
3728 for (i = 0; i < 16; i++) {
3729 u32 pxvidfreq = I915_READ(PXVFREQ_BASE + (i * 4));
3730 unsigned long freq = intel_pxfreq(pxvidfreq);
3731 unsigned long vid = (pxvidfreq & PXVFREQ_PX_MASK) >>
3732 PXVFREQ_PX_SHIFT;
3733 unsigned long val;
3734
3735 val = vid * vid;
3736 val *= (freq / 1000);
3737 val *= 255;
3738 val /= (127*127*900);
3739 if (val > 0xff)
3740 DRM_ERROR("bad pxval: %ld\n", val);
3741 pxw[i] = val;
3742 }
3743 /* Render standby states get 0 weight */
3744 pxw[14] = 0;
3745 pxw[15] = 0;
3746
3747 for (i = 0; i < 4; i++) {
3748 u32 val = (pxw[i*4] << 24) | (pxw[(i*4)+1] << 16) |
3749 (pxw[(i*4)+2] << 8) | (pxw[(i*4)+3]);
3750 I915_WRITE(PXW + (i * 4), val);
3751 }
3752
3753 /* Adjust magic regs to magic values (more experimental results) */
3754 I915_WRITE(OGW0, 0);
3755 I915_WRITE(OGW1, 0);
3756 I915_WRITE(EG0, 0x00007f00);
3757 I915_WRITE(EG1, 0x0000000e);
3758 I915_WRITE(EG2, 0x000e0000);
3759 I915_WRITE(EG3, 0x68000300);
3760 I915_WRITE(EG4, 0x42000000);
3761 I915_WRITE(EG5, 0x00140031);
3762 I915_WRITE(EG6, 0);
3763 I915_WRITE(EG7, 0);
3764
3765 for (i = 0; i < 8; i++)
3766 I915_WRITE(PXWL + (i * 4), 0);
3767
3768 /* Enable PMON + select events */
3769 I915_WRITE(ECR, 0x80000019);
3770
3771 lcfuse = I915_READ(LCFUSE02);
3772
20e4d407 3773 dev_priv->ips.corr = (lcfuse & LCFUSE_HIV_MASK);
dde18883
ED		 3774}
3775
8090c6b9
DV		 3776void intel_disable_gt_powersave(struct drm_device *dev)
3777{
1a01ab3b
JB		 3778 struct drm_i915_private *dev_priv = dev->dev_private;
3779
fd0c0642
DV		 3780 /* Interrupts should be disabled already to avoid re-arming. */
3781 WARN_ON(dev->irq_enabled);
3782
930ebb46 3783 if (IS_IRONLAKE_M(dev)) {
8090c6b9 3784 ironlake_disable_drps(dev);
930ebb46 3785 ironlake_disable_rc6(dev);
0a073b84 3786 } else if (INTEL_INFO(dev)->gen >= 6) {
1a01ab3b 3787 cancel_delayed_work_sync(&dev_priv->rps.delayed_resume_work);
250848ca 3788 cancel_work_sync(&dev_priv->rps.work);
52ceb908
JB		 3789 if (IS_VALLEYVIEW(dev))
3790 cancel_delayed_work_sync(&dev_priv->rps.vlv_work);
4fc688ce 3791 mutex_lock(&dev_priv->rps.hw_lock);
d20d4f0c
JB		 3792 if (IS_VALLEYVIEW(dev))
3793 valleyview_disable_rps(dev);
3794 else
3795 gen6_disable_rps(dev);
4fc688ce 3796 mutex_unlock(&dev_priv->rps.hw_lock);
930ebb46 3797 }
8090c6b9
DV		 3798}
3799
1a01ab3b
JB		 3800static void intel_gen6_powersave_work(struct work_struct *work)
3801{
3802 struct drm_i915_private *dev_priv =
3803 container_of(work, struct drm_i915_private,
3804 rps.delayed_resume_work.work);
3805 struct drm_device *dev = dev_priv->dev;
3806
4fc688ce 3807 mutex_lock(&dev_priv->rps.hw_lock);
0a073b84
JB		 3808
3809 if (IS_VALLEYVIEW(dev)) {
3810 valleyview_enable_rps(dev);
3811 } else {
3812 gen6_enable_rps(dev);
3813 gen6_update_ring_freq(dev);
3814 }
4fc688ce 3815 mutex_unlock(&dev_priv->rps.hw_lock);
1a01ab3b
JB		 3816}
3817
8090c6b9
DV		 3818void intel_enable_gt_powersave(struct drm_device *dev)
3819{
1a01ab3b
JB		 3820 struct drm_i915_private *dev_priv = dev->dev_private;
3821
8090c6b9
DV		 3822 if (IS_IRONLAKE_M(dev)) {
3823 ironlake_enable_drps(dev);
3824 ironlake_enable_rc6(dev);
3825 intel_init_emon(dev);
0a073b84 3826 } else if (IS_GEN6(dev) || IS_GEN7(dev)) {
1a01ab3b
JB		 3827 /*
3828 * PCU communication is slow and this doesn't need to be
3829 * done at any specific time, so do this out of our fast path
3830 * to make resume and init faster.
3831 */
3832 schedule_delayed_work(&dev_priv->rps.delayed_resume_work,
3833 round_jiffies_up_relative(HZ));
8090c6b9
DV		 3834 }
3835}
3836
3107bd48
DV		 3837static void ibx_init_clock_gating(struct drm_device *dev)
3838{
3839 struct drm_i915_private *dev_priv = dev->dev_private;
3840
3841 /*
3842 * On Ibex Peak and Cougar Point, we need to disable clock
3843 * gating for the panel power sequencer or it will fail to
3844 * start up when no ports are active.
3845 */
3846 I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE);
3847}
3848
1fa61106 3849static void ironlake_init_clock_gating(struct drm_device *dev)
6f1d69b0
ED		 3850{
3851 struct drm_i915_private *dev_priv = dev->dev_private;
231e54f6 3852 uint32_t dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
6f1d69b0
ED		 3853
3854 /* Required for FBC */
4d47e4f5
DL		 3855 dspclk_gate |= ILK_DPFCRUNIT_CLOCK_GATE_DISABLE |
3856 ILK_DPFCUNIT_CLOCK_GATE_DISABLE |
3857 ILK_DPFDUNIT_CLOCK_GATE_ENABLE;
6f1d69b0
ED		 3858
3859 I915_WRITE(PCH_3DCGDIS0,
3860 MARIUNIT_CLOCK_GATE_DISABLE |
3861 SVSMUNIT_CLOCK_GATE_DISABLE);
3862 I915_WRITE(PCH_3DCGDIS1,
3863 VFMUNIT_CLOCK_GATE_DISABLE);
3864
6f1d69b0
ED		 3865 /*
3866 * According to the spec the following bits should be set in
3867 * order to enable memory self-refresh
3868 * The bit 22/21 of 0x42004
3869 * The bit 5 of 0x42020
3870 * The bit 15 of 0x45000
3871 */
3872 I915_WRITE(ILK_DISPLAY_CHICKEN2,
3873 (I915_READ(ILK_DISPLAY_CHICKEN2) |
3874 ILK_DPARB_GATE | ILK_VSDPFD_FULL));
4d47e4f5 3875 dspclk_gate |= ILK_DPARBUNIT_CLOCK_GATE_ENABLE;
6f1d69b0
ED		 3876 I915_WRITE(DISP_ARB_CTL,
3877 (I915_READ(DISP_ARB_CTL) |
3878 DISP_FBC_WM_DIS));
3879 I915_WRITE(WM3_LP_ILK, 0);
3880 I915_WRITE(WM2_LP_ILK, 0);
3881 I915_WRITE(WM1_LP_ILK, 0);
3882
3883 /*
3884 * Based on the document from hardware guys the following bits
3885 * should be set unconditionally in order to enable FBC.
3886 * The bit 22 of 0x42000
3887 * The bit 22 of 0x42004
3888 * The bit 7,8,9 of 0x42020.
3889 */
3890 if (IS_IRONLAKE_M(dev)) {
3891 I915_WRITE(ILK_DISPLAY_CHICKEN1,
3892 I915_READ(ILK_DISPLAY_CHICKEN1) |
3893 ILK_FBCQ_DIS);
3894 I915_WRITE(ILK_DISPLAY_CHICKEN2,
3895 I915_READ(ILK_DISPLAY_CHICKEN2) |
3896 ILK_DPARB_GATE);
6f1d69b0
ED		 3897 }
3898
4d47e4f5
DL		 3899 I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate);
3900
6f1d69b0
ED		 3901 I915_WRITE(ILK_DISPLAY_CHICKEN2,
3902 I915_READ(ILK_DISPLAY_CHICKEN2) |
3903 ILK_ELPIN_409_SELECT);
3904 I915_WRITE(_3D_CHICKEN2,
3905 _3D_CHICKEN2_WM_READ_PIPELINED << 16 |
3906 _3D_CHICKEN2_WM_READ_PIPELINED);
4358a374 3907
ecdb4eb7 3908 /* WaDisableRenderCachePipelinedFlush:ilk */
4358a374
DV		 3909 I915_WRITE(CACHE_MODE_0,
3910 _MASKED_BIT_ENABLE(CM0_PIPELINED_RENDER_FLUSH_DISABLE));
3107bd48
DV		 3911
3912 ibx_init_clock_gating(dev);
3913}
3914
3915static void cpt_init_clock_gating(struct drm_device *dev)
3916{
3917 struct drm_i915_private *dev_priv = dev->dev_private;
3918 int pipe;
3f704fa2 3919 uint32_t val;
3107bd48
DV		 3920
3921 /*
3922 * On Ibex Peak and Cougar Point, we need to disable clock
3923 * gating for the panel power sequencer or it will fail to
3924 * start up when no ports are active.
3925 */
3926 I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE);
3927 I915_WRITE(SOUTH_CHICKEN2, I915_READ(SOUTH_CHICKEN2) |
3928 DPLS_EDP_PPS_FIX_DIS);
335c07b7
TI		 3929 /* The below fixes the weird display corruption, a few pixels shifted
3930 * downward, on (only) LVDS of some HP laptops with IVY.
3931 */
3f704fa2 3932 for_each_pipe(pipe) {
dc4bd2d1
PZ		 3933 val = I915_READ(TRANS_CHICKEN2(pipe));
3934 val |= TRANS_CHICKEN2_TIMING_OVERRIDE;
3935 val &= ~TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
41aa3448 3936 if (dev_priv->vbt.fdi_rx_polarity_inverted)
3f704fa2 3937 val |= TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
dc4bd2d1
PZ		 3938 val &= ~TRANS_CHICKEN2_FRAME_START_DELAY_MASK;
3939 val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_COUNTER;
3940 val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_MODESWITCH;
3f704fa2
PZ		 3941 I915_WRITE(TRANS_CHICKEN2(pipe), val);
3942 }
3107bd48
DV		 3943 /* WADP0ClockGatingDisable */
3944 for_each_pipe(pipe) {
3945 I915_WRITE(TRANS_CHICKEN1(pipe),
3946 TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
3947 }
6f1d69b0
ED		 3948}
3949
1d7aaa0c
DV		 3950static void gen6_check_mch_setup(struct drm_device *dev)
3951{
3952 struct drm_i915_private *dev_priv = dev->dev_private;
3953 uint32_t tmp;
3954
3955 tmp = I915_READ(MCH_SSKPD);
3956 if ((tmp & MCH_SSKPD_WM0_MASK) != MCH_SSKPD_WM0_VAL) {
3957 DRM_INFO("Wrong MCH_SSKPD value: 0x%08x\n", tmp);
3958 DRM_INFO("This can cause pipe underruns and display issues.\n");
3959 DRM_INFO("Please upgrade your BIOS to fix this.\n");
3960 }
3961}
3962
1fa61106 3963static void gen6_init_clock_gating(struct drm_device *dev)
6f1d69b0
ED		 3964{
3965 struct drm_i915_private *dev_priv = dev->dev_private;
3966 int pipe;
231e54f6 3967 uint32_t dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
6f1d69b0 3968
231e54f6 3969 I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate);
6f1d69b0
ED		 3970
3971 I915_WRITE(ILK_DISPLAY_CHICKEN2,
3972 I915_READ(ILK_DISPLAY_CHICKEN2) |
3973 ILK_ELPIN_409_SELECT);
3974
ecdb4eb7 3975 /* WaDisableHiZPlanesWhenMSAAEnabled:snb */
4283908e
DV		 3976 I915_WRITE(_3D_CHICKEN,
3977 _MASKED_BIT_ENABLE(_3D_CHICKEN_HIZ_PLANE_DISABLE_MSAA_4X_SNB));
3978
ecdb4eb7 3979 /* WaSetupGtModeTdRowDispatch:snb */
6547fbdb
DV		 3980 if (IS_SNB_GT1(dev))
3981 I915_WRITE(GEN6_GT_MODE,
3982 _MASKED_BIT_ENABLE(GEN6_TD_FOUR_ROW_DISPATCH_DISABLE));
3983
6f1d69b0
ED		 3984 I915_WRITE(WM3_LP_ILK, 0);
3985 I915_WRITE(WM2_LP_ILK, 0);
3986 I915_WRITE(WM1_LP_ILK, 0);
3987
6f1d69b0 3988 I915_WRITE(CACHE_MODE_0,
50743298 3989 _MASKED_BIT_DISABLE(CM0_STC_EVICT_DISABLE_LRA_SNB));
6f1d69b0
ED		 3990
3991 I915_WRITE(GEN6_UCGCTL1,
3992 I915_READ(GEN6_UCGCTL1) |
3993 GEN6_BLBUNIT_CLOCK_GATE_DISABLE |
3994 GEN6_CSUNIT_CLOCK_GATE_DISABLE);
3995
3996 /* According to the BSpec vol1g, bit 12 (RCPBUNIT) clock
3997 * gating disable must be set. Failure to set it results in
3998 * flickering pixels due to Z write ordering failures after
3999 * some amount of runtime in the Mesa "fire" demo, and Unigine
4000 * Sanctuary and Tropics, and apparently anything else with
4001 * alpha test or pixel discard.
4002 *
4003 * According to the spec, bit 11 (RCCUNIT) must also be set,
4004 * but we didn't debug actual testcases to find it out.
0f846f81 4005 *
ecdb4eb7
DL		 4006 * Also apply WaDisableVDSUnitClockGating:snb and
4007 * WaDisableRCPBUnitClockGating:snb.
6f1d69b0
ED		 4008 */
4009 I915_WRITE(GEN6_UCGCTL2,
0f846f81 4010 GEN7_VDSUNIT_CLOCK_GATE_DISABLE |
6f1d69b0
ED		 4011 GEN6_RCPBUNIT_CLOCK_GATE_DISABLE |
4012 GEN6_RCCUNIT_CLOCK_GATE_DISABLE);
4013
4014 /* Bspec says we need to always set all mask bits. */
26b6e44a
KG		 4015 I915_WRITE(_3D_CHICKEN3, (0xFFFF << 16) |
4016 _3D_CHICKEN3_SF_DISABLE_FASTCLIP_CULL);
6f1d69b0
ED		 4017
4018 /*
4019 * According to the spec the following bits should be
4020 * set in order to enable memory self-refresh and fbc:
4021 * The bit21 and bit22 of 0x42000
4022 * The bit21 and bit22 of 0x42004
4023 * The bit5 and bit7 of 0x42020
4024 * The bit14 of 0x70180
4025 * The bit14 of 0x71180
4026 */
4027 I915_WRITE(ILK_DISPLAY_CHICKEN1,
4028 I915_READ(ILK_DISPLAY_CHICKEN1) |
4029 ILK_FBCQ_DIS | ILK_PABSTRETCH_DIS);
4030 I915_WRITE(ILK_DISPLAY_CHICKEN2,
4031 I915_READ(ILK_DISPLAY_CHICKEN2) |
4032 ILK_DPARB_GATE | ILK_VSDPFD_FULL);
231e54f6
DL		 4033 I915_WRITE(ILK_DSPCLK_GATE_D,
4034 I915_READ(ILK_DSPCLK_GATE_D) |
4035 ILK_DPARBUNIT_CLOCK_GATE_ENABLE |
4036 ILK_DPFDUNIT_CLOCK_GATE_ENABLE);
6f1d69b0 4037
ecdb4eb7 4038 /* WaMbcDriverBootEnable:snb */
b4ae3f22
JB		 4039 I915_WRITE(GEN6_MBCTL, I915_READ(GEN6_MBCTL) |
4040 GEN6_MBCTL_ENABLE_BOOT_FETCH);
4041
6f1d69b0
ED		 4042 for_each_pipe(pipe) {
4043 I915_WRITE(DSPCNTR(pipe),
4044 I915_READ(DSPCNTR(pipe)) |
4045 DISPPLANE_TRICKLE_FEED_DISABLE);
4046 intel_flush_display_plane(dev_priv, pipe);
4047 }
f8f2ac9a
BW		 4048
4049 /* The default value should be 0x200 according to docs, but the two
4050 * platforms I checked have a 0 for this. (Maybe BIOS overrides?) */
4051 I915_WRITE(GEN6_GT_MODE, _MASKED_BIT_DISABLE(0xffff));
4052 I915_WRITE(GEN6_GT_MODE, _MASKED_BIT_ENABLE(GEN6_GT_MODE_HI));
3107bd48
DV		 4053
4054 cpt_init_clock_gating(dev);
1d7aaa0c
DV		 4055
4056 gen6_check_mch_setup(dev);
6f1d69b0
ED		 4057}
4058
4059static void gen7_setup_fixed_func_scheduler(struct drm_i915_private *dev_priv)
4060{
4061 uint32_t reg = I915_READ(GEN7_FF_THREAD_MODE);
4062
4063 reg &= ~GEN7_FF_SCHED_MASK;
4064 reg |= GEN7_FF_TS_SCHED_HW;
4065 reg |= GEN7_FF_VS_SCHED_HW;
4066 reg |= GEN7_FF_DS_SCHED_HW;
4067
41c0b3a8
BW		 4068 if (IS_HASWELL(dev_priv->dev))
4069 reg &= ~GEN7_FF_VS_REF_CNT_FFME;
4070
6f1d69b0
ED		 4071 I915_WRITE(GEN7_FF_THREAD_MODE, reg);
4072}
4073
17a303ec
PZ		 4074static void lpt_init_clock_gating(struct drm_device *dev)
4075{
4076 struct drm_i915_private *dev_priv = dev->dev_private;
4077
4078 /*
4079 * TODO: this bit should only be enabled when really needed, then
4080 * disabled when not needed anymore in order to save power.
4081 */
4082 if (dev_priv->pch_id == INTEL_PCH_LPT_LP_DEVICE_ID_TYPE)
4083 I915_WRITE(SOUTH_DSPCLK_GATE_D,
4084 I915_READ(SOUTH_DSPCLK_GATE_D) |
4085 PCH_LP_PARTITION_LEVEL_DISABLE);
4086}
4087
7d708ee4
ID		 4088static void lpt_suspend_hw(struct drm_device *dev)
4089{
4090 struct drm_i915_private *dev_priv = dev->dev_private;
4091
4092 if (dev_priv->pch_id == INTEL_PCH_LPT_LP_DEVICE_ID_TYPE) {
4093 uint32_t val = I915_READ(SOUTH_DSPCLK_GATE_D);
4094
4095 val &= ~PCH_LP_PARTITION_LEVEL_DISABLE;
4096 I915_WRITE(SOUTH_DSPCLK_GATE_D, val);
4097 }
4098}
4099
cad2a2d7
ED		 4100static void haswell_init_clock_gating(struct drm_device *dev)
4101{
4102 struct drm_i915_private *dev_priv = dev->dev_private;
4103 int pipe;
cad2a2d7
ED		 4104
4105 I915_WRITE(WM3_LP_ILK, 0);
4106 I915_WRITE(WM2_LP_ILK, 0);
4107 I915_WRITE(WM1_LP_ILK, 0);
4108
4109 /* According to the spec, bit 13 (RCZUNIT) must be set on IVB.
ecdb4eb7 4110 * This implements the WaDisableRCZUnitClockGating:hsw workaround.
cad2a2d7
ED		 4111 */
4112 I915_WRITE(GEN6_UCGCTL2, GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
4113
ecdb4eb7 4114 /* Apply the WaDisableRHWOOptimizationForRenderHang:hsw workaround. */
cad2a2d7
ED		 4115 I915_WRITE(GEN7_COMMON_SLICE_CHICKEN1,
4116 GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC);
4117
ecdb4eb7 4118 /* WaApplyL3ControlAndL3ChickenMode:hsw */
cad2a2d7
ED		 4119 I915_WRITE(GEN7_L3CNTLREG1,
4120 GEN7_WA_FOR_GEN7_L3_CONTROL);
4121 I915_WRITE(GEN7_L3_CHICKEN_MODE_REGISTER,
4122 GEN7_WA_L3_CHICKEN_MODE);
4123
ecdb4eb7 4124 /* This is required by WaCatErrorRejectionIssue:hsw */
cad2a2d7
ED		 4125 I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
4126 I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
4127 GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
4128
4129 for_each_pipe(pipe) {
4130 I915_WRITE(DSPCNTR(pipe),
4131 I915_READ(DSPCNTR(pipe)) |
4132 DISPPLANE_TRICKLE_FEED_DISABLE);
4133 intel_flush_display_plane(dev_priv, pipe);
4134 }
4135
ecdb4eb7 4136 /* WaVSRefCountFullforceMissDisable:hsw */
cad2a2d7
ED		 4137 gen7_setup_fixed_func_scheduler(dev_priv);
4138
ecdb4eb7 4139 /* WaDisable4x2SubspanOptimization:hsw */
cad2a2d7
ED		 4140 I915_WRITE(CACHE_MODE_1,
4141 _MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
1544d9d5 4142
ecdb4eb7 4143 /* WaMbcDriverBootEnable:hsw */
b3bf0766
PZ		 4144 I915_WRITE(GEN6_MBCTL, I915_READ(GEN6_MBCTL) |
4145 GEN6_MBCTL_ENABLE_BOOT_FETCH);
4146
ecdb4eb7 4147 /* WaSwitchSolVfFArbitrationPriority:hsw */
e3dff585
BW		 4148 I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) | HSW_ECOCHK_ARB_PRIO_SOL);
4149
1544d9d5
ED		 4150 /* XXX: This is a workaround for early silicon revisions and should be
4151 * removed later.
4152 */
4153 I915_WRITE(WM_DBG,
4154 I915_READ(WM_DBG) |
4155 WM_DBG_DISALLOW_MULTIPLE_LP |
4156 WM_DBG_DISALLOW_SPRITE |
4157 WM_DBG_DISALLOW_MAXFIFO);
4158
17a303ec 4159 lpt_init_clock_gating(dev);
cad2a2d7
ED		 4160}
4161
1fa61106 4162static void ivybridge_init_clock_gating(struct drm_device *dev)
6f1d69b0
ED		 4163{
4164 struct drm_i915_private *dev_priv = dev->dev_private;
4165 int pipe;
20848223 4166 uint32_t snpcr;
6f1d69b0 4167
6f1d69b0
ED		 4168 I915_WRITE(WM3_LP_ILK, 0);
4169 I915_WRITE(WM2_LP_ILK, 0);
4170 I915_WRITE(WM1_LP_ILK, 0);
4171
231e54f6 4172 I915_WRITE(ILK_DSPCLK_GATE_D, ILK_VRHUNIT_CLOCK_GATE_DISABLE);
6f1d69b0 4173
ecdb4eb7 4174 /* WaDisableEarlyCull:ivb */
87f8020e
JB		 4175 I915_WRITE(_3D_CHICKEN3,
4176 _MASKED_BIT_ENABLE(_3D_CHICKEN_SF_DISABLE_OBJEND_CULL));
4177
ecdb4eb7 4178 /* WaDisableBackToBackFlipFix:ivb */
6f1d69b0
ED		 4179 I915_WRITE(IVB_CHICKEN3,
4180 CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
4181 CHICKEN3_DGMG_DONE_FIX_DISABLE);
4182
ecdb4eb7 4183 /* WaDisablePSDDualDispatchEnable:ivb */
12f3382b
JB		 4184 if (IS_IVB_GT1(dev))
4185 I915_WRITE(GEN7_HALF_SLICE_CHICKEN1,
4186 _MASKED_BIT_ENABLE(GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE));
4187 else
4188 I915_WRITE(GEN7_HALF_SLICE_CHICKEN1_GT2,
4189 _MASKED_BIT_ENABLE(GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE));
4190
ecdb4eb7 4191 /* Apply the WaDisableRHWOOptimizationForRenderHang:ivb workaround. */
6f1d69b0
ED		 4192 I915_WRITE(GEN7_COMMON_SLICE_CHICKEN1,
4193 GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC);
4194
ecdb4eb7 4195 /* WaApplyL3ControlAndL3ChickenMode:ivb */
6f1d69b0
ED		 4196 I915_WRITE(GEN7_L3CNTLREG1,
4197 GEN7_WA_FOR_GEN7_L3_CONTROL);
4198 I915_WRITE(GEN7_L3_CHICKEN_MODE_REGISTER,
8ab43976
JB		 4199 GEN7_WA_L3_CHICKEN_MODE);
4200 if (IS_IVB_GT1(dev))
4201 I915_WRITE(GEN7_ROW_CHICKEN2,
4202 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
4203 else
4204 I915_WRITE(GEN7_ROW_CHICKEN2_GT2,
4205 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
4206
6f1d69b0 4207
ecdb4eb7 4208 /* WaForceL3Serialization:ivb */
61939d97
JB		 4209 I915_WRITE(GEN7_L3SQCREG4, I915_READ(GEN7_L3SQCREG4) &
4210 ~L3SQ_URB_READ_CAM_MATCH_DISABLE);
4211
0f846f81
JB		 4212 /* According to the BSpec vol1g, bit 12 (RCPBUNIT) clock
4213 * gating disable must be set. Failure to set it results in
4214 * flickering pixels due to Z write ordering failures after
4215 * some amount of runtime in the Mesa "fire" demo, and Unigine
4216 * Sanctuary and Tropics, and apparently anything else with
4217 * alpha test or pixel discard.
4218 *
4219 * According to the spec, bit 11 (RCCUNIT) must also be set,
4220 * but we didn't debug actual testcases to find it out.
4221 *
4222 * According to the spec, bit 13 (RCZUNIT) must be set on IVB.
ecdb4eb7 4223 * This implements the WaDisableRCZUnitClockGating:ivb workaround.
0f846f81
JB		 4224 */
4225 I915_WRITE(GEN6_UCGCTL2,
4226 GEN6_RCZUNIT_CLOCK_GATE_DISABLE |
4227 GEN6_RCCUNIT_CLOCK_GATE_DISABLE);
4228
ecdb4eb7 4229 /* This is required by WaCatErrorRejectionIssue:ivb */
6f1d69b0
ED		 4230 I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
4231 I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
4232 GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
4233
4234 for_each_pipe(pipe) {
4235 I915_WRITE(DSPCNTR(pipe),
4236 I915_READ(DSPCNTR(pipe)) |
4237 DISPPLANE_TRICKLE_FEED_DISABLE);
4238 intel_flush_display_plane(dev_priv, pipe);
4239 }
4240
ecdb4eb7 4241 /* WaMbcDriverBootEnable:ivb */
b4ae3f22
JB		 4242 I915_WRITE(GEN6_MBCTL, I915_READ(GEN6_MBCTL) |
4243 GEN6_MBCTL_ENABLE_BOOT_FETCH);
4244
ecdb4eb7 4245 /* WaVSRefCountFullforceMissDisable:ivb */
6f1d69b0 4246 gen7_setup_fixed_func_scheduler(dev_priv);
97e1930f 4247
ecdb4eb7 4248 /* WaDisable4x2SubspanOptimization:ivb */
97e1930f
DV		 4249 I915_WRITE(CACHE_MODE_1,
4250 _MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
20848223
BW		 4251
4252 snpcr = I915_READ(GEN6_MBCUNIT_SNPCR);
4253 snpcr &= ~GEN6_MBC_SNPCR_MASK;
4254 snpcr |= GEN6_MBC_SNPCR_MED;
4255 I915_WRITE(GEN6_MBCUNIT_SNPCR, snpcr);
3107bd48 4256
ab5c608b
BW		 4257 if (!HAS_PCH_NOP(dev))
4258 cpt_init_clock_gating(dev);
1d7aaa0c
DV		 4259
4260 gen6_check_mch_setup(dev);
6f1d69b0
ED		 4261}
4262
1fa61106 4263static void valleyview_init_clock_gating(struct drm_device *dev)
6f1d69b0
ED		 4264{
4265 struct drm_i915_private *dev_priv = dev->dev_private;
4266 int pipe;
6f1d69b0
ED		 4267
4268 I915_WRITE(WM3_LP_ILK, 0);
4269 I915_WRITE(WM2_LP_ILK, 0);
4270 I915_WRITE(WM1_LP_ILK, 0);
4271
231e54f6 4272 I915_WRITE(ILK_DSPCLK_GATE_D, ILK_VRHUNIT_CLOCK_GATE_DISABLE);
6f1d69b0 4273
ecdb4eb7 4274 /* WaDisableEarlyCull:vlv */
87f8020e
JB		 4275 I915_WRITE(_3D_CHICKEN3,
4276 _MASKED_BIT_ENABLE(_3D_CHICKEN_SF_DISABLE_OBJEND_CULL));
4277
ecdb4eb7 4278 /* WaDisableBackToBackFlipFix:vlv */
6f1d69b0
ED		 4279 I915_WRITE(IVB_CHICKEN3,
4280 CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
4281 CHICKEN3_DGMG_DONE_FIX_DISABLE);
4282
ecdb4eb7 4283 /* WaDisablePSDDualDispatchEnable:vlv */
12f3382b 4284 I915_WRITE(GEN7_HALF_SLICE_CHICKEN1,
d3bc0303
JB		 4285 _MASKED_BIT_ENABLE(GEN7_MAX_PS_THREAD_DEP |
4286 GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE));
12f3382b 4287
ecdb4eb7 4288 /* Apply the WaDisableRHWOOptimizationForRenderHang:vlv workaround. */
6f1d69b0
ED		 4289 I915_WRITE(GEN7_COMMON_SLICE_CHICKEN1,
4290 GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC);
4291
ecdb4eb7 4292 /* WaApplyL3ControlAndL3ChickenMode:vlv */
d0cf5ead 4293 I915_WRITE(GEN7_L3CNTLREG1, I915_READ(GEN7_L3CNTLREG1) | GEN7_L3AGDIS);
6f1d69b0
ED		 4294 I915_WRITE(GEN7_L3_CHICKEN_MODE_REGISTER, GEN7_WA_L3_CHICKEN_MODE);
4295
ecdb4eb7 4296 /* WaForceL3Serialization:vlv */
61939d97
JB		 4297 I915_WRITE(GEN7_L3SQCREG4, I915_READ(GEN7_L3SQCREG4) &
4298 ~L3SQ_URB_READ_CAM_MATCH_DISABLE);
4299
ecdb4eb7 4300 /* WaDisableDopClockGating:vlv */
8ab43976
JB		 4301 I915_WRITE(GEN7_ROW_CHICKEN2,
4302 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
4303
ecdb4eb7 4304 /* WaForceL3Serialization:vlv */
5c9664d7
JB		 4305 I915_WRITE(GEN7_L3SQCREG4, I915_READ(GEN7_L3SQCREG4) &
4306 ~L3SQ_URB_READ_CAM_MATCH_DISABLE);
4307
ecdb4eb7 4308 /* This is required by WaCatErrorRejectionIssue:vlv */
6f1d69b0
ED		 4309 I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
4310 I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
4311 GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
4312
ecdb4eb7 4313 /* WaMbcDriverBootEnable:vlv */
b4ae3f22
JB		 4314 I915_WRITE(GEN6_MBCTL, I915_READ(GEN6_MBCTL) |
4315 GEN6_MBCTL_ENABLE_BOOT_FETCH);
4316
0f846f81
JB		 4317
4318 /* According to the BSpec vol1g, bit 12 (RCPBUNIT) clock
4319 * gating disable must be set. Failure to set it results in
4320 * flickering pixels due to Z write ordering failures after
4321 * some amount of runtime in the Mesa "fire" demo, and Unigine
4322 * Sanctuary and Tropics, and apparently anything else with
4323 * alpha test or pixel discard.
4324 *
4325 * According to the spec, bit 11 (RCCUNIT) must also be set,
4326 * but we didn't debug actual testcases to find it out.
4327 *
4328 * According to the spec, bit 13 (RCZUNIT) must be set on IVB.
ecdb4eb7 4329 * This implements the WaDisableRCZUnitClockGating:vlv workaround.
0f846f81 4330 *
ecdb4eb7
DL		 4331 * Also apply WaDisableVDSUnitClockGating:vlv and
4332 * WaDisableRCPBUnitClockGating:vlv.
0f846f81
JB		 4333 */
4334 I915_WRITE(GEN6_UCGCTL2,
4335 GEN7_VDSUNIT_CLOCK_GATE_DISABLE |
6edaa7fc 4336 GEN7_TDLUNIT_CLOCK_GATE_DISABLE |
0f846f81
JB		 4337 GEN6_RCZUNIT_CLOCK_GATE_DISABLE |
4338 GEN6_RCPBUNIT_CLOCK_GATE_DISABLE |
4339 GEN6_RCCUNIT_CLOCK_GATE_DISABLE);
4340
e3f33d46
JB		 4341 I915_WRITE(GEN7_UCGCTL4, GEN7_L3BANK2X_CLOCK_GATE_DISABLE);
4342
6f1d69b0
ED		 4343 for_each_pipe(pipe) {
4344 I915_WRITE(DSPCNTR(pipe),
4345 I915_READ(DSPCNTR(pipe)) |
4346 DISPPLANE_TRICKLE_FEED_DISABLE);
4347 intel_flush_display_plane(dev_priv, pipe);
4348 }
4349
6b26c86d
DV		 4350 I915_WRITE(CACHE_MODE_1,
4351 _MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
7983117f 4352
2d809570 4353 /*
ecdb4eb7 4354 * WaDisableVLVClockGating_VBIIssue:vlv
2d809570
JB		 4355 * Disable clock gating on th GCFG unit to prevent a delay
4356 * in the reporting of vblank events.
4357 */
4e8c84a5
JB		 4358 I915_WRITE(VLV_GUNIT_CLOCK_GATE, 0xffffffff);
4359
4360 /* Conservative clock gating settings for now */
4361 I915_WRITE(0x9400, 0xffffffff);
4362 I915_WRITE(0x9404, 0xffffffff);
4363 I915_WRITE(0x9408, 0xffffffff);
4364 I915_WRITE(0x940c, 0xffffffff);
4365 I915_WRITE(0x9410, 0xffffffff);
4366 I915_WRITE(0x9414, 0xffffffff);
4367 I915_WRITE(0x9418, 0xffffffff);
6f1d69b0
ED		 4368}
4369
1fa61106 4370static void g4x_init_clock_gating(struct drm_device *dev)
6f1d69b0
ED		 4371{
4372 struct drm_i915_private *dev_priv = dev->dev_private;
4373 uint32_t dspclk_gate;
4374
4375 I915_WRITE(RENCLK_GATE_D1, 0);
4376 I915_WRITE(RENCLK_GATE_D2, VF_UNIT_CLOCK_GATE_DISABLE |
4377 GS_UNIT_CLOCK_GATE_DISABLE |
4378 CL_UNIT_CLOCK_GATE_DISABLE);
4379 I915_WRITE(RAMCLK_GATE_D, 0);
4380 dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE |
4381 OVRUNIT_CLOCK_GATE_DISABLE |
4382 OVCUNIT_CLOCK_GATE_DISABLE;
4383 if (IS_GM45(dev))
4384 dspclk_gate |= DSSUNIT_CLOCK_GATE_DISABLE;
4385 I915_WRITE(DSPCLK_GATE_D, dspclk_gate);
4358a374
DV		 4386
4387 /* WaDisableRenderCachePipelinedFlush */
4388 I915_WRITE(CACHE_MODE_0,
4389 _MASKED_BIT_ENABLE(CM0_PIPELINED_RENDER_FLUSH_DISABLE));
6f1d69b0
ED		 4390}
4391
1fa61106 4392static void crestline_init_clock_gating(struct drm_device *dev)
6f1d69b0
ED		 4393{
4394 struct drm_i915_private *dev_priv = dev->dev_private;
4395
4396 I915_WRITE(RENCLK_GATE_D1, I965_RCC_CLOCK_GATE_DISABLE);
4397 I915_WRITE(RENCLK_GATE_D2, 0);
4398 I915_WRITE(DSPCLK_GATE_D, 0);
4399 I915_WRITE(RAMCLK_GATE_D, 0);
4400 I915_WRITE16(DEUC, 0);
4401}
4402
1fa61106 4403static void broadwater_init_clock_gating(struct drm_device *dev)
6f1d69b0
ED		 4404{
4405 struct drm_i915_private *dev_priv = dev->dev_private;
4406
4407 I915_WRITE(RENCLK_GATE_D1, I965_RCZ_CLOCK_GATE_DISABLE |
4408 I965_RCC_CLOCK_GATE_DISABLE |
4409 I965_RCPB_CLOCK_GATE_DISABLE |
4410 I965_ISC_CLOCK_GATE_DISABLE |
4411 I965_FBC_CLOCK_GATE_DISABLE);
4412 I915_WRITE(RENCLK_GATE_D2, 0);
4413}
4414
1fa61106 4415static void gen3_init_clock_gating(struct drm_device *dev)
6f1d69b0
ED		 4416{
4417 struct drm_i915_private *dev_priv = dev->dev_private;
4418 u32 dstate = I915_READ(D_STATE);
4419
4420 dstate |= DSTATE_PLL_D3_OFF | DSTATE_GFX_CLOCK_GATING |
4421 DSTATE_DOT_CLOCK_GATING;
4422 I915_WRITE(D_STATE, dstate);
13a86b85
CW		 4423
4424 if (IS_PINEVIEW(dev))
4425 I915_WRITE(ECOSKPD, _MASKED_BIT_ENABLE(ECO_GATING_CX_ONLY));
974a3b0f
DV		 4426
4427 /* IIR "flip pending" means done if this bit is set */
4428 I915_WRITE(ECOSKPD, _MASKED_BIT_DISABLE(ECO_FLIP_DONE));
6f1d69b0
ED		 4429}
4430
1fa61106 4431static void i85x_init_clock_gating(struct drm_device *dev)
6f1d69b0
ED		 4432{
4433 struct drm_i915_private *dev_priv = dev->dev_private;
4434
4435 I915_WRITE(RENCLK_GATE_D1, SV_CLOCK_GATE_DISABLE);
4436}
4437
1fa61106 4438static void i830_init_clock_gating(struct drm_device *dev)
6f1d69b0
ED		 4439{
4440 struct drm_i915_private *dev_priv = dev->dev_private;
4441
4442 I915_WRITE(DSPCLK_GATE_D, OVRUNIT_CLOCK_GATE_DISABLE);
4443}
4444
6f1d69b0
ED		 4445void intel_init_clock_gating(struct drm_device *dev)
4446{
4447 struct drm_i915_private *dev_priv = dev->dev_private;
4448
4449 dev_priv->display.init_clock_gating(dev);
6f1d69b0
ED		 4450}
4451
7d708ee4
ID		 4452void intel_suspend_hw(struct drm_device *dev)
4453{
4454 if (HAS_PCH_LPT(dev))
4455 lpt_suspend_hw(dev);
4456}
4457
15d199ea
PZ		 4458/**
4459 * We should only use the power well if we explicitly asked the hardware to
4460 * enable it, so check if it's enabled and also check if we've requested it to
4461 * be enabled.
4462 */
b97186f0
PZ		 4463bool intel_display_power_enabled(struct drm_device *dev,
4464 enum intel_display_power_domain domain)
15d199ea
PZ		 4465{
4466 struct drm_i915_private *dev_priv = dev->dev_private;
4467
b97186f0
PZ		 4468 if (!HAS_POWER_WELL(dev))
4469 return true;
4470
4471 switch (domain) {
4472 case POWER_DOMAIN_PIPE_A:
4473 case POWER_DOMAIN_TRANSCODER_EDP:
4474 return true;
4475 case POWER_DOMAIN_PIPE_B:
4476 case POWER_DOMAIN_PIPE_C:
4477 case POWER_DOMAIN_PIPE_A_PANEL_FITTER:
4478 case POWER_DOMAIN_PIPE_B_PANEL_FITTER:
4479 case POWER_DOMAIN_PIPE_C_PANEL_FITTER:
4480 case POWER_DOMAIN_TRANSCODER_A:
4481 case POWER_DOMAIN_TRANSCODER_B:
4482 case POWER_DOMAIN_TRANSCODER_C:
15d199ea
PZ		 4483 return I915_READ(HSW_PWR_WELL_DRIVER) ==
4484 (HSW_PWR_WELL_ENABLE | HSW_PWR_WELL_STATE);
b97186f0
PZ		 4485 default:
4486 BUG();
4487 }
15d199ea
PZ		 4488}
4489
cb10799c 4490void intel_set_power_well(struct drm_device *dev, bool enable)
d0d3e513
ED		 4491{
4492 struct drm_i915_private *dev_priv = dev->dev_private;
fa42e23c
PZ		 4493 bool is_enabled, enable_requested;
4494 uint32_t tmp;
d0d3e513 4495
86d52df6 4496 if (!HAS_POWER_WELL(dev))
d0d3e513
ED		 4497 return;
4498
2124b72e
PZ		 4499 if (!i915_disable_power_well && !enable)
4500 return;
4501
fa42e23c
PZ		 4502 tmp = I915_READ(HSW_PWR_WELL_DRIVER);
4503 is_enabled = tmp & HSW_PWR_WELL_STATE;
4504 enable_requested = tmp & HSW_PWR_WELL_ENABLE;
d0d3e513 4505
fa42e23c
PZ		 4506 if (enable) {
4507 if (!enable_requested)
4508 I915_WRITE(HSW_PWR_WELL_DRIVER, HSW_PWR_WELL_ENABLE);
d0d3e513 4509
fa42e23c
PZ		 4510 if (!is_enabled) {
4511 DRM_DEBUG_KMS("Enabling power well\n");
4512 if (wait_for((I915_READ(HSW_PWR_WELL_DRIVER) &
4513 HSW_PWR_WELL_STATE), 20))
4514 DRM_ERROR("Timeout enabling power well\n");
4515 }
4516 } else {
4517 if (enable_requested) {
4518 I915_WRITE(HSW_PWR_WELL_DRIVER, 0);
4519 DRM_DEBUG_KMS("Requesting to disable the power well\n");
d0d3e513
ED		 4520 }
4521 }
fa42e23c 4522}
d0d3e513 4523
fa42e23c
PZ		 4524/*
4525 * Starting with Haswell, we have a "Power Down Well" that can be turned off
4526 * when not needed anymore. We have 4 registers that can request the power well
4527 * to be enabled, and it will only be disabled if none of the registers is
4528 * requesting it to be enabled.
d0d3e513 4529 */
fa42e23c 4530void intel_init_power_well(struct drm_device *dev)
d0d3e513
ED		 4531{
4532 struct drm_i915_private *dev_priv = dev->dev_private;
d0d3e513 4533
86d52df6 4534 if (!HAS_POWER_WELL(dev))
d0d3e513
ED		 4535 return;
4536
fa42e23c
PZ		 4537 /* For now, we need the power well to be always enabled. */
4538 intel_set_power_well(dev, true);
d0d3e513 4539
fa42e23c
PZ		 4540 /* We're taking over the BIOS, so clear any requests made by it since
4541 * the driver is in charge now. */
4542 if (I915_READ(HSW_PWR_WELL_BIOS) & HSW_PWR_WELL_ENABLE)
4543 I915_WRITE(HSW_PWR_WELL_BIOS, 0);
d0d3e513
ED		 4544}
4545
1fa61106
ED		 4546/* Set up chip specific power management-related functions */
4547void intel_init_pm(struct drm_device *dev)
4548{
4549 struct drm_i915_private *dev_priv = dev->dev_private;
4550
4551 if (I915_HAS_FBC(dev)) {
4552 if (HAS_PCH_SPLIT(dev)) {
4553 dev_priv->display.fbc_enabled = ironlake_fbc_enabled;
891348b2 4554 if (IS_IVYBRIDGE(dev) || IS_HASWELL(dev))
abe959c7
RV		 4555 dev_priv->display.enable_fbc =
4556 gen7_enable_fbc;
4557 else
4558 dev_priv->display.enable_fbc =
4559 ironlake_enable_fbc;
1fa61106
ED		 4560 dev_priv->display.disable_fbc = ironlake_disable_fbc;
4561 } else if (IS_GM45(dev)) {
4562 dev_priv->display.fbc_enabled = g4x_fbc_enabled;
4563 dev_priv->display.enable_fbc = g4x_enable_fbc;
4564 dev_priv->display.disable_fbc = g4x_disable_fbc;
4565 } else if (IS_CRESTLINE(dev)) {
4566 dev_priv->display.fbc_enabled = i8xx_fbc_enabled;
4567 dev_priv->display.enable_fbc = i8xx_enable_fbc;
4568 dev_priv->display.disable_fbc = i8xx_disable_fbc;
4569 }
4570 /* 855GM needs testing */
4571 }
4572
c921aba8
DV		 4573 /* For cxsr */
4574 if (IS_PINEVIEW(dev))
4575 i915_pineview_get_mem_freq(dev);
4576 else if (IS_GEN5(dev))
4577 i915_ironlake_get_mem_freq(dev);
4578
1fa61106
ED		 4579 /* For FIFO watermark updates */
4580 if (HAS_PCH_SPLIT(dev)) {
1fa61106
ED		 4581 if (IS_GEN5(dev)) {
4582 if (I915_READ(MLTR_ILK) & ILK_SRLT_MASK)
4583 dev_priv->display.update_wm = ironlake_update_wm;
4584 else {
4585 DRM_DEBUG_KMS("Failed to get proper latency. "
4586 "Disable CxSR\n");
4587 dev_priv->display.update_wm = NULL;
4588 }
4589 dev_priv->display.init_clock_gating = ironlake_init_clock_gating;
4590 } else if (IS_GEN6(dev)) {
4591 if (SNB_READ_WM0_LATENCY()) {
4592 dev_priv->display.update_wm = sandybridge_update_wm;
4593 dev_priv->display.update_sprite_wm = sandybridge_update_sprite_wm;
4594 } else {
4595 DRM_DEBUG_KMS("Failed to read display plane latency. "
4596 "Disable CxSR\n");
4597 dev_priv->display.update_wm = NULL;
4598 }
4599 dev_priv->display.init_clock_gating = gen6_init_clock_gating;
4600 } else if (IS_IVYBRIDGE(dev)) {
1fa61106 4601 if (SNB_READ_WM0_LATENCY()) {
c43d0188 4602 dev_priv->display.update_wm = ivybridge_update_wm;
1fa61106
ED		 4603 dev_priv->display.update_sprite_wm = sandybridge_update_sprite_wm;
4604 } else {
4605 DRM_DEBUG_KMS("Failed to read display plane latency. "
4606 "Disable CxSR\n");
4607 dev_priv->display.update_wm = NULL;
4608 }
4609 dev_priv->display.init_clock_gating = ivybridge_init_clock_gating;
6b8a5eeb
ED		 4610 } else if (IS_HASWELL(dev)) {
4611 if (SNB_READ_WM0_LATENCY()) {
4612 dev_priv->display.update_wm = sandybridge_update_wm;
4613 dev_priv->display.update_sprite_wm = sandybridge_update_sprite_wm;
1f8eeabf 4614 dev_priv->display.update_linetime_wm = haswell_update_linetime_wm;
6b8a5eeb
ED		 4615 } else {
4616 DRM_DEBUG_KMS("Failed to read display plane latency. "
4617 "Disable CxSR\n");
4618 dev_priv->display.update_wm = NULL;
4619 }
cad2a2d7 4620 dev_priv->display.init_clock_gating = haswell_init_clock_gating;
1fa61106
ED		 4621 } else
4622 dev_priv->display.update_wm = NULL;
4623 } else if (IS_VALLEYVIEW(dev)) {
4624 dev_priv->display.update_wm = valleyview_update_wm;
4625 dev_priv->display.init_clock_gating =
4626 valleyview_init_clock_gating;
1fa61106
ED		 4627 } else if (IS_PINEVIEW(dev)) {
4628 if (!intel_get_cxsr_latency(IS_PINEVIEW_G(dev),
4629 dev_priv->is_ddr3,
4630 dev_priv->fsb_freq,
4631 dev_priv->mem_freq)) {
4632 DRM_INFO("failed to find known CxSR latency "
4633 "(found ddr%s fsb freq %d, mem freq %d), "
4634 "disabling CxSR\n",
4635 (dev_priv->is_ddr3 == 1) ? "3" : "2",
4636 dev_priv->fsb_freq, dev_priv->mem_freq);
4637 /* Disable CxSR and never update its watermark again */
4638 pineview_disable_cxsr(dev);
4639 dev_priv->display.update_wm = NULL;
4640 } else
4641 dev_priv->display.update_wm = pineview_update_wm;
4642 dev_priv->display.init_clock_gating = gen3_init_clock_gating;
4643 } else if (IS_G4X(dev)) {
4644 dev_priv->display.update_wm = g4x_update_wm;
4645 dev_priv->display.init_clock_gating = g4x_init_clock_gating;
4646 } else if (IS_GEN4(dev)) {
4647 dev_priv->display.update_wm = i965_update_wm;
4648 if (IS_CRESTLINE(dev))
4649 dev_priv->display.init_clock_gating = crestline_init_clock_gating;
4650 else if (IS_BROADWATER(dev))
4651 dev_priv->display.init_clock_gating = broadwater_init_clock_gating;
4652 } else if (IS_GEN3(dev)) {
4653 dev_priv->display.update_wm = i9xx_update_wm;
4654 dev_priv->display.get_fifo_size = i9xx_get_fifo_size;
4655 dev_priv->display.init_clock_gating = gen3_init_clock_gating;
4656 } else if (IS_I865G(dev)) {
4657 dev_priv->display.update_wm = i830_update_wm;
4658 dev_priv->display.init_clock_gating = i85x_init_clock_gating;
4659 dev_priv->display.get_fifo_size = i830_get_fifo_size;
4660 } else if (IS_I85X(dev)) {
4661 dev_priv->display.update_wm = i9xx_update_wm;
4662 dev_priv->display.get_fifo_size = i85x_get_fifo_size;
4663 dev_priv->display.init_clock_gating = i85x_init_clock_gating;
4664 } else {
4665 dev_priv->display.update_wm = i830_update_wm;
4666 dev_priv->display.init_clock_gating = i830_init_clock_gating;
4667 if (IS_845G(dev))
4668 dev_priv->display.get_fifo_size = i845_get_fifo_size;
4669 else
4670 dev_priv->display.get_fifo_size = i830_get_fifo_size;
4671 }
4672}
4673
6590190d
ED		 4674static void __gen6_gt_wait_for_thread_c0(struct drm_i915_private *dev_priv)
4675{
4676 u32 gt_thread_status_mask;
4677
4678 if (IS_HASWELL(dev_priv->dev))
4679 gt_thread_status_mask = GEN6_GT_THREAD_STATUS_CORE_MASK_HSW;
4680 else
4681 gt_thread_status_mask = GEN6_GT_THREAD_STATUS_CORE_MASK;
4682
4683 /* w/a for a sporadic read returning 0 by waiting for the GT
4684 * thread to wake up.
4685 */
4686 if (wait_for_atomic_us((I915_READ_NOTRACE(GEN6_GT_THREAD_STATUS_REG) & gt_thread_status_mask) == 0, 500))
4687 DRM_ERROR("GT thread status wait timed out\n");
4688}
4689
16995a9f
CW		 4690static void __gen6_gt_force_wake_reset(struct drm_i915_private *dev_priv)
4691{
4692 I915_WRITE_NOTRACE(FORCEWAKE, 0);
4693 POSTING_READ(ECOBUS); /* something from same cacheline, but !FORCEWAKE */
4694}
4695
6590190d
ED		 4696static void __gen6_gt_force_wake_get(struct drm_i915_private *dev_priv)
4697{
ebd37ce1 4698 if (wait_for_atomic((I915_READ_NOTRACE(FORCEWAKE_ACK) & 1) == 0,
057d3860 4699 FORCEWAKE_ACK_TIMEOUT_MS))
8a038fd6 4700 DRM_ERROR("Timed out waiting for forcewake old ack to clear.\n");
6590190d 4701
30771e16 4702 I915_WRITE_NOTRACE(FORCEWAKE, 1);
8dee3eea 4703 POSTING_READ(ECOBUS); /* something from same cacheline, but !FORCEWAKE */
6590190d 4704
ebd37ce1 4705 if (wait_for_atomic((I915_READ_NOTRACE(FORCEWAKE_ACK) & 1),
057d3860 4706 FORCEWAKE_ACK_TIMEOUT_MS))
8a038fd6 4707 DRM_ERROR("Timed out waiting for forcewake to ack request.\n");
6590190d 4708
8693a824 4709 /* WaRsForcewakeWaitTC0:snb */
6590190d
ED		 4710 __gen6_gt_wait_for_thread_c0(dev_priv);
4711}
4712
16995a9f
CW		 4713static void __gen6_gt_force_wake_mt_reset(struct drm_i915_private *dev_priv)
4714{
4715 I915_WRITE_NOTRACE(FORCEWAKE_MT, _MASKED_BIT_DISABLE(0xffff));
b5144075
JN		 4716 /* something from same cacheline, but !FORCEWAKE_MT */
4717 POSTING_READ(ECOBUS);
16995a9f
CW		 4718}
4719
6590190d
ED		 4720static void __gen6_gt_force_wake_mt_get(struct drm_i915_private *dev_priv)
4721{
4722 u32 forcewake_ack;
4723
4724 if (IS_HASWELL(dev_priv->dev))
4725 forcewake_ack = FORCEWAKE_ACK_HSW;
4726 else
4727 forcewake_ack = FORCEWAKE_MT_ACK;
4728
83983c8b 4729 if (wait_for_atomic((I915_READ_NOTRACE(forcewake_ack) & FORCEWAKE_KERNEL) == 0,
057d3860 4730 FORCEWAKE_ACK_TIMEOUT_MS))
8a038fd6 4731 DRM_ERROR("Timed out waiting for forcewake old ack to clear.\n");
6590190d 4732
c5836c27 4733 I915_WRITE_NOTRACE(FORCEWAKE_MT, _MASKED_BIT_ENABLE(FORCEWAKE_KERNEL));
b5144075
JN		 4734 /* something from same cacheline, but !FORCEWAKE_MT */
4735 POSTING_READ(ECOBUS);
6590190d 4736
83983c8b 4737 if (wait_for_atomic((I915_READ_NOTRACE(forcewake_ack) & FORCEWAKE_KERNEL),
057d3860 4738 FORCEWAKE_ACK_TIMEOUT_MS))
8a038fd6 4739 DRM_ERROR("Timed out waiting for forcewake to ack request.\n");
6590190d 4740
8693a824 4741 /* WaRsForcewakeWaitTC0:ivb,hsw */
6590190d
ED		 4742 __gen6_gt_wait_for_thread_c0(dev_priv);
4743}
4744
4745/*
4746 * Generally this is called implicitly by the register read function. However,
4747 * if some sequence requires the GT to not power down then this function should
4748 * be called at the beginning of the sequence followed by a call to
4749 * gen6_gt_force_wake_put() at the end of the sequence.
4750 */
4751void gen6_gt_force_wake_get(struct drm_i915_private *dev_priv)
4752{
4753 unsigned long irqflags;
4754
4755 spin_lock_irqsave(&dev_priv->gt_lock, irqflags);
4756 if (dev_priv->forcewake_count++ == 0)
4757 dev_priv->gt.force_wake_get(dev_priv);
4758 spin_unlock_irqrestore(&dev_priv->gt_lock, irqflags);
4759}
4760
4761void gen6_gt_check_fifodbg(struct drm_i915_private *dev_priv)
4762{
4763 u32 gtfifodbg;
4764 gtfifodbg = I915_READ_NOTRACE(GTFIFODBG);
4765 if (WARN(gtfifodbg & GT_FIFO_CPU_ERROR_MASK,
4766 "MMIO read or write has been dropped %x\n", gtfifodbg))
4767 I915_WRITE_NOTRACE(GTFIFODBG, GT_FIFO_CPU_ERROR_MASK);
4768}
4769
4770static void __gen6_gt_force_wake_put(struct drm_i915_private *dev_priv)
4771{
4772 I915_WRITE_NOTRACE(FORCEWAKE, 0);
b5144075
JN		 4773 /* something from same cacheline, but !FORCEWAKE */
4774 POSTING_READ(ECOBUS);
6590190d
ED		 4775 gen6_gt_check_fifodbg(dev_priv);
4776}
4777
4778static void __gen6_gt_force_wake_mt_put(struct drm_i915_private *dev_priv)
4779{
c5836c27 4780 I915_WRITE_NOTRACE(FORCEWAKE_MT, _MASKED_BIT_DISABLE(FORCEWAKE_KERNEL));
b5144075
JN		 4781 /* something from same cacheline, but !FORCEWAKE_MT */
4782 POSTING_READ(ECOBUS);
6590190d
ED		 4783 gen6_gt_check_fifodbg(dev_priv);
4784}
4785
4786/*
4787 * see gen6_gt_force_wake_get()
4788 */
4789void gen6_gt_force_wake_put(struct drm_i915_private *dev_priv)
4790{
4791 unsigned long irqflags;
4792
4793 spin_lock_irqsave(&dev_priv->gt_lock, irqflags);
4794 if (--dev_priv->forcewake_count == 0)
4795 dev_priv->gt.force_wake_put(dev_priv);
4796 spin_unlock_irqrestore(&dev_priv->gt_lock, irqflags);
4797}
4798
4799int __gen6_gt_wait_for_fifo(struct drm_i915_private *dev_priv)
4800{
4801 int ret = 0;
4802
4803 if (dev_priv->gt_fifo_count < GT_FIFO_NUM_RESERVED_ENTRIES) {
4804 int loop = 500;
4805 u32 fifo = I915_READ_NOTRACE(GT_FIFO_FREE_ENTRIES);
4806 while (fifo <= GT_FIFO_NUM_RESERVED_ENTRIES && loop--) {
4807 udelay(10);
4808 fifo = I915_READ_NOTRACE(GT_FIFO_FREE_ENTRIES);
4809 }
4810 if (WARN_ON(loop < 0 && fifo <= GT_FIFO_NUM_RESERVED_ENTRIES))
4811 ++ret;
4812 dev_priv->gt_fifo_count = fifo;
4813 }
4814 dev_priv->gt_fifo_count--;
4815
4816 return ret;
4817}
4818
16995a9f
CW		 4819static void vlv_force_wake_reset(struct drm_i915_private *dev_priv)
4820{
4821 I915_WRITE_NOTRACE(FORCEWAKE_VLV, _MASKED_BIT_DISABLE(0xffff));
b5144075
JN		 4822 /* something from same cacheline, but !FORCEWAKE_VLV */
4823 POSTING_READ(FORCEWAKE_ACK_VLV);
16995a9f
CW		 4824}
4825
6590190d
ED		 4826static void vlv_force_wake_get(struct drm_i915_private *dev_priv)
4827{
83983c8b 4828 if (wait_for_atomic((I915_READ_NOTRACE(FORCEWAKE_ACK_VLV) & FORCEWAKE_KERNEL) == 0,
057d3860 4829 FORCEWAKE_ACK_TIMEOUT_MS))
8a038fd6 4830 DRM_ERROR("Timed out waiting for forcewake old ack to clear.\n");
6590190d 4831
c5836c27 4832 I915_WRITE_NOTRACE(FORCEWAKE_VLV, _MASKED_BIT_ENABLE(FORCEWAKE_KERNEL));
ed5de399
JB		 4833 I915_WRITE_NOTRACE(FORCEWAKE_MEDIA_VLV,
4834 _MASKED_BIT_ENABLE(FORCEWAKE_KERNEL));
6590190d 4835
83983c8b 4836 if (wait_for_atomic((I915_READ_NOTRACE(FORCEWAKE_ACK_VLV) & FORCEWAKE_KERNEL),
057d3860 4837 FORCEWAKE_ACK_TIMEOUT_MS))
ed5de399
JB		 4838 DRM_ERROR("Timed out waiting for GT to ack forcewake request.\n");
4839
4840 if (wait_for_atomic((I915_READ_NOTRACE(FORCEWAKE_ACK_MEDIA_VLV) &
4841 FORCEWAKE_KERNEL),
4842 FORCEWAKE_ACK_TIMEOUT_MS))
4843 DRM_ERROR("Timed out waiting for media to ack forcewake request.\n");
6590190d 4844
8693a824 4845 /* WaRsForcewakeWaitTC0:vlv */
6590190d
ED		 4846 __gen6_gt_wait_for_thread_c0(dev_priv);
4847}
4848
4849static void vlv_force_wake_put(struct drm_i915_private *dev_priv)
4850{
c5836c27 4851 I915_WRITE_NOTRACE(FORCEWAKE_VLV, _MASKED_BIT_DISABLE(FORCEWAKE_KERNEL));
ed5de399
JB		 4852 I915_WRITE_NOTRACE(FORCEWAKE_MEDIA_VLV,
4853 _MASKED_BIT_DISABLE(FORCEWAKE_KERNEL));
4854 /* The below doubles as a POSTING_READ */
5ab140a4 4855 gen6_gt_check_fifodbg(dev_priv);
6590190d
ED		 4856}
4857
16995a9f
CW		 4858void intel_gt_reset(struct drm_device *dev)
4859{
4860 struct drm_i915_private *dev_priv = dev->dev_private;
4861
4862 if (IS_VALLEYVIEW(dev)) {
4863 vlv_force_wake_reset(dev_priv);
4864 } else if (INTEL_INFO(dev)->gen >= 6) {
4865 __gen6_gt_force_wake_reset(dev_priv);
4866 if (IS_IVYBRIDGE(dev) || IS_HASWELL(dev))
4867 __gen6_gt_force_wake_mt_reset(dev_priv);
4868 }
4869}
4870
6590190d
ED		 4871void intel_gt_init(struct drm_device *dev)
4872{
4873 struct drm_i915_private *dev_priv = dev->dev_private;
4874
4875 spin_lock_init(&dev_priv->gt_lock);
4876
16995a9f
CW		 4877 intel_gt_reset(dev);
4878
6590190d
ED		 4879 if (IS_VALLEYVIEW(dev)) {
4880 dev_priv->gt.force_wake_get = vlv_force_wake_get;
4881 dev_priv->gt.force_wake_put = vlv_force_wake_put;
36ec8f87
DV		 4882 } else if (IS_IVYBRIDGE(dev) || IS_HASWELL(dev)) {
4883 dev_priv->gt.force_wake_get = __gen6_gt_force_wake_mt_get;
4884 dev_priv->gt.force_wake_put = __gen6_gt_force_wake_mt_put;
4885 } else if (IS_GEN6(dev)) {
6590190d
ED		 4886 dev_priv->gt.force_wake_get = __gen6_gt_force_wake_get;
4887 dev_priv->gt.force_wake_put = __gen6_gt_force_wake_put;
6590190d 4888 }
1a01ab3b
JB		 4889 INIT_DELAYED_WORK(&dev_priv->rps.delayed_resume_work,
4890 intel_gen6_powersave_work);
6590190d
ED		 4891}
4892
42c0526c
BW		 4893int sandybridge_pcode_read(struct drm_i915_private *dev_priv, u8 mbox, u32 *val)
4894{
4fc688ce 4895 WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
42c0526c
BW		 4896
4897 if (I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) {
4898 DRM_DEBUG_DRIVER("warning: pcode (read) mailbox access failed\n");
4899 return -EAGAIN;
4900 }
4901
4902 I915_WRITE(GEN6_PCODE_DATA, *val);
4903 I915_WRITE(GEN6_PCODE_MAILBOX, GEN6_PCODE_READY | mbox);
4904
4905 if (wait_for((I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) == 0,
4906 500)) {
4907 DRM_ERROR("timeout waiting for pcode read (%d) to finish\n", mbox);
4908 return -ETIMEDOUT;
4909 }
4910
4911 *val = I915_READ(GEN6_PCODE_DATA);
4912 I915_WRITE(GEN6_PCODE_DATA, 0);
4913
4914 return 0;
4915}
4916
4917int sandybridge_pcode_write(struct drm_i915_private *dev_priv, u8 mbox, u32 val)
4918{
4fc688ce 4919 WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
42c0526c
BW		 4920
4921 if (I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) {
4922 DRM_DEBUG_DRIVER("warning: pcode (write) mailbox access failed\n");
4923 return -EAGAIN;
4924 }
4925
4926 I915_WRITE(GEN6_PCODE_DATA, val);
4927 I915_WRITE(GEN6_PCODE_MAILBOX, GEN6_PCODE_READY | mbox);
4928
4929 if (wait_for((I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) == 0,
4930 500)) {
4931 DRM_ERROR("timeout waiting for pcode write (%d) to finish\n", mbox);
4932 return -ETIMEDOUT;
4933 }
4934
4935 I915_WRITE(GEN6_PCODE_DATA, 0);
4936
4937 return 0;
4938}
a0e4e199 4939
0a073b84 4940static int vlv_punit_rw(struct drm_i915_private *dev_priv, u32 port, u8 opcode,
a0e4e199
JB		 4941 u8 addr, u32 *val)
4942{
0a073b84 4943 u32 cmd, devfn, be, bar;
a0e4e199
JB		 4944
4945 bar = 0;
4946 be = 0xf;
a0e4e199
JB		 4947 devfn = PCI_DEVFN(2, 0);
4948
4949 cmd = (devfn << IOSF_DEVFN_SHIFT) | (opcode << IOSF_OPCODE_SHIFT) |
4950 (port << IOSF_PORT_SHIFT) | (be << IOSF_BYTE_ENABLES_SHIFT) |
4951 (bar << IOSF_BAR_SHIFT);
4952
4953 WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
4954
4955 if (I915_READ(VLV_IOSF_DOORBELL_REQ) & IOSF_SB_BUSY) {
4956 DRM_DEBUG_DRIVER("warning: pcode (%s) mailbox access failed\n",
4957 opcode == PUNIT_OPCODE_REG_READ ?
4958 "read" : "write");
4959 return -EAGAIN;
4960 }
4961
4962 I915_WRITE(VLV_IOSF_ADDR, addr);
4963 if (opcode == PUNIT_OPCODE_REG_WRITE)
4964 I915_WRITE(VLV_IOSF_DATA, *val);
4965 I915_WRITE(VLV_IOSF_DOORBELL_REQ, cmd);
4966
4967 if (wait_for((I915_READ(VLV_IOSF_DOORBELL_REQ) & IOSF_SB_BUSY) == 0,
0a073b84 4968 5)) {
a0e4e199
JB		 4969 DRM_ERROR("timeout waiting for pcode %s (%d) to finish\n",
4970 opcode == PUNIT_OPCODE_REG_READ ? "read" : "write",
4971 addr);
4972 return -ETIMEDOUT;
4973 }
4974
4975 if (opcode == PUNIT_OPCODE_REG_READ)
4976 *val = I915_READ(VLV_IOSF_DATA);
4977 I915_WRITE(VLV_IOSF_DATA, 0);
4978
4979 return 0;
4980}
4981
4982int valleyview_punit_read(struct drm_i915_private *dev_priv, u8 addr, u32 *val)
4983{
0a073b84
JB		 4984 return vlv_punit_rw(dev_priv, IOSF_PORT_PUNIT, PUNIT_OPCODE_REG_READ,
4985 addr, val);
a0e4e199
JB		 4986}
4987
4988int valleyview_punit_write(struct drm_i915_private *dev_priv, u8 addr, u32 val)
4989{
0a073b84
JB		 4990 return vlv_punit_rw(dev_priv, IOSF_PORT_PUNIT, PUNIT_OPCODE_REG_WRITE,
4991 addr, &val);
4992}
4993
4994int valleyview_nc_read(struct drm_i915_private *dev_priv, u8 addr, u32 *val)
4995{
4996 return vlv_punit_rw(dev_priv, IOSF_PORT_NC, PUNIT_OPCODE_REG_READ,
4997 addr, val);
a0e4e199 4998}
855ba3be
JB		 4999
5000int vlv_gpu_freq(int ddr_freq, int val)
5001{
5002 int mult, base;
5003
5004 switch (ddr_freq) {
5005 case 800:
5006 mult = 20;
5007 base = 120;
5008 break;
5009 case 1066:
5010 mult = 22;
5011 base = 133;
5012 break;
5013 case 1333:
5014 mult = 21;
5015 base = 125;
5016 break;
5017 default:
5018 return -1;
5019 }
5020
5021 return ((val - 0xbd) * mult) + base;
5022}
5023
5024int vlv_freq_opcode(int ddr_freq, int val)
5025{
5026 int mult, base;
5027
5028 switch (ddr_freq) {
5029 case 800:
5030 mult = 20;
5031 base = 120;
5032 break;
5033 case 1066:
5034 mult = 22;
5035 base = 133;
5036 break;
5037 case 1333:
5038 mult = 21;
5039 base = 125;
5040 break;
5041 default:
5042 return -1;
5043 }
5044
5045 val /= mult;
5046 val -= base / mult;
5047 val += 0xbd;
5048
5049 if (val > 0xea)
5050 val = 0xea;
5051
5052 return val;
5053}
5054
Motorola kernel-slsi