projects / GitHub / mt8127 / android_kernel_alcatel_ttab.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
drm/radeon: fix UVD on big endian
[GitHub/mt8127/android_kernel_alcatel_ttab.git] / drivers / gpu / drm / radeon / r600.c
CommitLineData
771fe6b9
JG		 1/*
2 * Copyright 2008 Advanced Micro Devices, Inc.
3 * Copyright 2008 Red Hat Inc.
4 * Copyright 2009 Jerome Glisse.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22 * OTHER DEALINGS IN THE SOFTWARE.
23 *
24 * Authors: Dave Airlie
25 * Alex Deucher
26 * Jerome Glisse
27 */
5a0e3ad6 28#include <linux/slab.h>
3ce0a23d
JG		 29#include <linux/seq_file.h>
30#include <linux/firmware.h>
31#include <linux/platform_device.h>
e0cd3608 32#include <linux/module.h>
760285e7
DH		 33#include <drm/drmP.h>
34#include <drm/radeon_drm.h>
771fe6b9 35#include "radeon.h"
e6990375 36#include "radeon_asic.h"
3ce0a23d 37#include "radeon_mode.h"
3ce0a23d 38#include "r600d.h"
3ce0a23d 39#include "atom.h"
d39c3b89 40#include "avivod.h"
771fe6b9 41
3ce0a23d
JG		 42#define PFP_UCODE_SIZE 576
43#define PM4_UCODE_SIZE 1792
d8f60cfc 44#define RLC_UCODE_SIZE 768
3ce0a23d
JG		 45#define R700_PFP_UCODE_SIZE 848
46#define R700_PM4_UCODE_SIZE 1360
d8f60cfc 47#define R700_RLC_UCODE_SIZE 1024
fe251e2f
AD		 48#define EVERGREEN_PFP_UCODE_SIZE 1120
49#define EVERGREEN_PM4_UCODE_SIZE 1376
45f9a39b 50#define EVERGREEN_RLC_UCODE_SIZE 768
12727809 51#define CAYMAN_RLC_UCODE_SIZE 1024
c420c745 52#define ARUBA_RLC_UCODE_SIZE 1536
3ce0a23d
JG		 53
54/* Firmware Names */
55MODULE_FIRMWARE("radeon/R600_pfp.bin");
56MODULE_FIRMWARE("radeon/R600_me.bin");
57MODULE_FIRMWARE("radeon/RV610_pfp.bin");
58MODULE_FIRMWARE("radeon/RV610_me.bin");
59MODULE_FIRMWARE("radeon/RV630_pfp.bin");
60MODULE_FIRMWARE("radeon/RV630_me.bin");
61MODULE_FIRMWARE("radeon/RV620_pfp.bin");
62MODULE_FIRMWARE("radeon/RV620_me.bin");
63MODULE_FIRMWARE("radeon/RV635_pfp.bin");
64MODULE_FIRMWARE("radeon/RV635_me.bin");
65MODULE_FIRMWARE("radeon/RV670_pfp.bin");
66MODULE_FIRMWARE("radeon/RV670_me.bin");
67MODULE_FIRMWARE("radeon/RS780_pfp.bin");
68MODULE_FIRMWARE("radeon/RS780_me.bin");
69MODULE_FIRMWARE("radeon/RV770_pfp.bin");
70MODULE_FIRMWARE("radeon/RV770_me.bin");
71MODULE_FIRMWARE("radeon/RV730_pfp.bin");
72MODULE_FIRMWARE("radeon/RV730_me.bin");
73MODULE_FIRMWARE("radeon/RV710_pfp.bin");
74MODULE_FIRMWARE("radeon/RV710_me.bin");
d8f60cfc
AD		 75MODULE_FIRMWARE("radeon/R600_rlc.bin");
76MODULE_FIRMWARE("radeon/R700_rlc.bin");
fe251e2f
AD		 77MODULE_FIRMWARE("radeon/CEDAR_pfp.bin");
78MODULE_FIRMWARE("radeon/CEDAR_me.bin");
45f9a39b 79MODULE_FIRMWARE("radeon/CEDAR_rlc.bin");
fe251e2f
AD		 80MODULE_FIRMWARE("radeon/REDWOOD_pfp.bin");
81MODULE_FIRMWARE("radeon/REDWOOD_me.bin");
45f9a39b 82MODULE_FIRMWARE("radeon/REDWOOD_rlc.bin");
fe251e2f
AD		 83MODULE_FIRMWARE("radeon/JUNIPER_pfp.bin");
84MODULE_FIRMWARE("radeon/JUNIPER_me.bin");
45f9a39b 85MODULE_FIRMWARE("radeon/JUNIPER_rlc.bin");
a7433742 86MODULE_FIRMWARE("radeon/CYPRESS_pfp.bin");
fe251e2f 87MODULE_FIRMWARE("radeon/CYPRESS_me.bin");
45f9a39b 88MODULE_FIRMWARE("radeon/CYPRESS_rlc.bin");
439bd6cd
AD		 89MODULE_FIRMWARE("radeon/PALM_pfp.bin");
90MODULE_FIRMWARE("radeon/PALM_me.bin");
91MODULE_FIRMWARE("radeon/SUMO_rlc.bin");
d5c5a72f
AD		 92MODULE_FIRMWARE("radeon/SUMO_pfp.bin");
93MODULE_FIRMWARE("radeon/SUMO_me.bin");
94MODULE_FIRMWARE("radeon/SUMO2_pfp.bin");
95MODULE_FIRMWARE("radeon/SUMO2_me.bin");
3ce0a23d 96
f13f7731
AD		 97static const u32 crtc_offsets[2] =
98{
99 0,
100 AVIVO_D2CRTC_H_TOTAL - AVIVO_D1CRTC_H_TOTAL
101};
102
3ce0a23d 103int r600_debugfs_mc_info_init(struct radeon_device *rdev);
771fe6b9 104
1a029b76 105/* r600,rv610,rv630,rv620,rv635,rv670 */
771fe6b9 106int r600_mc_wait_for_idle(struct radeon_device *rdev);
1109ca09 107static void r600_gpu_init(struct radeon_device *rdev);
3ce0a23d 108void r600_fini(struct radeon_device *rdev);
45f9a39b 109void r600_irq_disable(struct radeon_device *rdev);
9e46a48d 110static void r600_pcie_gen2_enable(struct radeon_device *rdev);
771fe6b9 111
454d2e2a
AD		 112/**
113 * r600_get_xclk - get the xclk
114 *
115 * @rdev: radeon_device pointer
116 *
117 * Returns the reference clock used by the gfx engine
118 * (r6xx, IGPs, APUs).
119 */
120u32 r600_get_xclk(struct radeon_device *rdev)
121{
122 return rdev->clock.spll.reference_freq;
123}
124
21a8122a 125/* get temperature in millidegrees */
20d391d7 126int rv6xx_get_temp(struct radeon_device *rdev)
21a8122a
AD		 127{
128 u32 temp = (RREG32(CG_THERMAL_STATUS) & ASIC_T_MASK) >>
129 ASIC_T_SHIFT;
20d391d7 130 int actual_temp = temp & 0xff;
21a8122a 131
20d391d7
AD		 132 if (temp & 0x100)
133 actual_temp -= 256;
134
135 return actual_temp * 1000;
21a8122a
AD		 136}
137
ce8f5370 138void r600_pm_get_dynpm_state(struct radeon_device *rdev)
a48b9b4e
AD		 139{
140 int i;
141
ce8f5370
AD		 142 rdev->pm.dynpm_can_upclock = true;
143 rdev->pm.dynpm_can_downclock = true;
a48b9b4e
AD		 144
145 /* power state array is low to high, default is first */
146 if ((rdev->flags & RADEON_IS_IGP) || (rdev->family == CHIP_R600)) {
147 int min_power_state_index = 0;
148
149 if (rdev->pm.num_power_states > 2)
150 min_power_state_index = 1;
151
ce8f5370
AD		 152 switch (rdev->pm.dynpm_planned_action) {
153 case DYNPM_ACTION_MINIMUM:
a48b9b4e
AD		 154 rdev->pm.requested_power_state_index = min_power_state_index;
155 rdev->pm.requested_clock_mode_index = 0;
ce8f5370 156 rdev->pm.dynpm_can_downclock = false;
a48b9b4e 157 break;
ce8f5370 158 case DYNPM_ACTION_DOWNCLOCK:
a48b9b4e
AD		 159 if (rdev->pm.current_power_state_index == min_power_state_index) {
160 rdev->pm.requested_power_state_index = rdev->pm.current_power_state_index;
ce8f5370 161 rdev->pm.dynpm_can_downclock = false;
a48b9b4e
AD		 162 } else {
163 if (rdev->pm.active_crtc_count > 1) {
164 for (i = 0; i < rdev->pm.num_power_states; i++) {
d7311171 165 if (rdev->pm.power_state[i].flags & RADEON_PM_STATE_SINGLE_DISPLAY_ONLY)
a48b9b4e
AD		 166 continue;
167 else if (i >= rdev->pm.current_power_state_index) {
168 rdev->pm.requested_power_state_index =
169 rdev->pm.current_power_state_index;
170 break;
171 } else {
172 rdev->pm.requested_power_state_index = i;
173 break;
174 }
175 }
773c3fa3
AD		 176 } else {
177 if (rdev->pm.current_power_state_index == 0)
178 rdev->pm.requested_power_state_index =
179 rdev->pm.num_power_states - 1;
180 else
181 rdev->pm.requested_power_state_index =
182 rdev->pm.current_power_state_index - 1;
183 }
a48b9b4e
AD		 184 }
185 rdev->pm.requested_clock_mode_index = 0;
d7311171
AD		 186 /* don't use the power state if crtcs are active and no display flag is set */
187 if ((rdev->pm.active_crtc_count > 0) &&
188 (rdev->pm.power_state[rdev->pm.requested_power_state_index].
189 clock_info[rdev->pm.requested_clock_mode_index].flags &
190 RADEON_PM_MODE_NO_DISPLAY)) {
191 rdev->pm.requested_power_state_index++;
192 }
a48b9b4e 193 break;
ce8f5370 194 case DYNPM_ACTION_UPCLOCK:
a48b9b4e
AD		 195 if (rdev->pm.current_power_state_index == (rdev->pm.num_power_states - 1)) {
196 rdev->pm.requested_power_state_index = rdev->pm.current_power_state_index;
ce8f5370 197 rdev->pm.dynpm_can_upclock = false;
a48b9b4e
AD		 198 } else {
199 if (rdev->pm.active_crtc_count > 1) {
200 for (i = (rdev->pm.num_power_states - 1); i >= 0; i--) {
d7311171 201 if (rdev->pm.power_state[i].flags & RADEON_PM_STATE_SINGLE_DISPLAY_ONLY)
a48b9b4e
AD		 202 continue;
203 else if (i <= rdev->pm.current_power_state_index) {
204 rdev->pm.requested_power_state_index =
205 rdev->pm.current_power_state_index;
206 break;
207 } else {
208 rdev->pm.requested_power_state_index = i;
209 break;
210 }
211 }
212 } else
213 rdev->pm.requested_power_state_index =
214 rdev->pm.current_power_state_index + 1;
215 }
216 rdev->pm.requested_clock_mode_index = 0;
217 break;
ce8f5370 218 case DYNPM_ACTION_DEFAULT:
58e21dff
AD		 219 rdev->pm.requested_power_state_index = rdev->pm.default_power_state_index;
220 rdev->pm.requested_clock_mode_index = 0;
ce8f5370 221 rdev->pm.dynpm_can_upclock = false;
58e21dff 222 break;
ce8f5370 223 case DYNPM_ACTION_NONE:
a48b9b4e
AD		 224 default:
225 DRM_ERROR("Requested mode for not defined action\n");
226 return;
227 }
228 } else {
229 /* XXX select a power state based on AC/DC, single/dualhead, etc. */
230 /* for now just select the first power state and switch between clock modes */
231 /* power state array is low to high, default is first (0) */
232 if (rdev->pm.active_crtc_count > 1) {
233 rdev->pm.requested_power_state_index = -1;
234 /* start at 1 as we don't want the default mode */
235 for (i = 1; i < rdev->pm.num_power_states; i++) {
d7311171 236 if (rdev->pm.power_state[i].flags & RADEON_PM_STATE_SINGLE_DISPLAY_ONLY)
a48b9b4e
AD		 237 continue;
238 else if ((rdev->pm.power_state[i].type == POWER_STATE_TYPE_PERFORMANCE) ||
239 (rdev->pm.power_state[i].type == POWER_STATE_TYPE_BATTERY)) {
240 rdev->pm.requested_power_state_index = i;
241 break;
242 }
243 }
244 /* if nothing selected, grab the default state. */
245 if (rdev->pm.requested_power_state_index == -1)
246 rdev->pm.requested_power_state_index = 0;
247 } else
248 rdev->pm.requested_power_state_index = 1;
249
ce8f5370
AD		 250 switch (rdev->pm.dynpm_planned_action) {
251 case DYNPM_ACTION_MINIMUM:
a48b9b4e 252 rdev->pm.requested_clock_mode_index = 0;
ce8f5370 253 rdev->pm.dynpm_can_downclock = false;
a48b9b4e 254 break;
ce8f5370 255 case DYNPM_ACTION_DOWNCLOCK:
a48b9b4e
AD		 256 if (rdev->pm.requested_power_state_index == rdev->pm.current_power_state_index) {
257 if (rdev->pm.current_clock_mode_index == 0) {
258 rdev->pm.requested_clock_mode_index = 0;
ce8f5370 259 rdev->pm.dynpm_can_downclock = false;
a48b9b4e
AD		 260 } else
261 rdev->pm.requested_clock_mode_index =
262 rdev->pm.current_clock_mode_index - 1;
263 } else {
264 rdev->pm.requested_clock_mode_index = 0;
ce8f5370 265 rdev->pm.dynpm_can_downclock = false;
a48b9b4e 266 }
d7311171
AD		 267 /* don't use the power state if crtcs are active and no display flag is set */
268 if ((rdev->pm.active_crtc_count > 0) &&
269 (rdev->pm.power_state[rdev->pm.requested_power_state_index].
270 clock_info[rdev->pm.requested_clock_mode_index].flags &
271 RADEON_PM_MODE_NO_DISPLAY)) {
272 rdev->pm.requested_clock_mode_index++;
273 }
a48b9b4e 274 break;
ce8f5370 275 case DYNPM_ACTION_UPCLOCK:
a48b9b4e
AD		 276 if (rdev->pm.requested_power_state_index == rdev->pm.current_power_state_index) {
277 if (rdev->pm.current_clock_mode_index ==
278 (rdev->pm.power_state[rdev->pm.requested_power_state_index].num_clock_modes - 1)) {
279 rdev->pm.requested_clock_mode_index = rdev->pm.current_clock_mode_index;
ce8f5370 280 rdev->pm.dynpm_can_upclock = false;
a48b9b4e
AD		 281 } else
282 rdev->pm.requested_clock_mode_index =
283 rdev->pm.current_clock_mode_index + 1;
284 } else {
285 rdev->pm.requested_clock_mode_index =
286 rdev->pm.power_state[rdev->pm.requested_power_state_index].num_clock_modes - 1;
ce8f5370 287 rdev->pm.dynpm_can_upclock = false;
a48b9b4e
AD		 288 }
289 break;
ce8f5370 290 case DYNPM_ACTION_DEFAULT:
58e21dff
AD		 291 rdev->pm.requested_power_state_index = rdev->pm.default_power_state_index;
292 rdev->pm.requested_clock_mode_index = 0;
ce8f5370 293 rdev->pm.dynpm_can_upclock = false;
58e21dff 294 break;
ce8f5370 295 case DYNPM_ACTION_NONE:
a48b9b4e
AD		 296 default:
297 DRM_ERROR("Requested mode for not defined action\n");
298 return;
299 }
300 }
301
d9fdaafb 302 DRM_DEBUG_DRIVER("Requested: e: %d m: %d p: %d\n",
ce8a3eb2
AD		 303 rdev->pm.power_state[rdev->pm.requested_power_state_index].
304 clock_info[rdev->pm.requested_clock_mode_index].sclk,
305 rdev->pm.power_state[rdev->pm.requested_power_state_index].
306 clock_info[rdev->pm.requested_clock_mode_index].mclk,
307 rdev->pm.power_state[rdev->pm.requested_power_state_index].
308 pcie_lanes);
a48b9b4e
AD		 309}
310
ce8f5370
AD		 311void rs780_pm_init_profile(struct radeon_device *rdev)
312{
313 if (rdev->pm.num_power_states == 2) {
314 /* default */
315 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
316 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
317 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
318 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0;
319 /* low sh */
320 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = 0;
321 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = 0;
322 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
323 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
c9e75b21
AD		 324 /* mid sh */
325 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = 0;
326 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = 0;
327 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
328 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0;
ce8f5370
AD		 329 /* high sh */
330 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = 0;
331 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = 1;
332 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
333 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 0;
334 /* low mh */
335 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = 0;
336 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = 0;
337 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
338 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
c9e75b21
AD		 339 /* mid mh */
340 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = 0;
341 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = 0;
342 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
343 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0;
ce8f5370
AD		 344 /* high mh */
345 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = 0;
346 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = 1;
347 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
348 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 0;
349 } else if (rdev->pm.num_power_states == 3) {
350 /* default */
351 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
352 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
353 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
354 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0;
355 /* low sh */
356 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = 1;
357 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = 1;
358 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
359 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
c9e75b21
AD		 360 /* mid sh */
361 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = 1;
362 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = 1;
363 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
364 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0;
ce8f5370
AD		 365 /* high sh */
366 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = 1;
367 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = 2;
368 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
369 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 0;
370 /* low mh */
371 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = 1;
372 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = 1;
373 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
374 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
c9e75b21
AD		 375 /* mid mh */
376 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = 1;
377 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = 1;
378 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
379 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0;
ce8f5370
AD		 380 /* high mh */
381 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = 1;
382 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = 2;
383 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
384 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 0;
385 } else {
386 /* default */
387 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
388 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
389 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
390 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0;
391 /* low sh */
392 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = 2;
393 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = 2;
394 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
395 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
c9e75b21
AD		 396 /* mid sh */
397 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = 2;
398 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = 2;
399 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
400 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0;
ce8f5370
AD		 401 /* high sh */
402 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = 2;
403 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = 3;
404 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
405 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 0;
406 /* low mh */
407 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = 2;
408 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = 0;
409 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
410 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
c9e75b21
AD		 411 /* mid mh */
412 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = 2;
413 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = 0;
414 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
415 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0;
ce8f5370
AD		 416 /* high mh */
417 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = 2;
418 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = 3;
419 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
420 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 0;
421 }
422}
bae6b562 423
ce8f5370
AD		 424void r600_pm_init_profile(struct radeon_device *rdev)
425{
bbe26ffe
AD		 426 int idx;
427
ce8f5370
AD		 428 if (rdev->family == CHIP_R600) {
429 /* XXX */
430 /* default */
431 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
432 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
433 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
4bff5171 434 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0;
ce8f5370
AD		 435 /* low sh */
436 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
437 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
438 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
4bff5171 439 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
c9e75b21
AD		 440 /* mid sh */
441 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
442 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
443 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
444 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0;
ce8f5370
AD		 445 /* high sh */
446 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
447 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
448 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
4bff5171 449 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 0;
ce8f5370
AD		 450 /* low mh */
451 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
452 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
453 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
4bff5171 454 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
c9e75b21
AD		 455 /* mid mh */
456 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
457 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
458 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
459 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0;
ce8f5370
AD		 460 /* high mh */
461 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
462 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
463 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
4bff5171 464 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 0;
ce8f5370
AD		 465 } else {
466 if (rdev->pm.num_power_states < 4) {
467 /* default */
468 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
469 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
470 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
471 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 2;
472 /* low sh */
4bff5171
AD		 473 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = 1;
474 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = 1;
ce8f5370 475 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
c9e75b21
AD		 476 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
477 /* mid sh */
478 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = 1;
479 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = 1;
480 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
481 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 1;
ce8f5370 482 /* high sh */
4bff5171
AD		 483 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = 1;
484 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = 1;
ce8f5370
AD		 485 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
486 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 2;
487 /* low mh */
4bff5171
AD		 488 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = 2;
489 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = 2;
ce8f5370 490 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
c9e75b21
AD		 491 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
492 /* low mh */
493 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = 2;
494 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = 2;
495 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
496 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 1;
ce8f5370 497 /* high mh */
4bff5171
AD		 498 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = 2;
499 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = 2;
ce8f5370
AD		 500 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
501 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 2;
502 } else {
503 /* default */
504 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
505 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
506 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
507 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 2;
508 /* low sh */
bbe26ffe
AD		 509 if (rdev->flags & RADEON_IS_MOBILITY)
510 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
511 else
512 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
513 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
514 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
515 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
516 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
c9e75b21 517 /* mid sh */
bbe26ffe
AD		 518 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
519 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
520 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
521 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 1;
ce8f5370 522 /* high sh */
bbe26ffe
AD		 523 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
524 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
525 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
ce8f5370
AD		 526 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
527 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 2;
528 /* low mh */
bbe26ffe
AD		 529 if (rdev->flags & RADEON_IS_MOBILITY)
530 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 1);
531 else
532 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 1);
533 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
534 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
535 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
536 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
c9e75b21 537 /* mid mh */
bbe26ffe
AD		 538 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
539 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
540 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
541 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 1;
ce8f5370 542 /* high mh */
bbe26ffe
AD		 543 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 1);
544 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
545 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
ce8f5370
AD		 546 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
547 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 2;
548 }
549 }
bae6b562
AD		 550}
551
49e02b73
AD		 552void r600_pm_misc(struct radeon_device *rdev)
553{
a081a9d6
RM		 554 int req_ps_idx = rdev->pm.requested_power_state_index;
555 int req_cm_idx = rdev->pm.requested_clock_mode_index;
556 struct radeon_power_state *ps = &rdev->pm.power_state[req_ps_idx];
557 struct radeon_voltage *voltage = &ps->clock_info[req_cm_idx].voltage;
7ac9aa5a 558
4d60173f 559 if ((voltage->type == VOLTAGE_SW) && voltage->voltage) {
a377e187
AD		 560 /* 0xff01 is a flag rather then an actual voltage */
561 if (voltage->voltage == 0xff01)
562 return;
4d60173f 563 if (voltage->voltage != rdev->pm.current_vddc) {
8a83ec5e 564 radeon_atom_set_voltage(rdev, voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC);
4d60173f 565 rdev->pm.current_vddc = voltage->voltage;
d9fdaafb 566 DRM_DEBUG_DRIVER("Setting: v: %d\n", voltage->voltage);
4d60173f
AD		 567 }
568 }
49e02b73
AD		 569}
570
def9ba9c
AD		 571bool r600_gui_idle(struct radeon_device *rdev)
572{
573 if (RREG32(GRBM_STATUS) & GUI_ACTIVE)
574 return false;
575 else
576 return true;
577}
578
e0df1ac5
AD		 579/* hpd for digital panel detect/disconnect */
580bool r600_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd)
581{
582 bool connected = false;
583
584 if (ASIC_IS_DCE3(rdev)) {
585 switch (hpd) {
586 case RADEON_HPD_1:
587 if (RREG32(DC_HPD1_INT_STATUS) & DC_HPDx_SENSE)
588 connected = true;
589 break;
590 case RADEON_HPD_2:
591 if (RREG32(DC_HPD2_INT_STATUS) & DC_HPDx_SENSE)
592 connected = true;
593 break;
594 case RADEON_HPD_3:
595 if (RREG32(DC_HPD3_INT_STATUS) & DC_HPDx_SENSE)
596 connected = true;
597 break;
598 case RADEON_HPD_4:
599 if (RREG32(DC_HPD4_INT_STATUS) & DC_HPDx_SENSE)
600 connected = true;
601 break;
602 /* DCE 3.2 */
603 case RADEON_HPD_5:
604 if (RREG32(DC_HPD5_INT_STATUS) & DC_HPDx_SENSE)
605 connected = true;
606 break;
607 case RADEON_HPD_6:
608 if (RREG32(DC_HPD6_INT_STATUS) & DC_HPDx_SENSE)
609 connected = true;
610 break;
611 default:
612 break;
613 }
614 } else {
615 switch (hpd) {
616 case RADEON_HPD_1:
617 if (RREG32(DC_HOT_PLUG_DETECT1_INT_STATUS) & DC_HOT_PLUG_DETECTx_SENSE)
618 connected = true;
619 break;
620 case RADEON_HPD_2:
621 if (RREG32(DC_HOT_PLUG_DETECT2_INT_STATUS) & DC_HOT_PLUG_DETECTx_SENSE)
622 connected = true;
623 break;
624 case RADEON_HPD_3:
625 if (RREG32(DC_HOT_PLUG_DETECT3_INT_STATUS) & DC_HOT_PLUG_DETECTx_SENSE)
626 connected = true;
627 break;
628 default:
629 break;
630 }
631 }
632 return connected;
633}
634
635void r600_hpd_set_polarity(struct radeon_device *rdev,
429770b3 636 enum radeon_hpd_id hpd)
e0df1ac5
AD		 637{
638 u32 tmp;
639 bool connected = r600_hpd_sense(rdev, hpd);
640
641 if (ASIC_IS_DCE3(rdev)) {
642 switch (hpd) {
643 case RADEON_HPD_1:
644 tmp = RREG32(DC_HPD1_INT_CONTROL);
645 if (connected)
646 tmp &= ~DC_HPDx_INT_POLARITY;
647 else
648 tmp |= DC_HPDx_INT_POLARITY;
649 WREG32(DC_HPD1_INT_CONTROL, tmp);
650 break;
651 case RADEON_HPD_2:
652 tmp = RREG32(DC_HPD2_INT_CONTROL);
653 if (connected)
654 tmp &= ~DC_HPDx_INT_POLARITY;
655 else
656 tmp |= DC_HPDx_INT_POLARITY;
657 WREG32(DC_HPD2_INT_CONTROL, tmp);
658 break;
659 case RADEON_HPD_3:
660 tmp = RREG32(DC_HPD3_INT_CONTROL);
661 if (connected)
662 tmp &= ~DC_HPDx_INT_POLARITY;
663 else
664 tmp |= DC_HPDx_INT_POLARITY;
665 WREG32(DC_HPD3_INT_CONTROL, tmp);
666 break;
667 case RADEON_HPD_4:
668 tmp = RREG32(DC_HPD4_INT_CONTROL);
669 if (connected)
670 tmp &= ~DC_HPDx_INT_POLARITY;
671 else
672 tmp |= DC_HPDx_INT_POLARITY;
673 WREG32(DC_HPD4_INT_CONTROL, tmp);
674 break;
675 case RADEON_HPD_5:
676 tmp = RREG32(DC_HPD5_INT_CONTROL);
677 if (connected)
678 tmp &= ~DC_HPDx_INT_POLARITY;
679 else
680 tmp |= DC_HPDx_INT_POLARITY;
681 WREG32(DC_HPD5_INT_CONTROL, tmp);
682 break;
683 /* DCE 3.2 */
684 case RADEON_HPD_6:
685 tmp = RREG32(DC_HPD6_INT_CONTROL);
686 if (connected)
687 tmp &= ~DC_HPDx_INT_POLARITY;
688 else
689 tmp |= DC_HPDx_INT_POLARITY;
690 WREG32(DC_HPD6_INT_CONTROL, tmp);
691 break;
692 default:
693 break;
694 }
695 } else {
696 switch (hpd) {
697 case RADEON_HPD_1:
698 tmp = RREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL);
699 if (connected)
700 tmp &= ~DC_HOT_PLUG_DETECTx_INT_POLARITY;
701 else
702 tmp |= DC_HOT_PLUG_DETECTx_INT_POLARITY;
703 WREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL, tmp);
704 break;
705 case RADEON_HPD_2:
706 tmp = RREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL);
707 if (connected)
708 tmp &= ~DC_HOT_PLUG_DETECTx_INT_POLARITY;
709 else
710 tmp |= DC_HOT_PLUG_DETECTx_INT_POLARITY;
711 WREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL, tmp);
712 break;
713 case RADEON_HPD_3:
714 tmp = RREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL);
715 if (connected)
716 tmp &= ~DC_HOT_PLUG_DETECTx_INT_POLARITY;
717 else
718 tmp |= DC_HOT_PLUG_DETECTx_INT_POLARITY;
719 WREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL, tmp);
720 break;
721 default:
722 break;
723 }
724 }
725}
726
727void r600_hpd_init(struct radeon_device *rdev)
728{
729 struct drm_device *dev = rdev->ddev;
730 struct drm_connector *connector;
fb98257a 731 unsigned enable = 0;
e0df1ac5 732
64912e99
AD		 733 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
734 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
735
455c89b9
JG		 736 if (connector->connector_type == DRM_MODE_CONNECTOR_eDP ||
737 connector->connector_type == DRM_MODE_CONNECTOR_LVDS) {
738 /* don't try to enable hpd on eDP or LVDS avoid breaking the
739 * aux dp channel on imac and help (but not completely fix)
740 * https://bugzilla.redhat.com/show_bug.cgi?id=726143
741 */
742 continue;
743 }
64912e99
AD		 744 if (ASIC_IS_DCE3(rdev)) {
745 u32 tmp = DC_HPDx_CONNECTION_TIMER(0x9c4) | DC_HPDx_RX_INT_TIMER(0xfa);
746 if (ASIC_IS_DCE32(rdev))
747 tmp |= DC_HPDx_EN;
e0df1ac5 748
e0df1ac5
AD		 749 switch (radeon_connector->hpd.hpd) {
750 case RADEON_HPD_1:
751 WREG32(DC_HPD1_CONTROL, tmp);
e0df1ac5
AD		 752 break;
753 case RADEON_HPD_2:
754 WREG32(DC_HPD2_CONTROL, tmp);
e0df1ac5
AD		 755 break;
756 case RADEON_HPD_3:
757 WREG32(DC_HPD3_CONTROL, tmp);
e0df1ac5
AD		 758 break;
759 case RADEON_HPD_4:
760 WREG32(DC_HPD4_CONTROL, tmp);
e0df1ac5
AD		 761 break;
762 /* DCE 3.2 */
763 case RADEON_HPD_5:
764 WREG32(DC_HPD5_CONTROL, tmp);
e0df1ac5
AD		 765 break;
766 case RADEON_HPD_6:
767 WREG32(DC_HPD6_CONTROL, tmp);
e0df1ac5
AD		 768 break;
769 default:
770 break;
771 }
64912e99 772 } else {
e0df1ac5
AD		 773 switch (radeon_connector->hpd.hpd) {
774 case RADEON_HPD_1:
775 WREG32(DC_HOT_PLUG_DETECT1_CONTROL, DC_HOT_PLUG_DETECTx_EN);
e0df1ac5
AD		 776 break;
777 case RADEON_HPD_2:
778 WREG32(DC_HOT_PLUG_DETECT2_CONTROL, DC_HOT_PLUG_DETECTx_EN);
e0df1ac5
AD		 779 break;
780 case RADEON_HPD_3:
781 WREG32(DC_HOT_PLUG_DETECT3_CONTROL, DC_HOT_PLUG_DETECTx_EN);
e0df1ac5
AD		 782 break;
783 default:
784 break;
785 }
786 }
fb98257a 787 enable |= 1 << radeon_connector->hpd.hpd;
64912e99 788 radeon_hpd_set_polarity(rdev, radeon_connector->hpd.hpd);
e0df1ac5 789 }
fb98257a 790 radeon_irq_kms_enable_hpd(rdev, enable);
e0df1ac5
AD		 791}
792
793void r600_hpd_fini(struct radeon_device *rdev)
794{
795 struct drm_device *dev = rdev->ddev;
796 struct drm_connector *connector;
fb98257a 797 unsigned disable = 0;
e0df1ac5 798
fb98257a
CK		 799 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
800 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
801 if (ASIC_IS_DCE3(rdev)) {
e0df1ac5
AD		 802 switch (radeon_connector->hpd.hpd) {
803 case RADEON_HPD_1:
804 WREG32(DC_HPD1_CONTROL, 0);
e0df1ac5
AD		 805 break;
806 case RADEON_HPD_2:
807 WREG32(DC_HPD2_CONTROL, 0);
e0df1ac5
AD		 808 break;
809 case RADEON_HPD_3:
810 WREG32(DC_HPD3_CONTROL, 0);
e0df1ac5
AD		 811 break;
812 case RADEON_HPD_4:
813 WREG32(DC_HPD4_CONTROL, 0);
e0df1ac5
AD		 814 break;
815 /* DCE 3.2 */
816 case RADEON_HPD_5:
817 WREG32(DC_HPD5_CONTROL, 0);
e0df1ac5
AD		 818 break;
819 case RADEON_HPD_6:
820 WREG32(DC_HPD6_CONTROL, 0);
e0df1ac5
AD		 821 break;
822 default:
823 break;
824 }
fb98257a 825 } else {
e0df1ac5
AD		 826 switch (radeon_connector->hpd.hpd) {
827 case RADEON_HPD_1:
828 WREG32(DC_HOT_PLUG_DETECT1_CONTROL, 0);
e0df1ac5
AD		 829 break;
830 case RADEON_HPD_2:
831 WREG32(DC_HOT_PLUG_DETECT2_CONTROL, 0);
e0df1ac5
AD		 832 break;
833 case RADEON_HPD_3:
834 WREG32(DC_HOT_PLUG_DETECT3_CONTROL, 0);
e0df1ac5
AD		 835 break;
836 default:
837 break;
838 }
839 }
fb98257a 840 disable |= 1 << radeon_connector->hpd.hpd;
e0df1ac5 841 }
fb98257a 842 radeon_irq_kms_disable_hpd(rdev, disable);
e0df1ac5
AD		 843}
844
771fe6b9 845/*
3ce0a23d 846 * R600 PCIE GART
771fe6b9 847 */
3ce0a23d
JG		 848void r600_pcie_gart_tlb_flush(struct radeon_device *rdev)
849{
850 unsigned i;
851 u32 tmp;
852
2e98f10a 853 /* flush hdp cache so updates hit vram */
f3886f85
AD		 854 if ((rdev->family >= CHIP_RV770) && (rdev->family <= CHIP_RV740) &&
855 !(rdev->flags & RADEON_IS_AGP)) {
c9a1be96 856 void __iomem *ptr = (void *)rdev->gart.ptr;
812d0469
AD		 857 u32 tmp;
858
859 /* r7xx hw bug. write to HDP_DEBUG1 followed by fb read
860 * rather than write to HDP_REG_COHERENCY_FLUSH_CNTL
f3886f85
AD		 861 * This seems to cause problems on some AGP cards. Just use the old
862 * method for them.
812d0469
AD		 863 */
864 WREG32(HDP_DEBUG1, 0);
865 tmp = readl((void __iomem *)ptr);
866 } else
867 WREG32(R_005480_HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
2e98f10a 868
3ce0a23d
JG		 869 WREG32(VM_CONTEXT0_INVALIDATION_LOW_ADDR, rdev->mc.gtt_start >> 12);
870 WREG32(VM_CONTEXT0_INVALIDATION_HIGH_ADDR, (rdev->mc.gtt_end - 1) >> 12);
871 WREG32(VM_CONTEXT0_REQUEST_RESPONSE, REQUEST_TYPE(1));
872 for (i = 0; i < rdev->usec_timeout; i++) {
873 /* read MC_STATUS */
874 tmp = RREG32(VM_CONTEXT0_REQUEST_RESPONSE);
875 tmp = (tmp & RESPONSE_TYPE_MASK) >> RESPONSE_TYPE_SHIFT;
876 if (tmp == 2) {
877 printk(KERN_WARNING "[drm] r600 flush TLB failed\n");
878 return;
879 }
880 if (tmp) {
881 return;
882 }
883 udelay(1);
884 }
885}
886
4aac0473 887int r600_pcie_gart_init(struct radeon_device *rdev)
3ce0a23d 888{
4aac0473 889 int r;
3ce0a23d 890
c9a1be96 891 if (rdev->gart.robj) {
fce7d61b 892 WARN(1, "R600 PCIE GART already initialized\n");
4aac0473
JG		 893 return 0;
894 }
3ce0a23d
JG		 895 /* Initialize common gart structure */
896 r = radeon_gart_init(rdev);
4aac0473 897 if (r)
3ce0a23d 898 return r;
3ce0a23d 899 rdev->gart.table_size = rdev->gart.num_gpu_pages * 8;
4aac0473
JG		 900 return radeon_gart_table_vram_alloc(rdev);
901}
902
1109ca09 903static int r600_pcie_gart_enable(struct radeon_device *rdev)
4aac0473
JG		 904{
905 u32 tmp;
906 int r, i;
907
c9a1be96 908 if (rdev->gart.robj == NULL) {
4aac0473
JG		 909 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
910 return -EINVAL;
771fe6b9 911 }
4aac0473
JG		 912 r = radeon_gart_table_vram_pin(rdev);
913 if (r)
914 return r;
82568565 915 radeon_gart_restore(rdev);
bc1a631e 916
3ce0a23d
JG		 917 /* Setup L2 cache */
918 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
919 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
920 EFFECTIVE_L2_QUEUE_SIZE(7));
921 WREG32(VM_L2_CNTL2, 0);
922 WREG32(VM_L2_CNTL3, BANK_SELECT_0(0) | BANK_SELECT_1(1));
923 /* Setup TLB control */
924 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
925 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
926 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5) |
927 ENABLE_WAIT_L2_QUERY;
928 WREG32(MC_VM_L1_TLB_MCB_RD_SYS_CNTL, tmp);
929 WREG32(MC_VM_L1_TLB_MCB_WR_SYS_CNTL, tmp);
930 WREG32(MC_VM_L1_TLB_MCB_RD_HDP_CNTL, tmp | ENABLE_L1_STRICT_ORDERING);
931 WREG32(MC_VM_L1_TLB_MCB_WR_HDP_CNTL, tmp);
932 WREG32(MC_VM_L1_TLB_MCD_RD_A_CNTL, tmp);
933 WREG32(MC_VM_L1_TLB_MCD_WR_A_CNTL, tmp);
934 WREG32(MC_VM_L1_TLB_MCD_RD_B_CNTL, tmp);
935 WREG32(MC_VM_L1_TLB_MCD_WR_B_CNTL, tmp);
936 WREG32(MC_VM_L1_TLB_MCB_RD_GFX_CNTL, tmp);
937 WREG32(MC_VM_L1_TLB_MCB_WR_GFX_CNTL, tmp);
938 WREG32(MC_VM_L1_TLB_MCB_RD_PDMA_CNTL, tmp);
939 WREG32(MC_VM_L1_TLB_MCB_WR_PDMA_CNTL, tmp);
940 WREG32(MC_VM_L1_TLB_MCB_RD_SEM_CNTL, tmp | ENABLE_SEMAPHORE_MODE);
941 WREG32(MC_VM_L1_TLB_MCB_WR_SEM_CNTL, tmp | ENABLE_SEMAPHORE_MODE);
942 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
1a029b76 943 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
3ce0a23d
JG		 944 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
945 WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
946 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
947 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
948 (u32)(rdev->dummy_page.addr >> 12));
949 for (i = 1; i < 7; i++)
950 WREG32(VM_CONTEXT0_CNTL + (i * 4), 0);
771fe6b9 951
3ce0a23d 952 r600_pcie_gart_tlb_flush(rdev);
fcf4de5a
TV		 953 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
954 (unsigned)(rdev->mc.gtt_size >> 20),
955 (unsigned long long)rdev->gart.table_addr);
3ce0a23d 956 rdev->gart.ready = true;
771fe6b9
JG		 957 return 0;
958}
959
1109ca09 960static void r600_pcie_gart_disable(struct radeon_device *rdev)
771fe6b9 961{
3ce0a23d 962 u32 tmp;
c9a1be96 963 int i;
771fe6b9 964
3ce0a23d
JG		 965 /* Disable all tables */
966 for (i = 0; i < 7; i++)
967 WREG32(VM_CONTEXT0_CNTL + (i * 4), 0);
771fe6b9 968
3ce0a23d
JG		 969 /* Disable L2 cache */
970 WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING |
971 EFFECTIVE_L2_QUEUE_SIZE(7));
972 WREG32(VM_L2_CNTL3, BANK_SELECT_0(0) | BANK_SELECT_1(1));
973 /* Setup L1 TLB control */
974 tmp = EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5) |
975 ENABLE_WAIT_L2_QUERY;
976 WREG32(MC_VM_L1_TLB_MCD_RD_A_CNTL, tmp);
977 WREG32(MC_VM_L1_TLB_MCD_WR_A_CNTL, tmp);
978 WREG32(MC_VM_L1_TLB_MCD_RD_B_CNTL, tmp);
979 WREG32(MC_VM_L1_TLB_MCD_WR_B_CNTL, tmp);
980 WREG32(MC_VM_L1_TLB_MCB_RD_GFX_CNTL, tmp);
981 WREG32(MC_VM_L1_TLB_MCB_WR_GFX_CNTL, tmp);
982 WREG32(MC_VM_L1_TLB_MCB_RD_PDMA_CNTL, tmp);
983 WREG32(MC_VM_L1_TLB_MCB_WR_PDMA_CNTL, tmp);
984 WREG32(MC_VM_L1_TLB_MCB_RD_SEM_CNTL, tmp);
985 WREG32(MC_VM_L1_TLB_MCB_WR_SEM_CNTL, tmp);
986 WREG32(MC_VM_L1_TLB_MCB_RD_SYS_CNTL, tmp);
987 WREG32(MC_VM_L1_TLB_MCB_WR_SYS_CNTL, tmp);
988 WREG32(MC_VM_L1_TLB_MCB_RD_HDP_CNTL, tmp);
989 WREG32(MC_VM_L1_TLB_MCB_WR_HDP_CNTL, tmp);
c9a1be96 990 radeon_gart_table_vram_unpin(rdev);
4aac0473
JG		 991}
992
1109ca09 993static void r600_pcie_gart_fini(struct radeon_device *rdev)
4aac0473 994{
f9274562 995 radeon_gart_fini(rdev);
4aac0473
JG		 996 r600_pcie_gart_disable(rdev);
997 radeon_gart_table_vram_free(rdev);
771fe6b9
JG		 998}
999
1109ca09 1000static void r600_agp_enable(struct radeon_device *rdev)
1a029b76
JG		 1001{
1002 u32 tmp;
1003 int i;
1004
1005 /* Setup L2 cache */
1006 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
1007 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
1008 EFFECTIVE_L2_QUEUE_SIZE(7));
1009 WREG32(VM_L2_CNTL2, 0);
1010 WREG32(VM_L2_CNTL3, BANK_SELECT_0(0) | BANK_SELECT_1(1));
1011 /* Setup TLB control */
1012 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
1013 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
1014 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5) |
1015 ENABLE_WAIT_L2_QUERY;
1016 WREG32(MC_VM_L1_TLB_MCB_RD_SYS_CNTL, tmp);
1017 WREG32(MC_VM_L1_TLB_MCB_WR_SYS_CNTL, tmp);
1018 WREG32(MC_VM_L1_TLB_MCB_RD_HDP_CNTL, tmp | ENABLE_L1_STRICT_ORDERING);
1019 WREG32(MC_VM_L1_TLB_MCB_WR_HDP_CNTL, tmp);
1020 WREG32(MC_VM_L1_TLB_MCD_RD_A_CNTL, tmp);
1021 WREG32(MC_VM_L1_TLB_MCD_WR_A_CNTL, tmp);
1022 WREG32(MC_VM_L1_TLB_MCD_RD_B_CNTL, tmp);
1023 WREG32(MC_VM_L1_TLB_MCD_WR_B_CNTL, tmp);
1024 WREG32(MC_VM_L1_TLB_MCB_RD_GFX_CNTL, tmp);
1025 WREG32(MC_VM_L1_TLB_MCB_WR_GFX_CNTL, tmp);
1026 WREG32(MC_VM_L1_TLB_MCB_RD_PDMA_CNTL, tmp);
1027 WREG32(MC_VM_L1_TLB_MCB_WR_PDMA_CNTL, tmp);
1028 WREG32(MC_VM_L1_TLB_MCB_RD_SEM_CNTL, tmp | ENABLE_SEMAPHORE_MODE);
1029 WREG32(MC_VM_L1_TLB_MCB_WR_SEM_CNTL, tmp | ENABLE_SEMAPHORE_MODE);
1030 for (i = 0; i < 7; i++)
1031 WREG32(VM_CONTEXT0_CNTL + (i * 4), 0);
1032}
1033
771fe6b9
JG		 1034int r600_mc_wait_for_idle(struct radeon_device *rdev)
1035{
3ce0a23d
JG		 1036 unsigned i;
1037 u32 tmp;
1038
1039 for (i = 0; i < rdev->usec_timeout; i++) {
1040 /* read MC_STATUS */
1041 tmp = RREG32(R_000E50_SRBM_STATUS) & 0x3F00;
1042 if (!tmp)
1043 return 0;
1044 udelay(1);
1045 }
1046 return -1;
771fe6b9
JG		 1047}
1048
65337e60
SL		 1049uint32_t rs780_mc_rreg(struct radeon_device *rdev, uint32_t reg)
1050{
1051 uint32_t r;
1052
1053 WREG32(R_0028F8_MC_INDEX, S_0028F8_MC_IND_ADDR(reg));
1054 r = RREG32(R_0028FC_MC_DATA);
1055 WREG32(R_0028F8_MC_INDEX, ~C_0028F8_MC_IND_ADDR);
1056 return r;
1057}
1058
1059void rs780_mc_wreg(struct radeon_device *rdev, uint32_t reg, uint32_t v)
1060{
1061 WREG32(R_0028F8_MC_INDEX, S_0028F8_MC_IND_ADDR(reg) |
1062 S_0028F8_MC_IND_WR_EN(1));
1063 WREG32(R_0028FC_MC_DATA, v);
1064 WREG32(R_0028F8_MC_INDEX, 0x7F);
1065}
1066
a3c1945a 1067static void r600_mc_program(struct radeon_device *rdev)
771fe6b9 1068{
a3c1945a 1069 struct rv515_mc_save save;
3ce0a23d
JG		 1070 u32 tmp;
1071 int i, j;
771fe6b9 1072
3ce0a23d
JG		 1073 /* Initialize HDP */
1074 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
1075 WREG32((0x2c14 + j), 0x00000000);
1076 WREG32((0x2c18 + j), 0x00000000);
1077 WREG32((0x2c1c + j), 0x00000000);
1078 WREG32((0x2c20 + j), 0x00000000);
1079 WREG32((0x2c24 + j), 0x00000000);
1080 }
1081 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
771fe6b9 1082
a3c1945a 1083 rv515_mc_stop(rdev, &save);
3ce0a23d 1084 if (r600_mc_wait_for_idle(rdev)) {
a3c1945a 1085 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3ce0a23d 1086 }
a3c1945a 1087 /* Lockout access through VGA aperture (doesn't exist before R600) */
3ce0a23d 1088 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
3ce0a23d 1089 /* Update configuration */
1a029b76
JG		 1090 if (rdev->flags & RADEON_IS_AGP) {
1091 if (rdev->mc.vram_start < rdev->mc.gtt_start) {
1092 /* VRAM before AGP */
1093 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
1094 rdev->mc.vram_start >> 12);
1095 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
1096 rdev->mc.gtt_end >> 12);
1097 } else {
1098 /* VRAM after AGP */
1099 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
1100 rdev->mc.gtt_start >> 12);
1101 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
1102 rdev->mc.vram_end >> 12);
1103 }
1104 } else {
1105 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR, rdev->mc.vram_start >> 12);
1106 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR, rdev->mc.vram_end >> 12);
1107 }
16cdf04d 1108 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, rdev->vram_scratch.gpu_addr >> 12);
1a029b76 1109 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
3ce0a23d
JG		 1110 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
1111 WREG32(MC_VM_FB_LOCATION, tmp);
1112 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
1113 WREG32(HDP_NONSURFACE_INFO, (2 << 7));
46fcd2b3 1114 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
3ce0a23d 1115 if (rdev->flags & RADEON_IS_AGP) {
1a029b76
JG		 1116 WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 22);
1117 WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 22);
3ce0a23d
JG		 1118 WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22);
1119 } else {
1120 WREG32(MC_VM_AGP_BASE, 0);
1121 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
1122 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
1123 }
3ce0a23d 1124 if (r600_mc_wait_for_idle(rdev)) {
a3c1945a 1125 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3ce0a23d 1126 }
a3c1945a 1127 rv515_mc_resume(rdev, &save);
698443d9
DA		 1128 /* we need to own VRAM, so turn off the VGA renderer here
1129 * to stop it overwriting our objects */
d39c3b89 1130 rv515_vga_render_disable(rdev);
3ce0a23d
JG		 1131}
1132
d594e46a
JG		 1133/**
1134 * r600_vram_gtt_location - try to find VRAM & GTT location
1135 * @rdev: radeon device structure holding all necessary informations
1136 * @mc: memory controller structure holding memory informations
1137 *
1138 * Function will place try to place VRAM at same place as in CPU (PCI)
1139 * address space as some GPU seems to have issue when we reprogram at
1140 * different address space.
1141 *
1142 * If there is not enough space to fit the unvisible VRAM after the
1143 * aperture then we limit the VRAM size to the aperture.
1144 *
1145 * If we are using AGP then place VRAM adjacent to AGP aperture are we need
1146 * them to be in one from GPU point of view so that we can program GPU to
1147 * catch access outside them (weird GPU policy see ??).
1148 *
1149 * This function will never fails, worst case are limiting VRAM or GTT.
1150 *
1151 * Note: GTT start, end, size should be initialized before calling this
1152 * function on AGP platform.
1153 */
0ef0c1f7 1154static void r600_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc)
d594e46a
JG		 1155{
1156 u64 size_bf, size_af;
1157
1158 if (mc->mc_vram_size > 0xE0000000) {
1159 /* leave room for at least 512M GTT */
1160 dev_warn(rdev->dev, "limiting VRAM\n");
1161 mc->real_vram_size = 0xE0000000;
1162 mc->mc_vram_size = 0xE0000000;
1163 }
1164 if (rdev->flags & RADEON_IS_AGP) {
1165 size_bf = mc->gtt_start;
9ed8b1f9 1166 size_af = mc->mc_mask - mc->gtt_end;
d594e46a
JG		 1167 if (size_bf > size_af) {
1168 if (mc->mc_vram_size > size_bf) {
1169 dev_warn(rdev->dev, "limiting VRAM\n");
1170 mc->real_vram_size = size_bf;
1171 mc->mc_vram_size = size_bf;
1172 }
1173 mc->vram_start = mc->gtt_start - mc->mc_vram_size;
1174 } else {
1175 if (mc->mc_vram_size > size_af) {
1176 dev_warn(rdev->dev, "limiting VRAM\n");
1177 mc->real_vram_size = size_af;
1178 mc->mc_vram_size = size_af;
1179 }
dfc6ae5b 1180 mc->vram_start = mc->gtt_end + 1;
d594e46a
JG		 1181 }
1182 mc->vram_end = mc->vram_start + mc->mc_vram_size - 1;
1183 dev_info(rdev->dev, "VRAM: %lluM 0x%08llX - 0x%08llX (%lluM used)\n",
1184 mc->mc_vram_size >> 20, mc->vram_start,
1185 mc->vram_end, mc->real_vram_size >> 20);
1186 } else {
1187 u64 base = 0;
8961d52d
AD		 1188 if (rdev->flags & RADEON_IS_IGP) {
1189 base = RREG32(MC_VM_FB_LOCATION) & 0xFFFF;
1190 base <<= 24;
1191 }
d594e46a 1192 radeon_vram_location(rdev, &rdev->mc, base);
8d369bb1 1193 rdev->mc.gtt_base_align = 0;
d594e46a
JG		 1194 radeon_gtt_location(rdev, mc);
1195 }
1196}
1197
1109ca09 1198static int r600_mc_init(struct radeon_device *rdev)
771fe6b9 1199{
3ce0a23d 1200 u32 tmp;
5885b7a9 1201 int chansize, numchan;
65337e60
SL		 1202 uint32_t h_addr, l_addr;
1203 unsigned long long k8_addr;
771fe6b9 1204
3ce0a23d 1205 /* Get VRAM informations */
771fe6b9 1206 rdev->mc.vram_is_ddr = true;
3ce0a23d
JG		 1207 tmp = RREG32(RAMCFG);
1208 if (tmp & CHANSIZE_OVERRIDE) {
771fe6b9 1209 chansize = 16;
3ce0a23d 1210 } else if (tmp & CHANSIZE_MASK) {
771fe6b9
JG		 1211 chansize = 64;
1212 } else {
1213 chansize = 32;
1214 }
5885b7a9
AD		 1215 tmp = RREG32(CHMAP);
1216 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
1217 case 0:
1218 default:
1219 numchan = 1;
1220 break;
1221 case 1:
1222 numchan = 2;
1223 break;
1224 case 2:
1225 numchan = 4;
1226 break;
1227 case 3:
1228 numchan = 8;
1229 break;
771fe6b9 1230 }
5885b7a9 1231 rdev->mc.vram_width = numchan * chansize;
3ce0a23d 1232 /* Could aper size report 0 ? */
01d73a69
JC		 1233 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
1234 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
3ce0a23d
JG		 1235 /* Setup GPU memory space */
1236 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
1237 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
51e5fcd3 1238 rdev->mc.visible_vram_size = rdev->mc.aper_size;
d594e46a 1239 r600_vram_gtt_location(rdev, &rdev->mc);
f47299c5 1240
f892034a
AD		 1241 if (rdev->flags & RADEON_IS_IGP) {
1242 rs690_pm_info(rdev);
06b6476d 1243 rdev->mc.igp_sideport_enabled = radeon_atombios_sideport_present(rdev);
65337e60
SL		 1244
1245 if (rdev->family == CHIP_RS780 || rdev->family == CHIP_RS880) {
1246 /* Use K8 direct mapping for fast fb access. */
1247 rdev->fastfb_working = false;
1248 h_addr = G_000012_K8_ADDR_EXT(RREG32_MC(R_000012_MC_MISC_UMA_CNTL));
1249 l_addr = RREG32_MC(R_000011_K8_FB_LOCATION);
1250 k8_addr = ((unsigned long long)h_addr) << 32 | l_addr;
1251#if defined(CONFIG_X86_32) && !defined(CONFIG_X86_PAE)
1252 if (k8_addr + rdev->mc.visible_vram_size < 0x100000000ULL)
1253#endif
1254 {
1255 /* FastFB shall be used with UMA memory. Here it is simply disabled when sideport
1256 * memory is present.
1257 */
1258 if (rdev->mc.igp_sideport_enabled == false && radeon_fastfb == 1) {
1259 DRM_INFO("Direct mapping: aper base at 0x%llx, replaced by direct mapping base 0x%llx.\n",
1260 (unsigned long long)rdev->mc.aper_base, k8_addr);
1261 rdev->mc.aper_base = (resource_size_t)k8_addr;
1262 rdev->fastfb_working = true;
1263 }
1264 }
1265 }
f892034a 1266 }
65337e60 1267
f47299c5 1268 radeon_update_bandwidth_info(rdev);
3ce0a23d 1269 return 0;
771fe6b9
JG		 1270}
1271
16cdf04d
AD		 1272int r600_vram_scratch_init(struct radeon_device *rdev)
1273{
1274 int r;
1275
1276 if (rdev->vram_scratch.robj == NULL) {
1277 r = radeon_bo_create(rdev, RADEON_GPU_PAGE_SIZE,
1278 PAGE_SIZE, true, RADEON_GEM_DOMAIN_VRAM,
40f5cf99 1279 NULL, &rdev->vram_scratch.robj);
16cdf04d
AD		 1280 if (r) {
1281 return r;
1282 }
1283 }
1284
1285 r = radeon_bo_reserve(rdev->vram_scratch.robj, false);
1286 if (unlikely(r != 0))
1287 return r;
1288 r = radeon_bo_pin(rdev->vram_scratch.robj,
1289 RADEON_GEM_DOMAIN_VRAM, &rdev->vram_scratch.gpu_addr);
1290 if (r) {
1291 radeon_bo_unreserve(rdev->vram_scratch.robj);
1292 return r;
1293 }
1294 r = radeon_bo_kmap(rdev->vram_scratch.robj,
1295 (void **)&rdev->vram_scratch.ptr);
1296 if (r)
1297 radeon_bo_unpin(rdev->vram_scratch.robj);
1298 radeon_bo_unreserve(rdev->vram_scratch.robj);
1299
1300 return r;
1301}
1302
1303void r600_vram_scratch_fini(struct radeon_device *rdev)
1304{
1305 int r;
1306
1307 if (rdev->vram_scratch.robj == NULL) {
1308 return;
1309 }
1310 r = radeon_bo_reserve(rdev->vram_scratch.robj, false);
1311 if (likely(r == 0)) {
1312 radeon_bo_kunmap(rdev->vram_scratch.robj);
1313 radeon_bo_unpin(rdev->vram_scratch.robj);
1314 radeon_bo_unreserve(rdev->vram_scratch.robj);
1315 }
1316 radeon_bo_unref(&rdev->vram_scratch.robj);
1317}
1318
410a3418
AD		 1319void r600_set_bios_scratch_engine_hung(struct radeon_device *rdev, bool hung)
1320{
1321 u32 tmp = RREG32(R600_BIOS_3_SCRATCH);
1322
1323 if (hung)
1324 tmp |= ATOM_S3_ASIC_GUI_ENGINE_HUNG;
1325 else
1326 tmp &= ~ATOM_S3_ASIC_GUI_ENGINE_HUNG;
1327
1328 WREG32(R600_BIOS_3_SCRATCH, tmp);
1329}
1330
d3cb781e 1331static void r600_print_gpu_status_regs(struct radeon_device *rdev)
771fe6b9 1332{
64c56e8c 1333 dev_info(rdev->dev, " R_008010_GRBM_STATUS = 0x%08X\n",
d3cb781e 1334 RREG32(R_008010_GRBM_STATUS));
64c56e8c 1335 dev_info(rdev->dev, " R_008014_GRBM_STATUS2 = 0x%08X\n",
d3cb781e 1336 RREG32(R_008014_GRBM_STATUS2));
64c56e8c 1337 dev_info(rdev->dev, " R_000E50_SRBM_STATUS = 0x%08X\n",
d3cb781e 1338 RREG32(R_000E50_SRBM_STATUS));
440a7cd8 1339 dev_info(rdev->dev, " R_008674_CP_STALLED_STAT1 = 0x%08X\n",
d3cb781e 1340 RREG32(CP_STALLED_STAT1));
440a7cd8 1341 dev_info(rdev->dev, " R_008678_CP_STALLED_STAT2 = 0x%08X\n",
d3cb781e 1342 RREG32(CP_STALLED_STAT2));
440a7cd8 1343 dev_info(rdev->dev, " R_00867C_CP_BUSY_STAT = 0x%08X\n",
d3cb781e 1344 RREG32(CP_BUSY_STAT));
440a7cd8 1345 dev_info(rdev->dev, " R_008680_CP_STAT = 0x%08X\n",
d3cb781e 1346 RREG32(CP_STAT));
71e3d157
AD		 1347 dev_info(rdev->dev, " R_00D034_DMA_STATUS_REG = 0x%08X\n",
1348 RREG32(DMA_STATUS_REG));
1349}
1350
f13f7731 1351static bool r600_is_display_hung(struct radeon_device *rdev)
71e3d157 1352{
f13f7731
AD		 1353 u32 crtc_hung = 0;
1354 u32 crtc_status[2];
1355 u32 i, j, tmp;
1356
1357 for (i = 0; i < rdev->num_crtc; i++) {
1358 if (RREG32(AVIVO_D1CRTC_CONTROL + crtc_offsets[i]) & AVIVO_CRTC_EN) {
1359 crtc_status[i] = RREG32(AVIVO_D1CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
1360 crtc_hung |= (1 << i);
1361 }
1362 }
1363
1364 for (j = 0; j < 10; j++) {
1365 for (i = 0; i < rdev->num_crtc; i++) {
1366 if (crtc_hung & (1 << i)) {
1367 tmp = RREG32(AVIVO_D1CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
1368 if (tmp != crtc_status[i])
1369 crtc_hung &= ~(1 << i);
1370 }
1371 }
1372 if (crtc_hung == 0)
1373 return false;
1374 udelay(100);
1375 }
1376
1377 return true;
1378}
1379
1380static u32 r600_gpu_check_soft_reset(struct radeon_device *rdev)
1381{
1382 u32 reset_mask = 0;
d3cb781e 1383 u32 tmp;
71e3d157 1384
f13f7731
AD		 1385 /* GRBM_STATUS */
1386 tmp = RREG32(R_008010_GRBM_STATUS);
1387 if (rdev->family >= CHIP_RV770) {
1388 if (G_008010_PA_BUSY(tmp) | G_008010_SC_BUSY(tmp) |
1389 G_008010_SH_BUSY(tmp) | G_008010_SX_BUSY(tmp) |
1390 G_008010_TA_BUSY(tmp) | G_008010_VGT_BUSY(tmp) |
1391 G_008010_DB03_BUSY(tmp) | G_008010_CB03_BUSY(tmp) |
1392 G_008010_SPI03_BUSY(tmp) | G_008010_VGT_BUSY_NO_DMA(tmp))
1393 reset_mask |= RADEON_RESET_GFX;
1394 } else {
1395 if (G_008010_PA_BUSY(tmp) | G_008010_SC_BUSY(tmp) |
1396 G_008010_SH_BUSY(tmp) | G_008010_SX_BUSY(tmp) |
1397 G_008010_TA03_BUSY(tmp) | G_008010_VGT_BUSY(tmp) |
1398 G_008010_DB03_BUSY(tmp) | G_008010_CB03_BUSY(tmp) |
1399 G_008010_SPI03_BUSY(tmp) | G_008010_VGT_BUSY_NO_DMA(tmp))
1400 reset_mask |= RADEON_RESET_GFX;
1401 }
1402
1403 if (G_008010_CF_RQ_PENDING(tmp) | G_008010_PF_RQ_PENDING(tmp) |
1404 G_008010_CP_BUSY(tmp) | G_008010_CP_COHERENCY_BUSY(tmp))
1405 reset_mask |= RADEON_RESET_CP;
1406
1407 if (G_008010_GRBM_EE_BUSY(tmp))
1408 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
1409
1410 /* DMA_STATUS_REG */
1411 tmp = RREG32(DMA_STATUS_REG);
1412 if (!(tmp & DMA_IDLE))
1413 reset_mask |= RADEON_RESET_DMA;
1414
1415 /* SRBM_STATUS */
1416 tmp = RREG32(R_000E50_SRBM_STATUS);
1417 if (G_000E50_RLC_RQ_PENDING(tmp) | G_000E50_RLC_BUSY(tmp))
1418 reset_mask |= RADEON_RESET_RLC;
1419
1420 if (G_000E50_IH_BUSY(tmp))
1421 reset_mask |= RADEON_RESET_IH;
1422
1423 if (G_000E50_SEM_BUSY(tmp))
1424 reset_mask |= RADEON_RESET_SEM;
19fc42ed 1425
f13f7731
AD		 1426 if (G_000E50_GRBM_RQ_PENDING(tmp))
1427 reset_mask |= RADEON_RESET_GRBM;
1428
1429 if (G_000E50_VMC_BUSY(tmp))
1430 reset_mask |= RADEON_RESET_VMC;
1431
1432 if (G_000E50_MCB_BUSY(tmp) | G_000E50_MCDZ_BUSY(tmp) |
1433 G_000E50_MCDY_BUSY(tmp) | G_000E50_MCDX_BUSY(tmp) |
1434 G_000E50_MCDW_BUSY(tmp))
1435 reset_mask |= RADEON_RESET_MC;
1436
1437 if (r600_is_display_hung(rdev))
1438 reset_mask |= RADEON_RESET_DISPLAY;
1439
d808fc88
AD		 1440 /* Skip MC reset as it's mostly likely not hung, just busy */
1441 if (reset_mask & RADEON_RESET_MC) {
1442 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
1443 reset_mask &= ~RADEON_RESET_MC;
1444 }
1445
f13f7731
AD		 1446 return reset_mask;
1447}
1448
1449static void r600_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
1450{
1451 struct rv515_mc_save save;
1452 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
1453 u32 tmp;
19fc42ed 1454
71e3d157 1455 if (reset_mask == 0)
f13f7731 1456 return;
71e3d157
AD		 1457
1458 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
1459
d3cb781e
AD		 1460 r600_print_gpu_status_regs(rdev);
1461
d3cb781e
AD		 1462 /* Disable CP parsing/prefetching */
1463 if (rdev->family >= CHIP_RV770)
1464 WREG32(R_0086D8_CP_ME_CNTL, S_0086D8_CP_ME_HALT(1) | S_0086D8_CP_PFP_HALT(1));
1465 else
1466 WREG32(R_0086D8_CP_ME_CNTL, S_0086D8_CP_ME_HALT(1));
1467
1468 /* disable the RLC */
1469 WREG32(RLC_CNTL, 0);
1470
1471 if (reset_mask & RADEON_RESET_DMA) {
1472 /* Disable DMA */
1473 tmp = RREG32(DMA_RB_CNTL);
1474 tmp &= ~DMA_RB_ENABLE;
1475 WREG32(DMA_RB_CNTL, tmp);
1476 }
1477
1478 mdelay(50);
1479
ca57802e
AD		 1480 rv515_mc_stop(rdev, &save);
1481 if (r600_mc_wait_for_idle(rdev)) {
1482 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
1483 }
1484
d3cb781e
AD		 1485 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE)) {
1486 if (rdev->family >= CHIP_RV770)
1487 grbm_soft_reset |= S_008020_SOFT_RESET_DB(1) |
1488 S_008020_SOFT_RESET_CB(1) |
1489 S_008020_SOFT_RESET_PA(1) |
1490 S_008020_SOFT_RESET_SC(1) |
1491 S_008020_SOFT_RESET_SPI(1) |
1492 S_008020_SOFT_RESET_SX(1) |
1493 S_008020_SOFT_RESET_SH(1) |
1494 S_008020_SOFT_RESET_TC(1) |
1495 S_008020_SOFT_RESET_TA(1) |
1496 S_008020_SOFT_RESET_VC(1) |
1497 S_008020_SOFT_RESET_VGT(1);
1498 else
1499 grbm_soft_reset |= S_008020_SOFT_RESET_CR(1) |
1500 S_008020_SOFT_RESET_DB(1) |
1501 S_008020_SOFT_RESET_CB(1) |
1502 S_008020_SOFT_RESET_PA(1) |
1503 S_008020_SOFT_RESET_SC(1) |
1504 S_008020_SOFT_RESET_SMX(1) |
1505 S_008020_SOFT_RESET_SPI(1) |
1506 S_008020_SOFT_RESET_SX(1) |
1507 S_008020_SOFT_RESET_SH(1) |
1508 S_008020_SOFT_RESET_TC(1) |
1509 S_008020_SOFT_RESET_TA(1) |
1510 S_008020_SOFT_RESET_VC(1) |
1511 S_008020_SOFT_RESET_VGT(1);
1512 }
1513
1514 if (reset_mask & RADEON_RESET_CP) {
1515 grbm_soft_reset |= S_008020_SOFT_RESET_CP(1) |
1516 S_008020_SOFT_RESET_VGT(1);
1517
1518 srbm_soft_reset |= S_000E60_SOFT_RESET_GRBM(1);
1519 }
1520
1521 if (reset_mask & RADEON_RESET_DMA) {
1522 if (rdev->family >= CHIP_RV770)
1523 srbm_soft_reset |= RV770_SOFT_RESET_DMA;
1524 else
1525 srbm_soft_reset |= SOFT_RESET_DMA;
1526 }
1527
f13f7731
AD		 1528 if (reset_mask & RADEON_RESET_RLC)
1529 srbm_soft_reset |= S_000E60_SOFT_RESET_RLC(1);
1530
1531 if (reset_mask & RADEON_RESET_SEM)
1532 srbm_soft_reset |= S_000E60_SOFT_RESET_SEM(1);
1533
1534 if (reset_mask & RADEON_RESET_IH)
1535 srbm_soft_reset |= S_000E60_SOFT_RESET_IH(1);
1536
1537 if (reset_mask & RADEON_RESET_GRBM)
1538 srbm_soft_reset |= S_000E60_SOFT_RESET_GRBM(1);
1539
24178ec4
AD		 1540 if (!(rdev->flags & RADEON_IS_IGP)) {
1541 if (reset_mask & RADEON_RESET_MC)
1542 srbm_soft_reset |= S_000E60_SOFT_RESET_MC(1);
1543 }
f13f7731
AD		 1544
1545 if (reset_mask & RADEON_RESET_VMC)
1546 srbm_soft_reset |= S_000E60_SOFT_RESET_VMC(1);
1547
d3cb781e
AD		 1548 if (grbm_soft_reset) {
1549 tmp = RREG32(R_008020_GRBM_SOFT_RESET);
1550 tmp |= grbm_soft_reset;
1551 dev_info(rdev->dev, "R_008020_GRBM_SOFT_RESET=0x%08X\n", tmp);
1552 WREG32(R_008020_GRBM_SOFT_RESET, tmp);
1553 tmp = RREG32(R_008020_GRBM_SOFT_RESET);
1554
1555 udelay(50);
1556
1557 tmp &= ~grbm_soft_reset;
1558 WREG32(R_008020_GRBM_SOFT_RESET, tmp);
1559 tmp = RREG32(R_008020_GRBM_SOFT_RESET);
1560 }
1561
1562 if (srbm_soft_reset) {
1563 tmp = RREG32(SRBM_SOFT_RESET);
1564 tmp |= srbm_soft_reset;
1565 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
1566 WREG32(SRBM_SOFT_RESET, tmp);
1567 tmp = RREG32(SRBM_SOFT_RESET);
1568
1569 udelay(50);
71e3d157 1570
d3cb781e
AD		 1571 tmp &= ~srbm_soft_reset;
1572 WREG32(SRBM_SOFT_RESET, tmp);
1573 tmp = RREG32(SRBM_SOFT_RESET);
1574 }
71e3d157
AD		 1575
1576 /* Wait a little for things to settle down */
1577 mdelay(1);
1578
a3c1945a 1579 rv515_mc_resume(rdev, &save);
d3cb781e 1580 udelay(50);
410a3418 1581
d3cb781e 1582 r600_print_gpu_status_regs(rdev);
d3cb781e
AD		 1583}
1584
1585int r600_asic_reset(struct radeon_device *rdev)
1586{
f13f7731
AD		 1587 u32 reset_mask;
1588
1589 reset_mask = r600_gpu_check_soft_reset(rdev);
1590
1591 if (reset_mask)
1592 r600_set_bios_scratch_engine_hung(rdev, true);
1593
1594 r600_gpu_soft_reset(rdev, reset_mask);
1595
1596 reset_mask = r600_gpu_check_soft_reset(rdev);
1597
1598 if (!reset_mask)
1599 r600_set_bios_scratch_engine_hung(rdev, false);
1600
1601 return 0;
3ce0a23d
JG		 1602}
1603
123bc183
AD		 1604/**
1605 * r600_gfx_is_lockup - Check if the GFX engine is locked up
1606 *
1607 * @rdev: radeon_device pointer
1608 * @ring: radeon_ring structure holding ring information
1609 *
1610 * Check if the GFX engine is locked up.
1611 * Returns true if the engine appears to be locked up, false if not.
1612 */
1613bool r600_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
225758d8 1614{
123bc183
AD		 1615 u32 reset_mask = r600_gpu_check_soft_reset(rdev);
1616
1617 if (!(reset_mask & (RADEON_RESET_GFX |
1618 RADEON_RESET_COMPUTE |
1619 RADEON_RESET_CP))) {
069211e5 1620 radeon_ring_lockup_update(ring);
225758d8
JG		 1621 return false;
1622 }
1623 /* force CP activities */
7b9ef16b 1624 radeon_ring_force_activity(rdev, ring);
069211e5 1625 return radeon_ring_test_lockup(rdev, ring);
225758d8
JG		 1626}
1627
4d75658b
AD		 1628/**
1629 * r600_dma_is_lockup - Check if the DMA engine is locked up
1630 *
1631 * @rdev: radeon_device pointer
1632 * @ring: radeon_ring structure holding ring information
1633 *
123bc183 1634 * Check if the async DMA engine is locked up.
4d75658b
AD		 1635 * Returns true if the engine appears to be locked up, false if not.
1636 */
1637bool r600_dma_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
1638{
123bc183 1639 u32 reset_mask = r600_gpu_check_soft_reset(rdev);
4d75658b 1640
123bc183 1641 if (!(reset_mask & RADEON_RESET_DMA)) {
4d75658b
AD		 1642 radeon_ring_lockup_update(ring);
1643 return false;
1644 }
1645 /* force ring activities */
1646 radeon_ring_force_activity(rdev, ring);
1647 return radeon_ring_test_lockup(rdev, ring);
1648}
1649
416a2bd2
AD		 1650u32 r6xx_remap_render_backend(struct radeon_device *rdev,
1651 u32 tiling_pipe_num,
1652 u32 max_rb_num,
1653 u32 total_max_rb_num,
1654 u32 disabled_rb_mask)
3ce0a23d 1655{
416a2bd2 1656 u32 rendering_pipe_num, rb_num_width, req_rb_num;
f689e3ac 1657 u32 pipe_rb_ratio, pipe_rb_remain, tmp;
416a2bd2
AD		 1658 u32 data = 0, mask = 1 << (max_rb_num - 1);
1659 unsigned i, j;
3ce0a23d 1660
416a2bd2 1661 /* mask out the RBs that don't exist on that asic */
f689e3ac
MT		 1662 tmp = disabled_rb_mask | ((0xff << max_rb_num) & 0xff);
1663 /* make sure at least one RB is available */
1664 if ((tmp & 0xff) != 0xff)
1665 disabled_rb_mask = tmp;
3ce0a23d 1666
416a2bd2
AD		 1667 rendering_pipe_num = 1 << tiling_pipe_num;
1668 req_rb_num = total_max_rb_num - r600_count_pipe_bits(disabled_rb_mask);
1669 BUG_ON(rendering_pipe_num < req_rb_num);
3ce0a23d 1670
416a2bd2
AD		 1671 pipe_rb_ratio = rendering_pipe_num / req_rb_num;
1672 pipe_rb_remain = rendering_pipe_num - pipe_rb_ratio * req_rb_num;
3ce0a23d 1673
416a2bd2
AD		 1674 if (rdev->family <= CHIP_RV740) {
1675 /* r6xx/r7xx */
1676 rb_num_width = 2;
1677 } else {
1678 /* eg+ */
1679 rb_num_width = 4;
1680 }
3ce0a23d 1681
416a2bd2
AD		 1682 for (i = 0; i < max_rb_num; i++) {
1683 if (!(mask & disabled_rb_mask)) {
1684 for (j = 0; j < pipe_rb_ratio; j++) {
1685 data <<= rb_num_width;
1686 data |= max_rb_num - i - 1;
1687 }
1688 if (pipe_rb_remain) {
1689 data <<= rb_num_width;
1690 data |= max_rb_num - i - 1;
1691 pipe_rb_remain--;
1692 }
1693 }
1694 mask >>= 1;
3ce0a23d
JG		 1695 }
1696
416a2bd2 1697 return data;
3ce0a23d
JG		 1698}
1699
1700int r600_count_pipe_bits(uint32_t val)
1701{
ef8cf3a1 1702 return hweight32(val);
771fe6b9
JG		 1703}
1704
1109ca09 1705static void r600_gpu_init(struct radeon_device *rdev)
3ce0a23d
JG		 1706{
1707 u32 tiling_config;
1708 u32 ramcfg;
d03f5d59
AD		 1709 u32 cc_rb_backend_disable;
1710 u32 cc_gc_shader_pipe_config;
3ce0a23d
JG		 1711 u32 tmp;
1712 int i, j;
1713 u32 sq_config;
1714 u32 sq_gpr_resource_mgmt_1 = 0;
1715 u32 sq_gpr_resource_mgmt_2 = 0;
1716 u32 sq_thread_resource_mgmt = 0;
1717 u32 sq_stack_resource_mgmt_1 = 0;
1718 u32 sq_stack_resource_mgmt_2 = 0;
416a2bd2 1719 u32 disabled_rb_mask;
3ce0a23d 1720
416a2bd2 1721 rdev->config.r600.tiling_group_size = 256;
3ce0a23d
JG		 1722 switch (rdev->family) {
1723 case CHIP_R600:
1724 rdev->config.r600.max_pipes = 4;
1725 rdev->config.r600.max_tile_pipes = 8;
1726 rdev->config.r600.max_simds = 4;
1727 rdev->config.r600.max_backends = 4;
1728 rdev->config.r600.max_gprs = 256;
1729 rdev->config.r600.max_threads = 192;
1730 rdev->config.r600.max_stack_entries = 256;
1731 rdev->config.r600.max_hw_contexts = 8;
1732 rdev->config.r600.max_gs_threads = 16;
1733 rdev->config.r600.sx_max_export_size = 128;
1734 rdev->config.r600.sx_max_export_pos_size = 16;
1735 rdev->config.r600.sx_max_export_smx_size = 128;
1736 rdev->config.r600.sq_num_cf_insts = 2;
1737 break;
1738 case CHIP_RV630:
1739 case CHIP_RV635:
1740 rdev->config.r600.max_pipes = 2;
1741 rdev->config.r600.max_tile_pipes = 2;
1742 rdev->config.r600.max_simds = 3;
1743 rdev->config.r600.max_backends = 1;
1744 rdev->config.r600.max_gprs = 128;
1745 rdev->config.r600.max_threads = 192;
1746 rdev->config.r600.max_stack_entries = 128;
1747 rdev->config.r600.max_hw_contexts = 8;
1748 rdev->config.r600.max_gs_threads = 4;
1749 rdev->config.r600.sx_max_export_size = 128;
1750 rdev->config.r600.sx_max_export_pos_size = 16;
1751 rdev->config.r600.sx_max_export_smx_size = 128;
1752 rdev->config.r600.sq_num_cf_insts = 2;
1753 break;
1754 case CHIP_RV610:
1755 case CHIP_RV620:
1756 case CHIP_RS780:
1757 case CHIP_RS880:
1758 rdev->config.r600.max_pipes = 1;
1759 rdev->config.r600.max_tile_pipes = 1;
1760 rdev->config.r600.max_simds = 2;
1761 rdev->config.r600.max_backends = 1;
1762 rdev->config.r600.max_gprs = 128;
1763 rdev->config.r600.max_threads = 192;
1764 rdev->config.r600.max_stack_entries = 128;
1765 rdev->config.r600.max_hw_contexts = 4;
1766 rdev->config.r600.max_gs_threads = 4;
1767 rdev->config.r600.sx_max_export_size = 128;
1768 rdev->config.r600.sx_max_export_pos_size = 16;
1769 rdev->config.r600.sx_max_export_smx_size = 128;
1770 rdev->config.r600.sq_num_cf_insts = 1;
1771 break;
1772 case CHIP_RV670:
1773 rdev->config.r600.max_pipes = 4;
1774 rdev->config.r600.max_tile_pipes = 4;
1775 rdev->config.r600.max_simds = 4;
1776 rdev->config.r600.max_backends = 4;
1777 rdev->config.r600.max_gprs = 192;
1778 rdev->config.r600.max_threads = 192;
1779 rdev->config.r600.max_stack_entries = 256;
1780 rdev->config.r600.max_hw_contexts = 8;
1781 rdev->config.r600.max_gs_threads = 16;
1782 rdev->config.r600.sx_max_export_size = 128;
1783 rdev->config.r600.sx_max_export_pos_size = 16;
1784 rdev->config.r600.sx_max_export_smx_size = 128;
1785 rdev->config.r600.sq_num_cf_insts = 2;
1786 break;
1787 default:
1788 break;
1789 }
1790
1791 /* Initialize HDP */
1792 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
1793 WREG32((0x2c14 + j), 0x00000000);
1794 WREG32((0x2c18 + j), 0x00000000);
1795 WREG32((0x2c1c + j), 0x00000000);
1796 WREG32((0x2c20 + j), 0x00000000);
1797 WREG32((0x2c24 + j), 0x00000000);
1798 }
1799
1800 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
1801
1802 /* Setup tiling */
1803 tiling_config = 0;
1804 ramcfg = RREG32(RAMCFG);
1805 switch (rdev->config.r600.max_tile_pipes) {
1806 case 1:
1807 tiling_config |= PIPE_TILING(0);
1808 break;
1809 case 2:
1810 tiling_config |= PIPE_TILING(1);
1811 break;
1812 case 4:
1813 tiling_config |= PIPE_TILING(2);
1814 break;
1815 case 8:
1816 tiling_config |= PIPE_TILING(3);
1817 break;
1818 default:
1819 break;
1820 }
d03f5d59 1821 rdev->config.r600.tiling_npipes = rdev->config.r600.max_tile_pipes;
961fb597 1822 rdev->config.r600.tiling_nbanks = 4 << ((ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT);
3ce0a23d 1823 tiling_config |= BANK_TILING((ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT);
881fe6c1 1824 tiling_config |= GROUP_SIZE((ramcfg & BURSTLENGTH_MASK) >> BURSTLENGTH_SHIFT);
416a2bd2 1825
3ce0a23d
JG		 1826 tmp = (ramcfg & NOOFROWS_MASK) >> NOOFROWS_SHIFT;
1827 if (tmp > 3) {
1828 tiling_config |= ROW_TILING(3);
1829 tiling_config |= SAMPLE_SPLIT(3);
1830 } else {
1831 tiling_config |= ROW_TILING(tmp);
1832 tiling_config |= SAMPLE_SPLIT(tmp);
1833 }
1834 tiling_config |= BANK_SWAPS(1);
d03f5d59
AD		 1835
1836 cc_rb_backend_disable = RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000;
416a2bd2
AD		 1837 tmp = R6XX_MAX_BACKENDS -
1838 r600_count_pipe_bits((cc_rb_backend_disable >> 16) & R6XX_MAX_BACKENDS_MASK);
1839 if (tmp < rdev->config.r600.max_backends) {
1840 rdev->config.r600.max_backends = tmp;
1841 }
1842
1843 cc_gc_shader_pipe_config = RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0x00ffff00;
1844 tmp = R6XX_MAX_PIPES -
1845 r600_count_pipe_bits((cc_gc_shader_pipe_config >> 8) & R6XX_MAX_PIPES_MASK);
1846 if (tmp < rdev->config.r600.max_pipes) {
1847 rdev->config.r600.max_pipes = tmp;
1848 }
1849 tmp = R6XX_MAX_SIMDS -
1850 r600_count_pipe_bits((cc_gc_shader_pipe_config >> 16) & R6XX_MAX_SIMDS_MASK);
1851 if (tmp < rdev->config.r600.max_simds) {
1852 rdev->config.r600.max_simds = tmp;
1853 }
1854
1855 disabled_rb_mask = (RREG32(CC_RB_BACKEND_DISABLE) >> 16) & R6XX_MAX_BACKENDS_MASK;
1856 tmp = (tiling_config & PIPE_TILING__MASK) >> PIPE_TILING__SHIFT;
1857 tmp = r6xx_remap_render_backend(rdev, tmp, rdev->config.r600.max_backends,
1858 R6XX_MAX_BACKENDS, disabled_rb_mask);
1859 tiling_config |= tmp << 16;
1860 rdev->config.r600.backend_map = tmp;
1861
e7aeeba6 1862 rdev->config.r600.tile_config = tiling_config;
3ce0a23d
JG		 1863 WREG32(GB_TILING_CONFIG, tiling_config);
1864 WREG32(DCP_TILING_CONFIG, tiling_config & 0xffff);
1865 WREG32(HDP_TILING_CONFIG, tiling_config & 0xffff);
4d75658b 1866 WREG32(DMA_TILING_CONFIG, tiling_config & 0xffff);
3ce0a23d 1867
d03f5d59 1868 tmp = R6XX_MAX_PIPES - r600_count_pipe_bits((cc_gc_shader_pipe_config & INACTIVE_QD_PIPES_MASK) >> 8);
3ce0a23d
JG		 1869 WREG32(VGT_OUT_DEALLOC_CNTL, (tmp * 4) & DEALLOC_DIST_MASK);
1870 WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, ((tmp * 4) - 2) & VTX_REUSE_DEPTH_MASK);
1871
1872 /* Setup some CP states */
1873 WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) | ROQ_IB2_START(0x2b)));
1874 WREG32(CP_MEQ_THRESHOLDS, (MEQ_END(0x40) | ROQ_END(0x40)));
1875
1876 WREG32(TA_CNTL_AUX, (DISABLE_CUBE_ANISO | SYNC_GRADIENT |
1877 SYNC_WALKER | SYNC_ALIGNER));
1878 /* Setup various GPU states */
1879 if (rdev->family == CHIP_RV670)
1880 WREG32(ARB_GDEC_RD_CNTL, 0x00000021);
1881
1882 tmp = RREG32(SX_DEBUG_1);
1883 tmp |= SMX_EVENT_RELEASE;
1884 if ((rdev->family > CHIP_R600))
1885 tmp |= ENABLE_NEW_SMX_ADDRESS;
1886 WREG32(SX_DEBUG_1, tmp);
1887
1888 if (((rdev->family) == CHIP_R600) ||
1889 ((rdev->family) == CHIP_RV630) ||
1890 ((rdev->family) == CHIP_RV610) ||
1891 ((rdev->family) == CHIP_RV620) ||
ee59f2b4
AD		 1892 ((rdev->family) == CHIP_RS780) ||
1893 ((rdev->family) == CHIP_RS880)) {
3ce0a23d
JG		 1894 WREG32(DB_DEBUG, PREZ_MUST_WAIT_FOR_POSTZ_DONE);
1895 } else {
1896 WREG32(DB_DEBUG, 0);
1897 }
1898 WREG32(DB_WATERMARKS, (DEPTH_FREE(4) | DEPTH_CACHELINE_FREE(16) |
1899 DEPTH_FLUSH(16) | DEPTH_PENDING_FREE(4)));
1900
1901 WREG32(PA_SC_MULTI_CHIP_CNTL, 0);
1902 WREG32(VGT_NUM_INSTANCES, 0);
1903
1904 WREG32(SPI_CONFIG_CNTL, GPR_WRITE_PRIORITY(0));
1905 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(0));
1906
1907 tmp = RREG32(SQ_MS_FIFO_SIZES);
1908 if (((rdev->family) == CHIP_RV610) ||
1909 ((rdev->family) == CHIP_RV620) ||
ee59f2b4
AD		 1910 ((rdev->family) == CHIP_RS780) ||
1911 ((rdev->family) == CHIP_RS880)) {
3ce0a23d
JG		 1912 tmp = (CACHE_FIFO_SIZE(0xa) |
1913 FETCH_FIFO_HIWATER(0xa) |
1914 DONE_FIFO_HIWATER(0xe0) |
1915 ALU_UPDATE_FIFO_HIWATER(0x8));
1916 } else if (((rdev->family) == CHIP_R600) ||
1917 ((rdev->family) == CHIP_RV630)) {
1918 tmp &= ~DONE_FIFO_HIWATER(0xff);
1919 tmp |= DONE_FIFO_HIWATER(0x4);
1920 }
1921 WREG32(SQ_MS_FIFO_SIZES, tmp);
1922
1923 /* SQ_CONFIG, SQ_GPR_RESOURCE_MGMT, SQ_THREAD_RESOURCE_MGMT, SQ_STACK_RESOURCE_MGMT
1924 * should be adjusted as needed by the 2D/3D drivers. This just sets default values
1925 */
1926 sq_config = RREG32(SQ_CONFIG);
1927 sq_config &= ~(PS_PRIO(3) |
1928 VS_PRIO(3) |
1929 GS_PRIO(3) |
1930 ES_PRIO(3));
1931 sq_config |= (DX9_CONSTS |
1932 VC_ENABLE |
1933 PS_PRIO(0) |
1934 VS_PRIO(1) |
1935 GS_PRIO(2) |
1936 ES_PRIO(3));
1937
1938 if ((rdev->family) == CHIP_R600) {
1939 sq_gpr_resource_mgmt_1 = (NUM_PS_GPRS(124) |
1940 NUM_VS_GPRS(124) |
1941 NUM_CLAUSE_TEMP_GPRS(4));
1942 sq_gpr_resource_mgmt_2 = (NUM_GS_GPRS(0) |
1943 NUM_ES_GPRS(0));
1944 sq_thread_resource_mgmt = (NUM_PS_THREADS(136) |
1945 NUM_VS_THREADS(48) |
1946 NUM_GS_THREADS(4) |
1947 NUM_ES_THREADS(4));
1948 sq_stack_resource_mgmt_1 = (NUM_PS_STACK_ENTRIES(128) |
1949 NUM_VS_STACK_ENTRIES(128));
1950 sq_stack_resource_mgmt_2 = (NUM_GS_STACK_ENTRIES(0) |
1951 NUM_ES_STACK_ENTRIES(0));
1952 } else if (((rdev->family) == CHIP_RV610) ||
1953 ((rdev->family) == CHIP_RV620) ||
ee59f2b4
AD		 1954 ((rdev->family) == CHIP_RS780) ||
1955 ((rdev->family) == CHIP_RS880)) {
3ce0a23d
JG		 1956 /* no vertex cache */
1957 sq_config &= ~VC_ENABLE;
1958
1959 sq_gpr_resource_mgmt_1 = (NUM_PS_GPRS(44) |
1960 NUM_VS_GPRS(44) |
1961 NUM_CLAUSE_TEMP_GPRS(2));
1962 sq_gpr_resource_mgmt_2 = (NUM_GS_GPRS(17) |
1963 NUM_ES_GPRS(17));
1964 sq_thread_resource_mgmt = (NUM_PS_THREADS(79) |
1965 NUM_VS_THREADS(78) |
1966 NUM_GS_THREADS(4) |
1967 NUM_ES_THREADS(31));
1968 sq_stack_resource_mgmt_1 = (NUM_PS_STACK_ENTRIES(40) |
1969 NUM_VS_STACK_ENTRIES(40));
1970 sq_stack_resource_mgmt_2 = (NUM_GS_STACK_ENTRIES(32) |
1971 NUM_ES_STACK_ENTRIES(16));
1972 } else if (((rdev->family) == CHIP_RV630) ||
1973 ((rdev->family) == CHIP_RV635)) {
1974 sq_gpr_resource_mgmt_1 = (NUM_PS_GPRS(44) |
1975 NUM_VS_GPRS(44) |
1976 NUM_CLAUSE_TEMP_GPRS(2));
1977 sq_gpr_resource_mgmt_2 = (NUM_GS_GPRS(18) |
1978 NUM_ES_GPRS(18));
1979 sq_thread_resource_mgmt = (NUM_PS_THREADS(79) |
1980 NUM_VS_THREADS(78) |
1981 NUM_GS_THREADS(4) |
1982 NUM_ES_THREADS(31));
1983 sq_stack_resource_mgmt_1 = (NUM_PS_STACK_ENTRIES(40) |
1984 NUM_VS_STACK_ENTRIES(40));
1985 sq_stack_resource_mgmt_2 = (NUM_GS_STACK_ENTRIES(32) |
1986 NUM_ES_STACK_ENTRIES(16));
1987 } else if ((rdev->family) == CHIP_RV670) {
1988 sq_gpr_resource_mgmt_1 = (NUM_PS_GPRS(44) |
1989 NUM_VS_GPRS(44) |
1990 NUM_CLAUSE_TEMP_GPRS(2));
1991 sq_gpr_resource_mgmt_2 = (NUM_GS_GPRS(17) |
1992 NUM_ES_GPRS(17));
1993 sq_thread_resource_mgmt = (NUM_PS_THREADS(79) |
1994 NUM_VS_THREADS(78) |
1995 NUM_GS_THREADS(4) |
1996 NUM_ES_THREADS(31));
1997 sq_stack_resource_mgmt_1 = (NUM_PS_STACK_ENTRIES(64) |
1998 NUM_VS_STACK_ENTRIES(64));
1999 sq_stack_resource_mgmt_2 = (NUM_GS_STACK_ENTRIES(64) |
2000 NUM_ES_STACK_ENTRIES(64));
2001 }
2002
2003 WREG32(SQ_CONFIG, sq_config);
2004 WREG32(SQ_GPR_RESOURCE_MGMT_1, sq_gpr_resource_mgmt_1);
2005 WREG32(SQ_GPR_RESOURCE_MGMT_2, sq_gpr_resource_mgmt_2);
2006 WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
2007 WREG32(SQ_STACK_RESOURCE_MGMT_1, sq_stack_resource_mgmt_1);
2008 WREG32(SQ_STACK_RESOURCE_MGMT_2, sq_stack_resource_mgmt_2);
2009
2010 if (((rdev->family) == CHIP_RV610) ||
2011 ((rdev->family) == CHIP_RV620) ||
ee59f2b4
AD		 2012 ((rdev->family) == CHIP_RS780) ||
2013 ((rdev->family) == CHIP_RS880)) {
3ce0a23d
JG		 2014 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(TC_ONLY));
2015 } else {
2016 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC));
2017 }
2018
2019 /* More default values. 2D/3D driver should adjust as needed */
2020 WREG32(PA_SC_AA_SAMPLE_LOCS_2S, (S0_X(0xc) | S0_Y(0x4) |
2021 S1_X(0x4) | S1_Y(0xc)));
2022 WREG32(PA_SC_AA_SAMPLE_LOCS_4S, (S0_X(0xe) | S0_Y(0xe) |
2023 S1_X(0x2) | S1_Y(0x2) |
2024 S2_X(0xa) | S2_Y(0x6) |
2025 S3_X(0x6) | S3_Y(0xa)));
2026 WREG32(PA_SC_AA_SAMPLE_LOCS_8S_WD0, (S0_X(0xe) | S0_Y(0xb) |
2027 S1_X(0x4) | S1_Y(0xc) |
2028 S2_X(0x1) | S2_Y(0x6) |
2029 S3_X(0xa) | S3_Y(0xe)));
2030 WREG32(PA_SC_AA_SAMPLE_LOCS_8S_WD1, (S4_X(0x6) | S4_Y(0x1) |
2031 S5_X(0x0) | S5_Y(0x0) |
2032 S6_X(0xb) | S6_Y(0x4) |
2033 S7_X(0x7) | S7_Y(0x8)));
2034
2035 WREG32(VGT_STRMOUT_EN, 0);
2036 tmp = rdev->config.r600.max_pipes * 16;
2037 switch (rdev->family) {
2038 case CHIP_RV610:
3ce0a23d 2039 case CHIP_RV620:
ee59f2b4
AD		 2040 case CHIP_RS780:
2041 case CHIP_RS880:
3ce0a23d
JG		 2042 tmp += 32;
2043 break;
2044 case CHIP_RV670:
2045 tmp += 128;
2046 break;
2047 default:
2048 break;
2049 }
2050 if (tmp > 256) {
2051 tmp = 256;
2052 }
2053 WREG32(VGT_ES_PER_GS, 128);
2054 WREG32(VGT_GS_PER_ES, tmp);
2055 WREG32(VGT_GS_PER_VS, 2);
2056 WREG32(VGT_GS_VERTEX_REUSE, 16);
2057
2058 /* more default values. 2D/3D driver should adjust as needed */
2059 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
2060 WREG32(VGT_STRMOUT_EN, 0);
2061 WREG32(SX_MISC, 0);
2062 WREG32(PA_SC_MODE_CNTL, 0);
2063 WREG32(PA_SC_AA_CONFIG, 0);
2064 WREG32(PA_SC_LINE_STIPPLE, 0);
2065 WREG32(SPI_INPUT_Z, 0);
2066 WREG32(SPI_PS_IN_CONTROL_0, NUM_INTERP(2));
2067 WREG32(CB_COLOR7_FRAG, 0);
2068
2069 /* Clear render buffer base addresses */
2070 WREG32(CB_COLOR0_BASE, 0);
2071 WREG32(CB_COLOR1_BASE, 0);
2072 WREG32(CB_COLOR2_BASE, 0);
2073 WREG32(CB_COLOR3_BASE, 0);
2074 WREG32(CB_COLOR4_BASE, 0);
2075 WREG32(CB_COLOR5_BASE, 0);
2076 WREG32(CB_COLOR6_BASE, 0);
2077 WREG32(CB_COLOR7_BASE, 0);
2078 WREG32(CB_COLOR7_FRAG, 0);
2079
2080 switch (rdev->family) {
2081 case CHIP_RV610:
3ce0a23d 2082 case CHIP_RV620:
ee59f2b4
AD		 2083 case CHIP_RS780:
2084 case CHIP_RS880:
3ce0a23d
JG		 2085 tmp = TC_L2_SIZE(8);
2086 break;
2087 case CHIP_RV630:
2088 case CHIP_RV635:
2089 tmp = TC_L2_SIZE(4);
2090 break;
2091 case CHIP_R600:
2092 tmp = TC_L2_SIZE(0) | L2_DISABLE_LATE_HIT;
2093 break;
2094 default:
2095 tmp = TC_L2_SIZE(0);
2096 break;
2097 }
2098 WREG32(TC_CNTL, tmp);
2099
2100 tmp = RREG32(HDP_HOST_PATH_CNTL);
2101 WREG32(HDP_HOST_PATH_CNTL, tmp);
2102
2103 tmp = RREG32(ARB_POP);
2104 tmp |= ENABLE_TC128;
2105 WREG32(ARB_POP, tmp);
2106
2107 WREG32(PA_SC_MULTI_CHIP_CNTL, 0);
2108 WREG32(PA_CL_ENHANCE, (CLIP_VTX_REORDER_ENA |
2109 NUM_CLIP_SEQ(3)));
2110 WREG32(PA_SC_ENHANCE, FORCE_EOV_MAX_CLK_CNT(4095));
b866d133 2111 WREG32(VC_ENHANCE, 0);
3ce0a23d
JG		 2112}
2113
2114
771fe6b9
JG		 2115/*
2116 * Indirect registers accessor
2117 */
3ce0a23d
JG		 2118u32 r600_pciep_rreg(struct radeon_device *rdev, u32 reg)
2119{
2120 u32 r;
2121
2122 WREG32(PCIE_PORT_INDEX, ((reg) & 0xff));
2123 (void)RREG32(PCIE_PORT_INDEX);
2124 r = RREG32(PCIE_PORT_DATA);
2125 return r;
2126}
2127
2128void r600_pciep_wreg(struct radeon_device *rdev, u32 reg, u32 v)
2129{
2130 WREG32(PCIE_PORT_INDEX, ((reg) & 0xff));
2131 (void)RREG32(PCIE_PORT_INDEX);
2132 WREG32(PCIE_PORT_DATA, (v));
2133 (void)RREG32(PCIE_PORT_DATA);
2134}
2135
3ce0a23d
JG		 2136/*
2137 * CP & Ring
2138 */
2139void r600_cp_stop(struct radeon_device *rdev)
2140{
53595338 2141 radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
3ce0a23d 2142 WREG32(R_0086D8_CP_ME_CNTL, S_0086D8_CP_ME_HALT(1));
724c80e1 2143 WREG32(SCRATCH_UMSK, 0);
4d75658b 2144 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3ce0a23d
JG		 2145}
2146
d8f60cfc 2147int r600_init_microcode(struct radeon_device *rdev)
3ce0a23d
JG		 2148{
2149 struct platform_device *pdev;
2150 const char *chip_name;
d8f60cfc
AD		 2151 const char *rlc_chip_name;
2152 size_t pfp_req_size, me_req_size, rlc_req_size;
3ce0a23d
JG		 2153 char fw_name[30];
2154 int err;
2155
2156 DRM_DEBUG("\n");
2157
2158 pdev = platform_device_register_simple("radeon_cp", 0, NULL, 0);
2159 err = IS_ERR(pdev);
2160 if (err) {
2161 printk(KERN_ERR "radeon_cp: Failed to register firmware\n");
2162 return -EINVAL;
2163 }
2164
2165 switch (rdev->family) {
d8f60cfc
AD		 2166 case CHIP_R600:
2167 chip_name = "R600";
2168 rlc_chip_name = "R600";
2169 break;
2170 case CHIP_RV610:
2171 chip_name = "RV610";
2172 rlc_chip_name = "R600";
2173 break;
2174 case CHIP_RV630:
2175 chip_name = "RV630";
2176 rlc_chip_name = "R600";
2177 break;
2178 case CHIP_RV620:
2179 chip_name = "RV620";
2180 rlc_chip_name = "R600";
2181 break;
2182 case CHIP_RV635:
2183 chip_name = "RV635";
2184 rlc_chip_name = "R600";
2185 break;
2186 case CHIP_RV670:
2187 chip_name = "RV670";
2188 rlc_chip_name = "R600";
2189 break;
3ce0a23d 2190 case CHIP_RS780:
d8f60cfc
AD		 2191 case CHIP_RS880:
2192 chip_name = "RS780";
2193 rlc_chip_name = "R600";
2194 break;
2195 case CHIP_RV770:
2196 chip_name = "RV770";
2197 rlc_chip_name = "R700";
2198 break;
3ce0a23d 2199 case CHIP_RV730:
d8f60cfc
AD		 2200 case CHIP_RV740:
2201 chip_name = "RV730";
2202 rlc_chip_name = "R700";
2203 break;
2204 case CHIP_RV710:
2205 chip_name = "RV710";
2206 rlc_chip_name = "R700";
2207 break;
fe251e2f
AD		 2208 case CHIP_CEDAR:
2209 chip_name = "CEDAR";
45f9a39b 2210 rlc_chip_name = "CEDAR";
fe251e2f
AD		 2211 break;
2212 case CHIP_REDWOOD:
2213 chip_name = "REDWOOD";
45f9a39b 2214 rlc_chip_name = "REDWOOD";
fe251e2f
AD		 2215 break;
2216 case CHIP_JUNIPER:
2217 chip_name = "JUNIPER";
45f9a39b 2218 rlc_chip_name = "JUNIPER";
fe251e2f
AD		 2219 break;
2220 case CHIP_CYPRESS:
2221 case CHIP_HEMLOCK:
2222 chip_name = "CYPRESS";
45f9a39b 2223 rlc_chip_name = "CYPRESS";
fe251e2f 2224 break;
439bd6cd
AD		 2225 case CHIP_PALM:
2226 chip_name = "PALM";
2227 rlc_chip_name = "SUMO";
2228 break;
d5c5a72f
AD		 2229 case CHIP_SUMO:
2230 chip_name = "SUMO";
2231 rlc_chip_name = "SUMO";
2232 break;
2233 case CHIP_SUMO2:
2234 chip_name = "SUMO2";
2235 rlc_chip_name = "SUMO";
2236 break;
3ce0a23d
JG		 2237 default: BUG();
2238 }
2239
fe251e2f
AD		 2240 if (rdev->family >= CHIP_CEDAR) {
2241 pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4;
2242 me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4;
45f9a39b 2243 rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4;
fe251e2f 2244 } else if (rdev->family >= CHIP_RV770) {
3ce0a23d
JG		 2245 pfp_req_size = R700_PFP_UCODE_SIZE * 4;
2246 me_req_size = R700_PM4_UCODE_SIZE * 4;
d8f60cfc 2247 rlc_req_size = R700_RLC_UCODE_SIZE * 4;
3ce0a23d
JG		 2248 } else {
2249 pfp_req_size = PFP_UCODE_SIZE * 4;
2250 me_req_size = PM4_UCODE_SIZE * 12;
d8f60cfc 2251 rlc_req_size = RLC_UCODE_SIZE * 4;
3ce0a23d
JG		 2252 }
2253
d8f60cfc 2254 DRM_INFO("Loading %s Microcode\n", chip_name);
3ce0a23d
JG		 2255
2256 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
2257 err = request_firmware(&rdev->pfp_fw, fw_name, &pdev->dev);
2258 if (err)
2259 goto out;
2260 if (rdev->pfp_fw->size != pfp_req_size) {
2261 printk(KERN_ERR
2262 "r600_cp: Bogus length %zu in firmware \"%s\"\n",
2263 rdev->pfp_fw->size, fw_name);
2264 err = -EINVAL;
2265 goto out;
2266 }
2267
2268 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
2269 err = request_firmware(&rdev->me_fw, fw_name, &pdev->dev);
2270 if (err)
2271 goto out;
2272 if (rdev->me_fw->size != me_req_size) {
2273 printk(KERN_ERR
2274 "r600_cp: Bogus length %zu in firmware \"%s\"\n",
2275 rdev->me_fw->size, fw_name);
2276 err = -EINVAL;
2277 }
d8f60cfc
AD		 2278
2279 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", rlc_chip_name);
2280 err = request_firmware(&rdev->rlc_fw, fw_name, &pdev->dev);
2281 if (err)
2282 goto out;
2283 if (rdev->rlc_fw->size != rlc_req_size) {
2284 printk(KERN_ERR
2285 "r600_rlc: Bogus length %zu in firmware \"%s\"\n",
2286 rdev->rlc_fw->size, fw_name);
2287 err = -EINVAL;
2288 }
2289
3ce0a23d
JG		 2290out:
2291 platform_device_unregister(pdev);
2292
2293 if (err) {
2294 if (err != -EINVAL)
2295 printk(KERN_ERR
2296 "r600_cp: Failed to load firmware \"%s\"\n",
2297 fw_name);
2298 release_firmware(rdev->pfp_fw);
2299 rdev->pfp_fw = NULL;
2300 release_firmware(rdev->me_fw);
2301 rdev->me_fw = NULL;
d8f60cfc
AD		 2302 release_firmware(rdev->rlc_fw);
2303 rdev->rlc_fw = NULL;
3ce0a23d
JG		 2304 }
2305 return err;
2306}
2307
2308static int r600_cp_load_microcode(struct radeon_device *rdev)
2309{
2310 const __be32 *fw_data;
2311 int i;
2312
2313 if (!rdev->me_fw || !rdev->pfp_fw)
2314 return -EINVAL;
2315
2316 r600_cp_stop(rdev);
2317
4eace7fd
CC		 2318 WREG32(CP_RB_CNTL,
2319#ifdef __BIG_ENDIAN
2320 BUF_SWAP_32BIT |
2321#endif
2322 RB_NO_UPDATE | RB_BLKSZ(15) | RB_BUFSZ(3));
3ce0a23d
JG		 2323
2324 /* Reset cp */
2325 WREG32(GRBM_SOFT_RESET, SOFT_RESET_CP);
2326 RREG32(GRBM_SOFT_RESET);
2327 mdelay(15);
2328 WREG32(GRBM_SOFT_RESET, 0);
2329
2330 WREG32(CP_ME_RAM_WADDR, 0);
2331
2332 fw_data = (const __be32 *)rdev->me_fw->data;
2333 WREG32(CP_ME_RAM_WADDR, 0);
2334 for (i = 0; i < PM4_UCODE_SIZE * 3; i++)
2335 WREG32(CP_ME_RAM_DATA,
2336 be32_to_cpup(fw_data++));
2337
2338 fw_data = (const __be32 *)rdev->pfp_fw->data;
2339 WREG32(CP_PFP_UCODE_ADDR, 0);
2340 for (i = 0; i < PFP_UCODE_SIZE; i++)
2341 WREG32(CP_PFP_UCODE_DATA,
2342 be32_to_cpup(fw_data++));
2343
2344 WREG32(CP_PFP_UCODE_ADDR, 0);
2345 WREG32(CP_ME_RAM_WADDR, 0);
2346 WREG32(CP_ME_RAM_RADDR, 0);
2347 return 0;
2348}
2349
2350int r600_cp_start(struct radeon_device *rdev)
2351{
e32eb50d 2352 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3ce0a23d
JG		 2353 int r;
2354 uint32_t cp_me;
2355
e32eb50d 2356 r = radeon_ring_lock(rdev, ring, 7);
3ce0a23d
JG		 2357 if (r) {
2358 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
2359 return r;
2360 }
e32eb50d
CK		 2361 radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
2362 radeon_ring_write(ring, 0x1);
7e7b41d2 2363 if (rdev->family >= CHIP_RV770) {
e32eb50d
CK		 2364 radeon_ring_write(ring, 0x0);
2365 radeon_ring_write(ring, rdev->config.rv770.max_hw_contexts - 1);
fe251e2f 2366 } else {
e32eb50d
CK		 2367 radeon_ring_write(ring, 0x3);
2368 radeon_ring_write(ring, rdev->config.r600.max_hw_contexts - 1);
3ce0a23d 2369 }
e32eb50d
CK		 2370 radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
2371 radeon_ring_write(ring, 0);
2372 radeon_ring_write(ring, 0);
2373 radeon_ring_unlock_commit(rdev, ring);
3ce0a23d
JG		 2374
2375 cp_me = 0xff;
2376 WREG32(R_0086D8_CP_ME_CNTL, cp_me);
2377 return 0;
2378}
2379
2380int r600_cp_resume(struct radeon_device *rdev)
2381{
e32eb50d 2382 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3ce0a23d
JG		 2383 u32 tmp;
2384 u32 rb_bufsz;
2385 int r;
2386
2387 /* Reset cp */
2388 WREG32(GRBM_SOFT_RESET, SOFT_RESET_CP);
2389 RREG32(GRBM_SOFT_RESET);
2390 mdelay(15);
2391 WREG32(GRBM_SOFT_RESET, 0);
2392
2393 /* Set ring buffer size */
e32eb50d 2394 rb_bufsz = drm_order(ring->ring_size / 8);
724c80e1 2395 tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3ce0a23d 2396#ifdef __BIG_ENDIAN
d6f28938 2397 tmp |= BUF_SWAP_32BIT;
3ce0a23d 2398#endif
d6f28938 2399 WREG32(CP_RB_CNTL, tmp);
15d3332f 2400 WREG32(CP_SEM_WAIT_TIMER, 0x0);
3ce0a23d
JG		 2401
2402 /* Set the write pointer delay */
2403 WREG32(CP_RB_WPTR_DELAY, 0);
2404
2405 /* Initialize the ring buffer's read and write pointers */
3ce0a23d
JG		 2406 WREG32(CP_RB_CNTL, tmp | RB_RPTR_WR_ENA);
2407 WREG32(CP_RB_RPTR_WR, 0);
e32eb50d
CK		 2408 ring->wptr = 0;
2409 WREG32(CP_RB_WPTR, ring->wptr);
724c80e1
AD		 2410
2411 /* set the wb address whether it's enabled or not */
4eace7fd 2412 WREG32(CP_RB_RPTR_ADDR,
4eace7fd 2413 ((rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC));
724c80e1
AD		 2414 WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
2415 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
2416
2417 if (rdev->wb.enabled)
2418 WREG32(SCRATCH_UMSK, 0xff);
2419 else {
2420 tmp |= RB_NO_UPDATE;
2421 WREG32(SCRATCH_UMSK, 0);
2422 }
2423
3ce0a23d
JG		 2424 mdelay(1);
2425 WREG32(CP_RB_CNTL, tmp);
2426
e32eb50d 2427 WREG32(CP_RB_BASE, ring->gpu_addr >> 8);
3ce0a23d
JG		 2428 WREG32(CP_DEBUG, (1 << 27) | (1 << 28));
2429
e32eb50d 2430 ring->rptr = RREG32(CP_RB_RPTR);
3ce0a23d
JG		 2431
2432 r600_cp_start(rdev);
e32eb50d 2433 ring->ready = true;
f712812e 2434 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, ring);
3ce0a23d 2435 if (r) {
e32eb50d 2436 ring->ready = false;
3ce0a23d
JG		 2437 return r;
2438 }
2439 return 0;
2440}
2441
e32eb50d 2442void r600_ring_init(struct radeon_device *rdev, struct radeon_ring *ring, unsigned ring_size)
3ce0a23d
JG		 2443{
2444 u32 rb_bufsz;
45df6803 2445 int r;
3ce0a23d
JG		 2446
2447 /* Align ring size */
2448 rb_bufsz = drm_order(ring_size / 8);
2449 ring_size = (1 << (rb_bufsz + 1)) * 4;
e32eb50d
CK		 2450 ring->ring_size = ring_size;
2451 ring->align_mask = 16 - 1;
45df6803 2452
89d35807
AD		 2453 if (radeon_ring_supports_scratch_reg(rdev, ring)) {
2454 r = radeon_scratch_get(rdev, &ring->rptr_save_reg);
2455 if (r) {
2456 DRM_ERROR("failed to get scratch reg for rptr save (%d).\n", r);
2457 ring->rptr_save_reg = 0;
2458 }
45df6803 2459 }
3ce0a23d
JG		 2460}
2461
655efd3d
JG		 2462void r600_cp_fini(struct radeon_device *rdev)
2463{
45df6803 2464 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
655efd3d 2465 r600_cp_stop(rdev);
45df6803
CK		 2466 radeon_ring_fini(rdev, ring);
2467 radeon_scratch_free(rdev, ring->rptr_save_reg);
655efd3d
JG		 2468}
2469
4d75658b
AD		 2470/*
2471 * DMA
2472 * Starting with R600, the GPU has an asynchronous
2473 * DMA engine. The programming model is very similar
2474 * to the 3D engine (ring buffer, IBs, etc.), but the
2475 * DMA controller has it's own packet format that is
2476 * different form the PM4 format used by the 3D engine.
2477 * It supports copying data, writing embedded data,
2478 * solid fills, and a number of other things. It also
2479 * has support for tiling/detiling of buffers.
2480 */
2481/**
2482 * r600_dma_stop - stop the async dma engine
2483 *
2484 * @rdev: radeon_device pointer
2485 *
2486 * Stop the async dma engine (r6xx-evergreen).
2487 */
2488void r600_dma_stop(struct radeon_device *rdev)
2489{
2490 u32 rb_cntl = RREG32(DMA_RB_CNTL);
2491
2492 radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
2493
2494 rb_cntl &= ~DMA_RB_ENABLE;
2495 WREG32(DMA_RB_CNTL, rb_cntl);
2496
2497 rdev->ring[R600_RING_TYPE_DMA_INDEX].ready = false;
2498}
2499
2500/**
2501 * r600_dma_resume - setup and start the async dma engine
2502 *
2503 * @rdev: radeon_device pointer
2504 *
2505 * Set up the DMA ring buffer and enable it. (r6xx-evergreen).
2506 * Returns 0 for success, error for failure.
2507 */
2508int r600_dma_resume(struct radeon_device *rdev)
2509{
2510 struct radeon_ring *ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
b3dfcb20 2511 u32 rb_cntl, dma_cntl, ib_cntl;
4d75658b
AD		 2512 u32 rb_bufsz;
2513 int r;
2514
2515 /* Reset dma */
2516 if (rdev->family >= CHIP_RV770)
2517 WREG32(SRBM_SOFT_RESET, RV770_SOFT_RESET_DMA);
2518 else
2519 WREG32(SRBM_SOFT_RESET, SOFT_RESET_DMA);
2520 RREG32(SRBM_SOFT_RESET);
2521 udelay(50);
2522 WREG32(SRBM_SOFT_RESET, 0);
2523
2524 WREG32(DMA_SEM_INCOMPLETE_TIMER_CNTL, 0);
2525 WREG32(DMA_SEM_WAIT_FAIL_TIMER_CNTL, 0);
2526
2527 /* Set ring buffer size in dwords */
2528 rb_bufsz = drm_order(ring->ring_size / 4);
2529 rb_cntl = rb_bufsz << 1;
2530#ifdef __BIG_ENDIAN
2531 rb_cntl |= DMA_RB_SWAP_ENABLE | DMA_RPTR_WRITEBACK_SWAP_ENABLE;
2532#endif
2533 WREG32(DMA_RB_CNTL, rb_cntl);
2534
2535 /* Initialize the ring buffer's read and write pointers */
2536 WREG32(DMA_RB_RPTR, 0);
2537 WREG32(DMA_RB_WPTR, 0);
2538
2539 /* set the wb address whether it's enabled or not */
2540 WREG32(DMA_RB_RPTR_ADDR_HI,
2541 upper_32_bits(rdev->wb.gpu_addr + R600_WB_DMA_RPTR_OFFSET) & 0xFF);
2542 WREG32(DMA_RB_RPTR_ADDR_LO,
2543 ((rdev->wb.gpu_addr + R600_WB_DMA_RPTR_OFFSET) & 0xFFFFFFFC));
2544
2545 if (rdev->wb.enabled)
2546 rb_cntl |= DMA_RPTR_WRITEBACK_ENABLE;
2547
2548 WREG32(DMA_RB_BASE, ring->gpu_addr >> 8);
2549
2550 /* enable DMA IBs */
b3dfcb20
MD		 2551 ib_cntl = DMA_IB_ENABLE;
2552#ifdef __BIG_ENDIAN
2553 ib_cntl |= DMA_IB_SWAP_ENABLE;
2554#endif
2555 WREG32(DMA_IB_CNTL, ib_cntl);
4d75658b
AD		 2556
2557 dma_cntl = RREG32(DMA_CNTL);
2558 dma_cntl &= ~CTXEMPTY_INT_ENABLE;
2559 WREG32(DMA_CNTL, dma_cntl);
2560
2561 if (rdev->family >= CHIP_RV770)
2562 WREG32(DMA_MODE, 1);
2563
2564 ring->wptr = 0;
2565 WREG32(DMA_RB_WPTR, ring->wptr << 2);
2566
2567 ring->rptr = RREG32(DMA_RB_RPTR) >> 2;
2568
2569 WREG32(DMA_RB_CNTL, rb_cntl | DMA_RB_ENABLE);
2570
2571 ring->ready = true;
2572
2573 r = radeon_ring_test(rdev, R600_RING_TYPE_DMA_INDEX, ring);
2574 if (r) {
2575 ring->ready = false;
2576 return r;
2577 }
2578
2579 radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);
2580
2581 return 0;
2582}
2583
2584/**
2585 * r600_dma_fini - tear down the async dma engine
2586 *
2587 * @rdev: radeon_device pointer
2588 *
2589 * Stop the async dma engine and free the ring (r6xx-evergreen).
2590 */
2591void r600_dma_fini(struct radeon_device *rdev)
2592{
2593 r600_dma_stop(rdev);
2594 radeon_ring_fini(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX]);
2595}
3ce0a23d 2596
f2ba57b5
CK		 2597/*
2598 * UVD
2599 */
2600int r600_uvd_rbc_start(struct radeon_device *rdev)
2601{
2602 struct radeon_ring *ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
2603 uint64_t rptr_addr;
2604 uint32_t rb_bufsz, tmp;
2605 int r;
2606
2607 rptr_addr = rdev->wb.gpu_addr + R600_WB_UVD_RPTR_OFFSET;
2608
2609 if (upper_32_bits(rptr_addr) != upper_32_bits(ring->gpu_addr)) {
2610 DRM_ERROR("UVD ring and rptr not in the same 4GB segment!\n");
2611 return -EINVAL;
2612 }
2613
2614 /* force RBC into idle state */
2615 WREG32(UVD_RBC_RB_CNTL, 0x11010101);
2616
2617 /* Set the write pointer delay */
2618 WREG32(UVD_RBC_RB_WPTR_CNTL, 0);
2619
2620 /* set the wb address */
2621 WREG32(UVD_RBC_RB_RPTR_ADDR, rptr_addr >> 2);
2622
2623 /* programm the 4GB memory segment for rptr and ring buffer */
2624 WREG32(UVD_LMI_EXT40_ADDR, upper_32_bits(rptr_addr) |
2625 (0x7 << 16) | (0x1 << 31));
2626
2627 /* Initialize the ring buffer's read and write pointers */
2628 WREG32(UVD_RBC_RB_RPTR, 0x0);
2629
2630 ring->wptr = ring->rptr = RREG32(UVD_RBC_RB_RPTR);
2631 WREG32(UVD_RBC_RB_WPTR, ring->wptr);
2632
2633 /* set the ring address */
2634 WREG32(UVD_RBC_RB_BASE, ring->gpu_addr);
2635
2636 /* Set ring buffer size */
2637 rb_bufsz = drm_order(ring->ring_size);
2638 rb_bufsz = (0x1 << 8) | rb_bufsz;
2639 WREG32(UVD_RBC_RB_CNTL, rb_bufsz);
2640
2641 ring->ready = true;
2642 r = radeon_ring_test(rdev, R600_RING_TYPE_UVD_INDEX, ring);
2643 if (r) {
2644 ring->ready = false;
2645 return r;
2646 }
2647
2648 r = radeon_ring_lock(rdev, ring, 10);
2649 if (r) {
2650 DRM_ERROR("radeon: ring failed to lock UVD ring (%d).\n", r);
2651 return r;
2652 }
2653
2654 tmp = PACKET0(UVD_SEMA_WAIT_FAULT_TIMEOUT_CNTL, 0);
2655 radeon_ring_write(ring, tmp);
2656 radeon_ring_write(ring, 0xFFFFF);
2657
2658 tmp = PACKET0(UVD_SEMA_WAIT_INCOMPLETE_TIMEOUT_CNTL, 0);
2659 radeon_ring_write(ring, tmp);
2660 radeon_ring_write(ring, 0xFFFFF);
2661
2662 tmp = PACKET0(UVD_SEMA_SIGNAL_INCOMPLETE_TIMEOUT_CNTL, 0);
2663 radeon_ring_write(ring, tmp);
2664 radeon_ring_write(ring, 0xFFFFF);
2665
2666 /* Clear timeout status bits */
2667 radeon_ring_write(ring, PACKET0(UVD_SEMA_TIMEOUT_STATUS, 0));
2668 radeon_ring_write(ring, 0x8);
2669
2670 radeon_ring_write(ring, PACKET0(UVD_SEMA_CNTL, 0));
03708b05 2671 radeon_ring_write(ring, 3);
f2ba57b5
CK		 2672
2673 radeon_ring_unlock_commit(rdev, ring);
2674
2675 return 0;
2676}
2677
2678void r600_uvd_rbc_stop(struct radeon_device *rdev)
2679{
2680 struct radeon_ring *ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
2681
2682 /* force RBC into idle state */
2683 WREG32(UVD_RBC_RB_CNTL, 0x11010101);
2684 ring->ready = false;
2685}
2686
2687int r600_uvd_init(struct radeon_device *rdev)
2688{
2689 int i, j, r;
c139b1ee
AD		 2690 /* disable byte swapping */
2691 u32 lmi_swap_cntl = 0;
2692 u32 mp_swap_cntl = 0;
f2ba57b5 2693
b05e9e4c
CK		 2694 /* raise clocks while booting up the VCPU */
2695 radeon_set_uvd_clocks(rdev, 53300, 40000);
2696
f2ba57b5
CK		 2697 /* disable clock gating */
2698 WREG32(UVD_CGC_GATE, 0);
2699
2700 /* disable interupt */
2701 WREG32_P(UVD_MASTINT_EN, 0, ~(1 << 1));
2702
2703 /* put LMI, VCPU, RBC etc... into reset */
2704 WREG32(UVD_SOFT_RESET, LMI_SOFT_RESET | VCPU_SOFT_RESET |
2705 LBSI_SOFT_RESET | RBC_SOFT_RESET | CSM_SOFT_RESET |
2706 CXW_SOFT_RESET | TAP_SOFT_RESET | LMI_UMC_SOFT_RESET);
2707 mdelay(5);
2708
2709 /* take UVD block out of reset */
2710 WREG32_P(SRBM_SOFT_RESET, 0, ~SOFT_RESET_UVD);
2711 mdelay(5);
2712
2713 /* initialize UVD memory controller */
2714 WREG32(UVD_LMI_CTRL, 0x40 | (1 << 8) | (1 << 13) |
2715 (1 << 21) | (1 << 9) | (1 << 20));
2716
c139b1ee
AD		 2717#ifdef __BIG_ENDIAN
2718 /* swap (8 in 32) RB and IB */
2719 lmi_swap_cntl = 0xa;
2720 mp_swap_cntl = 0;
2721#endif
2722 WREG32(UVD_LMI_SWAP_CNTL, lmi_swap_cntl);
2723 WREG32(UVD_MP_SWAP_CNTL, mp_swap_cntl);
f2ba57b5
CK		 2724
2725 WREG32(UVD_MPC_SET_MUXA0, 0x40c2040);
2726 WREG32(UVD_MPC_SET_MUXA1, 0x0);
2727 WREG32(UVD_MPC_SET_MUXB0, 0x40c2040);
2728 WREG32(UVD_MPC_SET_MUXB1, 0x0);
2729 WREG32(UVD_MPC_SET_ALU, 0);
2730 WREG32(UVD_MPC_SET_MUX, 0x88);
2731
2732 /* Stall UMC */
2733 WREG32_P(UVD_LMI_CTRL2, 1 << 8, ~(1 << 8));
2734 WREG32_P(UVD_RB_ARB_CTRL, 1 << 3, ~(1 << 3));
2735
2736 /* take all subblocks out of reset, except VCPU */
2737 WREG32(UVD_SOFT_RESET, VCPU_SOFT_RESET);
2738 mdelay(5);
2739
2740 /* enable VCPU clock */
2741 WREG32(UVD_VCPU_CNTL, 1 << 9);
2742
2743 /* enable UMC */
2744 WREG32_P(UVD_LMI_CTRL2, 0, ~(1 << 8));
2745
2746 /* boot up the VCPU */
2747 WREG32(UVD_SOFT_RESET, 0);
2748 mdelay(10);
2749
2750 WREG32_P(UVD_RB_ARB_CTRL, 0, ~(1 << 3));
2751
2752 for (i = 0; i < 10; ++i) {
2753 uint32_t status;
2754 for (j = 0; j < 100; ++j) {
2755 status = RREG32(UVD_STATUS);
2756 if (status & 2)
2757 break;
2758 mdelay(10);
2759 }
2760 r = 0;
2761 if (status & 2)
2762 break;
2763
2764 DRM_ERROR("UVD not responding, trying to reset the VCPU!!!\n");
2765 WREG32_P(UVD_SOFT_RESET, VCPU_SOFT_RESET, ~VCPU_SOFT_RESET);
2766 mdelay(10);
2767 WREG32_P(UVD_SOFT_RESET, 0, ~VCPU_SOFT_RESET);
2768 mdelay(10);
2769 r = -1;
2770 }
b05e9e4c 2771
f2ba57b5
CK		 2772 if (r) {
2773 DRM_ERROR("UVD not responding, giving up!!!\n");
b05e9e4c 2774 radeon_set_uvd_clocks(rdev, 0, 0);
f2ba57b5
CK		 2775 return r;
2776 }
b05e9e4c 2777
f2ba57b5
CK		 2778 /* enable interupt */
2779 WREG32_P(UVD_MASTINT_EN, 3<<1, ~(3 << 1));
2780
2781 r = r600_uvd_rbc_start(rdev);
b05e9e4c
CK		 2782 if (!r)
2783 DRM_INFO("UVD initialized successfully.\n");
f2ba57b5 2784
b05e9e4c
CK		 2785 /* lower clocks again */
2786 radeon_set_uvd_clocks(rdev, 0, 0);
2787
2788 return r;
f2ba57b5
CK		 2789}
2790
3ce0a23d
JG		 2791/*
2792 * GPU scratch registers helpers function.
2793 */
2794void r600_scratch_init(struct radeon_device *rdev)
2795{
2796 int i;
2797
2798 rdev->scratch.num_reg = 7;
724c80e1 2799 rdev->scratch.reg_base = SCRATCH_REG0;
3ce0a23d
JG		 2800 for (i = 0; i < rdev->scratch.num_reg; i++) {
2801 rdev->scratch.free[i] = true;
724c80e1 2802 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
3ce0a23d
JG		 2803 }
2804}
2805
e32eb50d 2806int r600_ring_test(struct radeon_device *rdev, struct radeon_ring *ring)
3ce0a23d
JG		 2807{
2808 uint32_t scratch;
2809 uint32_t tmp = 0;
8b25ed34 2810 unsigned i;
3ce0a23d
JG		 2811 int r;
2812
2813 r = radeon_scratch_get(rdev, &scratch);
2814 if (r) {
2815 DRM_ERROR("radeon: cp failed to get scratch reg (%d).\n", r);
2816 return r;
2817 }
2818 WREG32(scratch, 0xCAFEDEAD);
e32eb50d 2819 r = radeon_ring_lock(rdev, ring, 3);
3ce0a23d 2820 if (r) {
8b25ed34 2821 DRM_ERROR("radeon: cp failed to lock ring %d (%d).\n", ring->idx, r);
3ce0a23d
JG		 2822 radeon_scratch_free(rdev, scratch);
2823 return r;
2824 }
e32eb50d
CK		 2825 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
2826 radeon_ring_write(ring, ((scratch - PACKET3_SET_CONFIG_REG_OFFSET) >> 2));
2827 radeon_ring_write(ring, 0xDEADBEEF);
2828 radeon_ring_unlock_commit(rdev, ring);
3ce0a23d
JG		 2829 for (i = 0; i < rdev->usec_timeout; i++) {
2830 tmp = RREG32(scratch);
2831 if (tmp == 0xDEADBEEF)
2832 break;
2833 DRM_UDELAY(1);
2834 }
2835 if (i < rdev->usec_timeout) {
8b25ed34 2836 DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i);
3ce0a23d 2837 } else {
bf852799 2838 DRM_ERROR("radeon: ring %d test failed (scratch(0x%04X)=0x%08X)\n",
8b25ed34 2839 ring->idx, scratch, tmp);
3ce0a23d
JG		 2840 r = -EINVAL;
2841 }
2842 radeon_scratch_free(rdev, scratch);
2843 return r;
2844}
2845
4d75658b
AD		 2846/**
2847 * r600_dma_ring_test - simple async dma engine test
2848 *
2849 * @rdev: radeon_device pointer
2850 * @ring: radeon_ring structure holding ring information
2851 *
2852 * Test the DMA engine by writing using it to write an
2853 * value to memory. (r6xx-SI).
2854 * Returns 0 for success, error for failure.
2855 */
2856int r600_dma_ring_test(struct radeon_device *rdev,
2857 struct radeon_ring *ring)
2858{
2859 unsigned i;
2860 int r;
2861 void __iomem *ptr = (void *)rdev->vram_scratch.ptr;
2862 u32 tmp;
2863
2864 if (!ptr) {
2865 DRM_ERROR("invalid vram scratch pointer\n");
2866 return -EINVAL;
2867 }
2868
2869 tmp = 0xCAFEDEAD;
2870 writel(tmp, ptr);
2871
2872 r = radeon_ring_lock(rdev, ring, 4);
2873 if (r) {
2874 DRM_ERROR("radeon: dma failed to lock ring %d (%d).\n", ring->idx, r);
2875 return r;
2876 }
2877 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_WRITE, 0, 0, 1));
2878 radeon_ring_write(ring, rdev->vram_scratch.gpu_addr & 0xfffffffc);
2879 radeon_ring_write(ring, upper_32_bits(rdev->vram_scratch.gpu_addr) & 0xff);
2880 radeon_ring_write(ring, 0xDEADBEEF);
2881 radeon_ring_unlock_commit(rdev, ring);
2882
2883 for (i = 0; i < rdev->usec_timeout; i++) {
2884 tmp = readl(ptr);
2885 if (tmp == 0xDEADBEEF)
2886 break;
2887 DRM_UDELAY(1);
2888 }
2889
2890 if (i < rdev->usec_timeout) {
2891 DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i);
2892 } else {
2893 DRM_ERROR("radeon: ring %d test failed (0x%08X)\n",
2894 ring->idx, tmp);
2895 r = -EINVAL;
2896 }
2897 return r;
2898}
2899
f2ba57b5
CK		 2900int r600_uvd_ring_test(struct radeon_device *rdev, struct radeon_ring *ring)
2901{
2902 uint32_t tmp = 0;
2903 unsigned i;
2904 int r;
2905
2906 WREG32(UVD_CONTEXT_ID, 0xCAFEDEAD);
2907 r = radeon_ring_lock(rdev, ring, 3);
2908 if (r) {
2909 DRM_ERROR("radeon: cp failed to lock ring %d (%d).\n",
2910 ring->idx, r);
2911 return r;
2912 }
2913 radeon_ring_write(ring, PACKET0(UVD_CONTEXT_ID, 0));
2914 radeon_ring_write(ring, 0xDEADBEEF);
2915 radeon_ring_unlock_commit(rdev, ring);
2916 for (i = 0; i < rdev->usec_timeout; i++) {
2917 tmp = RREG32(UVD_CONTEXT_ID);
2918 if (tmp == 0xDEADBEEF)
2919 break;
2920 DRM_UDELAY(1);
2921 }
2922
2923 if (i < rdev->usec_timeout) {
2924 DRM_INFO("ring test on %d succeeded in %d usecs\n",
2925 ring->idx, i);
2926 } else {
2927 DRM_ERROR("radeon: ring %d test failed (0x%08X)\n",
2928 ring->idx, tmp);
2929 r = -EINVAL;
2930 }
2931 return r;
2932}
2933
4d75658b
AD		 2934/*
2935 * CP fences/semaphores
2936 */
2937
3ce0a23d
JG		 2938void r600_fence_ring_emit(struct radeon_device *rdev,
2939 struct radeon_fence *fence)
2940{
e32eb50d 2941 struct radeon_ring *ring = &rdev->ring[fence->ring];
7b1f2485 2942
d0f8a854 2943 if (rdev->wb.use_event) {
30eb77f4 2944 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
77b1bad4 2945 /* flush read cache over gart */
e32eb50d
CK		 2946 radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
2947 radeon_ring_write(ring, PACKET3_TC_ACTION_ENA |
2948 PACKET3_VC_ACTION_ENA |
2949 PACKET3_SH_ACTION_ENA);
2950 radeon_ring_write(ring, 0xFFFFFFFF);
2951 radeon_ring_write(ring, 0);
2952 radeon_ring_write(ring, 10); /* poll interval */
d0f8a854 2953 /* EVENT_WRITE_EOP - flush caches, send int */
e32eb50d
CK		 2954 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
2955 radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_EVENT_TS) | EVENT_INDEX(5));
2956 radeon_ring_write(ring, addr & 0xffffffff);
2957 radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) | DATA_SEL(1) | INT_SEL(2));
2958 radeon_ring_write(ring, fence->seq);
2959 radeon_ring_write(ring, 0);
d0f8a854 2960 } else {
77b1bad4 2961 /* flush read cache over gart */
e32eb50d
CK		 2962 radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
2963 radeon_ring_write(ring, PACKET3_TC_ACTION_ENA |
2964 PACKET3_VC_ACTION_ENA |
2965 PACKET3_SH_ACTION_ENA);
2966 radeon_ring_write(ring, 0xFFFFFFFF);
2967 radeon_ring_write(ring, 0);
2968 radeon_ring_write(ring, 10); /* poll interval */
2969 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE, 0));
2970 radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_EVENT) | EVENT_INDEX(0));
d0f8a854 2971 /* wait for 3D idle clean */
e32eb50d
CK		 2972 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
2973 radeon_ring_write(ring, (WAIT_UNTIL - PACKET3_SET_CONFIG_REG_OFFSET) >> 2);
2974 radeon_ring_write(ring, WAIT_3D_IDLE_bit | WAIT_3D_IDLECLEAN_bit);
d0f8a854 2975 /* Emit fence sequence & fire IRQ */
e32eb50d
CK		 2976 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
2977 radeon_ring_write(ring, ((rdev->fence_drv[fence->ring].scratch_reg - PACKET3_SET_CONFIG_REG_OFFSET) >> 2));
2978 radeon_ring_write(ring, fence->seq);
d0f8a854 2979 /* CP_INTERRUPT packet 3 no longer exists, use packet 0 */
e32eb50d
CK		 2980 radeon_ring_write(ring, PACKET0(CP_INT_STATUS, 0));
2981 radeon_ring_write(ring, RB_INT_STAT);
d0f8a854 2982 }
3ce0a23d
JG		 2983}
2984
f2ba57b5
CK		 2985void r600_uvd_fence_emit(struct radeon_device *rdev,
2986 struct radeon_fence *fence)
2987{
2988 struct radeon_ring *ring = &rdev->ring[fence->ring];
2989 uint32_t addr = rdev->fence_drv[fence->ring].gpu_addr;
2990
2991 radeon_ring_write(ring, PACKET0(UVD_CONTEXT_ID, 0));
2992 radeon_ring_write(ring, fence->seq);
2993 radeon_ring_write(ring, PACKET0(UVD_GPCOM_VCPU_DATA0, 0));
2994 radeon_ring_write(ring, addr & 0xffffffff);
2995 radeon_ring_write(ring, PACKET0(UVD_GPCOM_VCPU_DATA1, 0));
2996 radeon_ring_write(ring, upper_32_bits(addr) & 0xff);
2997 radeon_ring_write(ring, PACKET0(UVD_GPCOM_VCPU_CMD, 0));
2998 radeon_ring_write(ring, 0);
2999
3000 radeon_ring_write(ring, PACKET0(UVD_GPCOM_VCPU_DATA0, 0));
3001 radeon_ring_write(ring, 0);
3002 radeon_ring_write(ring, PACKET0(UVD_GPCOM_VCPU_DATA1, 0));
3003 radeon_ring_write(ring, 0);
3004 radeon_ring_write(ring, PACKET0(UVD_GPCOM_VCPU_CMD, 0));
3005 radeon_ring_write(ring, 2);
3006 return;
3007}
3008
15d3332f 3009void r600_semaphore_ring_emit(struct radeon_device *rdev,
e32eb50d 3010 struct radeon_ring *ring,
15d3332f 3011 struct radeon_semaphore *semaphore,
7b1f2485 3012 bool emit_wait)
15d3332f
CK		 3013{
3014 uint64_t addr = semaphore->gpu_addr;
3015 unsigned sel = emit_wait ? PACKET3_SEM_SEL_WAIT : PACKET3_SEM_SEL_SIGNAL;
3016
0be70439
CK		 3017 if (rdev->family < CHIP_CAYMAN)
3018 sel |= PACKET3_SEM_WAIT_ON_SIGNAL;
3019
e32eb50d
CK		 3020 radeon_ring_write(ring, PACKET3(PACKET3_MEM_SEMAPHORE, 1));
3021 radeon_ring_write(ring, addr & 0xffffffff);
3022 radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) | sel);
15d3332f
CK		 3023}
3024
4d75658b
AD		 3025/*
3026 * DMA fences/semaphores
3027 */
3028
3029/**
3030 * r600_dma_fence_ring_emit - emit a fence on the DMA ring
3031 *
3032 * @rdev: radeon_device pointer
3033 * @fence: radeon fence object
3034 *
3035 * Add a DMA fence packet to the ring to write
3036 * the fence seq number and DMA trap packet to generate
3037 * an interrupt if needed (r6xx-r7xx).
3038 */
3039void r600_dma_fence_ring_emit(struct radeon_device *rdev,
3040 struct radeon_fence *fence)
3041{
3042 struct radeon_ring *ring = &rdev->ring[fence->ring];
3043 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
86a1881d 3044
4d75658b
AD		 3045 /* write the fence */
3046 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_FENCE, 0, 0, 0));
3047 radeon_ring_write(ring, addr & 0xfffffffc);
3048 radeon_ring_write(ring, (upper_32_bits(addr) & 0xff));
86a1881d 3049 radeon_ring_write(ring, lower_32_bits(fence->seq));
4d75658b
AD		 3050 /* generate an interrupt */
3051 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_TRAP, 0, 0, 0));
3052}
3053
3054/**
3055 * r600_dma_semaphore_ring_emit - emit a semaphore on the dma ring
3056 *
3057 * @rdev: radeon_device pointer
3058 * @ring: radeon_ring structure holding ring information
3059 * @semaphore: radeon semaphore object
3060 * @emit_wait: wait or signal semaphore
3061 *
3062 * Add a DMA semaphore packet to the ring wait on or signal
3063 * other rings (r6xx-SI).
3064 */
3065void r600_dma_semaphore_ring_emit(struct radeon_device *rdev,
3066 struct radeon_ring *ring,
3067 struct radeon_semaphore *semaphore,
3068 bool emit_wait)
3069{
3070 u64 addr = semaphore->gpu_addr;
3071 u32 s = emit_wait ? 0 : 1;
3072
3073 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SEMAPHORE, 0, s, 0));
3074 radeon_ring_write(ring, addr & 0xfffffffc);
3075 radeon_ring_write(ring, upper_32_bits(addr) & 0xff);
3076}
3077
f2ba57b5
CK		 3078void r600_uvd_semaphore_emit(struct radeon_device *rdev,
3079 struct radeon_ring *ring,
3080 struct radeon_semaphore *semaphore,
3081 bool emit_wait)
3082{
3083 uint64_t addr = semaphore->gpu_addr;
3084
3085 radeon_ring_write(ring, PACKET0(UVD_SEMA_ADDR_LOW, 0));
3086 radeon_ring_write(ring, (addr >> 3) & 0x000FFFFF);
3087
3088 radeon_ring_write(ring, PACKET0(UVD_SEMA_ADDR_HIGH, 0));
3089 radeon_ring_write(ring, (addr >> 23) & 0x000FFFFF);
3090
3091 radeon_ring_write(ring, PACKET0(UVD_SEMA_CMD, 0));
3092 radeon_ring_write(ring, emit_wait ? 1 : 0);
3093}
3094
3ce0a23d 3095int r600_copy_blit(struct radeon_device *rdev,
003cefe0
AD		 3096 uint64_t src_offset,
3097 uint64_t dst_offset,
3098 unsigned num_gpu_pages,
876dc9f3 3099 struct radeon_fence **fence)
3ce0a23d 3100{
220907d9 3101 struct radeon_semaphore *sem = NULL;
f237750f 3102 struct radeon_sa_bo *vb = NULL;
ff82f052
JG		 3103 int r;
3104
220907d9 3105 r = r600_blit_prepare_copy(rdev, num_gpu_pages, fence, &vb, &sem);
ff82f052 3106 if (r) {
ff82f052
JG		 3107 return r;
3108 }
f237750f 3109 r600_kms_blit_copy(rdev, src_offset, dst_offset, num_gpu_pages, vb);
220907d9 3110 r600_blit_done_copy(rdev, fence, vb, sem);
3ce0a23d
JG		 3111 return 0;
3112}
3113
4d75658b
AD		 3114/**
3115 * r600_copy_dma - copy pages using the DMA engine
3116 *
3117 * @rdev: radeon_device pointer
3118 * @src_offset: src GPU address
3119 * @dst_offset: dst GPU address
3120 * @num_gpu_pages: number of GPU pages to xfer
3121 * @fence: radeon fence object
3122 *
43fb7787 3123 * Copy GPU paging using the DMA engine (r6xx).
4d75658b
AD		 3124 * Used by the radeon ttm implementation to move pages if
3125 * registered as the asic copy callback.
3126 */
3127int r600_copy_dma(struct radeon_device *rdev,
3128 uint64_t src_offset, uint64_t dst_offset,
3129 unsigned num_gpu_pages,
3130 struct radeon_fence **fence)
3131{
3132 struct radeon_semaphore *sem = NULL;
3133 int ring_index = rdev->asic->copy.dma_ring_index;
3134 struct radeon_ring *ring = &rdev->ring[ring_index];
3135 u32 size_in_dw, cur_size_in_dw;
3136 int i, num_loops;
3137 int r = 0;
3138
3139 r = radeon_semaphore_create(rdev, &sem);
3140 if (r) {
3141 DRM_ERROR("radeon: moving bo (%d).\n", r);
3142 return r;
3143 }
3144
3145 size_in_dw = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT) / 4;
43fb7787
AD		 3146 num_loops = DIV_ROUND_UP(size_in_dw, 0xFFFE);
3147 r = radeon_ring_lock(rdev, ring, num_loops * 4 + 8);
4d75658b
AD		 3148 if (r) {
3149 DRM_ERROR("radeon: moving bo (%d).\n", r);
3150 radeon_semaphore_free(rdev, &sem, NULL);
3151 return r;
3152 }
3153
3154 if (radeon_fence_need_sync(*fence, ring->idx)) {
3155 radeon_semaphore_sync_rings(rdev, sem, (*fence)->ring,
3156 ring->idx);
3157 radeon_fence_note_sync(*fence, ring->idx);
3158 } else {
3159 radeon_semaphore_free(rdev, &sem, NULL);
3160 }
3161
3162 for (i = 0; i < num_loops; i++) {
3163 cur_size_in_dw = size_in_dw;
909d9eb6
AD		 3164 if (cur_size_in_dw > 0xFFFE)
3165 cur_size_in_dw = 0xFFFE;
4d75658b
AD		 3166 size_in_dw -= cur_size_in_dw;
3167 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_COPY, 0, 0, cur_size_in_dw));
3168 radeon_ring_write(ring, dst_offset & 0xfffffffc);
3169 radeon_ring_write(ring, src_offset & 0xfffffffc);
43fb7787
AD		 3170 radeon_ring_write(ring, (((upper_32_bits(dst_offset) & 0xff) << 16) |
3171 (upper_32_bits(src_offset) & 0xff)));
4d75658b
AD		 3172 src_offset += cur_size_in_dw * 4;
3173 dst_offset += cur_size_in_dw * 4;
3174 }
3175
3176 r = radeon_fence_emit(rdev, fence, ring->idx);
3177 if (r) {
3178 radeon_ring_unlock_undo(rdev, ring);
3179 return r;
3180 }
3181
3182 radeon_ring_unlock_commit(rdev, ring);
3183 radeon_semaphore_free(rdev, &sem, *fence);
3184
3185 return r;
3186}
3187
3ce0a23d
JG		 3188int r600_set_surface_reg(struct radeon_device *rdev, int reg,
3189 uint32_t tiling_flags, uint32_t pitch,
3190 uint32_t offset, uint32_t obj_size)
3191{
3192 /* FIXME: implement */
3193 return 0;
3194}
3195
3196void r600_clear_surface_reg(struct radeon_device *rdev, int reg)
3197{
3198 /* FIXME: implement */
3199}
3200
1109ca09 3201static int r600_startup(struct radeon_device *rdev)
3ce0a23d 3202{
4d75658b 3203 struct radeon_ring *ring;
3ce0a23d
JG		 3204 int r;
3205
9e46a48d
AD		 3206 /* enable pcie gen2 link */
3207 r600_pcie_gen2_enable(rdev);
3208
779720a3
AD		 3209 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
3210 r = r600_init_microcode(rdev);
3211 if (r) {
3212 DRM_ERROR("Failed to load firmware!\n");
3213 return r;
3214 }
3215 }
3216
16cdf04d
AD		 3217 r = r600_vram_scratch_init(rdev);
3218 if (r)
3219 return r;
3220
a3c1945a 3221 r600_mc_program(rdev);
1a029b76
JG		 3222 if (rdev->flags & RADEON_IS_AGP) {
3223 r600_agp_enable(rdev);
3224 } else {
3225 r = r600_pcie_gart_enable(rdev);
3226 if (r)
3227 return r;
3228 }
3ce0a23d 3229 r600_gpu_init(rdev);
c38c7b64
JG		 3230 r = r600_blit_init(rdev);
3231 if (r) {
3232 r600_blit_fini(rdev);
27cd7769 3233 rdev->asic->copy.copy = NULL;
c38c7b64
JG		 3234 dev_warn(rdev->dev, "failed blitter (%d) falling back to memcpy\n", r);
3235 }
b70d6bb3 3236
724c80e1
AD		 3237 /* allocate wb buffer */
3238 r = radeon_wb_init(rdev);
3239 if (r)
3240 return r;
3241
30eb77f4
JG		 3242 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
3243 if (r) {
3244 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
3245 return r;
3246 }
3247
4d75658b
AD		 3248 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
3249 if (r) {
3250 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
3251 return r;
3252 }
3253
d8f60cfc 3254 /* Enable IRQ */
e49f3959
AH		 3255 if (!rdev->irq.installed) {
3256 r = radeon_irq_kms_init(rdev);
3257 if (r)
3258 return r;
3259 }
3260
d8f60cfc
AD		 3261 r = r600_irq_init(rdev);
3262 if (r) {
3263 DRM_ERROR("radeon: IH init failed (%d).\n", r);
3264 radeon_irq_kms_fini(rdev);
3265 return r;
3266 }
3267 r600_irq_set(rdev);
3268
4d75658b 3269 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
e32eb50d 3270 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
78c5560a
AD		 3271 R600_CP_RB_RPTR, R600_CP_RB_WPTR,
3272 0, 0xfffff, RADEON_CP_PACKET2);
4d75658b
AD		 3273 if (r)
3274 return r;
5596a9db 3275
4d75658b
AD		 3276 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
3277 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
3278 DMA_RB_RPTR, DMA_RB_WPTR,
3279 2, 0x3fffc, DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0));
3ce0a23d
JG		 3280 if (r)
3281 return r;
4d75658b 3282
3ce0a23d
JG		 3283 r = r600_cp_load_microcode(rdev);
3284 if (r)
3285 return r;
3286 r = r600_cp_resume(rdev);
3287 if (r)
3288 return r;
724c80e1 3289
4d75658b
AD		 3290 r = r600_dma_resume(rdev);
3291 if (r)
3292 return r;
3293
2898c348
CK		 3294 r = radeon_ib_pool_init(rdev);
3295 if (r) {
3296 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
b15ba512 3297 return r;
2898c348 3298 }
b15ba512 3299
d4e30ef0
AD		 3300 r = r600_audio_init(rdev);
3301 if (r) {
3302 DRM_ERROR("radeon: audio init failed\n");
3303 return r;
3304 }
3305
3ce0a23d
JG		 3306 return 0;
3307}
3308
28d52043
DA		 3309void r600_vga_set_state(struct radeon_device *rdev, bool state)
3310{
3311 uint32_t temp;
3312
3313 temp = RREG32(CONFIG_CNTL);
3314 if (state == false) {
3315 temp &= ~(1<<0);
3316 temp |= (1<<1);
3317 } else {
3318 temp &= ~(1<<1);
3319 }
3320 WREG32(CONFIG_CNTL, temp);
3321}
3322
fc30b8ef
DA		 3323int r600_resume(struct radeon_device *rdev)
3324{
3325 int r;
3326
1a029b76
JG		 3327 /* Do not reset GPU before posting, on r600 hw unlike on r500 hw,
3328 * posting will perform necessary task to bring back GPU into good
3329 * shape.
3330 */
fc30b8ef 3331 /* post card */
e7d40b9a 3332 atom_asic_init(rdev->mode_info.atom_context);
fc30b8ef 3333
b15ba512 3334 rdev->accel_working = true;
fc30b8ef
DA		 3335 r = r600_startup(rdev);
3336 if (r) {
3337 DRM_ERROR("r600 startup failed on resume\n");
6b7746e8 3338 rdev->accel_working = false;
fc30b8ef
DA		 3339 return r;
3340 }
3341
fc30b8ef
DA		 3342 return r;
3343}
3344
3ce0a23d
JG		 3345int r600_suspend(struct radeon_device *rdev)
3346{
38fd2c6f 3347 r600_audio_fini(rdev);
3ce0a23d 3348 r600_cp_stop(rdev);
4d75658b 3349 r600_dma_stop(rdev);
0c45249f 3350 r600_irq_suspend(rdev);
724c80e1 3351 radeon_wb_disable(rdev);
4aac0473 3352 r600_pcie_gart_disable(rdev);
6ddddfe7 3353
3ce0a23d
JG		 3354 return 0;
3355}
3356
3357/* Plan is to move initialization in that function and use
3358 * helper function so that radeon_device_init pretty much
3359 * do nothing more than calling asic specific function. This
3360 * should also allow to remove a bunch of callback function
3361 * like vram_info.
3362 */
3363int r600_init(struct radeon_device *rdev)
771fe6b9 3364{
3ce0a23d 3365 int r;
771fe6b9 3366
3ce0a23d
JG		 3367 if (r600_debugfs_mc_info_init(rdev)) {
3368 DRM_ERROR("Failed to register debugfs file for mc !\n");
3369 }
3ce0a23d
JG		 3370 /* Read BIOS */
3371 if (!radeon_get_bios(rdev)) {
3372 if (ASIC_IS_AVIVO(rdev))
3373 return -EINVAL;
3374 }
3375 /* Must be an ATOMBIOS */
e7d40b9a
JG		 3376 if (!rdev->is_atom_bios) {
3377 dev_err(rdev->dev, "Expecting atombios for R600 GPU\n");
3ce0a23d 3378 return -EINVAL;
e7d40b9a 3379 }
3ce0a23d
JG		 3380 r = radeon_atombios_init(rdev);
3381 if (r)
3382 return r;
3383 /* Post card if necessary */
fd909c37 3384 if (!radeon_card_posted(rdev)) {
72542d77
DA		 3385 if (!rdev->bios) {
3386 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
3387 return -EINVAL;
3388 }
3ce0a23d
JG		 3389 DRM_INFO("GPU not posted. posting now...\n");
3390 atom_asic_init(rdev->mode_info.atom_context);
3391 }
3392 /* Initialize scratch registers */
3393 r600_scratch_init(rdev);
3394 /* Initialize surface registers */
3395 radeon_surface_init(rdev);
7433874e 3396 /* Initialize clocks */
5e6dde7e 3397 radeon_get_clock_info(rdev->ddev);
3ce0a23d 3398 /* Fence driver */
30eb77f4 3399 r = radeon_fence_driver_init(rdev);
3ce0a23d
JG		 3400 if (r)
3401 return r;
700a0cc0
JG		 3402 if (rdev->flags & RADEON_IS_AGP) {
3403 r = radeon_agp_init(rdev);
3404 if (r)
3405 radeon_agp_disable(rdev);
3406 }
3ce0a23d 3407 r = r600_mc_init(rdev);
b574f251 3408 if (r)
3ce0a23d 3409 return r;
3ce0a23d 3410 /* Memory manager */
4c788679 3411 r = radeon_bo_init(rdev);
3ce0a23d
JG		 3412 if (r)
3413 return r;
d8f60cfc 3414
e32eb50d
CK		 3415 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ring_obj = NULL;
3416 r600_ring_init(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX], 1024 * 1024);
3ce0a23d 3417
4d75658b
AD		 3418 rdev->ring[R600_RING_TYPE_DMA_INDEX].ring_obj = NULL;
3419 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX], 64 * 1024);
3420
d8f60cfc
AD		 3421 rdev->ih.ring_obj = NULL;
3422 r600_ih_ring_init(rdev, 64 * 1024);
3ce0a23d 3423
4aac0473
JG		 3424 r = r600_pcie_gart_init(rdev);
3425 if (r)
3426 return r;
3427
779720a3 3428 rdev->accel_working = true;
fc30b8ef 3429 r = r600_startup(rdev);
3ce0a23d 3430 if (r) {
655efd3d
JG		 3431 dev_err(rdev->dev, "disabling GPU acceleration\n");
3432 r600_cp_fini(rdev);
4d75658b 3433 r600_dma_fini(rdev);
655efd3d 3434 r600_irq_fini(rdev);
724c80e1 3435 radeon_wb_fini(rdev);
2898c348 3436 radeon_ib_pool_fini(rdev);
655efd3d 3437 radeon_irq_kms_fini(rdev);
75c81298 3438 r600_pcie_gart_fini(rdev);
733289c2 3439 rdev->accel_working = false;
3ce0a23d 3440 }
dafc3bd5 3441
3ce0a23d
JG		 3442 return 0;
3443}
3444
3445void r600_fini(struct radeon_device *rdev)
3446{
dafc3bd5 3447 r600_audio_fini(rdev);
3ce0a23d 3448 r600_blit_fini(rdev);
655efd3d 3449 r600_cp_fini(rdev);
4d75658b 3450 r600_dma_fini(rdev);
d8f60cfc 3451 r600_irq_fini(rdev);
724c80e1 3452 radeon_wb_fini(rdev);
2898c348 3453 radeon_ib_pool_fini(rdev);
d8f60cfc 3454 radeon_irq_kms_fini(rdev);
4aac0473 3455 r600_pcie_gart_fini(rdev);
16cdf04d 3456 r600_vram_scratch_fini(rdev);
655efd3d 3457 radeon_agp_fini(rdev);
3ce0a23d
JG		 3458 radeon_gem_fini(rdev);
3459 radeon_fence_driver_fini(rdev);
4c788679 3460 radeon_bo_fini(rdev);
e7d40b9a 3461 radeon_atombios_fini(rdev);
3ce0a23d
JG		 3462 kfree(rdev->bios);
3463 rdev->bios = NULL;
3ce0a23d
JG		 3464}
3465
3466
3467/*
3468 * CS stuff
3469 */
3470void r600_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
3471{
876dc9f3 3472 struct radeon_ring *ring = &rdev->ring[ib->ring];
89d35807 3473 u32 next_rptr;
7b1f2485 3474
45df6803 3475 if (ring->rptr_save_reg) {
89d35807 3476 next_rptr = ring->wptr + 3 + 4;
45df6803
CK		 3477 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3478 radeon_ring_write(ring, ((ring->rptr_save_reg -
3479 PACKET3_SET_CONFIG_REG_OFFSET) >> 2));
3480 radeon_ring_write(ring, next_rptr);
89d35807
AD		 3481 } else if (rdev->wb.enabled) {
3482 next_rptr = ring->wptr + 5 + 4;
3483 radeon_ring_write(ring, PACKET3(PACKET3_MEM_WRITE, 3));
3484 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
3485 radeon_ring_write(ring, (upper_32_bits(ring->next_rptr_gpu_addr) & 0xff) | (1 << 18));
3486 radeon_ring_write(ring, next_rptr);
3487 radeon_ring_write(ring, 0);
45df6803
CK		 3488 }
3489
e32eb50d
CK		 3490 radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
3491 radeon_ring_write(ring,
4eace7fd
CC		 3492#ifdef __BIG_ENDIAN
3493 (2 << 0) |
3494#endif
3495 (ib->gpu_addr & 0xFFFFFFFC));
e32eb50d
CK		 3496 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF);
3497 radeon_ring_write(ring, ib->length_dw);
3ce0a23d
JG		 3498}
3499
f2ba57b5
CK		 3500void r600_uvd_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
3501{
3502 struct radeon_ring *ring = &rdev->ring[ib->ring];
3503
3504 radeon_ring_write(ring, PACKET0(UVD_RBC_IB_BASE, 0));
3505 radeon_ring_write(ring, ib->gpu_addr);
3506 radeon_ring_write(ring, PACKET0(UVD_RBC_IB_SIZE, 0));
3507 radeon_ring_write(ring, ib->length_dw);
3508}
3509
f712812e 3510int r600_ib_test(struct radeon_device *rdev, struct radeon_ring *ring)
3ce0a23d 3511{
f2e39221 3512 struct radeon_ib ib;
3ce0a23d
JG		 3513 uint32_t scratch;
3514 uint32_t tmp = 0;
3515 unsigned i;
3516 int r;
3517
3518 r = radeon_scratch_get(rdev, &scratch);
3519 if (r) {
3520 DRM_ERROR("radeon: failed to get scratch reg (%d).\n", r);
3521 return r;
3522 }
3523 WREG32(scratch, 0xCAFEDEAD);
4bf3dd92 3524 r = radeon_ib_get(rdev, ring->idx, &ib, NULL, 256);
3ce0a23d
JG		 3525 if (r) {
3526 DRM_ERROR("radeon: failed to get ib (%d).\n", r);
af026c5b 3527 goto free_scratch;
3ce0a23d 3528 }
f2e39221
JG		 3529 ib.ptr[0] = PACKET3(PACKET3_SET_CONFIG_REG, 1);
3530 ib.ptr[1] = ((scratch - PACKET3_SET_CONFIG_REG_OFFSET) >> 2);
3531 ib.ptr[2] = 0xDEADBEEF;
3532 ib.length_dw = 3;
4ef72566 3533 r = radeon_ib_schedule(rdev, &ib, NULL);
3ce0a23d 3534 if (r) {
3ce0a23d 3535 DRM_ERROR("radeon: failed to schedule ib (%d).\n", r);
af026c5b 3536 goto free_ib;
3ce0a23d 3537 }
f2e39221 3538 r = radeon_fence_wait(ib.fence, false);
3ce0a23d
JG		 3539 if (r) {
3540 DRM_ERROR("radeon: fence wait failed (%d).\n", r);
af026c5b 3541 goto free_ib;
3ce0a23d
JG		 3542 }
3543 for (i = 0; i < rdev->usec_timeout; i++) {
3544 tmp = RREG32(scratch);
3545 if (tmp == 0xDEADBEEF)
3546 break;
3547 DRM_UDELAY(1);
3548 }
3549 if (i < rdev->usec_timeout) {
f2e39221 3550 DRM_INFO("ib test on ring %d succeeded in %u usecs\n", ib.fence->ring, i);
3ce0a23d 3551 } else {
4417d7f6 3552 DRM_ERROR("radeon: ib test failed (scratch(0x%04X)=0x%08X)\n",
3ce0a23d
JG		 3553 scratch, tmp);
3554 r = -EINVAL;
3555 }
af026c5b 3556free_ib:
3ce0a23d 3557 radeon_ib_free(rdev, &ib);
af026c5b
MD		 3558free_scratch:
3559 radeon_scratch_free(rdev, scratch);
771fe6b9
JG		 3560 return r;
3561}
3562
4d75658b
AD		 3563/**
3564 * r600_dma_ib_test - test an IB on the DMA engine
3565 *
3566 * @rdev: radeon_device pointer
3567 * @ring: radeon_ring structure holding ring information
3568 *
3569 * Test a simple IB in the DMA ring (r6xx-SI).
3570 * Returns 0 on success, error on failure.
3571 */
3572int r600_dma_ib_test(struct radeon_device *rdev, struct radeon_ring *ring)
3573{
3574 struct radeon_ib ib;
3575 unsigned i;
3576 int r;
3577 void __iomem *ptr = (void *)rdev->vram_scratch.ptr;
3578 u32 tmp = 0;
3579
3580 if (!ptr) {
3581 DRM_ERROR("invalid vram scratch pointer\n");
3582 return -EINVAL;
3583 }
3584
3585 tmp = 0xCAFEDEAD;
3586 writel(tmp, ptr);
3587
3588 r = radeon_ib_get(rdev, ring->idx, &ib, NULL, 256);
3589 if (r) {
3590 DRM_ERROR("radeon: failed to get ib (%d).\n", r);
3591 return r;
3592 }
3593
3594 ib.ptr[0] = DMA_PACKET(DMA_PACKET_WRITE, 0, 0, 1);
3595 ib.ptr[1] = rdev->vram_scratch.gpu_addr & 0xfffffffc;
3596 ib.ptr[2] = upper_32_bits(rdev->vram_scratch.gpu_addr) & 0xff;
3597 ib.ptr[3] = 0xDEADBEEF;
3598 ib.length_dw = 4;
3599
3600 r = radeon_ib_schedule(rdev, &ib, NULL);
3601 if (r) {
3602 radeon_ib_free(rdev, &ib);
3603 DRM_ERROR("radeon: failed to schedule ib (%d).\n", r);
3604 return r;
3605 }
3606 r = radeon_fence_wait(ib.fence, false);
3607 if (r) {
3608 DRM_ERROR("radeon: fence wait failed (%d).\n", r);
3609 return r;
3610 }
3611 for (i = 0; i < rdev->usec_timeout; i++) {
3612 tmp = readl(ptr);
3613 if (tmp == 0xDEADBEEF)
3614 break;
3615 DRM_UDELAY(1);
3616 }
3617 if (i < rdev->usec_timeout) {
3618 DRM_INFO("ib test on ring %d succeeded in %u usecs\n", ib.fence->ring, i);
3619 } else {
3620 DRM_ERROR("radeon: ib test failed (0x%08X)\n", tmp);
3621 r = -EINVAL;
3622 }
3623 radeon_ib_free(rdev, &ib);
3624 return r;
3625}
3626
f2ba57b5
CK		 3627int r600_uvd_ib_test(struct radeon_device *rdev, struct radeon_ring *ring)
3628{
b05e9e4c 3629 struct radeon_fence *fence = NULL;
f2ba57b5
CK		 3630 int r;
3631
b05e9e4c
CK		 3632 r = radeon_set_uvd_clocks(rdev, 53300, 40000);
3633 if (r) {
3634 DRM_ERROR("radeon: failed to raise UVD clocks (%d).\n", r);
3635 return r;
3636 }
3637
f2ba57b5
CK		 3638 r = radeon_uvd_get_create_msg(rdev, ring->idx, 1, NULL);
3639 if (r) {
3640 DRM_ERROR("radeon: failed to get create msg (%d).\n", r);
b05e9e4c 3641 goto error;
f2ba57b5
CK		 3642 }
3643
3644 r = radeon_uvd_get_destroy_msg(rdev, ring->idx, 1, &fence);
3645 if (r) {
3646 DRM_ERROR("radeon: failed to get destroy ib (%d).\n", r);
b05e9e4c 3647 goto error;
f2ba57b5
CK		 3648 }
3649
3650 r = radeon_fence_wait(fence, false);
3651 if (r) {
3652 DRM_ERROR("radeon: fence wait failed (%d).\n", r);
b05e9e4c 3653 goto error;
f2ba57b5
CK		 3654 }
3655 DRM_INFO("ib test on ring %d succeeded\n", ring->idx);
b05e9e4c 3656error:
f2ba57b5 3657 radeon_fence_unref(&fence);
b05e9e4c 3658 radeon_set_uvd_clocks(rdev, 0, 0);
f2ba57b5
CK		 3659 return r;
3660}
3661
4d75658b
AD		 3662/**
3663 * r600_dma_ring_ib_execute - Schedule an IB on the DMA engine
3664 *
3665 * @rdev: radeon_device pointer
3666 * @ib: IB object to schedule
3667 *
3668 * Schedule an IB in the DMA ring (r6xx-r7xx).
3669 */
3670void r600_dma_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
3671{
3672 struct radeon_ring *ring = &rdev->ring[ib->ring];
3673
3674 if (rdev->wb.enabled) {
3675 u32 next_rptr = ring->wptr + 4;
3676 while ((next_rptr & 7) != 5)
3677 next_rptr++;
3678 next_rptr += 3;
3679 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_WRITE, 0, 0, 1));
3680 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
3681 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xff);
3682 radeon_ring_write(ring, next_rptr);
3683 }
3684
3685 /* The indirect buffer packet must end on an 8 DW boundary in the DMA ring.
3686 * Pad as necessary with NOPs.
3687 */
3688 while ((ring->wptr & 7) != 5)
3689 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0));
3690 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_INDIRECT_BUFFER, 0, 0, 0));
3691 radeon_ring_write(ring, (ib->gpu_addr & 0xFFFFFFE0));
3692 radeon_ring_write(ring, (ib->length_dw << 16) | (upper_32_bits(ib->gpu_addr) & 0xFF));
3693
3694}
3695
d8f60cfc
AD		 3696/*
3697 * Interrupts
3698 *
3699 * Interrupts use a ring buffer on r6xx/r7xx hardware. It works pretty
3700 * the same as the CP ring buffer, but in reverse. Rather than the CPU
3701 * writing to the ring and the GPU consuming, the GPU writes to the ring
3702 * and host consumes. As the host irq handler processes interrupts, it
3703 * increments the rptr. When the rptr catches up with the wptr, all the
3704 * current interrupts have been processed.
3705 */
3706
3707void r600_ih_ring_init(struct radeon_device *rdev, unsigned ring_size)
3708{
3709 u32 rb_bufsz;
3710
3711 /* Align ring size */
3712 rb_bufsz = drm_order(ring_size / 4);
3713 ring_size = (1 << rb_bufsz) * 4;
3714 rdev->ih.ring_size = ring_size;
0c45249f
JG		 3715 rdev->ih.ptr_mask = rdev->ih.ring_size - 1;
3716 rdev->ih.rptr = 0;
d8f60cfc
AD		 3717}
3718
25a857fb 3719int r600_ih_ring_alloc(struct radeon_device *rdev)
d8f60cfc
AD		 3720{
3721 int r;
3722
d8f60cfc
AD		 3723 /* Allocate ring buffer */
3724 if (rdev->ih.ring_obj == NULL) {
441921d5 3725 r = radeon_bo_create(rdev, rdev->ih.ring_size,
268b2510 3726 PAGE_SIZE, true,
4c788679 3727 RADEON_GEM_DOMAIN_GTT,
40f5cf99 3728 NULL, &rdev->ih.ring_obj);
d8f60cfc
AD		 3729 if (r) {
3730 DRM_ERROR("radeon: failed to create ih ring buffer (%d).\n", r);
3731 return r;
3732 }
4c788679
JG		 3733 r = radeon_bo_reserve(rdev->ih.ring_obj, false);
3734 if (unlikely(r != 0))
3735 return r;
3736 r = radeon_bo_pin(rdev->ih.ring_obj,
3737 RADEON_GEM_DOMAIN_GTT,
3738 &rdev->ih.gpu_addr);
d8f60cfc 3739 if (r) {
4c788679 3740 radeon_bo_unreserve(rdev->ih.ring_obj);
d8f60cfc
AD		 3741 DRM_ERROR("radeon: failed to pin ih ring buffer (%d).\n", r);
3742 return r;
3743 }
4c788679
JG		 3744 r = radeon_bo_kmap(rdev->ih.ring_obj,
3745 (void **)&rdev->ih.ring);
3746 radeon_bo_unreserve(rdev->ih.ring_obj);
d8f60cfc
AD		 3747 if (r) {
3748 DRM_ERROR("radeon: failed to map ih ring buffer (%d).\n", r);
3749 return r;
3750 }
3751 }
d8f60cfc
AD		 3752 return 0;
3753}
3754
25a857fb 3755void r600_ih_ring_fini(struct radeon_device *rdev)
d8f60cfc 3756{
4c788679 3757 int r;
d8f60cfc 3758 if (rdev->ih.ring_obj) {
4c788679
JG		 3759 r = radeon_bo_reserve(rdev->ih.ring_obj, false);
3760 if (likely(r == 0)) {
3761 radeon_bo_kunmap(rdev->ih.ring_obj);
3762 radeon_bo_unpin(rdev->ih.ring_obj);
3763 radeon_bo_unreserve(rdev->ih.ring_obj);
3764 }
3765 radeon_bo_unref(&rdev->ih.ring_obj);
d8f60cfc
AD		 3766 rdev->ih.ring = NULL;
3767 rdev->ih.ring_obj = NULL;
3768 }
3769}
3770
45f9a39b 3771void r600_rlc_stop(struct radeon_device *rdev)
d8f60cfc
AD		 3772{
3773
45f9a39b
AD		 3774 if ((rdev->family >= CHIP_RV770) &&
3775 (rdev->family <= CHIP_RV740)) {
d8f60cfc
AD		 3776 /* r7xx asics need to soft reset RLC before halting */
3777 WREG32(SRBM_SOFT_RESET, SOFT_RESET_RLC);
3778 RREG32(SRBM_SOFT_RESET);
4de833c3 3779 mdelay(15);
d8f60cfc
AD		 3780 WREG32(SRBM_SOFT_RESET, 0);
3781 RREG32(SRBM_SOFT_RESET);
3782 }
3783
3784 WREG32(RLC_CNTL, 0);
3785}
3786
3787static void r600_rlc_start(struct radeon_device *rdev)
3788{
3789 WREG32(RLC_CNTL, RLC_ENABLE);
3790}
3791
3792static int r600_rlc_init(struct radeon_device *rdev)
3793{
3794 u32 i;
3795 const __be32 *fw_data;
3796
3797 if (!rdev->rlc_fw)
3798 return -EINVAL;
3799
3800 r600_rlc_stop(rdev);
3801
d8f60cfc 3802 WREG32(RLC_HB_CNTL, 0);
c420c745
AD		 3803
3804 if (rdev->family == CHIP_ARUBA) {
3805 WREG32(TN_RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
3806 WREG32(TN_RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
3807 }
3808 if (rdev->family <= CHIP_CAYMAN) {
3809 WREG32(RLC_HB_BASE, 0);
3810 WREG32(RLC_HB_RPTR, 0);
3811 WREG32(RLC_HB_WPTR, 0);
3812 }
12727809
AD		 3813 if (rdev->family <= CHIP_CAICOS) {
3814 WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
3815 WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
3816 }
d8f60cfc
AD		 3817 WREG32(RLC_MC_CNTL, 0);
3818 WREG32(RLC_UCODE_CNTL, 0);
3819
3820 fw_data = (const __be32 *)rdev->rlc_fw->data;
c420c745
AD		 3821 if (rdev->family >= CHIP_ARUBA) {
3822 for (i = 0; i < ARUBA_RLC_UCODE_SIZE; i++) {
3823 WREG32(RLC_UCODE_ADDR, i);
3824 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
3825 }
3826 } else if (rdev->family >= CHIP_CAYMAN) {
12727809
AD		 3827 for (i = 0; i < CAYMAN_RLC_UCODE_SIZE; i++) {
3828 WREG32(RLC_UCODE_ADDR, i);
3829 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
3830 }
3831 } else if (rdev->family >= CHIP_CEDAR) {
45f9a39b
AD		 3832 for (i = 0; i < EVERGREEN_RLC_UCODE_SIZE; i++) {
3833 WREG32(RLC_UCODE_ADDR, i);
3834 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
3835 }
3836 } else if (rdev->family >= CHIP_RV770) {
d8f60cfc
AD		 3837 for (i = 0; i < R700_RLC_UCODE_SIZE; i++) {
3838 WREG32(RLC_UCODE_ADDR, i);
3839 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
3840 }
3841 } else {
3842 for (i = 0; i < RLC_UCODE_SIZE; i++) {
3843 WREG32(RLC_UCODE_ADDR, i);
3844 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
3845 }
3846 }
3847 WREG32(RLC_UCODE_ADDR, 0);
3848
3849 r600_rlc_start(rdev);
3850
3851 return 0;
3852}
3853
3854static void r600_enable_interrupts(struct radeon_device *rdev)
3855{
3856 u32 ih_cntl = RREG32(IH_CNTL);
3857 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
3858
3859 ih_cntl |= ENABLE_INTR;
3860 ih_rb_cntl |= IH_RB_ENABLE;
3861 WREG32(IH_CNTL, ih_cntl);
3862 WREG32(IH_RB_CNTL, ih_rb_cntl);
3863 rdev->ih.enabled = true;
3864}
3865
45f9a39b 3866void r600_disable_interrupts(struct radeon_device *rdev)
d8f60cfc
AD		 3867{
3868 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
3869 u32 ih_cntl = RREG32(IH_CNTL);
3870
3871 ih_rb_cntl &= ~IH_RB_ENABLE;
3872 ih_cntl &= ~ENABLE_INTR;
3873 WREG32(IH_RB_CNTL, ih_rb_cntl);
3874 WREG32(IH_CNTL, ih_cntl);
3875 /* set rptr, wptr to 0 */
3876 WREG32(IH_RB_RPTR, 0);
3877 WREG32(IH_RB_WPTR, 0);
3878 rdev->ih.enabled = false;
d8f60cfc
AD		 3879 rdev->ih.rptr = 0;
3880}
3881
e0df1ac5
AD		 3882static void r600_disable_interrupt_state(struct radeon_device *rdev)
3883{
3884 u32 tmp;
3885
3555e53b 3886 WREG32(CP_INT_CNTL, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4d75658b
AD		 3887 tmp = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
3888 WREG32(DMA_CNTL, tmp);
e0df1ac5
AD		 3889 WREG32(GRBM_INT_CNTL, 0);
3890 WREG32(DxMODE_INT_MASK, 0);
6f34be50
AD		 3891 WREG32(D1GRPH_INTERRUPT_CONTROL, 0);
3892 WREG32(D2GRPH_INTERRUPT_CONTROL, 0);
e0df1ac5
AD		 3893 if (ASIC_IS_DCE3(rdev)) {
3894 WREG32(DCE3_DACA_AUTODETECT_INT_CONTROL, 0);
3895 WREG32(DCE3_DACB_AUTODETECT_INT_CONTROL, 0);
3896 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
3897 WREG32(DC_HPD1_INT_CONTROL, tmp);
3898 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
3899 WREG32(DC_HPD2_INT_CONTROL, tmp);
3900 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
3901 WREG32(DC_HPD3_INT_CONTROL, tmp);
3902 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
3903 WREG32(DC_HPD4_INT_CONTROL, tmp);
3904 if (ASIC_IS_DCE32(rdev)) {
3905 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5898b1f3 3906 WREG32(DC_HPD5_INT_CONTROL, tmp);
e0df1ac5 3907 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5898b1f3 3908 WREG32(DC_HPD6_INT_CONTROL, tmp);
c6543a6e
RM		 3909 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET0) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
3910 WREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET0, tmp);
3911 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET1) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
3912 WREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET1, tmp);
f122c610
AD		 3913 } else {
3914 tmp = RREG32(HDMI0_AUDIO_PACKET_CONTROL) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
3915 WREG32(HDMI0_AUDIO_PACKET_CONTROL, tmp);
3916 tmp = RREG32(DCE3_HDMI1_AUDIO_PACKET_CONTROL) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
3917 WREG32(DCE3_HDMI1_AUDIO_PACKET_CONTROL, tmp);
e0df1ac5
AD		 3918 }
3919 } else {
3920 WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
3921 WREG32(DACB_AUTODETECT_INT_CONTROL, 0);
3922 tmp = RREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL) & DC_HOT_PLUG_DETECTx_INT_POLARITY;
5898b1f3 3923 WREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL, tmp);
e0df1ac5 3924 tmp = RREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL) & DC_HOT_PLUG_DETECTx_INT_POLARITY;
5898b1f3 3925 WREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL, tmp);
e0df1ac5 3926 tmp = RREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL) & DC_HOT_PLUG_DETECTx_INT_POLARITY;
5898b1f3 3927 WREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL, tmp);
f122c610
AD		 3928 tmp = RREG32(HDMI0_AUDIO_PACKET_CONTROL) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
3929 WREG32(HDMI0_AUDIO_PACKET_CONTROL, tmp);
3930 tmp = RREG32(HDMI1_AUDIO_PACKET_CONTROL) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
3931 WREG32(HDMI1_AUDIO_PACKET_CONTROL, tmp);
e0df1ac5
AD		 3932 }
3933}
3934
d8f60cfc
AD		 3935int r600_irq_init(struct radeon_device *rdev)
3936{
3937 int ret = 0;
3938 int rb_bufsz;
3939 u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
3940
3941 /* allocate ring */
0c45249f 3942 ret = r600_ih_ring_alloc(rdev);
d8f60cfc
AD		 3943 if (ret)
3944 return ret;
3945
3946 /* disable irqs */
3947 r600_disable_interrupts(rdev);
3948
3949 /* init rlc */
3950 ret = r600_rlc_init(rdev);
3951 if (ret) {
3952 r600_ih_ring_fini(rdev);
3953 return ret;
3954 }
3955
3956 /* setup interrupt control */
3957 /* set dummy read address to ring address */
3958 WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
3959 interrupt_cntl = RREG32(INTERRUPT_CNTL);
3960 /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
3961 * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
3962 */
3963 interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
3964 /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
3965 interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
3966 WREG32(INTERRUPT_CNTL, interrupt_cntl);
3967
3968 WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
3969 rb_bufsz = drm_order(rdev->ih.ring_size / 4);
3970
3971 ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
3972 IH_WPTR_OVERFLOW_CLEAR |
3973 (rb_bufsz << 1));
724c80e1
AD		 3974
3975 if (rdev->wb.enabled)
3976 ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
3977
3978 /* set the writeback address whether it's enabled or not */
3979 WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
3980 WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
d8f60cfc
AD		 3981
3982 WREG32(IH_RB_CNTL, ih_rb_cntl);
3983
3984 /* set rptr, wptr to 0 */
3985 WREG32(IH_RB_RPTR, 0);
3986 WREG32(IH_RB_WPTR, 0);
3987
3988 /* Default settings for IH_CNTL (disabled at first) */
3989 ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10);
3990 /* RPTR_REARM only works if msi's are enabled */
3991 if (rdev->msi_enabled)
3992 ih_cntl |= RPTR_REARM;
d8f60cfc
AD		 3993 WREG32(IH_CNTL, ih_cntl);
3994
3995 /* force the active interrupt state to all disabled */
45f9a39b
AD		 3996 if (rdev->family >= CHIP_CEDAR)
3997 evergreen_disable_interrupt_state(rdev);
3998 else
3999 r600_disable_interrupt_state(rdev);
d8f60cfc 4000
2099810f
DA		 4001 /* at this point everything should be setup correctly to enable master */
4002 pci_set_master(rdev->pdev);
4003
d8f60cfc
AD		 4004 /* enable irqs */
4005 r600_enable_interrupts(rdev);
4006
4007 return ret;
4008}
4009
0c45249f 4010void r600_irq_suspend(struct radeon_device *rdev)
d8f60cfc 4011{
45f9a39b 4012 r600_irq_disable(rdev);
d8f60cfc 4013 r600_rlc_stop(rdev);
0c45249f
JG		 4014}
4015
4016void r600_irq_fini(struct radeon_device *rdev)
4017{
4018 r600_irq_suspend(rdev);
d8f60cfc
AD		 4019 r600_ih_ring_fini(rdev);
4020}
4021
4022int r600_irq_set(struct radeon_device *rdev)
4023{
e0df1ac5
AD		 4024 u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE;
4025 u32 mode_int = 0;
4026 u32 hpd1, hpd2, hpd3, hpd4 = 0, hpd5 = 0, hpd6 = 0;
2031f77c 4027 u32 grbm_int_cntl = 0;
f122c610 4028 u32 hdmi0, hdmi1;
6f34be50 4029 u32 d1grph = 0, d2grph = 0;
4d75658b 4030 u32 dma_cntl;
d8f60cfc 4031
003e69f9 4032 if (!rdev->irq.installed) {
fce7d61b 4033 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
003e69f9
JG		 4034 return -EINVAL;
4035 }
d8f60cfc 4036 /* don't enable anything if the ih is disabled */
79c2bbc5
JG		 4037 if (!rdev->ih.enabled) {
4038 r600_disable_interrupts(rdev);
4039 /* force the active interrupt state to all disabled */
4040 r600_disable_interrupt_state(rdev);
d8f60cfc 4041 return 0;
79c2bbc5 4042 }
d8f60cfc 4043
e0df1ac5
AD		 4044 if (ASIC_IS_DCE3(rdev)) {
4045 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
4046 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
4047 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
4048 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
4049 if (ASIC_IS_DCE32(rdev)) {
4050 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
4051 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
c6543a6e
RM		 4052 hdmi0 = RREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET0) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4053 hdmi1 = RREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET1) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
f122c610
AD		 4054 } else {
4055 hdmi0 = RREG32(HDMI0_AUDIO_PACKET_CONTROL) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
4056 hdmi1 = RREG32(DCE3_HDMI1_AUDIO_PACKET_CONTROL) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
e0df1ac5
AD		 4057 }
4058 } else {
4059 hpd1 = RREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL) & ~DC_HPDx_INT_EN;
4060 hpd2 = RREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL) & ~DC_HPDx_INT_EN;
4061 hpd3 = RREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL) & ~DC_HPDx_INT_EN;
f122c610
AD		 4062 hdmi0 = RREG32(HDMI0_AUDIO_PACKET_CONTROL) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
4063 hdmi1 = RREG32(HDMI1_AUDIO_PACKET_CONTROL) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
e0df1ac5 4064 }
4d75658b 4065 dma_cntl = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
e0df1ac5 4066
736fc37f 4067 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
d8f60cfc
AD		 4068 DRM_DEBUG("r600_irq_set: sw int\n");
4069 cp_int_cntl |= RB_INT_ENABLE;
d0f8a854 4070 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
d8f60cfc 4071 }
4d75658b
AD		 4072
4073 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
4074 DRM_DEBUG("r600_irq_set: sw int dma\n");
4075 dma_cntl |= TRAP_ENABLE;
4076 }
4077
6f34be50 4078 if (rdev->irq.crtc_vblank_int[0] ||
736fc37f 4079 atomic_read(&rdev->irq.pflip[0])) {
d8f60cfc
AD		 4080 DRM_DEBUG("r600_irq_set: vblank 0\n");
4081 mode_int |= D1MODE_VBLANK_INT_MASK;
4082 }
6f34be50 4083 if (rdev->irq.crtc_vblank_int[1] ||
736fc37f 4084 atomic_read(&rdev->irq.pflip[1])) {
d8f60cfc
AD		 4085 DRM_DEBUG("r600_irq_set: vblank 1\n");
4086 mode_int |= D2MODE_VBLANK_INT_MASK;
4087 }
e0df1ac5
AD		 4088 if (rdev->irq.hpd[0]) {
4089 DRM_DEBUG("r600_irq_set: hpd 1\n");
4090 hpd1 |= DC_HPDx_INT_EN;
4091 }
4092 if (rdev->irq.hpd[1]) {
4093 DRM_DEBUG("r600_irq_set: hpd 2\n");
4094 hpd2 |= DC_HPDx_INT_EN;
4095 }
4096 if (rdev->irq.hpd[2]) {
4097 DRM_DEBUG("r600_irq_set: hpd 3\n");
4098 hpd3 |= DC_HPDx_INT_EN;
4099 }
4100 if (rdev->irq.hpd[3]) {
4101 DRM_DEBUG("r600_irq_set: hpd 4\n");
4102 hpd4 |= DC_HPDx_INT_EN;
4103 }
4104 if (rdev->irq.hpd[4]) {
4105 DRM_DEBUG("r600_irq_set: hpd 5\n");
4106 hpd5 |= DC_HPDx_INT_EN;
4107 }
4108 if (rdev->irq.hpd[5]) {
4109 DRM_DEBUG("r600_irq_set: hpd 6\n");
4110 hpd6 |= DC_HPDx_INT_EN;
4111 }
f122c610
AD		 4112 if (rdev->irq.afmt[0]) {
4113 DRM_DEBUG("r600_irq_set: hdmi 0\n");
4114 hdmi0 |= HDMI0_AZ_FORMAT_WTRIG_MASK;
f2594933 4115 }
f122c610
AD		 4116 if (rdev->irq.afmt[1]) {
4117 DRM_DEBUG("r600_irq_set: hdmi 0\n");
4118 hdmi1 |= HDMI0_AZ_FORMAT_WTRIG_MASK;
f2594933 4119 }
d8f60cfc
AD		 4120
4121 WREG32(CP_INT_CNTL, cp_int_cntl);
4d75658b 4122 WREG32(DMA_CNTL, dma_cntl);
d8f60cfc 4123 WREG32(DxMODE_INT_MASK, mode_int);
6f34be50
AD		 4124 WREG32(D1GRPH_INTERRUPT_CONTROL, d1grph);
4125 WREG32(D2GRPH_INTERRUPT_CONTROL, d2grph);
2031f77c 4126 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
e0df1ac5
AD		 4127 if (ASIC_IS_DCE3(rdev)) {
4128 WREG32(DC_HPD1_INT_CONTROL, hpd1);
4129 WREG32(DC_HPD2_INT_CONTROL, hpd2);
4130 WREG32(DC_HPD3_INT_CONTROL, hpd3);
4131 WREG32(DC_HPD4_INT_CONTROL, hpd4);
4132 if (ASIC_IS_DCE32(rdev)) {
4133 WREG32(DC_HPD5_INT_CONTROL, hpd5);
4134 WREG32(DC_HPD6_INT_CONTROL, hpd6);
c6543a6e
RM		 4135 WREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET0, hdmi0);
4136 WREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET1, hdmi1);
f122c610
AD		 4137 } else {
4138 WREG32(HDMI0_AUDIO_PACKET_CONTROL, hdmi0);
4139 WREG32(DCE3_HDMI1_AUDIO_PACKET_CONTROL, hdmi1);
e0df1ac5
AD		 4140 }
4141 } else {
4142 WREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL, hpd1);
4143 WREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL, hpd2);
4144 WREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL, hpd3);
f122c610
AD		 4145 WREG32(HDMI0_AUDIO_PACKET_CONTROL, hdmi0);
4146 WREG32(HDMI1_AUDIO_PACKET_CONTROL, hdmi1);
e0df1ac5 4147 }
d8f60cfc
AD		 4148
4149 return 0;
4150}
4151
ce580fab 4152static void r600_irq_ack(struct radeon_device *rdev)
d8f60cfc 4153{
e0df1ac5
AD		 4154 u32 tmp;
4155
4156 if (ASIC_IS_DCE3(rdev)) {
6f34be50
AD		 4157 rdev->irq.stat_regs.r600.disp_int = RREG32(DCE3_DISP_INTERRUPT_STATUS);
4158 rdev->irq.stat_regs.r600.disp_int_cont = RREG32(DCE3_DISP_INTERRUPT_STATUS_CONTINUE);
4159 rdev->irq.stat_regs.r600.disp_int_cont2 = RREG32(DCE3_DISP_INTERRUPT_STATUS_CONTINUE2);
f122c610 4160 if (ASIC_IS_DCE32(rdev)) {
c6543a6e
RM		 4161 rdev->irq.stat_regs.r600.hdmi0_status = RREG32(AFMT_STATUS + DCE3_HDMI_OFFSET0);
4162 rdev->irq.stat_regs.r600.hdmi1_status = RREG32(AFMT_STATUS + DCE3_HDMI_OFFSET1);
f122c610
AD		 4163 } else {
4164 rdev->irq.stat_regs.r600.hdmi0_status = RREG32(HDMI0_STATUS);
4165 rdev->irq.stat_regs.r600.hdmi1_status = RREG32(DCE3_HDMI1_STATUS);
4166 }
e0df1ac5 4167 } else {
6f34be50
AD		 4168 rdev->irq.stat_regs.r600.disp_int = RREG32(DISP_INTERRUPT_STATUS);
4169 rdev->irq.stat_regs.r600.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
4170 rdev->irq.stat_regs.r600.disp_int_cont2 = 0;
f122c610
AD		 4171 rdev->irq.stat_regs.r600.hdmi0_status = RREG32(HDMI0_STATUS);
4172 rdev->irq.stat_regs.r600.hdmi1_status = RREG32(HDMI1_STATUS);
6f34be50
AD		 4173 }
4174 rdev->irq.stat_regs.r600.d1grph_int = RREG32(D1GRPH_INTERRUPT_STATUS);
4175 rdev->irq.stat_regs.r600.d2grph_int = RREG32(D2GRPH_INTERRUPT_STATUS);
4176
4177 if (rdev->irq.stat_regs.r600.d1grph_int & DxGRPH_PFLIP_INT_OCCURRED)
4178 WREG32(D1GRPH_INTERRUPT_STATUS, DxGRPH_PFLIP_INT_CLEAR);
4179 if (rdev->irq.stat_regs.r600.d2grph_int & DxGRPH_PFLIP_INT_OCCURRED)
4180 WREG32(D2GRPH_INTERRUPT_STATUS, DxGRPH_PFLIP_INT_CLEAR);
4181 if (rdev->irq.stat_regs.r600.disp_int & LB_D1_VBLANK_INTERRUPT)
d8f60cfc 4182 WREG32(D1MODE_VBLANK_STATUS, DxMODE_VBLANK_ACK);
6f34be50 4183 if (rdev->irq.stat_regs.r600.disp_int & LB_D1_VLINE_INTERRUPT)
d8f60cfc 4184 WREG32(D1MODE_VLINE_STATUS, DxMODE_VLINE_ACK);
6f34be50 4185 if (rdev->irq.stat_regs.r600.disp_int & LB_D2_VBLANK_INTERRUPT)
d8f60cfc 4186 WREG32(D2MODE_VBLANK_STATUS, DxMODE_VBLANK_ACK);
6f34be50 4187 if (rdev->irq.stat_regs.r600.disp_int & LB_D2_VLINE_INTERRUPT)
d8f60cfc 4188 WREG32(D2MODE_VLINE_STATUS, DxMODE_VLINE_ACK);
6f34be50 4189 if (rdev->irq.stat_regs.r600.disp_int & DC_HPD1_INTERRUPT) {
e0df1ac5
AD		 4190 if (ASIC_IS_DCE3(rdev)) {
4191 tmp = RREG32(DC_HPD1_INT_CONTROL);
4192 tmp |= DC_HPDx_INT_ACK;
4193 WREG32(DC_HPD1_INT_CONTROL, tmp);
4194 } else {
4195 tmp = RREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL);
4196 tmp |= DC_HPDx_INT_ACK;
4197 WREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL, tmp);
4198 }
4199 }
6f34be50 4200 if (rdev->irq.stat_regs.r600.disp_int & DC_HPD2_INTERRUPT) {
e0df1ac5
AD		 4201 if (ASIC_IS_DCE3(rdev)) {
4202 tmp = RREG32(DC_HPD2_INT_CONTROL);
4203 tmp |= DC_HPDx_INT_ACK;
4204 WREG32(DC_HPD2_INT_CONTROL, tmp);
4205 } else {
4206 tmp = RREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL);
4207 tmp |= DC_HPDx_INT_ACK;
4208 WREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL, tmp);
4209 }
4210 }
6f34be50 4211 if (rdev->irq.stat_regs.r600.disp_int_cont & DC_HPD3_INTERRUPT) {
e0df1ac5
AD		 4212 if (ASIC_IS_DCE3(rdev)) {
4213 tmp = RREG32(DC_HPD3_INT_CONTROL);
4214 tmp |= DC_HPDx_INT_ACK;
4215 WREG32(DC_HPD3_INT_CONTROL, tmp);
4216 } else {
4217 tmp = RREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL);
4218 tmp |= DC_HPDx_INT_ACK;
4219 WREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL, tmp);
4220 }
4221 }
6f34be50 4222 if (rdev->irq.stat_regs.r600.disp_int_cont & DC_HPD4_INTERRUPT) {
e0df1ac5
AD		 4223 tmp = RREG32(DC_HPD4_INT_CONTROL);
4224 tmp |= DC_HPDx_INT_ACK;
4225 WREG32(DC_HPD4_INT_CONTROL, tmp);
4226 }
4227 if (ASIC_IS_DCE32(rdev)) {
6f34be50 4228 if (rdev->irq.stat_regs.r600.disp_int_cont2 & DC_HPD5_INTERRUPT) {
e0df1ac5
AD		 4229 tmp = RREG32(DC_HPD5_INT_CONTROL);
4230 tmp |= DC_HPDx_INT_ACK;
4231 WREG32(DC_HPD5_INT_CONTROL, tmp);
4232 }
6f34be50 4233 if (rdev->irq.stat_regs.r600.disp_int_cont2 & DC_HPD6_INTERRUPT) {
e0df1ac5
AD		 4234 tmp = RREG32(DC_HPD5_INT_CONTROL);
4235 tmp |= DC_HPDx_INT_ACK;
4236 WREG32(DC_HPD6_INT_CONTROL, tmp);
4237 }
f122c610 4238 if (rdev->irq.stat_regs.r600.hdmi0_status & AFMT_AZ_FORMAT_WTRIG) {
c6543a6e 4239 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET0);
f122c610 4240 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
c6543a6e 4241 WREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET0, tmp);
f122c610
AD		 4242 }
4243 if (rdev->irq.stat_regs.r600.hdmi1_status & AFMT_AZ_FORMAT_WTRIG) {
c6543a6e 4244 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET1);
f122c610 4245 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
c6543a6e 4246 WREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET1, tmp);
f2594933
CK		 4247 }
4248 } else {
f122c610
AD		 4249 if (rdev->irq.stat_regs.r600.hdmi0_status & HDMI0_AZ_FORMAT_WTRIG) {
4250 tmp = RREG32(HDMI0_AUDIO_PACKET_CONTROL);
4251 tmp |= HDMI0_AZ_FORMAT_WTRIG_ACK;
4252 WREG32(HDMI0_AUDIO_PACKET_CONTROL, tmp);
4253 }
4254 if (rdev->irq.stat_regs.r600.hdmi1_status & HDMI0_AZ_FORMAT_WTRIG) {
4255 if (ASIC_IS_DCE3(rdev)) {
4256 tmp = RREG32(DCE3_HDMI1_AUDIO_PACKET_CONTROL);
4257 tmp |= HDMI0_AZ_FORMAT_WTRIG_ACK;
4258 WREG32(DCE3_HDMI1_AUDIO_PACKET_CONTROL, tmp);
4259 } else {
4260 tmp = RREG32(HDMI1_AUDIO_PACKET_CONTROL);
4261 tmp |= HDMI0_AZ_FORMAT_WTRIG_ACK;
4262 WREG32(HDMI1_AUDIO_PACKET_CONTROL, tmp);
4263 }
f2594933
CK		 4264 }
4265 }
d8f60cfc
AD		 4266}
4267
4268void r600_irq_disable(struct radeon_device *rdev)
4269{
d8f60cfc
AD		 4270 r600_disable_interrupts(rdev);
4271 /* Wait and acknowledge irq */
4272 mdelay(1);
6f34be50 4273 r600_irq_ack(rdev);
e0df1ac5 4274 r600_disable_interrupt_state(rdev);
d8f60cfc
AD		 4275}
4276
ce580fab 4277static u32 r600_get_ih_wptr(struct radeon_device *rdev)
d8f60cfc
AD		 4278{
4279 u32 wptr, tmp;
3ce0a23d 4280
724c80e1 4281 if (rdev->wb.enabled)
204ae24d 4282 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
724c80e1
AD		 4283 else
4284 wptr = RREG32(IH_RB_WPTR);
3ce0a23d 4285
d8f60cfc 4286 if (wptr & RB_OVERFLOW) {
7924e5eb
JG		 4287 /* When a ring buffer overflow happen start parsing interrupt
4288 * from the last not overwritten vector (wptr + 16). Hopefully
4289 * this should allow us to catchup.
4290 */
4291 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
4292 wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
4293 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
d8f60cfc
AD		 4294 tmp = RREG32(IH_RB_CNTL);
4295 tmp |= IH_WPTR_OVERFLOW_CLEAR;
4296 WREG32(IH_RB_CNTL, tmp);
4297 }
0c45249f 4298 return (wptr & rdev->ih.ptr_mask);
d8f60cfc 4299}
3ce0a23d 4300
d8f60cfc
AD		 4301/* r600 IV Ring
4302 * Each IV ring entry is 128 bits:
4303 * [7:0] - interrupt source id
4304 * [31:8] - reserved
4305 * [59:32] - interrupt source data
4306 * [127:60] - reserved
4307 *
4308 * The basic interrupt vector entries
4309 * are decoded as follows:
4310 * src_id src_data description
4311 * 1 0 D1 Vblank
4312 * 1 1 D1 Vline
4313 * 5 0 D2 Vblank
4314 * 5 1 D2 Vline
4315 * 19 0 FP Hot plug detection A
4316 * 19 1 FP Hot plug detection B
4317 * 19 2 DAC A auto-detection
4318 * 19 3 DAC B auto-detection
f2594933
CK		 4319 * 21 4 HDMI block A
4320 * 21 5 HDMI block B
d8f60cfc
AD		 4321 * 176 - CP_INT RB
4322 * 177 - CP_INT IB1
4323 * 178 - CP_INT IB2
4324 * 181 - EOP Interrupt
4325 * 233 - GUI Idle
4326 *
4327 * Note, these are based on r600 and may need to be
4328 * adjusted or added to on newer asics
4329 */
4330
4331int r600_irq_process(struct radeon_device *rdev)
4332{
682f1a54
DA		 4333 u32 wptr;
4334 u32 rptr;
d8f60cfc 4335 u32 src_id, src_data;
6f34be50 4336 u32 ring_index;
d4877cf2 4337 bool queue_hotplug = false;
f122c610 4338 bool queue_hdmi = false;
d8f60cfc 4339
682f1a54 4340 if (!rdev->ih.enabled || rdev->shutdown)
79c2bbc5 4341 return IRQ_NONE;
d8f60cfc 4342
f6a56939
BH		 4343 /* No MSIs, need a dummy read to flush PCI DMAs */
4344 if (!rdev->msi_enabled)
4345 RREG32(IH_RB_WPTR);
4346
682f1a54 4347 wptr = r600_get_ih_wptr(rdev);
d8f60cfc 4348
c20dc369
CK		 4349restart_ih:
4350 /* is somebody else already processing irqs? */
4351 if (atomic_xchg(&rdev->ih.lock, 1))
d8f60cfc 4352 return IRQ_NONE;
d8f60cfc 4353
c20dc369
CK		 4354 rptr = rdev->ih.rptr;
4355 DRM_DEBUG("r600_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
4356
964f6645
BH		 4357 /* Order reading of wptr vs. reading of IH ring data */
4358 rmb();
4359
d8f60cfc 4360 /* display interrupts */
6f34be50 4361 r600_irq_ack(rdev);
d8f60cfc 4362
d8f60cfc
AD		 4363 while (rptr != wptr) {
4364 /* wptr/rptr are in bytes! */
4365 ring_index = rptr / 4;
4eace7fd
CC		 4366 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
4367 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
d8f60cfc
AD		 4368
4369 switch (src_id) {
4370 case 1: /* D1 vblank/vline */
4371 switch (src_data) {
4372 case 0: /* D1 vblank */
6f34be50 4373 if (rdev->irq.stat_regs.r600.disp_int & LB_D1_VBLANK_INTERRUPT) {
6f34be50
AD		 4374 if (rdev->irq.crtc_vblank_int[0]) {
4375 drm_handle_vblank(rdev->ddev, 0);
4376 rdev->pm.vblank_sync = true;
4377 wake_up(&rdev->irq.vblank_queue);
4378 }
736fc37f 4379 if (atomic_read(&rdev->irq.pflip[0]))
3e4ea742 4380 radeon_crtc_handle_flip(rdev, 0);
6f34be50 4381 rdev->irq.stat_regs.r600.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
d8f60cfc
AD		 4382 DRM_DEBUG("IH: D1 vblank\n");
4383 }
4384 break;
4385 case 1: /* D1 vline */
6f34be50
AD		 4386 if (rdev->irq.stat_regs.r600.disp_int & LB_D1_VLINE_INTERRUPT) {
4387 rdev->irq.stat_regs.r600.disp_int &= ~LB_D1_VLINE_INTERRUPT;
d8f60cfc
AD		 4388 DRM_DEBUG("IH: D1 vline\n");
4389 }
4390 break;
4391 default:
b042589c 4392 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
d8f60cfc
AD		 4393 break;
4394 }
4395 break;
4396 case 5: /* D2 vblank/vline */
4397 switch (src_data) {
4398 case 0: /* D2 vblank */
6f34be50 4399 if (rdev->irq.stat_regs.r600.disp_int & LB_D2_VBLANK_INTERRUPT) {
6f34be50
AD		 4400 if (rdev->irq.crtc_vblank_int[1]) {
4401 drm_handle_vblank(rdev->ddev, 1);
4402 rdev->pm.vblank_sync = true;
4403 wake_up(&rdev->irq.vblank_queue);
4404 }
736fc37f 4405 if (atomic_read(&rdev->irq.pflip[1]))
3e4ea742 4406 radeon_crtc_handle_flip(rdev, 1);
6f34be50 4407 rdev->irq.stat_regs.r600.disp_int &= ~LB_D2_VBLANK_INTERRUPT;
d8f60cfc
AD		 4408 DRM_DEBUG("IH: D2 vblank\n");
4409 }
4410 break;
4411 case 1: /* D1 vline */
6f34be50
AD		 4412 if (rdev->irq.stat_regs.r600.disp_int & LB_D2_VLINE_INTERRUPT) {
4413 rdev->irq.stat_regs.r600.disp_int &= ~LB_D2_VLINE_INTERRUPT;
d8f60cfc
AD		 4414 DRM_DEBUG("IH: D2 vline\n");
4415 }
4416 break;
4417 default:
b042589c 4418 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
d8f60cfc
AD		 4419 break;
4420 }
4421 break;
e0df1ac5
AD		 4422 case 19: /* HPD/DAC hotplug */
4423 switch (src_data) {
4424 case 0:
6f34be50
AD		 4425 if (rdev->irq.stat_regs.r600.disp_int & DC_HPD1_INTERRUPT) {
4426 rdev->irq.stat_regs.r600.disp_int &= ~DC_HPD1_INTERRUPT;
d4877cf2
AD		 4427 queue_hotplug = true;
4428 DRM_DEBUG("IH: HPD1\n");
e0df1ac5
AD		 4429 }
4430 break;
4431 case 1:
6f34be50
AD		 4432 if (rdev->irq.stat_regs.r600.disp_int & DC_HPD2_INTERRUPT) {
4433 rdev->irq.stat_regs.r600.disp_int &= ~DC_HPD2_INTERRUPT;
d4877cf2
AD		 4434 queue_hotplug = true;
4435 DRM_DEBUG("IH: HPD2\n");
e0df1ac5
AD		 4436 }
4437 break;
4438 case 4:
6f34be50
AD		 4439 if (rdev->irq.stat_regs.r600.disp_int_cont & DC_HPD3_INTERRUPT) {
4440 rdev->irq.stat_regs.r600.disp_int_cont &= ~DC_HPD3_INTERRUPT;
d4877cf2
AD		 4441 queue_hotplug = true;
4442 DRM_DEBUG("IH: HPD3\n");
e0df1ac5
AD		 4443 }
4444 break;
4445 case 5:
6f34be50
AD		 4446 if (rdev->irq.stat_regs.r600.disp_int_cont & DC_HPD4_INTERRUPT) {
4447 rdev->irq.stat_regs.r600.disp_int_cont &= ~DC_HPD4_INTERRUPT;
d4877cf2
AD		 4448 queue_hotplug = true;
4449 DRM_DEBUG("IH: HPD4\n");
e0df1ac5
AD		 4450 }
4451 break;
4452 case 10:
6f34be50
AD		 4453 if (rdev->irq.stat_regs.r600.disp_int_cont2 & DC_HPD5_INTERRUPT) {
4454 rdev->irq.stat_regs.r600.disp_int_cont2 &= ~DC_HPD5_INTERRUPT;
d4877cf2
AD		 4455 queue_hotplug = true;
4456 DRM_DEBUG("IH: HPD5\n");
e0df1ac5
AD		 4457 }
4458 break;
4459 case 12:
6f34be50
AD		 4460 if (rdev->irq.stat_regs.r600.disp_int_cont2 & DC_HPD6_INTERRUPT) {
4461 rdev->irq.stat_regs.r600.disp_int_cont2 &= ~DC_HPD6_INTERRUPT;
d4877cf2
AD		 4462 queue_hotplug = true;
4463 DRM_DEBUG("IH: HPD6\n");
e0df1ac5
AD		 4464 }
4465 break;
4466 default:
b042589c 4467 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
e0df1ac5
AD		 4468 break;
4469 }
4470 break;
f122c610
AD		 4471 case 21: /* hdmi */
4472 switch (src_data) {
4473 case 4:
4474 if (rdev->irq.stat_regs.r600.hdmi0_status & HDMI0_AZ_FORMAT_WTRIG) {
4475 rdev->irq.stat_regs.r600.hdmi0_status &= ~HDMI0_AZ_FORMAT_WTRIG;
4476 queue_hdmi = true;
4477 DRM_DEBUG("IH: HDMI0\n");
4478 }
4479 break;
4480 case 5:
4481 if (rdev->irq.stat_regs.r600.hdmi1_status & HDMI0_AZ_FORMAT_WTRIG) {
4482 rdev->irq.stat_regs.r600.hdmi1_status &= ~HDMI0_AZ_FORMAT_WTRIG;
4483 queue_hdmi = true;
4484 DRM_DEBUG("IH: HDMI1\n");
4485 }
4486 break;
4487 default:
4488 DRM_ERROR("Unhandled interrupt: %d %d\n", src_id, src_data);
4489 break;
4490 }
f2594933 4491 break;
d8f60cfc
AD		 4492 case 176: /* CP_INT in ring buffer */
4493 case 177: /* CP_INT in IB1 */
4494 case 178: /* CP_INT in IB2 */
4495 DRM_DEBUG("IH: CP int: 0x%08x\n", src_data);
7465280c 4496 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
d8f60cfc
AD		 4497 break;
4498 case 181: /* CP EOP event */
4499 DRM_DEBUG("IH: CP EOP\n");
7465280c 4500 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
d8f60cfc 4501 break;
4d75658b
AD		 4502 case 224: /* DMA trap event */
4503 DRM_DEBUG("IH: DMA trap\n");
4504 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
4505 break;
2031f77c 4506 case 233: /* GUI IDLE */
303c805c 4507 DRM_DEBUG("IH: GUI idle\n");
2031f77c 4508 break;
d8f60cfc 4509 default:
b042589c 4510 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
d8f60cfc
AD		 4511 break;
4512 }
4513
4514 /* wptr/rptr are in bytes! */
0c45249f
JG		 4515 rptr += 16;
4516 rptr &= rdev->ih.ptr_mask;
d8f60cfc 4517 }
d4877cf2 4518 if (queue_hotplug)
32c87fca 4519 schedule_work(&rdev->hotplug_work);
f122c610
AD		 4520 if (queue_hdmi)
4521 schedule_work(&rdev->audio_work);
d8f60cfc
AD		 4522 rdev->ih.rptr = rptr;
4523 WREG32(IH_RB_RPTR, rdev->ih.rptr);
c20dc369
CK		 4524 atomic_set(&rdev->ih.lock, 0);
4525
4526 /* make sure wptr hasn't changed while processing */
4527 wptr = r600_get_ih_wptr(rdev);
4528 if (wptr != rptr)
4529 goto restart_ih;
4530
d8f60cfc
AD		 4531 return IRQ_HANDLED;
4532}
3ce0a23d
JG		 4533
4534/*
4535 * Debugfs info
4536 */
4537#if defined(CONFIG_DEBUG_FS)
4538
3ce0a23d
JG		 4539static int r600_debugfs_mc_info(struct seq_file *m, void *data)
4540{
4541 struct drm_info_node *node = (struct drm_info_node *) m->private;
4542 struct drm_device *dev = node->minor->dev;
4543 struct radeon_device *rdev = dev->dev_private;
4544
4545 DREG32_SYS(m, rdev, R_000E50_SRBM_STATUS);
4546 DREG32_SYS(m, rdev, VM_L2_STATUS);
4547 return 0;
4548}
4549
4550static struct drm_info_list r600_mc_info_list[] = {
4551 {"r600_mc_info", r600_debugfs_mc_info, 0, NULL},
3ce0a23d
JG		 4552};
4553#endif
4554
4555int r600_debugfs_mc_info_init(struct radeon_device *rdev)
4556{
4557#if defined(CONFIG_DEBUG_FS)
4558 return radeon_debugfs_add_files(rdev, r600_mc_info_list, ARRAY_SIZE(r600_mc_info_list));
4559#else
4560 return 0;
4561#endif
771fe6b9 4562}
062b389c
JG		 4563
4564/**
4565 * r600_ioctl_wait_idle - flush host path cache on wait idle ioctl
4566 * rdev: radeon device structure
4567 * bo: buffer object struct which userspace is waiting for idle
4568 *
4569 * Some R6XX/R7XX doesn't seems to take into account HDP flush performed
4570 * through ring buffer, this leads to corruption in rendering, see
4571 * http://bugzilla.kernel.org/show_bug.cgi?id=15186 to avoid this we
4572 * directly perform HDP flush by writing register through MMIO.
4573 */
4574void r600_ioctl_wait_idle(struct radeon_device *rdev, struct radeon_bo *bo)
4575{
812d0469 4576 /* r7xx hw bug. write to HDP_DEBUG1 followed by fb read
f3886f85
AD		 4577 * rather than write to HDP_REG_COHERENCY_FLUSH_CNTL.
4578 * This seems to cause problems on some AGP cards. Just use the old
4579 * method for them.
812d0469 4580 */
e488459a 4581 if ((rdev->family >= CHIP_RV770) && (rdev->family <= CHIP_RV740) &&
f3886f85 4582 rdev->vram_scratch.ptr && !(rdev->flags & RADEON_IS_AGP)) {
87cbf8f2 4583 void __iomem *ptr = (void *)rdev->vram_scratch.ptr;
812d0469
AD		 4584 u32 tmp;
4585
4586 WREG32(HDP_DEBUG1, 0);
4587 tmp = readl((void __iomem *)ptr);
4588 } else
4589 WREG32(R_005480_HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
062b389c 4590}
3313e3d4
AD		 4591
4592void r600_set_pcie_lanes(struct radeon_device *rdev, int lanes)
4593{
d5445a17 4594 u32 link_width_cntl, mask;
3313e3d4
AD		 4595
4596 if (rdev->flags & RADEON_IS_IGP)
4597 return;
4598
4599 if (!(rdev->flags & RADEON_IS_PCIE))
4600 return;
4601
4602 /* x2 cards have a special sequence */
4603 if (ASIC_IS_X2(rdev))
4604 return;
4605
d5445a17 4606 radeon_gui_idle(rdev);
3313e3d4
AD		 4607
4608 switch (lanes) {
4609 case 0:
4610 mask = RADEON_PCIE_LC_LINK_WIDTH_X0;
4611 break;
4612 case 1:
4613 mask = RADEON_PCIE_LC_LINK_WIDTH_X1;
4614 break;
4615 case 2:
4616 mask = RADEON_PCIE_LC_LINK_WIDTH_X2;
4617 break;
4618 case 4:
4619 mask = RADEON_PCIE_LC_LINK_WIDTH_X4;
4620 break;
4621 case 8:
4622 mask = RADEON_PCIE_LC_LINK_WIDTH_X8;
4623 break;
4624 case 12:
d5445a17 4625 /* not actually supported */
3313e3d4
AD		 4626 mask = RADEON_PCIE_LC_LINK_WIDTH_X12;
4627 break;
4628 case 16:
3313e3d4
AD		 4629 mask = RADEON_PCIE_LC_LINK_WIDTH_X16;
4630 break;
d5445a17
AD		 4631 default:
4632 DRM_ERROR("invalid pcie lane request: %d\n", lanes);
4633 return;
3313e3d4
AD		 4634 }
4635
492d2b61 4636 link_width_cntl = RREG32_PCIE_PORT(RADEON_PCIE_LC_LINK_WIDTH_CNTL);
d5445a17
AD		 4637 link_width_cntl &= ~RADEON_PCIE_LC_LINK_WIDTH_MASK;
4638 link_width_cntl |= mask << RADEON_PCIE_LC_LINK_WIDTH_SHIFT;
4639 link_width_cntl |= (RADEON_PCIE_LC_RECONFIG_NOW |
4640 R600_PCIE_LC_RECONFIG_ARC_MISSING_ESCAPE);
3313e3d4 4641
492d2b61 4642 WREG32_PCIE_PORT(RADEON_PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
3313e3d4
AD		 4643}
4644
4645int r600_get_pcie_lanes(struct radeon_device *rdev)
4646{
4647 u32 link_width_cntl;
4648
4649 if (rdev->flags & RADEON_IS_IGP)
4650 return 0;
4651
4652 if (!(rdev->flags & RADEON_IS_PCIE))
4653 return 0;
4654
4655 /* x2 cards have a special sequence */
4656 if (ASIC_IS_X2(rdev))
4657 return 0;
4658
d5445a17 4659 radeon_gui_idle(rdev);
3313e3d4 4660
492d2b61 4661 link_width_cntl = RREG32_PCIE_PORT(RADEON_PCIE_LC_LINK_WIDTH_CNTL);
3313e3d4
AD		 4662
4663 switch ((link_width_cntl & RADEON_PCIE_LC_LINK_WIDTH_RD_MASK) >> RADEON_PCIE_LC_LINK_WIDTH_RD_SHIFT) {
3313e3d4
AD		 4664 case RADEON_PCIE_LC_LINK_WIDTH_X1:
4665 return 1;
4666 case RADEON_PCIE_LC_LINK_WIDTH_X2:
4667 return 2;
4668 case RADEON_PCIE_LC_LINK_WIDTH_X4:
4669 return 4;
4670 case RADEON_PCIE_LC_LINK_WIDTH_X8:
4671 return 8;
d5445a17
AD		 4672 case RADEON_PCIE_LC_LINK_WIDTH_X12:
4673 /* not actually supported */
4674 return 12;
4675 case RADEON_PCIE_LC_LINK_WIDTH_X0:
3313e3d4
AD		 4676 case RADEON_PCIE_LC_LINK_WIDTH_X16:
4677 default:
4678 return 16;
4679 }
4680}
4681
9e46a48d
AD		 4682static void r600_pcie_gen2_enable(struct radeon_device *rdev)
4683{
4684 u32 link_width_cntl, lanes, speed_cntl, training_cntl, tmp;
4685 u16 link_cntl2;
4686
d42dd579
AD		 4687 if (radeon_pcie_gen2 == 0)
4688 return;
4689
9e46a48d
AD		 4690 if (rdev->flags & RADEON_IS_IGP)
4691 return;
4692
4693 if (!(rdev->flags & RADEON_IS_PCIE))
4694 return;
4695
4696 /* x2 cards have a special sequence */
4697 if (ASIC_IS_X2(rdev))
4698 return;
4699
4700 /* only RV6xx+ chips are supported */
4701 if (rdev->family <= CHIP_R600)
4702 return;
4703
7e0e4196
KSS		 4704 if ((rdev->pdev->bus->max_bus_speed != PCIE_SPEED_5_0GT) &&
4705 (rdev->pdev->bus->max_bus_speed != PCIE_SPEED_8_0GT))
197bbb3d
DA		 4706 return;
4707
492d2b61 4708 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
3691feea
AD		 4709 if (speed_cntl & LC_CURRENT_DATA_RATE) {
4710 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
4711 return;
4712 }
4713
197bbb3d
DA		 4714 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
4715
9e46a48d
AD		 4716 /* 55 nm r6xx asics */
4717 if ((rdev->family == CHIP_RV670) ||
4718 (rdev->family == CHIP_RV620) ||
4719 (rdev->family == CHIP_RV635)) {
4720 /* advertise upconfig capability */
492d2b61 4721 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9e46a48d 4722 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
492d2b61
AD		 4723 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
4724 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9e46a48d
AD		 4725 if (link_width_cntl & LC_RENEGOTIATION_SUPPORT) {
4726 lanes = (link_width_cntl & LC_LINK_WIDTH_RD_MASK) >> LC_LINK_WIDTH_RD_SHIFT;
4727 link_width_cntl &= ~(LC_LINK_WIDTH_MASK |
4728 LC_RECONFIG_ARC_MISSING_ESCAPE);
4729 link_width_cntl |= lanes | LC_RECONFIG_NOW | LC_RENEGOTIATE_EN;
492d2b61 4730 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
9e46a48d
AD		 4731 } else {
4732 link_width_cntl |= LC_UPCONFIGURE_DIS;
492d2b61 4733 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
9e46a48d
AD		 4734 }
4735 }
4736
492d2b61 4737 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9e46a48d
AD		 4738 if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) &&
4739 (speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
4740
4741 /* 55 nm r6xx asics */
4742 if ((rdev->family == CHIP_RV670) ||
4743 (rdev->family == CHIP_RV620) ||
4744 (rdev->family == CHIP_RV635)) {
4745 WREG32(MM_CFGREGS_CNTL, 0x8);
4746 link_cntl2 = RREG32(0x4088);
4747 WREG32(MM_CFGREGS_CNTL, 0);
4748 /* not supported yet */
4749 if (link_cntl2 & SELECTABLE_DEEMPHASIS)
4750 return;
4751 }
4752
4753 speed_cntl &= ~LC_SPEED_CHANGE_ATTEMPTS_ALLOWED_MASK;
4754 speed_cntl |= (0x3 << LC_SPEED_CHANGE_ATTEMPTS_ALLOWED_SHIFT);
4755 speed_cntl &= ~LC_VOLTAGE_TIMER_SEL_MASK;
4756 speed_cntl &= ~LC_FORCE_DIS_HW_SPEED_CHANGE;
4757 speed_cntl |= LC_FORCE_EN_HW_SPEED_CHANGE;
492d2b61 4758 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9e46a48d
AD		 4759
4760 tmp = RREG32(0x541c);
4761 WREG32(0x541c, tmp | 0x8);
4762 WREG32(MM_CFGREGS_CNTL, MM_WR_TO_CFG_EN);
4763 link_cntl2 = RREG16(0x4088);
4764 link_cntl2 &= ~TARGET_LINK_SPEED_MASK;
4765 link_cntl2 |= 0x2;
4766 WREG16(0x4088, link_cntl2);
4767 WREG32(MM_CFGREGS_CNTL, 0);
4768
4769 if ((rdev->family == CHIP_RV670) ||
4770 (rdev->family == CHIP_RV620) ||
4771 (rdev->family == CHIP_RV635)) {
492d2b61 4772 training_cntl = RREG32_PCIE_PORT(PCIE_LC_TRAINING_CNTL);
9e46a48d 4773 training_cntl &= ~LC_POINT_7_PLUS_EN;
492d2b61 4774 WREG32_PCIE_PORT(PCIE_LC_TRAINING_CNTL, training_cntl);
9e46a48d 4775 } else {
492d2b61 4776 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9e46a48d 4777 speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN;
492d2b61 4778 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9e46a48d
AD		 4779 }
4780
492d2b61 4781 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9e46a48d 4782 speed_cntl |= LC_GEN2_EN_STRAP;
492d2b61 4783 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9e46a48d
AD		 4784
4785 } else {
492d2b61 4786 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9e46a48d
AD		 4787 /* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */
4788 if (1)
4789 link_width_cntl |= LC_UPCONFIGURE_DIS;
4790 else
4791 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
492d2b61 4792 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
9e46a48d
AD		 4793 }
4794}
6759a0a7
MO		 4795
4796/**
d0418894 4797 * r600_get_gpu_clock_counter - return GPU clock counter snapshot
6759a0a7
MO		 4798 *
4799 * @rdev: radeon_device pointer
4800 *
4801 * Fetches a GPU clock counter snapshot (R6xx-cayman).
4802 * Returns the 64 bit clock counter snapshot.
4803 */
d0418894 4804uint64_t r600_get_gpu_clock_counter(struct radeon_device *rdev)
6759a0a7
MO		 4805{
4806 uint64_t clock;
4807
4808 mutex_lock(&rdev->gpu_clock_mutex);
4809 WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
4810 clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
4811 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
4812 mutex_unlock(&rdev->gpu_clock_mutex);
4813 return clock;
4814}
kernel source for telekom puls (alcatel/mediatek)