2 * Copyright (C) 2012 Ben Skeggs.
5 * Permission is hereby granted, free of charge, to any person obtaining
6 * a copy of this software and associated documentation files (the
7 * "Software"), to deal in the Software without restriction, including
8 * without limitation the rights to use, copy, modify, merge, publish,
9 * distribute, sublicense, and/or sell copies of the Software, and to
10 * permit persons to whom the Software is furnished to do so, subject to
11 * the following conditions:
13 * The above copyright notice and this permission notice (including the
14 * next paragraph) shall be included in all copies or substantial
15 * portions of the Software.
17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
18 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
19 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
20 * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
21 * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
22 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
23 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
28 #include "nouveau_drv.h"
29 #include "nouveau_fifo.h"
30 #include "nouveau_util.h"
31 #include "nouveau_ramht.h"
32 #include "nouveau_software.h"
34 static struct ramfc_desc
{
41 { 32, 0, 0x00, 0, NV04_PFIFO_CACHE1_DMA_PUT
},
42 { 32, 0, 0x04, 0, NV04_PFIFO_CACHE1_DMA_GET
},
43 { 16, 0, 0x08, 0, NV04_PFIFO_CACHE1_DMA_INSTANCE
},
44 { 16, 16, 0x08, 0, NV04_PFIFO_CACHE1_DMA_DCOUNT
},
45 { 32, 0, 0x0c, 0, NV04_PFIFO_CACHE1_DMA_STATE
},
46 { 32, 0, 0x10, 0, NV04_PFIFO_CACHE1_DMA_FETCH
},
47 { 32, 0, 0x14, 0, NV04_PFIFO_CACHE1_ENGINE
},
48 { 32, 0, 0x18, 0, NV04_PFIFO_CACHE1_PULL1
},
52 struct nv04_fifo_priv
{
53 struct nouveau_fifo_priv base
;
54 struct ramfc_desc
*ramfc_desc
;
57 struct nv04_fifo_chan
{
58 struct nouveau_fifo_chan base
;
59 struct nouveau_gpuobj
*ramfc
;
63 nv04_fifo_cache_pull(struct drm_device
*dev
, bool enable
)
65 int pull
= nv_mask(dev
, NV04_PFIFO_CACHE1_PULL0
, 1, enable
);
68 /* In some cases the PFIFO puller may be left in an
69 * inconsistent state if you try to stop it when it's
70 * busy translating handles. Sometimes you get a
71 * PFIFO_CACHE_ERROR, sometimes it just fails silently
72 * sending incorrect instance offsets to PGRAPH after
73 * it's started up again. To avoid the latter we
74 * invalidate the most recently calculated instance.
76 if (!nv_wait(dev
, NV04_PFIFO_CACHE1_PULL0
,
77 NV04_PFIFO_CACHE1_PULL0_HASH_BUSY
, 0))
78 NV_ERROR(dev
, "Timeout idling the PFIFO puller.\n");
80 if (nv_rd32(dev
, NV04_PFIFO_CACHE1_PULL0
) &
81 NV04_PFIFO_CACHE1_PULL0_HASH_FAILED
)
82 nv_wr32(dev
, NV03_PFIFO_INTR_0
,
83 NV_PFIFO_INTR_CACHE_ERROR
);
85 nv_wr32(dev
, NV04_PFIFO_CACHE1_HASH
, 0);
92 nv04_fifo_context_new(struct nouveau_channel
*chan
, int engine
)
94 struct drm_device
*dev
= chan
->dev
;
95 struct drm_nouveau_private
*dev_priv
= dev
->dev_private
;
96 struct nv04_fifo_priv
*priv
= nv_engine(dev
, engine
);
97 struct nv04_fifo_chan
*fctx
;
101 fctx
= chan
->engctx
[engine
] = kzalloc(sizeof(*fctx
), GFP_KERNEL
);
105 /* map channel control registers */
106 chan
->user
= ioremap(pci_resource_start(dev
->pdev
, 0) +
107 NV03_USER(chan
->id
), PAGE_SIZE
);
113 /* initialise default fifo context */
114 ret
= nouveau_gpuobj_new_fake(dev
, dev_priv
->ramfc
->pinst
+
115 chan
->id
* 32, ~0, 32,
116 NVOBJ_FLAG_ZERO_FREE
, &fctx
->ramfc
);
120 nv_wo32(fctx
->ramfc
, 0x00, chan
->pushbuf_base
);
121 nv_wo32(fctx
->ramfc
, 0x04, chan
->pushbuf_base
);
122 nv_wo32(fctx
->ramfc
, 0x08, chan
->pushbuf
->pinst
>> 4);
123 nv_wo32(fctx
->ramfc
, 0x0c, 0x00000000);
124 nv_wo32(fctx
->ramfc
, 0x10, NV_PFIFO_CACHE1_DMA_FETCH_TRIG_128_BYTES
|
125 NV_PFIFO_CACHE1_DMA_FETCH_SIZE_128_BYTES
|
127 NV_PFIFO_CACHE1_BIG_ENDIAN
|
129 NV_PFIFO_CACHE1_DMA_FETCH_MAX_REQS_8
);
130 nv_wo32(fctx
->ramfc
, 0x14, 0x00000000);
131 nv_wo32(fctx
->ramfc
, 0x18, 0x00000000);
132 nv_wo32(fctx
->ramfc
, 0x1c, 0x00000000);
134 /* enable dma mode on the channel */
135 spin_lock_irqsave(&dev_priv
->context_switch_lock
, flags
);
136 nv_mask(dev
, NV04_PFIFO_MODE
, (1 << chan
->id
), (1 << chan
->id
));
137 spin_unlock_irqrestore(&dev_priv
->context_switch_lock
, flags
);
141 priv
->base
.base
.context_del(chan
, engine
);
146 nv04_fifo_context_del(struct nouveau_channel
*chan
, int engine
)
148 struct drm_device
*dev
= chan
->dev
;
149 struct drm_nouveau_private
*dev_priv
= dev
->dev_private
;
150 struct nv04_fifo_priv
*priv
= nv_engine(chan
->dev
, engine
);
151 struct nv04_fifo_chan
*fctx
= chan
->engctx
[engine
];
152 struct ramfc_desc
*c
= priv
->ramfc_desc
;
156 /* prevent fifo context switches */
157 spin_lock_irqsave(&dev_priv
->context_switch_lock
, flags
);
158 nv_wr32(dev
, NV03_PFIFO_CACHES
, 0);
160 /* if this channel is active, replace it with a null context */
161 chid
= nv_rd32(dev
, NV03_PFIFO_CACHE1_PUSH1
) & priv
->base
.channels
;
162 if (chid
== chan
->id
) {
163 nv_mask(dev
, NV04_PFIFO_CACHE1_DMA_PUSH
, 0x00000001, 0);
164 nv_wr32(dev
, NV03_PFIFO_CACHE1_PUSH0
, 0);
165 nv_mask(dev
, NV04_PFIFO_CACHE1_PULL0
, 0x00000001, 0);
168 u32 mask
= ((1ULL << c
->bits
) - 1) << c
->regs
;
169 nv_mask(dev
, c
->regp
, mask
, 0x00000000);
170 } while ((++c
)->bits
);
172 nv_wr32(dev
, NV03_PFIFO_CACHE1_GET
, 0);
173 nv_wr32(dev
, NV03_PFIFO_CACHE1_PUT
, 0);
174 nv_wr32(dev
, NV03_PFIFO_CACHE1_PUSH1
, priv
->base
.channels
);
175 nv_wr32(dev
, NV03_PFIFO_CACHE1_PUSH0
, 1);
176 nv_wr32(dev
, NV04_PFIFO_CACHE1_PULL0
, 1);
179 /* restore normal operation, after disabling dma mode */
180 nv_mask(dev
, NV04_PFIFO_MODE
, 1 << chan
->id
, 0);
181 nv_wr32(dev
, NV03_PFIFO_CACHES
, 1);
182 spin_unlock_irqrestore(&dev_priv
->context_switch_lock
, flags
);
185 nouveau_gpuobj_ref(NULL
, &fctx
->ramfc
);
186 nouveau_gpuobj_ref(NULL
, &chan
->ramfc
); /*XXX: nv40 */
194 nv04_fifo_init(struct drm_device
*dev
, int engine
)
196 struct drm_nouveau_private
*dev_priv
= dev
->dev_private
;
197 struct nv04_fifo_priv
*priv
= nv_engine(dev
, engine
);
200 nv_mask(dev
, NV03_PMC_ENABLE
, NV_PMC_ENABLE_PFIFO
, 0);
201 nv_mask(dev
, NV03_PMC_ENABLE
, NV_PMC_ENABLE_PFIFO
, NV_PMC_ENABLE_PFIFO
);
203 nv_wr32(dev
, NV04_PFIFO_DELAY_0
, 0x000000ff);
204 nv_wr32(dev
, NV04_PFIFO_DMA_TIMESLICE
, 0x0101ffff);
206 nv_wr32(dev
, NV03_PFIFO_RAMHT
, (0x03 << 24) /* search 128 */ |
207 ((dev_priv
->ramht
->bits
- 9) << 16) |
208 (dev_priv
->ramht
->gpuobj
->pinst
>> 8));
209 nv_wr32(dev
, NV03_PFIFO_RAMRO
, dev_priv
->ramro
->pinst
>> 8);
210 nv_wr32(dev
, NV03_PFIFO_RAMFC
, dev_priv
->ramfc
->pinst
>> 8);
212 nv_wr32(dev
, NV03_PFIFO_CACHE1_PUSH1
, priv
->base
.channels
);
214 nv_wr32(dev
, NV03_PFIFO_INTR_0
, 0xffffffff);
215 nv_wr32(dev
, NV03_PFIFO_INTR_EN_0
, 0xffffffff);
217 nv_wr32(dev
, NV03_PFIFO_CACHE1_PUSH0
, 1);
218 nv_wr32(dev
, NV04_PFIFO_CACHE1_PULL0
, 1);
219 nv_wr32(dev
, NV03_PFIFO_CACHES
, 1);
221 for (i
= 0; i
< priv
->base
.channels
; i
++) {
222 if (dev_priv
->channels
.ptr
[i
])
223 nv_mask(dev
, NV04_PFIFO_MODE
, (1 << i
), (1 << i
));
230 nv04_fifo_fini(struct drm_device
*dev
, int engine
, bool suspend
)
232 struct drm_nouveau_private
*dev_priv
= dev
->dev_private
;
233 struct nv04_fifo_priv
*priv
= nv_engine(dev
, engine
);
234 struct nouveau_channel
*chan
;
237 /* prevent context switches and halt fifo operation */
238 nv_wr32(dev
, NV03_PFIFO_CACHES
, 0);
239 nv_wr32(dev
, NV04_PFIFO_CACHE1_DMA_PUSH
, 0);
240 nv_wr32(dev
, NV03_PFIFO_CACHE1_PUSH0
, 0);
241 nv_wr32(dev
, NV04_PFIFO_CACHE1_PULL0
, 0);
243 /* store current fifo context in ramfc */
244 chid
= nv_rd32(dev
, NV03_PFIFO_CACHE1_PUSH1
) & priv
->base
.channels
;
245 chan
= dev_priv
->channels
.ptr
[chid
];
246 if (suspend
&& chid
!= priv
->base
.channels
&& chan
) {
247 struct nv04_fifo_chan
*fctx
= chan
->engctx
[engine
];
248 struct nouveau_gpuobj
*ctx
= fctx
->ramfc
;
249 struct ramfc_desc
*c
= priv
->ramfc_desc
;
251 u32 rm
= ((1ULL << c
->bits
) - 1) << c
->regs
;
252 u32 cm
= ((1ULL << c
->bits
) - 1) << c
->ctxs
;
253 u32 rv
= (nv_rd32(dev
, c
->regp
) & rm
) >> c
->regs
;
254 u32 cv
= (nv_ro32(ctx
, c
->ctxp
) & ~cm
);
255 nv_wo32(ctx
, c
->ctxp
, cv
| (rv
<< c
->ctxs
));
256 } while ((++c
)->bits
);
259 nv_wr32(dev
, NV03_PFIFO_INTR_EN_0
, 0x00000000);
264 nouveau_fifo_swmthd(struct drm_device
*dev
, u32 chid
, u32 addr
, u32 data
)
266 struct nouveau_fifo_priv
*pfifo
= nv_engine(dev
, NVOBJ_ENGINE_FIFO
);
267 struct drm_nouveau_private
*dev_priv
= dev
->dev_private
;
268 struct nouveau_channel
*chan
= NULL
;
269 struct nouveau_gpuobj
*obj
;
271 const int subc
= (addr
>> 13) & 0x7;
272 const int mthd
= addr
& 0x1ffc;
273 bool handled
= false;
276 spin_lock_irqsave(&dev_priv
->channels
.lock
, flags
);
277 if (likely(chid
>= 0 && chid
< pfifo
->channels
))
278 chan
= dev_priv
->channels
.ptr
[chid
];
283 case 0x0000: /* bind object to subchannel */
284 obj
= nouveau_ramht_find(chan
, data
);
285 if (unlikely(!obj
|| obj
->engine
!= NVOBJ_ENGINE_SW
))
288 engine
= 0x0000000f << (subc
* 4);
290 nv_mask(dev
, NV04_PFIFO_CACHE1_ENGINE
, engine
, 0x00000000);
294 engine
= nv_rd32(dev
, NV04_PFIFO_CACHE1_ENGINE
);
295 if (unlikely(((engine
>> (subc
* 4)) & 0xf) != 0))
298 if (!nouveau_gpuobj_mthd_call(chan
, nouveau_software_class(dev
),
305 spin_unlock_irqrestore(&dev_priv
->channels
.lock
, flags
);
309 static const char *nv_dma_state_err(u32 state
)
311 static const char * const desc
[] = {
312 "NONE", "CALL_SUBR_ACTIVE", "INVALID_MTHD", "RET_SUBR_INACTIVE",
313 "INVALID_CMD", "IB_EMPTY"/* NV50+ */, "MEM_FAULT", "UNK"
315 return desc
[(state
>> 29) & 0x7];
319 nv04_fifo_isr(struct drm_device
*dev
)
321 struct nouveau_fifo_priv
*pfifo
= nv_engine(dev
, NVOBJ_ENGINE_FIFO
);
322 struct drm_nouveau_private
*dev_priv
= dev
->dev_private
;
323 uint32_t status
, reassign
;
326 reassign
= nv_rd32(dev
, NV03_PFIFO_CACHES
) & 1;
327 while ((status
= nv_rd32(dev
, NV03_PFIFO_INTR_0
)) && (cnt
++ < 100)) {
330 nv_wr32(dev
, NV03_PFIFO_CACHES
, 0);
332 chid
= nv_rd32(dev
, NV03_PFIFO_CACHE1_PUSH1
) & pfifo
->channels
;
333 get
= nv_rd32(dev
, NV03_PFIFO_CACHE1_GET
);
335 if (status
& NV_PFIFO_INTR_CACHE_ERROR
) {
339 /* NV_PFIFO_CACHE1_GET actually goes to 0xffc before
340 * wrapping on my G80 chips, but CACHE1 isn't big
341 * enough for this much data.. Tests show that it
342 * wraps around to the start at GET=0x800.. No clue
345 ptr
= (get
& 0x7ff) >> 2;
347 if (dev_priv
->card_type
< NV_40
) {
349 NV04_PFIFO_CACHE1_METHOD(ptr
));
351 NV04_PFIFO_CACHE1_DATA(ptr
));
354 NV40_PFIFO_CACHE1_METHOD(ptr
));
356 NV40_PFIFO_CACHE1_DATA(ptr
));
359 if (!nouveau_fifo_swmthd(dev
, chid
, mthd
, data
)) {
360 NV_INFO(dev
, "PFIFO_CACHE_ERROR - Ch %d/%d "
361 "Mthd 0x%04x Data 0x%08x\n",
362 chid
, (mthd
>> 13) & 7, mthd
& 0x1ffc,
366 nv_wr32(dev
, NV04_PFIFO_CACHE1_DMA_PUSH
, 0);
367 nv_wr32(dev
, NV03_PFIFO_INTR_0
,
368 NV_PFIFO_INTR_CACHE_ERROR
);
370 nv_wr32(dev
, NV03_PFIFO_CACHE1_PUSH0
,
371 nv_rd32(dev
, NV03_PFIFO_CACHE1_PUSH0
) & ~1);
372 nv_wr32(dev
, NV03_PFIFO_CACHE1_GET
, get
+ 4);
373 nv_wr32(dev
, NV03_PFIFO_CACHE1_PUSH0
,
374 nv_rd32(dev
, NV03_PFIFO_CACHE1_PUSH0
) | 1);
375 nv_wr32(dev
, NV04_PFIFO_CACHE1_HASH
, 0);
377 nv_wr32(dev
, NV04_PFIFO_CACHE1_DMA_PUSH
,
378 nv_rd32(dev
, NV04_PFIFO_CACHE1_DMA_PUSH
) | 1);
379 nv_wr32(dev
, NV04_PFIFO_CACHE1_PULL0
, 1);
381 status
&= ~NV_PFIFO_INTR_CACHE_ERROR
;
384 if (status
& NV_PFIFO_INTR_DMA_PUSHER
) {
385 u32 dma_get
= nv_rd32(dev
, 0x003244);
386 u32 dma_put
= nv_rd32(dev
, 0x003240);
387 u32 push
= nv_rd32(dev
, 0x003220);
388 u32 state
= nv_rd32(dev
, 0x003228);
390 if (dev_priv
->card_type
== NV_50
) {
391 u32 ho_get
= nv_rd32(dev
, 0x003328);
392 u32 ho_put
= nv_rd32(dev
, 0x003320);
393 u32 ib_get
= nv_rd32(dev
, 0x003334);
394 u32 ib_put
= nv_rd32(dev
, 0x003330);
396 if (nouveau_ratelimit())
397 NV_INFO(dev
, "PFIFO_DMA_PUSHER - Ch %d Get 0x%02x%08x "
398 "Put 0x%02x%08x IbGet 0x%08x IbPut 0x%08x "
399 "State 0x%08x (err: %s) Push 0x%08x\n",
400 chid
, ho_get
, dma_get
, ho_put
,
401 dma_put
, ib_get
, ib_put
, state
,
402 nv_dma_state_err(state
),
405 /* METHOD_COUNT, in DMA_STATE on earlier chipsets */
406 nv_wr32(dev
, 0x003364, 0x00000000);
407 if (dma_get
!= dma_put
|| ho_get
!= ho_put
) {
408 nv_wr32(dev
, 0x003244, dma_put
);
409 nv_wr32(dev
, 0x003328, ho_put
);
411 if (ib_get
!= ib_put
) {
412 nv_wr32(dev
, 0x003334, ib_put
);
415 NV_INFO(dev
, "PFIFO_DMA_PUSHER - Ch %d Get 0x%08x "
416 "Put 0x%08x State 0x%08x (err: %s) Push 0x%08x\n",
417 chid
, dma_get
, dma_put
, state
,
418 nv_dma_state_err(state
), push
);
420 if (dma_get
!= dma_put
)
421 nv_wr32(dev
, 0x003244, dma_put
);
424 nv_wr32(dev
, 0x003228, 0x00000000);
425 nv_wr32(dev
, 0x003220, 0x00000001);
426 nv_wr32(dev
, 0x002100, NV_PFIFO_INTR_DMA_PUSHER
);
427 status
&= ~NV_PFIFO_INTR_DMA_PUSHER
;
430 if (status
& NV_PFIFO_INTR_SEMAPHORE
) {
433 status
&= ~NV_PFIFO_INTR_SEMAPHORE
;
434 nv_wr32(dev
, NV03_PFIFO_INTR_0
,
435 NV_PFIFO_INTR_SEMAPHORE
);
437 sem
= nv_rd32(dev
, NV10_PFIFO_CACHE1_SEMAPHORE
);
438 nv_wr32(dev
, NV10_PFIFO_CACHE1_SEMAPHORE
, sem
| 0x1);
440 nv_wr32(dev
, NV03_PFIFO_CACHE1_GET
, get
+ 4);
441 nv_wr32(dev
, NV04_PFIFO_CACHE1_PULL0
, 1);
444 if (dev_priv
->card_type
== NV_50
) {
445 if (status
& 0x00000010) {
446 nv50_fb_vm_trap(dev
, nouveau_ratelimit());
447 status
&= ~0x00000010;
448 nv_wr32(dev
, 0x002100, 0x00000010);
453 if (nouveau_ratelimit())
454 NV_INFO(dev
, "PFIFO_INTR 0x%08x - Ch %d\n",
456 nv_wr32(dev
, NV03_PFIFO_INTR_0
, status
);
460 nv_wr32(dev
, NV03_PFIFO_CACHES
, reassign
);
464 NV_INFO(dev
, "PFIFO still angry after %d spins, halt\n", cnt
);
465 nv_wr32(dev
, 0x2140, 0);
466 nv_wr32(dev
, 0x140, 0);
469 nv_wr32(dev
, NV03_PMC_INTR_0
, NV_PMC_INTR_0_PFIFO_PENDING
);
473 nv04_fifo_destroy(struct drm_device
*dev
, int engine
)
475 struct drm_nouveau_private
*dev_priv
= dev
->dev_private
;
476 struct nv04_fifo_priv
*priv
= nv_engine(dev
, engine
);
478 nouveau_irq_unregister(dev
, 8);
480 dev_priv
->eng
[engine
] = NULL
;
485 nv04_fifo_create(struct drm_device
*dev
)
487 struct drm_nouveau_private
*dev_priv
= dev
->dev_private
;
488 struct nv04_fifo_priv
*priv
;
490 priv
= kzalloc(sizeof(*priv
), GFP_KERNEL
);
494 priv
->base
.base
.destroy
= nv04_fifo_destroy
;
495 priv
->base
.base
.init
= nv04_fifo_init
;
496 priv
->base
.base
.fini
= nv04_fifo_fini
;
497 priv
->base
.base
.context_new
= nv04_fifo_context_new
;
498 priv
->base
.base
.context_del
= nv04_fifo_context_del
;
499 priv
->base
.channels
= 15;
500 priv
->ramfc_desc
= nv04_ramfc
;
501 dev_priv
->eng
[NVOBJ_ENGINE_FIFO
] = &priv
->base
.base
;
503 nouveau_irq_register(dev
, 8, nv04_fifo_isr
);