projects / GitHub / mt8127 / android_kernel_alcatel_ttab.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge tag 'v3.10.107' into update
[GitHub/mt8127/android_kernel_alcatel_ttab.git] / drivers / usb / dwc3 / gadget.c
CommitLineData
72246da4
FB		 1/**
2 * gadget.c - DesignWare USB3 DRD Controller Gadget Framework Link
3 *
4 * Copyright (C) 2010-2011 Texas Instruments Incorporated - http://www.ti.com
72246da4
FB		 5 *
6 * Authors: Felipe Balbi <balbi@ti.com>,
7 * Sebastian Andrzej Siewior <bigeasy@linutronix.de>
8 *
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
11 * are met:
12 * 1. Redistributions of source code must retain the above copyright
13 * notice, this list of conditions, and the following disclaimer,
14 * without modification.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 * 3. The names of the above-listed copyright holders may not be used
19 * to endorse or promote products derived from this software without
20 * specific prior written permission.
21 *
22 * ALTERNATIVELY, this software may be distributed under the terms of the
23 * GNU General Public License ("GPL") version 2, as published by the Free
24 * Software Foundation.
25 *
26 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
27 * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
28 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
29 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
30 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
31 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
32 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
33 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
34 * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
35 * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
36 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
37 */
38
39#include <linux/kernel.h>
40#include <linux/delay.h>
41#include <linux/slab.h>
42#include <linux/spinlock.h>
43#include <linux/platform_device.h>
44#include <linux/pm_runtime.h>
45#include <linux/interrupt.h>
46#include <linux/io.h>
47#include <linux/list.h>
48#include <linux/dma-mapping.h>
49
50#include <linux/usb/ch9.h>
51#include <linux/usb/gadget.h>
52
53#include "core.h"
54#include "gadget.h"
55#include "io.h"
56
04a9bfcd
FB		 57/**
58 * dwc3_gadget_set_test_mode - Enables USB2 Test Modes
59 * @dwc: pointer to our context structure
60 * @mode: the mode to set (J, K SE0 NAK, Force Enable)
61 *
62 * Caller should take care of locking. This function will
63 * return 0 on success or -EINVAL if wrong Test Selector
64 * is passed
65 */
66int dwc3_gadget_set_test_mode(struct dwc3 *dwc, int mode)
67{
68 u32 reg;
69
70 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
71 reg &= ~DWC3_DCTL_TSTCTRL_MASK;
72
73 switch (mode) {
74 case TEST_J:
75 case TEST_K:
76 case TEST_SE0_NAK:
77 case TEST_PACKET:
78 case TEST_FORCE_EN:
79 reg |= mode << 1;
80 break;
81 default:
82 return -EINVAL;
83 }
84
85 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
86
87 return 0;
88}
89
8598bde7
FB		 90/**
91 * dwc3_gadget_set_link_state - Sets USB Link to a particular State
92 * @dwc: pointer to our context structure
93 * @state: the state to put link into
94 *
95 * Caller should take care of locking. This function will
aee63e3c 96 * return 0 on success or -ETIMEDOUT.
8598bde7
FB		 97 */
98int dwc3_gadget_set_link_state(struct dwc3 *dwc, enum dwc3_link_state state)
99{
aee63e3c 100 int retries = 10000;
8598bde7
FB		 101 u32 reg;
102
802fde98
PZ		 103 /*
104 * Wait until device controller is ready. Only applies to 1.94a and
105 * later RTL.
106 */
107 if (dwc->revision >= DWC3_REVISION_194A) {
108 while (--retries) {
109 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
110 if (reg & DWC3_DSTS_DCNRD)
111 udelay(5);
112 else
113 break;
114 }
115
116 if (retries <= 0)
117 return -ETIMEDOUT;
118 }
119
8598bde7
FB		 120 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
121 reg &= ~DWC3_DCTL_ULSTCHNGREQ_MASK;
122
123 /* set requested state */
124 reg |= DWC3_DCTL_ULSTCHNGREQ(state);
125 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
126
802fde98
PZ		 127 /*
128 * The following code is racy when called from dwc3_gadget_wakeup,
129 * and is not needed, at least on newer versions
130 */
131 if (dwc->revision >= DWC3_REVISION_194A)
132 return 0;
133
8598bde7 134 /* wait for a change in DSTS */
aed430e5 135 retries = 10000;
8598bde7
FB		 136 while (--retries) {
137 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
138
8598bde7
FB		 139 if (DWC3_DSTS_USBLNKST(reg) == state)
140 return 0;
141
aee63e3c 142 udelay(5);
8598bde7
FB		 143 }
144
145 dev_vdbg(dwc->dev, "link state change request timed out\n");
146
147 return -ETIMEDOUT;
148}
149
457e84b6
FB		 150/**
151 * dwc3_gadget_resize_tx_fifos - reallocate fifo spaces for current use-case
152 * @dwc: pointer to our context structure
153 *
154 * This function will a best effort FIFO allocation in order
155 * to improve FIFO usage and throughput, while still allowing
156 * us to enable as many endpoints as possible.
157 *
158 * Keep in mind that this operation will be highly dependent
159 * on the configured size for RAM1 - which contains TxFifo -,
160 * the amount of endpoints enabled on coreConsultant tool, and
161 * the width of the Master Bus.
162 *
163 * In the ideal world, we would always be able to satisfy the
164 * following equation:
165 *
166 * ((512 + 2 * MDWIDTH-Bytes) + (Number of IN Endpoints - 1) * \
167 * (3 * (1024 + MDWIDTH-Bytes) + MDWIDTH-Bytes)) / MDWIDTH-Bytes
168 *
169 * Unfortunately, due to many variables that's not always the case.
170 */
171int dwc3_gadget_resize_tx_fifos(struct dwc3 *dwc)
172{
173 int last_fifo_depth = 0;
174 int ram1_depth;
175 int fifo_size;
176 int mdwidth;
177 int num;
178
179 if (!dwc->needs_fifo_resize)
180 return 0;
181
182 ram1_depth = DWC3_RAM1_DEPTH(dwc->hwparams.hwparams7);
183 mdwidth = DWC3_MDWIDTH(dwc->hwparams.hwparams0);
184
185 /* MDWIDTH is represented in bits, we need it in bytes */
186 mdwidth >>= 3;
187
188 /*
189 * FIXME For now we will only allocate 1 wMaxPacketSize space
190 * for each enabled endpoint, later patches will come to
191 * improve this algorithm so that we better use the internal
192 * FIFO space
193 */
194 for (num = 0; num < DWC3_ENDPOINTS_NUM; num++) {
195 struct dwc3_ep *dep = dwc->eps[num];
196 int fifo_number = dep->number >> 1;
2e81c36a 197 int mult = 1;
457e84b6
FB		 198 int tmp;
199
200 if (!(dep->number & 1))
201 continue;
202
203 if (!(dep->flags & DWC3_EP_ENABLED))
204 continue;
205
16e78db7
IS		 206 if (usb_endpoint_xfer_bulk(dep->endpoint.desc)
207 || usb_endpoint_xfer_isoc(dep->endpoint.desc))
2e81c36a
FB		 208 mult = 3;
209
210 /*
211 * REVISIT: the following assumes we will always have enough
212 * space available on the FIFO RAM for all possible use cases.
213 * Make sure that's true somehow and change FIFO allocation
214 * accordingly.
215 *
216 * If we have Bulk or Isochronous endpoints, we want
217 * them to be able to be very, very fast. So we're giving
218 * those endpoints a fifo_size which is enough for 3 full
219 * packets
220 */
221 tmp = mult * (dep->endpoint.maxpacket + mdwidth);
457e84b6
FB		 222 tmp += mdwidth;
223
224 fifo_size = DIV_ROUND_UP(tmp, mdwidth);
2e81c36a 225
457e84b6
FB		 226 fifo_size |= (last_fifo_depth << 16);
227
228 dev_vdbg(dwc->dev, "%s: Fifo Addr %04x Size %d\n",
229 dep->name, last_fifo_depth, fifo_size & 0xffff);
230
231 dwc3_writel(dwc->regs, DWC3_GTXFIFOSIZ(fifo_number),
232 fifo_size);
233
234 last_fifo_depth += (fifo_size & 0xffff);
235 }
236
237 return 0;
238}
239
72246da4
FB		 240void dwc3_gadget_giveback(struct dwc3_ep *dep, struct dwc3_request *req,
241 int status)
242{
243 struct dwc3 *dwc = dep->dwc;
91c86d70 244 unsigned int unmap_after_complete = false;
e5ba5ec8 245 int i;
72246da4
FB		 246
247 if (req->queued) {
e5ba5ec8
PA		 248 i = 0;
249 do {
eeb720fb 250 dep->busy_slot++;
e5ba5ec8
PA		 251 /*
252 * Skip LINK TRB. We can't use req->trb and check for
253 * DWC3_TRBCTL_LINK_TRB because it points the TRB we
254 * just completed (not the LINK TRB).
255 */
256 if (((dep->busy_slot & DWC3_TRB_MASK) ==
257 DWC3_TRB_NUM- 1) &&
16e78db7 258 usb_endpoint_xfer_isoc(dep->endpoint.desc))
e5ba5ec8
PA		 259 dep->busy_slot++;
260 } while(++i < req->request.num_mapped_sgs);
c9fda7d6 261 req->queued = false;
72246da4
FB		 262 }
263 list_del(&req->list);
eeb720fb 264 req->trb = NULL;
72246da4
FB		 265
266 if (req->request.status == -EINPROGRESS)
267 req->request.status = status;
268
91c86d70
JD		 269 /*
270 * NOTICE we don't want to unmap before calling ->complete() if we're
271 * dealing with a bounced ep0 request. If we unmap it here, we would end
272 * up overwritting the contents of req->buf and this could confuse the
273 * gadget driver.
274 */
275 if (dwc->ep0_bounced && dep->number <= 1) {
0416e494 276 dwc->ep0_bounced = false;
91c86d70
JD		 277 unmap_after_complete = true;
278 } else {
279 usb_gadget_unmap_request(&dwc->gadget,
280 &req->request, req->direction);
281 }
72246da4
FB		 282
283 dev_dbg(dwc->dev, "request %p from %s completed %d/%d ===> %d\n",
284 req, dep->name, req->request.actual,
285 req->request.length, status);
286
287 spin_unlock(&dwc->lock);
0fc9a1be 288 req->request.complete(&dep->endpoint, &req->request);
72246da4 289 spin_lock(&dwc->lock);
91c86d70
JD		 290
291 if (unmap_after_complete)
292 usb_gadget_unmap_request(&dwc->gadget,
293 &req->request, req->direction);
72246da4
FB		 294}
295
296static const char *dwc3_gadget_ep_cmd_string(u8 cmd)
297{
298 switch (cmd) {
299 case DWC3_DEPCMD_DEPSTARTCFG:
300 return "Start New Configuration";
301 case DWC3_DEPCMD_ENDTRANSFER:
302 return "End Transfer";
303 case DWC3_DEPCMD_UPDATETRANSFER:
304 return "Update Transfer";
305 case DWC3_DEPCMD_STARTTRANSFER:
306 return "Start Transfer";
307 case DWC3_DEPCMD_CLEARSTALL:
308 return "Clear Stall";
309 case DWC3_DEPCMD_SETSTALL:
310 return "Set Stall";
802fde98
PZ		 311 case DWC3_DEPCMD_GETEPSTATE:
312 return "Get Endpoint State";
72246da4
FB		 313 case DWC3_DEPCMD_SETTRANSFRESOURCE:
314 return "Set Endpoint Transfer Resource";
315 case DWC3_DEPCMD_SETEPCONFIG:
316 return "Set Endpoint Configuration";
317 default:
318 return "UNKNOWN command";
319 }
320}
321
b09bb642
FB		 322int dwc3_send_gadget_generic_command(struct dwc3 *dwc, int cmd, u32 param)
323{
324 u32 timeout = 500;
325 u32 reg;
326
327 dwc3_writel(dwc->regs, DWC3_DGCMDPAR, param);
328 dwc3_writel(dwc->regs, DWC3_DGCMD, cmd | DWC3_DGCMD_CMDACT);
329
330 do {
331 reg = dwc3_readl(dwc->regs, DWC3_DGCMD);
332 if (!(reg & DWC3_DGCMD_CMDACT)) {
333 dev_vdbg(dwc->dev, "Command Complete --> %d\n",
334 DWC3_DGCMD_STATUS(reg));
265f7f39
SSB		 335 if (DWC3_DGCMD_STATUS(reg))
336 return -EINVAL;
b09bb642
FB		 337 return 0;
338 }
339
340 /*
341 * We can't sleep here, because it's also called from
342 * interrupt context.
343 */
344 timeout--;
345 if (!timeout)
346 return -ETIMEDOUT;
347 udelay(1);
348 } while (1);
349}
350
72246da4
FB		 351int dwc3_send_gadget_ep_cmd(struct dwc3 *dwc, unsigned ep,
352 unsigned cmd, struct dwc3_gadget_ep_cmd_params *params)
353{
354 struct dwc3_ep *dep = dwc->eps[ep];
61d58242 355 u32 timeout = 500;
72246da4
FB		 356 u32 reg;
357
358 dev_vdbg(dwc->dev, "%s: cmd '%s' params %08x %08x %08x\n",
359 dep->name,
dc1c70a7
FB		 360 dwc3_gadget_ep_cmd_string(cmd), params->param0,
361 params->param1, params->param2);
72246da4 362
dc1c70a7
FB		 363 dwc3_writel(dwc->regs, DWC3_DEPCMDPAR0(ep), params->param0);
364 dwc3_writel(dwc->regs, DWC3_DEPCMDPAR1(ep), params->param1);
365 dwc3_writel(dwc->regs, DWC3_DEPCMDPAR2(ep), params->param2);
72246da4
FB		 366
367 dwc3_writel(dwc->regs, DWC3_DEPCMD(ep), cmd | DWC3_DEPCMD_CMDACT);
368 do {
369 reg = dwc3_readl(dwc->regs, DWC3_DEPCMD(ep));
370 if (!(reg & DWC3_DEPCMD_CMDACT)) {
164f6e14
FB		 371 dev_vdbg(dwc->dev, "Command Complete --> %d\n",
372 DWC3_DEPCMD_STATUS(reg));
de03d218
SSB		 373 if (DWC3_DEPCMD_STATUS(reg))
374 return -EINVAL;
72246da4
FB		 375 return 0;
376 }
377
378 /*
72246da4
FB		 379 * We can't sleep here, because it is also called from
380 * interrupt context.
381 */
382 timeout--;
383 if (!timeout)
384 return -ETIMEDOUT;
385
61d58242 386 udelay(1);
72246da4
FB		 387 } while (1);
388}
389
390static dma_addr_t dwc3_trb_dma_offset(struct dwc3_ep *dep,
f6bafc6a 391 struct dwc3_trb *trb)
72246da4 392{
c439ef87 393 u32 offset = (char *) trb - (char *) dep->trb_pool;
72246da4
FB		 394
395 return dep->trb_pool_dma + offset;
396}
397
398static int dwc3_alloc_trb_pool(struct dwc3_ep *dep)
399{
400 struct dwc3 *dwc = dep->dwc;
401
402 if (dep->trb_pool)
403 return 0;
404
405 if (dep->number == 0 || dep->number == 1)
406 return 0;
407
408 dep->trb_pool = dma_alloc_coherent(dwc->dev,
409 sizeof(struct dwc3_trb) * DWC3_TRB_NUM,
410 &dep->trb_pool_dma, GFP_KERNEL);
411 if (!dep->trb_pool) {
412 dev_err(dep->dwc->dev, "failed to allocate trb pool for %s\n",
413 dep->name);
414 return -ENOMEM;
415 }
416
417 return 0;
418}
419
420static void dwc3_free_trb_pool(struct dwc3_ep *dep)
421{
422 struct dwc3 *dwc = dep->dwc;
423
424 dma_free_coherent(dwc->dev, sizeof(struct dwc3_trb) * DWC3_TRB_NUM,
425 dep->trb_pool, dep->trb_pool_dma);
426
427 dep->trb_pool = NULL;
428 dep->trb_pool_dma = 0;
429}
430
431static int dwc3_gadget_start_config(struct dwc3 *dwc, struct dwc3_ep *dep)
432{
433 struct dwc3_gadget_ep_cmd_params params;
434 u32 cmd;
435
436 memset(&params, 0x00, sizeof(params));
437
438 if (dep->number != 1) {
439 cmd = DWC3_DEPCMD_DEPSTARTCFG;
440 /* XferRscIdx == 0 for ep0 and 2 for the remaining */
b23c8439
PZ		 441 if (dep->number > 1) {
442 if (dwc->start_config_issued)
443 return 0;
444 dwc->start_config_issued = true;
72246da4 445 cmd |= DWC3_DEPCMD_PARAM(2);
b23c8439 446 }
72246da4
FB		 447
448 return dwc3_send_gadget_ep_cmd(dwc, 0, cmd, &params);
449 }
450
451 return 0;
452}
453
454static int dwc3_gadget_set_ep_config(struct dwc3 *dwc, struct dwc3_ep *dep,
c90bfaec 455 const struct usb_endpoint_descriptor *desc,
4b345c9a
FB		 456 const struct usb_ss_ep_comp_descriptor *comp_desc,
457 bool ignore)
72246da4
FB		 458{
459 struct dwc3_gadget_ep_cmd_params params;
460
461 memset(&params, 0x00, sizeof(params));
462
dc1c70a7 463 params.param0 = DWC3_DEPCFG_EP_TYPE(usb_endpoint_type(desc))
d2e9a13a
CP		 464 | DWC3_DEPCFG_MAX_PACKET_SIZE(usb_endpoint_maxp(desc));
465
466 /* Burst size is only needed in SuperSpeed mode */
467 if (dwc->gadget.speed == USB_SPEED_SUPER) {
468 u32 burst = dep->endpoint.maxburst - 1;
469
470 params.param0 |= DWC3_DEPCFG_BURST_SIZE(burst);
471 }
72246da4 472
4b345c9a
FB		 473 if (ignore)
474 params.param0 |= DWC3_DEPCFG_IGN_SEQ_NUM;
475
dc1c70a7
FB		 476 params.param1 = DWC3_DEPCFG_XFER_COMPLETE_EN
477 | DWC3_DEPCFG_XFER_NOT_READY_EN;
72246da4 478
18b7ede5 479 if (usb_ss_max_streams(comp_desc) && usb_endpoint_xfer_bulk(desc)) {
dc1c70a7
FB		 480 params.param1 |= DWC3_DEPCFG_STREAM_CAPABLE
481 | DWC3_DEPCFG_STREAM_EVENT_EN;
879631aa
FB		 482 dep->stream_capable = true;
483 }
484
72246da4 485 if (usb_endpoint_xfer_isoc(desc))
dc1c70a7 486 params.param1 |= DWC3_DEPCFG_XFER_IN_PROGRESS_EN;
72246da4
FB		 487
488 /*
489 * We are doing 1:1 mapping for endpoints, meaning
490 * Physical Endpoints 2 maps to Logical Endpoint 2 and
491 * so on. We consider the direction bit as part of the physical
492 * endpoint number. So USB endpoint 0x81 is 0x03.
493 */
dc1c70a7 494 params.param1 |= DWC3_DEPCFG_EP_NUMBER(dep->number);
72246da4
FB		 495
496 /*
497 * We must use the lower 16 TX FIFOs even though
498 * HW might have more
499 */
500 if (dep->direction)
dc1c70a7 501 params.param0 |= DWC3_DEPCFG_FIFO_NUMBER(dep->number >> 1);
72246da4
FB		 502
503 if (desc->bInterval) {
dc1c70a7 504 params.param1 |= DWC3_DEPCFG_BINTERVAL_M1(desc->bInterval - 1);
72246da4
FB		 505 dep->interval = 1 << (desc->bInterval - 1);
506 }
507
508 return dwc3_send_gadget_ep_cmd(dwc, dep->number,
509 DWC3_DEPCMD_SETEPCONFIG, &params);
510}
511
512static int dwc3_gadget_set_xfer_resource(struct dwc3 *dwc, struct dwc3_ep *dep)
513{
514 struct dwc3_gadget_ep_cmd_params params;
515
516 memset(&params, 0x00, sizeof(params));
517
dc1c70a7 518 params.param0 = DWC3_DEPXFERCFG_NUM_XFER_RES(1);
72246da4
FB		 519
520 return dwc3_send_gadget_ep_cmd(dwc, dep->number,
521 DWC3_DEPCMD_SETTRANSFRESOURCE, &params);
522}
523
524/**
525 * __dwc3_gadget_ep_enable - Initializes a HW endpoint
526 * @dep: endpoint to be initialized
527 * @desc: USB Endpoint Descriptor
528 *
529 * Caller should take care of locking
530 */
531static int __dwc3_gadget_ep_enable(struct dwc3_ep *dep,
c90bfaec 532 const struct usb_endpoint_descriptor *desc,
4b345c9a
FB		 533 const struct usb_ss_ep_comp_descriptor *comp_desc,
534 bool ignore)
72246da4
FB		 535{
536 struct dwc3 *dwc = dep->dwc;
537 u32 reg;
538 int ret = -ENOMEM;
539
540 if (!(dep->flags & DWC3_EP_ENABLED)) {
541 ret = dwc3_gadget_start_config(dwc, dep);
542 if (ret)
543 return ret;
544 }
545
4b345c9a 546 ret = dwc3_gadget_set_ep_config(dwc, dep, desc, comp_desc, ignore);
72246da4
FB		 547 if (ret)
548 return ret;
549
550 if (!(dep->flags & DWC3_EP_ENABLED)) {
f6bafc6a
FB		 551 struct dwc3_trb *trb_st_hw;
552 struct dwc3_trb *trb_link;
72246da4
FB		 553
554 ret = dwc3_gadget_set_xfer_resource(dwc, dep);
555 if (ret)
556 return ret;
557
16e78db7 558 dep->endpoint.desc = desc;
c90bfaec 559 dep->comp_desc = comp_desc;
72246da4
FB		 560 dep->type = usb_endpoint_type(desc);
561 dep->flags |= DWC3_EP_ENABLED;
562
563 reg = dwc3_readl(dwc->regs, DWC3_DALEPENA);
564 reg |= DWC3_DALEPENA_EP(dep->number);
565 dwc3_writel(dwc->regs, DWC3_DALEPENA, reg);
566
567 if (!usb_endpoint_xfer_isoc(desc))
568 return 0;
569
1d046793 570 /* Link TRB for ISOC. The HWO bit is never reset */
72246da4
FB		 571 trb_st_hw = &dep->trb_pool[0];
572
f6bafc6a 573 trb_link = &dep->trb_pool[DWC3_TRB_NUM - 1];
6b93e366 574 memset(trb_link, 0, sizeof(*trb_link));
72246da4 575
f6bafc6a
FB		 576 trb_link->bpl = lower_32_bits(dwc3_trb_dma_offset(dep, trb_st_hw));
577 trb_link->bph = upper_32_bits(dwc3_trb_dma_offset(dep, trb_st_hw));
578 trb_link->ctrl |= DWC3_TRBCTL_LINK_TRB;
579 trb_link->ctrl |= DWC3_TRB_CTRL_HWO;
72246da4
FB		 580 }
581
582 return 0;
583}
584
624407f9
SAS		 585static void dwc3_stop_active_transfer(struct dwc3 *dwc, u32 epnum);
586static void dwc3_remove_requests(struct dwc3 *dwc, struct dwc3_ep *dep)
72246da4
FB		 587{
588 struct dwc3_request *req;
589
ea53b882 590 if (!list_empty(&dep->req_queued)) {
624407f9
SAS		 591 dwc3_stop_active_transfer(dwc, dep->number);
592
57911504 593 /* - giveback all requests to gadget driver */
1591633e
PA		 594 while (!list_empty(&dep->req_queued)) {
595 req = next_request(&dep->req_queued);
596
597 dwc3_gadget_giveback(dep, req, -ESHUTDOWN);
598 }
ea53b882
FB		 599 }
600
72246da4
FB		 601 while (!list_empty(&dep->request_list)) {
602 req = next_request(&dep->request_list);
603
624407f9 604 dwc3_gadget_giveback(dep, req, -ESHUTDOWN);
72246da4 605 }
72246da4
FB		 606}
607
608/**
609 * __dwc3_gadget_ep_disable - Disables a HW endpoint
610 * @dep: the endpoint to disable
611 *
624407f9
SAS		 612 * This function also removes requests which are currently processed ny the
613 * hardware and those which are not yet scheduled.
614 * Caller should take care of locking.
72246da4 615 */
72246da4
FB		 616static int __dwc3_gadget_ep_disable(struct dwc3_ep *dep)
617{
618 struct dwc3 *dwc = dep->dwc;
619 u32 reg;
620
624407f9 621 dwc3_remove_requests(dwc, dep);
72246da4 622
c3ea18ca
FB		 623 /* make sure HW endpoint isn't stalled */
624 if (dep->flags & DWC3_EP_STALL)
7d513758 625 __dwc3_gadget_ep_set_halt(dep, 0, false);
c3ea18ca 626
72246da4
FB		 627 reg = dwc3_readl(dwc->regs, DWC3_DALEPENA);
628 reg &= ~DWC3_DALEPENA_EP(dep->number);
629 dwc3_writel(dwc->regs, DWC3_DALEPENA, reg);
630
879631aa 631 dep->stream_capable = false;
f9c56cdd 632 dep->endpoint.desc = NULL;
c90bfaec 633 dep->comp_desc = NULL;
72246da4 634 dep->type = 0;
879631aa 635 dep->flags = 0;
72246da4
FB		 636
637 return 0;
638}
639
640/* -------------------------------------------------------------------------- */
641
642static int dwc3_gadget_ep0_enable(struct usb_ep *ep,
643 const struct usb_endpoint_descriptor *desc)
644{
645 return -EINVAL;
646}
647
648static int dwc3_gadget_ep0_disable(struct usb_ep *ep)
649{
650 return -EINVAL;
651}
652
653/* -------------------------------------------------------------------------- */
654
655static int dwc3_gadget_ep_enable(struct usb_ep *ep,
656 const struct usb_endpoint_descriptor *desc)
657{
658 struct dwc3_ep *dep;
659 struct dwc3 *dwc;
660 unsigned long flags;
661 int ret;
662
663 if (!ep || !desc || desc->bDescriptorType != USB_DT_ENDPOINT) {
664 pr_debug("dwc3: invalid parameters\n");
665 return -EINVAL;
666 }
667
668 if (!desc->wMaxPacketSize) {
669 pr_debug("dwc3: missing wMaxPacketSize\n");
670 return -EINVAL;
671 }
672
673 dep = to_dwc3_ep(ep);
674 dwc = dep->dwc;
675
c6f83f38
FB		 676 if (dep->flags & DWC3_EP_ENABLED) {
677 dev_WARN_ONCE(dwc->dev, true, "%s is already enabled\n",
678 dep->name);
679 return 0;
680 }
681
72246da4
FB		 682 switch (usb_endpoint_type(desc)) {
683 case USB_ENDPOINT_XFER_CONTROL:
27a78d6a 684 strlcat(dep->name, "-control", sizeof(dep->name));
72246da4
FB		 685 break;
686 case USB_ENDPOINT_XFER_ISOC:
27a78d6a 687 strlcat(dep->name, "-isoc", sizeof(dep->name));
72246da4
FB		 688 break;
689 case USB_ENDPOINT_XFER_BULK:
27a78d6a 690 strlcat(dep->name, "-bulk", sizeof(dep->name));
72246da4
FB		 691 break;
692 case USB_ENDPOINT_XFER_INT:
27a78d6a 693 strlcat(dep->name, "-int", sizeof(dep->name));
72246da4
FB		 694 break;
695 default:
696 dev_err(dwc->dev, "invalid endpoint transfer type\n");
697 }
698
72246da4
FB		 699 dev_vdbg(dwc->dev, "Enabling %s\n", dep->name);
700
701 spin_lock_irqsave(&dwc->lock, flags);
4b345c9a 702 ret = __dwc3_gadget_ep_enable(dep, desc, ep->comp_desc, false);
72246da4
FB		 703 spin_unlock_irqrestore(&dwc->lock, flags);
704
705 return ret;
706}
707
708static int dwc3_gadget_ep_disable(struct usb_ep *ep)
709{
710 struct dwc3_ep *dep;
711 struct dwc3 *dwc;
712 unsigned long flags;
713 int ret;
714
715 if (!ep) {
716 pr_debug("dwc3: invalid parameters\n");
717 return -EINVAL;
718 }
719
720 dep = to_dwc3_ep(ep);
721 dwc = dep->dwc;
722
723 if (!(dep->flags & DWC3_EP_ENABLED)) {
724 dev_WARN_ONCE(dwc->dev, true, "%s is already disabled\n",
725 dep->name);
726 return 0;
727 }
728
729 snprintf(dep->name, sizeof(dep->name), "ep%d%s",
730 dep->number >> 1,
731 (dep->number & 1) ? "in" : "out");
732
733 spin_lock_irqsave(&dwc->lock, flags);
734 ret = __dwc3_gadget_ep_disable(dep);
735 spin_unlock_irqrestore(&dwc->lock, flags);
736
737 return ret;
738}
739
740static struct usb_request *dwc3_gadget_ep_alloc_request(struct usb_ep *ep,
741 gfp_t gfp_flags)
742{
743 struct dwc3_request *req;
744 struct dwc3_ep *dep = to_dwc3_ep(ep);
745 struct dwc3 *dwc = dep->dwc;
746
747 req = kzalloc(sizeof(*req), gfp_flags);
748 if (!req) {
749 dev_err(dwc->dev, "not enough memory\n");
750 return NULL;
751 }
752
753 req->epnum = dep->number;
754 req->dep = dep;
72246da4
FB		 755
756 return &req->request;
757}
758
759static void dwc3_gadget_ep_free_request(struct usb_ep *ep,
760 struct usb_request *request)
761{
762 struct dwc3_request *req = to_dwc3_request(request);
763
764 kfree(req);
765}
766
c71fc37c
FB		 767/**
768 * dwc3_prepare_one_trb - setup one TRB from one request
769 * @dep: endpoint for which this request is prepared
770 * @req: dwc3_request pointer
771 */
68e823e2 772static void dwc3_prepare_one_trb(struct dwc3_ep *dep,
eeb720fb 773 struct dwc3_request *req, dma_addr_t dma,
e5ba5ec8 774 unsigned length, unsigned last, unsigned chain, unsigned node)
c71fc37c 775{
eeb720fb 776 struct dwc3 *dwc = dep->dwc;
f6bafc6a 777 struct dwc3_trb *trb;
c71fc37c 778
eeb720fb
FB		 779 dev_vdbg(dwc->dev, "%s: req %p dma %08llx length %d%s%s\n",
780 dep->name, req, (unsigned long long) dma,
781 length, last ? " last" : "",
782 chain ? " chain" : "");
783
c71fc37c 784 /* Skip the LINK-TRB on ISOC */
915e202a 785 if (((dep->free_slot & DWC3_TRB_MASK) == DWC3_TRB_NUM - 1) &&
16e78db7 786 usb_endpoint_xfer_isoc(dep->endpoint.desc))
915e202a
PA		 787 dep->free_slot++;
788
789 trb = &dep->trb_pool[dep->free_slot & DWC3_TRB_MASK];
c71fc37c 790
eeb720fb
FB		 791 if (!req->trb) {
792 dwc3_gadget_move_request_queued(req);
f6bafc6a
FB		 793 req->trb = trb;
794 req->trb_dma = dwc3_trb_dma_offset(dep, trb);
e5ba5ec8 795 req->start_slot = dep->free_slot & DWC3_TRB_MASK;
eeb720fb 796 }
c71fc37c 797
e5ba5ec8
PA		 798 dep->free_slot++;
799
f6bafc6a
FB		 800 trb->size = DWC3_TRB_SIZE_LENGTH(length);
801 trb->bpl = lower_32_bits(dma);
802 trb->bph = upper_32_bits(dma);
c71fc37c 803
16e78db7 804 switch (usb_endpoint_type(dep->endpoint.desc)) {
c71fc37c 805 case USB_ENDPOINT_XFER_CONTROL:
f6bafc6a 806 trb->ctrl = DWC3_TRBCTL_CONTROL_SETUP;
c71fc37c
FB		 807 break;
808
809 case USB_ENDPOINT_XFER_ISOC:
e5ba5ec8
PA		 810 if (!node)
811 trb->ctrl = DWC3_TRBCTL_ISOCHRONOUS_FIRST;
812 else
813 trb->ctrl = DWC3_TRBCTL_ISOCHRONOUS;
c71fc37c 814
e5ba5ec8 815 if (!req->request.no_interrupt && !chain)
f6bafc6a 816 trb->ctrl |= DWC3_TRB_CTRL_IOC;
c71fc37c
FB		 817 break;
818
819 case USB_ENDPOINT_XFER_BULK:
820 case USB_ENDPOINT_XFER_INT:
f6bafc6a 821 trb->ctrl = DWC3_TRBCTL_NORMAL;
c71fc37c
FB		 822 break;
823 default:
824 /*
825 * This is only possible with faulty memory because we
826 * checked it already :)
827 */
828 BUG();
829 }
830
16e78db7 831 if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
f6bafc6a
FB		 832 trb->ctrl |= DWC3_TRB_CTRL_ISP_IMI;
833 trb->ctrl |= DWC3_TRB_CTRL_CSP;
e5ba5ec8
PA		 834 } else if (last) {
835 trb->ctrl |= DWC3_TRB_CTRL_LST;
f6bafc6a 836 }
c71fc37c 837
e5ba5ec8
PA		 838 if (chain)
839 trb->ctrl |= DWC3_TRB_CTRL_CHN;
840
16e78db7 841 if (usb_endpoint_xfer_bulk(dep->endpoint.desc) && dep->stream_capable)
f6bafc6a 842 trb->ctrl |= DWC3_TRB_CTRL_SID_SOFN(req->request.stream_id);
c71fc37c 843
f6bafc6a 844 trb->ctrl |= DWC3_TRB_CTRL_HWO;
c71fc37c
FB		 845}
846
72246da4
FB		 847/*
848 * dwc3_prepare_trbs - setup TRBs from requests
849 * @dep: endpoint for which requests are being prepared
850 * @starting: true if the endpoint is idle and no requests are queued.
851 *
1d046793
PZ		 852 * The function goes through the requests list and sets up TRBs for the
853 * transfers. The function returns once there are no more TRBs available or
854 * it runs out of requests.
72246da4 855 */
68e823e2 856static void dwc3_prepare_trbs(struct dwc3_ep *dep, bool starting)
72246da4 857{
68e823e2 858 struct dwc3_request *req, *n;
72246da4 859 u32 trbs_left;
8d62cd65 860 u32 max;
c71fc37c 861 unsigned int last_one = 0;
72246da4
FB		 862
863 BUILD_BUG_ON_NOT_POWER_OF_2(DWC3_TRB_NUM);
864
865 /* the first request must not be queued */
866 trbs_left = (dep->busy_slot - dep->free_slot) & DWC3_TRB_MASK;
c71fc37c 867
8d62cd65 868 /* Can't wrap around on a non-isoc EP since there's no link TRB */
16e78db7 869 if (!usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
8d62cd65
PZ		 870 max = DWC3_TRB_NUM - (dep->free_slot & DWC3_TRB_MASK);
871 if (trbs_left > max)
872 trbs_left = max;
873 }
874
72246da4 875 /*
1d046793
PZ		 876 * If busy & slot are equal than it is either full or empty. If we are
877 * starting to process requests then we are empty. Otherwise we are
72246da4
FB		 878 * full and don't do anything
879 */
880 if (!trbs_left) {
881 if (!starting)
68e823e2 882 return;
72246da4
FB		 883 trbs_left = DWC3_TRB_NUM;
884 /*
885 * In case we start from scratch, we queue the ISOC requests
886 * starting from slot 1. This is done because we use ring
887 * buffer and have no LST bit to stop us. Instead, we place
1d046793 888 * IOC bit every TRB_NUM/4. We try to avoid having an interrupt
72246da4
FB		 889 * after the first request so we start at slot 1 and have
890 * 7 requests proceed before we hit the first IOC.
891 * Other transfer types don't use the ring buffer and are
892 * processed from the first TRB until the last one. Since we
893 * don't wrap around we have to start at the beginning.
894 */
16e78db7 895 if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
72246da4
FB		 896 dep->busy_slot = 1;
897 dep->free_slot = 1;
898 } else {
899 dep->busy_slot = 0;
900 dep->free_slot = 0;
901 }
902 }
903
904 /* The last TRB is a link TRB, not used for xfer */
16e78db7 905 if ((trbs_left <= 1) && usb_endpoint_xfer_isoc(dep->endpoint.desc))
68e823e2 906 return;
72246da4
FB		 907
908 list_for_each_entry_safe(req, n, &dep->request_list, list) {
eeb720fb
FB		 909 unsigned length;
910 dma_addr_t dma;
e5ba5ec8 911 last_one = false;
72246da4 912
eeb720fb
FB		 913 if (req->request.num_mapped_sgs > 0) {
914 struct usb_request *request = &req->request;
915 struct scatterlist *sg = request->sg;
916 struct scatterlist *s;
917 int i;
72246da4 918
eeb720fb
FB		 919 for_each_sg(sg, s, request->num_mapped_sgs, i) {
920 unsigned chain = true;
72246da4 921
eeb720fb
FB		 922 length = sg_dma_len(s);
923 dma = sg_dma_address(s);
72246da4 924
1d046793
PZ		 925 if (i == (request->num_mapped_sgs - 1) ||
926 sg_is_last(s)) {
94b762f0 927 if (list_empty(&dep->request_list))
e5ba5ec8 928 last_one = true;
eeb720fb
FB		 929 chain = false;
930 }
72246da4 931
eeb720fb
FB		 932 trbs_left--;
933 if (!trbs_left)
934 last_one = true;
72246da4 935
eeb720fb
FB		 936 if (last_one)
937 chain = false;
72246da4 938
eeb720fb 939 dwc3_prepare_one_trb(dep, req, dma, length,
e5ba5ec8 940 last_one, chain, i);
72246da4 941
eeb720fb
FB		 942 if (last_one)
943 break;
944 }
a3664806
AV		 945
946 if (last_one)
947 break;
72246da4 948 } else {
eeb720fb
FB		 949 dma = req->request.dma;
950 length = req->request.length;
951 trbs_left--;
72246da4 952
eeb720fb
FB		 953 if (!trbs_left)
954 last_one = 1;
879631aa 955
eeb720fb
FB		 956 /* Is this the last request? */
957 if (list_is_last(&req->list, &dep->request_list))
958 last_one = 1;
72246da4 959
eeb720fb 960 dwc3_prepare_one_trb(dep, req, dma, length,
e5ba5ec8 961 last_one, false, 0);
72246da4 962
eeb720fb
FB		 963 if (last_one)
964 break;
72246da4 965 }
72246da4 966 }
72246da4
FB		 967}
968
969static int __dwc3_gadget_kick_transfer(struct dwc3_ep *dep, u16 cmd_param,
970 int start_new)
971{
972 struct dwc3_gadget_ep_cmd_params params;
973 struct dwc3_request *req;
974 struct dwc3 *dwc = dep->dwc;
975 int ret;
976 u32 cmd;
977
978 if (start_new && (dep->flags & DWC3_EP_BUSY)) {
979 dev_vdbg(dwc->dev, "%s: endpoint busy\n", dep->name);
980 return -EBUSY;
981 }
982 dep->flags &= ~DWC3_EP_PENDING_REQUEST;
983
984 /*
985 * If we are getting here after a short-out-packet we don't enqueue any
986 * new requests as we try to set the IOC bit only on the last request.
987 */
988 if (start_new) {
989 if (list_empty(&dep->req_queued))
990 dwc3_prepare_trbs(dep, start_new);
991
992 /* req points to the first request which will be sent */
993 req = next_request(&dep->req_queued);
994 } else {
68e823e2
FB		 995 dwc3_prepare_trbs(dep, start_new);
996
72246da4 997 /*
1d046793 998 * req points to the first request where HWO changed from 0 to 1
72246da4 999 */
68e823e2 1000 req = next_request(&dep->req_queued);
72246da4
FB		 1001 }
1002 if (!req) {
1003 dep->flags |= DWC3_EP_PENDING_REQUEST;
1004 return 0;
1005 }
1006
1007 memset(&params, 0, sizeof(params));
72246da4 1008
1877d6c9
PA		 1009 if (start_new) {
1010 params.param0 = upper_32_bits(req->trb_dma);
1011 params.param1 = lower_32_bits(req->trb_dma);
72246da4 1012 cmd = DWC3_DEPCMD_STARTTRANSFER;
1877d6c9 1013 } else {
72246da4 1014 cmd = DWC3_DEPCMD_UPDATETRANSFER;
1877d6c9 1015 }
72246da4
FB		 1016
1017 cmd |= DWC3_DEPCMD_PARAM(cmd_param);
1018 ret = dwc3_send_gadget_ep_cmd(dwc, dep->number, cmd, &params);
1019 if (ret < 0) {
1020 dev_dbg(dwc->dev, "failed to send STARTTRANSFER command\n");
1021
1022 /*
1023 * FIXME we need to iterate over the list of requests
1024 * here and stop, unmap, free and del each of the linked
1d046793 1025 * requests instead of what we do now.
72246da4 1026 */
0fc9a1be
FB		 1027 usb_gadget_unmap_request(&dwc->gadget, &req->request,
1028 req->direction);
72246da4
FB		 1029 list_del(&req->list);
1030 return ret;
1031 }
1032
1033 dep->flags |= DWC3_EP_BUSY;
25b8ff68 1034
f898ae09 1035 if (start_new) {
b4996a86 1036 dep->resource_index = dwc3_gadget_ep_get_transfer_index(dwc,
f898ae09 1037 dep->number);
b4996a86 1038 WARN_ON_ONCE(!dep->resource_index);
f898ae09 1039 }
25b8ff68 1040
72246da4
FB		 1041 return 0;
1042}
1043
d6d6ec7b
PA		 1044static void __dwc3_gadget_start_isoc(struct dwc3 *dwc,
1045 struct dwc3_ep *dep, u32 cur_uf)
1046{
1047 u32 uf;
1048
1049 if (list_empty(&dep->request_list)) {
1050 dev_vdbg(dwc->dev, "ISOC ep %s run out for requests.\n",
1051 dep->name);
f4a53c55 1052 dep->flags |= DWC3_EP_PENDING_REQUEST;
d6d6ec7b
PA		 1053 return;
1054 }
1055
1056 /* 4 micro frames in the future */
1057 uf = cur_uf + dep->interval * 4;
1058
1059 __dwc3_gadget_kick_transfer(dep, uf, 1);
1060}
1061
1062static void dwc3_gadget_start_isoc(struct dwc3 *dwc,
1063 struct dwc3_ep *dep, const struct dwc3_event_depevt *event)
1064{
1065 u32 cur_uf, mask;
1066
1067 mask = ~(dep->interval - 1);
1068 cur_uf = event->parameters & mask;
1069
1070 __dwc3_gadget_start_isoc(dwc, dep, cur_uf);
1071}
1072
72246da4
FB		 1073static int __dwc3_gadget_ep_queue(struct dwc3_ep *dep, struct dwc3_request *req)
1074{
0fc9a1be
FB		 1075 struct dwc3 *dwc = dep->dwc;
1076 int ret;
1077
72246da4
FB		 1078 req->request.actual = 0;
1079 req->request.status = -EINPROGRESS;
1080 req->direction = dep->direction;
1081 req->epnum = dep->number;
1082
1083 /*
1084 * We only add to our list of requests now and
1085 * start consuming the list once we get XferNotReady
1086 * IRQ.
1087 *
1088 * That way, we avoid doing anything that we don't need
1089 * to do now and defer it until the point we receive a
1090 * particular token from the Host side.
1091 *
1092 * This will also avoid Host cancelling URBs due to too
1d046793 1093 * many NAKs.
72246da4 1094 */
0fc9a1be
FB		 1095 ret = usb_gadget_map_request(&dwc->gadget, &req->request,
1096 dep->direction);
1097 if (ret)
1098 return ret;
1099
72246da4
FB		 1100 list_add_tail(&req->list, &dep->request_list);
1101
1102 /*
b511e5e7 1103 * There are a few special cases:
72246da4 1104 *
f898ae09
PZ		 1105 * 1. XferNotReady with empty list of requests. We need to kick the
1106 * transfer here in that situation, otherwise we will be NAKing
1107 * forever. If we get XferNotReady before gadget driver has a
1108 * chance to queue a request, we will ACK the IRQ but won't be
1109 * able to receive the data until the next request is queued.
1110 * The following code is handling exactly that.
72246da4 1111 *
72246da4
FB		 1112 */
1113 if (dep->flags & DWC3_EP_PENDING_REQUEST) {
f4a53c55
PA		 1114 /*
1115 * If xfernotready is already elapsed and it is a case
1116 * of isoc transfer, then issue END TRANSFER, so that
1117 * you can receive xfernotready again and can have
1118 * notion of current microframe.
1119 */
1120 if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
cdc359dd
PA		 1121 if (list_empty(&dep->req_queued)) {
1122 dwc3_stop_active_transfer(dwc, dep->number);
1123 dep->flags = DWC3_EP_ENABLED;
1124 }
f4a53c55
PA		 1125 return 0;
1126 }
1127
b511e5e7 1128 ret = __dwc3_gadget_kick_transfer(dep, 0, true);
348e026f 1129 if (ret && ret != -EBUSY)
b511e5e7
FB		 1130 dev_dbg(dwc->dev, "%s: failed to kick transfers\n",
1131 dep->name);
15f86bde 1132 return ret;
b511e5e7 1133 }
72246da4 1134
b511e5e7
FB		 1135 /*
1136 * 2. XferInProgress on Isoc EP with an active transfer. We need to
1137 * kick the transfer here after queuing a request, otherwise the
1138 * core may not see the modified TRB(s).
1139 */
1140 if (usb_endpoint_xfer_isoc(dep->endpoint.desc) &&
79c9046e
PA		 1141 (dep->flags & DWC3_EP_BUSY) &&
1142 !(dep->flags & DWC3_EP_MISSED_ISOC)) {
b4996a86
FB		 1143 WARN_ON_ONCE(!dep->resource_index);
1144 ret = __dwc3_gadget_kick_transfer(dep, dep->resource_index,
b511e5e7 1145 false);
348e026f 1146 if (ret && ret != -EBUSY)
72246da4
FB		 1147 dev_dbg(dwc->dev, "%s: failed to kick transfers\n",
1148 dep->name);
15f86bde 1149 return ret;
a0925324 1150 }
72246da4
FB		 1151
1152 return 0;
1153}
1154
1155static int dwc3_gadget_ep_queue(struct usb_ep *ep, struct usb_request *request,
1156 gfp_t gfp_flags)
1157{
1158 struct dwc3_request *req = to_dwc3_request(request);
1159 struct dwc3_ep *dep = to_dwc3_ep(ep);
1160 struct dwc3 *dwc = dep->dwc;
1161
1162 unsigned long flags;
1163
1164 int ret;
1165
16e78db7 1166 if (!dep->endpoint.desc) {
72246da4
FB		 1167 dev_dbg(dwc->dev, "trying to queue request %p to disabled %s\n",
1168 request, ep->name);
1169 return -ESHUTDOWN;
1170 }
1171
1172 dev_vdbg(dwc->dev, "queing request %p to %s length %d\n",
1173 request, ep->name, request->length);
1174
1175 spin_lock_irqsave(&dwc->lock, flags);
1176 ret = __dwc3_gadget_ep_queue(dep, req);
1177 spin_unlock_irqrestore(&dwc->lock, flags);
1178
1179 return ret;
1180}
1181
1182static int dwc3_gadget_ep_dequeue(struct usb_ep *ep,
1183 struct usb_request *request)
1184{
1185 struct dwc3_request *req = to_dwc3_request(request);
1186 struct dwc3_request *r = NULL;
1187
1188 struct dwc3_ep *dep = to_dwc3_ep(ep);
1189 struct dwc3 *dwc = dep->dwc;
1190
1191 unsigned long flags;
1192 int ret = 0;
1193
1194 spin_lock_irqsave(&dwc->lock, flags);
1195
1196 list_for_each_entry(r, &dep->request_list, list) {
1197 if (r == req)
1198 break;
1199 }
1200
1201 if (r != req) {
1202 list_for_each_entry(r, &dep->req_queued, list) {
1203 if (r == req)
1204 break;
1205 }
1206 if (r == req) {
1207 /* wait until it is processed */
1208 dwc3_stop_active_transfer(dwc, dep->number);
e8d4e8be 1209 goto out1;
72246da4
FB		 1210 }
1211 dev_err(dwc->dev, "request %p was not queued to %s\n",
1212 request, ep->name);
1213 ret = -EINVAL;
1214 goto out0;
1215 }
1216
e8d4e8be 1217out1:
72246da4
FB		 1218 /* giveback the request */
1219 dwc3_gadget_giveback(dep, req, -ECONNRESET);
1220
1221out0:
1222 spin_unlock_irqrestore(&dwc->lock, flags);
1223
1224 return ret;
1225}
1226
7d513758 1227int __dwc3_gadget_ep_set_halt(struct dwc3_ep *dep, int value, int protocol)
72246da4
FB		 1228{
1229 struct dwc3_gadget_ep_cmd_params params;
1230 struct dwc3 *dwc = dep->dwc;
1231 int ret;
1232
1233 memset(&params, 0x00, sizeof(params));
1234
1235 if (value) {
7d513758
FB		 1236 if (!protocol && ((dep->direction && dep->flags & DWC3_EP_BUSY) ||
1237 (!list_empty(&dep->req_queued) ||
1238 !list_empty(&dep->request_list)))) {
1239 dev_dbg(dwc->dev, "%s: pending request, cannot halt\n",
1240 dep->name);
1241 return -EAGAIN;
1242 }
1243
72246da4
FB		 1244 ret = dwc3_send_gadget_ep_cmd(dwc, dep->number,
1245 DWC3_DEPCMD_SETSTALL, &params);
1246 if (ret)
1247 dev_err(dwc->dev, "failed to %s STALL on %s\n",
1248 value ? "set" : "clear",
1249 dep->name);
1250 else
1251 dep->flags |= DWC3_EP_STALL;
1252 } else {
1253 ret = dwc3_send_gadget_ep_cmd(dwc, dep->number,
1254 DWC3_DEPCMD_CLEARSTALL, &params);
1255 if (ret)
1256 dev_err(dwc->dev, "failed to %s STALL on %s\n",
1257 value ? "set" : "clear",
1258 dep->name);
1259 else
e6303463 1260 dep->flags &= ~(DWC3_EP_STALL | DWC3_EP_WEDGE);
72246da4 1261 }
5275455a 1262
72246da4
FB		 1263 return ret;
1264}
1265
1266static int dwc3_gadget_ep_set_halt(struct usb_ep *ep, int value)
1267{
1268 struct dwc3_ep *dep = to_dwc3_ep(ep);
1269 struct dwc3 *dwc = dep->dwc;
1270
1271 unsigned long flags;
1272
1273 int ret;
1274
1275 spin_lock_irqsave(&dwc->lock, flags);
1276
16e78db7 1277 if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
72246da4
FB		 1278 dev_err(dwc->dev, "%s is of Isochronous type\n", dep->name);
1279 ret = -EINVAL;
1280 goto out;
1281 }
1282
7d513758 1283 ret = __dwc3_gadget_ep_set_halt(dep, value, false);
72246da4
FB		 1284out:
1285 spin_unlock_irqrestore(&dwc->lock, flags);
1286
1287 return ret;
1288}
1289
1290static int dwc3_gadget_ep_set_wedge(struct usb_ep *ep)
1291{
1292 struct dwc3_ep *dep = to_dwc3_ep(ep);
249a4569
PZ		 1293 struct dwc3 *dwc = dep->dwc;
1294 unsigned long flags;
72246da4 1295
249a4569 1296 spin_lock_irqsave(&dwc->lock, flags);
72246da4 1297 dep->flags |= DWC3_EP_WEDGE;
249a4569 1298 spin_unlock_irqrestore(&dwc->lock, flags);
72246da4 1299
08f0d966
PA		 1300 if (dep->number == 0 || dep->number == 1)
1301 return dwc3_gadget_ep0_set_halt(ep, 1);
1302 else
7d513758 1303 return __dwc3_gadget_ep_set_halt(dep, 1, false);
72246da4
FB		 1304}
1305
1306/* -------------------------------------------------------------------------- */
1307
1308static struct usb_endpoint_descriptor dwc3_gadget_ep0_desc = {
1309 .bLength = USB_DT_ENDPOINT_SIZE,
1310 .bDescriptorType = USB_DT_ENDPOINT,
1311 .bmAttributes = USB_ENDPOINT_XFER_CONTROL,
1312};
1313
1314static const struct usb_ep_ops dwc3_gadget_ep0_ops = {
1315 .enable = dwc3_gadget_ep0_enable,
1316 .disable = dwc3_gadget_ep0_disable,
1317 .alloc_request = dwc3_gadget_ep_alloc_request,
1318 .free_request = dwc3_gadget_ep_free_request,
1319 .queue = dwc3_gadget_ep0_queue,
1320 .dequeue = dwc3_gadget_ep_dequeue,
08f0d966 1321 .set_halt = dwc3_gadget_ep0_set_halt,
72246da4
FB		 1322 .set_wedge = dwc3_gadget_ep_set_wedge,
1323};
1324
1325static const struct usb_ep_ops dwc3_gadget_ep_ops = {
1326 .enable = dwc3_gadget_ep_enable,
1327 .disable = dwc3_gadget_ep_disable,
1328 .alloc_request = dwc3_gadget_ep_alloc_request,
1329 .free_request = dwc3_gadget_ep_free_request,
1330 .queue = dwc3_gadget_ep_queue,
1331 .dequeue = dwc3_gadget_ep_dequeue,
1332 .set_halt = dwc3_gadget_ep_set_halt,
1333 .set_wedge = dwc3_gadget_ep_set_wedge,
1334};
1335
1336/* -------------------------------------------------------------------------- */
1337
1338static int dwc3_gadget_get_frame(struct usb_gadget *g)
1339{
1340 struct dwc3 *dwc = gadget_to_dwc(g);
1341 u32 reg;
1342
1343 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
1344 return DWC3_DSTS_SOFFN(reg);
1345}
1346
1347static int dwc3_gadget_wakeup(struct usb_gadget *g)
1348{
1349 struct dwc3 *dwc = gadget_to_dwc(g);
1350
1351 unsigned long timeout;
1352 unsigned long flags;
1353
1354 u32 reg;
1355
1356 int ret = 0;
1357
1358 u8 link_state;
1359 u8 speed;
1360
1361 spin_lock_irqsave(&dwc->lock, flags);
1362
1363 /*
1364 * According to the Databook Remote wakeup request should
1365 * be issued only when the device is in early suspend state.
1366 *
1367 * We can check that via USB Link State bits in DSTS register.
1368 */
1369 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
1370
1371 speed = reg & DWC3_DSTS_CONNECTSPD;
1372 if (speed == DWC3_DSTS_SUPERSPEED) {
1373 dev_dbg(dwc->dev, "no wakeup on SuperSpeed\n");
1374 ret = -EINVAL;
1375 goto out;
1376 }
1377
1378 link_state = DWC3_DSTS_USBLNKST(reg);
1379
1380 switch (link_state) {
1381 case DWC3_LINK_STATE_RX_DET: /* in HS, means Early Suspend */
1382 case DWC3_LINK_STATE_U3: /* in HS, means SUSPEND */
1383 break;
1384 default:
1385 dev_dbg(dwc->dev, "can't wakeup from link state %d\n",
1386 link_state);
1387 ret = -EINVAL;
1388 goto out;
1389 }
1390
8598bde7
FB		 1391 ret = dwc3_gadget_set_link_state(dwc, DWC3_LINK_STATE_RECOV);
1392 if (ret < 0) {
1393 dev_err(dwc->dev, "failed to put link in Recovery\n");
1394 goto out;
1395 }
72246da4 1396
802fde98
PZ		 1397 /* Recent versions do this automatically */
1398 if (dwc->revision < DWC3_REVISION_194A) {
1399 /* write zeroes to Link Change Request */
fcc023c7 1400 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
802fde98
PZ		 1401 reg &= ~DWC3_DCTL_ULSTCHNGREQ_MASK;
1402 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
1403 }
72246da4 1404
1d046793 1405 /* poll until Link State changes to ON */
72246da4
FB		 1406 timeout = jiffies + msecs_to_jiffies(100);
1407
1d046793 1408 while (!time_after(jiffies, timeout)) {
72246da4
FB		 1409 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
1410
1411 /* in HS, means ON */
1412 if (DWC3_DSTS_USBLNKST(reg) == DWC3_LINK_STATE_U0)
1413 break;
1414 }
1415
1416 if (DWC3_DSTS_USBLNKST(reg) != DWC3_LINK_STATE_U0) {
1417 dev_err(dwc->dev, "failed to send remote wakeup\n");
1418 ret = -EINVAL;
1419 }
1420
1421out:
1422 spin_unlock_irqrestore(&dwc->lock, flags);
1423
1424 return ret;
1425}
1426
1427static int dwc3_gadget_set_selfpowered(struct usb_gadget *g,
1428 int is_selfpowered)
1429{
1430 struct dwc3 *dwc = gadget_to_dwc(g);
249a4569 1431 unsigned long flags;
72246da4 1432
249a4569 1433 spin_lock_irqsave(&dwc->lock, flags);
72246da4 1434 dwc->is_selfpowered = !!is_selfpowered;
249a4569 1435 spin_unlock_irqrestore(&dwc->lock, flags);
72246da4
FB		 1436
1437 return 0;
1438}
1439
6f17f74b 1440static int dwc3_gadget_run_stop(struct dwc3 *dwc, int is_on)
72246da4
FB		 1441{
1442 u32 reg;
61d58242 1443 u32 timeout = 500;
72246da4
FB		 1444
1445 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
8db7ed15 1446 if (is_on) {
802fde98
PZ		 1447 if (dwc->revision <= DWC3_REVISION_187A) {
1448 reg &= ~DWC3_DCTL_TRGTULST_MASK;
1449 reg |= DWC3_DCTL_TRGTULST_RX_DET;
1450 }
1451
1452 if (dwc->revision >= DWC3_REVISION_194A)
1453 reg &= ~DWC3_DCTL_KEEP_CONNECT;
1454 reg |= DWC3_DCTL_RUN_STOP;
9fcb3bd8 1455 dwc->pullups_connected = true;
8db7ed15 1456 } else {
72246da4 1457 reg &= ~DWC3_DCTL_RUN_STOP;
9fcb3bd8 1458 dwc->pullups_connected = false;
8db7ed15 1459 }
72246da4
FB		 1460
1461 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
1462
1463 do {
1464 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
1465 if (is_on) {
1466 if (!(reg & DWC3_DSTS_DEVCTRLHLT))
1467 break;
1468 } else {
1469 if (reg & DWC3_DSTS_DEVCTRLHLT)
1470 break;
1471 }
72246da4
FB		 1472 timeout--;
1473 if (!timeout)
6f17f74b 1474 return -ETIMEDOUT;
61d58242 1475 udelay(1);
72246da4
FB		 1476 } while (1);
1477
1478 dev_vdbg(dwc->dev, "gadget %s data soft-%s\n",
1479 dwc->gadget_driver
1480 ? dwc->gadget_driver->function : "no-function",
1481 is_on ? "connect" : "disconnect");
6f17f74b
PA		 1482
1483 return 0;
72246da4
FB		 1484}
1485
1486static int dwc3_gadget_pullup(struct usb_gadget *g, int is_on)
1487{
1488 struct dwc3 *dwc = gadget_to_dwc(g);
1489 unsigned long flags;
6f17f74b 1490 int ret;
72246da4
FB		 1491
1492 is_on = !!is_on;
1493
1494 spin_lock_irqsave(&dwc->lock, flags);
6f17f74b 1495 ret = dwc3_gadget_run_stop(dwc, is_on);
72246da4
FB		 1496 spin_unlock_irqrestore(&dwc->lock, flags);
1497
6f17f74b 1498 return ret;
72246da4
FB		 1499}
1500
8698e2ac
FB		 1501static void dwc3_gadget_enable_irq(struct dwc3 *dwc)
1502{
1503 u32 reg;
1504
1505 /* Enable all but Start and End of Frame IRQs */
1506 reg = (DWC3_DEVTEN_VNDRDEVTSTRCVEDEN |
1507 DWC3_DEVTEN_EVNTOVERFLOWEN |
1508 DWC3_DEVTEN_CMDCMPLTEN |
1509 DWC3_DEVTEN_ERRTICERREN |
1510 DWC3_DEVTEN_WKUPEVTEN |
1511 DWC3_DEVTEN_ULSTCNGEN |
1512 DWC3_DEVTEN_CONNECTDONEEN |
1513 DWC3_DEVTEN_USBRSTEN |
1514 DWC3_DEVTEN_DISCONNEVTEN);
1515
1516 dwc3_writel(dwc->regs, DWC3_DEVTEN, reg);
1517}
1518
1519static void dwc3_gadget_disable_irq(struct dwc3 *dwc)
1520{
1521 /* mask all interrupts */
1522 dwc3_writel(dwc->regs, DWC3_DEVTEN, 0x00);
1523}
1524
1525static irqreturn_t dwc3_interrupt(int irq, void *_dwc);
b15a762f 1526static irqreturn_t dwc3_thread_interrupt(int irq, void *_dwc);
8698e2ac 1527
72246da4
FB		 1528static int dwc3_gadget_start(struct usb_gadget *g,
1529 struct usb_gadget_driver *driver)
1530{
1531 struct dwc3 *dwc = gadget_to_dwc(g);
1532 struct dwc3_ep *dep;
1533 unsigned long flags;
1534 int ret = 0;
8698e2ac 1535 int irq;
72246da4
FB		 1536 u32 reg;
1537
734b2fe9
FB		 1538 irq = platform_get_irq(to_platform_device(dwc->dev), 0);
1539 ret = request_threaded_irq(irq, dwc3_interrupt, dwc3_thread_interrupt,
1540 IRQF_SHARED | IRQF_ONESHOT, "dwc3", dwc);
1541 if (ret) {
1542 dev_err(dwc->dev, "failed to request irq #%d --> %d\n",
1543 irq, ret);
1544 goto err0;
1545 }
1546
72246da4
FB		 1547 spin_lock_irqsave(&dwc->lock, flags);
1548
1549 if (dwc->gadget_driver) {
1550 dev_err(dwc->dev, "%s is already bound to %s\n",
1551 dwc->gadget.name,
1552 dwc->gadget_driver->driver.name);
1553 ret = -EBUSY;
734b2fe9 1554 goto err1;
72246da4
FB		 1555 }
1556
1557 dwc->gadget_driver = driver;
72246da4 1558
72246da4
FB		 1559 reg = dwc3_readl(dwc->regs, DWC3_DCFG);
1560 reg &= ~(DWC3_DCFG_SPEED_MASK);
07e7f47b
FB		 1561
1562 /**
1563 * WORKAROUND: DWC3 revision < 2.20a have an issue
1564 * which would cause metastability state on Run/Stop
1565 * bit if we try to force the IP to USB2-only mode.
1566 *
1567 * Because of that, we cannot configure the IP to any
1568 * speed other than the SuperSpeed
1569 *
1570 * Refers to:
1571 *
1572 * STAR#9000525659: Clock Domain Crossing on DCTL in
1573 * USB 2.0 Mode
1574 */
1575 if (dwc->revision < DWC3_REVISION_220A)
1576 reg |= DWC3_DCFG_SUPERSPEED;
1577 else
1578 reg |= dwc->maximum_speed;
72246da4
FB		 1579 dwc3_writel(dwc->regs, DWC3_DCFG, reg);
1580
b23c8439
PZ		 1581 dwc->start_config_issued = false;
1582
72246da4
FB		 1583 /* Start with SuperSpeed Default */
1584 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
1585
1586 dep = dwc->eps[0];
4b345c9a 1587 ret = __dwc3_gadget_ep_enable(dep, &dwc3_gadget_ep0_desc, NULL, false);
72246da4
FB		 1588 if (ret) {
1589 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
734b2fe9 1590 goto err2;
72246da4
FB		 1591 }
1592
1593 dep = dwc->eps[1];
4b345c9a 1594 ret = __dwc3_gadget_ep_enable(dep, &dwc3_gadget_ep0_desc, NULL, false);
72246da4
FB		 1595 if (ret) {
1596 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
734b2fe9 1597 goto err3;
72246da4
FB		 1598 }
1599
1600 /* begin to receive SETUP packets */
c7fcdeb2 1601 dwc->ep0state = EP0_SETUP_PHASE;
72246da4
FB		 1602 dwc3_ep0_out_start(dwc);
1603
8698e2ac
FB		 1604 dwc3_gadget_enable_irq(dwc);
1605
72246da4
FB		 1606 spin_unlock_irqrestore(&dwc->lock, flags);
1607
1608 return 0;
1609
734b2fe9 1610err3:
72246da4
FB		 1611 __dwc3_gadget_ep_disable(dwc->eps[0]);
1612
734b2fe9 1613err2:
003dda8a 1614 dwc->gadget_driver = NULL;
734b2fe9
FB		 1615
1616err1:
72246da4
FB		 1617 spin_unlock_irqrestore(&dwc->lock, flags);
1618
734b2fe9
FB		 1619 free_irq(irq, dwc);
1620
1621err0:
72246da4
FB		 1622 return ret;
1623}
1624
1625static int dwc3_gadget_stop(struct usb_gadget *g,
1626 struct usb_gadget_driver *driver)
1627{
1628 struct dwc3 *dwc = gadget_to_dwc(g);
1629 unsigned long flags;
8698e2ac 1630 int irq;
72246da4
FB		 1631
1632 spin_lock_irqsave(&dwc->lock, flags);
1633
8698e2ac 1634 dwc3_gadget_disable_irq(dwc);
72246da4
FB		 1635 __dwc3_gadget_ep_disable(dwc->eps[0]);
1636 __dwc3_gadget_ep_disable(dwc->eps[1]);
1637
1638 dwc->gadget_driver = NULL;
72246da4
FB		 1639
1640 spin_unlock_irqrestore(&dwc->lock, flags);
1641
734b2fe9
FB		 1642 irq = platform_get_irq(to_platform_device(dwc->dev), 0);
1643 free_irq(irq, dwc);
1644
72246da4
FB		 1645 return 0;
1646}
802fde98 1647
72246da4
FB		 1648static const struct usb_gadget_ops dwc3_gadget_ops = {
1649 .get_frame = dwc3_gadget_get_frame,
1650 .wakeup = dwc3_gadget_wakeup,
1651 .set_selfpowered = dwc3_gadget_set_selfpowered,
1652 .pullup = dwc3_gadget_pullup,
1653 .udc_start = dwc3_gadget_start,
1654 .udc_stop = dwc3_gadget_stop,
1655};
1656
1657/* -------------------------------------------------------------------------- */
1658
6a1e3ef4
FB		 1659static int dwc3_gadget_init_hw_endpoints(struct dwc3 *dwc,
1660 u8 num, u32 direction)
72246da4
FB		 1661{
1662 struct dwc3_ep *dep;
6a1e3ef4 1663 u8 i;
72246da4 1664
6a1e3ef4
FB		 1665 for (i = 0; i < num; i++) {
1666 u8 epnum = (i << 1) | (!!direction);
72246da4 1667
72246da4
FB		 1668 dep = kzalloc(sizeof(*dep), GFP_KERNEL);
1669 if (!dep) {
1670 dev_err(dwc->dev, "can't allocate endpoint %d\n",
1671 epnum);
1672 return -ENOMEM;
1673 }
1674
1675 dep->dwc = dwc;
1676 dep->number = epnum;
1677 dwc->eps[epnum] = dep;
1678
1679 snprintf(dep->name, sizeof(dep->name), "ep%d%s", epnum >> 1,
1680 (epnum & 1) ? "in" : "out");
6a1e3ef4 1681
72246da4
FB		 1682 dep->endpoint.name = dep->name;
1683 dep->direction = (epnum & 1);
1684
1685 if (epnum == 0 || epnum == 1) {
1686 dep->endpoint.maxpacket = 512;
6048e4c6 1687 dep->endpoint.maxburst = 1;
72246da4
FB		 1688 dep->endpoint.ops = &dwc3_gadget_ep0_ops;
1689 if (!epnum)
1690 dwc->gadget.ep0 = &dep->endpoint;
1691 } else {
1692 int ret;
1693
1694 dep->endpoint.maxpacket = 1024;
12d36c16 1695 dep->endpoint.max_streams = 15;
72246da4
FB		 1696 dep->endpoint.ops = &dwc3_gadget_ep_ops;
1697 list_add_tail(&dep->endpoint.ep_list,
1698 &dwc->gadget.ep_list);
1699
1700 ret = dwc3_alloc_trb_pool(dep);
25b8ff68 1701 if (ret)
72246da4 1702 return ret;
72246da4 1703 }
25b8ff68 1704
72246da4
FB		 1705 INIT_LIST_HEAD(&dep->request_list);
1706 INIT_LIST_HEAD(&dep->req_queued);
1707 }
1708
1709 return 0;
1710}
1711
6a1e3ef4
FB		 1712static int dwc3_gadget_init_endpoints(struct dwc3 *dwc)
1713{
1714 int ret;
1715
1716 INIT_LIST_HEAD(&dwc->gadget.ep_list);
1717
1718 ret = dwc3_gadget_init_hw_endpoints(dwc, dwc->num_out_eps, 0);
1719 if (ret < 0) {
1720 dev_vdbg(dwc->dev, "failed to allocate OUT endpoints\n");
1721 return ret;
1722 }
1723
1724 ret = dwc3_gadget_init_hw_endpoints(dwc, dwc->num_in_eps, 1);
1725 if (ret < 0) {
1726 dev_vdbg(dwc->dev, "failed to allocate IN endpoints\n");
1727 return ret;
1728 }
1729
1730 return 0;
1731}
1732
72246da4
FB		 1733static void dwc3_gadget_free_endpoints(struct dwc3 *dwc)
1734{
1735 struct dwc3_ep *dep;
1736 u8 epnum;
1737
1738 for (epnum = 0; epnum < DWC3_ENDPOINTS_NUM; epnum++) {
1739 dep = dwc->eps[epnum];
6a1e3ef4
FB		 1740 if (!dep)
1741 continue;
5bf8fae3
GC		 1742 /*
1743 * Physical endpoints 0 and 1 are special; they form the
1744 * bi-directional USB endpoint 0.
1745 *
1746 * For those two physical endpoints, we don't allocate a TRB
1747 * pool nor do we add them the endpoints list. Due to that, we
1748 * shouldn't do these two operations otherwise we would end up
1749 * with all sorts of bugs when removing dwc3.ko.
1750 */
1751 if (epnum != 0 && epnum != 1) {
1752 dwc3_free_trb_pool(dep);
72246da4 1753 list_del(&dep->endpoint.ep_list);
5bf8fae3 1754 }
72246da4
FB		 1755
1756 kfree(dep);
1757 }
1758}
1759
72246da4 1760/* -------------------------------------------------------------------------- */
e5caff68 1761
e5ba5ec8
PA		 1762static int __dwc3_cleanup_done_trbs(struct dwc3 *dwc, struct dwc3_ep *dep,
1763 struct dwc3_request *req, struct dwc3_trb *trb,
72246da4
FB		 1764 const struct dwc3_event_depevt *event, int status)
1765{
72246da4
FB		 1766 unsigned int count;
1767 unsigned int s_pkt = 0;
d6d6ec7b 1768 unsigned int trb_status;
72246da4 1769
e5ba5ec8
PA		 1770 if ((trb->ctrl & DWC3_TRB_CTRL_HWO) && status != -ESHUTDOWN)
1771 /*
1772 * We continue despite the error. There is not much we
1773 * can do. If we don't clean it up we loop forever. If
1774 * we skip the TRB then it gets overwritten after a
1775 * while since we use them in a ring buffer. A BUG()
1776 * would help. Lets hope that if this occurs, someone
1777 * fixes the root cause instead of looking away :)
1778 */
1779 dev_err(dwc->dev, "%s's TRB (%p) still owned by HW\n",
1780 dep->name, trb);
1781 count = trb->size & DWC3_TRB_SIZE_MASK;
1782
1783 if (dep->direction) {
1784 if (count) {
1785 trb_status = DWC3_TRB_SIZE_TRBSTS(trb->size);
1786 if (trb_status == DWC3_TRBSTS_MISSED_ISOC) {
1787 dev_dbg(dwc->dev, "incomplete IN transfer %s\n",
1788 dep->name);
1789 /*
1790 * If missed isoc occurred and there is
1791 * no request queued then issue END
1792 * TRANSFER, so that core generates
1793 * next xfernotready and we will issue
1794 * a fresh START TRANSFER.
1795 * If there are still queued request
1796 * then wait, do not issue either END
1797 * or UPDATE TRANSFER, just attach next
1798 * request in request_list during
1799 * giveback.If any future queued request
1800 * is successfully transferred then we
1801 * will issue UPDATE TRANSFER for all
1802 * request in the request_list.
1803 */
1804 dep->flags |= DWC3_EP_MISSED_ISOC;
1805 } else {
1806 dev_err(dwc->dev, "incomplete IN transfer %s\n",
1807 dep->name);
1808 status = -ECONNRESET;
1809 }
1810 } else {
1811 dep->flags &= ~DWC3_EP_MISSED_ISOC;
1812 }
1813 } else {
1814 if (count && (event->status & DEPEVT_STATUS_SHORT))
1815 s_pkt = 1;
1816 }
1817
e5ba5ec8
PA		 1818 if (s_pkt)
1819 return 1;
1820 if ((event->status & DEPEVT_STATUS_LST) &&
1821 (trb->ctrl & (DWC3_TRB_CTRL_LST |
1822 DWC3_TRB_CTRL_HWO)))
1823 return 1;
1824 if ((event->status & DEPEVT_STATUS_IOC) &&
1825 (trb->ctrl & DWC3_TRB_CTRL_IOC))
1826 return 1;
1827 return 0;
1828}
1829
1830static int dwc3_cleanup_done_reqs(struct dwc3 *dwc, struct dwc3_ep *dep,
1831 const struct dwc3_event_depevt *event, int status)
1832{
1833 struct dwc3_request *req;
1834 struct dwc3_trb *trb;
1835 unsigned int slot;
1836 unsigned int i;
1cb7105d 1837 int count = 0;
e5ba5ec8
PA		 1838 int ret;
1839
72246da4
FB		 1840 do {
1841 req = next_request(&dep->req_queued);
d39ee7be
SAS		 1842 if (!req) {
1843 WARN_ON_ONCE(1);
1844 return 1;
1845 }
e5ba5ec8
PA		 1846 i = 0;
1847 do {
1848 slot = req->start_slot + i;
1849 if ((slot == DWC3_TRB_NUM - 1) &&
1850 usb_endpoint_xfer_isoc(dep->endpoint.desc))
1851 slot++;
1852 slot %= DWC3_TRB_NUM;
1853 trb = &dep->trb_pool[slot];
1cb7105d
FB		 1854 count += trb->size & DWC3_TRB_SIZE_MASK;
1855
72246da4 1856
e5ba5ec8
PA		 1857 ret = __dwc3_cleanup_done_trbs(dwc, dep, req, trb,
1858 event, status);
1859 if (ret)
1860 break;
1861 }while (++i < req->request.num_mapped_sgs);
72246da4 1862
1cb7105d
FB		 1863 /*
1864 * We assume here we will always receive the entire data block
1865 * which we should receive. Meaning, if we program RX to
1866 * receive 4K but we receive only 2K, we assume that's all we
1867 * should receive and we simply bounce the request back to the
1868 * gadget driver for further processing.
1869 */
1870 req->request.actual += req->request.length - count;
72246da4 1871 dwc3_gadget_giveback(dep, req, status);
e5ba5ec8
PA		 1872
1873 if (ret)
72246da4
FB		 1874 break;
1875 } while (1);
1876
cdc359dd
PA		 1877 if (usb_endpoint_xfer_isoc(dep->endpoint.desc) &&
1878 list_empty(&dep->req_queued)) {
1879 if (list_empty(&dep->request_list)) {
1880 /*
1881 * If there is no entry in request list then do
1882 * not issue END TRANSFER now. Just set PENDING
1883 * flag, so that END TRANSFER is issued when an
1884 * entry is added into request list.
1885 */
1886 dep->flags = DWC3_EP_PENDING_REQUEST;
1887 } else {
1888 dwc3_stop_active_transfer(dwc, dep->number);
1889 dep->flags = DWC3_EP_ENABLED;
1890 }
7efea86c
PA		 1891 return 1;
1892 }
1893
f6bafc6a
FB		 1894 if ((event->status & DEPEVT_STATUS_IOC) &&
1895 (trb->ctrl & DWC3_TRB_CTRL_IOC))
72246da4
FB		 1896 return 0;
1897 return 1;
1898}
1899
1900static void dwc3_endpoint_transfer_complete(struct dwc3 *dwc,
1901 struct dwc3_ep *dep, const struct dwc3_event_depevt *event,
1902 int start_new)
1903{
1904 unsigned status = 0;
1905 int clean_busy;
1906
1907 if (event->status & DEPEVT_STATUS_BUSERR)
1908 status = -ECONNRESET;
1909
1d046793 1910 clean_busy = dwc3_cleanup_done_reqs(dwc, dep, event, status);
c2df85ca 1911 if (clean_busy)
72246da4 1912 dep->flags &= ~DWC3_EP_BUSY;
fae2b904
FB		 1913
1914 /*
1915 * WORKAROUND: This is the 2nd half of U1/U2 -> U0 workaround.
1916 * See dwc3_gadget_linksts_change_interrupt() for 1st half.
1917 */
1918 if (dwc->revision < DWC3_REVISION_183A) {
1919 u32 reg;
1920 int i;
1921
1922 for (i = 0; i < DWC3_ENDPOINTS_NUM; i++) {
348e026f 1923 dep = dwc->eps[i];
fae2b904
FB		 1924
1925 if (!(dep->flags & DWC3_EP_ENABLED))
1926 continue;
1927
1928 if (!list_empty(&dep->req_queued))
1929 return;
1930 }
1931
1932 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
1933 reg |= dwc->u1u2;
1934 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
1935
1936 dwc->u1u2 = 0;
1937 }
72246da4
FB		 1938}
1939
72246da4
FB		 1940static void dwc3_endpoint_interrupt(struct dwc3 *dwc,
1941 const struct dwc3_event_depevt *event)
1942{
1943 struct dwc3_ep *dep;
1944 u8 epnum = event->endpoint_number;
1945
1946 dep = dwc->eps[epnum];
1947
3336abb5
FB		 1948 if (!(dep->flags & DWC3_EP_ENABLED))
1949 return;
1950
72246da4
FB		 1951 dev_vdbg(dwc->dev, "%s: %s\n", dep->name,
1952 dwc3_ep_event_string(event->endpoint_event));
1953
1954 if (epnum == 0 || epnum == 1) {
1955 dwc3_ep0_interrupt(dwc, event);
1956 return;
1957 }
1958
1959 switch (event->endpoint_event) {
1960 case DWC3_DEPEVT_XFERCOMPLETE:
b4996a86 1961 dep->resource_index = 0;
c2df85ca 1962
16e78db7 1963 if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
72246da4
FB		 1964 dev_dbg(dwc->dev, "%s is an Isochronous endpoint\n",
1965 dep->name);
1966 return;
1967 }
1968
1969 dwc3_endpoint_transfer_complete(dwc, dep, event, 1);
1970 break;
1971 case DWC3_DEPEVT_XFERINPROGRESS:
16e78db7 1972 if (!usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
72246da4
FB		 1973 dev_dbg(dwc->dev, "%s is not an Isochronous endpoint\n",
1974 dep->name);
1975 return;
1976 }
1977
1978 dwc3_endpoint_transfer_complete(dwc, dep, event, 0);
1979 break;
1980 case DWC3_DEPEVT_XFERNOTREADY:
16e78db7 1981 if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
72246da4
FB		 1982 dwc3_gadget_start_isoc(dwc, dep, event);
1983 } else {
1984 int ret;
1985
1986 dev_vdbg(dwc->dev, "%s: reason %s\n",
40aa41fb
FB		 1987 dep->name, event->status &
1988 DEPEVT_STATUS_TRANSFER_ACTIVE
72246da4
FB		 1989 ? "Transfer Active"
1990 : "Transfer Not Active");
1991
1992 ret = __dwc3_gadget_kick_transfer(dep, 0, 1);
1993 if (!ret || ret == -EBUSY)
1994 return;
1995
1996 dev_dbg(dwc->dev, "%s: failed to kick transfers\n",
1997 dep->name);
1998 }
1999
879631aa
FB		 2000 break;
2001 case DWC3_DEPEVT_STREAMEVT:
16e78db7 2002 if (!usb_endpoint_xfer_bulk(dep->endpoint.desc)) {
879631aa
FB		 2003 dev_err(dwc->dev, "Stream event for non-Bulk %s\n",
2004 dep->name);
2005 return;
2006 }
2007
2008 switch (event->status) {
2009 case DEPEVT_STREAMEVT_FOUND:
2010 dev_vdbg(dwc->dev, "Stream %d found and started\n",
2011 event->parameters);
2012
2013 break;
2014 case DEPEVT_STREAMEVT_NOTFOUND:
2015 /* FALLTHROUGH */
2016 default:
2017 dev_dbg(dwc->dev, "Couldn't find suitable stream\n");
2018 }
72246da4
FB		 2019 break;
2020 case DWC3_DEPEVT_RXTXFIFOEVT:
2021 dev_dbg(dwc->dev, "%s FIFO Overrun\n", dep->name);
2022 break;
72246da4 2023 case DWC3_DEPEVT_EPCMDCMPLT:
ea53b882 2024 dev_vdbg(dwc->dev, "Endpoint Command Complete\n");
72246da4
FB		 2025 break;
2026 }
2027}
2028
2029static void dwc3_disconnect_gadget(struct dwc3 *dwc)
2030{
2031 if (dwc->gadget_driver && dwc->gadget_driver->disconnect) {
2032 spin_unlock(&dwc->lock);
2033 dwc->gadget_driver->disconnect(&dwc->gadget);
2034 spin_lock(&dwc->lock);
2035 }
2036}
2037
2038static void dwc3_stop_active_transfer(struct dwc3 *dwc, u32 epnum)
2039{
2040 struct dwc3_ep *dep;
2041 struct dwc3_gadget_ep_cmd_params params;
2042 u32 cmd;
2043 int ret;
2044
2045 dep = dwc->eps[epnum];
2046
b4996a86 2047 if (!dep->resource_index)
3daf74d7
PA		 2048 return;
2049
57911504
PA		 2050 /*
2051 * NOTICE: We are violating what the Databook says about the
2052 * EndTransfer command. Ideally we would _always_ wait for the
2053 * EndTransfer Command Completion IRQ, but that's causing too
2054 * much trouble synchronizing between us and gadget driver.
2055 *
2056 * We have discussed this with the IP Provider and it was
2057 * suggested to giveback all requests here, but give HW some
2058 * extra time to synchronize with the interconnect. We're using
2059 * an arbitraty 100us delay for that.
2060 *
2061 * Note also that a similar handling was tested by Synopsys
2062 * (thanks a lot Paul) and nothing bad has come out of it.
2063 * In short, what we're doing is:
2064 *
2065 * - Issue EndTransfer WITH CMDIOC bit set
2066 * - Wait 100us
2067 */
2068
3daf74d7
PA		 2069 cmd = DWC3_DEPCMD_ENDTRANSFER;
2070 cmd |= DWC3_DEPCMD_HIPRI_FORCERM | DWC3_DEPCMD_CMDIOC;
b4996a86 2071 cmd |= DWC3_DEPCMD_PARAM(dep->resource_index);
3daf74d7
PA		 2072 memset(&params, 0, sizeof(params));
2073 ret = dwc3_send_gadget_ep_cmd(dwc, dep->number, cmd, &params);
2074 WARN_ON_ONCE(ret);
b4996a86 2075 dep->resource_index = 0;
041d81f4 2076 dep->flags &= ~DWC3_EP_BUSY;
57911504 2077 udelay(100);
72246da4
FB		 2078}
2079
2080static void dwc3_stop_active_transfers(struct dwc3 *dwc)
2081{
2082 u32 epnum;
2083
2084 for (epnum = 2; epnum < DWC3_ENDPOINTS_NUM; epnum++) {
2085 struct dwc3_ep *dep;
2086
2087 dep = dwc->eps[epnum];
6a1e3ef4
FB		 2088 if (!dep)
2089 continue;
2090
72246da4
FB		 2091 if (!(dep->flags & DWC3_EP_ENABLED))
2092 continue;
2093
624407f9 2094 dwc3_remove_requests(dwc, dep);
72246da4
FB		 2095 }
2096}
2097
2098static void dwc3_clear_stall_all_ep(struct dwc3 *dwc)
2099{
2100 u32 epnum;
2101
2102 for (epnum = 1; epnum < DWC3_ENDPOINTS_NUM; epnum++) {
2103 struct dwc3_ep *dep;
2104 struct dwc3_gadget_ep_cmd_params params;
2105 int ret;
2106
2107 dep = dwc->eps[epnum];
6a1e3ef4
FB		 2108 if (!dep)
2109 continue;
72246da4
FB		 2110
2111 if (!(dep->flags & DWC3_EP_STALL))
2112 continue;
2113
2114 dep->flags &= ~DWC3_EP_STALL;
2115
2116 memset(&params, 0, sizeof(params));
2117 ret = dwc3_send_gadget_ep_cmd(dwc, dep->number,
2118 DWC3_DEPCMD_CLEARSTALL, &params);
2119 WARN_ON_ONCE(ret);
2120 }
2121}
2122
2123static void dwc3_gadget_disconnect_interrupt(struct dwc3 *dwc)
2124{
c4430a26
FB		 2125 int reg;
2126
72246da4 2127 dev_vdbg(dwc->dev, "%s\n", __func__);
72246da4
FB		 2128
2129 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2130 reg &= ~DWC3_DCTL_INITU1ENA;
2131 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2132
2133 reg &= ~DWC3_DCTL_INITU2ENA;
2134 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
72246da4 2135
72246da4 2136 dwc3_disconnect_gadget(dwc);
b23c8439 2137 dwc->start_config_issued = false;
72246da4
FB		 2138
2139 dwc->gadget.speed = USB_SPEED_UNKNOWN;
df62df56 2140 dwc->setup_packet_pending = false;
72246da4
FB		 2141}
2142
d7a46a8d 2143static void dwc3_gadget_usb3_phy_suspend(struct dwc3 *dwc, int suspend)
72246da4
FB		 2144{
2145 u32 reg;
2146
2147 reg = dwc3_readl(dwc->regs, DWC3_GUSB3PIPECTL(0));
2148
d7a46a8d 2149 if (suspend)
72246da4 2150 reg |= DWC3_GUSB3PIPECTL_SUSPHY;
d7a46a8d
PZ		 2151 else
2152 reg &= ~DWC3_GUSB3PIPECTL_SUSPHY;
72246da4
FB		 2153
2154 dwc3_writel(dwc->regs, DWC3_GUSB3PIPECTL(0), reg);
2155}
2156
d7a46a8d 2157static void dwc3_gadget_usb2_phy_suspend(struct dwc3 *dwc, int suspend)
72246da4
FB		 2158{
2159 u32 reg;
2160
2161 reg = dwc3_readl(dwc->regs, DWC3_GUSB2PHYCFG(0));
2162
d7a46a8d 2163 if (suspend)
72246da4 2164 reg |= DWC3_GUSB2PHYCFG_SUSPHY;
d7a46a8d
PZ		 2165 else
2166 reg &= ~DWC3_GUSB2PHYCFG_SUSPHY;
72246da4
FB		 2167
2168 dwc3_writel(dwc->regs, DWC3_GUSB2PHYCFG(0), reg);
2169}
2170
2171static void dwc3_gadget_reset_interrupt(struct dwc3 *dwc)
2172{
2173 u32 reg;
2174
2175 dev_vdbg(dwc->dev, "%s\n", __func__);
2176
df62df56
FB		 2177 /*
2178 * WORKAROUND: DWC3 revisions <1.88a have an issue which
2179 * would cause a missing Disconnect Event if there's a
2180 * pending Setup Packet in the FIFO.
2181 *
2182 * There's no suggested workaround on the official Bug
2183 * report, which states that "unless the driver/application
2184 * is doing any special handling of a disconnect event,
2185 * there is no functional issue".
2186 *
2187 * Unfortunately, it turns out that we _do_ some special
2188 * handling of a disconnect event, namely complete all
2189 * pending transfers, notify gadget driver of the
2190 * disconnection, and so on.
2191 *
2192 * Our suggested workaround is to follow the Disconnect
2193 * Event steps here, instead, based on a setup_packet_pending
2194 * flag. Such flag gets set whenever we have a XferNotReady
2195 * event on EP0 and gets cleared on XferComplete for the
2196 * same endpoint.
2197 *
2198 * Refers to:
2199 *
2200 * STAR#9000466709: RTL: Device : Disconnect event not
2201 * generated if setup packet pending in FIFO
2202 */
2203 if (dwc->revision < DWC3_REVISION_188A) {
2204 if (dwc->setup_packet_pending)
2205 dwc3_gadget_disconnect_interrupt(dwc);
2206 }
2207
961906ed 2208 /* after reset -> Default State */
14cd592f 2209 usb_gadget_set_state(&dwc->gadget, USB_STATE_DEFAULT);
961906ed 2210
802fde98
PZ		 2211 /* Recent versions support automatic phy suspend and don't need this */
2212 if (dwc->revision < DWC3_REVISION_194A) {
2213 /* Resume PHYs */
2214 dwc3_gadget_usb2_phy_suspend(dwc, false);
2215 dwc3_gadget_usb3_phy_suspend(dwc, false);
2216 }
72246da4
FB		 2217
2218 if (dwc->gadget.speed != USB_SPEED_UNKNOWN)
2219 dwc3_disconnect_gadget(dwc);
2220
2221 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2222 reg &= ~DWC3_DCTL_TSTCTRL_MASK;
2223 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
3b637367 2224 dwc->test_mode = false;
72246da4
FB		 2225
2226 dwc3_stop_active_transfers(dwc);
2227 dwc3_clear_stall_all_ep(dwc);
b23c8439 2228 dwc->start_config_issued = false;
72246da4
FB		 2229
2230 /* Reset device address to zero */
2231 reg = dwc3_readl(dwc->regs, DWC3_DCFG);
2232 reg &= ~(DWC3_DCFG_DEVADDR_MASK);
2233 dwc3_writel(dwc->regs, DWC3_DCFG, reg);
72246da4
FB		 2234}
2235
2236static void dwc3_update_ram_clk_sel(struct dwc3 *dwc, u32 speed)
2237{
2238 u32 reg;
2239 u32 usb30_clock = DWC3_GCTL_CLK_BUS;
2240
2241 /*
2242 * We change the clock only at SS but I dunno why I would want to do
2243 * this. Maybe it becomes part of the power saving plan.
2244 */
2245
2246 if (speed != DWC3_DSTS_SUPERSPEED)
2247 return;
2248
2249 /*
2250 * RAMClkSel is reset to 0 after USB reset, so it must be reprogrammed
2251 * each time on Connect Done.
2252 */
2253 if (!usb30_clock)
2254 return;
2255
2256 reg = dwc3_readl(dwc->regs, DWC3_GCTL);
2257 reg |= DWC3_GCTL_RAMCLKSEL(usb30_clock);
2258 dwc3_writel(dwc->regs, DWC3_GCTL, reg);
2259}
2260
d7a46a8d 2261static void dwc3_gadget_phy_suspend(struct dwc3 *dwc, u8 speed)
72246da4
FB		 2262{
2263 switch (speed) {
2264 case USB_SPEED_SUPER:
d7a46a8d 2265 dwc3_gadget_usb2_phy_suspend(dwc, true);
72246da4
FB		 2266 break;
2267 case USB_SPEED_HIGH:
2268 case USB_SPEED_FULL:
2269 case USB_SPEED_LOW:
d7a46a8d 2270 dwc3_gadget_usb3_phy_suspend(dwc, true);
72246da4
FB		 2271 break;
2272 }
2273}
2274
2275static void dwc3_gadget_conndone_interrupt(struct dwc3 *dwc)
2276{
72246da4
FB		 2277 struct dwc3_ep *dep;
2278 int ret;
2279 u32 reg;
2280 u8 speed;
2281
2282 dev_vdbg(dwc->dev, "%s\n", __func__);
2283
72246da4
FB		 2284 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
2285 speed = reg & DWC3_DSTS_CONNECTSPD;
2286 dwc->speed = speed;
2287
2288 dwc3_update_ram_clk_sel(dwc, speed);
2289
2290 switch (speed) {
2291 case DWC3_DCFG_SUPERSPEED:
05870c5b
FB		 2292 /*
2293 * WORKAROUND: DWC3 revisions <1.90a have an issue which
2294 * would cause a missing USB3 Reset event.
2295 *
2296 * In such situations, we should force a USB3 Reset
2297 * event by calling our dwc3_gadget_reset_interrupt()
2298 * routine.
2299 *
2300 * Refers to:
2301 *
2302 * STAR#9000483510: RTL: SS : USB3 reset event may
2303 * not be generated always when the link enters poll
2304 */
2305 if (dwc->revision < DWC3_REVISION_190A)
2306 dwc3_gadget_reset_interrupt(dwc);
2307
72246da4
FB		 2308 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
2309 dwc->gadget.ep0->maxpacket = 512;
2310 dwc->gadget.speed = USB_SPEED_SUPER;
2311 break;
2312 case DWC3_DCFG_HIGHSPEED:
2313 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(64);
2314 dwc->gadget.ep0->maxpacket = 64;
2315 dwc->gadget.speed = USB_SPEED_HIGH;
2316 break;
2317 case DWC3_DCFG_FULLSPEED2:
2318 case DWC3_DCFG_FULLSPEED1:
2319 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(64);
2320 dwc->gadget.ep0->maxpacket = 64;
2321 dwc->gadget.speed = USB_SPEED_FULL;
2322 break;
2323 case DWC3_DCFG_LOWSPEED:
2324 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(8);
2325 dwc->gadget.ep0->maxpacket = 8;
2326 dwc->gadget.speed = USB_SPEED_LOW;
2327 break;
2328 }
2329
2b758350
PA		 2330 /* Enable USB2 LPM Capability */
2331
2332 if ((dwc->revision > DWC3_REVISION_194A)
2333 && (speed != DWC3_DCFG_SUPERSPEED)) {
2334 reg = dwc3_readl(dwc->regs, DWC3_DCFG);
2335 reg |= DWC3_DCFG_LPM_CAP;
2336 dwc3_writel(dwc->regs, DWC3_DCFG, reg);
2337
2338 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2339 reg &= ~(DWC3_DCTL_HIRD_THRES_MASK | DWC3_DCTL_L1_HIBER_EN);
2340
1a947746
FB		 2341 /*
2342 * TODO: This should be configurable. For now using
2343 * maximum allowed HIRD threshold value of 0b1100
2344 */
2345 reg |= DWC3_DCTL_HIRD_THRES(12);
2b758350
PA		 2346
2347 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2348 }
2349
802fde98
PZ		 2350 /* Recent versions support automatic phy suspend and don't need this */
2351 if (dwc->revision < DWC3_REVISION_194A) {
2352 /* Suspend unneeded PHY */
2353 dwc3_gadget_phy_suspend(dwc, dwc->gadget.speed);
2354 }
72246da4
FB		 2355
2356 dep = dwc->eps[0];
4b345c9a 2357 ret = __dwc3_gadget_ep_enable(dep, &dwc3_gadget_ep0_desc, NULL, true);
72246da4
FB		 2358 if (ret) {
2359 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
2360 return;
2361 }
2362
2363 dep = dwc->eps[1];
4b345c9a 2364 ret = __dwc3_gadget_ep_enable(dep, &dwc3_gadget_ep0_desc, NULL, true);
72246da4
FB		 2365 if (ret) {
2366 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
2367 return;
2368 }
2369
2370 /*
2371 * Configure PHY via GUSB3PIPECTLn if required.
2372 *
2373 * Update GTXFIFOSIZn
2374 *
2375 * In both cases reset values should be sufficient.
2376 */
2377}
2378
2379static void dwc3_gadget_wakeup_interrupt(struct dwc3 *dwc)
2380{
2381 dev_vdbg(dwc->dev, "%s\n", __func__);
2382
2383 /*
2384 * TODO take core out of low power mode when that's
2385 * implemented.
2386 */
2387
2388 dwc->gadget_driver->resume(&dwc->gadget);
2389}
2390
2391static void dwc3_gadget_linksts_change_interrupt(struct dwc3 *dwc,
2392 unsigned int evtinfo)
2393{
fae2b904 2394 enum dwc3_link_state next = evtinfo & DWC3_LINK_STATE_MASK;
0b0cc1cd
FB		 2395 unsigned int pwropt;
2396
2397 /*
2398 * WORKAROUND: DWC3 < 2.50a have an issue when configured without
2399 * Hibernation mode enabled which would show up when device detects
2400 * host-initiated U3 exit.
2401 *
2402 * In that case, device will generate a Link State Change Interrupt
2403 * from U3 to RESUME which is only necessary if Hibernation is
2404 * configured in.
2405 *
2406 * There are no functional changes due to such spurious event and we
2407 * just need to ignore it.
2408 *
2409 * Refers to:
2410 *
2411 * STAR#9000570034 RTL: SS Resume event generated in non-Hibernation
2412 * operational mode
2413 */
2414 pwropt = DWC3_GHWPARAMS1_EN_PWROPT(dwc->hwparams.hwparams1);
2415 if ((dwc->revision < DWC3_REVISION_250A) &&
2416 (pwropt != DWC3_GHWPARAMS1_EN_PWROPT_HIB)) {
2417 if ((dwc->link_state == DWC3_LINK_STATE_U3) &&
2418 (next == DWC3_LINK_STATE_RESUME)) {
2419 dev_vdbg(dwc->dev, "ignoring transition U3 -> Resume\n");
2420 return;
2421 }
2422 }
fae2b904
FB		 2423
2424 /*
2425 * WORKAROUND: DWC3 Revisions <1.83a have an issue which, depending
2426 * on the link partner, the USB session might do multiple entry/exit
2427 * of low power states before a transfer takes place.
2428 *
2429 * Due to this problem, we might experience lower throughput. The
2430 * suggested workaround is to disable DCTL[12:9] bits if we're
2431 * transitioning from U1/U2 to U0 and enable those bits again
2432 * after a transfer completes and there are no pending transfers
2433 * on any of the enabled endpoints.
2434 *
2435 * This is the first half of that workaround.
2436 *
2437 * Refers to:
2438 *
2439 * STAR#9000446952: RTL: Device SS : if U1/U2 ->U0 takes >128us
2440 * core send LGO_Ux entering U0
2441 */
2442 if (dwc->revision < DWC3_REVISION_183A) {
2443 if (next == DWC3_LINK_STATE_U0) {
2444 u32 u1u2;
2445 u32 reg;
2446
2447 switch (dwc->link_state) {
2448 case DWC3_LINK_STATE_U1:
2449 case DWC3_LINK_STATE_U2:
2450 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2451 u1u2 = reg & (DWC3_DCTL_INITU2ENA
2452 | DWC3_DCTL_ACCEPTU2ENA
2453 | DWC3_DCTL_INITU1ENA
2454 | DWC3_DCTL_ACCEPTU1ENA);
2455
2456 if (!dwc->u1u2)
2457 dwc->u1u2 = reg & u1u2;
2458
2459 reg &= ~u1u2;
2460
2461 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2462 break;
2463 default:
2464 /* do nothing */
2465 break;
2466 }
2467 }
2468 }
2469
2470 dwc->link_state = next;
019ac832
FB		 2471
2472 dev_vdbg(dwc->dev, "%s link %d\n", __func__, dwc->link_state);
72246da4
FB		 2473}
2474
2475static void dwc3_gadget_interrupt(struct dwc3 *dwc,
2476 const struct dwc3_event_devt *event)
2477{
2478 switch (event->type) {
2479 case DWC3_DEVICE_EVENT_DISCONNECT:
2480 dwc3_gadget_disconnect_interrupt(dwc);
2481 break;
2482 case DWC3_DEVICE_EVENT_RESET:
2483 dwc3_gadget_reset_interrupt(dwc);
2484 break;
2485 case DWC3_DEVICE_EVENT_CONNECT_DONE:
2486 dwc3_gadget_conndone_interrupt(dwc);
2487 break;
2488 case DWC3_DEVICE_EVENT_WAKEUP:
2489 dwc3_gadget_wakeup_interrupt(dwc);
2490 break;
2491 case DWC3_DEVICE_EVENT_LINK_STATUS_CHANGE:
2492 dwc3_gadget_linksts_change_interrupt(dwc, event->event_info);
2493 break;
2494 case DWC3_DEVICE_EVENT_EOPF:
2495 dev_vdbg(dwc->dev, "End of Periodic Frame\n");
2496 break;
2497 case DWC3_DEVICE_EVENT_SOF:
2498 dev_vdbg(dwc->dev, "Start of Periodic Frame\n");
2499 break;
2500 case DWC3_DEVICE_EVENT_ERRATIC_ERROR:
2501 dev_vdbg(dwc->dev, "Erratic Error\n");
2502 break;
2503 case DWC3_DEVICE_EVENT_CMD_CMPL:
2504 dev_vdbg(dwc->dev, "Command Complete\n");
2505 break;
2506 case DWC3_DEVICE_EVENT_OVERFLOW:
2507 dev_vdbg(dwc->dev, "Overflow\n");
2508 break;
2509 default:
2510 dev_dbg(dwc->dev, "UNKNOWN IRQ %d\n", event->type);
2511 }
2512}
2513
2514static void dwc3_process_event_entry(struct dwc3 *dwc,
2515 const union dwc3_event *event)
2516{
2517 /* Endpoint IRQ, handle it and return early */
2518 if (event->type.is_devspec == 0) {
2519 /* depevt */
2520 return dwc3_endpoint_interrupt(dwc, &event->depevt);
2521 }
2522
2523 switch (event->type.type) {
2524 case DWC3_EVENT_TYPE_DEV:
2525 dwc3_gadget_interrupt(dwc, &event->devt);
2526 break;
2527 /* REVISIT what to do with Carkit and I2C events ? */
2528 default:
2529 dev_err(dwc->dev, "UNKNOWN IRQ type %d\n", event->raw);
2530 }
2531}
2532
b15a762f
FB		 2533static irqreturn_t dwc3_thread_interrupt(int irq, void *_dwc)
2534{
2535 struct dwc3 *dwc = _dwc;
2536 unsigned long flags;
2537 irqreturn_t ret = IRQ_NONE;
2538 int i;
2539
2540 spin_lock_irqsave(&dwc->lock, flags);
2541
2542 for (i = 0; i < dwc->num_event_buffers; i++) {
2543 struct dwc3_event_buffer *evt;
2544 int left;
2545
2546 evt = dwc->ev_buffs[i];
2547 left = evt->count;
2548
2549 if (!(evt->flags & DWC3_EVENT_PENDING))
2550 continue;
2551
2552 while (left > 0) {
2553 union dwc3_event event;
2554
2555 event.raw = *(u32 *) (evt->buf + evt->lpos);
2556
2557 dwc3_process_event_entry(dwc, &event);
2558
2559 /*
2560 * FIXME we wrap around correctly to the next entry as
2561 * almost all entries are 4 bytes in size. There is one
2562 * entry which has 12 bytes which is a regular entry
2563 * followed by 8 bytes data. ATM I don't know how
2564 * things are organized if we get next to the a
2565 * boundary so I worry about that once we try to handle
2566 * that.
2567 */
2568 evt->lpos = (evt->lpos + 4) % DWC3_EVENT_BUFFERS_SIZE;
2569 left -= 4;
2570
2571 dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(i), 4);
2572 }
2573
2574 evt->count = 0;
2575 evt->flags &= ~DWC3_EVENT_PENDING;
2576 ret = IRQ_HANDLED;
2577 }
2578
2579 spin_unlock_irqrestore(&dwc->lock, flags);
2580
2581 return ret;
2582}
2583
72246da4
FB		 2584static irqreturn_t dwc3_process_event_buf(struct dwc3 *dwc, u32 buf)
2585{
2586 struct dwc3_event_buffer *evt;
72246da4
FB		 2587 u32 count;
2588
b15a762f
FB		 2589 evt = dwc->ev_buffs[buf];
2590
72246da4
FB		 2591 count = dwc3_readl(dwc->regs, DWC3_GEVNTCOUNT(buf));
2592 count &= DWC3_GEVNTCOUNT_MASK;
2593 if (!count)
2594 return IRQ_NONE;
2595
b15a762f
FB		 2596 evt->count = count;
2597 evt->flags |= DWC3_EVENT_PENDING;
72246da4 2598
b15a762f 2599 return IRQ_WAKE_THREAD;
72246da4
FB		 2600}
2601
2602static irqreturn_t dwc3_interrupt(int irq, void *_dwc)
2603{
2604 struct dwc3 *dwc = _dwc;
2605 int i;
2606 irqreturn_t ret = IRQ_NONE;
2607
2608 spin_lock(&dwc->lock);
2609
9f622b2a 2610 for (i = 0; i < dwc->num_event_buffers; i++) {
72246da4
FB		 2611 irqreturn_t status;
2612
2613 status = dwc3_process_event_buf(dwc, i);
b15a762f 2614 if (status == IRQ_WAKE_THREAD)
72246da4
FB		 2615 ret = status;
2616 }
2617
2618 spin_unlock(&dwc->lock);
2619
2620 return ret;
2621}
2622
2623/**
2624 * dwc3_gadget_init - Initializes gadget related registers
1d046793 2625 * @dwc: pointer to our controller context structure
72246da4
FB		 2626 *
2627 * Returns 0 on success otherwise negative errno.
2628 */
41ac7b3a 2629int dwc3_gadget_init(struct dwc3 *dwc)
72246da4
FB		 2630{
2631 u32 reg;
2632 int ret;
72246da4
FB		 2633
2634 dwc->ctrl_req = dma_alloc_coherent(dwc->dev, sizeof(*dwc->ctrl_req),
2635 &dwc->ctrl_req_addr, GFP_KERNEL);
2636 if (!dwc->ctrl_req) {
2637 dev_err(dwc->dev, "failed to allocate ctrl request\n");
2638 ret = -ENOMEM;
2639 goto err0;
2640 }
2641
2642 dwc->ep0_trb = dma_alloc_coherent(dwc->dev, sizeof(*dwc->ep0_trb),
2643 &dwc->ep0_trb_addr, GFP_KERNEL);
2644 if (!dwc->ep0_trb) {
2645 dev_err(dwc->dev, "failed to allocate ep0 trb\n");
2646 ret = -ENOMEM;
2647 goto err1;
2648 }
2649
3ef35faf 2650 dwc->setup_buf = kzalloc(DWC3_EP0_BOUNCE_SIZE, GFP_KERNEL);
72246da4
FB		 2651 if (!dwc->setup_buf) {
2652 dev_err(dwc->dev, "failed to allocate setup buffer\n");
2653 ret = -ENOMEM;
2654 goto err2;
2655 }
2656
5812b1c2 2657 dwc->ep0_bounce = dma_alloc_coherent(dwc->dev,
3ef35faf
FB		 2658 DWC3_EP0_BOUNCE_SIZE, &dwc->ep0_bounce_addr,
2659 GFP_KERNEL);
5812b1c2
FB		 2660 if (!dwc->ep0_bounce) {
2661 dev_err(dwc->dev, "failed to allocate ep0 bounce buffer\n");
2662 ret = -ENOMEM;
2663 goto err3;
2664 }
2665
72246da4 2666 dwc->gadget.ops = &dwc3_gadget_ops;
d327ab5b 2667 dwc->gadget.max_speed = USB_SPEED_SUPER;
72246da4 2668 dwc->gadget.speed = USB_SPEED_UNKNOWN;
eeb720fb 2669 dwc->gadget.sg_supported = true;
72246da4
FB		 2670 dwc->gadget.name = "dwc3-gadget";
2671
2672 /*
2673 * REVISIT: Here we should clear all pending IRQs to be
2674 * sure we're starting from a well known location.
2675 */
2676
2677 ret = dwc3_gadget_init_endpoints(dwc);
2678 if (ret)
5812b1c2 2679 goto err4;
72246da4 2680
e6a3b5e2
SAS		 2681 reg = dwc3_readl(dwc->regs, DWC3_DCFG);
2682 reg |= DWC3_DCFG_LPM_CAP;
2683 dwc3_writel(dwc->regs, DWC3_DCFG, reg);
2684
8698e2ac 2685 /* Enable USB2 LPM and automatic phy suspend only on recent versions */
802fde98 2686 if (dwc->revision >= DWC3_REVISION_194A) {
dcae3573
PA		 2687 dwc3_gadget_usb2_phy_suspend(dwc, false);
2688 dwc3_gadget_usb3_phy_suspend(dwc, false);
802fde98
PZ		 2689 }
2690
72246da4
FB		 2691 ret = usb_add_gadget_udc(dwc->dev, &dwc->gadget);
2692 if (ret) {
2693 dev_err(dwc->dev, "failed to register udc\n");
8698e2ac 2694 goto err5;
72246da4
FB		 2695 }
2696
2697 return 0;
2698
5812b1c2 2699err5:
72246da4
FB		 2700 dwc3_gadget_free_endpoints(dwc);
2701
5812b1c2 2702err4:
3ef35faf
FB		 2703 dma_free_coherent(dwc->dev, DWC3_EP0_BOUNCE_SIZE,
2704 dwc->ep0_bounce, dwc->ep0_bounce_addr);
5812b1c2 2705
72246da4 2706err3:
0fc9a1be 2707 kfree(dwc->setup_buf);
72246da4
FB		 2708
2709err2:
2710 dma_free_coherent(dwc->dev, sizeof(*dwc->ep0_trb),
2711 dwc->ep0_trb, dwc->ep0_trb_addr);
2712
2713err1:
2714 dma_free_coherent(dwc->dev, sizeof(*dwc->ctrl_req),
2715 dwc->ctrl_req, dwc->ctrl_req_addr);
2716
2717err0:
2718 return ret;
2719}
2720
7415f17c
FB		 2721/* -------------------------------------------------------------------------- */
2722
72246da4
FB		 2723void dwc3_gadget_exit(struct dwc3 *dwc)
2724{
72246da4 2725 usb_del_gadget_udc(&dwc->gadget);
72246da4 2726
72246da4
FB		 2727 dwc3_gadget_free_endpoints(dwc);
2728
3ef35faf
FB		 2729 dma_free_coherent(dwc->dev, DWC3_EP0_BOUNCE_SIZE,
2730 dwc->ep0_bounce, dwc->ep0_bounce_addr);
5812b1c2 2731
0fc9a1be 2732 kfree(dwc->setup_buf);
72246da4
FB		 2733
2734 dma_free_coherent(dwc->dev, sizeof(*dwc->ep0_trb),
2735 dwc->ep0_trb, dwc->ep0_trb_addr);
2736
2737 dma_free_coherent(dwc->dev, sizeof(*dwc->ctrl_req),
2738 dwc->ctrl_req, dwc->ctrl_req_addr);
72246da4 2739}
7415f17c
FB		 2740
2741int dwc3_gadget_prepare(struct dwc3 *dwc)
2742{
2743 if (dwc->pullups_connected)
2744 dwc3_gadget_disable_irq(dwc);
2745
2746 return 0;
2747}
2748
2749void dwc3_gadget_complete(struct dwc3 *dwc)
2750{
2751 if (dwc->pullups_connected) {
2752 dwc3_gadget_enable_irq(dwc);
2753 dwc3_gadget_run_stop(dwc, true);
2754 }
2755}
2756
2757int dwc3_gadget_suspend(struct dwc3 *dwc)
2758{
2759 __dwc3_gadget_ep_disable(dwc->eps[0]);
2760 __dwc3_gadget_ep_disable(dwc->eps[1]);
2761
2762 dwc->dcfg = dwc3_readl(dwc->regs, DWC3_DCFG);
2763
2764 return 0;
2765}
2766
2767int dwc3_gadget_resume(struct dwc3 *dwc)
2768{
2769 struct dwc3_ep *dep;
2770 int ret;
2771
2772 /* Start with SuperSpeed Default */
2773 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
2774
2775 dep = dwc->eps[0];
2776 ret = __dwc3_gadget_ep_enable(dep, &dwc3_gadget_ep0_desc, NULL, false);
2777 if (ret)
2778 goto err0;
2779
2780 dep = dwc->eps[1];
2781 ret = __dwc3_gadget_ep_enable(dep, &dwc3_gadget_ep0_desc, NULL, false);
2782 if (ret)
2783 goto err1;
2784
2785 /* begin to receive SETUP packets */
2786 dwc->ep0state = EP0_SETUP_PHASE;
2787 dwc3_ep0_out_start(dwc);
2788
2789 dwc3_writel(dwc->regs, DWC3_DCFG, dwc->dcfg);
2790
2791 return 0;
2792
2793err1:
2794 __dwc3_gadget_ep_disable(dwc->eps[0]);
2795
2796err0:
2797 return ret;
2798}
kernel source for telekom puls (alcatel/mediatek)