2 * libahci.c - Common AHCI SATA low-level routines
4 * Maintained by: Tejun Heo <tj@kernel.org>
5 * Please ALWAYS copy linux-ide@vger.kernel.org
8 * Copyright 2004-2005 Red Hat, Inc.
11 * This program is free software; you can redistribute it and/or modify
12 * it under the terms of the GNU General Public License as published by
13 * the Free Software Foundation; either version 2, or (at your option)
16 * This program is distributed in the hope that it will be useful,
17 * but WITHOUT ANY WARRANTY; without even the implied warranty of
18 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19 * GNU General Public License for more details.
21 * You should have received a copy of the GNU General Public License
22 * along with this program; see the file COPYING. If not, write to
23 * the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
26 * libata documentation is available via 'make {ps|pdf}docs',
27 * as Documentation/driver-api/libata.rst
29 * AHCI hardware documentation:
30 * http://www.intel.com/technology/serialata/pdf/rev1_0.pdf
31 * http://www.intel.com/technology/serialata/pdf/rev1_1.pdf
35 #include <linux/kernel.h>
36 #include <linux/gfp.h>
37 #include <linux/module.h>
38 #include <linux/nospec.h>
39 #include <linux/blkdev.h>
40 #include <linux/delay.h>
41 #include <linux/interrupt.h>
42 #include <linux/dma-mapping.h>
43 #include <linux/device.h>
44 #include <scsi/scsi_host.h>
45 #include <scsi/scsi_cmnd.h>
46 #include <linux/libata.h>
47 #include <linux/pci.h>
51 static int ahci_skip_host_reset
;
53 EXPORT_SYMBOL_GPL(ahci_ignore_sss
);
55 module_param_named(skip_host_reset
, ahci_skip_host_reset
, int, 0444);
56 MODULE_PARM_DESC(skip_host_reset
, "skip global host reset (0=don't skip, 1=skip)");
58 module_param_named(ignore_sss
, ahci_ignore_sss
, int, 0444);
59 MODULE_PARM_DESC(ignore_sss
, "Ignore staggered spinup flag (0=don't ignore, 1=ignore)");
61 static int ahci_set_lpm(struct ata_link
*link
, enum ata_lpm_policy policy
,
63 static ssize_t
ahci_led_show(struct ata_port
*ap
, char *buf
);
64 static ssize_t
ahci_led_store(struct ata_port
*ap
, const char *buf
,
66 static ssize_t
ahci_transmit_led_message(struct ata_port
*ap
, u32 state
,
71 static int ahci_scr_read(struct ata_link
*link
, unsigned int sc_reg
, u32
*val
);
72 static int ahci_scr_write(struct ata_link
*link
, unsigned int sc_reg
, u32 val
);
73 static bool ahci_qc_fill_rtf(struct ata_queued_cmd
*qc
);
74 static int ahci_port_start(struct ata_port
*ap
);
75 static void ahci_port_stop(struct ata_port
*ap
);
76 static void ahci_qc_prep(struct ata_queued_cmd
*qc
);
77 static int ahci_pmp_qc_defer(struct ata_queued_cmd
*qc
);
78 static void ahci_freeze(struct ata_port
*ap
);
79 static void ahci_thaw(struct ata_port
*ap
);
80 static void ahci_set_aggressive_devslp(struct ata_port
*ap
, bool sleep
);
81 static void ahci_enable_fbs(struct ata_port
*ap
);
82 static void ahci_disable_fbs(struct ata_port
*ap
);
83 static void ahci_pmp_attach(struct ata_port
*ap
);
84 static void ahci_pmp_detach(struct ata_port
*ap
);
85 static int ahci_softreset(struct ata_link
*link
, unsigned int *class,
86 unsigned long deadline
);
87 static int ahci_pmp_retry_softreset(struct ata_link
*link
, unsigned int *class,
88 unsigned long deadline
);
89 static int ahci_hardreset(struct ata_link
*link
, unsigned int *class,
90 unsigned long deadline
);
91 static void ahci_postreset(struct ata_link
*link
, unsigned int *class);
92 static void ahci_post_internal_cmd(struct ata_queued_cmd
*qc
);
93 static void ahci_dev_config(struct ata_device
*dev
);
95 static int ahci_port_suspend(struct ata_port
*ap
, pm_message_t mesg
);
97 static ssize_t
ahci_activity_show(struct ata_device
*dev
, char *buf
);
98 static ssize_t
ahci_activity_store(struct ata_device
*dev
,
99 enum sw_activity val
);
100 static void ahci_init_sw_activity(struct ata_link
*link
);
102 static ssize_t
ahci_show_host_caps(struct device
*dev
,
103 struct device_attribute
*attr
, char *buf
);
104 static ssize_t
ahci_show_host_cap2(struct device
*dev
,
105 struct device_attribute
*attr
, char *buf
);
106 static ssize_t
ahci_show_host_version(struct device
*dev
,
107 struct device_attribute
*attr
, char *buf
);
108 static ssize_t
ahci_show_port_cmd(struct device
*dev
,
109 struct device_attribute
*attr
, char *buf
);
110 static ssize_t
ahci_read_em_buffer(struct device
*dev
,
111 struct device_attribute
*attr
, char *buf
);
112 static ssize_t
ahci_store_em_buffer(struct device
*dev
,
113 struct device_attribute
*attr
,
114 const char *buf
, size_t size
);
115 static ssize_t
ahci_show_em_supported(struct device
*dev
,
116 struct device_attribute
*attr
, char *buf
);
117 static irqreturn_t
ahci_single_level_irq_intr(int irq
, void *dev_instance
);
119 static DEVICE_ATTR(ahci_host_caps
, S_IRUGO
, ahci_show_host_caps
, NULL
);
120 static DEVICE_ATTR(ahci_host_cap2
, S_IRUGO
, ahci_show_host_cap2
, NULL
);
121 static DEVICE_ATTR(ahci_host_version
, S_IRUGO
, ahci_show_host_version
, NULL
);
122 static DEVICE_ATTR(ahci_port_cmd
, S_IRUGO
, ahci_show_port_cmd
, NULL
);
123 static DEVICE_ATTR(em_buffer
, S_IWUSR
| S_IRUGO
,
124 ahci_read_em_buffer
, ahci_store_em_buffer
);
125 static DEVICE_ATTR(em_message_supported
, S_IRUGO
, ahci_show_em_supported
, NULL
);
127 struct device_attribute
*ahci_shost_attrs
[] = {
128 &dev_attr_link_power_management_policy
,
129 &dev_attr_em_message_type
,
130 &dev_attr_em_message
,
131 &dev_attr_ahci_host_caps
,
132 &dev_attr_ahci_host_cap2
,
133 &dev_attr_ahci_host_version
,
134 &dev_attr_ahci_port_cmd
,
136 &dev_attr_em_message_supported
,
139 EXPORT_SYMBOL_GPL(ahci_shost_attrs
);
141 struct device_attribute
*ahci_sdev_attrs
[] = {
142 &dev_attr_sw_activity
,
143 &dev_attr_unload_heads
,
144 &dev_attr_ncq_prio_enable
,
147 EXPORT_SYMBOL_GPL(ahci_sdev_attrs
);
149 struct ata_port_operations ahci_ops
= {
150 .inherits
= &sata_pmp_port_ops
,
152 .qc_defer
= ahci_pmp_qc_defer
,
153 .qc_prep
= ahci_qc_prep
,
154 .qc_issue
= ahci_qc_issue
,
155 .qc_fill_rtf
= ahci_qc_fill_rtf
,
157 .freeze
= ahci_freeze
,
159 .softreset
= ahci_softreset
,
160 .hardreset
= ahci_hardreset
,
161 .postreset
= ahci_postreset
,
162 .pmp_softreset
= ahci_softreset
,
163 .error_handler
= ahci_error_handler
,
164 .post_internal_cmd
= ahci_post_internal_cmd
,
165 .dev_config
= ahci_dev_config
,
167 .scr_read
= ahci_scr_read
,
168 .scr_write
= ahci_scr_write
,
169 .pmp_attach
= ahci_pmp_attach
,
170 .pmp_detach
= ahci_pmp_detach
,
172 .set_lpm
= ahci_set_lpm
,
173 .em_show
= ahci_led_show
,
174 .em_store
= ahci_led_store
,
175 .sw_activity_show
= ahci_activity_show
,
176 .sw_activity_store
= ahci_activity_store
,
177 .transmit_led_message
= ahci_transmit_led_message
,
179 .port_suspend
= ahci_port_suspend
,
180 .port_resume
= ahci_port_resume
,
182 .port_start
= ahci_port_start
,
183 .port_stop
= ahci_port_stop
,
185 EXPORT_SYMBOL_GPL(ahci_ops
);
187 struct ata_port_operations ahci_pmp_retry_srst_ops
= {
188 .inherits
= &ahci_ops
,
189 .softreset
= ahci_pmp_retry_softreset
,
191 EXPORT_SYMBOL_GPL(ahci_pmp_retry_srst_ops
);
193 static bool ahci_em_messages __read_mostly
= true;
194 EXPORT_SYMBOL_GPL(ahci_em_messages
);
195 module_param(ahci_em_messages
, bool, 0444);
196 /* add other LED protocol types when they become supported */
197 MODULE_PARM_DESC(ahci_em_messages
,
198 "AHCI Enclosure Management Message control (0 = off, 1 = on)");
200 /* device sleep idle timeout in ms */
201 static int devslp_idle_timeout __read_mostly
= 1000;
202 module_param(devslp_idle_timeout
, int, 0644);
203 MODULE_PARM_DESC(devslp_idle_timeout
, "device sleep idle timeout");
205 static void ahci_enable_ahci(void __iomem
*mmio
)
210 /* turn on AHCI_EN */
211 tmp
= readl(mmio
+ HOST_CTL
);
212 if (tmp
& HOST_AHCI_EN
)
215 /* Some controllers need AHCI_EN to be written multiple times.
216 * Try a few times before giving up.
218 for (i
= 0; i
< 5; i
++) {
220 writel(tmp
, mmio
+ HOST_CTL
);
221 tmp
= readl(mmio
+ HOST_CTL
); /* flush && sanity check */
222 if (tmp
& HOST_AHCI_EN
)
231 * ahci_rpm_get_port - Make sure the port is powered on
232 * @ap: Port to power on
234 * Whenever there is need to access the AHCI host registers outside of
235 * normal execution paths, call this function to make sure the host is
236 * actually powered on.
238 static int ahci_rpm_get_port(struct ata_port
*ap
)
240 return pm_runtime_get_sync(ap
->dev
);
244 * ahci_rpm_put_port - Undoes ahci_rpm_get_port()
245 * @ap: Port to power down
247 * Undoes ahci_rpm_get_port() and possibly powers down the AHCI host
248 * if it has no more active users.
250 static void ahci_rpm_put_port(struct ata_port
*ap
)
252 pm_runtime_put(ap
->dev
);
255 static ssize_t
ahci_show_host_caps(struct device
*dev
,
256 struct device_attribute
*attr
, char *buf
)
258 struct Scsi_Host
*shost
= class_to_shost(dev
);
259 struct ata_port
*ap
= ata_shost_to_port(shost
);
260 struct ahci_host_priv
*hpriv
= ap
->host
->private_data
;
262 return sprintf(buf
, "%x\n", hpriv
->cap
);
265 static ssize_t
ahci_show_host_cap2(struct device
*dev
,
266 struct device_attribute
*attr
, char *buf
)
268 struct Scsi_Host
*shost
= class_to_shost(dev
);
269 struct ata_port
*ap
= ata_shost_to_port(shost
);
270 struct ahci_host_priv
*hpriv
= ap
->host
->private_data
;
272 return sprintf(buf
, "%x\n", hpriv
->cap2
);
275 static ssize_t
ahci_show_host_version(struct device
*dev
,
276 struct device_attribute
*attr
, char *buf
)
278 struct Scsi_Host
*shost
= class_to_shost(dev
);
279 struct ata_port
*ap
= ata_shost_to_port(shost
);
280 struct ahci_host_priv
*hpriv
= ap
->host
->private_data
;
282 return sprintf(buf
, "%x\n", hpriv
->version
);
285 static ssize_t
ahci_show_port_cmd(struct device
*dev
,
286 struct device_attribute
*attr
, char *buf
)
288 struct Scsi_Host
*shost
= class_to_shost(dev
);
289 struct ata_port
*ap
= ata_shost_to_port(shost
);
290 void __iomem
*port_mmio
= ahci_port_base(ap
);
293 ahci_rpm_get_port(ap
);
294 ret
= sprintf(buf
, "%x\n", readl(port_mmio
+ PORT_CMD
));
295 ahci_rpm_put_port(ap
);
300 static ssize_t
ahci_read_em_buffer(struct device
*dev
,
301 struct device_attribute
*attr
, char *buf
)
303 struct Scsi_Host
*shost
= class_to_shost(dev
);
304 struct ata_port
*ap
= ata_shost_to_port(shost
);
305 struct ahci_host_priv
*hpriv
= ap
->host
->private_data
;
306 void __iomem
*mmio
= hpriv
->mmio
;
307 void __iomem
*em_mmio
= mmio
+ hpriv
->em_loc
;
313 ahci_rpm_get_port(ap
);
314 spin_lock_irqsave(ap
->lock
, flags
);
316 em_ctl
= readl(mmio
+ HOST_EM_CTL
);
317 if (!(ap
->flags
& ATA_FLAG_EM
) || em_ctl
& EM_CTL_XMT
||
318 !(hpriv
->em_msg_type
& EM_MSG_TYPE_SGPIO
)) {
319 spin_unlock_irqrestore(ap
->lock
, flags
);
320 ahci_rpm_put_port(ap
);
324 if (!(em_ctl
& EM_CTL_MR
)) {
325 spin_unlock_irqrestore(ap
->lock
, flags
);
326 ahci_rpm_put_port(ap
);
330 if (!(em_ctl
& EM_CTL_SMB
))
331 em_mmio
+= hpriv
->em_buf_sz
;
333 count
= hpriv
->em_buf_sz
;
335 /* the count should not be larger than PAGE_SIZE */
336 if (count
> PAGE_SIZE
) {
337 if (printk_ratelimit())
339 "EM read buffer size too large: "
340 "buffer size %u, page size %lu\n",
341 hpriv
->em_buf_sz
, PAGE_SIZE
);
345 for (i
= 0; i
< count
; i
+= 4) {
346 msg
= readl(em_mmio
+ i
);
348 buf
[i
+ 1] = (msg
>> 8) & 0xff;
349 buf
[i
+ 2] = (msg
>> 16) & 0xff;
350 buf
[i
+ 3] = (msg
>> 24) & 0xff;
353 spin_unlock_irqrestore(ap
->lock
, flags
);
354 ahci_rpm_put_port(ap
);
359 static ssize_t
ahci_store_em_buffer(struct device
*dev
,
360 struct device_attribute
*attr
,
361 const char *buf
, size_t size
)
363 struct Scsi_Host
*shost
= class_to_shost(dev
);
364 struct ata_port
*ap
= ata_shost_to_port(shost
);
365 struct ahci_host_priv
*hpriv
= ap
->host
->private_data
;
366 void __iomem
*mmio
= hpriv
->mmio
;
367 void __iomem
*em_mmio
= mmio
+ hpriv
->em_loc
;
368 const unsigned char *msg_buf
= buf
;
373 /* check size validity */
374 if (!(ap
->flags
& ATA_FLAG_EM
) ||
375 !(hpriv
->em_msg_type
& EM_MSG_TYPE_SGPIO
) ||
376 size
% 4 || size
> hpriv
->em_buf_sz
)
379 ahci_rpm_get_port(ap
);
380 spin_lock_irqsave(ap
->lock
, flags
);
382 em_ctl
= readl(mmio
+ HOST_EM_CTL
);
383 if (em_ctl
& EM_CTL_TM
) {
384 spin_unlock_irqrestore(ap
->lock
, flags
);
385 ahci_rpm_put_port(ap
);
389 for (i
= 0; i
< size
; i
+= 4) {
390 msg
= msg_buf
[i
] | msg_buf
[i
+ 1] << 8 |
391 msg_buf
[i
+ 2] << 16 | msg_buf
[i
+ 3] << 24;
392 writel(msg
, em_mmio
+ i
);
395 writel(em_ctl
| EM_CTL_TM
, mmio
+ HOST_EM_CTL
);
397 spin_unlock_irqrestore(ap
->lock
, flags
);
398 ahci_rpm_put_port(ap
);
403 static ssize_t
ahci_show_em_supported(struct device
*dev
,
404 struct device_attribute
*attr
, char *buf
)
406 struct Scsi_Host
*shost
= class_to_shost(dev
);
407 struct ata_port
*ap
= ata_shost_to_port(shost
);
408 struct ahci_host_priv
*hpriv
= ap
->host
->private_data
;
409 void __iomem
*mmio
= hpriv
->mmio
;
412 ahci_rpm_get_port(ap
);
413 em_ctl
= readl(mmio
+ HOST_EM_CTL
);
414 ahci_rpm_put_port(ap
);
416 return sprintf(buf
, "%s%s%s%s\n",
417 em_ctl
& EM_CTL_LED
? "led " : "",
418 em_ctl
& EM_CTL_SAFTE
? "saf-te " : "",
419 em_ctl
& EM_CTL_SES
? "ses-2 " : "",
420 em_ctl
& EM_CTL_SGPIO
? "sgpio " : "");
424 * ahci_save_initial_config - Save and fixup initial config values
425 * @dev: target AHCI device
426 * @hpriv: host private area to store config values
428 * Some registers containing configuration info might be setup by
429 * BIOS and might be cleared on reset. This function saves the
430 * initial values of those registers into @hpriv such that they
431 * can be restored after controller reset.
433 * If inconsistent, config values are fixed up by this function.
435 * If it is not set already this function sets hpriv->start_engine to
441 void ahci_save_initial_config(struct device
*dev
, struct ahci_host_priv
*hpriv
)
443 void __iomem
*mmio
= hpriv
->mmio
;
444 u32 cap
, cap2
, vers
, port_map
;
447 /* make sure AHCI mode is enabled before accessing CAP */
448 ahci_enable_ahci(mmio
);
450 /* Values prefixed with saved_ are written back to host after
451 * reset. Values without are used for driver operation.
453 hpriv
->saved_cap
= cap
= readl(mmio
+ HOST_CAP
);
454 hpriv
->saved_port_map
= port_map
= readl(mmio
+ HOST_PORTS_IMPL
);
456 /* CAP2 register is only defined for AHCI 1.2 and later */
457 vers
= readl(mmio
+ HOST_VERSION
);
458 if ((vers
>> 16) > 1 ||
459 ((vers
>> 16) == 1 && (vers
& 0xFFFF) >= 0x200))
460 hpriv
->saved_cap2
= cap2
= readl(mmio
+ HOST_CAP2
);
462 hpriv
->saved_cap2
= cap2
= 0;
464 /* some chips have errata preventing 64bit use */
465 if ((cap
& HOST_CAP_64
) && (hpriv
->flags
& AHCI_HFLAG_32BIT_ONLY
)) {
466 dev_info(dev
, "controller can't do 64bit DMA, forcing 32bit\n");
470 if ((cap
& HOST_CAP_NCQ
) && (hpriv
->flags
& AHCI_HFLAG_NO_NCQ
)) {
471 dev_info(dev
, "controller can't do NCQ, turning off CAP_NCQ\n");
472 cap
&= ~HOST_CAP_NCQ
;
475 if (!(cap
& HOST_CAP_NCQ
) && (hpriv
->flags
& AHCI_HFLAG_YES_NCQ
)) {
476 dev_info(dev
, "controller can do NCQ, turning on CAP_NCQ\n");
480 if ((cap
& HOST_CAP_PMP
) && (hpriv
->flags
& AHCI_HFLAG_NO_PMP
)) {
481 dev_info(dev
, "controller can't do PMP, turning off CAP_PMP\n");
482 cap
&= ~HOST_CAP_PMP
;
485 if ((cap
& HOST_CAP_SNTF
) && (hpriv
->flags
& AHCI_HFLAG_NO_SNTF
)) {
487 "controller can't do SNTF, turning off CAP_SNTF\n");
488 cap
&= ~HOST_CAP_SNTF
;
491 if ((cap2
& HOST_CAP2_SDS
) && (hpriv
->flags
& AHCI_HFLAG_NO_DEVSLP
)) {
493 "controller can't do DEVSLP, turning off\n");
494 cap2
&= ~HOST_CAP2_SDS
;
495 cap2
&= ~HOST_CAP2_SADM
;
498 if (!(cap
& HOST_CAP_FBS
) && (hpriv
->flags
& AHCI_HFLAG_YES_FBS
)) {
499 dev_info(dev
, "controller can do FBS, turning on CAP_FBS\n");
503 if ((cap
& HOST_CAP_FBS
) && (hpriv
->flags
& AHCI_HFLAG_NO_FBS
)) {
504 dev_info(dev
, "controller can't do FBS, turning off CAP_FBS\n");
505 cap
&= ~HOST_CAP_FBS
;
508 if (!(cap
& HOST_CAP_ALPM
) && (hpriv
->flags
& AHCI_HFLAG_YES_ALPM
)) {
509 dev_info(dev
, "controller can do ALPM, turning on CAP_ALPM\n");
510 cap
|= HOST_CAP_ALPM
;
513 if (hpriv
->force_port_map
&& port_map
!= hpriv
->force_port_map
) {
514 dev_info(dev
, "forcing port_map 0x%x -> 0x%x\n",
515 port_map
, hpriv
->force_port_map
);
516 port_map
= hpriv
->force_port_map
;
517 hpriv
->saved_port_map
= port_map
;
520 if (hpriv
->mask_port_map
) {
521 dev_warn(dev
, "masking port_map 0x%x -> 0x%x\n",
523 port_map
& hpriv
->mask_port_map
);
524 port_map
&= hpriv
->mask_port_map
;
527 /* cross check port_map and cap.n_ports */
531 for (i
= 0; i
< AHCI_MAX_PORTS
; i
++)
532 if (port_map
& (1 << i
))
535 /* If PI has more ports than n_ports, whine, clear
536 * port_map and let it be generated from n_ports.
538 if (map_ports
> ahci_nr_ports(cap
)) {
540 "implemented port map (0x%x) contains more ports than nr_ports (%u), using nr_ports\n",
541 port_map
, ahci_nr_ports(cap
));
546 /* fabricate port_map from cap.nr_ports for < AHCI 1.3 */
547 if (!port_map
&& vers
< 0x10300) {
548 port_map
= (1 << ahci_nr_ports(cap
)) - 1;
549 dev_warn(dev
, "forcing PORTS_IMPL to 0x%x\n", port_map
);
551 /* write the fixed up value to the PI register */
552 hpriv
->saved_port_map
= port_map
;
555 /* record values to use during operation */
558 hpriv
->version
= readl(mmio
+ HOST_VERSION
);
559 hpriv
->port_map
= port_map
;
561 if (!hpriv
->start_engine
)
562 hpriv
->start_engine
= ahci_start_engine
;
564 if (!hpriv
->stop_engine
)
565 hpriv
->stop_engine
= ahci_stop_engine
;
567 if (!hpriv
->irq_handler
)
568 hpriv
->irq_handler
= ahci_single_level_irq_intr
;
570 EXPORT_SYMBOL_GPL(ahci_save_initial_config
);
573 * ahci_restore_initial_config - Restore initial config
574 * @host: target ATA host
576 * Restore initial config stored by ahci_save_initial_config().
581 static void ahci_restore_initial_config(struct ata_host
*host
)
583 struct ahci_host_priv
*hpriv
= host
->private_data
;
584 void __iomem
*mmio
= hpriv
->mmio
;
586 writel(hpriv
->saved_cap
, mmio
+ HOST_CAP
);
587 if (hpriv
->saved_cap2
)
588 writel(hpriv
->saved_cap2
, mmio
+ HOST_CAP2
);
589 writel(hpriv
->saved_port_map
, mmio
+ HOST_PORTS_IMPL
);
590 (void) readl(mmio
+ HOST_PORTS_IMPL
); /* flush */
593 static unsigned ahci_scr_offset(struct ata_port
*ap
, unsigned int sc_reg
)
595 static const int offset
[] = {
596 [SCR_STATUS
] = PORT_SCR_STAT
,
597 [SCR_CONTROL
] = PORT_SCR_CTL
,
598 [SCR_ERROR
] = PORT_SCR_ERR
,
599 [SCR_ACTIVE
] = PORT_SCR_ACT
,
600 [SCR_NOTIFICATION
] = PORT_SCR_NTF
,
602 struct ahci_host_priv
*hpriv
= ap
->host
->private_data
;
604 if (sc_reg
< ARRAY_SIZE(offset
) &&
605 (sc_reg
!= SCR_NOTIFICATION
|| (hpriv
->cap
& HOST_CAP_SNTF
)))
606 return offset
[sc_reg
];
610 static int ahci_scr_read(struct ata_link
*link
, unsigned int sc_reg
, u32
*val
)
612 void __iomem
*port_mmio
= ahci_port_base(link
->ap
);
613 int offset
= ahci_scr_offset(link
->ap
, sc_reg
);
616 *val
= readl(port_mmio
+ offset
);
622 static int ahci_scr_write(struct ata_link
*link
, unsigned int sc_reg
, u32 val
)
624 void __iomem
*port_mmio
= ahci_port_base(link
->ap
);
625 int offset
= ahci_scr_offset(link
->ap
, sc_reg
);
628 writel(val
, port_mmio
+ offset
);
634 void ahci_start_engine(struct ata_port
*ap
)
636 void __iomem
*port_mmio
= ahci_port_base(ap
);
640 tmp
= readl(port_mmio
+ PORT_CMD
);
641 tmp
|= PORT_CMD_START
;
642 writel(tmp
, port_mmio
+ PORT_CMD
);
643 readl(port_mmio
+ PORT_CMD
); /* flush */
645 EXPORT_SYMBOL_GPL(ahci_start_engine
);
647 int ahci_stop_engine(struct ata_port
*ap
)
649 void __iomem
*port_mmio
= ahci_port_base(ap
);
650 struct ahci_host_priv
*hpriv
= ap
->host
->private_data
;
654 * On some controllers, stopping a port's DMA engine while the port
655 * is in ALPM state (partial or slumber) results in failures on
656 * subsequent DMA engine starts. For those controllers, put the
657 * port back in active state before stopping its DMA engine.
659 if ((hpriv
->flags
& AHCI_HFLAG_WAKE_BEFORE_STOP
) &&
660 (ap
->link
.lpm_policy
> ATA_LPM_MAX_POWER
) &&
661 ahci_set_lpm(&ap
->link
, ATA_LPM_MAX_POWER
, ATA_LPM_WAKE_ONLY
)) {
662 dev_err(ap
->host
->dev
, "Failed to wake up port before engine stop\n");
666 tmp
= readl(port_mmio
+ PORT_CMD
);
668 /* check if the HBA is idle */
669 if ((tmp
& (PORT_CMD_START
| PORT_CMD_LIST_ON
)) == 0)
672 /* setting HBA to idle */
673 tmp
&= ~PORT_CMD_START
;
674 writel(tmp
, port_mmio
+ PORT_CMD
);
676 /* wait for engine to stop. This could be as long as 500 msec */
677 tmp
= ata_wait_register(ap
, port_mmio
+ PORT_CMD
,
678 PORT_CMD_LIST_ON
, PORT_CMD_LIST_ON
, 1, 500);
679 if (tmp
& PORT_CMD_LIST_ON
)
684 EXPORT_SYMBOL_GPL(ahci_stop_engine
);
686 void ahci_start_fis_rx(struct ata_port
*ap
)
688 void __iomem
*port_mmio
= ahci_port_base(ap
);
689 struct ahci_host_priv
*hpriv
= ap
->host
->private_data
;
690 struct ahci_port_priv
*pp
= ap
->private_data
;
693 /* set FIS registers */
694 if (hpriv
->cap
& HOST_CAP_64
)
695 writel((pp
->cmd_slot_dma
>> 16) >> 16,
696 port_mmio
+ PORT_LST_ADDR_HI
);
697 writel(pp
->cmd_slot_dma
& 0xffffffff, port_mmio
+ PORT_LST_ADDR
);
699 if (hpriv
->cap
& HOST_CAP_64
)
700 writel((pp
->rx_fis_dma
>> 16) >> 16,
701 port_mmio
+ PORT_FIS_ADDR_HI
);
702 writel(pp
->rx_fis_dma
& 0xffffffff, port_mmio
+ PORT_FIS_ADDR
);
704 /* enable FIS reception */
705 tmp
= readl(port_mmio
+ PORT_CMD
);
706 tmp
|= PORT_CMD_FIS_RX
;
707 writel(tmp
, port_mmio
+ PORT_CMD
);
710 readl(port_mmio
+ PORT_CMD
);
712 EXPORT_SYMBOL_GPL(ahci_start_fis_rx
);
714 static int ahci_stop_fis_rx(struct ata_port
*ap
)
716 void __iomem
*port_mmio
= ahci_port_base(ap
);
719 /* disable FIS reception */
720 tmp
= readl(port_mmio
+ PORT_CMD
);
721 tmp
&= ~PORT_CMD_FIS_RX
;
722 writel(tmp
, port_mmio
+ PORT_CMD
);
724 /* wait for completion, spec says 500ms, give it 1000 */
725 tmp
= ata_wait_register(ap
, port_mmio
+ PORT_CMD
, PORT_CMD_FIS_ON
,
726 PORT_CMD_FIS_ON
, 10, 1000);
727 if (tmp
& PORT_CMD_FIS_ON
)
733 static void ahci_power_up(struct ata_port
*ap
)
735 struct ahci_host_priv
*hpriv
= ap
->host
->private_data
;
736 void __iomem
*port_mmio
= ahci_port_base(ap
);
739 cmd
= readl(port_mmio
+ PORT_CMD
) & ~PORT_CMD_ICC_MASK
;
742 if (hpriv
->cap
& HOST_CAP_SSS
) {
743 cmd
|= PORT_CMD_SPIN_UP
;
744 writel(cmd
, port_mmio
+ PORT_CMD
);
748 writel(cmd
| PORT_CMD_ICC_ACTIVE
, port_mmio
+ PORT_CMD
);
751 static int ahci_set_lpm(struct ata_link
*link
, enum ata_lpm_policy policy
,
754 struct ata_port
*ap
= link
->ap
;
755 struct ahci_host_priv
*hpriv
= ap
->host
->private_data
;
756 struct ahci_port_priv
*pp
= ap
->private_data
;
757 void __iomem
*port_mmio
= ahci_port_base(ap
);
759 if (policy
!= ATA_LPM_MAX_POWER
) {
760 /* wakeup flag only applies to the max power policy */
761 hints
&= ~ATA_LPM_WAKE_ONLY
;
764 * Disable interrupts on Phy Ready. This keeps us from
765 * getting woken up due to spurious phy ready
768 pp
->intr_mask
&= ~PORT_IRQ_PHYRDY
;
769 writel(pp
->intr_mask
, port_mmio
+ PORT_IRQ_MASK
);
771 sata_link_scr_lpm(link
, policy
, false);
774 if (hpriv
->cap
& HOST_CAP_ALPM
) {
775 u32 cmd
= readl(port_mmio
+ PORT_CMD
);
777 if (policy
== ATA_LPM_MAX_POWER
|| !(hints
& ATA_LPM_HIPM
)) {
778 if (!(hints
& ATA_LPM_WAKE_ONLY
))
779 cmd
&= ~(PORT_CMD_ASP
| PORT_CMD_ALPE
);
780 cmd
|= PORT_CMD_ICC_ACTIVE
;
782 writel(cmd
, port_mmio
+ PORT_CMD
);
783 readl(port_mmio
+ PORT_CMD
);
785 /* wait 10ms to be sure we've come out of LPM state */
788 if (hints
& ATA_LPM_WAKE_ONLY
)
791 cmd
|= PORT_CMD_ALPE
;
792 if (policy
== ATA_LPM_MIN_POWER
)
795 /* write out new cmd value */
796 writel(cmd
, port_mmio
+ PORT_CMD
);
800 /* set aggressive device sleep */
801 if ((hpriv
->cap2
& HOST_CAP2_SDS
) &&
802 (hpriv
->cap2
& HOST_CAP2_SADM
) &&
803 (link
->device
->flags
& ATA_DFLAG_DEVSLP
)) {
804 if (policy
== ATA_LPM_MIN_POWER
)
805 ahci_set_aggressive_devslp(ap
, true);
807 ahci_set_aggressive_devslp(ap
, false);
810 if (policy
== ATA_LPM_MAX_POWER
) {
811 sata_link_scr_lpm(link
, policy
, false);
813 /* turn PHYRDY IRQ back on */
814 pp
->intr_mask
|= PORT_IRQ_PHYRDY
;
815 writel(pp
->intr_mask
, port_mmio
+ PORT_IRQ_MASK
);
822 static void ahci_power_down(struct ata_port
*ap
)
824 struct ahci_host_priv
*hpriv
= ap
->host
->private_data
;
825 void __iomem
*port_mmio
= ahci_port_base(ap
);
828 if (!(hpriv
->cap
& HOST_CAP_SSS
))
831 /* put device into listen mode, first set PxSCTL.DET to 0 */
832 scontrol
= readl(port_mmio
+ PORT_SCR_CTL
);
834 writel(scontrol
, port_mmio
+ PORT_SCR_CTL
);
836 /* then set PxCMD.SUD to 0 */
837 cmd
= readl(port_mmio
+ PORT_CMD
) & ~PORT_CMD_ICC_MASK
;
838 cmd
&= ~PORT_CMD_SPIN_UP
;
839 writel(cmd
, port_mmio
+ PORT_CMD
);
843 static void ahci_start_port(struct ata_port
*ap
)
845 struct ahci_host_priv
*hpriv
= ap
->host
->private_data
;
846 struct ahci_port_priv
*pp
= ap
->private_data
;
847 struct ata_link
*link
;
848 struct ahci_em_priv
*emp
;
852 /* enable FIS reception */
853 ahci_start_fis_rx(ap
);
856 if (!(hpriv
->flags
& AHCI_HFLAG_DELAY_ENGINE
))
857 hpriv
->start_engine(ap
);
860 if (ap
->flags
& ATA_FLAG_EM
) {
861 ata_for_each_link(link
, ap
, EDGE
) {
862 emp
= &pp
->em_priv
[link
->pmp
];
864 /* EM Transmit bit maybe busy during init */
865 for (i
= 0; i
< EM_MAX_RETRY
; i
++) {
866 rc
= ap
->ops
->transmit_led_message(ap
,
870 * If busy, give a breather but do not
871 * release EH ownership by using msleep()
872 * instead of ata_msleep(). EM Transmit
873 * bit is busy for the whole host and
874 * releasing ownership will cause other
875 * ports to fail the same way.
885 if (ap
->flags
& ATA_FLAG_SW_ACTIVITY
)
886 ata_for_each_link(link
, ap
, EDGE
)
887 ahci_init_sw_activity(link
);
891 static int ahci_deinit_port(struct ata_port
*ap
, const char **emsg
)
894 struct ahci_host_priv
*hpriv
= ap
->host
->private_data
;
897 rc
= hpriv
->stop_engine(ap
);
899 *emsg
= "failed to stop engine";
903 /* disable FIS reception */
904 rc
= ahci_stop_fis_rx(ap
);
906 *emsg
= "failed stop FIS RX";
913 int ahci_reset_controller(struct ata_host
*host
)
915 struct ahci_host_priv
*hpriv
= host
->private_data
;
916 void __iomem
*mmio
= hpriv
->mmio
;
919 /* we must be in AHCI mode, before using anything
920 * AHCI-specific, such as HOST_RESET.
922 ahci_enable_ahci(mmio
);
924 /* global controller reset */
925 if (!ahci_skip_host_reset
) {
926 tmp
= readl(mmio
+ HOST_CTL
);
927 if ((tmp
& HOST_RESET
) == 0) {
928 writel(tmp
| HOST_RESET
, mmio
+ HOST_CTL
);
929 readl(mmio
+ HOST_CTL
); /* flush */
933 * to perform host reset, OS should set HOST_RESET
934 * and poll until this bit is read to be "0".
935 * reset must complete within 1 second, or
936 * the hardware should be considered fried.
938 tmp
= ata_wait_register(NULL
, mmio
+ HOST_CTL
, HOST_RESET
,
939 HOST_RESET
, 10, 1000);
941 if (tmp
& HOST_RESET
) {
942 dev_err(host
->dev
, "controller reset failed (0x%x)\n",
947 /* turn on AHCI mode */
948 ahci_enable_ahci(mmio
);
950 /* Some registers might be cleared on reset. Restore
953 if (!(hpriv
->flags
& AHCI_HFLAG_NO_WRITE_TO_RO
))
954 ahci_restore_initial_config(host
);
956 dev_info(host
->dev
, "skipping global host reset\n");
960 EXPORT_SYMBOL_GPL(ahci_reset_controller
);
962 static void ahci_sw_activity(struct ata_link
*link
)
964 struct ata_port
*ap
= link
->ap
;
965 struct ahci_port_priv
*pp
= ap
->private_data
;
966 struct ahci_em_priv
*emp
= &pp
->em_priv
[link
->pmp
];
968 if (!(link
->flags
& ATA_LFLAG_SW_ACTIVITY
))
972 if (!timer_pending(&emp
->timer
))
973 mod_timer(&emp
->timer
, jiffies
+ msecs_to_jiffies(10));
976 static void ahci_sw_activity_blink(unsigned long arg
)
978 struct ata_link
*link
= (struct ata_link
*)arg
;
979 struct ata_port
*ap
= link
->ap
;
980 struct ahci_port_priv
*pp
= ap
->private_data
;
981 struct ahci_em_priv
*emp
= &pp
->em_priv
[link
->pmp
];
982 unsigned long led_message
= emp
->led_state
;
983 u32 activity_led_state
;
986 led_message
&= EM_MSG_LED_VALUE
;
987 led_message
|= ap
->port_no
| (link
->pmp
<< 8);
989 /* check to see if we've had activity. If so,
990 * toggle state of LED and reset timer. If not,
991 * turn LED to desired idle state.
993 spin_lock_irqsave(ap
->lock
, flags
);
994 if (emp
->saved_activity
!= emp
->activity
) {
995 emp
->saved_activity
= emp
->activity
;
996 /* get the current LED state */
997 activity_led_state
= led_message
& EM_MSG_LED_VALUE_ON
;
999 if (activity_led_state
)
1000 activity_led_state
= 0;
1002 activity_led_state
= 1;
1004 /* clear old state */
1005 led_message
&= ~EM_MSG_LED_VALUE_ACTIVITY
;
1008 led_message
|= (activity_led_state
<< 16);
1009 mod_timer(&emp
->timer
, jiffies
+ msecs_to_jiffies(100));
1011 /* switch to idle */
1012 led_message
&= ~EM_MSG_LED_VALUE_ACTIVITY
;
1013 if (emp
->blink_policy
== BLINK_OFF
)
1014 led_message
|= (1 << 16);
1016 spin_unlock_irqrestore(ap
->lock
, flags
);
1017 ap
->ops
->transmit_led_message(ap
, led_message
, 4);
1020 static void ahci_init_sw_activity(struct ata_link
*link
)
1022 struct ata_port
*ap
= link
->ap
;
1023 struct ahci_port_priv
*pp
= ap
->private_data
;
1024 struct ahci_em_priv
*emp
= &pp
->em_priv
[link
->pmp
];
1026 /* init activity stats, setup timer */
1027 emp
->saved_activity
= emp
->activity
= 0;
1028 setup_timer(&emp
->timer
, ahci_sw_activity_blink
, (unsigned long)link
);
1030 /* check our blink policy and set flag for link if it's enabled */
1031 if (emp
->blink_policy
)
1032 link
->flags
|= ATA_LFLAG_SW_ACTIVITY
;
1035 int ahci_reset_em(struct ata_host
*host
)
1037 struct ahci_host_priv
*hpriv
= host
->private_data
;
1038 void __iomem
*mmio
= hpriv
->mmio
;
1041 em_ctl
= readl(mmio
+ HOST_EM_CTL
);
1042 if ((em_ctl
& EM_CTL_TM
) || (em_ctl
& EM_CTL_RST
))
1045 writel(em_ctl
| EM_CTL_RST
, mmio
+ HOST_EM_CTL
);
1048 EXPORT_SYMBOL_GPL(ahci_reset_em
);
1050 static ssize_t
ahci_transmit_led_message(struct ata_port
*ap
, u32 state
,
1053 struct ahci_host_priv
*hpriv
= ap
->host
->private_data
;
1054 struct ahci_port_priv
*pp
= ap
->private_data
;
1055 void __iomem
*mmio
= hpriv
->mmio
;
1057 u32 message
[] = {0, 0};
1058 unsigned long flags
;
1060 struct ahci_em_priv
*emp
;
1062 /* get the slot number from the message */
1063 pmp
= (state
& EM_MSG_LED_PMP_SLOT
) >> 8;
1064 if (pmp
< EM_MAX_SLOTS
)
1065 emp
= &pp
->em_priv
[pmp
];
1069 ahci_rpm_get_port(ap
);
1070 spin_lock_irqsave(ap
->lock
, flags
);
1073 * if we are still busy transmitting a previous message,
1076 em_ctl
= readl(mmio
+ HOST_EM_CTL
);
1077 if (em_ctl
& EM_CTL_TM
) {
1078 spin_unlock_irqrestore(ap
->lock
, flags
);
1079 ahci_rpm_put_port(ap
);
1083 if (hpriv
->em_msg_type
& EM_MSG_TYPE_LED
) {
1085 * create message header - this is all zero except for
1086 * the message size, which is 4 bytes.
1088 message
[0] |= (4 << 8);
1090 /* ignore 0:4 of byte zero, fill in port info yourself */
1091 message
[1] = ((state
& ~EM_MSG_LED_HBA_PORT
) | ap
->port_no
);
1093 /* write message to EM_LOC */
1094 writel(message
[0], mmio
+ hpriv
->em_loc
);
1095 writel(message
[1], mmio
+ hpriv
->em_loc
+4);
1098 * tell hardware to transmit the message
1100 writel(em_ctl
| EM_CTL_TM
, mmio
+ HOST_EM_CTL
);
1103 /* save off new led state for port/slot */
1104 emp
->led_state
= state
;
1106 spin_unlock_irqrestore(ap
->lock
, flags
);
1107 ahci_rpm_put_port(ap
);
1112 static ssize_t
ahci_led_show(struct ata_port
*ap
, char *buf
)
1114 struct ahci_port_priv
*pp
= ap
->private_data
;
1115 struct ata_link
*link
;
1116 struct ahci_em_priv
*emp
;
1119 ata_for_each_link(link
, ap
, EDGE
) {
1120 emp
= &pp
->em_priv
[link
->pmp
];
1121 rc
+= sprintf(buf
, "%lx\n", emp
->led_state
);
1126 static ssize_t
ahci_led_store(struct ata_port
*ap
, const char *buf
,
1131 struct ahci_port_priv
*pp
= ap
->private_data
;
1132 struct ahci_em_priv
*emp
;
1134 if (kstrtouint(buf
, 0, &state
) < 0)
1137 /* get the slot number from the message */
1138 pmp
= (state
& EM_MSG_LED_PMP_SLOT
) >> 8;
1139 if (pmp
< EM_MAX_SLOTS
) {
1140 pmp
= array_index_nospec(pmp
, EM_MAX_SLOTS
);
1141 emp
= &pp
->em_priv
[pmp
];
1146 /* mask off the activity bits if we are in sw_activity
1147 * mode, user should turn off sw_activity before setting
1148 * activity led through em_message
1150 if (emp
->blink_policy
)
1151 state
&= ~EM_MSG_LED_VALUE_ACTIVITY
;
1153 return ap
->ops
->transmit_led_message(ap
, state
, size
);
1156 static ssize_t
ahci_activity_store(struct ata_device
*dev
, enum sw_activity val
)
1158 struct ata_link
*link
= dev
->link
;
1159 struct ata_port
*ap
= link
->ap
;
1160 struct ahci_port_priv
*pp
= ap
->private_data
;
1161 struct ahci_em_priv
*emp
= &pp
->em_priv
[link
->pmp
];
1162 u32 port_led_state
= emp
->led_state
;
1164 /* save the desired Activity LED behavior */
1167 link
->flags
&= ~(ATA_LFLAG_SW_ACTIVITY
);
1169 /* set the LED to OFF */
1170 port_led_state
&= EM_MSG_LED_VALUE_OFF
;
1171 port_led_state
|= (ap
->port_no
| (link
->pmp
<< 8));
1172 ap
->ops
->transmit_led_message(ap
, port_led_state
, 4);
1174 link
->flags
|= ATA_LFLAG_SW_ACTIVITY
;
1175 if (val
== BLINK_OFF
) {
1176 /* set LED to ON for idle */
1177 port_led_state
&= EM_MSG_LED_VALUE_OFF
;
1178 port_led_state
|= (ap
->port_no
| (link
->pmp
<< 8));
1179 port_led_state
|= EM_MSG_LED_VALUE_ON
; /* check this */
1180 ap
->ops
->transmit_led_message(ap
, port_led_state
, 4);
1183 emp
->blink_policy
= val
;
1187 static ssize_t
ahci_activity_show(struct ata_device
*dev
, char *buf
)
1189 struct ata_link
*link
= dev
->link
;
1190 struct ata_port
*ap
= link
->ap
;
1191 struct ahci_port_priv
*pp
= ap
->private_data
;
1192 struct ahci_em_priv
*emp
= &pp
->em_priv
[link
->pmp
];
1194 /* display the saved value of activity behavior for this
1197 return sprintf(buf
, "%d\n", emp
->blink_policy
);
1200 static void ahci_port_init(struct device
*dev
, struct ata_port
*ap
,
1201 int port_no
, void __iomem
*mmio
,
1202 void __iomem
*port_mmio
)
1204 struct ahci_host_priv
*hpriv
= ap
->host
->private_data
;
1205 const char *emsg
= NULL
;
1209 /* make sure port is not active */
1210 rc
= ahci_deinit_port(ap
, &emsg
);
1212 dev_warn(dev
, "%s (%d)\n", emsg
, rc
);
1215 tmp
= readl(port_mmio
+ PORT_SCR_ERR
);
1216 VPRINTK("PORT_SCR_ERR 0x%x\n", tmp
);
1217 writel(tmp
, port_mmio
+ PORT_SCR_ERR
);
1219 /* clear port IRQ */
1220 tmp
= readl(port_mmio
+ PORT_IRQ_STAT
);
1221 VPRINTK("PORT_IRQ_STAT 0x%x\n", tmp
);
1223 writel(tmp
, port_mmio
+ PORT_IRQ_STAT
);
1225 writel(1 << port_no
, mmio
+ HOST_IRQ_STAT
);
1227 /* mark esata ports */
1228 tmp
= readl(port_mmio
+ PORT_CMD
);
1229 if ((tmp
& PORT_CMD_ESP
) && (hpriv
->cap
& HOST_CAP_SXS
))
1230 ap
->pflags
|= ATA_PFLAG_EXTERNAL
;
1233 void ahci_init_controller(struct ata_host
*host
)
1235 struct ahci_host_priv
*hpriv
= host
->private_data
;
1236 void __iomem
*mmio
= hpriv
->mmio
;
1238 void __iomem
*port_mmio
;
1241 for (i
= 0; i
< host
->n_ports
; i
++) {
1242 struct ata_port
*ap
= host
->ports
[i
];
1244 port_mmio
= ahci_port_base(ap
);
1245 if (ata_port_is_dummy(ap
))
1248 ahci_port_init(host
->dev
, ap
, i
, mmio
, port_mmio
);
1251 tmp
= readl(mmio
+ HOST_CTL
);
1252 VPRINTK("HOST_CTL 0x%x\n", tmp
);
1253 writel(tmp
| HOST_IRQ_EN
, mmio
+ HOST_CTL
);
1254 tmp
= readl(mmio
+ HOST_CTL
);
1255 VPRINTK("HOST_CTL 0x%x\n", tmp
);
1257 EXPORT_SYMBOL_GPL(ahci_init_controller
);
1259 static void ahci_dev_config(struct ata_device
*dev
)
1261 struct ahci_host_priv
*hpriv
= dev
->link
->ap
->host
->private_data
;
1263 if (hpriv
->flags
& AHCI_HFLAG_SECT255
) {
1264 dev
->max_sectors
= 255;
1266 "SB600 AHCI: limiting to 255 sectors per cmd\n");
1270 unsigned int ahci_dev_classify(struct ata_port
*ap
)
1272 void __iomem
*port_mmio
= ahci_port_base(ap
);
1273 struct ata_taskfile tf
;
1276 tmp
= readl(port_mmio
+ PORT_SIG
);
1277 tf
.lbah
= (tmp
>> 24) & 0xff;
1278 tf
.lbam
= (tmp
>> 16) & 0xff;
1279 tf
.lbal
= (tmp
>> 8) & 0xff;
1280 tf
.nsect
= (tmp
) & 0xff;
1282 return ata_dev_classify(&tf
);
1284 EXPORT_SYMBOL_GPL(ahci_dev_classify
);
1286 void ahci_fill_cmd_slot(struct ahci_port_priv
*pp
, unsigned int tag
,
1289 dma_addr_t cmd_tbl_dma
;
1291 cmd_tbl_dma
= pp
->cmd_tbl_dma
+ tag
* AHCI_CMD_TBL_SZ
;
1293 pp
->cmd_slot
[tag
].opts
= cpu_to_le32(opts
);
1294 pp
->cmd_slot
[tag
].status
= 0;
1295 pp
->cmd_slot
[tag
].tbl_addr
= cpu_to_le32(cmd_tbl_dma
& 0xffffffff);
1296 pp
->cmd_slot
[tag
].tbl_addr_hi
= cpu_to_le32((cmd_tbl_dma
>> 16) >> 16);
1298 EXPORT_SYMBOL_GPL(ahci_fill_cmd_slot
);
1300 int ahci_kick_engine(struct ata_port
*ap
)
1302 void __iomem
*port_mmio
= ahci_port_base(ap
);
1303 struct ahci_host_priv
*hpriv
= ap
->host
->private_data
;
1304 u8 status
= readl(port_mmio
+ PORT_TFDATA
) & 0xFF;
1309 rc
= hpriv
->stop_engine(ap
);
1314 * always do CLO if PMP is attached (AHCI-1.3 9.2)
1316 busy
= status
& (ATA_BUSY
| ATA_DRQ
);
1317 if (!busy
&& !sata_pmp_attached(ap
)) {
1322 if (!(hpriv
->cap
& HOST_CAP_CLO
)) {
1328 tmp
= readl(port_mmio
+ PORT_CMD
);
1329 tmp
|= PORT_CMD_CLO
;
1330 writel(tmp
, port_mmio
+ PORT_CMD
);
1333 tmp
= ata_wait_register(ap
, port_mmio
+ PORT_CMD
,
1334 PORT_CMD_CLO
, PORT_CMD_CLO
, 1, 500);
1335 if (tmp
& PORT_CMD_CLO
)
1338 /* restart engine */
1340 hpriv
->start_engine(ap
);
1343 EXPORT_SYMBOL_GPL(ahci_kick_engine
);
1345 static int ahci_exec_polled_cmd(struct ata_port
*ap
, int pmp
,
1346 struct ata_taskfile
*tf
, int is_cmd
, u16 flags
,
1347 unsigned long timeout_msec
)
1349 const u32 cmd_fis_len
= 5; /* five dwords */
1350 struct ahci_port_priv
*pp
= ap
->private_data
;
1351 void __iomem
*port_mmio
= ahci_port_base(ap
);
1352 u8
*fis
= pp
->cmd_tbl
;
1355 /* prep the command */
1356 ata_tf_to_fis(tf
, pmp
, is_cmd
, fis
);
1357 ahci_fill_cmd_slot(pp
, 0, cmd_fis_len
| flags
| (pmp
<< 12));
1359 /* set port value for softreset of Port Multiplier */
1360 if (pp
->fbs_enabled
&& pp
->fbs_last_dev
!= pmp
) {
1361 tmp
= readl(port_mmio
+ PORT_FBS
);
1362 tmp
&= ~(PORT_FBS_DEV_MASK
| PORT_FBS_DEC
);
1363 tmp
|= pmp
<< PORT_FBS_DEV_OFFSET
;
1364 writel(tmp
, port_mmio
+ PORT_FBS
);
1365 pp
->fbs_last_dev
= pmp
;
1369 writel(1, port_mmio
+ PORT_CMD_ISSUE
);
1372 tmp
= ata_wait_register(ap
, port_mmio
+ PORT_CMD_ISSUE
,
1373 0x1, 0x1, 1, timeout_msec
);
1375 ahci_kick_engine(ap
);
1379 readl(port_mmio
+ PORT_CMD_ISSUE
); /* flush */
1384 int ahci_do_softreset(struct ata_link
*link
, unsigned int *class,
1385 int pmp
, unsigned long deadline
,
1386 int (*check_ready
)(struct ata_link
*link
))
1388 struct ata_port
*ap
= link
->ap
;
1389 struct ahci_host_priv
*hpriv
= ap
->host
->private_data
;
1390 struct ahci_port_priv
*pp
= ap
->private_data
;
1391 const char *reason
= NULL
;
1392 unsigned long now
, msecs
;
1393 struct ata_taskfile tf
;
1394 bool fbs_disabled
= false;
1399 /* prepare for SRST (AHCI-1.1 10.4.1) */
1400 rc
= ahci_kick_engine(ap
);
1401 if (rc
&& rc
!= -EOPNOTSUPP
)
1402 ata_link_warn(link
, "failed to reset engine (errno=%d)\n", rc
);
1405 * According to AHCI-1.2 9.3.9: if FBS is enable, software shall
1406 * clear PxFBS.EN to '0' prior to issuing software reset to devices
1407 * that is attached to port multiplier.
1409 if (!ata_is_host_link(link
) && pp
->fbs_enabled
) {
1410 ahci_disable_fbs(ap
);
1411 fbs_disabled
= true;
1414 ata_tf_init(link
->device
, &tf
);
1416 /* issue the first H2D Register FIS */
1419 if (time_after(deadline
, now
))
1420 msecs
= jiffies_to_msecs(deadline
- now
);
1423 if (ahci_exec_polled_cmd(ap
, pmp
, &tf
, 0,
1424 AHCI_CMD_RESET
| AHCI_CMD_CLR_BUSY
, msecs
)) {
1426 reason
= "1st FIS failed";
1430 /* spec says at least 5us, but be generous and sleep for 1ms */
1433 /* issue the second H2D Register FIS */
1434 tf
.ctl
&= ~ATA_SRST
;
1435 ahci_exec_polled_cmd(ap
, pmp
, &tf
, 0, 0, 0);
1437 /* wait for link to become ready */
1438 rc
= ata_wait_after_reset(link
, deadline
, check_ready
);
1439 if (rc
== -EBUSY
&& hpriv
->flags
& AHCI_HFLAG_SRST_TOUT_IS_OFFLINE
) {
1441 * Workaround for cases where link online status can't
1442 * be trusted. Treat device readiness timeout as link
1445 ata_link_info(link
, "device not ready, treating as offline\n");
1446 *class = ATA_DEV_NONE
;
1448 /* link occupied, -ENODEV too is an error */
1449 reason
= "device not ready";
1452 *class = ahci_dev_classify(ap
);
1454 /* re-enable FBS if disabled before */
1456 ahci_enable_fbs(ap
);
1458 DPRINTK("EXIT, class=%u\n", *class);
1462 ata_link_err(link
, "softreset failed (%s)\n", reason
);
1466 int ahci_check_ready(struct ata_link
*link
)
1468 void __iomem
*port_mmio
= ahci_port_base(link
->ap
);
1469 u8 status
= readl(port_mmio
+ PORT_TFDATA
) & 0xFF;
1471 return ata_check_ready(status
);
1473 EXPORT_SYMBOL_GPL(ahci_check_ready
);
1475 static int ahci_softreset(struct ata_link
*link
, unsigned int *class,
1476 unsigned long deadline
)
1478 int pmp
= sata_srst_pmp(link
);
1482 return ahci_do_softreset(link
, class, pmp
, deadline
, ahci_check_ready
);
1484 EXPORT_SYMBOL_GPL(ahci_do_softreset
);
1486 static int ahci_bad_pmp_check_ready(struct ata_link
*link
)
1488 void __iomem
*port_mmio
= ahci_port_base(link
->ap
);
1489 u8 status
= readl(port_mmio
+ PORT_TFDATA
) & 0xFF;
1490 u32 irq_status
= readl(port_mmio
+ PORT_IRQ_STAT
);
1493 * There is no need to check TFDATA if BAD PMP is found due to HW bug,
1494 * which can save timeout delay.
1496 if (irq_status
& PORT_IRQ_BAD_PMP
)
1499 return ata_check_ready(status
);
1502 static int ahci_pmp_retry_softreset(struct ata_link
*link
, unsigned int *class,
1503 unsigned long deadline
)
1505 struct ata_port
*ap
= link
->ap
;
1506 void __iomem
*port_mmio
= ahci_port_base(ap
);
1507 int pmp
= sata_srst_pmp(link
);
1513 rc
= ahci_do_softreset(link
, class, pmp
, deadline
,
1514 ahci_bad_pmp_check_ready
);
1517 * Soft reset fails with IPMS set when PMP is enabled but
1518 * SATA HDD/ODD is connected to SATA port, do soft reset
1522 irq_sts
= readl(port_mmio
+ PORT_IRQ_STAT
);
1523 if (irq_sts
& PORT_IRQ_BAD_PMP
) {
1525 "applying PMP SRST workaround "
1527 rc
= ahci_do_softreset(link
, class, 0, deadline
,
1535 int ahci_do_hardreset(struct ata_link
*link
, unsigned int *class,
1536 unsigned long deadline
, bool *online
)
1538 const unsigned long *timing
= sata_ehc_deb_timing(&link
->eh_context
);
1539 struct ata_port
*ap
= link
->ap
;
1540 struct ahci_port_priv
*pp
= ap
->private_data
;
1541 struct ahci_host_priv
*hpriv
= ap
->host
->private_data
;
1542 u8
*d2h_fis
= pp
->rx_fis
+ RX_FIS_D2H_REG
;
1543 struct ata_taskfile tf
;
1548 hpriv
->stop_engine(ap
);
1550 /* clear D2H reception area to properly wait for D2H FIS */
1551 ata_tf_init(link
->device
, &tf
);
1552 tf
.command
= ATA_BUSY
;
1553 ata_tf_to_fis(&tf
, 0, 0, d2h_fis
);
1555 rc
= sata_link_hardreset(link
, timing
, deadline
, online
,
1558 hpriv
->start_engine(ap
);
1561 *class = ahci_dev_classify(ap
);
1563 DPRINTK("EXIT, rc=%d, class=%u\n", rc
, *class);
1566 EXPORT_SYMBOL_GPL(ahci_do_hardreset
);
1568 static int ahci_hardreset(struct ata_link
*link
, unsigned int *class,
1569 unsigned long deadline
)
1573 return ahci_do_hardreset(link
, class, deadline
, &online
);
1576 static void ahci_postreset(struct ata_link
*link
, unsigned int *class)
1578 struct ata_port
*ap
= link
->ap
;
1579 void __iomem
*port_mmio
= ahci_port_base(ap
);
1582 ata_std_postreset(link
, class);
1584 /* Make sure port's ATAPI bit is set appropriately */
1585 new_tmp
= tmp
= readl(port_mmio
+ PORT_CMD
);
1586 if (*class == ATA_DEV_ATAPI
)
1587 new_tmp
|= PORT_CMD_ATAPI
;
1589 new_tmp
&= ~PORT_CMD_ATAPI
;
1590 if (new_tmp
!= tmp
) {
1591 writel(new_tmp
, port_mmio
+ PORT_CMD
);
1592 readl(port_mmio
+ PORT_CMD
); /* flush */
1596 static unsigned int ahci_fill_sg(struct ata_queued_cmd
*qc
, void *cmd_tbl
)
1598 struct scatterlist
*sg
;
1599 struct ahci_sg
*ahci_sg
= cmd_tbl
+ AHCI_CMD_TBL_HDR_SZ
;
1605 * Next, the S/G list.
1607 for_each_sg(qc
->sg
, sg
, qc
->n_elem
, si
) {
1608 dma_addr_t addr
= sg_dma_address(sg
);
1609 u32 sg_len
= sg_dma_len(sg
);
1611 ahci_sg
[si
].addr
= cpu_to_le32(addr
& 0xffffffff);
1612 ahci_sg
[si
].addr_hi
= cpu_to_le32((addr
>> 16) >> 16);
1613 ahci_sg
[si
].flags_size
= cpu_to_le32(sg_len
- 1);
1619 static int ahci_pmp_qc_defer(struct ata_queued_cmd
*qc
)
1621 struct ata_port
*ap
= qc
->ap
;
1622 struct ahci_port_priv
*pp
= ap
->private_data
;
1624 if (!sata_pmp_attached(ap
) || pp
->fbs_enabled
)
1625 return ata_std_qc_defer(qc
);
1627 return sata_pmp_qc_defer_cmd_switch(qc
);
1630 static void ahci_qc_prep(struct ata_queued_cmd
*qc
)
1632 struct ata_port
*ap
= qc
->ap
;
1633 struct ahci_port_priv
*pp
= ap
->private_data
;
1634 int is_atapi
= ata_is_atapi(qc
->tf
.protocol
);
1637 const u32 cmd_fis_len
= 5; /* five dwords */
1638 unsigned int n_elem
;
1641 * Fill in command table information. First, the header,
1642 * a SATA Register - Host to Device command FIS.
1644 cmd_tbl
= pp
->cmd_tbl
+ qc
->tag
* AHCI_CMD_TBL_SZ
;
1646 ata_tf_to_fis(&qc
->tf
, qc
->dev
->link
->pmp
, 1, cmd_tbl
);
1648 memset(cmd_tbl
+ AHCI_CMD_TBL_CDB
, 0, 32);
1649 memcpy(cmd_tbl
+ AHCI_CMD_TBL_CDB
, qc
->cdb
, qc
->dev
->cdb_len
);
1653 if (qc
->flags
& ATA_QCFLAG_DMAMAP
)
1654 n_elem
= ahci_fill_sg(qc
, cmd_tbl
);
1657 * Fill in command slot information.
1659 opts
= cmd_fis_len
| n_elem
<< 16 | (qc
->dev
->link
->pmp
<< 12);
1660 if (qc
->tf
.flags
& ATA_TFLAG_WRITE
)
1661 opts
|= AHCI_CMD_WRITE
;
1663 opts
|= AHCI_CMD_ATAPI
| AHCI_CMD_PREFETCH
;
1665 ahci_fill_cmd_slot(pp
, qc
->tag
, opts
);
1668 static void ahci_fbs_dec_intr(struct ata_port
*ap
)
1670 struct ahci_port_priv
*pp
= ap
->private_data
;
1671 void __iomem
*port_mmio
= ahci_port_base(ap
);
1672 u32 fbs
= readl(port_mmio
+ PORT_FBS
);
1676 BUG_ON(!pp
->fbs_enabled
);
1678 /* time to wait for DEC is not specified by AHCI spec,
1679 * add a retry loop for safety.
1681 writel(fbs
| PORT_FBS_DEC
, port_mmio
+ PORT_FBS
);
1682 fbs
= readl(port_mmio
+ PORT_FBS
);
1683 while ((fbs
& PORT_FBS_DEC
) && retries
--) {
1685 fbs
= readl(port_mmio
+ PORT_FBS
);
1688 if (fbs
& PORT_FBS_DEC
)
1689 dev_err(ap
->host
->dev
, "failed to clear device error\n");
1692 static void ahci_error_intr(struct ata_port
*ap
, u32 irq_stat
)
1694 struct ahci_host_priv
*hpriv
= ap
->host
->private_data
;
1695 struct ahci_port_priv
*pp
= ap
->private_data
;
1696 struct ata_eh_info
*host_ehi
= &ap
->link
.eh_info
;
1697 struct ata_link
*link
= NULL
;
1698 struct ata_queued_cmd
*active_qc
;
1699 struct ata_eh_info
*active_ehi
;
1700 bool fbs_need_dec
= false;
1703 /* determine active link with error */
1704 if (pp
->fbs_enabled
) {
1705 void __iomem
*port_mmio
= ahci_port_base(ap
);
1706 u32 fbs
= readl(port_mmio
+ PORT_FBS
);
1707 int pmp
= fbs
>> PORT_FBS_DWE_OFFSET
;
1709 if ((fbs
& PORT_FBS_SDE
) && (pmp
< ap
->nr_pmp_links
)) {
1710 link
= &ap
->pmp_link
[pmp
];
1711 fbs_need_dec
= true;
1715 ata_for_each_link(link
, ap
, EDGE
)
1716 if (ata_link_active(link
))
1722 active_qc
= ata_qc_from_tag(ap
, link
->active_tag
);
1723 active_ehi
= &link
->eh_info
;
1725 /* record irq stat */
1726 ata_ehi_clear_desc(host_ehi
);
1727 ata_ehi_push_desc(host_ehi
, "irq_stat 0x%08x", irq_stat
);
1729 /* AHCI needs SError cleared; otherwise, it might lock up */
1730 ahci_scr_read(&ap
->link
, SCR_ERROR
, &serror
);
1731 ahci_scr_write(&ap
->link
, SCR_ERROR
, serror
);
1732 host_ehi
->serror
|= serror
;
1734 /* some controllers set IRQ_IF_ERR on device errors, ignore it */
1735 if (hpriv
->flags
& AHCI_HFLAG_IGN_IRQ_IF_ERR
)
1736 irq_stat
&= ~PORT_IRQ_IF_ERR
;
1738 if (irq_stat
& PORT_IRQ_TF_ERR
) {
1739 /* If qc is active, charge it; otherwise, the active
1740 * link. There's no active qc on NCQ errors. It will
1741 * be determined by EH by reading log page 10h.
1744 active_qc
->err_mask
|= AC_ERR_DEV
;
1746 active_ehi
->err_mask
|= AC_ERR_DEV
;
1748 if (hpriv
->flags
& AHCI_HFLAG_IGN_SERR_INTERNAL
)
1749 host_ehi
->serror
&= ~SERR_INTERNAL
;
1752 if (irq_stat
& PORT_IRQ_UNK_FIS
) {
1753 u32
*unk
= pp
->rx_fis
+ RX_FIS_UNK
;
1755 active_ehi
->err_mask
|= AC_ERR_HSM
;
1756 active_ehi
->action
|= ATA_EH_RESET
;
1757 ata_ehi_push_desc(active_ehi
,
1758 "unknown FIS %08x %08x %08x %08x" ,
1759 unk
[0], unk
[1], unk
[2], unk
[3]);
1762 if (sata_pmp_attached(ap
) && (irq_stat
& PORT_IRQ_BAD_PMP
)) {
1763 active_ehi
->err_mask
|= AC_ERR_HSM
;
1764 active_ehi
->action
|= ATA_EH_RESET
;
1765 ata_ehi_push_desc(active_ehi
, "incorrect PMP");
1768 if (irq_stat
& (PORT_IRQ_HBUS_ERR
| PORT_IRQ_HBUS_DATA_ERR
)) {
1769 host_ehi
->err_mask
|= AC_ERR_HOST_BUS
;
1770 host_ehi
->action
|= ATA_EH_RESET
;
1771 ata_ehi_push_desc(host_ehi
, "host bus error");
1774 if (irq_stat
& PORT_IRQ_IF_ERR
) {
1776 active_ehi
->err_mask
|= AC_ERR_DEV
;
1778 host_ehi
->err_mask
|= AC_ERR_ATA_BUS
;
1779 host_ehi
->action
|= ATA_EH_RESET
;
1782 ata_ehi_push_desc(host_ehi
, "interface fatal error");
1785 if (irq_stat
& (PORT_IRQ_CONNECT
| PORT_IRQ_PHYRDY
)) {
1786 ata_ehi_hotplugged(host_ehi
);
1787 ata_ehi_push_desc(host_ehi
, "%s",
1788 irq_stat
& PORT_IRQ_CONNECT
?
1789 "connection status changed" : "PHY RDY changed");
1792 /* okay, let's hand over to EH */
1794 if (irq_stat
& PORT_IRQ_FREEZE
)
1795 ata_port_freeze(ap
);
1796 else if (fbs_need_dec
) {
1797 ata_link_abort(link
);
1798 ahci_fbs_dec_intr(ap
);
1803 static void ahci_handle_port_interrupt(struct ata_port
*ap
,
1804 void __iomem
*port_mmio
, u32 status
)
1806 struct ata_eh_info
*ehi
= &ap
->link
.eh_info
;
1807 struct ahci_port_priv
*pp
= ap
->private_data
;
1808 struct ahci_host_priv
*hpriv
= ap
->host
->private_data
;
1809 int resetting
= !!(ap
->pflags
& ATA_PFLAG_RESETTING
);
1813 /* ignore BAD_PMP while resetting */
1814 if (unlikely(resetting
))
1815 status
&= ~PORT_IRQ_BAD_PMP
;
1817 if (sata_lpm_ignore_phy_events(&ap
->link
)) {
1818 status
&= ~PORT_IRQ_PHYRDY
;
1819 ahci_scr_write(&ap
->link
, SCR_ERROR
, SERR_PHYRDY_CHG
);
1822 if (unlikely(status
& PORT_IRQ_ERROR
)) {
1823 ahci_error_intr(ap
, status
);
1827 if (status
& PORT_IRQ_SDB_FIS
) {
1828 /* If SNotification is available, leave notification
1829 * handling to sata_async_notification(). If not,
1830 * emulate it by snooping SDB FIS RX area.
1832 * Snooping FIS RX area is probably cheaper than
1833 * poking SNotification but some constrollers which
1834 * implement SNotification, ICH9 for example, don't
1835 * store AN SDB FIS into receive area.
1837 if (hpriv
->cap
& HOST_CAP_SNTF
)
1838 sata_async_notification(ap
);
1840 /* If the 'N' bit in word 0 of the FIS is set,
1841 * we just received asynchronous notification.
1842 * Tell libata about it.
1844 * Lack of SNotification should not appear in
1845 * ahci 1.2, so the workaround is unnecessary
1846 * when FBS is enabled.
1848 if (pp
->fbs_enabled
)
1851 const __le32
*f
= pp
->rx_fis
+ RX_FIS_SDB
;
1852 u32 f0
= le32_to_cpu(f
[0]);
1854 sata_async_notification(ap
);
1859 /* pp->active_link is not reliable once FBS is enabled, both
1860 * PORT_SCR_ACT and PORT_CMD_ISSUE should be checked because
1861 * NCQ and non-NCQ commands may be in flight at the same time.
1863 if (pp
->fbs_enabled
) {
1864 if (ap
->qc_active
) {
1865 qc_active
= readl(port_mmio
+ PORT_SCR_ACT
);
1866 qc_active
|= readl(port_mmio
+ PORT_CMD_ISSUE
);
1869 /* pp->active_link is valid iff any command is in flight */
1870 if (ap
->qc_active
&& pp
->active_link
->sactive
)
1871 qc_active
= readl(port_mmio
+ PORT_SCR_ACT
);
1873 qc_active
= readl(port_mmio
+ PORT_CMD_ISSUE
);
1877 rc
= ata_qc_complete_multiple(ap
, qc_active
);
1879 /* while resetting, invalid completions are expected */
1880 if (unlikely(rc
< 0 && !resetting
)) {
1881 ehi
->err_mask
|= AC_ERR_HSM
;
1882 ehi
->action
|= ATA_EH_RESET
;
1883 ata_port_freeze(ap
);
1887 static void ahci_port_intr(struct ata_port
*ap
)
1889 void __iomem
*port_mmio
= ahci_port_base(ap
);
1892 status
= readl(port_mmio
+ PORT_IRQ_STAT
);
1893 writel(status
, port_mmio
+ PORT_IRQ_STAT
);
1895 ahci_handle_port_interrupt(ap
, port_mmio
, status
);
1898 static irqreturn_t
ahci_multi_irqs_intr_hard(int irq
, void *dev_instance
)
1900 struct ata_port
*ap
= dev_instance
;
1901 void __iomem
*port_mmio
= ahci_port_base(ap
);
1906 status
= readl(port_mmio
+ PORT_IRQ_STAT
);
1907 writel(status
, port_mmio
+ PORT_IRQ_STAT
);
1909 spin_lock(ap
->lock
);
1910 ahci_handle_port_interrupt(ap
, port_mmio
, status
);
1911 spin_unlock(ap
->lock
);
1918 u32
ahci_handle_port_intr(struct ata_host
*host
, u32 irq_masked
)
1920 unsigned int i
, handled
= 0;
1922 for (i
= 0; i
< host
->n_ports
; i
++) {
1923 struct ata_port
*ap
;
1925 if (!(irq_masked
& (1 << i
)))
1928 ap
= host
->ports
[i
];
1931 VPRINTK("port %u\n", i
);
1933 VPRINTK("port %u (no irq)\n", i
);
1934 if (ata_ratelimit())
1936 "interrupt on disabled port %u\n", i
);
1944 EXPORT_SYMBOL_GPL(ahci_handle_port_intr
);
1946 static irqreturn_t
ahci_single_level_irq_intr(int irq
, void *dev_instance
)
1948 struct ata_host
*host
= dev_instance
;
1949 struct ahci_host_priv
*hpriv
;
1950 unsigned int rc
= 0;
1952 u32 irq_stat
, irq_masked
;
1956 hpriv
= host
->private_data
;
1959 /* sigh. 0xffffffff is a valid return from h/w */
1960 irq_stat
= readl(mmio
+ HOST_IRQ_STAT
);
1964 irq_masked
= irq_stat
& hpriv
->port_map
;
1966 spin_lock(&host
->lock
);
1968 rc
= ahci_handle_port_intr(host
, irq_masked
);
1970 /* HOST_IRQ_STAT behaves as level triggered latch meaning that
1971 * it should be cleared after all the port events are cleared;
1972 * otherwise, it will raise a spurious interrupt after each
1973 * valid one. Please read section 10.6.2 of ahci 1.1 for more
1976 * Also, use the unmasked value to clear interrupt as spurious
1977 * pending event on a dummy port might cause screaming IRQ.
1979 writel(irq_stat
, mmio
+ HOST_IRQ_STAT
);
1981 spin_unlock(&host
->lock
);
1985 return IRQ_RETVAL(rc
);
1988 unsigned int ahci_qc_issue(struct ata_queued_cmd
*qc
)
1990 struct ata_port
*ap
= qc
->ap
;
1991 void __iomem
*port_mmio
= ahci_port_base(ap
);
1992 struct ahci_port_priv
*pp
= ap
->private_data
;
1994 /* Keep track of the currently active link. It will be used
1995 * in completion path to determine whether NCQ phase is in
1998 pp
->active_link
= qc
->dev
->link
;
2000 if (ata_is_ncq(qc
->tf
.protocol
))
2001 writel(1 << qc
->tag
, port_mmio
+ PORT_SCR_ACT
);
2003 if (pp
->fbs_enabled
&& pp
->fbs_last_dev
!= qc
->dev
->link
->pmp
) {
2004 u32 fbs
= readl(port_mmio
+ PORT_FBS
);
2005 fbs
&= ~(PORT_FBS_DEV_MASK
| PORT_FBS_DEC
);
2006 fbs
|= qc
->dev
->link
->pmp
<< PORT_FBS_DEV_OFFSET
;
2007 writel(fbs
, port_mmio
+ PORT_FBS
);
2008 pp
->fbs_last_dev
= qc
->dev
->link
->pmp
;
2011 writel(1 << qc
->tag
, port_mmio
+ PORT_CMD_ISSUE
);
2013 ahci_sw_activity(qc
->dev
->link
);
2017 EXPORT_SYMBOL_GPL(ahci_qc_issue
);
2019 static bool ahci_qc_fill_rtf(struct ata_queued_cmd
*qc
)
2021 struct ahci_port_priv
*pp
= qc
->ap
->private_data
;
2022 u8
*rx_fis
= pp
->rx_fis
;
2024 if (pp
->fbs_enabled
)
2025 rx_fis
+= qc
->dev
->link
->pmp
* AHCI_RX_FIS_SZ
;
2028 * After a successful execution of an ATA PIO data-in command,
2029 * the device doesn't send D2H Reg FIS to update the TF and
2030 * the host should take TF and E_Status from the preceding PIO
2033 if (qc
->tf
.protocol
== ATA_PROT_PIO
&& qc
->dma_dir
== DMA_FROM_DEVICE
&&
2034 !(qc
->flags
& ATA_QCFLAG_FAILED
)) {
2035 ata_tf_from_fis(rx_fis
+ RX_FIS_PIO_SETUP
, &qc
->result_tf
);
2036 qc
->result_tf
.command
= (rx_fis
+ RX_FIS_PIO_SETUP
)[15];
2038 ata_tf_from_fis(rx_fis
+ RX_FIS_D2H_REG
, &qc
->result_tf
);
2043 static void ahci_freeze(struct ata_port
*ap
)
2045 void __iomem
*port_mmio
= ahci_port_base(ap
);
2048 writel(0, port_mmio
+ PORT_IRQ_MASK
);
2051 static void ahci_thaw(struct ata_port
*ap
)
2053 struct ahci_host_priv
*hpriv
= ap
->host
->private_data
;
2054 void __iomem
*mmio
= hpriv
->mmio
;
2055 void __iomem
*port_mmio
= ahci_port_base(ap
);
2057 struct ahci_port_priv
*pp
= ap
->private_data
;
2060 tmp
= readl(port_mmio
+ PORT_IRQ_STAT
);
2061 writel(tmp
, port_mmio
+ PORT_IRQ_STAT
);
2062 writel(1 << ap
->port_no
, mmio
+ HOST_IRQ_STAT
);
2064 /* turn IRQ back on */
2065 writel(pp
->intr_mask
, port_mmio
+ PORT_IRQ_MASK
);
2068 void ahci_error_handler(struct ata_port
*ap
)
2070 struct ahci_host_priv
*hpriv
= ap
->host
->private_data
;
2072 if (!(ap
->pflags
& ATA_PFLAG_FROZEN
)) {
2073 /* restart engine */
2074 hpriv
->stop_engine(ap
);
2075 hpriv
->start_engine(ap
);
2078 sata_pmp_error_handler(ap
);
2080 if (!ata_dev_enabled(ap
->link
.device
))
2081 hpriv
->stop_engine(ap
);
2083 EXPORT_SYMBOL_GPL(ahci_error_handler
);
2085 static void ahci_post_internal_cmd(struct ata_queued_cmd
*qc
)
2087 struct ata_port
*ap
= qc
->ap
;
2089 /* make DMA engine forget about the failed command */
2090 if (qc
->flags
& ATA_QCFLAG_FAILED
)
2091 ahci_kick_engine(ap
);
2094 static void ahci_set_aggressive_devslp(struct ata_port
*ap
, bool sleep
)
2096 struct ahci_host_priv
*hpriv
= ap
->host
->private_data
;
2097 void __iomem
*port_mmio
= ahci_port_base(ap
);
2098 struct ata_device
*dev
= ap
->link
.device
;
2099 u32 devslp
, dm
, dito
, mdat
, deto
, dito_conf
;
2101 unsigned int err_mask
;
2103 devslp
= readl(port_mmio
+ PORT_DEVSLP
);
2104 if (!(devslp
& PORT_DEVSLP_DSP
)) {
2105 dev_info(ap
->host
->dev
, "port does not support device sleep\n");
2109 /* disable device sleep */
2111 if (devslp
& PORT_DEVSLP_ADSE
) {
2112 writel(devslp
& ~PORT_DEVSLP_ADSE
,
2113 port_mmio
+ PORT_DEVSLP
);
2114 err_mask
= ata_dev_set_feature(dev
,
2115 SETFEATURES_SATA_DISABLE
,
2117 if (err_mask
&& err_mask
!= AC_ERR_DEV
)
2118 ata_dev_warn(dev
, "failed to disable DEVSLP\n");
2123 dm
= (devslp
& PORT_DEVSLP_DM_MASK
) >> PORT_DEVSLP_DM_OFFSET
;
2124 dito
= devslp_idle_timeout
/ (dm
+ 1);
2128 dito_conf
= (devslp
>> PORT_DEVSLP_DITO_OFFSET
) & 0x3FF;
2130 /* device sleep was already enabled and same dito */
2131 if ((devslp
& PORT_DEVSLP_ADSE
) && (dito_conf
== dito
))
2134 /* set DITO, MDAT, DETO and enable DevSlp, need to stop engine first */
2135 rc
= hpriv
->stop_engine(ap
);
2139 /* Use the nominal value 10 ms if the read MDAT is zero,
2140 * the nominal value of DETO is 20 ms.
2142 if (dev
->devslp_timing
[ATA_LOG_DEVSLP_VALID
] &
2143 ATA_LOG_DEVSLP_VALID_MASK
) {
2144 mdat
= dev
->devslp_timing
[ATA_LOG_DEVSLP_MDAT
] &
2145 ATA_LOG_DEVSLP_MDAT_MASK
;
2148 deto
= dev
->devslp_timing
[ATA_LOG_DEVSLP_DETO
];
2156 /* Make dito, mdat, deto bits to 0s */
2157 devslp
&= ~GENMASK_ULL(24, 2);
2158 devslp
|= ((dito
<< PORT_DEVSLP_DITO_OFFSET
) |
2159 (mdat
<< PORT_DEVSLP_MDAT_OFFSET
) |
2160 (deto
<< PORT_DEVSLP_DETO_OFFSET
) |
2162 writel(devslp
, port_mmio
+ PORT_DEVSLP
);
2164 hpriv
->start_engine(ap
);
2166 /* enable device sleep feature for the drive */
2167 err_mask
= ata_dev_set_feature(dev
,
2168 SETFEATURES_SATA_ENABLE
,
2170 if (err_mask
&& err_mask
!= AC_ERR_DEV
)
2171 ata_dev_warn(dev
, "failed to enable DEVSLP\n");
2174 static void ahci_enable_fbs(struct ata_port
*ap
)
2176 struct ahci_host_priv
*hpriv
= ap
->host
->private_data
;
2177 struct ahci_port_priv
*pp
= ap
->private_data
;
2178 void __iomem
*port_mmio
= ahci_port_base(ap
);
2182 if (!pp
->fbs_supported
)
2185 fbs
= readl(port_mmio
+ PORT_FBS
);
2186 if (fbs
& PORT_FBS_EN
) {
2187 pp
->fbs_enabled
= true;
2188 pp
->fbs_last_dev
= -1; /* initialization */
2192 rc
= hpriv
->stop_engine(ap
);
2196 writel(fbs
| PORT_FBS_EN
, port_mmio
+ PORT_FBS
);
2197 fbs
= readl(port_mmio
+ PORT_FBS
);
2198 if (fbs
& PORT_FBS_EN
) {
2199 dev_info(ap
->host
->dev
, "FBS is enabled\n");
2200 pp
->fbs_enabled
= true;
2201 pp
->fbs_last_dev
= -1; /* initialization */
2203 dev_err(ap
->host
->dev
, "Failed to enable FBS\n");
2205 hpriv
->start_engine(ap
);
2208 static void ahci_disable_fbs(struct ata_port
*ap
)
2210 struct ahci_host_priv
*hpriv
= ap
->host
->private_data
;
2211 struct ahci_port_priv
*pp
= ap
->private_data
;
2212 void __iomem
*port_mmio
= ahci_port_base(ap
);
2216 if (!pp
->fbs_supported
)
2219 fbs
= readl(port_mmio
+ PORT_FBS
);
2220 if ((fbs
& PORT_FBS_EN
) == 0) {
2221 pp
->fbs_enabled
= false;
2225 rc
= hpriv
->stop_engine(ap
);
2229 writel(fbs
& ~PORT_FBS_EN
, port_mmio
+ PORT_FBS
);
2230 fbs
= readl(port_mmio
+ PORT_FBS
);
2231 if (fbs
& PORT_FBS_EN
)
2232 dev_err(ap
->host
->dev
, "Failed to disable FBS\n");
2234 dev_info(ap
->host
->dev
, "FBS is disabled\n");
2235 pp
->fbs_enabled
= false;
2238 hpriv
->start_engine(ap
);
2241 static void ahci_pmp_attach(struct ata_port
*ap
)
2243 void __iomem
*port_mmio
= ahci_port_base(ap
);
2244 struct ahci_port_priv
*pp
= ap
->private_data
;
2247 cmd
= readl(port_mmio
+ PORT_CMD
);
2248 cmd
|= PORT_CMD_PMP
;
2249 writel(cmd
, port_mmio
+ PORT_CMD
);
2251 ahci_enable_fbs(ap
);
2253 pp
->intr_mask
|= PORT_IRQ_BAD_PMP
;
2256 * We must not change the port interrupt mask register if the
2257 * port is marked frozen, the value in pp->intr_mask will be
2258 * restored later when the port is thawed.
2260 * Note that during initialization, the port is marked as
2261 * frozen since the irq handler is not yet registered.
2263 if (!(ap
->pflags
& ATA_PFLAG_FROZEN
))
2264 writel(pp
->intr_mask
, port_mmio
+ PORT_IRQ_MASK
);
2267 static void ahci_pmp_detach(struct ata_port
*ap
)
2269 void __iomem
*port_mmio
= ahci_port_base(ap
);
2270 struct ahci_port_priv
*pp
= ap
->private_data
;
2273 ahci_disable_fbs(ap
);
2275 cmd
= readl(port_mmio
+ PORT_CMD
);
2276 cmd
&= ~PORT_CMD_PMP
;
2277 writel(cmd
, port_mmio
+ PORT_CMD
);
2279 pp
->intr_mask
&= ~PORT_IRQ_BAD_PMP
;
2281 /* see comment above in ahci_pmp_attach() */
2282 if (!(ap
->pflags
& ATA_PFLAG_FROZEN
))
2283 writel(pp
->intr_mask
, port_mmio
+ PORT_IRQ_MASK
);
2286 int ahci_port_resume(struct ata_port
*ap
)
2288 ahci_rpm_get_port(ap
);
2291 ahci_start_port(ap
);
2293 if (sata_pmp_attached(ap
))
2294 ahci_pmp_attach(ap
);
2296 ahci_pmp_detach(ap
);
2300 EXPORT_SYMBOL_GPL(ahci_port_resume
);
2303 static int ahci_port_suspend(struct ata_port
*ap
, pm_message_t mesg
)
2305 const char *emsg
= NULL
;
2308 rc
= ahci_deinit_port(ap
, &emsg
);
2310 ahci_power_down(ap
);
2312 ata_port_err(ap
, "%s (%d)\n", emsg
, rc
);
2313 ata_port_freeze(ap
);
2316 ahci_rpm_put_port(ap
);
2321 static int ahci_port_start(struct ata_port
*ap
)
2323 struct ahci_host_priv
*hpriv
= ap
->host
->private_data
;
2324 struct device
*dev
= ap
->host
->dev
;
2325 struct ahci_port_priv
*pp
;
2328 size_t dma_sz
, rx_fis_sz
;
2330 pp
= devm_kzalloc(dev
, sizeof(*pp
), GFP_KERNEL
);
2334 if (ap
->host
->n_ports
> 1) {
2335 pp
->irq_desc
= devm_kzalloc(dev
, 8, GFP_KERNEL
);
2336 if (!pp
->irq_desc
) {
2337 devm_kfree(dev
, pp
);
2340 snprintf(pp
->irq_desc
, 8,
2341 "%s%d", dev_driver_string(dev
), ap
->port_no
);
2344 /* check FBS capability */
2345 if ((hpriv
->cap
& HOST_CAP_FBS
) && sata_pmp_supported(ap
)) {
2346 void __iomem
*port_mmio
= ahci_port_base(ap
);
2347 u32 cmd
= readl(port_mmio
+ PORT_CMD
);
2348 if (cmd
& PORT_CMD_FBSCP
)
2349 pp
->fbs_supported
= true;
2350 else if (hpriv
->flags
& AHCI_HFLAG_YES_FBS
) {
2351 dev_info(dev
, "port %d can do FBS, forcing FBSCP\n",
2353 pp
->fbs_supported
= true;
2355 dev_warn(dev
, "port %d is not capable of FBS\n",
2359 if (pp
->fbs_supported
) {
2360 dma_sz
= AHCI_PORT_PRIV_FBS_DMA_SZ
;
2361 rx_fis_sz
= AHCI_RX_FIS_SZ
* 16;
2363 dma_sz
= AHCI_PORT_PRIV_DMA_SZ
;
2364 rx_fis_sz
= AHCI_RX_FIS_SZ
;
2367 mem
= dmam_alloc_coherent(dev
, dma_sz
, &mem_dma
, GFP_KERNEL
);
2370 memset(mem
, 0, dma_sz
);
2373 * First item in chunk of DMA memory: 32-slot command table,
2374 * 32 bytes each in size
2377 pp
->cmd_slot_dma
= mem_dma
;
2379 mem
+= AHCI_CMD_SLOT_SZ
;
2380 mem_dma
+= AHCI_CMD_SLOT_SZ
;
2383 * Second item: Received-FIS area
2386 pp
->rx_fis_dma
= mem_dma
;
2389 mem_dma
+= rx_fis_sz
;
2392 * Third item: data area for storing a single command
2393 * and its scatter-gather table
2396 pp
->cmd_tbl_dma
= mem_dma
;
2399 * Save off initial list of interrupts to be enabled.
2400 * This could be changed later
2402 pp
->intr_mask
= DEF_PORT_IRQ
;
2405 * Switch to per-port locking in case each port has its own MSI vector.
2407 if (hpriv
->flags
& AHCI_HFLAG_MULTI_MSI
) {
2408 spin_lock_init(&pp
->lock
);
2409 ap
->lock
= &pp
->lock
;
2412 ap
->private_data
= pp
;
2414 /* engage engines, captain */
2415 return ahci_port_resume(ap
);
2418 static void ahci_port_stop(struct ata_port
*ap
)
2420 const char *emsg
= NULL
;
2421 struct ahci_host_priv
*hpriv
= ap
->host
->private_data
;
2422 void __iomem
*host_mmio
= hpriv
->mmio
;
2425 /* de-initialize port */
2426 rc
= ahci_deinit_port(ap
, &emsg
);
2428 ata_port_warn(ap
, "%s (%d)\n", emsg
, rc
);
2431 * Clear GHC.IS to prevent stuck INTx after disabling MSI and
2434 writel(1 << ap
->port_no
, host_mmio
+ HOST_IRQ_STAT
);
2437 void ahci_print_info(struct ata_host
*host
, const char *scc_s
)
2439 struct ahci_host_priv
*hpriv
= host
->private_data
;
2440 u32 vers
, cap
, cap2
, impl
, speed
;
2441 const char *speed_s
;
2443 vers
= hpriv
->version
;
2446 impl
= hpriv
->port_map
;
2448 speed
= (cap
>> 20) & 0xf;
2451 else if (speed
== 2)
2453 else if (speed
== 3)
2459 "AHCI %02x%02x.%02x%02x "
2460 "%u slots %u ports %s Gbps 0x%x impl %s mode\n"
2463 (vers
>> 24) & 0xff,
2464 (vers
>> 16) & 0xff,
2468 ((cap
>> 8) & 0x1f) + 1,
2482 cap
& HOST_CAP_64
? "64bit " : "",
2483 cap
& HOST_CAP_NCQ
? "ncq " : "",
2484 cap
& HOST_CAP_SNTF
? "sntf " : "",
2485 cap
& HOST_CAP_MPS
? "ilck " : "",
2486 cap
& HOST_CAP_SSS
? "stag " : "",
2487 cap
& HOST_CAP_ALPM
? "pm " : "",
2488 cap
& HOST_CAP_LED
? "led " : "",
2489 cap
& HOST_CAP_CLO
? "clo " : "",
2490 cap
& HOST_CAP_ONLY
? "only " : "",
2491 cap
& HOST_CAP_PMP
? "pmp " : "",
2492 cap
& HOST_CAP_FBS
? "fbs " : "",
2493 cap
& HOST_CAP_PIO_MULTI
? "pio " : "",
2494 cap
& HOST_CAP_SSC
? "slum " : "",
2495 cap
& HOST_CAP_PART
? "part " : "",
2496 cap
& HOST_CAP_CCC
? "ccc " : "",
2497 cap
& HOST_CAP_EMS
? "ems " : "",
2498 cap
& HOST_CAP_SXS
? "sxs " : "",
2499 cap2
& HOST_CAP2_DESO
? "deso " : "",
2500 cap2
& HOST_CAP2_SADM
? "sadm " : "",
2501 cap2
& HOST_CAP2_SDS
? "sds " : "",
2502 cap2
& HOST_CAP2_APST
? "apst " : "",
2503 cap2
& HOST_CAP2_NVMHCI
? "nvmp " : "",
2504 cap2
& HOST_CAP2_BOH
? "boh " : ""
2507 EXPORT_SYMBOL_GPL(ahci_print_info
);
2509 void ahci_set_em_messages(struct ahci_host_priv
*hpriv
,
2510 struct ata_port_info
*pi
)
2513 void __iomem
*mmio
= hpriv
->mmio
;
2514 u32 em_loc
= readl(mmio
+ HOST_EM_LOC
);
2515 u32 em_ctl
= readl(mmio
+ HOST_EM_CTL
);
2517 if (!ahci_em_messages
|| !(hpriv
->cap
& HOST_CAP_EMS
))
2520 messages
= (em_ctl
& EM_CTRL_MSG_TYPE
) >> 16;
2524 hpriv
->em_loc
= ((em_loc
>> 16) * 4);
2525 hpriv
->em_buf_sz
= ((em_loc
& 0xff) * 4);
2526 hpriv
->em_msg_type
= messages
;
2527 pi
->flags
|= ATA_FLAG_EM
;
2528 if (!(em_ctl
& EM_CTL_ALHD
))
2529 pi
->flags
|= ATA_FLAG_SW_ACTIVITY
;
2532 EXPORT_SYMBOL_GPL(ahci_set_em_messages
);
2534 static int ahci_host_activate_multi_irqs(struct ata_host
*host
,
2535 struct scsi_host_template
*sht
)
2537 struct ahci_host_priv
*hpriv
= host
->private_data
;
2540 rc
= ata_host_start(host
);
2544 * Requests IRQs according to AHCI-1.1 when multiple MSIs were
2545 * allocated. That is one MSI per port, starting from @irq.
2547 for (i
= 0; i
< host
->n_ports
; i
++) {
2548 struct ahci_port_priv
*pp
= host
->ports
[i
]->private_data
;
2549 int irq
= hpriv
->get_irq_vector(host
, i
);
2551 /* Do not receive interrupts sent by dummy ports */
2557 rc
= devm_request_irq(host
->dev
, irq
, ahci_multi_irqs_intr_hard
,
2558 0, pp
->irq_desc
, host
->ports
[i
]);
2562 ata_port_desc(host
->ports
[i
], "irq %d", irq
);
2565 return ata_host_register(host
, sht
);
2569 * ahci_host_activate - start AHCI host, request IRQs and register it
2570 * @host: target ATA host
2571 * @sht: scsi_host_template to use when registering the host
2574 * Inherited from calling layer (may sleep).
2577 * 0 on success, -errno otherwise.
2579 int ahci_host_activate(struct ata_host
*host
, struct scsi_host_template
*sht
)
2581 struct ahci_host_priv
*hpriv
= host
->private_data
;
2582 int irq
= hpriv
->irq
;
2585 if (hpriv
->flags
& AHCI_HFLAG_MULTI_MSI
) {
2586 if (hpriv
->irq_handler
)
2588 "both AHCI_HFLAG_MULTI_MSI flag set and custom irq handler implemented\n");
2589 if (!hpriv
->get_irq_vector
) {
2591 "AHCI_HFLAG_MULTI_MSI requires ->get_irq_vector!\n");
2595 rc
= ahci_host_activate_multi_irqs(host
, sht
);
2597 rc
= ata_host_activate(host
, irq
, hpriv
->irq_handler
,
2604 EXPORT_SYMBOL_GPL(ahci_host_activate
);
2606 MODULE_AUTHOR("Jeff Garzik");
2607 MODULE_DESCRIPTION("Common AHCI SATA low-level routines");
2608 MODULE_LICENSE("GPL");