usr/src/uts/common/io/sata/adapters/ahci/ahci.c

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright (c) 2006, 2010, Oracle and/or its affiliates. All rights reserved.
 * Copyright 2018 Nexenta Systems, Inc.  All rights reserved.
 * Copyright 2018 OmniOS Community Edition (OmniOSce) Association.
 */

/*
 * AHCI (Advanced Host Controller Interface) SATA HBA Driver
 *
 * Power Management Support
 * ------------------------
 *
 * At the moment, the ahci driver only implements suspend/resume to
 * support Suspend to RAM on X86 feature. Device power management isn't
 * implemented, link power management is disabled, and hot plug isn't
 * allowed during the period from suspend to resume.
 *
 * For s/r support, the ahci driver only need to implement DDI_SUSPEND
 * and DDI_RESUME entries, and don't need to take care of new requests
 * sent down after suspend because the target driver (sd) has already
 * handled these conditions, and blocked these requests. For the detailed
 * information, please check with sdopen, sdclose and sdioctl routines.
 *
 */

#include <sys/note.h>
#include <sys/scsi/scsi.h>
#include <sys/pci.h>
#include <sys/disp.h>
#include <sys/sata/sata_hba.h>
#include <sys/sata/adapters/ahci/ahcireg.h>
#include <sys/sata/adapters/ahci/ahcivar.h>

/*
 * FMA header files
 */
#include <sys/ddifm.h>
#include <sys/fm/protocol.h>
#include <sys/fm/util.h>
#include <sys/fm/io/ddi.h>

/*
 * This is the string displayed by modinfo, etc.
 */
static char ahci_ident[] = "ahci driver";

/*
 * Function prototypes for driver entry points
 */
static	int ahci_attach(dev_info_t *, ddi_attach_cmd_t);
static	int ahci_detach(dev_info_t *, ddi_detach_cmd_t);
static	int ahci_getinfo(dev_info_t *, ddi_info_cmd_t, void *, void **);
static	int ahci_quiesce(dev_info_t *);

/*
 * Function prototypes for SATA Framework interfaces
 */
static	int ahci_register_sata_hba_tran(ahci_ctl_t *, uint32_t);
static	int ahci_unregister_sata_hba_tran(ahci_ctl_t *);

static	int ahci_tran_probe_port(dev_info_t *, sata_device_t *);
static	int ahci_tran_start(dev_info_t *, sata_pkt_t *spkt);
static	int ahci_tran_abort(dev_info_t *, sata_pkt_t *, int);
static	int ahci_tran_reset_dport(dev_info_t *, sata_device_t *);
static	int ahci_tran_hotplug_port_activate(dev_info_t *, sata_device_t *);
static	int ahci_tran_hotplug_port_deactivate(dev_info_t *, sata_device_t *);
#if defined(__lock_lint)
static	int ahci_selftest(dev_info_t *, sata_device_t *);
#endif

/*
 * FMA Prototypes
 */
static	void ahci_fm_init(ahci_ctl_t *);
static	void ahci_fm_fini(ahci_ctl_t *);
static	int ahci_fm_error_cb(dev_info_t *, ddi_fm_error_t *, const void*);
int	ahci_check_acc_handle(ddi_acc_handle_t);
int	ahci_check_dma_handle(ddi_dma_handle_t);
void	ahci_fm_ereport(ahci_ctl_t *, char *);
static	int ahci_check_all_handle(ahci_ctl_t *);
static	int ahci_check_ctl_handle(ahci_ctl_t *);
static	int ahci_check_port_handle(ahci_ctl_t *, int);
static	int ahci_check_slot_handle(ahci_port_t *, int);

/*
 * Local function prototypes
 */
static	int ahci_setup_port_base_addresses(ahci_ctl_t *, ahci_port_t *);
static	int ahci_alloc_ports_state(ahci_ctl_t *);
static	void ahci_dealloc_ports_state(ahci_ctl_t *);
static	int ahci_alloc_port_state(ahci_ctl_t *, uint8_t);
static	void ahci_dealloc_port_state(ahci_ctl_t *, uint8_t);
static	int ahci_alloc_rcvd_fis(ahci_ctl_t *, ahci_port_t *);
static	void ahci_dealloc_rcvd_fis(ahci_port_t *);
static	int ahci_alloc_cmd_list(ahci_ctl_t *, ahci_port_t *);
static	void ahci_dealloc_cmd_list(ahci_ctl_t *, ahci_port_t *);
static  int ahci_alloc_cmd_tables(ahci_ctl_t *, ahci_port_t *);
static  void ahci_dealloc_cmd_tables(ahci_ctl_t *, ahci_port_t *);
static	void ahci_alloc_pmult(ahci_ctl_t *, ahci_port_t *);
static	void ahci_dealloc_pmult(ahci_ctl_t *, ahci_port_t *);

static	int ahci_initialize_controller(ahci_ctl_t *);
static	void ahci_uninitialize_controller(ahci_ctl_t *);
static	int ahci_initialize_port(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *);
static	int ahci_config_space_init(ahci_ctl_t *);
static	void ahci_staggered_spin_up(ahci_ctl_t *, uint8_t);

static	void ahci_drain_ports_taskq(ahci_ctl_t *);
static	int ahci_rdwr_pmult(ahci_ctl_t *, ahci_addr_t *, uint8_t, uint32_t *,
    uint8_t);
static	int ahci_read_pmult(ahci_ctl_t *, ahci_addr_t *, uint8_t, uint32_t *);
static	int ahci_write_pmult(ahci_ctl_t *, ahci_addr_t *, uint8_t, uint32_t);
static	int ahci_update_pmult_pscr(ahci_ctl_t *, ahci_addr_t *,
    sata_device_t *);
static	int ahci_update_pmult_gscr(ahci_ctl_t *, ahci_addr_t *,
    sata_pmult_gscr_t *);
static	int ahci_initialize_pmult(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *,
    sata_device_t *);
static	int ahci_initialize_pmport(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *);
static	int ahci_probe_pmult(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *);
static	int ahci_probe_pmport(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *,
    sata_device_t *);

static	void ahci_disable_interface_pm(ahci_ctl_t *, uint8_t);
static	int ahci_start_port(ahci_ctl_t *, ahci_port_t *, uint8_t);
static	void ahci_find_dev_signature(ahci_ctl_t *, ahci_port_t *,
    ahci_addr_t *);
static	void ahci_update_sata_registers(ahci_ctl_t *, uint8_t, sata_device_t *);
static	int ahci_deliver_satapkt(ahci_ctl_t *, ahci_port_t *,
    ahci_addr_t *, sata_pkt_t *);
static	int ahci_do_sync_start(ahci_ctl_t *, ahci_port_t *,
    ahci_addr_t *, sata_pkt_t *);
static	int ahci_claim_free_slot(ahci_ctl_t *, ahci_port_t *,
    ahci_addr_t *, int);
static  void ahci_copy_err_cnxt(sata_cmd_t *, ahci_fis_d2h_register_t *);
static	void ahci_copy_ncq_err_page(sata_cmd_t *,
    struct sata_ncq_error_recovery_page *);
static	void ahci_copy_out_regs(sata_cmd_t *, ahci_fis_d2h_register_t *);
static	void ahci_add_doneq(ahci_port_t *, sata_pkt_t *, int);
static	void ahci_flush_doneq(ahci_port_t *);

static	int ahci_software_reset(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *);
static	int ahci_hba_reset(ahci_ctl_t *);
static	int ahci_port_reset(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *);
static	int ahci_pmport_reset(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *);
static	void ahci_reject_all_abort_pkts(ahci_ctl_t *, ahci_port_t *, uint8_t);
static	int ahci_reset_device_reject_pkts(ahci_ctl_t *, ahci_port_t *,
    ahci_addr_t *);
static	int ahci_reset_pmdevice_reject_pkts(ahci_ctl_t *, ahci_port_t *,
    ahci_addr_t *);
static	int ahci_reset_port_reject_pkts(ahci_ctl_t *, ahci_port_t *,
    ahci_addr_t *);
static	int ahci_reset_hba_reject_pkts(ahci_ctl_t *);
static	int ahci_put_port_into_notrunning_state(ahci_ctl_t *, ahci_port_t *,
    uint8_t);
static	int ahci_restart_port_wait_till_ready(ahci_ctl_t *, ahci_port_t *,
    uint8_t, int, int *);
static	void ahci_mop_commands(ahci_ctl_t *, ahci_port_t *, uint32_t,
    uint32_t, uint32_t, uint32_t, uint32_t);
static	uint32_t ahci_get_rdlogext_data(ahci_ctl_t *, ahci_port_t *, uint8_t);
static void ahci_get_rqsense_data(ahci_ctl_t *, ahci_port_t *,
    uint8_t, sata_pkt_t *);
static	void ahci_fatal_error_recovery_handler(ahci_ctl_t *, ahci_port_t *,
    ahci_addr_t *, uint32_t);
static	void ahci_pmult_error_recovery_handler(ahci_ctl_t *, ahci_port_t *,
    uint8_t, uint32_t);
static	void ahci_timeout_pkts(ahci_ctl_t *, ahci_port_t *,
    uint8_t, uint32_t);
static	void ahci_events_handler(void *);
static	void ahci_watchdog_handler(ahci_ctl_t *);

static	uint_t ahci_intr(caddr_t, caddr_t);
static	void ahci_port_intr(ahci_ctl_t *, ahci_port_t *, uint8_t);
static	int ahci_add_intrs(ahci_ctl_t *, int);
static	void ahci_rem_intrs(ahci_ctl_t *);
static	void ahci_enable_all_intrs(ahci_ctl_t *);
static	void ahci_disable_all_intrs(ahci_ctl_t *);
static	void ahci_enable_port_intrs(ahci_ctl_t *, uint8_t);
static	void ahci_disable_port_intrs(ahci_ctl_t *, uint8_t);

static  int ahci_intr_cmd_cmplt(ahci_ctl_t *, ahci_port_t *, uint8_t);
static	int ahci_intr_set_device_bits(ahci_ctl_t *, ahci_port_t *, uint8_t);
static	int ahci_intr_ncq_events(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *);
static	int ahci_intr_pmult_sntf_events(ahci_ctl_t *, ahci_port_t *, uint8_t);
static	int ahci_intr_port_connect_change(ahci_ctl_t *, ahci_port_t *, uint8_t);
static	int ahci_intr_device_mechanical_presence_status(ahci_ctl_t *,
    ahci_port_t *, uint8_t);
static	int ahci_intr_phyrdy_change(ahci_ctl_t *, ahci_port_t *, uint8_t);
static	int ahci_intr_non_fatal_error(ahci_ctl_t *, ahci_port_t *,
    uint8_t, uint32_t);
static  int ahci_intr_fatal_error(ahci_ctl_t *, ahci_port_t *,
    uint8_t, uint32_t);
static	int ahci_intr_cold_port_detect(ahci_ctl_t *, ahci_port_t *, uint8_t);

static	void ahci_get_ahci_addr(ahci_ctl_t *, sata_device_t *, ahci_addr_t *);
static	int ahci_get_num_implemented_ports(uint32_t);
static  void ahci_log_fatal_error_message(ahci_ctl_t *, uint8_t, uint32_t);
static	void ahci_dump_commands(ahci_ctl_t *, uint8_t, uint32_t);
static	void ahci_log_serror_message(ahci_ctl_t *, uint8_t, uint32_t, int);
#if AHCI_DEBUG
static	void ahci_log(ahci_ctl_t *, uint_t, char *, ...);
#endif


/*
 * DMA attributes for the data buffer
 *
 * dma_attr_addr_hi will be changed to 0xffffffffull if the HBA
 * does not support 64-bit addressing
 */
static ddi_dma_attr_t buffer_dma_attr = {
	DMA_ATTR_V0,		/* dma_attr_version */
	0x0ull,			/* dma_attr_addr_lo: lowest bus address */
	0xffffffffffffffffull,	/* dma_attr_addr_hi: highest bus address */
	0x3fffffull,		/* dma_attr_count_max i.e. for one cookie */
	0x2ull,			/* dma_attr_align: word aligned */
	1,			/* dma_attr_burstsizes */
	1,			/* dma_attr_minxfer */
	0xffffffffull,		/* dma_attr_maxxfer i.e. includes all cookies */
	0xffffffffull,		/* dma_attr_seg */
	AHCI_PRDT_NUMBER,	/* dma_attr_sgllen */
	512,			/* dma_attr_granular */
	0,			/* dma_attr_flags */
};

/*
 * DMA attributes for the rcvd FIS
 *
 * dma_attr_addr_hi will be changed to 0xffffffffull if the HBA
 * does not support 64-bit addressing
 */
static ddi_dma_attr_t rcvd_fis_dma_attr = {
	DMA_ATTR_V0,		/* dma_attr_version */
	0x0ull,			/* dma_attr_addr_lo: lowest bus address */
	0xffffffffffffffffull,	/* dma_attr_addr_hi: highest bus address */
	0xffffffffull,		/* dma_attr_count_max i.e. for one cookie */
	0x100ull,		/* dma_attr_align: 256-byte aligned */
	1,			/* dma_attr_burstsizes */
	1,			/* dma_attr_minxfer */
	0xffffffffull,		/* dma_attr_maxxfer i.e. includes all cookies */
	0xffffffffull,		/* dma_attr_seg */
	1,			/* dma_attr_sgllen */
	1,			/* dma_attr_granular */
	0,			/* dma_attr_flags */
};

/*
 * DMA attributes for the command list
 *
 * dma_attr_addr_hi will be changed to 0xffffffffull if the HBA
 * does not support 64-bit addressing
 */
static ddi_dma_attr_t cmd_list_dma_attr = {
	DMA_ATTR_V0,		/* dma_attr_version */
	0x0ull,			/* dma_attr_addr_lo: lowest bus address */
	0xffffffffffffffffull,	/* dma_attr_addr_hi: highest bus address */
	0xffffffffull,		/* dma_attr_count_max i.e. for one cookie */
	0x400ull,		/* dma_attr_align: 1K-byte aligned */
	1,			/* dma_attr_burstsizes */
	1,			/* dma_attr_minxfer */
	0xffffffffull,		/* dma_attr_maxxfer i.e. includes all cookies */
	0xffffffffull,		/* dma_attr_seg */
	1,			/* dma_attr_sgllen */
	1,			/* dma_attr_granular */
	0,			/* dma_attr_flags */
};

/*
 * DMA attributes for cmd tables
 *
 * dma_attr_addr_hi will be changed to 0xffffffffull if the HBA
 * does not support 64-bit addressing
 */
static ddi_dma_attr_t cmd_table_dma_attr = {
	DMA_ATTR_V0,		/* dma_attr_version */
	0x0ull,			/* dma_attr_addr_lo: lowest bus address */
	0xffffffffffffffffull,	/* dma_attr_addr_hi: highest bus address */
	0xffffffffull,		/* dma_attr_count_max i.e. for one cookie */
	0x80ull,		/* dma_attr_align: 128-byte aligned */
	1,			/* dma_attr_burstsizes */
	1,			/* dma_attr_minxfer */
	0xffffffffull,		/* dma_attr_maxxfer i.e. includes all cookies */
	0xffffffffull,		/* dma_attr_seg */
	1,			/* dma_attr_sgllen */
	1,			/* dma_attr_granular */
	0,			/* dma_attr_flags */
};


/* Device access attributes */
static ddi_device_acc_attr_t accattr = {
	DDI_DEVICE_ATTR_V1,
	DDI_STRUCTURE_LE_ACC,
	DDI_STRICTORDER_ACC,
	DDI_DEFAULT_ACC
};


static struct dev_ops ahcictl_dev_ops = {
	DEVO_REV,		/* devo_rev */
	0,			/* refcnt  */
	ahci_getinfo,		/* info */
	nulldev,		/* identify */
	nulldev,		/* probe */
	ahci_attach,		/* attach */
	ahci_detach,		/* detach */
	nodev,			/* no reset */
	(struct cb_ops *)0,	/* driver operations */
	NULL,			/* bus operations */
	NULL,			/* power */
	ahci_quiesce,		/* quiesce */
};

static sata_tran_hotplug_ops_t ahci_tran_hotplug_ops = {
	SATA_TRAN_HOTPLUG_OPS_REV_1,
	ahci_tran_hotplug_port_activate,
	ahci_tran_hotplug_port_deactivate
};

extern struct mod_ops mod_driverops;

static  struct modldrv modldrv = {
	&mod_driverops,		/* driverops */
	ahci_ident,		/* short description */
	&ahcictl_dev_ops,	/* driver ops */
};

static  struct modlinkage modlinkage = {
	MODREV_1,
	&modldrv,
	NULL
};

/* The following variables are watchdog handler related */
static clock_t ahci_watchdog_timeout = 5; /* 5 seconds */
static clock_t ahci_watchdog_tick;

/*
 * This static variable indicates the size of command table,
 * and it's changeable with prdt number, which ahci_dma_prdt_number
 * indicates.
 */
static size_t ahci_cmd_table_size;

/*
 * The below global variables are tunable via /etc/system
 *
 *	ahci_dma_prdt_number
 *	ahci_msi_enabled
 *	ahci_buf_64bit_dma
 *	ahci_commu_64bit_dma
 */

/* The number of Physical Region Descriptor Table(PRDT) in Command Table */
int ahci_dma_prdt_number = AHCI_PRDT_NUMBER;

/* AHCI MSI is tunable */
boolean_t ahci_msi_enabled = B_TRUE;

/*
 * 64-bit dma addressing for data buffer is tunable
 *
 * The variable controls only the below value:
 *	DBAU (upper 32-bits physical address of data block)
 */
boolean_t ahci_buf_64bit_dma = B_TRUE;

/*
 * 64-bit dma addressing for communication system descriptors is tunable
 *
 * The variable controls the below three values:
 *
 *	PxCLBU (upper 32-bits for the command list base physical address)
 *	PxFBU (upper 32-bits for the received FIS base physical address)
 *	CTBAU (upper 32-bits of command table base)
 */
boolean_t ahci_commu_64bit_dma = B_TRUE;

/*
 * By default, 64-bit dma for data buffer will be disabled for AMD/ATI SB600
 * chipset. If the users want to have a try with 64-bit dma, please change
 * the below variable value to enable it.
 */
boolean_t sb600_buf_64bit_dma_disable = B_TRUE;

/*
 * By default, 64-bit dma for command buffer will be disabled for AMD/ATI
 * SB600/700/710/750/800. If the users want to have a try with 64-bit dma,
 * please change the below value to enable it.
 */
boolean_t sbxxx_commu_64bit_dma_disable = B_TRUE;


/*
 * End of global tunable variable definition
 */

#if AHCI_DEBUG
uint32_t ahci_debug_flags = 0;
#else
uint32_t ahci_debug_flags = (AHCIDBG_ERRS|AHCIDBG_TIMEOUT);
#endif


#if AHCI_DEBUG
/* The following is needed for ahci_log() */
static kmutex_t ahci_log_mutex;
static char ahci_log_buf[512];
#endif

/* Opaque state pointer initialized by ddi_soft_state_init() */
static void *ahci_statep = NULL;

/*
 *  ahci module initialization.
 */
int
_init(void)
{
	int	ret;

	ret = ddi_soft_state_init(&ahci_statep, sizeof (ahci_ctl_t), 0);
	if (ret != 0) {
		goto err_out;
	}

#if AHCI_DEBUG
	mutex_init(&ahci_log_mutex, NULL, MUTEX_DRIVER, NULL);
#endif

	if ((ret = sata_hba_init(&modlinkage)) != 0) {
#if AHCI_DEBUG
		mutex_destroy(&ahci_log_mutex);
#endif
		ddi_soft_state_fini(&ahci_statep);
		goto err_out;
	}

	/* watchdog tick */
	ahci_watchdog_tick = drv_usectohz(
	    (clock_t)ahci_watchdog_timeout * 1000000);

	ret = mod_install(&modlinkage);
	if (ret != 0) {
		sata_hba_fini(&modlinkage);
#if AHCI_DEBUG
		mutex_destroy(&ahci_log_mutex);
#endif
		ddi_soft_state_fini(&ahci_statep);
		goto err_out;
	}

	return (ret);

err_out:
	cmn_err(CE_WARN, "!ahci: Module init failed");
	return (ret);
}

/*
 * ahci module uninitialize.
 */
int
_fini(void)
{
	int	ret;

	ret = mod_remove(&modlinkage);
	if (ret != 0) {
		return (ret);
	}

	/* Remove the resources allocated in _init(). */
	sata_hba_fini(&modlinkage);
#if AHCI_DEBUG
	mutex_destroy(&ahci_log_mutex);
#endif
	ddi_soft_state_fini(&ahci_statep);

	return (ret);
}

/*
 * _info entry point
 */
int
_info(struct modinfo *modinfop)
{
	return (mod_info(&modlinkage, modinfop));
}

/*
 * The attach entry point for dev_ops.
 */
static int
ahci_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
{
	ahci_ctl_t *ahci_ctlp = NULL;
	int instance = ddi_get_instance(dip);
	int status;
	int attach_state;
	uint32_t cap_status, ahci_version;
	uint32_t ghc_control;
	int intr_types;
	int i;
	pci_regspec_t *regs;
	int regs_length;
	int rnumber;
#if AHCI_DEBUG
	int speed;
#endif

	AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, "ahci_attach enter",
	    NULL);

	switch (cmd) {
	case DDI_ATTACH:
		break;

	case DDI_RESUME:

		/*
		 * During DDI_RESUME, the hardware state of the device
		 * (power may have been removed from the device) must be
		 * restored, allow pending requests to continue, and
		 * service new requests.
		 */
		ahci_ctlp = ddi_get_soft_state(ahci_statep, instance);
		mutex_enter(&ahci_ctlp->ahcictl_mutex);

		/*
		 * GHC.AE must be set to 1 before any other AHCI register
		 * is accessed
		 */
		ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
		    (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp));
		ghc_control |= AHCI_HBA_GHC_AE;
		ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
		    (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control);

		/* Restart watch thread */
		if (ahci_ctlp->ahcictl_timeout_id == 0)
			ahci_ctlp->ahcictl_timeout_id = timeout(
			    (void (*)(void *))ahci_watchdog_handler,
			    (caddr_t)ahci_ctlp, ahci_watchdog_tick);

		mutex_exit(&ahci_ctlp->ahcictl_mutex);

		/*
		 * Re-initialize the controller and enable the interrupts and
		 * restart all the ports.
		 *
		 * Note that so far we don't support hot-plug during
		 * suspend/resume.
		 */
		if (ahci_initialize_controller(ahci_ctlp) != AHCI_SUCCESS) {
			AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PM, ahci_ctlp,
			    "Failed to initialize the controller "
			    "during DDI_RESUME", NULL);
			return (DDI_FAILURE);
		}

		mutex_enter(&ahci_ctlp->ahcictl_mutex);
		ahci_ctlp->ahcictl_flags &= ~AHCI_SUSPEND;
		mutex_exit(&ahci_ctlp->ahcictl_mutex);

		return (DDI_SUCCESS);

	default:
		return (DDI_FAILURE);
	}

	attach_state = AHCI_ATTACH_STATE_NONE;

	/* Allocate soft state */
	status = ddi_soft_state_zalloc(ahci_statep, instance);
	if (status != DDI_SUCCESS) {
		cmn_err(CE_WARN, "!ahci%d: Cannot allocate soft state",
		    instance);
		goto err_out;
	}

	ahci_ctlp = ddi_get_soft_state(ahci_statep, instance);
	ahci_ctlp->ahcictl_flags |= AHCI_ATTACH;
	ahci_ctlp->ahcictl_dip = dip;

	/* Initialize the cport/port mapping */
	for (i = 0; i < AHCI_MAX_PORTS; i++) {
		ahci_ctlp->ahcictl_port_to_cport[i] = 0xff;
		ahci_ctlp->ahcictl_cport_to_port[i] = 0xff;
	}

	attach_state |= AHCI_ATTACH_STATE_STATEP_ALLOC;

	/* Initialize FMA properties */
	ahci_fm_init(ahci_ctlp);

	attach_state |= AHCI_ATTACH_STATE_FMA;

	/*
	 * Now map the AHCI base address; which includes global
	 * registers and port control registers
	 *
	 * According to the spec, the AHCI Base Address is BAR5,
	 * but BAR0-BAR4 are optional, so we need to check which
	 * rnumber is used for BAR5.
	 */

	/*
	 * search through DDI "reg" property for the AHCI register set
	 */
	if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, dip,
	    DDI_PROP_DONTPASS, "reg", (int **)&regs,
	    (uint_t *)&regs_length) != DDI_PROP_SUCCESS) {
		cmn_err(CE_WARN, "!ahci%d: Cannot lookup reg property",
		    instance);
		goto err_out;
	}

	/* AHCI Base Address is located at 0x24 offset */
	for (rnumber = 0; rnumber < regs_length; ++rnumber) {
		if ((regs[rnumber].pci_phys_hi & PCI_REG_REG_M)
		    == AHCI_PCI_RNUM)
			break;
	}

	ddi_prop_free(regs);

	if (rnumber == regs_length) {
		cmn_err(CE_WARN, "!ahci%d: Cannot find AHCI register set",
		    instance);
		goto err_out;
	}

	AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "rnumber = %d", rnumber);

	status = ddi_regs_map_setup(dip,
	    rnumber,
	    (caddr_t *)&ahci_ctlp->ahcictl_ahci_addr,
	    0,
	    0,
	    &accattr,
	    &ahci_ctlp->ahcictl_ahci_acc_handle);
	if (status != DDI_SUCCESS) {
		cmn_err(CE_WARN, "!ahci%d: Cannot map register space",
		    instance);
		goto err_out;
	}

	attach_state |= AHCI_ATTACH_STATE_REG_MAP;

	/*
	 * GHC.AE must be set to 1 before any other AHCI register
	 * is accessed
	 */
	ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
	    (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp));
	ghc_control |= AHCI_HBA_GHC_AE;
	ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
	    (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control);

	/* Get the AHCI version information */
	ahci_version = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
	    (uint32_t *)AHCI_GLOBAL_VS(ahci_ctlp));

	cmn_err(CE_NOTE, "!ahci%d: hba AHCI version = %x.%x", instance,
	    (ahci_version & 0xffff0000) >> 16,
	    ((ahci_version & 0x0000ff00) >> 4 |
	    (ahci_version & 0x000000ff)));

	/* We don't support controllers whose versions are lower than 1.0 */
	if (!(ahci_version & 0xffff0000)) {
		cmn_err(CE_WARN, "ahci%d: Don't support AHCI HBA with lower "
		    "than version 1.0", instance);
		goto err_out;
	}

	/* Get the HBA capabilities information */
	cap_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
	    (uint32_t *)AHCI_GLOBAL_CAP(ahci_ctlp));

	AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "hba capabilities = 0x%x",
	    cap_status);

	/* CAP2 (HBA Capabilities Extended) is available since AHCI spec 1.2 */
	if (ahci_version >= 0x00010200) {
		uint32_t cap2_status;

		/* Get the HBA capabilities extended information */
		cap2_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
		    (uint32_t *)AHCI_GLOBAL_CAP2(ahci_ctlp));

		AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
		    "hba capabilities extended = 0x%x", cap2_status);
	}

#if AHCI_DEBUG
	/* Get the interface speed supported by the HBA */
	speed = (cap_status & AHCI_HBA_CAP_ISS) >> AHCI_HBA_CAP_ISS_SHIFT;
	if (speed == 0x01) {
		AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
		    "hba interface speed support: Gen 1 (1.5Gbps)", NULL);
	} else if (speed == 0x10) {
		AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
		    "hba interface speed support: Gen 2 (3 Gbps)", NULL);
	} else if (speed == 0x11) {
		AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
		    "hba interface speed support: Gen 3 (6 Gbps)", NULL);
	}
#endif

	/* Get the number of command slots supported by the HBA */
	ahci_ctlp->ahcictl_num_cmd_slots =
	    ((cap_status & AHCI_HBA_CAP_NCS) >>
	    AHCI_HBA_CAP_NCS_SHIFT) + 1;

	AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "hba number of cmd slots: %d",
	    ahci_ctlp->ahcictl_num_cmd_slots);

	/* Get the bit map which indicates ports implemented by the HBA */
	ahci_ctlp->ahcictl_ports_implemented =
	    ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
	    (uint32_t *)AHCI_GLOBAL_PI(ahci_ctlp));

	AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "hba implementation of ports: 0x%x",
	    ahci_ctlp->ahcictl_ports_implemented);

	/* Max port number implemented */
	ahci_ctlp->ahcictl_num_ports =
	    ddi_fls(ahci_ctlp->ahcictl_ports_implemented);

	AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "hba number of ports: %d",
	    (cap_status & AHCI_HBA_CAP_NP) + 1);

	/* Get the number of implemented ports by the HBA */
	ahci_ctlp->ahcictl_num_implemented_ports =
	    ahci_get_num_implemented_ports(
	    ahci_ctlp->ahcictl_ports_implemented);

	AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
	    "hba number of implemented ports: %d",
	    ahci_ctlp->ahcictl_num_implemented_ports);

	/* Check whether HBA supports 64bit DMA addressing */
	if (!(cap_status & AHCI_HBA_CAP_S64A)) {
		ahci_ctlp->ahcictl_cap |= AHCI_CAP_BUF_32BIT_DMA;
		ahci_ctlp->ahcictl_cap |= AHCI_CAP_COMMU_32BIT_DMA;
		AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
		    "hba does not support 64-bit addressing", NULL);
	}

	/* Checking for the support of Port Multiplier */
	if (cap_status & AHCI_HBA_CAP_SPM) {
		ahci_ctlp->ahcictl_cap |= AHCI_CAP_PMULT_CBSS;
		AHCIDBG(AHCIDBG_INIT|AHCIDBG_PMULT, ahci_ctlp,
		    "hba supports port multiplier (CBSS)", NULL);

		/* Support FIS-based switching ? */
		if (cap_status & AHCI_HBA_CAP_FBSS) {
			ahci_ctlp->ahcictl_cap |= AHCI_CAP_PMULT_FBSS;
			AHCIDBG(AHCIDBG_INIT|AHCIDBG_PMULT, ahci_ctlp,
			    "hba supports FIS-based switching (FBSS)", NULL);
		}
	}

	/* Checking for Support Command List Override */
	if (cap_status & AHCI_HBA_CAP_SCLO) {
		ahci_ctlp->ahcictl_cap |= AHCI_CAP_SCLO;
		AHCIDBG(AHCIDBG_INIT|AHCIDBG_PMULT, ahci_ctlp,
		    "hba supports command list override.", NULL);
	}

	/* Checking for Asynchronous Notification */
	if (cap_status & AHCI_HBA_CAP_SSNTF) {
		ahci_ctlp->ahcictl_cap |= AHCI_CAP_SNTF;
		AHCIDBG(AHCIDBG_INIT|AHCIDBG_PMULT, ahci_ctlp,
		    "hba supports asynchronous notification.", NULL);
	}

	if (pci_config_setup(dip, &ahci_ctlp->ahcictl_pci_conf_handle)
	    != DDI_SUCCESS) {
		cmn_err(CE_WARN, "!ahci%d: Cannot set up pci configure space",
		    instance);
		goto err_out;
	}

	attach_state |= AHCI_ATTACH_STATE_PCICFG_SETUP;

	/*
	 * Check the pci configuration space, and set caps. We also
	 * handle the hardware defect in this function.
	 *
	 * For example, force ATI SB600 to use 32-bit dma addressing
	 * since it doesn't support 64-bit dma though its CAP register
	 * declares it support.
	 */
	if (ahci_config_space_init(ahci_ctlp) == AHCI_FAILURE) {
		cmn_err(CE_WARN, "!ahci%d: ahci_config_space_init failed",
		    instance);
		goto err_out;
	}

	/*
	 * Disable the whole controller interrupts before adding
	 * interrupt handlers(s).
	 */
	ahci_disable_all_intrs(ahci_ctlp);

	/* Get supported interrupt types */
	if (ddi_intr_get_supported_types(dip, &intr_types) != DDI_SUCCESS) {
		cmn_err(CE_WARN, "!ahci%d: ddi_intr_get_supported_types failed",
		    instance);
		goto err_out;
	}

	AHCIDBG(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp,
	    "ddi_intr_get_supported_types() returned: 0x%x",
	    intr_types);

	if (ahci_msi_enabled && (intr_types & DDI_INTR_TYPE_MSI)) {
		/*
		 * Try MSI first, but fall back to FIXED if failed
		 */
		if (ahci_add_intrs(ahci_ctlp, DDI_INTR_TYPE_MSI) ==
		    DDI_SUCCESS) {
			ahci_ctlp->ahcictl_intr_type = DDI_INTR_TYPE_MSI;
			AHCIDBG(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp,
			    "Using MSI interrupt type", NULL);
			goto intr_done;
		}

		AHCIDBG(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp,
		    "MSI registration failed, "
		    "trying FIXED interrupts", NULL);
	}

	if (intr_types & DDI_INTR_TYPE_FIXED) {
		if (ahci_add_intrs(ahci_ctlp, DDI_INTR_TYPE_FIXED) ==
		    DDI_SUCCESS) {
			ahci_ctlp->ahcictl_intr_type = DDI_INTR_TYPE_FIXED;
			AHCIDBG(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp,
			    "Using FIXED interrupt type", NULL);
			goto intr_done;
		}

		AHCIDBG(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp,
		    "FIXED interrupt registration failed", NULL);
	}

	cmn_err(CE_WARN, "!ahci%d: Interrupt registration failed", instance);

	goto err_out;

intr_done:

	attach_state |= AHCI_ATTACH_STATE_INTR_ADDED;

	/* Initialize the controller mutex */
	mutex_init(&ahci_ctlp->ahcictl_mutex, NULL, MUTEX_DRIVER,
	    (void *)(uintptr_t)ahci_ctlp->ahcictl_intr_pri);

	attach_state |= AHCI_ATTACH_STATE_MUTEX_INIT;

	if (ahci_dma_prdt_number < AHCI_MIN_PRDT_NUMBER) {
		ahci_dma_prdt_number = AHCI_MIN_PRDT_NUMBER;
	} else if (ahci_dma_prdt_number > AHCI_MAX_PRDT_NUMBER) {
		ahci_dma_prdt_number = AHCI_MAX_PRDT_NUMBER;
	}

	ahci_cmd_table_size = (sizeof (ahci_cmd_table_t) +
	    (ahci_dma_prdt_number - AHCI_PRDT_NUMBER) *
	    sizeof (ahci_prdt_item_t));

	AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
	    "ahci_attach: ahci_dma_prdt_number set by user is 0x%x,"
	    " ahci_cmd_table_size is 0x%x",
	    ahci_dma_prdt_number, ahci_cmd_table_size);

	if (ahci_dma_prdt_number != AHCI_PRDT_NUMBER)
		ahci_ctlp->ahcictl_buffer_dma_attr.dma_attr_sgllen =
		    ahci_dma_prdt_number;

	ahci_ctlp->ahcictl_buffer_dma_attr = buffer_dma_attr;
	ahci_ctlp->ahcictl_rcvd_fis_dma_attr = rcvd_fis_dma_attr;
	ahci_ctlp->ahcictl_cmd_list_dma_attr = cmd_list_dma_attr;
	ahci_ctlp->ahcictl_cmd_table_dma_attr = cmd_table_dma_attr;

	/*
	 * enable 64bit dma for data buffer for SB600 if
	 * sb600_buf_64bit_dma_disable is B_FALSE
	 */
	if ((ahci_buf_64bit_dma == B_FALSE) ||
	    ((ahci_ctlp->ahcictl_cap & AHCI_CAP_BUF_32BIT_DMA) &&
	    !(sb600_buf_64bit_dma_disable == B_FALSE &&
	    ahci_ctlp->ahcictl_venid == 0x1002 &&
	    ahci_ctlp->ahcictl_devid == 0x4380))) {
		ahci_ctlp->ahcictl_buffer_dma_attr.dma_attr_addr_hi =
		    0xffffffffull;
	}

	/*
	 * enable 64bit dma for command buffer for SB600/700/710/800
	 * if sbxxx_commu_64bit_dma_disable is B_FALSE
	 */
	if ((ahci_commu_64bit_dma == B_FALSE) ||
	    ((ahci_ctlp->ahcictl_cap & AHCI_CAP_COMMU_32BIT_DMA) &&
	    !(sbxxx_commu_64bit_dma_disable == B_FALSE &&
	    ahci_ctlp->ahcictl_venid == 0x1002 &&
	    (ahci_ctlp->ahcictl_devid == 0x4380 ||
	    ahci_ctlp->ahcictl_devid == 0x4391)))) {
		ahci_ctlp->ahcictl_rcvd_fis_dma_attr.dma_attr_addr_hi =
		    0xffffffffull;
		ahci_ctlp->ahcictl_cmd_list_dma_attr.dma_attr_addr_hi =
		    0xffffffffull;
		ahci_ctlp->ahcictl_cmd_table_dma_attr.dma_attr_addr_hi =
		    0xffffffffull;
	}

	/* Allocate the ports structure */
	status = ahci_alloc_ports_state(ahci_ctlp);
	if (status != AHCI_SUCCESS) {
		cmn_err(CE_WARN, "!ahci%d: Cannot allocate ports structure",
		    instance);
		goto err_out;
	}

	attach_state |= AHCI_ATTACH_STATE_PORT_ALLOC;

	/*
	 * Initialize the controller and ports.
	 */
	status = ahci_initialize_controller(ahci_ctlp);
	if (status != AHCI_SUCCESS) {
		cmn_err(CE_WARN, "!ahci%d: HBA initialization failed",
		    instance);
		goto err_out;
	}

	attach_state |= AHCI_ATTACH_STATE_HW_INIT;

	/* Start one thread to check packet timeouts */
	ahci_ctlp->ahcictl_timeout_id = timeout(
	    (void (*)(void *))ahci_watchdog_handler,
	    (caddr_t)ahci_ctlp, ahci_watchdog_tick);

	attach_state |= AHCI_ATTACH_STATE_TIMEOUT_ENABLED;

	if (ahci_register_sata_hba_tran(ahci_ctlp, cap_status)) {
		cmn_err(CE_WARN, "!ahci%d: sata hba tran registration failed",
		    instance);
		goto err_out;
	}

	/* Check all handles at the end of the attach operation. */
	if (ahci_check_all_handle(ahci_ctlp) != DDI_SUCCESS) {
		cmn_err(CE_WARN, "!ahci%d: invalid dma/acc handles",
		    instance);
		goto err_out;
	}

	ahci_ctlp->ahcictl_flags &= ~AHCI_ATTACH;

	AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "ahci_attach success!", NULL);

	return (DDI_SUCCESS);

err_out:
	/* FMA message */
	ahci_fm_ereport(ahci_ctlp, DDI_FM_DEVICE_NO_RESPONSE);
	ddi_fm_service_impact(ahci_ctlp->ahcictl_dip, DDI_SERVICE_LOST);

	if (attach_state & AHCI_ATTACH_STATE_TIMEOUT_ENABLED) {
		mutex_enter(&ahci_ctlp->ahcictl_mutex);
		(void) untimeout(ahci_ctlp->ahcictl_timeout_id);
		ahci_ctlp->ahcictl_timeout_id = 0;
		mutex_exit(&ahci_ctlp->ahcictl_mutex);
	}

	if (attach_state & AHCI_ATTACH_STATE_HW_INIT) {
		ahci_uninitialize_controller(ahci_ctlp);
	}

	if (attach_state & AHCI_ATTACH_STATE_PORT_ALLOC) {
		ahci_dealloc_ports_state(ahci_ctlp);
	}

	if (attach_state & AHCI_ATTACH_STATE_MUTEX_INIT) {
		mutex_destroy(&ahci_ctlp->ahcictl_mutex);
	}

	if (attach_state & AHCI_ATTACH_STATE_INTR_ADDED) {
		ahci_rem_intrs(ahci_ctlp);
	}

	if (attach_state & AHCI_ATTACH_STATE_PCICFG_SETUP) {
		pci_config_teardown(&ahci_ctlp->ahcictl_pci_conf_handle);
	}

	if (attach_state & AHCI_ATTACH_STATE_REG_MAP) {
		ddi_regs_map_free(&ahci_ctlp->ahcictl_ahci_acc_handle);
	}

	if (attach_state & AHCI_ATTACH_STATE_FMA) {
		ahci_fm_fini(ahci_ctlp);
	}

	if (attach_state & AHCI_ATTACH_STATE_STATEP_ALLOC) {
		ddi_soft_state_free(ahci_statep, instance);
	}

	return (DDI_FAILURE);
}

/*
 * The detach entry point for dev_ops.
 */
static int
ahci_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
{
	ahci_ctl_t *ahci_ctlp;
	int instance;
	int ret;

	instance = ddi_get_instance(dip);
	ahci_ctlp = ddi_get_soft_state(ahci_statep, instance);

	AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_detach enter", NULL);

	switch (cmd) {
	case DDI_DETACH:

		/* disable the interrupts for an uninterrupted detach */
		mutex_enter(&ahci_ctlp->ahcictl_mutex);
		ahci_disable_all_intrs(ahci_ctlp);
		mutex_exit(&ahci_ctlp->ahcictl_mutex);

		/* unregister from the sata framework. */
		ret = ahci_unregister_sata_hba_tran(ahci_ctlp);
		if (ret != AHCI_SUCCESS) {
			mutex_enter(&ahci_ctlp->ahcictl_mutex);
			ahci_enable_all_intrs(ahci_ctlp);
			mutex_exit(&ahci_ctlp->ahcictl_mutex);
			return (DDI_FAILURE);
		}

		mutex_enter(&ahci_ctlp->ahcictl_mutex);

		/* stop the watchdog handler */
		(void) untimeout(ahci_ctlp->ahcictl_timeout_id);
		ahci_ctlp->ahcictl_timeout_id = 0;

		mutex_exit(&ahci_ctlp->ahcictl_mutex);

		/* uninitialize the controller */
		ahci_uninitialize_controller(ahci_ctlp);

		/* remove the interrupts */
		ahci_rem_intrs(ahci_ctlp);

		/* deallocate the ports structures */
		ahci_dealloc_ports_state(ahci_ctlp);

		/* destroy mutex */
		mutex_destroy(&ahci_ctlp->ahcictl_mutex);

		/* teardown the pci config */
		pci_config_teardown(&ahci_ctlp->ahcictl_pci_conf_handle);

		/* remove the reg maps. */
		ddi_regs_map_free(&ahci_ctlp->ahcictl_ahci_acc_handle);

		/* release fma resource */
		ahci_fm_fini(ahci_ctlp);

		/* free the soft state. */
		ddi_soft_state_free(ahci_statep, instance);

		return (DDI_SUCCESS);

	case DDI_SUSPEND:

		/*
		 * The steps associated with suspension must include putting
		 * the underlying device into a quiescent state so that it
		 * will not generate interrupts or modify or access memory.
		 */
		mutex_enter(&ahci_ctlp->ahcictl_mutex);
		if (ahci_ctlp->ahcictl_flags & AHCI_SUSPEND) {
			mutex_exit(&ahci_ctlp->ahcictl_mutex);
			return (DDI_SUCCESS);
		}

		ahci_ctlp->ahcictl_flags |= AHCI_SUSPEND;

		/* stop the watchdog handler */
		if (ahci_ctlp->ahcictl_timeout_id) {
			(void) untimeout(ahci_ctlp->ahcictl_timeout_id);
			ahci_ctlp->ahcictl_timeout_id = 0;
		}

		mutex_exit(&ahci_ctlp->ahcictl_mutex);

		/*
		 * drain the taskq
		 */
		ahci_drain_ports_taskq(ahci_ctlp);

		/*
		 * Disable the interrupts and stop all the ports.
		 */
		ahci_uninitialize_controller(ahci_ctlp);

		return (DDI_SUCCESS);

	default:
		return (DDI_FAILURE);
	}
}

/*
 * The info entry point for dev_ops.
 *
 */
static int
ahci_getinfo(dev_info_t *dip, ddi_info_cmd_t infocmd,
    void *arg, void **result)
{
#ifndef __lock_lint
	_NOTE(ARGUNUSED(dip))
#endif /* __lock_lint */

	ahci_ctl_t *ahci_ctlp;
	int instance;
	dev_t dev;

	dev = (dev_t)arg;
	instance = getminor(dev);

	switch (infocmd) {
		case DDI_INFO_DEVT2DEVINFO:
			ahci_ctlp = ddi_get_soft_state(ahci_statep,  instance);
			if (ahci_ctlp != NULL) {
				*result = ahci_ctlp->ahcictl_dip;
				return (DDI_SUCCESS);
			} else {
				*result = NULL;
				return (DDI_FAILURE);
			}
		case DDI_INFO_DEVT2INSTANCE:
			*(int *)result = instance;
			break;
		default:
			break;
	}

	return (DDI_SUCCESS);
}

/*
 * Registers the ahci with sata framework.
 */
static int
ahci_register_sata_hba_tran(ahci_ctl_t *ahci_ctlp, uint32_t cap_status)
{
	struct	sata_hba_tran	*sata_hba_tran;

	AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp,
	    "ahci_register_sata_hba_tran enter", NULL);

	mutex_enter(&ahci_ctlp->ahcictl_mutex);

	/* Allocate memory for the sata_hba_tran  */
	sata_hba_tran = kmem_zalloc(sizeof (sata_hba_tran_t), KM_SLEEP);

	sata_hba_tran->sata_tran_hba_rev = SATA_TRAN_HBA_REV;
	sata_hba_tran->sata_tran_hba_dip = ahci_ctlp->ahcictl_dip;
	sata_hba_tran->sata_tran_hba_dma_attr =
	    &ahci_ctlp->ahcictl_buffer_dma_attr;

	/* Report the number of implemented ports */
	sata_hba_tran->sata_tran_hba_num_cports =
	    ahci_ctlp->ahcictl_num_implemented_ports;

	/* Support ATAPI device */
	sata_hba_tran->sata_tran_hba_features_support = SATA_CTLF_ATAPI;

	/* Get the data transfer capability for PIO command by the HBA */
	if (cap_status & AHCI_HBA_CAP_PMD) {
		ahci_ctlp->ahcictl_cap |= AHCI_CAP_PIO_MDRQ;
		AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "HBA supports multiple "
		    "DRQ block data transfer for PIO command protocol", NULL);
	}

	/*
	 * According to the AHCI spec, the ATA/ATAPI-7 queued feature set
	 * is not supported by AHCI (including the READ QUEUED (EXT), WRITE
	 * QUEUED (EXT), and SERVICE commands). Queued operations are
	 * supported in AHCI using the READ FPDMA QUEUED and WRITE FPDMA
	 * QUEUED commands when the HBA and device support native command
	 * queuing(NCQ).
	 *
	 * SATA_CTLF_NCQ will be set to sata_tran_hba_features_support if the
	 * CAP register of the HBA indicates NCQ is supported.
	 *
	 * SATA_CTLF_NCQ cannot be set if AHCI_CAP_NO_MCMDLIST_NONQUEUE is
	 * set because the previous register content of PxCI can be re-written
	 * in the register write.
	 */
	if ((cap_status & AHCI_HBA_CAP_SNCQ) &&
	    !(ahci_ctlp->ahcictl_cap & AHCI_CAP_NO_MCMDLIST_NONQUEUE)) {
		sata_hba_tran->sata_tran_hba_features_support |= SATA_CTLF_NCQ;
		ahci_ctlp->ahcictl_cap |= AHCI_CAP_NCQ;
		AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "HBA supports Native "
		    "Command Queuing", NULL);
	}

	/* Support port multiplier? */
	if (cap_status & AHCI_HBA_CAP_SPM) {
		sata_hba_tran->sata_tran_hba_features_support |=
		    SATA_CTLF_PORT_MULTIPLIER;

		/* Support FIS-based switching for port multiplier? */
		if (cap_status & AHCI_HBA_CAP_FBSS) {
			sata_hba_tran->sata_tran_hba_features_support |=
			    SATA_CTLF_PMULT_FBS;
		}
	}

	/* Report the number of command slots */
	sata_hba_tran->sata_tran_hba_qdepth = ahci_ctlp->ahcictl_num_cmd_slots;

	sata_hba_tran->sata_tran_probe_port = ahci_tran_probe_port;
	sata_hba_tran->sata_tran_start = ahci_tran_start;
	sata_hba_tran->sata_tran_abort = ahci_tran_abort;
	sata_hba_tran->sata_tran_reset_dport = ahci_tran_reset_dport;
	sata_hba_tran->sata_tran_hotplug_ops = &ahci_tran_hotplug_ops;
#ifdef __lock_lint
	sata_hba_tran->sata_tran_selftest = ahci_selftest;
#endif
	/*
	 * When SATA framework adds support for pwrmgt the
	 * pwrmgt_ops needs to be updated
	 */
	sata_hba_tran->sata_tran_pwrmgt_ops = NULL;
	sata_hba_tran->sata_tran_ioctl = NULL;

	ahci_ctlp->ahcictl_sata_hba_tran = sata_hba_tran;

	mutex_exit(&ahci_ctlp->ahcictl_mutex);

	/* Attach it to SATA framework */
	if (sata_hba_attach(ahci_ctlp->ahcictl_dip, sata_hba_tran, DDI_ATTACH)
	    != DDI_SUCCESS) {
		kmem_free((void *)sata_hba_tran, sizeof (sata_hba_tran_t));
		mutex_enter(&ahci_ctlp->ahcictl_mutex);
		ahci_ctlp->ahcictl_sata_hba_tran = NULL;
		mutex_exit(&ahci_ctlp->ahcictl_mutex);
		return (AHCI_FAILURE);
	}

	return (AHCI_SUCCESS);
}

/*
 * Unregisters the ahci with sata framework.
 */
static int
ahci_unregister_sata_hba_tran(ahci_ctl_t *ahci_ctlp)
{
	AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
	    "ahci_unregister_sata_hba_tran enter", NULL);

	/* Detach from the SATA framework. */
	if (sata_hba_detach(ahci_ctlp->ahcictl_dip, DDI_DETACH) !=
	    DDI_SUCCESS) {
		return (AHCI_FAILURE);
	}

	/* Deallocate sata_hba_tran. */
	kmem_free((void *)ahci_ctlp->ahcictl_sata_hba_tran,
	    sizeof (sata_hba_tran_t));

	mutex_enter(&ahci_ctlp->ahcictl_mutex);
	ahci_ctlp->ahcictl_sata_hba_tran = NULL;
	mutex_exit(&ahci_ctlp->ahcictl_mutex);

	return (AHCI_SUCCESS);
}

#define	SET_PORTSTR(str, addrp)						\
	if (AHCI_ADDR_IS_PORT(addrp))					\
		(void) sprintf((str), "%d", (addrp)->aa_port);		\
	else if (AHCI_ADDR_IS_PMULT(addrp))				\
		(void) sprintf((str), "%d (pmult)", (addrp)->aa_port);	\
	else								\
		(void) sprintf((str), "%d:%d", (addrp)->aa_port,	\
		    (addrp)->aa_pmport);

/*
 * ahci_tran_probe_port is called by SATA framework. It returns port state,
 * port status registers and an attached device type via sata_device
 * structure.
 *
 * We return the cached information from a previous hardware probe. The
 * actual hardware probing itself was done either from within
 * ahci_initialize_controller() during the driver attach or from a phy
 * ready change interrupt handler.
 */
static int
ahci_tran_probe_port(dev_info_t *dip, sata_device_t *sd)
{
	ahci_ctl_t *ahci_ctlp;
	ahci_port_t *ahci_portp;
	ahci_addr_t addr, pmult_addr;
	uint8_t cport = sd->satadev_addr.cport;
	char portstr[10];
	uint8_t device_type;
	uint32_t port_state;
	uint8_t port;
	int rval = SATA_SUCCESS, rval_init;

	ahci_ctlp = ddi_get_soft_state(ahci_statep, ddi_get_instance(dip));
	port = ahci_ctlp->ahcictl_cport_to_port[cport];

	ahci_portp = ahci_ctlp->ahcictl_ports[port];

	mutex_enter(&ahci_portp->ahciport_mutex);

	ahci_get_ahci_addr(ahci_ctlp, sd, &addr);
	ASSERT(AHCI_ADDR_IS_VALID(&addr));
	SET_PORTSTR(portstr, &addr);

	AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
	    "ahci_tran_probe_port enter: port %s", portstr);

	if ((AHCI_ADDR_IS_PMULT(&addr) || AHCI_ADDR_IS_PMPORT(&addr)) &&
	    (ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT ||
	    ahci_portp->ahciport_pmult_info == NULL)) {
		/* port mutliplier is removed. */
		AHCIDBG(AHCIDBG_PMULT, ahci_ctlp,
		    "ahci_tran_probe_port: "
		    "pmult is removed from port %s", portstr);
		mutex_exit(&ahci_portp->ahciport_mutex);
		return (SATA_FAILURE);
	}

	/*
	 * The sata_device may refer to
	 * 1. A controller port.
	 *    A controller port should be ready here.
	 * 2. A port multiplier.
	 *    SATA_ADDR_PMULT_SPEC - if it is not initialized yet, initialize
	 *    it and register the port multiplier to the framework.
	 *    SATA_ADDR_PMULT - check the status of all its device ports.
	 * 3. A port multiplier port.
	 *    If it has not been initialized, initialized it.
	 *
	 * A port multiplier or a port multiplier port may require some
	 * initialization because we cannot do these time-consuming jobs in an
	 * interrupt context.
	 */
	if (sd->satadev_addr.qual & SATA_ADDR_PMULT_SPEC) {
		AHCI_ADDR_SET_PMULT(&pmult_addr, port);
		/* Initialize registers on a port multiplier */
		rval_init = ahci_initialize_pmult(ahci_ctlp,
		    ahci_portp, &pmult_addr, sd);
		if (rval_init != AHCI_SUCCESS) {
			AHCIDBG(AHCIDBG_PMULT, ahci_ctlp,
			    "ahci_tran_probe_port: "
			    "pmult initialization failed.", NULL);
			mutex_exit(&ahci_portp->ahciport_mutex);
			return (SATA_FAILURE);
		}
	} else if (sd->satadev_addr.qual & SATA_ADDR_PMULT) {
		/* Check pmports hotplug events */
		(void) ahci_probe_pmult(ahci_ctlp, ahci_portp, &addr);
	} else if (sd->satadev_addr.qual & (SATA_ADDR_PMPORT |
	    SATA_ADDR_DPMPORT)) {
		if (ahci_probe_pmport(ahci_ctlp, ahci_portp,
		    &addr, sd) != AHCI_SUCCESS) {
			rval = SATA_FAILURE;
			goto out;
		}
	}

	/* Update port state and device type */
	port_state = AHCIPORT_GET_STATE(ahci_portp, &addr);

	switch (port_state) {

	case SATA_PSTATE_FAILED:
		sd->satadev_state = SATA_PSTATE_FAILED;
		AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
		    "ahci_tran_probe_port: port %s PORT FAILED", portstr);
		goto out;

	case SATA_PSTATE_SHUTDOWN:
		sd->satadev_state = SATA_PSTATE_SHUTDOWN;
		AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
		    "ahci_tran_probe_port: port %s PORT SHUTDOWN", portstr);
		goto out;

	case SATA_PSTATE_PWROFF:
		sd->satadev_state = SATA_PSTATE_PWROFF;
		AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
		    "ahci_tran_probe_port: port %s PORT PWROFF", portstr);
		goto out;

	case SATA_PSTATE_PWRON:
		sd->satadev_state = SATA_PSTATE_PWRON;
		AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
		    "ahci_tran_probe_port: port %s PORT PWRON", portstr);
		break;

	default:
		sd->satadev_state = port_state;
		AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
		    "ahci_tran_probe_port: port %s PORT NORMAL %x",
		    portstr, port_state);
		break;
	}

	device_type = AHCIPORT_GET_DEV_TYPE(ahci_portp, &addr);

	switch (device_type) {

	case SATA_DTYPE_ATADISK:
		sd->satadev_type = SATA_DTYPE_ATADISK;
		AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
		    "ahci_tran_probe_port: port %s DISK found", portstr);
		break;

	case SATA_DTYPE_ATAPI:
		/*
		 * HBA driver only knows it's an ATAPI device, and don't know
		 * it's CD/DVD, tape or ATAPI disk because the ATAPI device
		 * type need to be determined by checking IDENTIFY PACKET
		 * DEVICE data
		 */
		sd->satadev_type = SATA_DTYPE_ATAPI;
		AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
		    "ahci_tran_probe_port: port %s ATAPI found", portstr);
		break;

	case SATA_DTYPE_PMULT:
		ASSERT(AHCI_ADDR_IS_PORT(&addr) || AHCI_ADDR_IS_PMULT(&addr));
		sd->satadev_type = SATA_DTYPE_PMULT;

		/* Update the number of pmports. */
		ASSERT(ahci_portp->ahciport_pmult_info != NULL);
		sd->satadev_add_info = ahci_portp->
		    ahciport_pmult_info->ahcipmi_num_dev_ports;

		AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
		    "ahci_tran_probe_port: port %s Port Multiplier found",
		    portstr);
		break;

	case SATA_DTYPE_UNKNOWN:
		sd->satadev_type = SATA_DTYPE_UNKNOWN;
		AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
		    "ahci_tran_probe_port: port %s Unknown device found",
		    portstr);
		break;

	default:
		/* we don't support any other device types */
		sd->satadev_type = SATA_DTYPE_NONE;
		AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
		    "ahci_tran_probe_port: port %s No device found", portstr);
		break;
	}

out:
	/* Register update only fails while probing a pmult/pmport */
	if (AHCI_ADDR_IS_PORT(&addr) || AHCI_ADDR_IS_PMULT(&addr)) {
		ahci_update_sata_registers(ahci_ctlp, port, sd);
	} else if (AHCI_ADDR_IS_PMPORT(&addr)) {
		if (port_state & SATA_STATE_READY)
			if (ahci_update_pmult_pscr(ahci_ctlp,
			    &addr, sd) != AHCI_SUCCESS)
				rval = SATA_FAILURE;
	}

	/* Check handles for the sata registers access */
	if ((ahci_check_ctl_handle(ahci_ctlp) != DDI_SUCCESS) ||
	    (ahci_check_port_handle(ahci_ctlp, port) != DDI_SUCCESS)) {
		ddi_fm_service_impact(ahci_ctlp->ahcictl_dip,
		    DDI_SERVICE_UNAFFECTED);
		rval = SATA_FAILURE;
	}

	mutex_exit(&ahci_portp->ahciport_mutex);
	return (rval);
}

/*
 * There are four operation modes in sata framework:
 * SATA_OPMODE_INTERRUPTS
 * SATA_OPMODE_POLLING
 * SATA_OPMODE_ASYNCH
 * SATA_OPMODE_SYNCH
 *
 * Their combined meanings as following:
 *
 * SATA_OPMODE_SYNCH
 * The command has to be completed before sata_tran_start functions returns.
 * Either interrupts or polling could be used - it's up to the driver.
 * Mode used currently for internal, sata-module initiated operations.
 *
 * SATA_OPMODE_SYNCH | SATA_OPMODE_INTERRUPTS
 * It is the same as the one above.
 *
 * SATA_OPMODE_SYNCH | SATA_OPMODE_POLLING
 * The command has to be completed before sata_tran_start function returns.
 * No interrupt used, polling only. This should be the mode used for scsi
 * packets with FLAG_NOINTR.
 *
 * SATA_OPMODE_ASYNCH | SATA_OPMODE_INTERRUPTS
 * The command may be queued (callback function specified). Interrupts could
 * be used. It's normal operation mode.
 */
/*
 * Called by sata framework to transport a sata packet down stream.
 */
static int
ahci_tran_start(dev_info_t *dip, sata_pkt_t *spkt)
{
	ahci_ctl_t *ahci_ctlp;
	ahci_port_t *ahci_portp;
	ahci_addr_t addr;
	uint8_t	cport = spkt->satapkt_device.satadev_addr.cport;
	uint8_t port;
	char portstr[10];

	ahci_ctlp = ddi_get_soft_state(ahci_statep, ddi_get_instance(dip));
	port = ahci_ctlp->ahcictl_cport_to_port[cport];

	AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
	    "ahci_tran_start enter: cport %d satapkt 0x%p",
	    cport, (void *)spkt);

	ahci_portp = ahci_ctlp->ahcictl_ports[port];

	mutex_enter(&ahci_portp->ahciport_mutex);
	ahci_get_ahci_addr(ahci_ctlp, &spkt->satapkt_device, &addr);
	SET_PORTSTR(portstr, &addr);

	/* Sanity check */
	if (AHCI_ADDR_IS_PMPORT(&addr)) {
		if (ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT ||
		    ahci_portp->ahciport_pmult_info == NULL) {

			spkt->satapkt_reason = SATA_PKT_PORT_ERROR;
			spkt->satapkt_device.satadev_type = SATA_DTYPE_NONE;
			spkt->satapkt_device.satadev_state = SATA_STATE_UNKNOWN;
			ahci_update_sata_registers(ahci_ctlp, port,
			    &spkt->satapkt_device);
			AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
			    "ahci_tran_start returning PORT_ERROR while "
			    "pmult removed: port: %s", portstr);
			mutex_exit(&ahci_portp->ahciport_mutex);
			return (SATA_TRAN_PORT_ERROR);
		}

		if (!(AHCIPORT_GET_STATE(ahci_portp, &addr) &
		    SATA_STATE_READY)) {
			if (!ddi_in_panic() ||
			    ahci_initialize_pmport(ahci_ctlp,
			    ahci_portp, &addr) != AHCI_SUCCESS) {
				spkt->satapkt_reason = SATA_PKT_PORT_ERROR;
				spkt->satapkt_device.satadev_type =
				    AHCIPORT_GET_DEV_TYPE(ahci_portp, &addr);
				spkt->satapkt_device.satadev_state =
				    AHCIPORT_GET_STATE(ahci_portp, &addr);
				ahci_update_sata_registers(ahci_ctlp, port,
				    &spkt->satapkt_device);
				AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
				    "ahci_tran_start returning PORT_ERROR "
				    "while sub-link is not initialized "
				    "at port: %s", portstr);
				mutex_exit(&ahci_portp->ahciport_mutex);
				return (SATA_TRAN_PORT_ERROR);
			}
		}
	}

	if (AHCIPORT_GET_STATE(ahci_portp, &addr) & SATA_PSTATE_FAILED ||
	    AHCIPORT_GET_STATE(ahci_portp, &addr) & SATA_PSTATE_SHUTDOWN||
	    AHCIPORT_GET_STATE(ahci_portp, &addr) & SATA_PSTATE_PWROFF) {
		/*
		 * In case the target driver would send the packet before
		 * sata framework can have the opportunity to process those
		 * event reports.
		 */
		spkt->satapkt_reason = SATA_PKT_PORT_ERROR;
		spkt->satapkt_device.satadev_state =
		    ahci_portp->ahciport_port_state;
		ahci_update_sata_registers(ahci_ctlp, port,
		    &spkt->satapkt_device);
		AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
		    "ahci_tran_start returning PORT_ERROR while "
		    "port in FAILED/SHUTDOWN/PWROFF state: "
		    "port: %s", portstr);
		mutex_exit(&ahci_portp->ahciport_mutex);
		return (SATA_TRAN_PORT_ERROR);
	}

	if (AHCIPORT_GET_DEV_TYPE(ahci_portp, &addr) == SATA_DTYPE_NONE) {
		/*
		 * ahci_intr_phyrdy_change() may have rendered it to
		 * SATA_DTYPE_NONE.
		 */
		spkt->satapkt_reason = SATA_PKT_PORT_ERROR;
		spkt->satapkt_device.satadev_type = SATA_DTYPE_NONE;
		spkt->satapkt_device.satadev_state =
		    ahci_portp->ahciport_port_state;
		ahci_update_sata_registers(ahci_ctlp, port,
		    &spkt->satapkt_device);
		AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
		    "ahci_tran_start returning PORT_ERROR while "
		    "no device attached: port: %s", portstr);
		mutex_exit(&ahci_portp->ahciport_mutex);
		return (SATA_TRAN_PORT_ERROR);
	}

	/* R/W PMULT command will occupy the whole HBA port */
	if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) {
		AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
		    "ahci_tran_start returning BUSY while "
		    "executing READ/WRITE PORT-MULT command: "
		    "port: %s", portstr);
		spkt->satapkt_reason = SATA_PKT_BUSY;
		mutex_exit(&ahci_portp->ahciport_mutex);
		return (SATA_TRAN_BUSY);
	}

	if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_HOTPLUG) {
		AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
		    "ahci_tran_start returning BUSY while "
		    "hot-plug in progress: port: %s", portstr);
		spkt->satapkt_reason = SATA_PKT_BUSY;
		mutex_exit(&ahci_portp->ahciport_mutex);
		return (SATA_TRAN_BUSY);
	}

	/*
	 * SATA HBA driver should remember that a device was reset and it
	 * is supposed to reject any packets which do not specify either
	 * SATA_IGNORE_DEV_RESET_STATE or SATA_CLEAR_DEV_RESET_STATE.
	 *
	 * This is to prevent a race condition when a device was arbitrarily
	 * reset by the HBA driver (and lost it's setting) and a target
	 * driver sending some commands to a device before the sata framework
	 * has a chance to restore the device setting (such as cache enable/
	 * disable or other resettable stuff).
	 */
	/*
	 * It is unnecessary to use specific flags to indicate
	 * reset_in_progress for a pmport. While mopping, all command will be
	 * mopped so that the entire HBA port is being dealt as a single
	 * object.
	 */
	if (spkt->satapkt_cmd.satacmd_flags.sata_clear_dev_reset) {
		ahci_portp->ahciport_reset_in_progress = 0;
		AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
		    "ahci_tran_start [CLEAR] the "
		    "reset_in_progress for port: %d", port);
	}

	if (ahci_portp->ahciport_reset_in_progress &&
	    ! spkt->satapkt_cmd.satacmd_flags.sata_ignore_dev_reset &&
	    ! ddi_in_panic()) {
		spkt->satapkt_reason = SATA_PKT_BUSY;
		AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
		    "ahci_tran_start returning BUSY while "
		    "reset in progress: port: %d", port);
		mutex_exit(&ahci_portp->ahciport_mutex);
		return (SATA_TRAN_BUSY);
	}

#ifdef AHCI_DEBUG
	if (spkt->satapkt_cmd.satacmd_flags.sata_ignore_dev_reset) {
		AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
		    "ahci_tran_start: packet 0x%p [PASSTHRU] at port %d",
		    spkt, port);
	}
#endif

	if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) {
		spkt->satapkt_reason = SATA_PKT_BUSY;
		AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
		    "ahci_tran_start returning BUSY while "
		    "mopping in progress: port: %d", port);
		mutex_exit(&ahci_portp->ahciport_mutex);
		return (SATA_TRAN_BUSY);
	}

	if (ahci_check_ctl_handle(ahci_ctlp) != DDI_SUCCESS) {
		ddi_fm_service_impact(ahci_ctlp->ahcictl_dip,
		    DDI_SERVICE_UNAFFECTED);
		mutex_exit(&ahci_portp->ahciport_mutex);
		return (SATA_TRAN_BUSY);
	}

	if (spkt->satapkt_op_mode &
	    (SATA_OPMODE_SYNCH | SATA_OPMODE_POLLING)) {
		/*
		 * If a SYNC command to be executed in interrupt context,
		 * bounce it back to sata module.
		 */
		if (!(spkt->satapkt_op_mode & SATA_OPMODE_POLLING) &&
		    servicing_interrupt()) {
			spkt->satapkt_reason = SATA_PKT_BUSY;
			AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
			    "ahci_tran_start returning BUSY while "
			    "sending SYNC mode under interrupt context: "
			    "port : %d", port);
			mutex_exit(&ahci_portp->ahciport_mutex);
			return (SATA_TRAN_BUSY);
		}

		/* We need to do the sync start now */
		if (ahci_do_sync_start(ahci_ctlp, ahci_portp, &addr,
		    spkt) == AHCI_FAILURE) {
			goto fail_out;
		}
	} else {
		/* Async start, using interrupt */
		if (ahci_deliver_satapkt(ahci_ctlp, ahci_portp, &addr, spkt)
		    == AHCI_FAILURE) {
			spkt->satapkt_reason = SATA_PKT_QUEUE_FULL;
			goto fail_out;
		}
	}

	AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "ahci_tran_start "
	    "sata tran accepted: port %s", portstr);

	mutex_exit(&ahci_portp->ahciport_mutex);
	return (SATA_TRAN_ACCEPTED);

fail_out:
	/*
	 * Failed to deliver packet to the controller.
	 * Check if it's caused by invalid handles.
	 */
	if (ahci_check_ctl_handle(ahci_ctlp) != DDI_SUCCESS ||
	    ahci_check_port_handle(ahci_ctlp, port) != DDI_SUCCESS) {
		spkt->satapkt_device.satadev_type =
		    AHCIPORT_GET_DEV_TYPE(ahci_portp, &addr);
		spkt->satapkt_device.satadev_state =
		    AHCIPORT_GET_STATE(ahci_portp, &addr);
		spkt->satapkt_reason = SATA_PKT_DEV_ERROR;
		mutex_exit(&ahci_portp->ahciport_mutex);
		return (SATA_TRAN_PORT_ERROR);
	}

	AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_tran_start "
	    "return QUEUE_FULL: port %d", port);
	mutex_exit(&ahci_portp->ahciport_mutex);
	return (SATA_TRAN_QUEUE_FULL);
}

/*
 * SATA_OPMODE_SYNCH flag is set
 *
 * If SATA_OPMODE_POLLING flag is set, then we must poll the command
 * without interrupt, otherwise we can still use the interrupt.
 */
static int
ahci_do_sync_start(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
    ahci_addr_t *addrp, sata_pkt_t *spkt)
{
	int pkt_timeout_ticks;
	uint32_t timeout_tags;
	int rval;
	int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip);
	uint8_t port = addrp->aa_port;

	ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));

	AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_do_sync_start enter: "
	    "port %d:%d spkt 0x%p", port, addrp->aa_pmport, spkt);

	if (spkt->satapkt_op_mode & SATA_OPMODE_POLLING) {
		ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_POLLING;
		if ((rval = ahci_deliver_satapkt(ahci_ctlp, ahci_portp,
		    addrp, spkt)) == AHCI_FAILURE) {
			ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_POLLING;
			return (rval);
		}

		pkt_timeout_ticks =
		    drv_usectohz((clock_t)spkt->satapkt_time * 1000000);

		while (spkt->satapkt_reason == SATA_PKT_BUSY) {
			/* Simulate the interrupt */
			mutex_exit(&ahci_portp->ahciport_mutex);
			ahci_port_intr(ahci_ctlp, ahci_portp, port);
			mutex_enter(&ahci_portp->ahciport_mutex);

			if (spkt->satapkt_reason != SATA_PKT_BUSY)
				break;

			mutex_exit(&ahci_portp->ahciport_mutex);
			drv_usecwait(AHCI_1MS_USECS);
			mutex_enter(&ahci_portp->ahciport_mutex);

			pkt_timeout_ticks -= AHCI_1MS_TICKS;
			if (pkt_timeout_ticks < 0) {
				cmn_err(CE_WARN, "!ahci%d: ahci_do_sync_start "
				    "port %d satapkt 0x%p timed out\n",
				    instance, port, (void *)spkt);
				timeout_tags = (0x1 << rval);
				mutex_exit(&ahci_portp->ahciport_mutex);
				ahci_timeout_pkts(ahci_ctlp, ahci_portp,
				    port, timeout_tags);
				mutex_enter(&ahci_portp->ahciport_mutex);
			}
		}

		ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_POLLING;
		return (AHCI_SUCCESS);

	} else {
		if ((rval = ahci_deliver_satapkt(ahci_ctlp, ahci_portp,
		    addrp, spkt)) == AHCI_FAILURE)
			return (rval);

#if AHCI_DEBUG
		/*
		 * Note that the driver always uses the slot 0 to deliver
		 * REQUEST SENSE or READ LOG EXT command
		 */
		if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp))
			ASSERT(rval == 0);
#endif

		while (spkt->satapkt_reason == SATA_PKT_BUSY)
			cv_wait(&ahci_portp->ahciport_cv,
			    &ahci_portp->ahciport_mutex);

		return (AHCI_SUCCESS);
	}
}

/*
 * Searches for and claims a free command slot.
 *
 * Returns value:
 *
 * AHCI_FAILURE returned only if
 *	1. No empty slot left
 *	2. Non-queued command requested while queued command(s) is outstanding
 *	3. Queued command requested while non-queued command(s) is outstanding
 *	4. HBA doesn't support multiple-use of command list while already a
 *	   non-queued command is oustanding
 *	5. Queued command requested while some queued command(s) has been
 *	   outstanding on a different port multiplier port. (AHCI spec 1.2,
 *	   9.1.2)
 *
 * claimed slot number returned if succeeded
 *
 * NOTE: it will always return slot 0 for following commands to simplify the
 * algorithm.
 *	1. REQUEST SENSE or READ LOG EXT command during error recovery process
 *	2. READ/WRITE PORTMULT command
 */
static int
ahci_claim_free_slot(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
    ahci_addr_t *addrp, int command_type)
{
	uint32_t port_cmd_issue;
	uint32_t free_slots;
	int slot;

	ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));

	AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_claim_free_slot enter "
	    "ahciport_pending_tags = 0x%x "
	    "ahciport_pending_ncq_tags = 0x%x",
	    ahci_portp->ahciport_pending_tags,
	    ahci_portp->ahciport_pending_ncq_tags);

	/*
	 * According to the AHCI spec, system software is responsible to
	 * ensure that queued and non-queued commands are not mixed in
	 * the command list.
	 */
	if (command_type == AHCI_NON_NCQ_CMD) {
		/* Non-NCQ command request */
		if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
			AHCIDBG(AHCIDBG_INFO|AHCIDBG_NCQ, ahci_ctlp,
			    "ahci_claim_free_slot: there is still pending "
			    "queued command(s) in the command list, "
			    "so no available slot for the non-queued "
			    "command", NULL);
			return (AHCI_FAILURE);
		}
		if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) {
			AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
			    "ahci_claim_free_slot: there is still pending "
			    "read/write port-mult command(s) in command list, "
			    "so no available slot for the non-queued command",
			    NULL);
			return (AHCI_FAILURE);
		}
		if ((ahci_ctlp->ahcictl_cap & AHCI_CAP_NO_MCMDLIST_NONQUEUE) &&
		    NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
			AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
			    "ahci_claim_free_slot: HBA cannot support multiple-"
			    "use of the command list for non-queued commands",
			    NULL);
			return (AHCI_FAILURE);
		}
		free_slots = (~ahci_portp->ahciport_pending_tags) &
		    AHCI_SLOT_MASK(ahci_ctlp);
	} else if (command_type == AHCI_NCQ_CMD) {
		/* NCQ command request */
		if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
			AHCIDBG(AHCIDBG_INFO|AHCIDBG_NCQ, ahci_ctlp,
			    "ahci_claim_free_slot: there is still pending "
			    "non-queued command(s) in the command list, "
			    "so no available slot for the queued command",
			    NULL);
			return (AHCI_FAILURE);
		}

		/*
		 * NCQ commands cannot be sent to different port multiplier
		 * ports in Command-Based Switching mode
		 */
		/*
		 * NOTE: In Command-Based Switching mode, AHCI controller
		 * usually reports a 'Handshake Error' when multiple NCQ
		 * commands are outstanding simultaneously.
		 */
		if (AHCIPORT_DEV_TYPE(ahci_portp, addrp) == SATA_DTYPE_PMULT) {
			ASSERT(ahci_portp->ahciport_pmult_info != NULL);
			if (!(ahci_ctlp->ahcictl_cap & AHCI_CAP_PMULT_FBSS) &&
			    NCQ_CMD_IN_PROGRESS(ahci_portp) &&
			    AHCIPORT_NCQ_PMPORT(ahci_portp) !=
			    addrp->aa_pmport) {
				AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
				    "ahci_claim_free_slot: there is still "
				    "pending queued command(s) in the "
				    "command list for another Port Multiplier "
				    "port, so no available slot.", NULL);
				return (AHCI_FAILURE);
			}
		}

		free_slots = (~ahci_portp->ahciport_pending_ncq_tags) &
		    AHCI_NCQ_SLOT_MASK(ahci_portp);
	} else if (command_type == AHCI_ERR_RETRI_CMD) {
		/* Error retrieval command request */
		AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
		    "ahci_claim_free_slot: slot 0 is allocated for REQUEST "
		    "SENSE or READ LOG EXT command", NULL);
		slot = 0;
		goto out;
	} else if (command_type == AHCI_RDWR_PMULT_CMD) {
		/*
		 * An extra check on PxCI. Sometimes PxCI bits may not be
		 * cleared during hot-plug or error recovery process.
		 */
		port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
		    (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, addrp->aa_port));

		if (port_cmd_issue != 0) {
			AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
			    "ahci_claim_free_slot: there is still pending "
			    "command(s) in command list (0x%x/0x%x, PxCI %x),"
			    "so no available slot for R/W PMULT command.",
			    NON_NCQ_CMD_IN_PROGRESS(ahci_portp),
			    NCQ_CMD_IN_PROGRESS(ahci_portp),
			    port_cmd_issue);
			return (AHCI_FAILURE);
		}

		AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
		    "ahci_claim_free_slot: slot 0 is allocated for "
		    "READ/WRITE PORTMULT command", NULL);
		slot = 0;
		goto out;
	}

	slot = ddi_ffs(free_slots) - 1;
	if (slot == -1) {
		AHCIDBG(AHCIDBG_VERBOSE, ahci_ctlp,
		    "ahci_claim_free_slot: no empty slots", NULL);
		return (AHCI_FAILURE);
	}

	/*
	 * According to the AHCI spec, to allow a simple mechanism for the
	 * HBA to map command list slots to queue entries, software must
	 * match the tag number it uses to the slot it is placing the command
	 * in. For example, if a queued command is placed in slot 5, the tag
	 * for that command must be 5.
	 */
	if (command_type == AHCI_NCQ_CMD) {
		ahci_portp->ahciport_pending_ncq_tags |= (0x1 << slot);
		if (AHCI_ADDR_IS_PMPORT(addrp)) {
			ASSERT(ahci_portp->ahciport_pmult_info != NULL);
			AHCIPORT_NCQ_PMPORT(ahci_portp) = addrp->aa_pmport;
		}
	}

	ahci_portp->ahciport_pending_tags |= (0x1 << slot);

out:
	AHCIDBG(AHCIDBG_VERBOSE, ahci_ctlp,
	    "ahci_claim_free_slot: found slot: 0x%x", slot);

	return (slot);
}

/*
 * Builds the Command Table for the sata packet and delivers it to controller.