usr/src/uts/common/io/fibre-channel/ulp/fcip.c

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 * Copyright (c) 2016 by Delphix. All rights reserved.
 */

/*
 * SunOS 5.x Multithreaded STREAMS DLPI FCIP Module
 * This is a pseudo driver module to handle encapsulation of IP and ARP
 * datagrams over FibreChannel interfaces. FCIP is a cloneable STREAMS
 * driver module which interfaces with IP/ARP using DLPI. This module
 * is a Style-2 DLS provider.
 *
 * The implementation of this module is based on RFC 2625 which gives
 * details on the encapsulation of IP/ARP data over FibreChannel.
 * The fcip module needs to resolve an IP address to a port address before
 * sending data to a destination port. A FC device port has 2 addresses
 * associated with it: A 8 byte World Wide unique Port Name and a 3 byte
 * volatile Port number or Port_ID.
 *
 * The mapping between a IP address and the World Wide Port Name is handled
 * by the ARP layer since the IP over FC draft requires the MAC address to
 * be the least significant six bytes of the WorldWide Port Names. The
 * fcip module however needs to identify the destination port uniquely when
 * the destination FC device has multiple FC ports.
 *
 * The FC layer mapping between the World Wide Port Name and the Port_ID
 * will be handled through the use of a fabric name server or through the
 * use of the FARP ELS command as described in the draft. Since the Port_IDs
 * are volatile, the mapping between the World Wide Port Name and Port_IDs
 * must be maintained and validated before use each time a datagram
 * needs to be sent to the destination ports. The FC transport module
 * informs the fcip module of all changes to states of ports on the
 * fabric through registered callbacks. This enables the fcip module
 * to maintain the WW_PN to Port_ID mappings current.
 *
 * For details on how this module interfaces with the FibreChannel Transport
 * modules, refer to PSARC/1997/385. Chapter 3 of the FibreChannel Transport
 * Programming guide details the APIs between ULPs and the Transport.
 *
 * Now for some Caveats:
 *
 * RFC 2625 requires that a FibreChannel Port name (the Port WWN) have
 * the NAA bits set to '0001' indicating a IEEE 48bit address which
 * corresponds to a ULA (Universal LAN MAC address). But with FibreChannel
 * adapters containing 2 or more ports, IEEE naming cannot identify the
 * ports on an adapter uniquely so we will in the first implementation
 * be operating only on Port 0 of each adapter.
 */

#include	<sys/types.h>
#include	<sys/errno.h>
#include	<sys/debug.h>
#include	<sys/time.h>
#include	<sys/sysmacros.h>
#include	<sys/systm.h>
#include	<sys/user.h>
#include	<sys/stropts.h>
#include	<sys/stream.h>
#include	<sys/strlog.h>
#include	<sys/strsubr.h>
#include	<sys/cmn_err.h>
#include	<sys/cpu.h>
#include	<sys/kmem.h>
#include	<sys/conf.h>
#include	<sys/ddi.h>
#include	<sys/sunddi.h>
#include	<sys/ksynch.h>
#include	<sys/stat.h>
#include	<sys/kstat.h>
#include	<sys/vtrace.h>
#include	<sys/strsun.h>
#include	<sys/varargs.h>
#include	<sys/modctl.h>
#include 	<sys/thread.h>
#include 	<sys/var.h>
#include 	<sys/proc.h>
#include	<inet/common.h>
#include	<netinet/ip6.h>
#include	<inet/ip.h>
#include	<inet/arp.h>
#include	<inet/mi.h>
#include	<inet/nd.h>
#include	<sys/dlpi.h>
#include	<sys/ethernet.h>
#include	<sys/file.h>
#include	<sys/syslog.h>
#include	<sys/disp.h>
#include	<sys/taskq.h>

/*
 * Leadville includes
 */

#include	<sys/fibre-channel/fc.h>
#include	<sys/fibre-channel/impl/fc_ulpif.h>
#include	<sys/fibre-channel/ulp/fcip.h>

/*
 * TNF Probe/trace facility include
 */
#if defined(lint) || defined(FCIP_TNF_ENABLED)
#include <sys/tnf_probe.h>
#endif

#define	FCIP_ESBALLOC

/*
 * Function prototypes
 */

/* standard loadable modules entry points */
static int	fcip_attach(dev_info_t *, ddi_attach_cmd_t);
static int 	fcip_detach(dev_info_t *, ddi_detach_cmd_t);
static void 	fcip_dodetach(struct fcipstr *slp);
static int fcip_getinfo(dev_info_t *dip, ddi_info_cmd_t cmd,
    void *arg, void **result);


/* streams specific */
static void fcip_setipq(struct fcip *fptr);
static int fcip_wput(queue_t *, mblk_t *);
static int fcip_wsrv(queue_t *);
static void fcip_proto(queue_t *, mblk_t *);
static void fcip_ioctl(queue_t *, mblk_t *);
static int fcip_open(queue_t *wq, dev_t *devp, int flag,
		int sflag, cred_t *credp);
static int fcip_close(queue_t *rq, int flag, int otyp, cred_t *credp);
static int fcip_start(queue_t *wq, mblk_t *mp, struct fcip *fptr,
    struct fcip_dest *fdestp, int flags);
static void fcip_sendup(struct fcip *fptr, mblk_t *mp,
    struct fcipstr *(*acceptfunc)());
static struct fcipstr *fcip_accept(struct fcipstr *slp, struct fcip *fptr,
    int type, la_wwn_t *dhostp);
static mblk_t *fcip_addudind(struct fcip *fptr, mblk_t *mp,
    fcph_network_hdr_t *nhdr, int type);
static int fcip_setup_mac_addr(struct fcip *fptr);
static void fcip_kstat_init(struct fcip *fptr);
static int fcip_stat_update(kstat_t *, int);


/* dlpi specific */
static void fcip_spareq(queue_t *wq, mblk_t *mp);
static void fcip_pareq(queue_t *wq, mblk_t *mp);
static void fcip_ubreq(queue_t *wq, mblk_t *mp);
static void fcip_breq(queue_t *wq, mblk_t *mp);
static void fcip_dreq(queue_t *wq, mblk_t *mp);
static void fcip_areq(queue_t *wq, mblk_t *mp);
static void fcip_udreq(queue_t *wq, mblk_t *mp);
static void fcip_ireq(queue_t *wq, mblk_t *mp);
static void fcip_dl_ioc_hdr_info(queue_t *wq, mblk_t *mp);


/* solaris sundry, DR/CPR etc */
static int fcip_cache_constructor(void *buf, void *arg, int size);
static void fcip_cache_destructor(void *buf, void *size);
static int fcip_handle_suspend(fcip_port_info_t *fport, fc_detach_cmd_t cmd);
static int fcip_handle_resume(fcip_port_info_t *fport,
    fc_ulp_port_info_t *port_info, fc_attach_cmd_t cmd);
static fcip_port_info_t *fcip_softstate_free(fcip_port_info_t *fport);
static int fcip_port_attach_handler(struct fcip *fptr);


/*
 * ulp - transport interface function prototypes
 */
static int fcip_port_attach(opaque_t ulp_handle, fc_ulp_port_info_t *,
    fc_attach_cmd_t cmd, uint32_t sid);
static int fcip_port_detach(opaque_t ulp_handle, fc_ulp_port_info_t *,
    fc_detach_cmd_t cmd);
static int fcip_port_ioctl(opaque_t ulp_handle,  opaque_t port_handle,
    dev_t dev, int cmd, intptr_t data, int mode, cred_t *credp, int *rval,
    uint32_t claimed);
static void fcip_statec_cb(opaque_t ulp_handle, opaque_t phandle,
    uint32_t port_state, uint32_t port_top, fc_portmap_t changelist[],
    uint32_t listlen, uint32_t sid);
static int fcip_els_cb(opaque_t ulp_handle, opaque_t phandle,
    fc_unsol_buf_t *buf, uint32_t claimed);
static int fcip_data_cb(opaque_t ulp_handle, opaque_t phandle,
    fc_unsol_buf_t *payload, uint32_t claimed);


/* Routing table specific */
static void fcip_handle_topology(struct fcip *fptr);
static int fcip_init_port(struct fcip *fptr);
struct fcip_routing_table *fcip_lookup_rtable(struct fcip *fptr,
    la_wwn_t *pwwn, int matchflag);
static void fcip_rt_update(struct fcip *fptr, fc_portmap_t *devlist,
    uint32_t listlen);
static void fcip_rt_flush(struct fcip *fptr);
static void fcip_rte_remove_deferred(void *arg);
static int fcip_do_plogi(struct fcip *fptr, struct fcip_routing_table *frp);


/* dest table specific */
static struct fcip_dest *fcip_get_dest(struct fcip *fptr,
    la_wwn_t *dlphys);
static struct fcip_dest *fcip_add_dest(struct fcip *fptr,
    struct fcip_routing_table *frp);
static int fcip_dest_add_broadcast_entry(struct fcip *fptr, int new_flag);
static uint32_t fcip_get_broadcast_did(struct fcip *fptr);
static void fcip_cleanup_dest(struct fcip *fptr);


/* helper functions */
static fcip_port_info_t *fcip_get_port(opaque_t phandle);
static int fcip_wwn_compare(la_wwn_t *wwn1, la_wwn_t *wwn2, int flag);
static void fcip_ether_to_str(struct ether_addr *e, caddr_t s);
static int fcip_port_get_num_pkts(struct fcip *fptr);
static int fcip_check_port_busy(struct fcip *fptr);
static void fcip_check_remove_minor_node(void);
static int fcip_set_wwn(la_wwn_t *pwwn);
static int fcip_plogi_in_progress(struct fcip *fptr);
static int fcip_check_port_exists(struct fcip *fptr);
static int fcip_is_supported_fc_topology(int fc_topology);


/* pkt specific */
static fcip_pkt_t *fcip_pkt_alloc(struct fcip *fptr, mblk_t *bp,
    int flags, int datalen);
static void fcip_pkt_free(struct fcip_pkt *fcip_pkt, int flags);
static fcip_pkt_t *fcip_ipkt_alloc(struct fcip *fptr, int cmdlen,
    int resplen, opaque_t pd, int flags);
static void fcip_ipkt_free(fcip_pkt_t *fcip_pkt);
static void fcip_ipkt_callback(fc_packet_t *fc_pkt);
static void fcip_free_pkt_dma(fcip_pkt_t *fcip_pkt);
static void fcip_pkt_callback(fc_packet_t *fc_pkt);
static void fcip_init_unicast_pkt(fcip_pkt_t *fcip_pkt, fc_portid_t sid,
    fc_portid_t did, void (*comp) ());
static int fcip_transport(fcip_pkt_t *fcip_pkt);
static void fcip_pkt_timeout(void *arg);
static void fcip_timeout(void *arg);
static void fcip_fdestp_enqueue_pkt(struct fcip_dest *fdestp,
    fcip_pkt_t *fcip_pkt);
static int fcip_fdestp_dequeue_pkt(struct fcip_dest *fdestp,
    fcip_pkt_t *fcip_pkt);
static int fcip_sendup_constructor(void *buf, void *arg, int flags);
static void fcip_sendup_thr(void *arg);
static int fcip_sendup_alloc_enque(struct fcip *ftpr, mblk_t *mp,
    struct fcipstr *(*f)());

/*
 * zero copy inbound data handling
 */
#ifdef FCIP_ESBALLOC
static void fcip_ubfree(char *arg);
#endif /* FCIP_ESBALLOC */

#if !defined(FCIP_ESBALLOC)
static void *fcip_allocb(size_t size, uint_t pri);
#endif


/* FCIP FARP support functions */
static struct fcip_dest *fcip_do_farp(struct fcip *fptr, la_wwn_t *pwwn,
    char *ip_addr, size_t ip_addr_len, int flags);
static void fcip_init_broadcast_pkt(fcip_pkt_t *fcip_pkt, void (*comp) (),
    int is_els);
static int fcip_handle_farp_request(struct fcip *fptr, la_els_farp_t *fcmd);
static int fcip_handle_farp_response(struct fcip *fptr, la_els_farp_t *fcmd);
static void fcip_cache_arp_broadcast(struct fcip *ftpr, fc_unsol_buf_t *buf);
static void fcip_port_ns(void *arg);

#ifdef DEBUG

#include <sys/debug.h>

#define	FCIP_DEBUG_DEFAULT	0x1
#define	FCIP_DEBUG_ATTACH	0x2
#define	FCIP_DEBUG_INIT		0x4
#define	FCIP_DEBUG_DETACH	0x8
#define	FCIP_DEBUG_DLPI		0x10
#define	FCIP_DEBUG_ELS		0x20
#define	FCIP_DEBUG_DOWNSTREAM	0x40
#define	FCIP_DEBUG_UPSTREAM	0x80
#define	FCIP_DEBUG_MISC		0x100

#define	FCIP_DEBUG_STARTUP	(FCIP_DEBUG_ATTACH|FCIP_DEBUG_INIT)
#define	FCIP_DEBUG_DATAOUT	(FCIP_DEBUG_DLPI|FCIP_DEBUG_DOWNSTREAM)
#define	FCIP_DEBUG_DATAIN	(FCIP_DEBUG_ELS|FCIP_DEBUG_UPSTREAM)

static int fcip_debug = FCIP_DEBUG_DEFAULT;

#define	FCIP_DEBUG(level, args)	\
	if (fcip_debug & (level))	cmn_err args;

#else	/* DEBUG */

#define	FCIP_DEBUG(level, args)		/* do nothing */

#endif	/* DEBUG */

#define	KIOIP	KSTAT_INTR_PTR(fcip->fcip_intrstats)

/*
 * Endian independent ethernet to WWN copy
 */
#define	ether_to_wwn(E, W)	\
	bzero((void *)(W), sizeof (la_wwn_t)); \
	bcopy((void *)(E), (void *)&((W)->raw_wwn[2]), ETHERADDRL); \
	(W)->raw_wwn[0] |= 0x10

/*
 * wwn_to_ether : Endian independent, copies a WWN to struct ether_addr.
 * The args to the macro are pointers to WWN and ether_addr structures
 */
#define	wwn_to_ether(W, E)	\
	bcopy((void *)&((W)->raw_wwn[2]), (void *)E, ETHERADDRL)

/*
 * The module_info structure contains identification and limit values.
 * All queues associated with a certain driver share the same module_info
 * structures. This structure defines the characteristics of that driver/
 * module's queues. The module name must be unique. The max and min packet
 * sizes limit the no. of characters in M_DATA messages. The Hi and Lo
 * water marks are for flow control when a module has a service procedure.
 */
static struct module_info	fcipminfo = {
	FCIPIDNUM,	/* mi_idnum : Module ID num */
	FCIPNAME, 	/* mi_idname: Module Name */
	FCIPMINPSZ,	/* mi_minpsz: Min packet size */
	FCIPMAXPSZ,	/* mi_maxpsz: Max packet size */
	FCIPHIWAT,	/* mi_hiwat : High water mark */
	FCIPLOWAT	/* mi_lowat : Low water mark */
};

/*
 * The qinit structres contain the module put, service. open and close
 * procedure pointers. All modules and drivers with the same streamtab
 * file (i.e same fmodsw or cdevsw entry points) point to the same
 * upstream (read) and downstream (write) qinit structs.
 */
static struct qinit	fcip_rinit = {
	NULL,		/* qi_putp */
	NULL,		/* qi_srvp */
	fcip_open,	/* qi_qopen */
	fcip_close,	/* qi_qclose */
	NULL,		/* qi_qadmin */
	&fcipminfo,	/* qi_minfo */
	NULL		/* qi_mstat */
};

static struct qinit	fcip_winit = {
	fcip_wput,	/* qi_putp */
	fcip_wsrv,	/* qi_srvp */
	NULL,		/* qi_qopen */
	NULL,		/* qi_qclose */
	NULL,		/* qi_qadmin */
	&fcipminfo,	/* qi_minfo */
	NULL		/* qi_mstat */
};

/*
 * streamtab contains pointers to the read and write qinit structures
 */

static struct streamtab fcip_info = {
	&fcip_rinit,	/* st_rdinit */
	&fcip_winit,	/* st_wrinit */
	NULL,		/* st_muxrinit */
	NULL,		/* st_muxwrinit */
};

static struct cb_ops  fcip_cb_ops = {
	nodev,				/* open */
	nodev,				/* close */
	nodev,				/* strategy */
	nodev,				/* print */
	nodev,				/* dump */
	nodev,				/* read */
	nodev,				/* write */
	nodev,				/* ioctl */
	nodev,				/* devmap */
	nodev,				/* mmap */
	nodev,				/* segmap */
	nochpoll,			/* poll */
	ddi_prop_op,			/* cb_prop_op */
	&fcip_info,			/* streamtab  */
	D_MP | D_HOTPLUG,		/* Driver compatibility flag */
	CB_REV,				/* rev */
	nodev,				/* int (*cb_aread)() */
	nodev				/* int (*cb_awrite)() */
};

/*
 * autoconfiguration routines.
 */
static struct dev_ops fcip_ops = {
	DEVO_REV,		/* devo_rev, */
	0,			/* refcnt  */
	fcip_getinfo,		/* info */
	nulldev,		/* identify */
	nulldev,		/* probe */
	fcip_attach,		/* attach */
	fcip_detach,		/* detach */
	nodev,			/* RESET */
	&fcip_cb_ops,		/* driver operations */
	NULL,			/* bus operations */
	ddi_power		/* power management */
};

#define	FCIP_VERSION	"1.61"
#define	FCIP_NAME	"SunFC FCIP v" FCIP_VERSION

#define	PORT_DRIVER	"fp"

#define	GETSTRUCT(struct, number)	\
	((struct *)kmem_zalloc((size_t)(sizeof (struct) * (number)), \
		KM_SLEEP))

static struct modldrv modldrv = {
	&mod_driverops,			/* Type of module - driver */
	FCIP_NAME,			/* Name of module */
	&fcip_ops,			/* driver ops */
};

static struct modlinkage modlinkage = {
	MODREV_1, (void *)&modldrv, NULL
};


/*
 * Now for some global statics
 */
static uint32_t	fcip_ub_nbufs = FCIP_UB_NBUFS;
static uint32_t fcip_ub_size = FCIP_UB_SIZE;
static int fcip_pkt_ttl_ticks = FCIP_PKT_TTL;
static int fcip_tick_incr = 1;
static int fcip_wait_cmds = FCIP_WAIT_CMDS;
static int fcip_num_attaching = 0;
static int fcip_port_attach_pending = 0;
static int fcip_create_nodes_on_demand = 1;	/* keep it similar to fcp */
static int fcip_cache_on_arp_broadcast = 0;
static int fcip_farp_supported = 0;
static int fcip_minor_node_created = 0;

/*
 * Supported FCAs
 */
#define	QLC_PORT_1_ID_BITS		0x100
#define	QLC_PORT_2_ID_BITS		0x101
#define	QLC_PORT_NAA			0x2
#define	QLC_MODULE_NAME			"qlc"
#define	IS_QLC_PORT(port_dip)		\
			(strcmp(ddi_driver_name(ddi_get_parent((port_dip))),\
			QLC_MODULE_NAME) == 0)


/*
 * fcip softstate structures head.
 */

static void *fcip_softp = NULL;

/*
 * linked list of active (inuse) driver streams
 */

static int fcip_num_instances = 0;
static dev_info_t *fcip_module_dip = (dev_info_t *)0;


/*
 * Ethernet broadcast address: Broadcast addressing in IP over fibre
 * channel should be the IEEE ULA (also the low 6 bytes of the Port WWN).
 *
 * The broadcast addressing varies for differing topologies a node may be in:
 *	- On a private loop the ARP broadcast is a class 3 sequence sent
 *	  using OPNfr (Open Broadcast Replicate primitive) followed by
 *	  the ARP frame to D_ID 0xFFFFFF
 *
 *	- On a public Loop the broadcast sequence is sent to AL_PA 0x00
 *	  (no OPNfr primitive).
 *
 *	- For direct attach and point to point topologies we just send
 *	  the frame to D_ID 0xFFFFFF
 *
 * For public loop the handling would probably be different - for now
 * I'll just declare this struct - It can be deleted if not necessary.
 *
 */


/*
 * DL_INFO_ACK template for the fcip module. The dl_info_ack_t structure is
 * returned as a part of an  DL_INFO_ACK message which is a M_PCPROTO message
 * returned in response to a DL_INFO_REQ message sent to us from a DLS user
 * Let us fake an ether header as much as possible.
 *
 * dl_addr_length is the Provider's DLSAP addr which is SAP addr +
 *                Physical addr of the provider. We set this to
 *                ushort_t + sizeof (la_wwn_t) for Fibre Channel ports.
 * dl_mac_type    Lets just use DL_ETHER - we can try using DL_IPFC, a new
 *		  dlpi.h define later.
 * dl_sap_length  -2 indicating the SAP address follows the Physical addr
 *		  component in the DLSAP addr.
 * dl_service_mode: DLCLDS - connectionless data link service.
 *
 */

static dl_info_ack_t fcip_infoack = {
	DL_INFO_ACK,				/* dl_primitive */
	FCIPMTU,				/* dl_max_sdu */
	0,					/* dl_min_sdu */
	FCIPADDRL,				/* dl_addr_length */
	DL_ETHER,				/* dl_mac_type */
	0,					/* dl_reserved */
	0,					/* dl_current_state */
	-2,					/* dl_sap_length */
	DL_CLDLS,				/* dl_service_mode */
	0,					/* dl_qos_length */
	0,					/* dl_qos_offset */
	0,					/* dl_range_length */
	0,					/* dl_range_offset */
	DL_STYLE2,				/* dl_provider_style */
	sizeof (dl_info_ack_t),			/* dl_addr_offset */
	DL_VERSION_2,				/* dl_version */
	ETHERADDRL,				/* dl_brdcst_addr_length */
	sizeof (dl_info_ack_t) + FCIPADDRL,	/* dl_brdcst_addr_offset */
	0					/* dl_growth */
};

/*
 * FCIP broadcast address definition.
 */
static	struct ether_addr	fcipnhbroadcastaddr = {
	0xff, 0xff, 0xff, 0xff, 0xff, 0xff
};

/*
 * RFC2625 requires the broadcast ARP address in the ARP data payload to
 * be set to 0x00 00 00 00 00 00 for ARP broadcast packets
 */
static	struct ether_addr	fcip_arpbroadcast_addr = {
	0x00, 0x00, 0x00, 0x00, 0x00, 0x00
};


#define	ether_bcopy(src, dest)	bcopy((src), (dest), ETHERADDRL);

/*
 * global kernel locks
 */
static kcondvar_t	fcip_global_cv;
static kmutex_t		fcip_global_mutex;

/*
 * fctl external defines
 */
extern int fc_ulp_add(fc_ulp_modinfo_t *);

/*
 * fctl data structures
 */

#define	FCIP_REV	0x07

/* linked list of port info structures */
static fcip_port_info_t *fcip_port_head = NULL;

/* linked list of fcip structures */
static struct fcipstr	*fcipstrup = NULL;
static krwlock_t	fcipstruplock;


/*
 * Module information structure. This structure gives the FC Transport modules
 * information about an ULP that registers with it.
 */
static fc_ulp_modinfo_t	fcip_modinfo = {
	0,			/* for xref checks? */
	FCTL_ULP_MODREV_4,	/* FCIP revision */
	FC_TYPE_IS8802_SNAP,	/* type 5 for SNAP encapsulated datagrams */
	FCIP_NAME,		/* module name as in the modldrv struct */
	0x0,			/* get all statec callbacks for now */
	fcip_port_attach,	/* port attach callback */
	fcip_port_detach,	/* port detach callback */
	fcip_port_ioctl,	/* port ioctl callback */
	fcip_els_cb,		/* els callback */
	fcip_data_cb,		/* data callback */
	fcip_statec_cb		/* state change callback */
};


/*
 * Solaris 9 and up, the /kernel/drv/fp.conf file will have the following entry
 *
 * ddi-forceattach=1;
 *
 * This will ensure that fp is loaded at bootup. No additional checks are needed
 */
int
_init(void)
{
	int	rval;

	FCIP_TNF_LOAD();

	/*
	 * Initialize the mutexs used by port attach and other callbacks.
	 * The transport can call back into our port_attach_callback
	 * routine even before _init() completes and bad things can happen.
	 */
	mutex_init(&fcip_global_mutex, NULL, MUTEX_DRIVER, NULL);
	cv_init(&fcip_global_cv, NULL, CV_DRIVER, NULL);
	rw_init(&fcipstruplock, NULL, RW_DRIVER, NULL);

	mutex_enter(&fcip_global_mutex);
	fcip_port_attach_pending = 1;
	mutex_exit(&fcip_global_mutex);

	/*
	 * Now attempt to register fcip with the transport.
	 * If fc_ulp_add fails, fcip module will not be loaded.
	 */
	rval = fc_ulp_add(&fcip_modinfo);
	if (rval != FC_SUCCESS) {
		mutex_destroy(&fcip_global_mutex);
		cv_destroy(&fcip_global_cv);
		rw_destroy(&fcipstruplock);
		switch (rval) {
		case FC_ULP_SAMEMODULE:
			FCIP_DEBUG(FCIP_DEBUG_DEFAULT, (CE_WARN,
			    "!fcip: module is already registered with"
			    " transport"));
			rval = EEXIST;
			break;
		case FC_ULP_SAMETYPE:
			FCIP_DEBUG(FCIP_DEBUG_DEFAULT, (CE_WARN,
			    "!fcip: Another module of the same ULP type 0x%x"
			    " is already registered with the transport",
			    fcip_modinfo.ulp_type));
			rval = EEXIST;
			break;
		case FC_BADULP:
			FCIP_DEBUG(FCIP_DEBUG_DEFAULT, (CE_WARN,
			    "!fcip: Current fcip version 0x%x does not match"
			    " fctl version",
			    fcip_modinfo.ulp_rev));
			rval = ENODEV;
			break;
		default:
			FCIP_DEBUG(FCIP_DEBUG_DEFAULT, (CE_WARN,
			    "!fcip: fc_ulp_add failed with status 0x%x", rval));
			rval = ENODEV;
			break;
		}
		FCIP_TNF_UNLOAD(&modlinkage);
		return (rval);
	}

	if ((rval = ddi_soft_state_init(&fcip_softp, sizeof (struct fcip),
			FCIP_NUM_INSTANCES)) != 0) {
		mutex_destroy(&fcip_global_mutex);
		cv_destroy(&fcip_global_cv);
		rw_destroy(&fcipstruplock);
		(void) fc_ulp_remove(&fcip_modinfo);
		FCIP_TNF_UNLOAD(&modlinkage);
		return (rval);
	}

	if ((rval = mod_install(&modlinkage)) != 0) {
		FCIP_TNF_UNLOAD(&modlinkage);
		(void) fc_ulp_remove(&fcip_modinfo);
		mutex_destroy(&fcip_global_mutex);
		cv_destroy(&fcip_global_cv);
		rw_destroy(&fcipstruplock);
		ddi_soft_state_fini(&fcip_softp);
	}
	return (rval);
}

/*
 * Unload the port driver if this was the only ULP loaded and then
 * deregister with the transport.
 */
int
_fini(void)
{
	int	rval;
	int	rval1;

	/*
	 * Do not permit the module to be unloaded before a port
	 * attach callback has happened.
	 */
	mutex_enter(&fcip_global_mutex);
	if (fcip_num_attaching || fcip_port_attach_pending) {
		mutex_exit(&fcip_global_mutex);
		return (EBUSY);
	}
	mutex_exit(&fcip_global_mutex);

	if ((rval = mod_remove(&modlinkage)) != 0) {
		return (rval);
	}

	/*
	 * unregister with the transport layer
	 */
	rval1 = fc_ulp_remove(&fcip_modinfo);

	/*
	 * If the ULP was not registered with the transport, init should
	 * have failed. If transport has no knowledge of our existence
	 * we should simply bail out and succeed
	 */
#ifdef DEBUG
	if (rval1 == FC_BADULP) {
		FCIP_DEBUG(FCIP_DEBUG_DEFAULT, (CE_WARN,
		"fcip: ULP was never registered with the transport"));
		rval = ENODEV;
	} else if (rval1 == FC_BADTYPE) {
		FCIP_DEBUG(FCIP_DEBUG_DEFAULT, (CE_WARN,
			"fcip: No ULP of this type 0x%x was registered with "
			"transport", fcip_modinfo.ulp_type));
		rval = ENODEV;
	}
#endif /* DEBUG */

	mutex_destroy(&fcip_global_mutex);
	rw_destroy(&fcipstruplock);
	cv_destroy(&fcip_global_cv);
	ddi_soft_state_fini(&fcip_softp);

	FCIP_TNF_UNLOAD(&modlinkage);

	return (rval);
}

/*
 * Info about this loadable module
 */
int
_info(struct modinfo *modinfop)
{
	return (mod_info(&modlinkage, modinfop));
}

/*
 * The port attach callback is invoked by the port driver when a FCA
 * port comes online and binds with the transport layer. The transport
 * then callsback into all ULP modules registered with it. The Port attach
 * call back will also provide the ULP module with the Port's WWN and S_ID
 */
/* ARGSUSED */
static int
fcip_port_attach(opaque_t ulp_handle, fc_ulp_port_info_t *port_info,
    fc_attach_cmd_t cmd, uint32_t sid)
{
	int 			rval = FC_FAILURE;
	int 			instance;
	struct fcip		*fptr;
	fcip_port_info_t	*fport = NULL;
	fcip_port_info_t	*cur_fport;
	fc_portid_t		src_id;

	switch (cmd) {
	case FC_CMD_ATTACH: {
		la_wwn_t	*ww_pn = NULL;
		/*
		 * It was determined that, as per spec, the lower 48 bits of
		 * the port-WWN will always be unique. This will make the MAC
		 * address (i.e the lower 48 bits of the WWN), that IP/ARP
		 * depend on, unique too. Hence we should be able to remove the
		 * restriction of attaching to only one of the ports of
		 * multi port FCAs.
		 *
		 * Earlier, fcip used to attach only to qlc module and fail
		 * silently for attach failures resulting from unknown FCAs or
		 * unsupported FCA ports. Now, we'll do no such checks.
		 */
		ww_pn = &port_info->port_pwwn;

		FCIP_TNF_PROBE_2((fcip_port_attach, "fcip io", /* CSTYLED */,
			tnf_string, msg, "port id bits",
			tnf_opaque, nport_id, ww_pn->w.nport_id));
		FCIP_DEBUG(FCIP_DEBUG_ATTACH, (CE_NOTE,
		    "port id bits: 0x%x", ww_pn->w.nport_id));
		/*
		 * A port has come online
		 */
		mutex_enter(&fcip_global_mutex);
		fcip_num_instances++;
		fcip_num_attaching++;

		if (fcip_port_head == NULL) {
			/* OK to sleep here ? */
			fport = kmem_zalloc(sizeof (fcip_port_info_t),
						KM_NOSLEEP);
			if (fport == NULL) {
				fcip_num_instances--;
				fcip_num_attaching--;
				ASSERT(fcip_num_attaching >= 0);
				mutex_exit(&fcip_global_mutex);
				rval = FC_FAILURE;
				cmn_err(CE_WARN, "!fcip(%d): port attach "
				    "failed: alloc failed",
				    ddi_get_instance(port_info->port_dip));
				goto done;
			}
			fcip_port_head = fport;
		} else {
			/*
			 * traverse the port list and also check for
			 * duplicate port attaches - Nothing wrong in being
			 * paranoid Heh Heh.
			 */
			cur_fport = fcip_port_head;
			while (cur_fport != NULL) {
				if (cur_fport->fcipp_handle ==
				    port_info->port_handle) {
					fcip_num_instances--;
					fcip_num_attaching--;
					ASSERT(fcip_num_attaching >= 0);
					mutex_exit(&fcip_global_mutex);
					FCIP_DEBUG(FCIP_DEBUG_ATTACH, (CE_WARN,
					    "!fcip(%d): port already "
					    "attached!!", ddi_get_instance(
					    port_info->port_dip)));
					rval = FC_FAILURE;
					goto done;
				}
				cur_fport = cur_fport->fcipp_next;
			}
			fport = kmem_zalloc(sizeof (fcip_port_info_t),
						KM_NOSLEEP);
			if (fport == NULL) {
				rval = FC_FAILURE;
				fcip_num_instances--;
				fcip_num_attaching--;
				ASSERT(fcip_num_attaching >= 0);
				mutex_exit(&fcip_global_mutex);
				cmn_err(CE_WARN, "!fcip(%d): port attach "
				    "failed: alloc failed",
				    ddi_get_instance(port_info->port_dip));
				goto done;
			}
			fport->fcipp_next = fcip_port_head;
			fcip_port_head = fport;
		}

		mutex_exit(&fcip_global_mutex);

		/*
		 * now fill in the details about the port itself
		 */
		fport->fcipp_linkage = *port_info->port_linkage;
		fport->fcipp_handle = port_info->port_handle;
		fport->fcipp_dip = port_info->port_dip;
		fport->fcipp_topology = port_info->port_flags;
		fport->fcipp_pstate = port_info->port_state;
		fport->fcipp_naa = port_info->port_pwwn.w.naa_id;
		bcopy(&port_info->port_pwwn, &fport->fcipp_pwwn,
		    sizeof (la_wwn_t));
		bcopy(&port_info->port_nwwn, &fport->fcipp_nwwn,
		    sizeof (la_wwn_t));
		fport->fcipp_fca_pkt_size = port_info->port_fca_pkt_size;
		fport->fcipp_cmd_dma_attr = *port_info->port_cmd_dma_attr;
		fport->fcipp_resp_dma_attr = *port_info->port_resp_dma_attr;
		fport->fcipp_fca_acc_attr = *port_info->port_acc_attr;
		src_id.port_id = sid;
		src_id.priv_lilp_posit = 0;
		fport->fcipp_sid = src_id;

		/*
		 * allocate soft state for this instance
		 */
		instance = ddi_get_instance(fport->fcipp_dip);
		if (ddi_soft_state_zalloc(fcip_softp,
		    instance) != DDI_SUCCESS) {
			rval = FC_FAILURE;
			cmn_err(CE_WARN, "!fcip(%d): port attach failed: "
			    "soft state alloc failed", instance);
			goto failure;
		}

		fptr = ddi_get_soft_state(fcip_softp, instance);

		if (fptr == NULL) {
			rval = FC_FAILURE;
			cmn_err(CE_WARN, "!fcip(%d): port attach failed: "
			    "failure to get soft state", instance);
			goto failure;
		}

		/*
		 * initialize all mutexes and locks required for this module
		 */
		mutex_init(&fptr->fcip_mutex, NULL, MUTEX_DRIVER, NULL);
		mutex_init(&fptr->fcip_ub_mutex, NULL, MUTEX_DRIVER, NULL);
		mutex_init(&fptr->fcip_rt_mutex, NULL, MUTEX_DRIVER, NULL);
		mutex_init(&fptr->fcip_dest_mutex, NULL, MUTEX_DRIVER, NULL);
		mutex_init(&fptr->fcip_sendup_mutex, NULL, MUTEX_DRIVER, NULL);
		cv_init(&fptr->fcip_farp_cv, NULL, CV_DRIVER, NULL);
		cv_init(&fptr->fcip_sendup_cv, NULL, CV_DRIVER, NULL);
		cv_init(&fptr->fcip_ub_cv, NULL, CV_DRIVER, NULL);

		mutex_enter(&fptr->fcip_mutex);

		fptr->fcip_dip = fport->fcipp_dip;	/* parent's dip */
		fptr->fcip_instance = instance;
		fptr->fcip_ub_upstream = 0;

		if (FC_PORT_STATE_MASK(port_info->port_state) ==
		    FC_STATE_ONLINE) {
			fptr->fcip_port_state = FCIP_PORT_ONLINE;
			if (fptr->fcip_flags & FCIP_LINK_DOWN) {
				fptr->fcip_flags &= ~FCIP_LINK_DOWN;
			}
		} else {
			fptr->fcip_port_state = FCIP_PORT_OFFLINE;
		}

		fptr->fcip_flags |= FCIP_ATTACHING;
		fptr->fcip_port_info = fport;

		/*
		 * Extract our MAC addr from our port's WWN. The lower 48
		 * bits will be our MAC address
		 */
		wwn_to_ether(&fport->fcipp_nwwn, &fptr->fcip_macaddr);

		fport->fcipp_fcip = fptr;

		FCIP_DEBUG(FCIP_DEBUG_ATTACH,
		    (CE_NOTE, "fcipdest : 0x%lx, rtable : 0x%lx",
		    (long)(sizeof (fptr->fcip_dest)),
		    (long)(sizeof (fptr->fcip_rtable))));

		bzero(fptr->fcip_dest, sizeof (fptr->fcip_dest));
		bzero(fptr->fcip_rtable, sizeof (fptr->fcip_rtable));

		/*
		 * create a taskq to handle sundry jobs for the driver
		 * This way we can have jobs run in parallel
		 */
		fptr->fcip_tq = taskq_create("fcip_tasks",
		    FCIP_NUM_THREADS, MINCLSYSPRI, FCIP_MIN_TASKS,
		    FCIP_MAX_TASKS, TASKQ_PREPOPULATE);

		mutex_exit(&fptr->fcip_mutex);

		/*
		 * create a separate thread to handle all unsolicited
		 * callback handling. This is because unsolicited_callback
		 * can happen from an interrupt context and the upstream
		 * modules can put new messages right back in the same
		 * thread context. This usually works fine, but sometimes
		 * we may have to block to obtain the dest struct entries
		 * for some remote ports.
		 */
		mutex_enter(&fptr->fcip_sendup_mutex);
		if (thread_create(NULL, DEFAULTSTKSZ,
		    (void (*)())fcip_sendup_thr, (caddr_t)fptr, 0, &p0,
		    TS_RUN, minclsyspri) == NULL) {
			mutex_exit(&fptr->fcip_sendup_mutex);
			cmn_err(CE_WARN,
			    "!unable to create fcip sendup thread for "
			    " instance: 0x%x", instance);
			rval = FC_FAILURE;
			goto done;
		}
		fptr->fcip_sendup_thr_initted = 1;
		fptr->fcip_sendup_head = fptr->fcip_sendup_tail = NULL;
		mutex_exit(&fptr->fcip_sendup_mutex);


		/* Let the attach handler do the rest */
		if (fcip_port_attach_handler(fptr) != FC_SUCCESS) {
			/*
			 * We have already cleaned up so return
			 */
			rval = FC_FAILURE;
			cmn_err(CE_WARN, "!fcip(%d): port attach failed",
			    instance);
			goto done;
		}

		FCIP_DEBUG(FCIP_DEBUG_ATTACH, (CE_CONT,
		    "!fcip attach for port instance (0x%x) successful",
		    instance));

		rval = FC_SUCCESS;
		goto done;
	}
	case FC_CMD_POWER_UP:
	/* FALLTHROUGH */
	case FC_CMD_RESUME:
		mutex_enter(&fcip_global_mutex);
		fport = fcip_port_head;
		while (fport != NULL) {
			if (fport->fcipp_handle == port_info->port_handle) {
				break;
			}
			fport = fport->fcipp_next;
		}
		if (fport == NULL) {
			rval = FC_SUCCESS;
			mutex_exit(&fcip_global_mutex);
			goto done;
		}
		rval = fcip_handle_resume(fport, port_info, cmd);
		mutex_exit(&fcip_global_mutex);
		goto done;

	default:
		FCIP_TNF_PROBE_2((fcip_port_attach, "fcip io", /* CSTYLED */,
			tnf_string, msg, "unknown command type",
			tnf_uint, cmd, cmd));
		FCIP_DEBUG(FCIP_DEBUG_ATTACH, (CE_WARN,
		    "unknown cmd type 0x%x in port_attach", cmd));
		rval = FC_FAILURE;
		goto done;
	}

failure:
	if (fport) {
		mutex_enter(&fcip_global_mutex);
		fcip_num_attaching--;
		ASSERT(fcip_num_attaching >= 0);
		(void) fcip_softstate_free(fport);
		fcip_port_attach_pending = 0;
		mutex_exit(&fcip_global_mutex);
	}
	return (rval);

done:
	mutex_enter(&fcip_global_mutex);
	fcip_port_attach_pending = 0;
	mutex_exit(&fcip_global_mutex);
	return (rval);
}

/*
 * fcip_port_attach_handler : Completes the port attach operation after
 * the ulp_port_attach routine has completed its ground work. The job
 * of this function among other things is to obtain and handle topology
 * specifics, initialize a port, setup broadcast address entries in
 * the fcip tables etc. This routine cleans up behind itself on failures.
 * Returns FC_SUCCESS or FC_FAILURE.
 */
static int
fcip_port_attach_handler(struct fcip *fptr)
{
	fcip_port_info_t		*fport = fptr->fcip_port_info;
	int				rval = FC_FAILURE;

	ASSERT(fport != NULL);

	mutex_enter(&fcip_global_mutex);

	FCIP_DEBUG(FCIP_DEBUG_ATTACH, (CE_NOTE,
	    "fcip module dip: %p instance: %d",
	    (void *)fcip_module_dip, ddi_get_instance(fptr->fcip_dip)));

	if (fcip_module_dip == NULL) {
		clock_t		fcip_lbolt;

		fcip_lbolt = ddi_get_lbolt();
		/*
		 * we need to use the fcip devinfo for creating
		 * the clone device node, but the fcip attach
		 * (from its conf file entry claiming to be a
		 * child of pseudo) may not have happened yet.
		 * wait here for 10 seconds and fail port attach
		 * if the fcip devinfo is not attached yet
		 */
		fcip_lbolt += drv_usectohz(FCIP_INIT_DELAY);

		FCIP_DEBUG(FCIP_DEBUG_ATTACH,
		    (CE_WARN, "cv_timedwait lbolt %lx", fcip_lbolt));

		(void) cv_timedwait(&fcip_global_cv, &fcip_global_mutex,
		    fcip_lbolt);

		if (fcip_module_dip == NULL) {
			mutex_exit(&fcip_global_mutex);

			FCIP_DEBUG(FCIP_DEBUG_ATTACH, (CE_WARN,
				"fcip attach did not happen"));
			goto port_attach_cleanup;
		}
	}

	if ((!fcip_minor_node_created) &&
	    fcip_is_supported_fc_topology(fport->fcipp_topology)) {
		/*
		 * Checking for same topologies which are considered valid
		 * by fcip_handle_topology(). Dont create a minor node if
		 * nothing is hanging off the FC port.
		 */
		if (ddi_create_minor_node(fcip_module_dip, "fcip", S_IFCHR,
		    ddi_get_instance(fptr->fcip_dip), DDI_PSEUDO,
		    CLONE_DEV) == DDI_FAILURE) {
			mutex_exit(&fcip_global_mutex);
			FCIP_DEBUG(FCIP_DEBUG_ATTACH, (CE_WARN,
			    "failed to create minor node for fcip(%d)",
			    ddi_get_instance(fptr->fcip_dip)));
			goto port_attach_cleanup;
		}
		fcip_minor_node_created++;
	}
	mutex_exit(&fcip_global_mutex);

	/*
	 * initialize port for traffic
	 */
	if (fcip_init_port(fptr) != FC_SUCCESS) {
		/* fcip_init_port has already cleaned up its stuff */

		mutex_enter(&fcip_global_mutex);

		if ((fcip_num_instances == 1) &&
		    (fcip_minor_node_created == 1)) {
			/* Remove minor node iff this is the last instance */
			ddi_remove_minor_node(fcip_module_dip, NULL);
		}

		mutex_exit(&fcip_global_mutex);

		goto port_attach_cleanup;
	}

	mutex_enter(&fptr->fcip_mutex);
	fptr->fcip_flags &= ~FCIP_ATTACHING;
	fptr->fcip_flags |= FCIP_INITED;
	fptr->fcip_timeout_ticks = 0;

	/*
	 * start the timeout threads
	 */
	fptr->fcip_timeout_id = timeout(fcip_timeout, fptr,
	    drv_usectohz(1000000));

	mutex_exit(&fptr->fcip_mutex);
	mutex_enter(&fcip_global_mutex);
	fcip_num_attaching--;
	ASSERT(fcip_num_attaching >= 0);
	mutex_exit(&fcip_global_mutex);
	rval = FC_SUCCESS;
	return (rval);

port_attach_cleanup:
	mutex_enter(&fcip_global_mutex);
	(void) fcip_softstate_free(fport);
	fcip_num_attaching--;
	ASSERT(fcip_num_attaching >= 0);
	mutex_exit(&fcip_global_mutex);
	rval = FC_FAILURE;
	return (rval);
}


/*
 * Handler for DDI_RESUME operations. Port must be ready to restart IP
 * traffic on resume
 */
static int
fcip_handle_resume(fcip_port_info_t *fport, fc_ulp_port_info_t *port_info,
    fc_attach_cmd_t cmd)
{
	int 		rval = FC_SUCCESS;
	struct fcip	*fptr = fport->fcipp_fcip;
	struct fcipstr	*tslp;
	int		index;


	ASSERT(fptr != NULL);

	mutex_enter(&fptr->fcip_mutex);

	if (cmd == FC_CMD_POWER_UP) {
		fptr->fcip_flags &= ~(FCIP_POWER_DOWN);
		if (fptr->fcip_flags & FCIP_SUSPENDED) {
			mutex_exit(&fptr->fcip_mutex);
			return (FC_SUCCESS);
		}
	} else if (cmd == FC_CMD_RESUME) {
		fptr->fcip_flags &= ~(FCIP_SUSPENDED);
	} else {
		mutex_exit(&fptr->fcip_mutex);
		return (FC_FAILURE);
	}

	/*
	 * set the current port state and topology
	 */
	fport->fcipp_topology = port_info->port_flags;
	fport->fcipp_pstate = port_info->port_state;

	rw_enter(&fcipstruplock, RW_READER);
	for (tslp = fcipstrup; tslp; tslp = tslp->sl_nextp) {
		if (tslp->sl_fcip == fptr) {
			break;
		}
	}
	rw_exit(&fcipstruplock);

	/*
	 * No active streams on this port
	 */
	if (tslp == NULL) {
		rval = FC_SUCCESS;
		goto done;
	}

	mutex_enter(&fptr->fcip_rt_mutex);
	for (index = 0; index < FCIP_RT_HASH_ELEMS; index++) {
		struct fcip_routing_table 	*frp;

		frp = fptr->fcip_rtable[index];
		while (frp) {
			uint32_t		did;
			/*
			 * Mark the broadcast RTE available again. It
			 * was marked SUSPENDED during SUSPEND.
			 */
			did = fcip_get_broadcast_did(fptr);
			if (frp->fcipr_d_id.port_id == did) {
				frp->fcipr_state = 0;
				index = FCIP_RT_HASH_ELEMS;
				break;
			}
			frp = frp->fcipr_next;
		}
	}
	mutex_exit(&fptr->fcip_rt_mutex);

	/*
	 * fcip_handle_topology will update the port entries in the
	 * routing table.
	 * fcip_handle_topology also takes care of resetting the
	 * fcipr_state field in the routing table structure. The entries
	 * were set to RT_INVALID during suspend.
	 */
	fcip_handle_topology(fptr);

done:
	/*
	 * Restart the timeout thread
	 */
	fptr->fcip_timeout_id = timeout(fcip_timeout, fptr,
	    drv_usectohz(1000000));
	mutex_exit(&fptr->fcip_mutex);
	return (rval);
}


/*
 * Insert a destination port entry into the routing table for
 * this port
 */
static void
fcip_rt_update(struct fcip *fptr, fc_portmap_t *devlist, uint32_t listlen)
{
	struct fcip_routing_table	*frp;
	fcip_port_info_t		*fport = fptr->fcip_port_info;
	int				hash_bucket, i;
	fc_portmap_t			*pmap;
	char				wwn_buf[20];

	FCIP_TNF_PROBE_2((fcip_rt_update, "fcip io", /* CSTYLED */,
		tnf_string, msg, "enter",
		tnf_int, listlen, listlen));

	ASSERT(!mutex_owned(&fptr->fcip_mutex));
	mutex_enter(&fptr->fcip_rt_mutex);

	for (i = 0; i < listlen; i++) {
		pmap = &(devlist[i]);

		frp = fcip_lookup_rtable(fptr, &(pmap->map_pwwn),
		    FCIP_COMPARE_PWWN);
		/*
		 * If an entry for a port in the devlist exists in the
		 * in the per port routing table, make sure the data
		 * is current. We need to do this irrespective of the
		 * underlying port topology.
		 */
		switch (pmap->map_type) {
		/* FALLTHROUGH */
		case PORT_DEVICE_NOCHANGE:
		/* FALLTHROUGH */
		case PORT_DEVICE_USER_LOGIN:
		/* FALLTHROUGH */
		case PORT_DEVICE_CHANGED:
		/* FALLTHROUGH */
		case PORT_DEVICE_NEW:
			if (frp == NULL) {
				goto add_new_entry;
			} else if (frp) {
				goto update_entry;
			} else {
				continue;
			}

		case PORT_DEVICE_OLD:
		/* FALLTHROUGH */
		case PORT_DEVICE_USER_LOGOUT:
			/*
			 * Mark entry for removal from Routing Table if
			 * one exists. Let the timeout thread actually
			 * remove the entry after we've given up hopes
			 * of the port ever showing up.
			 */
			if (frp) {
				uint32_t		did;

				/*
				 * Mark the routing table as invalid to bail
				 * the packets early that are in transit
				 */
				did = fptr->fcip_broadcast_did;
				if (frp->fcipr_d_id.port_id != did) {
					frp->fcipr_pd = NULL;
					frp->fcipr_state = FCIP_RT_INVALID;
					frp->fcipr_invalid_timeout =
					    fptr->fcip_timeout_ticks +
					    FCIP_RTE_TIMEOUT;
				}
			}
			continue;

		default:
			FCIP_DEBUG(FCIP_DEBUG_INIT, (CE_WARN,
			    "unknown map flags in rt_update"));
			continue;
		}
add_new_entry:
		ASSERT(frp == NULL);
		hash_bucket = FCIP_RT_HASH(pmap->map_pwwn.raw_wwn);

		ASSERT(hash_bucket < FCIP_RT_HASH_ELEMS);

		FCIP_TNF_PROBE_2((fcip_rt_update, "cfip io", /* CSTYLED */,
			tnf_string, msg,
			"add new entry",
			tnf_int, hashbucket, hash_bucket));

		frp = (struct fcip_routing_table *)
		    kmem_zalloc(sizeof (struct fcip_routing_table), KM_SLEEP);
		/* insert at beginning of hash bucket */
		frp->fcipr_next = fptr->fcip_rtable[hash_bucket];
		fptr->fcip_rtable[hash_bucket] = frp;
		fc_wwn_to_str(&pmap->map_pwwn, wwn_buf);
		FCIP_DEBUG(FCIP_DEBUG_ATTACH, (CE_NOTE,
		    "added entry for pwwn %s and d_id 0x%x",
		    wwn_buf, pmap->map_did.port_id));
update_entry:
		bcopy((void *)&pmap->map_pwwn,
		    (void *)&frp->fcipr_pwwn, sizeof (la_wwn_t));
		bcopy((void *)&pmap->map_nwwn, (void *)&frp->fcipr_nwwn,
		    sizeof (la_wwn_t));
		frp->fcipr_d_id = pmap->map_did;
		frp->fcipr_state = pmap->map_state;
		frp->fcipr_pd = pmap->map_pd;

		/*
		 * If there is no pd for a destination port that is not
		 * a broadcast entry, the port is pretty much unusable - so
		 * mark the port for removal so we can try adding back the
		 * entry again.
		 */
		if ((frp->fcipr_pd == NULL) &&
		    (frp->fcipr_d_id.port_id != fptr->fcip_broadcast_did)) {
			frp->fcipr_state = PORT_DEVICE_INVALID;
			frp->fcipr_invalid_timeout = fptr->fcip_timeout_ticks +
			    (FCIP_RTE_TIMEOUT / 2);
		}
		frp->fcipr_fca_dev =
		    fc_ulp_get_fca_device(fport->fcipp_handle, pmap->map_did);

		/*
		 * login to the remote port. Don't worry about
		 * plogi failures for now
		 */
		if (pmap->map_pd != NULL) {
			(void) fcip_do_plogi(fptr, frp);
		} else if (FC_TOP_EXTERNAL(fport->fcipp_topology)) {
			fc_wwn_to_str(&frp->fcipr_pwwn, wwn_buf);
			FCIP_DEBUG(FCIP_DEBUG_MISC, (CE_NOTE,
			    "logging into pwwn %s, d_id 0x%x",
			    wwn_buf, frp->fcipr_d_id.port_id));
			(void) fcip_do_plogi(fptr, frp);
		}

		FCIP_TNF_BYTE_ARRAY(fcip_rt_update, "fcip io", "detail",
			"new wwn in rt", pwwn,
			&frp->fcipr_pwwn, sizeof (la_wwn_t));
	}
	mutex_exit(&fptr->fcip_rt_mutex);
}


/*
 * return a matching routing table entry for a given fcip instance
 */
struct fcip_routing_table *
fcip_lookup_rtable(struct fcip *fptr, la_wwn_t *wwn, int matchflag)
{
	struct fcip_routing_table	*frp = NULL;
	int				hash_bucket;


	FCIP_TNF_PROBE_1((fcip_lookup_rtable, "fcip io", /* CSTYLED */,
		tnf_string, msg, "enter"));
	FCIP_TNF_BYTE_ARRAY(fcip_lookup_rtable, "fcip io", "detail",
		"rtable lookup for", wwn,
		&wwn->raw_wwn, sizeof (la_wwn_t));
	FCIP_TNF_PROBE_2((fcip_lookup_rtable, "fcip io", /* CSTYLED */,
		tnf_string, msg, "match by",
		tnf_int, matchflag, matchflag));

	ASSERT(mutex_owned(&fptr->fcip_rt_mutex));

	hash_bucket = FCIP_RT_HASH(wwn->raw_wwn);
	frp = fptr->fcip_rtable[hash_bucket];
	while (frp != NULL) {

		FCIP_TNF_BYTE_ARRAY(fcip_lookup_rtable, "fcip io", "detail",
			"rtable entry", nwwn,
			&(frp->fcipr_nwwn.raw_wwn), sizeof (la_wwn_t));

		if (fcip_wwn_compare(&frp->fcipr_pwwn, wwn, matchflag) == 0) {
			break;
		}

		frp = frp->fcipr_next;
	}
	FCIP_TNF_PROBE_2((fcip_lookup_rtable, "fcip io", /* CSTYLED */,
		tnf_string, msg, "lookup result",
		tnf_opaque, frp, frp));
	return (frp);
}

/*
 * Attach of fcip under pseudo. The actual setup of the interface
 * actually happens in fcip_port_attach on a callback from the
 * transport. The port_attach callback however can proceed only
 * after the devinfo for fcip has been created under pseudo
 */
static int
fcip_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
{
	switch ((int)cmd) {

	case DDI_ATTACH: {
		ASSERT(fcip_module_dip == NULL);
		fcip_module_dip = dip;

		/*
		 * this call originates as a result of fcip's conf
		 * file entry and will result in a fcip instance being
		 * a child of pseudo. We should ensure here that the port
		 * driver (fp) has been loaded and initted since we would
		 * never get a port attach callback without fp being loaded.
		 * If we are unable to succesfully load and initalize fp -
		 * just fail this attach.
		 */
		mutex_enter(&fcip_global_mutex);

		FCIP_DEBUG(FCIP_DEBUG_ATTACH,
		    (CE_WARN, "global cv - signaling"));

		cv_signal(&fcip_global_cv);

		FCIP_DEBUG(FCIP_DEBUG_ATTACH,
		    (CE_WARN, "global cv - signaled"));
		mutex_exit(&fcip_global_mutex);
		return (DDI_SUCCESS);
	}
	case DDI_RESUME:
		/*
		 * Resume appears trickier
		 */
		return (DDI_SUCCESS);
	default:
		return (DDI_FAILURE);
	}
}


/*
 * The detach entry point to permit unloading fcip. We make sure
 * there are no active streams before we proceed with the detach
 */
/* ARGSUSED */
static int
fcip_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
{
	struct fcip		*fptr;
	fcip_port_info_t	*fport;
	int			detached;

	switch (cmd) {
	case DDI_DETACH: {
		/*
		 * If we got here, any active streams should have been
		 * unplumbed but check anyway
		 */
		mutex_enter(&fcip_global_mutex);
		if (fcipstrup != NULL) {
			mutex_exit(&fcip_global_mutex);
			return (DDI_FAILURE);
		}

		if (fcip_port_head != NULL) {
			/*
			 * Check to see if we have unattached/unbound
			 * ports. If all the ports are unattached/unbound go
			 * ahead and unregister with the transport
			 */
			fport = fcip_port_head;
			while (fport != NULL) {
				fptr = fport->fcipp_fcip;
				if (fptr == NULL) {
					continue;
				}
				mutex_enter(&fptr->fcip_mutex);
				fptr->fcip_flags |= FCIP_DETACHING;
				if (fptr->fcip_ipq ||
				    fptr->fcip_flags & (FCIP_IN_TIMEOUT |
				    FCIP_IN_CALLBACK | FCIP_ATTACHING |
				    FCIP_SUSPENDED | FCIP_POWER_DOWN |
				    FCIP_REG_INPROGRESS)) {
					FCIP_TNF_PROBE_1((fcip_detach,
					    "fcip io", /* CSTYLED */,
					    tnf_string, msg,
					    "fcip instance busy"));

					mutex_exit(&fptr->fcip_mutex);
					FCIP_DEBUG(FCIP_DEBUG_DETACH, (CE_WARN,
					    "fcip instance busy"));
					break;
				}
				/*
				 * Check for any outstanding pkts. If yes
				 * fail the detach
				 */
				mutex_enter(&fptr->fcip_dest_mutex);
				if (fcip_port_get_num_pkts(fptr) > 0) {
					mutex_exit(&fptr->fcip_dest_mutex);
					mutex_exit(&fptr->fcip_mutex);
					FCIP_DEBUG(FCIP_DEBUG_DETACH, (CE_WARN,
					    "fcip instance busy - pkts "
					    "pending"));
					break;
				}
				mutex_exit(&fptr->fcip_dest_mutex);

				mutex_enter(&fptr->fcip_rt_mutex);
				if (fcip_plogi_in_progress(fptr)) {
					mutex_exit(&fptr->fcip_rt_mutex);
					mutex_exit(&fptr->fcip_mutex);
					FCIP_DEBUG(FCIP_DEBUG_DETACH, (CE_WARN,
					    "fcip instance busy - plogi in "
					    "progress"));
					break;
				}
				mutex_exit(&fptr->fcip_rt_mutex);

				mutex_exit(&fptr->fcip_mutex);
				fport = fport->fcipp_next;
			}
			/*
			 * if fport is non NULL - we have active ports
			 */
			if (fport != NULL) {
				/*
				 * Remove the DETACHING flags on the ports
				 */
				fport = fcip_port_head;
				while (fport != NULL) {
					fptr = fport->fcipp_fcip;
					mutex_enter(&fptr->fcip_mutex);
					fptr->fcip_flags &= ~(FCIP_DETACHING);
					mutex_exit(&fptr->fcip_mutex);
					fport = fport->fcipp_next;
				}
				mutex_exit(&fcip_global_mutex);
				return (DDI_FAILURE);
			}
		}

		/*
		 * free up all softstate structures
		 */
		fport = fcip_port_head;
		while (fport != NULL) {
			detached = 1;

			fptr = fport->fcipp_fcip;
			if (fptr) {
				mutex_enter(&fptr->fcip_mutex);
				/*
				 * Check to see if somebody beat us to the
				 * punch
				 */
				detached = fptr->fcip_flags & FCIP_DETACHED;
				fptr->fcip_flags &= ~(FCIP_DETACHING);
				fptr->fcip_flags |= FCIP_DETACHED;
				mutex_exit(&fptr->fcip_mutex);
			}

			if (!detached) {
				fport = fcip_softstate_free(fport);
			} else {
				/*
				 * If the port was marked as detached
				 * but it was still in the list, that
				 * means another thread has marked it
				 * but we got in while it released the
				 * fcip_global_mutex in softstate_free.
				 * Given that, we're still safe to use
				 * fport->fcipp_next to find out what
				 * the next port on the list is.
				 */
				fport = fport->fcipp_next;
			}

			FCIP_DEBUG(FCIP_DEBUG_DETACH,
			    (CE_NOTE, "detaching port"));

			FCIP_TNF_PROBE_1((fcip_detach,
				"fcip io", /* CSTYLED */, tnf_string,
				msg, "detaching port"));
		}

		/*
		 * If we haven't removed all the port structures, we
		 * aren't yet ready to be detached.
		 */
		if (fcip_port_head != NULL) {
			mutex_exit(&fcip_global_mutex);
			return (DDI_FAILURE);
		}

		fcip_num_instances = 0;
		mutex_exit(&fcip_global_mutex);
		fcip_module_dip = NULL;
		return (DDI_SUCCESS);
	}
	case DDI_SUSPEND:
		return (DDI_SUCCESS);
	default:
		return (DDI_FAILURE);
	}
}

/*
 * The port_detach callback is called from the transport when a
 * FC port is being removed from the transport's control. This routine
 * provides fcip with an opportunity to cleanup all activities and
 * structures on the port marked for removal.
 */
/* ARGSUSED */
static int
fcip_port_detach(opaque_t ulp_handle, fc_ulp_port_info_t *port_info,
    fc_detach_cmd_t cmd)
{
	int 			rval = FC_FAILURE;
	fcip_port_info_t	*fport;
	struct fcip		*fptr;
	struct fcipstr		*strp;

	switch (cmd) {
	case FC_CMD_DETACH: {
		mutex_enter(&fcip_global_mutex);

		if (fcip_port_head == NULL) {
			/*
			 * we are all done but our fini has not been
			 * called yet!! Let's hope we have no active
			 * fcip instances here. - strange secnario but
			 * no harm in having this return a success.
			 */
			fcip_check_remove_minor_node();

			mutex_exit(&fcip_global_mutex);
			return (FC_SUCCESS);
		} else {
			/*
			 * traverse the port list
			 */
			fport = fcip_port_head;
			while (fport != NULL) {
				if (fport->fcipp_handle ==
				    port_info->port_handle) {
					fptr = fport->fcipp_fcip;

					/*
					 * Fail the port detach if there is
					 * still an attached, bound stream on
					 * this interface.
					 */

					rw_enter(&fcipstruplock, RW_READER);

					for (strp = fcipstrup; strp != NULL;
					    strp = strp->sl_nextp) {
						if (strp->sl_fcip == fptr) {
							rw_exit(&fcipstruplock);
							mutex_exit(
							    &fcip_global_mutex);
							return (FC_FAILURE);
						}
					}

					rw_exit(&fcipstruplock);

					/*
					 * fail port detach if we are in
					 * the middle of a deferred port attach
					 * or if the port has outstanding pkts
					 */
					if (fptr != NULL) {
						mutex_enter(&fptr->fcip_mutex);
						if (fcip_check_port_busy
						    (fptr) ||
						    (fptr->fcip_flags &
						    FCIP_DETACHED)) {
							mutex_exit(
							    &fptr->fcip_mutex);
							mutex_exit(
							    &fcip_global_mutex);
							return (FC_FAILURE);
						}

						fptr->fcip_flags |=
						    FCIP_DETACHED;
						mutex_exit(&fptr->fcip_mutex);
					}
					(void) fcip_softstate_free(fport);

					fcip_check_remove_minor_node();
					mutex_exit(&fcip_global_mutex);
					return (FC_SUCCESS);
				}
				fport = fport->fcipp_next;
			}
			ASSERT(fport == NULL);
		}
		mutex_exit(&fcip_global_mutex);
		break;
	}
	case FC_CMD_POWER_DOWN:
	/* FALLTHROUGH */
	case FC_CMD_SUSPEND:
		mutex_enter(&fcip_global_mutex);
		fport = fcip_port_head;
		while (fport != NULL) {
			if (fport->fcipp_handle == port_info->port_handle) {
				break;
			}
			fport = fport->fcipp_next;
		}
		if (fport == NULL) {
			mutex_exit(&fcip_global_mutex);
			break;
		}
		rval = fcip_handle_suspend(fport, cmd);
		mutex_exit(&fcip_global_mutex);
		break;
	default:
		FCIP_DEBUG(FCIP_DEBUG_DETACH,
		    (CE_WARN, "unknown port detach command!!"));
		break;
	}
	return (rval);
}


/*
 * Returns 0 if the port is not busy, else returns non zero.
 */
static int
fcip_check_port_busy(struct fcip *fptr)
{
	int rval = 0, num_pkts = 0;

	ASSERT(fptr != NULL);
	ASSERT(MUTEX_HELD(&fptr->fcip_mutex));

	mutex_enter(&fptr->fcip_dest_mutex);

	if (fptr->fcip_flags & FCIP_PORT_BUSY ||
	    ((num_pkts = fcip_port_get_num_pkts(fptr)) > 0) ||
	    fptr->fcip_num_ipkts_pending) {
		rval = 1;
		FCIP_DEBUG(FCIP_DEBUG_DETACH,
		    (CE_NOTE, "!fcip_check_port_busy: port is busy "
		    "fcip_flags: 0x%x, num_pkts: 0x%x, ipkts_pending: 0x%lx!",
		    fptr->fcip_flags, num_pkts, fptr->fcip_num_ipkts_pending));
	}

	mutex_exit(&fptr->fcip_dest_mutex);
	return (rval);
}

/*
 * Helper routine to remove fcip's minor node
 * There is one minor node per system and it should be removed if there are no
 * other fcip instances (which has a 1:1 mapping for fp instances) present
 */
static void
fcip_check_remove_minor_node(void)
{
	ASSERT(MUTEX_HELD(&fcip_global_mutex));

	/*
	 * If there are no more fcip (fp) instances, remove the
	 * minor node for fcip.
	 * Reset fcip_minor_node_created to invalidate it.
	 */
	if (fcip_num_instances == 0 && (fcip_module_dip != NULL)) {
		ddi_remove_minor_node(fcip_module_dip, NULL);
		fcip_minor_node_created = 0;
	}
}

/*
 * This routine permits the suspend operation during a CPR/System
 * power management operation. The routine basically quiesces I/Os
 * on all active interfaces
 */
static int
fcip_handle_suspend(fcip_port_info_t *fport, fc_detach_cmd_t cmd)
{
	struct fcip	*fptr = fport->fcipp_fcip;
	timeout_id_t	tid;
	int 		index;
	int		tryagain = 0;
	int		count;
	struct fcipstr	*tslp;


	ASSERT(fptr != NULL);
	mutex_enter(&fptr->fcip_mutex);

	/*
	 * Fail if we are in the middle of a callback. Don't use delay during
	 * suspend since clock intrs are not available so busy wait
	 */
	count = 0;
	while (count++ < 15 &&
	    ((fptr->fcip_flags & FCIP_IN_CALLBACK) ||
	    (fptr->fcip_flags & FCIP_IN_TIMEOUT))) {
		mutex_exit(&fptr->fcip_mutex);
		drv_usecwait(1000000);
		mutex_enter(&fptr->fcip_mutex);
	}

	if (fptr->fcip_flags & FCIP_IN_CALLBACK ||
	    fptr->fcip_flags & FCIP_IN_TIMEOUT) {
		mutex_exit(&fptr->fcip_mutex);
		return (FC_FAILURE);
	}

	if (cmd == FC_CMD_POWER_DOWN) {
		if (fptr->fcip_flags & FCIP_SUSPENDED) {
			fptr->fcip_flags |= FCIP_POWER_DOWN;
			mutex_exit(&fptr->fcip_mutex);
			goto success;
		} else {
			fptr->fcip_flags |= FCIP_POWER_DOWN;
		}
	} else if (cmd == FC_CMD_SUSPEND) {
		fptr->fcip_flags |= FCIP_SUSPENDED;
	} else {
		mutex_exit(&fptr->fcip_mutex);
		return (FC_FAILURE);
	}

	mutex_exit(&fptr->fcip_mutex);
	/*
	 * If no streams are plumbed - its the easiest case - Just
	 * bail out without having to do much
	 */

	rw_enter(&fcipstruplock, RW_READER);
	for (tslp = fcipstrup; tslp; tslp = tslp->sl_nextp) {
		if (tslp->sl_fcip == fptr) {
			break;
		}
	}
	rw_exit(&fcipstruplock);

	/*
	 * No active streams on this port
	 */
	if (tslp == NULL) {
		goto success;
	}

	/*
	 * Walk through each Routing table structure and check if
	 * the destination table has any outstanding commands. If yes
	 * wait for the commands to drain. Since we go through each
	 * routing table entry in succession, it may be wise to wait
	 * only a few seconds for each entry.
	 */
	mutex_enter(&fptr->fcip_rt_mutex);
	while (!tryagain) {

		tryagain = 0;
		for (index = 0; index < FCIP_RT_HASH_ELEMS; index++) {
			struct fcip_routing_table 	*frp;
			struct fcip_dest 		*fdestp;
			la_wwn_t			*pwwn;
			int				hash_bucket;

			frp = fptr->fcip_rtable[index];
			while (frp) {
				/*
				 * Mark the routing table as SUSPENDED. Even
				 * mark the broadcast entry SUSPENDED to
				 * prevent any ARP or other broadcasts. We
				 * can reset the state of the broadcast
				 * RTE when we resume.
				 */
				frp->fcipr_state = FCIP_RT_SUSPENDED;
				pwwn = &frp->fcipr_pwwn;

				/*
				 * Get hold of destination pointer
				 */
				mutex_enter(&fptr->fcip_dest_mutex);

				hash_bucket = FCIP_DEST_HASH(pwwn->raw_wwn);
				ASSERT(hash_bucket < FCIP_DEST_HASH_ELEMS);

				fdestp = fptr->fcip_dest[hash_bucket];
				while (fdestp != NULL) {
					mutex_enter(&fdestp->fcipd_mutex);
					if (fdestp->fcipd_rtable) {
						if (fcip_wwn_compare(pwwn,
						    &fdestp->fcipd_pwwn,
						    FCIP_COMPARE_PWWN) == 0) {
							mutex_exit(
							&fdestp->fcipd_mutex);
							break;
						}
					}
					mutex_exit(&fdestp->fcipd_mutex);
					fdestp = fdestp->fcipd_next;
				}

				mutex_exit(&fptr->fcip_dest_mutex);
				if (fdestp == NULL) {
					frp = frp->fcipr_next;
					continue;
				}

				/*
				 * Wait for fcip_wait_cmds seconds for
				 * the commands to drain.
				 */
				count = 0;
				mutex_enter(&fdestp->fcipd_mutex);
				while (fdestp->fcipd_ncmds &&
				    count < fcip_wait_cmds) {
					mutex_exit(&fdestp->fcipd_mutex);
					mutex_exit(&fptr->fcip_rt_mutex);
					drv_usecwait(1000000);
					mutex_enter(&fptr->fcip_rt_mutex);
					mutex_enter(&fdestp->fcipd_mutex);
					count++;
				}
				/*
				 * Check if we were able to drain all cmds
				 * successfully. Else continue with other
				 * ports and try during the second pass
				 */
				if (fdestp->fcipd_ncmds) {
					tryagain++;
				}
				mutex_exit(&fdestp->fcipd_mutex);

				frp = frp->fcipr_next;
			}
		}
		if (tryagain == 0) {
			break;
		}
	}
	mutex_exit(&fptr->fcip_rt_mutex);

	if (tryagain) {
		mutex_enter(&fptr->fcip_mutex);
		fptr->fcip_flags &= ~(FCIP_SUSPENDED | FCIP_POWER_DOWN);
		mutex_exit(&fptr->fcip_mutex);
		return (FC_FAILURE);
	}

success:
	mutex_enter(&fptr->fcip_mutex);
	tid = fptr->fcip_timeout_id;
	fptr->fcip_timeout_id = NULL;
	mutex_exit(&fptr->fcip_mutex);

	(void) untimeout(tid);

	return (FC_SUCCESS);
}

/*
 * the getinfo(9E) entry point
 */
/* ARGSUSED */
static int
fcip_getinfo(dev_info_t *dip, ddi_info_cmd_t cmd, void *arg, void **result)
{
	int rval = DDI_FAILURE;

	switch (cmd) {
	case DDI_INFO_DEVT2DEVINFO:
		*result = fcip_module_dip;
		if (*result)
			rval = DDI_SUCCESS;
		break;

	case DDI_INFO_DEVT2INSTANCE:
		*result = (void *)0;
		rval = DDI_SUCCESS;
		break;
	default:
		break;
	}

	return (rval);
}

/*
 * called from fcip_attach to initialize kstats for the link
 */
/* ARGSUSED */
static void
fcip_kstat_init(struct fcip *fptr)
{
	int instance;
	char buf[16];
	struct fcipstat	*fcipstatp;

	ASSERT(mutex_owned(&fptr->fcip_mutex));

	instance = ddi_get_instance(fptr->fcip_dip);
	(void) sprintf(buf, "fcip%d", instance);

#ifdef	kstat
	fptr->fcip_kstatp = kstat_create("fcip", instance, buf, "net",
	    KSTAT_TYPE_NAMED,
	    (sizeof (struct fcipstat)/ sizeof (kstat_named_t)),
	    KSTAT_FLAG_PERSISTENT);
#else
	fptr->fcip_kstatp = kstat_create("fcip", instance, buf, "net",
	    KSTAT_TYPE_NAMED,
	    (sizeof (struct fcipstat)/ sizeof (kstat_named_t)), 0);
#endif
	if (fptr->fcip_kstatp == NULL) {
		FCIP_DEBUG(FCIP_DEBUG_INIT, (CE_WARN, "kstat created failed"));
		return;
	}

	fcipstatp = (struct  fcipstat *)fptr->fcip_kstatp->ks_data;
	kstat_named_init(&fcipstatp->fcips_ipackets,	"ipackets",
		KSTAT_DATA_ULONG);
	kstat_named_init(&fcipstatp->fcips_ierrors,	"ierrors",
		KSTAT_DATA_ULONG);
	kstat_named_init(&fcipstatp->fcips_opackets,	"opackets",
		KSTAT_DATA_ULONG);
	kstat_named_init(&fcipstatp->fcips_oerrors,	"oerrors",
		KSTAT_DATA_ULONG);
	kstat_named_init(&fcipstatp->fcips_collisions,	"collisions",
		KSTAT_DATA_ULONG);
	kstat_named_init(&fcipstatp->fcips_nocanput,	"nocanput",
		KSTAT_DATA_ULONG);
	kstat_named_init(&fcipstatp->fcips_allocbfail,	"allocbfail",
		KSTAT_DATA_ULONG);

	kstat_named_init(&fcipstatp->fcips_defer, "defer",
		KSTAT_DATA_ULONG);
	kstat_named_init(&fcipstatp->fcips_fram, "fram",
		KSTAT_DATA_ULONG);
	kstat_named_init(&fcipstatp->fcips_crc, "crc",
		KSTAT_DATA_ULONG);
	kstat_named_init(&fcipstatp->fcips_oflo, "oflo",
		KSTAT_DATA_ULONG);
	kstat_named_init(&fcipstatp->fcips_uflo, "uflo",
		KSTAT_DATA_ULONG);
	kstat_named_init(&fcipstatp->fcips_missed, "missed",
		KSTAT_DATA_ULONG);
	kstat_named_init(&fcipstatp->fcips_tlcol, "tlcol",
		KSTAT_DATA_ULONG);
	kstat_named_init(&fcipstatp->fcips_trtry, "trtry",
		KSTAT_DATA_ULONG);
	kstat_named_init(&fcipstatp->fcips_tnocar, "tnocar",
		KSTAT_DATA_ULONG);
	kstat_named_init(&fcipstatp->fcips_inits, "inits",
		KSTAT_DATA_ULONG);
	kstat_named_init(&fcipstatp->fcips_notbufs, "notbufs",
		KSTAT_DATA_ULONG);
	kstat_named_init(&fcipstatp->fcips_norbufs, "norbufs",
		KSTAT_DATA_ULONG);
	kstat_named_init(&fcipstatp->fcips_allocbfail, "allocbfail",
		KSTAT_DATA_ULONG);

	/*
	 * required by kstat for MIB II objects(RFC 1213)
	 */
	kstat_named_init(&fcipstatp->fcips_rcvbytes, "fcips_rcvbytes",
		KSTAT_DATA_ULONG);	/* # octets received */
					/* MIB - ifInOctets */
	kstat_named_init(&fcipstatp->fcips_xmtbytes, "fcips_xmtbytes",
		KSTAT_DATA_ULONG);	/* # octets xmitted */
					/* MIB - ifOutOctets */
	kstat_named_init(&fcipstatp->fcips_multircv,	"fcips_multircv",
		KSTAT_DATA_ULONG);	/* # multicast packets */
					/* delivered to upper layer */
					/* MIB - ifInNUcastPkts */
	kstat_named_init(&fcipstatp->fcips_multixmt,	"fcips_multixmt",
		KSTAT_DATA_ULONG);	/* # multicast packets */
					/* requested to be sent */
					/* MIB - ifOutNUcastPkts */
	kstat_named_init(&fcipstatp->fcips_brdcstrcv, "fcips_brdcstrcv",
		KSTAT_DATA_ULONG); /* # broadcast packets */
					/* delivered to upper layer */
					/* MIB - ifInNUcastPkts */
	kstat_named_init(&fcipstatp->fcips_brdcstxmt, "fcips_brdcstxmt",
		KSTAT_DATA_ULONG);	/* # broadcast packets */
					/* requested to be sent */
					/* MIB - ifOutNUcastPkts */
	kstat_named_init(&fcipstatp->fcips_norcvbuf,	"fcips_norcvbuf",
		KSTAT_DATA_ULONG);	/* # rcv packets discarded */
					/* MIB - ifInDiscards */
	kstat_named_init(&fcipstatp->fcips_noxmtbuf,	"fcips_noxmtbuf",
		KSTAT_DATA_ULONG);	/* # xmt packets discarded */

	fptr->fcip_kstatp->ks_update = fcip_stat_update;
	fptr->fcip_kstatp->ks_private = (void *) fptr;
	kstat_install(fptr->fcip_kstatp);
}

/*
 * Update the defined kstats for netstat et al to use
 */
/* ARGSUSED */
static int
fcip_stat_update(kstat_t *fcip_statp, int val)
{
	struct fcipstat	*fcipstatp;
	struct fcip	*fptr;

	fptr = (struct fcip *)fcip_statp->ks_private;
	fcipstatp = (struct fcipstat *)fcip_statp->ks_data;

	if (val == KSTAT_WRITE) {
		fptr->fcip_ipackets	= fcipstatp->fcips_ipackets.value.ul;
		fptr->fcip_ierrors	= fcipstatp->fcips_ierrors.value.ul;
		fptr->fcip_opackets	= fcipstatp->fcips_opackets.value.ul;
		fptr->fcip_oerrors	= fcipstatp->fcips_oerrors.value.ul;
		fptr->fcip_collisions	= fcipstatp->fcips_collisions.value.ul;
		fptr->fcip_defer	= fcipstatp->fcips_defer.value.ul;
		fptr->fcip_fram	= fcipstatp->fcips_fram.value.ul;
		fptr->fcip_crc	= fcipstatp->fcips_crc.value.ul;
		fptr->fcip_oflo	= fcipstatp->fcips_oflo.value.ul;
		fptr->fcip_uflo	= fcipstatp->fcips_uflo.value.ul;
		fptr->fcip_missed	= fcipstatp->fcips_missed.value.ul;
		fptr->fcip_tlcol	= fcipstatp->fcips_tlcol.value.ul;
		fptr->fcip_trtry	= fcipstatp->fcips_trtry.value.ul;
		fptr->fcip_tnocar	= fcipstatp->fcips_tnocar.value.ul;
		fptr->fcip_inits	= fcipstatp->fcips_inits.value.ul;
		fptr->fcip_notbufs	= fcipstatp->fcips_notbufs.value.ul;
		fptr->fcip_norbufs	= fcipstatp->fcips_norbufs.value.ul;
		fptr->fcip_nocanput	= fcipstatp->fcips_nocanput.value.ul;
		fptr->fcip_allocbfail	= fcipstatp->fcips_allocbfail.value.ul;
		fptr->fcip_rcvbytes	= fcipstatp->fcips_rcvbytes.value.ul;
		fptr->fcip_xmtbytes	= fcipstatp->fcips_xmtbytes.value.ul;
		fptr->fcip_multircv	= fcipstatp->fcips_multircv.value.ul;
		fptr->fcip_multixmt	= fcipstatp->fcips_multixmt.value.ul;
		fptr->fcip_brdcstrcv	= fcipstatp->fcips_brdcstrcv.value.ul;
		fptr->fcip_norcvbuf	= fcipstatp->fcips_norcvbuf.value.ul;
		fptr->fcip_noxmtbuf	= fcipstatp->fcips_noxmtbuf.value.ul;
		fptr->fcip_allocbfail	= fcipstatp->fcips_allocbfail.value.ul;
		fptr->fcip_allocbfail	= fcipstatp->fcips_allocbfail.value.ul;
		fptr->fcip_allocbfail	= fcipstatp->fcips_allocbfail.value.ul;
		fptr->fcip_allocbfail	= fcipstatp->fcips_allocbfail.value.ul;
		fptr->fcip_allocbfail	= fcipstatp->fcips_allocbfail.value.ul;
		fptr->fcip_allocbfail	= fcipstatp->fcips_allocbfail.value.ul;
		fptr->fcip_allocbfail	= fcipstatp->fcips_allocbfail.value.ul;
		fptr->fcip_allocbfail	= fcipstatp->fcips_allocbfail.value.ul;

	} else {
		fcipstatp->fcips_ipackets.value.ul	= fptr->fcip_ipackets;
		fcipstatp->fcips_ierrors.value.ul	= fptr->fcip_ierrors;
		fcipstatp->fcips_opackets.value.ul	= fptr->fcip_opackets;
		fcipstatp->fcips_oerrors.value.ul	= fptr->fcip_oerrors;
		fcipstatp->fcips_collisions.value.ul	= fptr->fcip_collisions;
		fcipstatp->fcips_nocanput.value.ul	= fptr->fcip_nocanput;
		fcipstatp->fcips_allocbfail.value.ul	= fptr->fcip_allocbfail;
		fcipstatp->fcips_defer.value.ul	= fptr->fcip_defer;
		fcipstatp->fcips_fram.value.ul	= fptr->fcip_fram;
		fcipstatp->fcips_crc.value.ul	= fptr->fcip_crc;
		fcipstatp->fcips_oflo.value.ul	= fptr->fcip_oflo;
		fcipstatp->fcips_uflo.value.ul	= fptr->fcip_uflo;
		fcipstatp->fcips_missed.value.ul	= fptr->fcip_missed;
		fcipstatp->fcips_tlcol.value.ul	= fptr->fcip_tlcol;
		fcipstatp->fcips_trtry.value.ul	= fptr->fcip_trtry;
		fcipstatp->fcips_tnocar.value.ul	= fptr->fcip_tnocar;
		fcipstatp->fcips_inits.value.ul	= fptr->fcip_inits;
		fcipstatp->fcips_norbufs.value.ul	= fptr->fcip_norbufs;
		fcipstatp->fcips_notbufs.value.ul	= fptr->fcip_notbufs;
		fcipstatp->fcips_rcvbytes.value.ul	= fptr->fcip_rcvbytes;
		fcipstatp->fcips_xmtbytes.value.ul	= fptr->fcip_xmtbytes;
		fcipstatp->fcips_multircv.value.ul	= fptr->fcip_multircv;
		fcipstatp->fcips_multixmt.value.ul	= fptr->fcip_multixmt;
		fcipstatp->fcips_brdcstrcv.value.ul	= fptr->fcip_brdcstrcv;
		fcipstatp->fcips_brdcstxmt.value.ul	= fptr->fcip_brdcstxmt;
		fcipstatp->fcips_norcvbuf.value.ul	= fptr->fcip_norcvbuf;
		fcipstatp->fcips_noxmtbuf.value.ul	= fptr->fcip_noxmtbuf;

	}
	return (0);
}


/*
 * fcip_statec_cb: handles all required state change callback notifications
 * it receives from the transport
 */
/* ARGSUSED */
static void
fcip_statec_cb(opaque_t ulp_handle, opaque_t phandle,
    uint32_t port_state, uint32_t port_top, fc_portmap_t changelist[],
    uint32_t listlen, uint32_t sid)
{
	fcip_port_info_t	*fport;
	struct fcip 		*fptr;
	struct fcipstr		*slp;
	queue_t			*wrq;
	int			instance;
	int 			index;
	struct fcip_routing_table 	*frtp;

	fport = fcip_get_port(phandle);

	if (fport == NULL) {
		return;
	}

	fptr = fport->fcipp_fcip;
	ASSERT(fptr != NULL);

	if (fptr == NULL) {
		return;
	}

	instance = ddi_get_instance(fport->fcipp_dip);

	FCIP_TNF_PROBE_4((fcip_statec_cb, "fcip io", /* CSTYLED */,
		tnf_string, msg, "state change callback",
		tnf_uint, instance, instance,
		tnf_uint, S_ID, sid,
		tnf_int, count, listlen));
	FCIP_DEBUG(FCIP_DEBUG_ELS,
	    (CE_NOTE, "fcip%d, state change callback: state:0x%x, "
	    "S_ID:0x%x, count:0x%x", instance, port_state, sid, listlen));

	mutex_enter(&fptr->fcip_mutex);

	if ((fptr->fcip_flags & (FCIP_DETACHING | FCIP_DETACHED)) ||
	    (fptr->fcip_flags & (FCIP_SUSPENDED | FCIP_POWER_DOWN))) {
		mutex_exit(&fptr->fcip_mutex);
		return;
	}

	/*
	 * set fcip flags to indicate we are in the middle of a
	 * state change callback so we can wait till the statechange
	 * is handled before succeeding/failing the SUSPEND/POWER DOWN.
	 */
	fptr->fcip_flags |= FCIP_IN_SC_CB;

	fport->fcipp_pstate = port_state;

	/*
	 * Check if topology changed. If Yes - Modify the broadcast
	 * RTE entries to understand the new broadcast D_IDs
	 */
	if (fport->fcipp_topology != port_top &&
	    (port_top != FC_TOP_UNKNOWN)) {
		/* REMOVE later */
		FCIP_DEBUG(FCIP_DEBUG_ELS, (CE_NOTE,
		    "topology changed: Old topology: 0x%x New topology 0x%x",
		    fport->fcipp_topology, port_top));
		/*
		 * If topology changed - attempt a rediscovery of
		 * devices. Helps specially in Fabric/Public loops
		 * and if on_demand_node_creation is disabled
		 */
		fport->fcipp_topology = port_top;
		fcip_handle_topology(fptr);
	}

	mutex_exit(&fptr->fcip_mutex);

	switch (FC_PORT_STATE_MASK(port_state)) {
	case FC_STATE_ONLINE:
	/* FALLTHROUGH */
	case FC_STATE_LIP:
	/* FALLTHROUGH */
	case FC_STATE_LIP_LBIT_SET:

		/*
		 * nothing to do here actually other than if we
		 * were actually logged onto a port in the devlist
		 * (which indicates active communication between
		 * the host port and the port in the changelist).
		 * If however we are in a private loop or point to
		 * point mode, we need to check for any IP capable
		 * ports and update our routing table.
		 */
		switch (port_top) {
		case FC_TOP_FABRIC:
			/*
			 * This indicates a fabric port with a NameServer.
			 * Check the devlist to see if we are in active
			 * communication with a port on the devlist.
			 */
			FCIP_DEBUG(FCIP_DEBUG_ELS, (CE_NOTE,
			    "Statec_cb: fabric topology"));
			fcip_rt_update(fptr, changelist, listlen);
			break;
		case FC_TOP_NO_NS:
			/*
			 * No nameserver - so treat it like a Private loop
			 * or point to point topology and get a map of
			 * devices on the link and get IP capable ports to
			 * to update the routing table.
			 */
			FCIP_DEBUG(FCIP_DEBUG_ELS,
			    (CE_NOTE, "Statec_cb: NO_NS topology"));
		/* FALLTHROUGH */
		case FC_TOP_PRIVATE_LOOP:
			FCIP_DEBUG(FCIP_DEBUG_ELS, (CE_NOTE,
			    "Statec_cb: Pvt_Loop topology"));
		/* FALLTHROUGH */
		case FC_TOP_PT_PT:
			/*
			 * call get_port_map() and update routing table
			 */
			fcip_rt_update(fptr, changelist, listlen);
			break;
		default:
			FCIP_DEBUG(FCIP_DEBUG_ELS,
			    (CE_NOTE, "Statec_cb: Unknown topology"));
		}

		/*
		 * We should now enable the Queues and permit I/Os
		 * to flow through downstream. The update of routing
		 * table should have flushed out any port entries that
		 * don't exist or are not available after the state change
		 */
		mutex_enter(&fptr->fcip_mutex);
		fptr->fcip_port_state = FCIP_PORT_ONLINE;
		if (fptr->fcip_flags & FCIP_LINK_DOWN) {
			fptr->fcip_flags &= ~FCIP_LINK_DOWN;
		}
		mutex_exit(&fptr->fcip_mutex);

		/*
		 * Enable write queues
		 */
		rw_enter(&fcipstruplock, RW_READER);
		for (slp = fcipstrup; slp != NULL; slp = slp->sl_nextp) {
			if (slp && slp->sl_fcip == fptr) {
				wrq = WR(slp->sl_rq);
				if (wrq->q_flag & QFULL) {
					qenable(wrq);
				}
			}
		}
		rw_exit(&fcipstruplock);
		break;
	case FC_STATE_OFFLINE:
		/*
		 * mark the port_state OFFLINE and wait for it to
		 * become online. Any new messages in this state will
		 * simply be queued back up. If the port does not
		 * come online in a short while, we can begin failing
		 * messages and flush the routing table
		 */
		mutex_enter(&fptr->fcip_mutex);
		fptr->fcip_mark_offline = fptr->fcip_timeout_ticks +
		    FCIP_OFFLINE_TIMEOUT;
		fptr->fcip_port_state = FCIP_PORT_OFFLINE;
		mutex_exit(&fptr->fcip_mutex);

		/*
		 * Mark all Routing table entries as invalid to prevent
		 * any commands from trickling through to ports that
		 * have disappeared from under us
		 */
		mutex_enter(&fptr->fcip_rt_mutex);
		for (index = 0; index < FCIP_RT_HASH_ELEMS; index++) {
			frtp = fptr->fcip_rtable[index];
			while (frtp) {
				frtp->fcipr_state = PORT_DEVICE_INVALID;
				frtp = frtp->fcipr_next;
			}
		}
		mutex_exit(&fptr->fcip_rt_mutex);

		break;

	case FC_STATE_RESET_REQUESTED:
		/*
		 * Release all Unsolicited buffers back to transport/FCA.
		 * This also means the port state is marked offline - so
		 * we may have to do what OFFLINE state requires us to do.
		 * Care must be taken to wait for any active unsolicited
		 * buffer with the other Streams modules - so wait for
		 * a freeb if the unsolicited buffer is passed back all
		 * the way upstream.
		 */
		mutex_enter(&fptr->fcip_mutex);

#ifdef FCIP_ESBALLOC
		while (fptr->fcip_ub_upstream) {
			cv_wait(&fptr->fcip_ub_cv, &fptr->fcip_mutex);
		}
#endif	/* FCIP_ESBALLOC */

		fptr->fcip_mark_offline = fptr->fcip_timeout_ticks +
		    FCIP_OFFLINE_TIMEOUT;
		fptr->fcip_port_state = FCIP_PORT_OFFLINE;
		mutex_exit(&fptr->fcip_mutex);
		break;

	case FC_STATE_DEVICE_CHANGE:
		if (listlen) {
			fcip_rt_update(fptr, changelist, listlen);
		}
		break;
	case FC_STATE_RESET:
		/*
		 * Not much to do I guess - wait for port to become
		 * ONLINE. If the port doesn't become online in a short
		 * while, the upper layers abort any request themselves.
		 * We can just putback the messages in the streams queues
		 * if the link is offline
		 */
		break;
	}
	mutex_enter(&fptr->fcip_mutex);
	fptr->fcip_flags &= ~(FCIP_IN_SC_CB);
	mutex_exit(&fptr->fcip_mutex);
}

/*
 * Given a port handle, return the fcip_port_info structure corresponding
 * to that port handle. The transport allocates and communicates with
 * ULPs using port handles
 */
static fcip_port_info_t *
fcip_get_port(opaque_t phandle)
{
	fcip_port_info_t *fport;

	ASSERT(phandle != NULL);

	mutex_enter(&fcip_global_mutex);
	fport = fcip_port_head;

	while (fport != NULL) {
		if (fport->fcipp_handle == phandle) {
			/* found */
			break;
		}
		fport = fport->fcipp_next;
	}

	mutex_exit(&fcip_global_mutex);

	return (fport);
}

/*
 * Handle inbound ELS requests received by the transport. We are only
 * intereseted in FARP/InARP mostly.
 */
/* ARGSUSED */
static int
fcip_els_cb(opaque_t ulp_handle, opaque_t phandle,
    fc_unsol_buf_t *buf, uint32_t claimed)
{
	fcip_port_info_t	*fport;
	struct fcip 		*fptr;
	int			instance;
	uchar_t			r_ctl;
	uchar_t			ls_code;
	la_els_farp_t		farp_cmd;
	la_els_farp_t		*fcmd;
	int			rval = FC_UNCLAIMED;

	fport = fcip_get_port(phandle);
	if (fport == NULL) {
		return (FC_UNCLAIMED);
	}

	fptr = fport->fcipp_fcip;
	ASSERT(fptr != NULL);
	if (fptr == NULL) {
		return (FC_UNCLAIMED);
	}

	instance = ddi_get_instance(fport->fcipp_dip);

	mutex_enter(&fptr->fcip_mutex);
	if ((fptr->fcip_flags & (FCIP_DETACHING | FCIP_DETACHED)) ||
	    (fptr->fcip_flags & (FCIP_SUSPENDED | FCIP_POWER_DOWN))) {
		mutex_exit(&fptr->fcip_mutex);
		return (FC_UNCLAIMED);
	}

	/*
	 * set fcip flags to indicate we are in the middle of a
	 * ELS callback so we can wait till the statechange
	 * is handled before succeeding/failing the SUSPEND/POWER DOWN.
	 */
	fptr->fcip_flags |= FCIP_IN_ELS_CB;
	mutex_exit(&fptr->fcip_mutex);

	FCIP_TNF_PROBE_2((fcip_els_cb, "fcip io", /* CSTYLED */,
		tnf_string, msg, "ELS callback",
		tnf_uint, instance, instance));

	FCIP_DEBUG(FCIP_DEBUG_ELS,
	    (CE_NOTE, "fcip%d, ELS callback , ", instance));

	r_ctl = buf->ub_frame.r_ctl;
	switch (r_ctl & R_CTL_ROUTING) {
	case R_CTL_EXTENDED_SVC:
		if (r_ctl == R_CTL_ELS_REQ) {
			ls_code = buf->ub_buffer[0];
			if (ls_code == LA_ELS_FARP_REQ) {
				/*
				 * Inbound FARP broadcast request
				 */
				if (buf->ub_bufsize != sizeof (la_els_farp_t)) {
					FCIP_DEBUG(FCIP_DEBUG_ELS, (CE_WARN,
					    "Invalid FARP req buffer size "
					    "expected 0x%lx, got 0x%x",
					    (long)(sizeof (la_els_farp_t)),
					    buf->ub_bufsize));
					rval = FC_UNCLAIMED;
					goto els_cb_done;
				}
				fcmd = (la_els_farp_t *)buf;
				if (fcip_wwn_compare(&fcmd->resp_nwwn,
				    &fport->fcipp_nwwn,
				    FCIP_COMPARE_NWWN) != 0) {
					rval = FC_UNCLAIMED;
					goto els_cb_done;
				}
				/*
				 * copy the FARP request and release the
				 * unsolicited buffer
				 */
				fcmd = &farp_cmd;
				bcopy((void *)buf, (void *)fcmd,
				    sizeof (la_els_farp_t));
				(void) fc_ulp_ubrelease(fport->fcipp_handle, 1,
				    &buf->ub_token);

				if (fcip_farp_supported &&
				    fcip_handle_farp_request(fptr, fcmd) ==
				    FC_SUCCESS) {
					/*
					 * We successfully sent out a FARP
					 * reply to the requesting port
					 */
					rval = FC_SUCCESS;
					goto els_cb_done;
				} else {
					rval = FC_UNCLAIMED;
					goto els_cb_done;
				}
			}
		} else if (r_ctl == R_CTL_ELS_RSP) {
			ls_code = buf->ub_buffer[0];
			if (ls_code == LA_ELS_FARP_REPLY) {
				/*
				 * We received a REPLY to our FARP request
				 */
				if (buf->ub_bufsize != sizeof (la_els_farp_t)) {
					FCIP_DEBUG(FCIP_DEBUG_ELS, (CE_WARN,
					    "Invalid FARP req buffer size "
					    "expected 0x%lx, got 0x%x",
					    (long)(sizeof (la_els_farp_t)),
					    buf->ub_bufsize));
					rval = FC_UNCLAIMED;
					goto els_cb_done;
				}
				fcmd = &farp_cmd;
				bcopy((void *)buf, (void *)fcmd,
				    sizeof (la_els_farp_t));
				(void) fc_ulp_ubrelease(fport->fcipp_handle, 1,
				    &buf->ub_token);
				if (fcip_farp_supported &&
				    fcip_handle_farp_response(fptr, fcmd) ==
				    FC_SUCCESS) {
					FCIP_DEBUG(FCIP_DEBUG_ELS, (CE_NOTE,
					    "Successfully recevied a FARP "
					    "response"));
					mutex_enter(&fptr->fcip_mutex);
					fptr->fcip_farp_rsp_flag = 1;
					cv_signal(&fptr->fcip_farp_cv);
					mutex_exit(&fptr->fcip_mutex);
					rval = FC_SUCCESS;
					goto els_cb_done;
				} else {
					FCIP_DEBUG(FCIP_DEBUG_ELS, (CE_WARN,
					    "Unable to handle a FARP response "
					    "receive"));
					rval = FC_UNCLAIMED;
					goto els_cb_done;
				}
			}
		}
		break;
	default:
		break;
	}
els_cb_done:
	mutex_enter(&fptr->fcip_mutex);
	fptr->fcip_flags &= ~(FCIP_IN_ELS_CB);
	mutex_exit(&fptr->fcip_mutex);
	return (rval);
}


/*
 * Handle inbound FARP requests
 */
static int
fcip_handle_farp_request(struct fcip *fptr, la_els_farp_t *fcmd)
{
	fcip_pkt_t		*fcip_pkt;
	fc_packet_t		*fc_pkt;
	fcip_port_info_t	*fport = fptr->fcip_port_info;
	int			rval = FC_FAILURE;
	opaque_t		fca_dev;
	fc_portmap_t 		map;
	struct fcip_routing_table *frp;
	struct fcip_dest *fdestp;

	/*
	 * Add an entry for the remote port into our routing and destination
	 * tables.
	 */
	map.map_did = fcmd->req_id;
	map.map_hard_addr.hard_addr = fcmd->req_id.port_id;
	map.map_state = PORT_DEVICE_VALID;
	map.map_type = PORT_DEVICE_NEW;
	map.map_flags = 0;
	map.map_pd = NULL;
	bcopy((void *)&fcmd->req_pwwn, (void *)&map.map_pwwn,
	    sizeof (la_wwn_t));
	bcopy((void *)&fcmd->req_nwwn, (void *)&map.map_nwwn,
	    sizeof (la_wwn_t));
	fcip_rt_update(fptr, &map, 1);
	mutex_enter(&fptr->fcip_rt_mutex);
	frp = fcip_lookup_rtable(fptr, &fcmd->req_pwwn, FCIP_COMPARE_NWWN);
	mutex_exit(&fptr->fcip_rt_mutex);

	fdestp = fcip_add_dest(fptr, frp);

	fcip_pkt = fcip_ipkt_alloc(fptr, sizeof (la_els_farp_t),
	    sizeof (la_els_farp_t), NULL, KM_SLEEP);
	if (fcip_pkt == NULL) {
		rval = FC_FAILURE;
		goto farp_done;
	}
	/*
	 * Fill in our port's PWWN and NWWN
	 */
	fcmd->resp_pwwn = fport->fcipp_pwwn;
	fcmd->resp_nwwn = fport->fcipp_nwwn;

	fcip_init_unicast_pkt(fcip_pkt, fport->fcipp_sid,
	    fcmd->req_id, NULL);

	fca_dev =
	    fc_ulp_get_fca_device(fport->fcipp_handle, fcmd->req_id);
	fc_pkt = FCIP_PKT_TO_FC_PKT(fcip_pkt);
	fc_pkt->pkt_cmd_fhdr.r_ctl = R_CTL_ELS_RSP;
	fc_pkt->pkt_fca_device = fca_dev;
	fcip_pkt->fcip_pkt_dest = fdestp;

	/*
	 * Attempt a PLOGI again
	 */
	if (fcmd->resp_flags & FARP_INIT_P_LOGI) {
		if (fcip_do_plogi(fptr, frp) != FC_SUCCESS) {
			/*
			 * Login to the remote port failed. There is no
			 * point continuing with the FARP request further
			 * so bail out here.
			 */
			frp->fcipr_state = PORT_DEVICE_INVALID;
			rval = FC_FAILURE;
			goto farp_done;
		}
	}

	FCIP_CP_OUT(fcmd, fc_pkt->pkt_cmd, fc_pkt->pkt_cmd_acc,
	    sizeof (la_els_farp_t));

	rval = fc_ulp_issue_els(fport->fcipp_handle, fc_pkt);
	if (rval != FC_SUCCESS) {
		FCIP_TNF_PROBE_2((fcip_handle_farp_request, "fcip io",
		    /* CSTYLED */, tnf_string, msg,
		    "fcip_transport of farp reply failed",
		    tnf_uint, rval, rval));
		FCIP_DEBUG(FCIP_DEBUG_ELS, (CE_WARN,
		    "fcip_transport of farp reply failed 0x%x", rval));
	}

farp_done:
	return (rval);
}


/*
 * Handle FARP responses to our FARP requests. When we receive a FARP
 * reply, we need to add the entry for the Port that replied into our
 * routing and destination hash tables. It is possible that the remote
 * port did not login into us (FARP responses can be received without
 * a PLOGI)
 */
static int
fcip_handle_farp_response(struct fcip *fptr, la_els_farp_t *fcmd)
{
	int			rval = FC_FAILURE;
	fc_portmap_t 		map;
	struct fcip_routing_table *frp;
	struct fcip_dest *fdestp;

	/*
	 * Add an entry for the remote port into our routing and destination
	 * tables.
	 */
	map.map_did = fcmd->dest_id;
	map.map_hard_addr.hard_addr = fcmd->dest_id.port_id;
	map.map_state = PORT_DEVICE_VALID;
	map.map_type = PORT_DEVICE_NEW;
	map.map_flags = 0;
	map.map_pd = NULL;
	bcopy((void *)&fcmd->resp_pwwn, (void *)&map.map_pwwn,
	    sizeof (la_wwn_t));
	bcopy((void *)&fcmd->resp_nwwn, (void *)&map.map_nwwn,
	    sizeof (la_wwn_t));
	fcip_rt_update(fptr, &map, 1);
	mutex_enter(&fptr->fcip_rt_mutex);
	frp = fcip_lookup_rtable(fptr, &fcmd->resp_pwwn, FCIP_COMPARE_NWWN);
	mutex_exit(&fptr->fcip_rt_mutex);

	fdestp = fcip_add_dest(fptr, frp);

	if (fdestp != NULL) {
		rval = FC_SUCCESS;
	}
	return (rval);
}


#define	FCIP_HDRS_LENGTH	\
	sizeof (fcph_network_hdr_t)+sizeof (llc_snap_hdr_t)+sizeof (ipha_t)

/*
 * fcip_data_cb is the heart of most IP operations. This routine is called
 * by the transport when any unsolicited IP data arrives at a port (which
 * is almost all IP data). This routine then strips off the Network header
 * from the payload (after authenticating the received payload ofcourse),
 * creates a message blk and sends the data upstream. You will see ugly
 * #defines because of problems with using esballoc() as opposed to
 * allocb to prevent an extra copy of data. We should probably move to
 * esballoc entirely when the MTU eventually will be larger than 1500 bytes
 * since copies will get more expensive then. At 1500 byte MTUs, there is
 * no noticable difference between using allocb and esballoc. The other
 * caveat is that the qlc firmware still cannot tell us accurately the
 * no. of valid bytes in the unsol buffer it DMA'ed so we have to resort
 * to looking into the IP header and hoping that the no. of bytes speficified
 * in the header was actually received.
 */
/* ARGSUSED */
static int
fcip_data_cb(opaque_t ulp_handle, opaque_t phandle,
    fc_unsol_buf_t *buf, uint32_t claimed)
{
	fcip_port_info_t		*fport;
	struct fcip 			*fptr;
	fcph_network_hdr_t		*nhdr;
	llc_snap_hdr_t			*snaphdr;
	mblk_t				*bp;
	uint32_t 			len;
	uint32_t			hdrlen;
	ushort_t			type;
	ipha_t				*iphdr;
	int				rval;

#ifdef FCIP_ESBALLOC
	frtn_t				*free_ubuf;
	struct fcip_esballoc_arg	*fesb_argp;
#endif /* FCIP_ESBALLOC */

	fport = fcip_get_port(phandle);
	if (fport == NULL) {
		return (FC_UNCLAIMED);
	}

	fptr = fport->fcipp_fcip;
	ASSERT(fptr != NULL);

	if (fptr == NULL) {
		return (FC_UNCLAIMED);
	}

	mutex_enter(&fptr->fcip_mutex);
	if ((fptr->fcip_flags & (FCIP_DETACHING | FCIP_DETACHED)) ||
	    (fptr->fcip_flags & (FCIP_SUSPENDED | FCIP_POWER_DOWN))) {
		mutex_exit(&fptr->fcip_mutex);
		rval = FC_UNCLAIMED;
		goto data_cb_done;
	}

	/*
	 * set fcip flags to indicate we are in the middle of a
	 * data callback so we can wait till the statechange
	 * is handled before succeeding/failing the SUSPEND/POWER DOWN.
	 */
	fptr->fcip_flags |= FCIP_IN_DATA_CB;
	mutex_exit(&fptr->fcip_mutex);

	FCIP_TNF_PROBE_2((fcip_data_cb, "fcip io", /* CSTYLED */,
		tnf_string, msg, "data callback",
		tnf_int, instance, ddi_get_instance(fport->fcipp_dip)));
	FCIP_DEBUG(FCIP_DEBUG_UPSTREAM,
	    (CE_NOTE, "fcip%d, data callback",
	    ddi_get_instance(fport->fcipp_dip)));

	/*
	 * get to the network and snap headers in the payload
	 */
	nhdr = (fcph_network_hdr_t *)buf->ub_buffer;
	snaphdr = (llc_snap_hdr_t *)(buf->ub_buffer +
	    sizeof (fcph_network_hdr_t));

	hdrlen = sizeof (fcph_network_hdr_t) + sizeof (llc_snap_hdr_t);

	/*
	 * get the IP header to obtain the no. of bytes we need to read
	 * off from the unsol buffer. This obviously is because not all
	 * data fills up the unsol buffer completely and the firmware
	 * doesn't tell us how many valid bytes are in there as well
	 */
	iphdr = (ipha_t *)(buf->ub_buffer + hdrlen);
	snaphdr->pid = BE_16(snaphdr->pid);
	type = snaphdr->pid;

	FCIP_DEBUG(FCIP_DEBUG_UPSTREAM,
	    (CE_CONT, "SNAPHDR: dsap %x, ssap %x, ctrl %x\n",
	    snaphdr->dsap, snaphdr->ssap, snaphdr->ctrl));

	FCIP_DEBUG(FCIP_DEBUG_UPSTREAM,
	    (CE_CONT, "oui[0] 0x%x oui[1] 0x%x oui[2] 0x%x pid 0x%x\n",
	    snaphdr->oui[0], snaphdr->oui[1], snaphdr->oui[2], snaphdr->pid));

	/* Authneticate, Authenticate */
	if (type == ETHERTYPE_IP) {
		len = hdrlen + BE_16(iphdr->ipha_length);
	} else if (type == ETHERTYPE_ARP) {
		len = hdrlen + 28;
	} else {
		len = buf->ub_bufsize;
	}

	FCIP_DEBUG(FCIP_DEBUG_UPSTREAM,
	    (CE_CONT, "effective packet length is %d bytes.\n", len));

	if (len < hdrlen || len > FCIP_UB_SIZE) {
		FCIP_DEBUG(FCIP_DEBUG_UPSTREAM,
		    (CE_NOTE, "Incorrect buffer size %d bytes", len));
		rval = FC_UNCLAIMED;
		goto data_cb_done;
	}

	if (buf->ub_frame.type != FC_TYPE_IS8802_SNAP) {
		FCIP_DEBUG(FCIP_DEBUG_UPSTREAM, (CE_NOTE, "Not IP/ARP data"));
		rval = FC_UNCLAIMED;