blob: ab9c479af326dfaf4307013441299e87e624d4d4 [file] [log] [blame]
Yu Xiangning0f1702c2008-12-11 20:04:13 -08001/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22/*
Anders Persson3e95bd42010-06-17 17:22:09 -070023 * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
Yu Xiangning0f1702c2008-12-11 20:04:13 -080024 */
Dan McDonaldd690b622014-07-23 19:50:51 -040025/*
26 * Copyright 2014, OmniTI Computer Consulting, Inc. All rights reserved.
Robert Mustacchi907c2822021-02-01 13:50:02 +000027 * Copyright 2019 Joyent, Inc.
Dan McDonaldd690b622014-07-23 19:50:51 -040028 */
Yu Xiangning0f1702c2008-12-11 20:04:13 -080029
30#include <sys/types.h>
31#include <sys/param.h>
32#include <sys/signal.h>
33#include <sys/cmn_err.h>
34
35#include <sys/stropts.h>
36#include <sys/socket.h>
37#include <sys/socketvar.h>
38#include <sys/sockio.h>
Yu Xiangning0f1702c2008-12-11 20:04:13 -080039#include <sys/strsubr.h>
40#include <sys/strsun.h>
41#include <sys/atomic.h>
Anders Persson41174432009-02-12 17:35:05 -080042#include <sys/tihdr.h>
Yu Xiangning0f1702c2008-12-11 20:04:13 -080043
44#include <fs/sockfs/sockcommon.h>
Anders Persson3e95bd42010-06-17 17:22:09 -070045#include <fs/sockfs/sockfilter_impl.h>
Yu Xiangning0f1702c2008-12-11 20:04:13 -080046#include <fs/sockfs/socktpi.h>
Anders Perssonbbc000e2009-04-28 12:10:59 -070047#include <fs/sockfs/sodirect.h>
Yu Xiangning0f1702c2008-12-11 20:04:13 -080048#include <sys/ddi.h>
49#include <inet/ip.h>
50#include <sys/time.h>
51#include <sys/cmn_err.h>
52
53#ifdef SOCK_TEST
54extern int do_useracc;
55extern clock_t sock_test_timelimit;
56#endif /* SOCK_TEST */
57
58#define MBLK_PULL_LEN 64
59uint32_t so_mblk_pull_len = MBLK_PULL_LEN;
60
61#ifdef DEBUG
62boolean_t so_debug_length = B_FALSE;
63static boolean_t so_check_length(sonode_t *so);
64#endif
65
Yu Xiangning0f1702c2008-12-11 20:04:13 -080066static int
67so_acceptq_dequeue_locked(struct sonode *so, boolean_t dontblock,
68 struct sonode **nsop)
69{
70 struct sonode *nso = NULL;
71
72 *nsop = NULL;
73 ASSERT(MUTEX_HELD(&so->so_acceptq_lock));
Anders Persson3e95bd42010-06-17 17:22:09 -070074 while ((nso = list_remove_head(&so->so_acceptq_list)) == NULL) {
Yu Xiangning0f1702c2008-12-11 20:04:13 -080075 /*
76 * No need to check so_error here, because it is not
77 * possible for a listening socket to be reset or otherwise
78 * disconnected.
79 *
80 * So now we just need check if it's ok to wait.
81 */
82 if (dontblock)
83 return (EWOULDBLOCK);
84 if (so->so_state & (SS_CLOSING | SS_FALLBACK_PENDING))
85 return (EINTR);
86
87 if (cv_wait_sig_swap(&so->so_acceptq_cv,
88 &so->so_acceptq_lock) == 0)
89 return (EINTR);
90 }
91
92 ASSERT(nso != NULL);
Yu Xiangning0f1702c2008-12-11 20:04:13 -080093 ASSERT(so->so_acceptq_len > 0);
Anders Persson3e95bd42010-06-17 17:22:09 -070094 so->so_acceptq_len--;
95 nso->so_listener = NULL;
Yu Xiangning0f1702c2008-12-11 20:04:13 -080096
97 *nsop = nso;
98
99 return (0);
100}
101
102/*
103 * int so_acceptq_dequeue(struct sonode *, boolean_t, struct sonode **)
104 *
105 * Pulls a connection off of the accept queue.
106 *
107 * Arguments:
108 * so - listening socket
109 * dontblock - indicate whether it's ok to sleep if there are no
110 * connections on the queue
111 * nsop - Value-return argument
112 *
113 * Return values:
114 * 0 when a connection is successfully dequeued, in which case nsop
115 * is set to point to the new connection. Upon failure a non-zero
116 * value is returned, and the value of nsop is set to NULL.
117 *
118 * Note:
119 * so_acceptq_dequeue() may return prematurly if the socket is falling
120 * back to TPI.
121 */
122int
123so_acceptq_dequeue(struct sonode *so, boolean_t dontblock,
124 struct sonode **nsop)
125{
126 int error;
127
128 mutex_enter(&so->so_acceptq_lock);
129 error = so_acceptq_dequeue_locked(so, dontblock, nsop);
130 mutex_exit(&so->so_acceptq_lock);
131
132 return (error);
133}
134
Anders Persson3e95bd42010-06-17 17:22:09 -0700135static void
136so_acceptq_flush_impl(struct sonode *so, list_t *list, boolean_t doclose)
137{
138 struct sonode *nso;
139
140 while ((nso = list_remove_head(list)) != NULL) {
141 nso->so_listener = NULL;
142 if (doclose) {
143 (void) socket_close(nso, 0, CRED());
144 } else {
145 /*
146 * Only used for fallback - not possible when filters
147 * are present.
148 */
149 ASSERT(so->so_filter_active == 0);
150 /*
151 * Since the socket is on the accept queue, there can
152 * only be one reference. We drop the reference and
153 * just blow off the socket.
154 */
155 ASSERT(nso->so_count == 1);
156 nso->so_count--;
157 /* drop the proto ref */
158 VN_RELE(SOTOV(nso));
159 }
160 socket_destroy(nso);
161 }
162}
Yu Xiangning0f1702c2008-12-11 20:04:13 -0800163/*
Anders Persson3e95bd42010-06-17 17:22:09 -0700164 * void so_acceptq_flush(struct sonode *so)
Yu Xiangning0f1702c2008-12-11 20:04:13 -0800165 *
166 * Removes all pending connections from a listening socket, and
167 * frees the associated resources.
168 *
169 * Arguments
Anders Persson2320a8c2009-10-21 19:52:57 -0700170 * so - listening socket
171 * doclose - make a close downcall for each socket on the accept queue
Yu Xiangning0f1702c2008-12-11 20:04:13 -0800172 *
173 * Return values:
174 * None.
175 *
176 * Note:
177 * The caller has to ensure that no calls to so_acceptq_enqueue() or
178 * so_acceptq_dequeue() occur while the accept queue is being flushed.
179 * So either the socket needs to be in a state where no operations
180 * would come in, or so_lock needs to be obtained.
181 */
182void
Anders Persson2320a8c2009-10-21 19:52:57 -0700183so_acceptq_flush(struct sonode *so, boolean_t doclose)
Yu Xiangning0f1702c2008-12-11 20:04:13 -0800184{
Anders Persson3e95bd42010-06-17 17:22:09 -0700185 so_acceptq_flush_impl(so, &so->so_acceptq_list, doclose);
186 so_acceptq_flush_impl(so, &so->so_acceptq_defer, doclose);
Yu Xiangning0f1702c2008-12-11 20:04:13 -0800187
Yu Xiangning0f1702c2008-12-11 20:04:13 -0800188 so->so_acceptq_len = 0;
189}
190
191int
192so_wait_connected_locked(struct sonode *so, boolean_t nonblock,
193 sock_connid_t id)
194{
195 ASSERT(MUTEX_HELD(&so->so_lock));
196
197 /*
198 * The protocol has notified us that a connection attempt is being
199 * made, so before we wait for a notification to arrive we must
200 * clear out any errors associated with earlier connection attempts.
201 */
202 if (so->so_error != 0 && SOCK_CONNID_LT(so->so_proto_connid, id))
203 so->so_error = 0;
204
205 while (SOCK_CONNID_LT(so->so_proto_connid, id)) {
206 if (nonblock)
207 return (EINPROGRESS);
208
209 if (so->so_state & (SS_CLOSING | SS_FALLBACK_PENDING))
210 return (EINTR);
211
212 if (cv_wait_sig_swap(&so->so_state_cv, &so->so_lock) == 0)
213 return (EINTR);
214 }
215
216 if (so->so_error != 0)
217 return (sogeterr(so, B_TRUE));
218 /*
219 * Under normal circumstances, so_error should contain an error
220 * in case the connect failed. However, it is possible for another
221 * thread to come in a consume the error, so generate a sensible
222 * error in that case.
223 */
224 if ((so->so_state & SS_ISCONNECTED) == 0)
225 return (ECONNREFUSED);
226
227 return (0);
228}
229
230/*
231 * int so_wait_connected(struct sonode *so, boolean_t nonblock,
232 * sock_connid_t id)
233 *
234 * Wait until the socket is connected or an error has occured.
235 *
236 * Arguments:
237 * so - socket
238 * nonblock - indicate whether it's ok to sleep if the connection has
239 * not yet been established
240 * gen - generation number that was returned by the protocol
241 * when the operation was started
242 *
243 * Returns:
244 * 0 if the connection attempt was successful, or an error indicating why
245 * the connection attempt failed.
246 */
247int
248so_wait_connected(struct sonode *so, boolean_t nonblock, sock_connid_t id)
249{
250 int error;
251
252 mutex_enter(&so->so_lock);
253 error = so_wait_connected_locked(so, nonblock, id);
254 mutex_exit(&so->so_lock);
255
256 return (error);
257}
258
259int
260so_snd_wait_qnotfull_locked(struct sonode *so, boolean_t dontblock)
261{
262 int error;
263
264 ASSERT(MUTEX_HELD(&so->so_lock));
Anders Persson3e95bd42010-06-17 17:22:09 -0700265 while (SO_SND_FLOWCTRLD(so)) {
Yu Xiangning0f1702c2008-12-11 20:04:13 -0800266 if (so->so_state & SS_CANTSENDMORE)
267 return (EPIPE);
268 if (dontblock)
269 return (EWOULDBLOCK);
270
271 if (so->so_state & (SS_CLOSING | SS_FALLBACK_PENDING))
272 return (EINTR);
273
274 if (so->so_sndtimeo == 0) {
275 /*
276 * Zero means disable timeout.
277 */
278 error = cv_wait_sig(&so->so_snd_cv, &so->so_lock);
279 } else {
Rafael Vanonid3d50732009-11-13 01:32:32 -0800280 error = cv_reltimedwait_sig(&so->so_snd_cv,
281 &so->so_lock, so->so_sndtimeo, TR_CLOCK_TICK);
Yu Xiangning0f1702c2008-12-11 20:04:13 -0800282 }
283 if (error == 0)
284 return (EINTR);
285 else if (error == -1)
shenjian34dfe682009-01-21 10:04:42 +0800286 return (EAGAIN);
Yu Xiangning0f1702c2008-12-11 20:04:13 -0800287 }
288 return (0);
289}
290
291/*
292 * int so_wait_sendbuf(struct sonode *so, boolean_t dontblock)
293 *
294 * Wait for the transport to notify us about send buffers becoming
295 * available.
296 */
297int
298so_snd_wait_qnotfull(struct sonode *so, boolean_t dontblock)
299{
300 int error = 0;
301
302 mutex_enter(&so->so_lock);
Anders Persson3e95bd42010-06-17 17:22:09 -0700303 so->so_snd_wakeup = B_TRUE;
304 error = so_snd_wait_qnotfull_locked(so, dontblock);
305 so->so_snd_wakeup = B_FALSE;
Yu Xiangning0f1702c2008-12-11 20:04:13 -0800306 mutex_exit(&so->so_lock);
307
308 return (error);
309}
310
311void
312so_snd_qfull(struct sonode *so)
313{
314 mutex_enter(&so->so_lock);
315 so->so_snd_qfull = B_TRUE;
316 mutex_exit(&so->so_lock);
317}
318
319void
320so_snd_qnotfull(struct sonode *so)
321{
322 mutex_enter(&so->so_lock);
323 so->so_snd_qfull = B_FALSE;
324 /* wake up everyone waiting for buffers */
325 cv_broadcast(&so->so_snd_cv);
326 mutex_exit(&so->so_lock);
327}
328
329/*
330 * Change the process/process group to which SIGIO is sent.
331 */
332int
333socket_chgpgrp(struct sonode *so, pid_t pid)
334{
335 int error;
336
337 ASSERT(MUTEX_HELD(&so->so_lock));
338 if (pid != 0) {
339 /*
340 * Permissions check by sending signal 0.
341 * Note that when kill fails it does a
342 * set_errno causing the system call to fail.
343 */
344 error = kill(pid, 0);
345 if (error != 0) {
346 return (error);
347 }
348 }
349 so->so_pgrp = pid;
350 return (0);
351}
352
353
354/*
355 * Generate a SIGIO, for 'writable' events include siginfo structure,
356 * for read events just send the signal.
357 */
358/*ARGSUSED*/
359static void
360socket_sigproc(proc_t *proc, int event)
361{
362 k_siginfo_t info;
363
364 ASSERT(event & (SOCKETSIG_WRITE | SOCKETSIG_READ | SOCKETSIG_URG));
365
366 if (event & SOCKETSIG_WRITE) {
367 info.si_signo = SIGPOLL;
368 info.si_code = POLL_OUT;
369 info.si_errno = 0;
370 info.si_fd = 0;
371 info.si_band = 0;
372 sigaddq(proc, NULL, &info, KM_NOSLEEP);
373 }
374 if (event & SOCKETSIG_READ) {
375 sigtoproc(proc, NULL, SIGPOLL);
376 }
377 if (event & SOCKETSIG_URG) {
378 sigtoproc(proc, NULL, SIGURG);
379 }
380}
381
382void
383socket_sendsig(struct sonode *so, int event)
384{
385 proc_t *proc;
386
387 ASSERT(MUTEX_HELD(&so->so_lock));
388
389 if (so->so_pgrp == 0 || (!(so->so_state & SS_ASYNC) &&
390 event != SOCKETSIG_URG)) {
391 return;
392 }
393
394 dprint(3, ("sending sig %d to %d\n", event, so->so_pgrp));
395
396 if (so->so_pgrp > 0) {
397 /*
398 * XXX This unfortunately still generates
399 * a signal when a fd is closed but
400 * the proc is active.
401 */
402 mutex_enter(&pidlock);
Dan McDonaldd690b622014-07-23 19:50:51 -0400403 /*
404 * Even if the thread started in another zone, we're receiving
405 * on behalf of this socket's zone, so find the proc using the
406 * socket's zone ID.
407 */
408 proc = prfind_zone(so->so_pgrp, so->so_zoneid);
Yu Xiangning0f1702c2008-12-11 20:04:13 -0800409 if (proc == NULL) {
410 mutex_exit(&pidlock);
411 return;
412 }
413 mutex_enter(&proc->p_lock);
414 mutex_exit(&pidlock);
415 socket_sigproc(proc, event);
416 mutex_exit(&proc->p_lock);
417 } else {
418 /*
419 * Send to process group. Hold pidlock across
420 * calls to socket_sigproc().
421 */
422 pid_t pgrp = -so->so_pgrp;
423
424 mutex_enter(&pidlock);
Dan McDonaldd690b622014-07-23 19:50:51 -0400425 /*
426 * Even if the thread started in another zone, we're receiving
427 * on behalf of this socket's zone, so find the pgrp using the
428 * socket's zone ID.
429 */
430 proc = pgfind_zone(pgrp, so->so_zoneid);
Yu Xiangning0f1702c2008-12-11 20:04:13 -0800431 while (proc != NULL) {
432 mutex_enter(&proc->p_lock);
433 socket_sigproc(proc, event);
434 mutex_exit(&proc->p_lock);
435 proc = proc->p_pglink;
436 }
437 mutex_exit(&pidlock);
438 }
439}
440
441#define MIN(a, b) ((a) < (b) ? (a) : (b))
442/* Copy userdata into a new mblk_t */
443mblk_t *
444socopyinuio(uio_t *uiop, ssize_t iosize, size_t wroff, ssize_t maxblk,
Erik Nordmarkbd670b32009-11-11 11:49:49 -0800445 size_t tail_len, int *errorp)
Yu Xiangning0f1702c2008-12-11 20:04:13 -0800446{
447 mblk_t *head = NULL, **tail = &head;
448
449 ASSERT(iosize == INFPSZ || iosize > 0);
450
451 if (iosize == INFPSZ || iosize > uiop->uio_resid)
452 iosize = uiop->uio_resid;
453
454 if (maxblk == INFPSZ)
455 maxblk = iosize;
456
457 /* Nothing to do in these cases, so we're done */
458 if (iosize < 0 || maxblk < 0 || (maxblk == 0 && iosize > 0))
459 goto done;
460
461 /*
462 * We will enter the loop below if iosize is 0; it will allocate an
463 * empty message block and call uiomove(9F) which will just return.
464 * We could avoid that with an extra check but would only slow
465 * down the much more likely case where iosize is larger than 0.
466 */
467 do {
468 ssize_t blocksize;
469 mblk_t *mp;
470
471 blocksize = MIN(iosize, maxblk);
472 ASSERT(blocksize >= 0);
Erik Nordmarkbd670b32009-11-11 11:49:49 -0800473 mp = allocb(wroff + blocksize + tail_len, BPRI_MED);
Erik Nordmarkde8c4a12009-02-12 08:42:06 -0800474 if (mp == NULL) {
Yu Xiangning0f1702c2008-12-11 20:04:13 -0800475 *errorp = ENOMEM;
476 return (head);
477 }
478 mp->b_rptr += wroff;
479 mp->b_wptr = mp->b_rptr + blocksize;
480
481 *tail = mp;
482 tail = &mp->b_cont;
483
484 /* uiomove(9F) either returns 0 or EFAULT */
485 if ((*errorp = uiomove(mp->b_rptr, (size_t)blocksize,
486 UIO_WRITE, uiop)) != 0) {
487 ASSERT(*errorp != ENOMEM);
488 freemsg(head);
489 return (NULL);
490 }
491
492 iosize -= blocksize;
493 } while (iosize > 0);
494
495done:
496 *errorp = 0;
497 return (head);
498}
499
500mblk_t *
501socopyoutuio(mblk_t *mp, struct uio *uiop, ssize_t max_read, int *errorp)
502{
503 int error;
504 ptrdiff_t n;
505 mblk_t *nmp;
506
507 ASSERT(mp->b_wptr >= mp->b_rptr);
508
509 /*
510 * max_read is the offset of the oobmark and read can not go pass
511 * the oobmark.
512 */
513 if (max_read == INFPSZ || max_read > uiop->uio_resid)
514 max_read = uiop->uio_resid;
515
516 do {
517 if ((n = MIN(max_read, MBLKL(mp))) != 0) {
518 ASSERT(n > 0);
519
520 error = uiomove(mp->b_rptr, n, UIO_READ, uiop);
521 if (error != 0) {
522 freemsg(mp);
523 *errorp = error;
524 return (NULL);
525 }
526 }
527
528 mp->b_rptr += n;
529 max_read -= n;
530 while (mp != NULL && (mp->b_rptr >= mp->b_wptr)) {
531 /*
532 * get rid of zero length mblks
533 */
534 nmp = mp;
535 mp = mp->b_cont;
536 freeb(nmp);
537 }
538 } while (mp != NULL && max_read > 0);
539
540 *errorp = 0;
541 return (mp);
542}
543
544static void
545so_prepend_msg(struct sonode *so, mblk_t *mp, mblk_t *last_tail)
546{
547 ASSERT(last_tail != NULL);
548 mp->b_next = so->so_rcv_q_head;
549 mp->b_prev = last_tail;
550 ASSERT(!(DB_FLAGS(mp) & DBLK_UIOA));
551
552 if (so->so_rcv_q_head == NULL) {
553 ASSERT(so->so_rcv_q_last_head == NULL);
554 so->so_rcv_q_last_head = mp;
555#ifdef DEBUG
556 } else {
557 ASSERT(!(DB_FLAGS(so->so_rcv_q_head) & DBLK_UIOA));
558#endif
559 }
560 so->so_rcv_q_head = mp;
561
562#ifdef DEBUG
563 if (so_debug_length) {
564 mutex_enter(&so->so_lock);
565 ASSERT(so_check_length(so));
566 mutex_exit(&so->so_lock);
567 }
568#endif
569}
570
Anders Perssone4b767e2009-03-26 17:08:33 -0700571/*
572 * Move a mblk chain (mp_head, mp_last_head) to the sonode's rcv queue so it
573 * can be processed by so_dequeue_msg().
574 */
575void
576so_process_new_message(struct sonode *so, mblk_t *mp_head, mblk_t *mp_last_head)
Yu Xiangning0f1702c2008-12-11 20:04:13 -0800577{
Anders Persson3e95bd42010-06-17 17:22:09 -0700578 if (so->so_filter_active > 0 &&
579 (mp_head = sof_filter_data_in_proc(so, mp_head,
580 &mp_last_head)) == NULL)
581 return;
582
Yu Xiangning0f1702c2008-12-11 20:04:13 -0800583 ASSERT(mp_head->b_prev != NULL);
Anders Persson3e95bd42010-06-17 17:22:09 -0700584 if (so->so_rcv_q_head == NULL) {
Yu Xiangning0f1702c2008-12-11 20:04:13 -0800585 so->so_rcv_q_head = mp_head;
586 so->so_rcv_q_last_head = mp_last_head;
587 ASSERT(so->so_rcv_q_last_head->b_prev != NULL);
588 } else {
589 boolean_t flag_equal = ((DB_FLAGS(mp_head) & DBLK_UIOA) ==
590 (DB_FLAGS(so->so_rcv_q_last_head) & DBLK_UIOA));
591
592 if (mp_head->b_next == NULL &&
593 DB_TYPE(mp_head) == M_DATA &&
594 DB_TYPE(so->so_rcv_q_last_head) == M_DATA && flag_equal) {
595 so->so_rcv_q_last_head->b_prev->b_cont = mp_head;
596 so->so_rcv_q_last_head->b_prev = mp_head->b_prev;
597 mp_head->b_prev = NULL;
598 } else if (flag_equal && (DB_FLAGS(mp_head) & DBLK_UIOA)) {
599 /*
600 * Append to last_head if more than one mblks, and both
601 * mp_head and last_head are I/OAT mblks.
602 */
603 ASSERT(mp_head->b_next != NULL);
604 so->so_rcv_q_last_head->b_prev->b_cont = mp_head;
605 so->so_rcv_q_last_head->b_prev = mp_head->b_prev;
606 mp_head->b_prev = NULL;
607
608 so->so_rcv_q_last_head->b_next = mp_head->b_next;
609 mp_head->b_next = NULL;
610 so->so_rcv_q_last_head = mp_last_head;
611 } else {
612#ifdef DEBUG
613 {
614 mblk_t *tmp_mblk;
615 tmp_mblk = mp_head;
616 while (tmp_mblk != NULL) {
617 ASSERT(tmp_mblk->b_prev != NULL);
618 tmp_mblk = tmp_mblk->b_next;
619 }
620 }
621#endif
622 so->so_rcv_q_last_head->b_next = mp_head;
623 so->so_rcv_q_last_head = mp_last_head;
624 }
625 }
626}
627
Rao Shoaib5795faa2009-07-28 13:53:49 -0700628/*
629 * Check flow control on a given sonode. Must have so_lock held, and
Kacheong Poona215d4e2010-08-09 20:07:20 -0700630 * this function will release the hold. Return true if flow control
631 * is cleared.
Rao Shoaib5795faa2009-07-28 13:53:49 -0700632 */
Kacheong Poona215d4e2010-08-09 20:07:20 -0700633boolean_t
Rao Shoaib5795faa2009-07-28 13:53:49 -0700634so_check_flow_control(struct sonode *so)
635{
636 ASSERT(MUTEX_HELD(&so->so_lock));
637
Anders Persson3e95bd42010-06-17 17:22:09 -0700638 if (so->so_flowctrld && (so->so_rcv_queued < so->so_rcvlowat &&
639 !(so->so_state & SS_FIL_RCV_FLOWCTRL))) {
Rao Shoaib5795faa2009-07-28 13:53:49 -0700640 so->so_flowctrld = B_FALSE;
641 mutex_exit(&so->so_lock);
642 /*
643 * Open up flow control. SCTP does not have any downcalls, and
644 * it will clr flow ctrl in sosctp_recvmsg().
645 */
646 if (so->so_downcalls != NULL &&
647 so->so_downcalls->sd_clr_flowctrl != NULL) {
648 (*so->so_downcalls->sd_clr_flowctrl)
649 (so->so_proto_handle);
650 }
Anders Persson3e95bd42010-06-17 17:22:09 -0700651 /* filters can start injecting data */
652 sof_sonode_notify_filters(so, SOF_EV_INJECT_DATA_IN_OK, 0);
Kacheong Poona215d4e2010-08-09 20:07:20 -0700653 return (B_TRUE);
Rao Shoaib5795faa2009-07-28 13:53:49 -0700654 } else {
655 mutex_exit(&so->so_lock);
Kacheong Poona215d4e2010-08-09 20:07:20 -0700656 return (B_FALSE);
Rao Shoaib5795faa2009-07-28 13:53:49 -0700657 }
658}
659
Yu Xiangning0f1702c2008-12-11 20:04:13 -0800660int
661so_dequeue_msg(struct sonode *so, mblk_t **mctlp, struct uio *uiop,
662 rval_t *rvalp, int flags)
663{
664 mblk_t *mp, *nmp;
665 mblk_t *savemp, *savemptail;
666 mblk_t *new_msg_head;
667 mblk_t *new_msg_last_head;
668 mblk_t *last_tail;
669 boolean_t partial_read;
670 boolean_t reset_atmark = B_FALSE;
671 int more = 0;
672 int error;
673 ssize_t oobmark;
674 sodirect_t *sodp = so->so_direct;
675
676 partial_read = B_FALSE;
677 *mctlp = NULL;
678again:
679 mutex_enter(&so->so_lock);
680again1:
681#ifdef DEBUG
682 if (so_debug_length) {
683 ASSERT(so_check_length(so));
684 }
685#endif
Anders Persson8591a192009-05-29 09:33:18 -0700686 if (so->so_state & SS_RCVATMARK) {
687 /* Check whether the caller is OK to read past the mark */
688 if (flags & MSG_NOMARK) {
689 mutex_exit(&so->so_lock);
690 return (EWOULDBLOCK);
691 }
692 reset_atmark = B_TRUE;
693 }
Yu Xiangning0f1702c2008-12-11 20:04:13 -0800694 /*
695 * First move messages from the dump area to processing area
696 */
697 if (sodp != NULL) {
Anders Perssonbbc000e2009-04-28 12:10:59 -0700698 if (sodp->sod_enabled) {
Yu Xiangning0f1702c2008-12-11 20:04:13 -0800699 if (sodp->sod_uioa.uioa_state & UIOA_ALLOC) {
700 /* nothing to uioamove */
701 sodp = NULL;
702 } else if (sodp->sod_uioa.uioa_state & UIOA_INIT) {
703 sodp->sod_uioa.uioa_state &= UIOA_CLR;
704 sodp->sod_uioa.uioa_state |= UIOA_ENABLED;
705 /*
706 * try to uioamove() the data that
707 * has already queued.
708 */
709 sod_uioa_so_init(so, sodp, uiop);
710 }
711 } else {
712 sodp = NULL;
713 }
714 }
715 new_msg_head = so->so_rcv_head;
716 new_msg_last_head = so->so_rcv_last_head;
717 so->so_rcv_head = NULL;
718 so->so_rcv_last_head = NULL;
719 oobmark = so->so_oobmark;
720 /*
721 * We can release the lock as there can only be one reader
722 */
723 mutex_exit(&so->so_lock);
724
Yu Xiangning0f1702c2008-12-11 20:04:13 -0800725 if (new_msg_head != NULL) {
Anders Perssone4b767e2009-03-26 17:08:33 -0700726 so_process_new_message(so, new_msg_head, new_msg_last_head);
Yu Xiangning0f1702c2008-12-11 20:04:13 -0800727 }
728 savemp = savemptail = NULL;
Kacheong Poona215d4e2010-08-09 20:07:20 -0700729 rvalp->r_vals = 0;
Yu Xiangning0f1702c2008-12-11 20:04:13 -0800730 error = 0;
731 mp = so->so_rcv_q_head;
732
733 if (mp != NULL &&
734 (so->so_rcv_timer_tid == 0 ||
735 so->so_rcv_queued >= so->so_rcv_thresh)) {
736 partial_read = B_FALSE;
737
738 if (flags & MSG_PEEK) {
739 if ((nmp = dupmsg(mp)) == NULL &&
740 (nmp = copymsg(mp)) == NULL) {
741 size_t size = msgsize(mp);
742
743 error = strwaitbuf(size, BPRI_HI);
744 if (error) {
745 return (error);
746 }
747 goto again;
748 }
749 mp = nmp;
750 } else {
751 ASSERT(mp->b_prev != NULL);
752 last_tail = mp->b_prev;
753 mp->b_prev = NULL;
754 so->so_rcv_q_head = mp->b_next;
755 if (so->so_rcv_q_head == NULL) {
756 so->so_rcv_q_last_head = NULL;
757 }
758 mp->b_next = NULL;
759 }
760
761 ASSERT(mctlp != NULL);
762 /*
763 * First process PROTO or PCPROTO blocks, if any.
764 */
765 if (DB_TYPE(mp) != M_DATA) {
766 *mctlp = mp;
767 savemp = mp;
768 savemptail = mp;
769 ASSERT(DB_TYPE(mp) == M_PROTO ||
770 DB_TYPE(mp) == M_PCPROTO);
771 while (mp->b_cont != NULL &&
772 DB_TYPE(mp->b_cont) != M_DATA) {
773 ASSERT(DB_TYPE(mp->b_cont) == M_PROTO ||
774 DB_TYPE(mp->b_cont) == M_PCPROTO);
775 mp = mp->b_cont;
776 savemptail = mp;
777 }
778 mp = savemptail->b_cont;
779 savemptail->b_cont = NULL;
780 }
781
782 ASSERT(DB_TYPE(mp) == M_DATA);
783 /*
784 * Now process DATA blocks, if any. Note that for sodirect
785 * enabled socket, uio_resid can be 0.
786 */
787 if (uiop->uio_resid >= 0) {
788 ssize_t copied = 0;
789
790 if (sodp != NULL && (DB_FLAGS(mp) & DBLK_UIOA)) {
Anders Perssonbbc000e2009-04-28 12:10:59 -0700791 mutex_enter(&so->so_lock);
Yu Xiangning0f1702c2008-12-11 20:04:13 -0800792 ASSERT(uiop == (uio_t *)&sodp->sod_uioa);
793 copied = sod_uioa_mblk(so, mp);
794 if (copied > 0)
795 partial_read = B_TRUE;
Anders Perssonbbc000e2009-04-28 12:10:59 -0700796 mutex_exit(&so->so_lock);
Yu Xiangning0f1702c2008-12-11 20:04:13 -0800797 /* mark this mblk as processed */
798 mp = NULL;
799 } else {
800 ssize_t oldresid = uiop->uio_resid;
801
802 if (MBLKL(mp) < so_mblk_pull_len) {
803 if (pullupmsg(mp, -1) == 1) {
804 last_tail = mp;
805 }
806 }
807 /*
808 * Can not read beyond the oobmark
809 */
810 mp = socopyoutuio(mp, uiop,
811 oobmark == 0 ? INFPSZ : oobmark, &error);
812 if (error != 0) {
813 freemsg(*mctlp);
814 *mctlp = NULL;
815 more = 0;
816 goto done;
817 }
818 ASSERT(oldresid >= uiop->uio_resid);
819 copied = oldresid - uiop->uio_resid;
820 if (oldresid > uiop->uio_resid)
821 partial_read = B_TRUE;
822 }
823 ASSERT(copied >= 0);
824 if (copied > 0 && !(flags & MSG_PEEK)) {
825 mutex_enter(&so->so_lock);
826 so->so_rcv_queued -= copied;
827 ASSERT(so->so_oobmark >= 0);
828 if (so->so_oobmark > 0) {
829 so->so_oobmark -= copied;
830 ASSERT(so->so_oobmark >= 0);
831 if (so->so_oobmark == 0) {
832 ASSERT(so->so_state &
833 SS_OOBPEND);
834 so->so_oobmark = 0;
835 so->so_state |= SS_RCVATMARK;
836 }
837 }
Rao Shoaib5795faa2009-07-28 13:53:49 -0700838 /*
839 * so_check_flow_control() will drop
840 * so->so_lock.
841 */
Kacheong Poona215d4e2010-08-09 20:07:20 -0700842 rvalp->r_val2 = so_check_flow_control(so);
Yu Xiangning0f1702c2008-12-11 20:04:13 -0800843 }
844 }
845 if (mp != NULL) { /* more data blocks in msg */
846 more |= MOREDATA;
847 if ((flags & (MSG_PEEK|MSG_TRUNC))) {
Rao Shoaib5795faa2009-07-28 13:53:49 -0700848 if (flags & MSG_PEEK) {
849 freemsg(mp);
850 } else {
851 unsigned int msize = msgdsize(mp);
852
853 freemsg(mp);
Yu Xiangning0f1702c2008-12-11 20:04:13 -0800854 mutex_enter(&so->so_lock);
Rao Shoaib5795faa2009-07-28 13:53:49 -0700855 so->so_rcv_queued -= msize;
856 /*
857 * so_check_flow_control() will drop
858 * so->so_lock.
859 */
Kacheong Poona215d4e2010-08-09 20:07:20 -0700860 rvalp->r_val2 =
861 so_check_flow_control(so);
Yu Xiangning0f1702c2008-12-11 20:04:13 -0800862 }
Yu Xiangning0f1702c2008-12-11 20:04:13 -0800863 } else if (partial_read && !somsghasdata(mp)) {
864 /*
865 * Avoid queuing a zero-length tail part of
866 * a message. partial_read == 1 indicates that
867 * we read some of the message.
868 */
869 freemsg(mp);
870 more &= ~MOREDATA;
871 } else {
872 if (savemp != NULL &&
873 (flags & MSG_DUPCTRL)) {
874 mblk_t *nmp;
875 /*
876 * There should only be non data mblks
877 */
878 ASSERT(DB_TYPE(savemp) != M_DATA &&
879 DB_TYPE(savemptail) != M_DATA);
880try_again:
881 if ((nmp = dupmsg(savemp)) == NULL &&
882 (nmp = copymsg(savemp)) == NULL) {
883
884 size_t size = msgsize(savemp);
885
886 error = strwaitbuf(size,
887 BPRI_HI);
888 if (error != 0) {
889 /*
890 * In case we
891 * cannot copy
892 * control data
893 * free the remaining
894 * data.
895 */
896 freemsg(mp);
897 goto done;
898 }
899 goto try_again;
900 }
901
902 ASSERT(nmp != NULL);
903 ASSERT(DB_TYPE(nmp) != M_DATA);
904 savemptail->b_cont = mp;
905 *mctlp = nmp;
906 mp = savemp;
907 }
908 /*
909 * putback mp
910 */
911 so_prepend_msg(so, mp, last_tail);
912 }
913 }
914
915 /* fast check so_rcv_head if there is more data */
916 if (partial_read && !(so->so_state & SS_RCVATMARK) &&
917 *mctlp == NULL && uiop->uio_resid > 0 &&
918 !(flags & MSG_PEEK) && so->so_rcv_head != NULL) {
919 goto again;
920 }
921 } else if (!partial_read) {
922 mutex_enter(&so->so_lock);
923 if (so->so_error != 0) {
924 error = sogeterr(so, !(flags & MSG_PEEK));
925 mutex_exit(&so->so_lock);
926 return (error);
927 }
928 /*
929 * No pending data. Return right away for nonblocking
930 * socket, otherwise sleep waiting for data.
931 */
Mike Cheng2caa6592008-12-29 14:01:03 +0800932 if (!(so->so_state & SS_CANTRCVMORE) && uiop->uio_resid > 0) {
Yu Xiangning0f1702c2008-12-11 20:04:13 -0800933 if ((uiop->uio_fmode & (FNDELAY|FNONBLOCK)) ||
934 (flags & MSG_DONTWAIT)) {
935 error = EWOULDBLOCK;
936 } else {
937 if (so->so_state & (SS_CLOSING |
938 SS_FALLBACK_PENDING)) {
939 mutex_exit(&so->so_lock);
940 error = EINTR;
941 goto done;
942 }
943
944 if (so->so_rcv_head != NULL) {
945 goto again1;
946 }
947 so->so_rcv_wakeup = B_TRUE;
948 so->so_rcv_wanted = uiop->uio_resid;
949 if (so->so_rcvtimeo == 0) {
950 /*
951 * Zero means disable timeout.
952 */
953 error = cv_wait_sig(&so->so_rcv_cv,
954 &so->so_lock);
955 } else {
Rafael Vanonid3d50732009-11-13 01:32:32 -0800956 error = cv_reltimedwait_sig(
957 &so->so_rcv_cv, &so->so_lock,
958 so->so_rcvtimeo, TR_CLOCK_TICK);
Yu Xiangning0f1702c2008-12-11 20:04:13 -0800959 }
960 so->so_rcv_wakeup = B_FALSE;
961 so->so_rcv_wanted = 0;
962
963 if (error == 0) {
964 error = EINTR;
965 } else if (error == -1) {
shenjian34dfe682009-01-21 10:04:42 +0800966 error = EAGAIN;
Yu Xiangning0f1702c2008-12-11 20:04:13 -0800967 } else {
968 goto again1;
969 }
970 }
971 }
972 mutex_exit(&so->so_lock);
973 }
974 if (reset_atmark && partial_read && !(flags & MSG_PEEK)) {
975 /*
976 * We are passed the mark, update state
977 * 4.3BSD and 4.4BSD clears the mark when peeking across it.
978 * The draft Posix socket spec states that the mark should
979 * not be cleared when peeking. We follow the latter.
980 */
981 mutex_enter(&so->so_lock);
982 ASSERT(so_verify_oobstate(so));
983 so->so_state &= ~(SS_OOBPEND|SS_HAVEOOBDATA|SS_RCVATMARK);
984 freemsg(so->so_oobmsg);
985 so->so_oobmsg = NULL;
986 ASSERT(so_verify_oobstate(so));
987 mutex_exit(&so->so_lock);
988 }
989 ASSERT(so->so_rcv_wakeup == B_FALSE);
990done:
991 if (sodp != NULL) {
Anders Perssonbbc000e2009-04-28 12:10:59 -0700992 mutex_enter(&so->so_lock);
993 if (sodp->sod_enabled &&
Yu Xiangning0f1702c2008-12-11 20:04:13 -0800994 (sodp->sod_uioa.uioa_state & UIOA_ENABLED)) {
995 SOD_UIOAFINI(sodp);
996 if (sodp->sod_uioa.uioa_mbytes > 0) {
997 ASSERT(so->so_rcv_q_head != NULL ||
998 so->so_rcv_head != NULL);
999 so->so_rcv_queued -= sod_uioa_mblk(so, NULL);
1000 if (error == EWOULDBLOCK)
1001 error = 0;
1002 }
1003 }
Anders Perssonbbc000e2009-04-28 12:10:59 -07001004 mutex_exit(&so->so_lock);
Yu Xiangning0f1702c2008-12-11 20:04:13 -08001005 }
1006#ifdef DEBUG
1007 if (so_debug_length) {
1008 mutex_enter(&so->so_lock);
1009 ASSERT(so_check_length(so));
1010 mutex_exit(&so->so_lock);
1011 }
1012#endif
1013 rvalp->r_val1 = more;
Rao Shoaib5795faa2009-07-28 13:53:49 -07001014 ASSERT(MUTEX_NOT_HELD(&so->so_lock));
Yu Xiangning0f1702c2008-12-11 20:04:13 -08001015 return (error);
1016}
1017
Anders Perssone4b767e2009-03-26 17:08:33 -07001018/*
1019 * Enqueue data from the protocol on the socket's rcv queue.
1020 *
1021 * We try to hook new M_DATA mblks onto an existing chain, however,
1022 * that cannot be done if the existing chain has already been
1023 * processed by I/OAT. Non-M_DATA mblks are just linked together via
1024 * b_next. In all cases the b_prev of the enqueued mblk is set to
1025 * point to the last mblk in its b_cont chain.
1026 */
Yu Xiangning0f1702c2008-12-11 20:04:13 -08001027void
1028so_enqueue_msg(struct sonode *so, mblk_t *mp, size_t msg_size)
1029{
1030 ASSERT(MUTEX_HELD(&so->so_lock));
1031
1032#ifdef DEBUG
1033 if (so_debug_length) {
1034 ASSERT(so_check_length(so));
1035 }
1036#endif
1037 so->so_rcv_queued += msg_size;
1038
1039 if (so->so_rcv_head == NULL) {
1040 ASSERT(so->so_rcv_last_head == NULL);
1041 so->so_rcv_head = mp;
1042 so->so_rcv_last_head = mp;
1043 } else if ((DB_TYPE(mp) == M_DATA &&
1044 DB_TYPE(so->so_rcv_last_head) == M_DATA) &&
1045 ((DB_FLAGS(mp) & DBLK_UIOA) ==
1046 (DB_FLAGS(so->so_rcv_last_head) & DBLK_UIOA))) {
1047 /* Added to the end */
1048 ASSERT(so->so_rcv_last_head != NULL);
1049 ASSERT(so->so_rcv_last_head->b_prev != NULL);
1050 so->so_rcv_last_head->b_prev->b_cont = mp;
1051 } else {
1052 /* Start a new end */
1053 so->so_rcv_last_head->b_next = mp;
1054 so->so_rcv_last_head = mp;
1055 }
1056 while (mp->b_cont != NULL)
1057 mp = mp->b_cont;
1058
1059 so->so_rcv_last_head->b_prev = mp;
1060#ifdef DEBUG
1061 if (so_debug_length) {
1062 ASSERT(so_check_length(so));
1063 }
1064#endif
1065}
1066
1067/*
1068 * Return B_TRUE if there is data in the message, B_FALSE otherwise.
1069 */
1070boolean_t
1071somsghasdata(mblk_t *mp)
1072{
1073 for (; mp; mp = mp->b_cont)
1074 if (mp->b_datap->db_type == M_DATA) {
1075 ASSERT(mp->b_wptr >= mp->b_rptr);
1076 if (mp->b_wptr > mp->b_rptr)
1077 return (B_TRUE);
1078 }
1079 return (B_FALSE);
1080}
1081
1082/*
1083 * Flush the read side of sockfs.
1084 *
1085 * The caller must be sure that a reader is not already active when the
1086 * buffer is being flushed.
1087 */
1088void
1089so_rcv_flush(struct sonode *so)
1090{
1091 mblk_t *mp;
1092
1093 ASSERT(MUTEX_HELD(&so->so_lock));
1094
1095 if (so->so_oobmsg != NULL) {
1096 freemsg(so->so_oobmsg);
1097 so->so_oobmsg = NULL;
1098 so->so_oobmark = 0;
1099 so->so_state &=
1100 ~(SS_OOBPEND|SS_HAVEOOBDATA|SS_HADOOBDATA|SS_RCVATMARK);
1101 }
1102
1103 /*
Anders Persson3e95bd42010-06-17 17:22:09 -07001104 * Free messages sitting in the recv queues
Yu Xiangning0f1702c2008-12-11 20:04:13 -08001105 */
1106 while (so->so_rcv_q_head != NULL) {
1107 mp = so->so_rcv_q_head;
1108 so->so_rcv_q_head = mp->b_next;
1109 mp->b_next = mp->b_prev = NULL;
1110 freemsg(mp);
1111 }
1112 while (so->so_rcv_head != NULL) {
1113 mp = so->so_rcv_head;
1114 so->so_rcv_head = mp->b_next;
1115 mp->b_next = mp->b_prev = NULL;
1116 freemsg(mp);
1117 }
1118 so->so_rcv_queued = 0;
1119 so->so_rcv_q_head = NULL;
1120 so->so_rcv_q_last_head = NULL;
1121 so->so_rcv_head = NULL;
1122 so->so_rcv_last_head = NULL;
1123}
1124
1125/*
1126 * Handle recv* calls that set MSG_OOB or MSG_OOB together with MSG_PEEK.
1127 */
1128int
1129sorecvoob(struct sonode *so, struct nmsghdr *msg, struct uio *uiop, int flags,
1130 boolean_t oob_inline)
1131{
1132 mblk_t *mp, *nmp;
1133 int error;
1134
1135 dprintso(so, 1, ("sorecvoob(%p, %p, 0x%x)\n", (void *)so, (void *)msg,
1136 flags));
1137
1138 if (msg != NULL) {
1139 /*
1140 * There is never any oob data with addresses or control since
1141 * the T_EXDATA_IND does not carry any options.
1142 */
1143 msg->msg_controllen = 0;
1144 msg->msg_namelen = 0;
1145 msg->msg_flags = 0;
1146 }
1147
1148 mutex_enter(&so->so_lock);
1149 ASSERT(so_verify_oobstate(so));
1150 if (oob_inline ||
1151 (so->so_state & (SS_OOBPEND|SS_HADOOBDATA)) != SS_OOBPEND) {
1152 dprintso(so, 1, ("sorecvoob: inline or data consumed\n"));
1153 mutex_exit(&so->so_lock);
1154 return (EINVAL);
1155 }
1156 if (!(so->so_state & SS_HAVEOOBDATA)) {
1157 dprintso(so, 1, ("sorecvoob: no data yet\n"));
1158 mutex_exit(&so->so_lock);
1159 return (EWOULDBLOCK);
1160 }
1161 ASSERT(so->so_oobmsg != NULL);
1162 mp = so->so_oobmsg;
1163 if (flags & MSG_PEEK) {
1164 /*
1165 * Since recv* can not return ENOBUFS we can not use dupmsg.
1166 * Instead we revert to the consolidation private
1167 * allocb_wait plus bcopy.
1168 */
1169 mblk_t *mp1;
1170
1171 mp1 = allocb_wait(msgdsize(mp), BPRI_MED, STR_NOSIG, NULL);
1172 ASSERT(mp1);
1173
1174 while (mp != NULL) {
1175 ssize_t size;
1176
1177 size = MBLKL(mp);
1178 bcopy(mp->b_rptr, mp1->b_wptr, size);
1179 mp1->b_wptr += size;
1180 ASSERT(mp1->b_wptr <= mp1->b_datap->db_lim);
1181 mp = mp->b_cont;
1182 }
1183 mp = mp1;
1184 } else {
1185 /*
1186 * Update the state indicating that the data has been consumed.
1187 * Keep SS_OOBPEND set until data is consumed past the mark.
1188 */
1189 so->so_oobmsg = NULL;
1190 so->so_state ^= SS_HAVEOOBDATA|SS_HADOOBDATA;
1191 }
1192 ASSERT(so_verify_oobstate(so));
1193 mutex_exit(&so->so_lock);
1194
1195 error = 0;
1196 nmp = mp;
1197 while (nmp != NULL && uiop->uio_resid > 0) {
1198 ssize_t n = MBLKL(nmp);
1199
1200 n = MIN(n, uiop->uio_resid);
1201 if (n > 0)
1202 error = uiomove(nmp->b_rptr, n,
1203 UIO_READ, uiop);
1204 if (error)
1205 break;
1206 nmp = nmp->b_cont;
1207 }
1208 ASSERT(mp->b_next == NULL && mp->b_prev == NULL);
1209 freemsg(mp);
1210 return (error);
1211}
1212
1213/*
1214 * Allocate and initializ sonode
1215 */
1216/* ARGSUSED */
1217struct sonode *
1218socket_sonode_create(struct sockparams *sp, int family, int type,
1219 int protocol, int version, int sflags, int *errorp, struct cred *cr)
1220{
1221 sonode_t *so;
1222 int kmflags;
1223
1224 /*
1225 * Choose the right set of sonodeops based on the upcall and
1226 * down call version that the protocol has provided
1227 */
1228 if (SOCK_UC_VERSION != sp->sp_smod_info->smod_uc_version ||
1229 SOCK_DC_VERSION != sp->sp_smod_info->smod_dc_version) {
1230 /*
1231 * mismatch
1232 */
1233#ifdef DEBUG
1234 cmn_err(CE_CONT, "protocol and socket module version mismatch");
1235#endif
1236 *errorp = EINVAL;
1237 return (NULL);
1238 }
1239
1240 kmflags = (sflags & SOCKET_NOSLEEP) ? KM_NOSLEEP : KM_SLEEP;
1241
1242 so = kmem_cache_alloc(socket_cache, kmflags);
1243 if (so == NULL) {
1244 *errorp = ENOMEM;
1245 return (NULL);
1246 }
1247
1248 sonode_init(so, sp, family, type, protocol, &so_sonodeops);
1249
1250 if (version == SOV_DEFAULT)
1251 version = so_default_version;
1252
1253 so->so_version = (short)version;
1254
1255 /*
1256 * set the default values to be INFPSZ
1257 * if a protocol desires it can change the value later
1258 */
1259 so->so_proto_props.sopp_rxhiwat = SOCKET_RECVHIWATER;
1260 so->so_proto_props.sopp_rxlowat = SOCKET_RECVLOWATER;
1261 so->so_proto_props.sopp_maxpsz = INFPSZ;
1262 so->so_proto_props.sopp_maxblk = INFPSZ;
1263
1264 return (so);
1265}
1266
1267int
1268socket_init_common(struct sonode *so, struct sonode *pso, int flags, cred_t *cr)
1269{
1270 int error = 0;
1271
1272 if (pso != NULL) {
1273 /*
1274 * We have a passive open, so inherit basic state from
1275 * the parent (listener).
1276 *
1277 * No need to grab the new sonode's lock, since there is no
1278 * one that can have a reference to it.
1279 */
1280 mutex_enter(&pso->so_lock);
1281
1282 so->so_state |= SS_ISCONNECTED | (pso->so_state & SS_ASYNC);
1283 so->so_pgrp = pso->so_pgrp;
1284 so->so_rcvtimeo = pso->so_rcvtimeo;
1285 so->so_sndtimeo = pso->so_sndtimeo;
Yu Xiangninga5adac42008-12-29 13:56:29 +08001286 so->so_xpg_rcvbuf = pso->so_xpg_rcvbuf;
Yu Xiangning0f1702c2008-12-11 20:04:13 -08001287 /*
1288 * Make note of the socket level options. TCP and IP level
1289 * options are already inherited. We could do all this after
1290 * accept is successful but doing it here simplifies code and
1291 * no harm done for error case.
1292 */
1293 so->so_options = pso->so_options & (SO_DEBUG|SO_REUSEADDR|
Yu Xiangninga5adac42008-12-29 13:56:29 +08001294 SO_KEEPALIVE|SO_DONTROUTE|SO_BROADCAST|SO_USELOOPBACK|
Yu Xiangning0f1702c2008-12-11 20:04:13 -08001295 SO_OOBINLINE|SO_DGRAM_ERRIND|SO_LINGER);
1296 so->so_proto_props = pso->so_proto_props;
1297 so->so_mode = pso->so_mode;
andersf0267582008-12-20 22:46:32 -08001298 so->so_pollev = pso->so_pollev & SO_POLLEV_ALWAYS;
Yu Xiangning0f1702c2008-12-11 20:04:13 -08001299
1300 mutex_exit(&pso->so_lock);
Anders Persson3e95bd42010-06-17 17:22:09 -07001301
1302 /*
1303 * If the parent has any filters, try to inherit them.
1304 */
1305 if (pso->so_filter_active > 0 &&
1306 (error = sof_sonode_inherit_filters(so, pso)) != 0)
1307 return (error);
1308
Yu Xiangning0f1702c2008-12-11 20:04:13 -08001309 } else {
1310 struct sockparams *sp = so->so_sockparams;
1311 sock_upcalls_t *upcalls_to_use;
1312
1313 /*
Anders Persson3e95bd42010-06-17 17:22:09 -07001314 * Attach automatic filters, if there are any.
1315 */
1316 if (!list_is_empty(&sp->sp_auto_filters) &&
1317 (error = sof_sonode_autoattach_filters(so, cr)) != 0)
1318 return (error);
1319
1320 /* OK to attach filters */
1321 so->so_state |= SS_FILOP_OK;
1322
1323 /*
Yu Xiangning0f1702c2008-12-11 20:04:13 -08001324 * Based on the version number select the right upcalls to
1325 * pass down. Currently we only have one version so choose
1326 * default
1327 */
1328 upcalls_to_use = &so_upcalls;
1329
1330 /* active open, so create a lower handle */
1331 so->so_proto_handle =
1332 sp->sp_smod_info->smod_proto_create_func(so->so_family,
1333 so->so_type, so->so_protocol, &so->so_downcalls,
1334 &so->so_mode, &error, flags, cr);
1335
1336 if (so->so_proto_handle == NULL) {
1337 ASSERT(error != 0);
1338 /*
1339 * To be safe; if a lower handle cannot be created, and
1340 * the proto does not give a reason why, assume there
1341 * was a lack of memory.
1342 */
1343 return ((error == 0) ? ENOMEM : error);
1344 }
1345 ASSERT(so->so_downcalls != NULL);
1346 ASSERT(so->so_downcalls->sd_send != NULL ||
1347 so->so_downcalls->sd_send_uio != NULL);
1348 if (so->so_downcalls->sd_recv_uio != NULL) {
1349 ASSERT(so->so_downcalls->sd_poll != NULL);
1350 so->so_pollev |= SO_POLLEV_ALWAYS;
1351 }
1352
1353 (*so->so_downcalls->sd_activate)(so->so_proto_handle,
1354 (sock_upper_handle_t)so, upcalls_to_use, 0, cr);
1355
1356 /* Wildcard */
1357
1358 /*
1359 * FIXME No need for this, the protocol can deal with it in
1360 * sd_create(). Should update ICMP.
1361 */
1362 if (so->so_protocol != so->so_sockparams->sp_protocol) {
1363 int protocol = so->so_protocol;
1364 int error;
1365 /*
1366 * Issue SO_PROTOTYPE setsockopt.
1367 */
1368 error = socket_setsockopt(so, SOL_SOCKET, SO_PROTOTYPE,
1369 &protocol, (t_uscalar_t)sizeof (protocol), cr);
1370 if (error) {
1371 (void) (*so->so_downcalls->sd_close)
1372 (so->so_proto_handle, 0, cr);
1373
1374 mutex_enter(&so->so_lock);
1375 so_rcv_flush(so);
1376 mutex_exit(&so->so_lock);
1377 /*
1378 * Setsockopt often fails with ENOPROTOOPT but
1379 * socket() should fail with
1380 * EPROTONOSUPPORT/EPROTOTYPE.
1381 */
1382 return (EPROTONOSUPPORT);
1383 }
1384 }
Yu Xiangning0f1702c2008-12-11 20:04:13 -08001385 }
Anders Perssonbbc000e2009-04-28 12:10:59 -07001386
1387 if (uioasync.enabled)
1388 sod_sock_init(so);
1389
Anders Persson3e95bd42010-06-17 17:22:09 -07001390 /* put an extra reference on the socket for the protocol */
1391 VN_HOLD(SOTOV(so));
1392
Anders Perssonbbc000e2009-04-28 12:10:59 -07001393 return (0);
Yu Xiangning0f1702c2008-12-11 20:04:13 -08001394}
1395
1396/*
1397 * int socket_ioctl_common(struct sonode *so, int cmd, intptr_t arg, int mode,
1398 * struct cred *cr, int32_t *rvalp)
1399 *
1400 * Handle ioctls that manipulate basic socket state; non-blocking,
1401 * async, etc.
1402 *
1403 * Returns:
1404 * < 0 - ioctl was not handle
1405 * >= 0 - ioctl was handled, if > 0, then it is an errno
1406 *
1407 * Notes:
1408 * Assumes the standard receive buffer is used to obtain info for
1409 * NREAD.
1410 */
1411/* ARGSUSED */
1412int
1413socket_ioctl_common(struct sonode *so, int cmd, intptr_t arg, int mode,
1414 struct cred *cr, int32_t *rvalp)
1415{
1416 switch (cmd) {
Rao Shoaibbfcb55b2009-01-05 10:51:43 -08001417 case SIOCSQPTR:
1418 /*
1419 * SIOCSQPTR is valid only when helper stream is created
1420 * by the protocol.
1421 */
1422
1423 return (EOPNOTSUPP);
Yu Xiangning0f1702c2008-12-11 20:04:13 -08001424 case FIONBIO: {
1425 int32_t value;
1426
1427 if (so_copyin((void *)arg, &value, sizeof (int32_t),
1428 (mode & (int)FKIOCTL)))
1429 return (EFAULT);
1430
1431 mutex_enter(&so->so_lock);
1432 if (value) {
1433 so->so_state |= SS_NDELAY;
1434 } else {
1435 so->so_state &= ~SS_NDELAY;
1436 }
1437 mutex_exit(&so->so_lock);
1438 return (0);
1439 }
1440 case FIOASYNC: {
1441 int32_t value;
1442
1443 if (so_copyin((void *)arg, &value, sizeof (int32_t),
1444 (mode & (int)FKIOCTL)))
1445 return (EFAULT);
1446
1447 mutex_enter(&so->so_lock);
1448
1449 if (value) {
1450 /* Turn on SIGIO */
1451 so->so_state |= SS_ASYNC;
1452 } else {
1453 /* Turn off SIGIO */
1454 so->so_state &= ~SS_ASYNC;
1455 }
1456 mutex_exit(&so->so_lock);
1457
1458 return (0);
1459 }
1460
1461 case SIOCSPGRP:
1462 case FIOSETOWN: {
1463 int error;
1464 pid_t pid;
1465
1466 if (so_copyin((void *)arg, &pid, sizeof (pid_t),
1467 (mode & (int)FKIOCTL)))
1468 return (EFAULT);
1469
1470 mutex_enter(&so->so_lock);
1471 error = (pid != so->so_pgrp) ? socket_chgpgrp(so, pid) : 0;
1472 mutex_exit(&so->so_lock);
1473 return (error);
1474 }
1475 case SIOCGPGRP:
1476 case FIOGETOWN:
1477 if (so_copyout(&so->so_pgrp, (void *)arg,
1478 sizeof (pid_t), (mode & (int)FKIOCTL)))
1479 return (EFAULT);
1480
1481 return (0);
1482 case SIOCATMARK: {
1483 int retval;
1484
1485 /*
1486 * Only protocols that support urgent data can handle ATMARK.
1487 */
1488 if ((so->so_mode & SM_EXDATA) == 0)
1489 return (EINVAL);
1490
1491 /*
1492 * If the protocol is maintaining its own buffer, then the
1493 * request must be passed down.
1494 */
1495 if (so->so_downcalls->sd_recv_uio != NULL)
1496 return (-1);
1497
1498 retval = (so->so_state & SS_RCVATMARK) != 0;
1499
1500 if (so_copyout(&retval, (void *)arg, sizeof (int),
1501 (mode & (int)FKIOCTL))) {
1502 return (EFAULT);
1503 }
1504 return (0);
1505 }
1506
1507 case FIONREAD: {
1508 int retval;
1509
1510 /*
1511 * If the protocol is maintaining its own buffer, then the
1512 * request must be passed down.
1513 */
1514 if (so->so_downcalls->sd_recv_uio != NULL)
1515 return (-1);
1516
1517 retval = MIN(so->so_rcv_queued, INT_MAX);
1518
1519 if (so_copyout(&retval, (void *)arg,
1520 sizeof (retval), (mode & (int)FKIOCTL))) {
1521 return (EFAULT);
1522 }
1523 return (0);
1524 }
1525
1526 case _I_GETPEERCRED: {
1527 int error = 0;
1528
1529 if ((mode & FKIOCTL) == 0)
1530 return (EINVAL);
1531
1532 mutex_enter(&so->so_lock);
1533 if ((so->so_mode & SM_CONNREQUIRED) == 0) {
1534 error = ENOTSUP;
1535 } else if ((so->so_state & SS_ISCONNECTED) == 0) {
1536 error = ENOTCONN;
1537 } else if (so->so_peercred != NULL) {
1538 k_peercred_t *kp = (k_peercred_t *)arg;
1539 kp->pc_cr = so->so_peercred;
1540 kp->pc_cpid = so->so_cpid;
1541 crhold(so->so_peercred);
1542 } else {
1543 error = EINVAL;
1544 }
1545 mutex_exit(&so->so_lock);
1546 return (error);
1547 }
1548 default:
1549 return (-1);
1550 }
1551}
1552
1553/*
Anders Persson41174432009-02-12 17:35:05 -08001554 * Handle the I_NREAD STREAM ioctl.
1555 */
1556static int
1557so_strioc_nread(struct sonode *so, intptr_t arg, int mode, int32_t *rvalp)
1558{
1559 size_t size = 0;
1560 int retval;
1561 int count = 0;
1562 mblk_t *mp;
Rafael Vanonid3d50732009-11-13 01:32:32 -08001563 clock_t wakeup = drv_usectohz(10);
Anders Persson41174432009-02-12 17:35:05 -08001564
1565 if (so->so_downcalls == NULL ||
1566 so->so_downcalls->sd_recv_uio != NULL)
1567 return (EINVAL);
1568
1569 mutex_enter(&so->so_lock);
1570 /* Wait for reader to get out of the way. */
1571 while (so->so_flag & SOREADLOCKED) {
1572 /*
1573 * If reader is waiting for data, then there should be nothing
1574 * on the rcv queue.
1575 */
1576 if (so->so_rcv_wakeup)
1577 goto out;
1578
Anders Persson41174432009-02-12 17:35:05 -08001579 /* Do a timed sleep, in case the reader goes to sleep. */
Anders Perssondecd6cc2010-01-15 20:20:59 -08001580 (void) cv_reltimedwait(&so->so_read_cv, &so->so_lock, wakeup,
Rafael Vanonid3d50732009-11-13 01:32:32 -08001581 TR_CLOCK_TICK);
Anders Persson41174432009-02-12 17:35:05 -08001582 }
1583
1584 /*
1585 * Since we are holding so_lock no new reader will come in, and the
1586 * protocol will not be able to enqueue data. So it's safe to walk
1587 * both rcv queues.
1588 */
1589 mp = so->so_rcv_q_head;
1590 if (mp != NULL) {
1591 size = msgdsize(so->so_rcv_q_head);
1592 for (; mp != NULL; mp = mp->b_next)
1593 count++;
1594 } else {
1595 /*
1596 * In case the processing list was empty, get the size of the
1597 * next msg in line.
1598 */
1599 size = msgdsize(so->so_rcv_head);
1600 }
1601
1602 for (mp = so->so_rcv_head; mp != NULL; mp = mp->b_next)
1603 count++;
1604out:
1605 mutex_exit(&so->so_lock);
1606
1607 /*
1608 * Drop down from size_t to the "int" required by the
1609 * interface. Cap at INT_MAX.
1610 */
1611 retval = MIN(size, INT_MAX);
1612 if (so_copyout(&retval, (void *)arg, sizeof (retval),
1613 (mode & (int)FKIOCTL))) {
1614 return (EFAULT);
1615 } else {
1616 *rvalp = count;
1617 return (0);
1618 }
1619}
1620
1621/*
1622 * Process STREAM ioctls.
Yu Xiangning0f1702c2008-12-11 20:04:13 -08001623 *
1624 * Returns:
1625 * < 0 - ioctl was not handle
1626 * >= 0 - ioctl was handled, if > 0, then it is an errno
1627 */
1628int
1629socket_strioc_common(struct sonode *so, int cmd, intptr_t arg, int mode,
1630 struct cred *cr, int32_t *rvalp)
1631{
Anders Persson41174432009-02-12 17:35:05 -08001632 int retval;
1633
1634 /* Only STREAM iotcls are handled here */
1635 if ((cmd & 0xffffff00U) != STR)
1636 return (-1);
1637
Yu Xiangning0f1702c2008-12-11 20:04:13 -08001638 switch (cmd) {
Anders Persson41174432009-02-12 17:35:05 -08001639 case I_CANPUT:
1640 /*
1641 * We return an error for I_CANPUT so that isastream(3C) will
1642 * not report the socket as being a STREAM.
1643 */
Yu Xiangning0f1702c2008-12-11 20:04:13 -08001644 return (EOPNOTSUPP);
Anders Persson41174432009-02-12 17:35:05 -08001645 case I_NREAD:
1646 /* Avoid doing a fallback for I_NREAD. */
1647 return (so_strioc_nread(so, arg, mode, rvalp));
Yu Xiangning0f1702c2008-12-11 20:04:13 -08001648 case I_LOOK:
Anders Persson41174432009-02-12 17:35:05 -08001649 /* Avoid doing a fallback for I_LOOK. */
Yu Xiangning0f1702c2008-12-11 20:04:13 -08001650 if (so_copyout("sockmod", (void *)arg, strlen("sockmod") + 1,
1651 (mode & (int)FKIOCTL))) {
1652 return (EFAULT);
1653 }
1654 return (0);
1655 default:
Anders Persson41174432009-02-12 17:35:05 -08001656 break;
1657 }
1658
1659 /*
1660 * Try to fall back to TPI, and if successful, reissue the ioctl.
1661 */
1662 if ((retval = so_tpi_fallback(so, cr)) == 0) {
1663 /* Reissue the ioctl */
1664 ASSERT(so->so_rcv_q_head == NULL);
1665 return (SOP_IOCTL(so, cmd, arg, mode, cr, rvalp));
1666 } else {
1667 return (retval);
Yu Xiangning0f1702c2008-12-11 20:04:13 -08001668 }
1669}
1670
Anders Persson2c632ad2009-10-21 19:52:57 -07001671/*
1672 * This is called for all socket types to verify that the buffer size is large
1673 * enough for the option, and if we can, handle the request as well. Most
1674 * options will be forwarded to the protocol.
1675 */
Yu Xiangning0f1702c2008-12-11 20:04:13 -08001676int
1677socket_getopt_common(struct sonode *so, int level, int option_name,
Yu Xiangninga5adac42008-12-29 13:56:29 +08001678 void *optval, socklen_t *optlenp, int flags)
Yu Xiangning0f1702c2008-12-11 20:04:13 -08001679{
1680 if (level != SOL_SOCKET)
1681 return (-1);
1682
1683 switch (option_name) {
1684 case SO_ERROR:
1685 case SO_DOMAIN:
1686 case SO_TYPE:
1687 case SO_ACCEPTCONN: {
1688 int32_t value;
1689 socklen_t optlen = *optlenp;
1690
1691 if (optlen < (t_uscalar_t)sizeof (int32_t)) {
1692 return (EINVAL);
1693 }
1694
1695 switch (option_name) {
1696 case SO_ERROR:
1697 mutex_enter(&so->so_lock);
1698 value = sogeterr(so, B_TRUE);
1699 mutex_exit(&so->so_lock);
1700 break;
1701 case SO_DOMAIN:
1702 value = so->so_family;
1703 break;
1704 case SO_TYPE:
1705 value = so->so_type;
1706 break;
1707 case SO_ACCEPTCONN:
1708 if (so->so_state & SS_ACCEPTCONN)
1709 value = SO_ACCEPTCONN;
1710 else
1711 value = 0;
1712 break;
1713 }
1714
1715 bcopy(&value, optval, sizeof (value));
1716 *optlenp = sizeof (value);
1717
1718 return (0);
1719 }
1720 case SO_SNDTIMEO:
1721 case SO_RCVTIMEO: {
1722 clock_t value;
1723 socklen_t optlen = *optlenp;
shenjiane5083e82009-01-20 14:46:11 +08001724
1725 if (get_udatamodel() == DATAMODEL_NONE ||
1726 get_udatamodel() == DATAMODEL_NATIVE) {
shenjian22238f72009-01-07 13:45:08 +08001727 if (optlen < sizeof (struct timeval))
1728 return (EINVAL);
1729 } else {
1730 if (optlen < sizeof (struct timeval32))
1731 return (EINVAL);
Yu Xiangning0f1702c2008-12-11 20:04:13 -08001732 }
1733 if (option_name == SO_RCVTIMEO)
1734 value = drv_hztousec(so->so_rcvtimeo);
1735 else
1736 value = drv_hztousec(so->so_sndtimeo);
shenjian22238f72009-01-07 13:45:08 +08001737
shenjiane5083e82009-01-20 14:46:11 +08001738 if (get_udatamodel() == DATAMODEL_NONE ||
1739 get_udatamodel() == DATAMODEL_NATIVE) {
shenjian22238f72009-01-07 13:45:08 +08001740 ((struct timeval *)(optval))->tv_sec =
1741 value / (1000 * 1000);
1742 ((struct timeval *)(optval))->tv_usec =
1743 value % (1000 * 1000);
1744 *optlenp = sizeof (struct timeval);
1745 } else {
1746 ((struct timeval32 *)(optval))->tv_sec =
1747 value / (1000 * 1000);
1748 ((struct timeval32 *)(optval))->tv_usec =
1749 value % (1000 * 1000);
1750 *optlenp = sizeof (struct timeval32);
1751 }
Yu Xiangning0f1702c2008-12-11 20:04:13 -08001752 return (0);
1753 }
1754 case SO_DEBUG:
1755 case SO_REUSEADDR:
1756 case SO_KEEPALIVE:
1757 case SO_DONTROUTE:
1758 case SO_BROADCAST:
1759 case SO_USELOOPBACK:
1760 case SO_OOBINLINE:
1761 case SO_SNDBUF:
Yu Xiangning0f1702c2008-12-11 20:04:13 -08001762#ifdef notyet
1763 case SO_SNDLOWAT:
1764 case SO_RCVLOWAT:
1765#endif /* notyet */
1766 case SO_DGRAM_ERRIND: {
1767 socklen_t optlen = *optlenp;
1768
1769 if (optlen < (t_uscalar_t)sizeof (int32_t))
1770 return (EINVAL);
1771 break;
1772 }
Yu Xiangninga5adac42008-12-29 13:56:29 +08001773 case SO_RCVBUF: {
1774 socklen_t optlen = *optlenp;
1775
1776 if (optlen < (t_uscalar_t)sizeof (int32_t))
1777 return (EINVAL);
1778
1779 if ((flags & _SOGETSOCKOPT_XPG4_2) && so->so_xpg_rcvbuf != 0) {
1780 /*
1781 * XXX If SO_RCVBUF has been set and this is an
1782 * XPG 4.2 application then do not ask the transport
1783 * since the transport might adjust the value and not
1784 * return exactly what was set by the application.
1785 * For non-XPG 4.2 application we return the value
1786 * that the transport is actually using.
1787 */
1788 *(int32_t *)optval = so->so_xpg_rcvbuf;
1789 *optlenp = sizeof (so->so_xpg_rcvbuf);
1790 return (0);
1791 }
1792 /*
1793 * If the option has not been set then get a default
1794 * value from the transport.
1795 */
1796 break;
1797 }
Yu Xiangning0f1702c2008-12-11 20:04:13 -08001798 case SO_LINGER: {
1799 socklen_t optlen = *optlenp;
1800
1801 if (optlen < (t_uscalar_t)sizeof (struct linger))
1802 return (EINVAL);
1803 break;
1804 }
1805 case SO_SND_BUFINFO: {
1806 socklen_t optlen = *optlenp;
1807
1808 if (optlen < (t_uscalar_t)sizeof (struct so_snd_bufinfo))
1809 return (EINVAL);
1810 ((struct so_snd_bufinfo *)(optval))->sbi_wroff =
1811 (so->so_proto_props).sopp_wroff;
1812 ((struct so_snd_bufinfo *)(optval))->sbi_maxblk =
1813 (so->so_proto_props).sopp_maxblk;
1814 ((struct so_snd_bufinfo *)(optval))->sbi_maxpsz =
1815 (so->so_proto_props).sopp_maxpsz;
1816 ((struct so_snd_bufinfo *)(optval))->sbi_tail =
1817 (so->so_proto_props).sopp_tail;
1818 *optlenp = sizeof (struct so_snd_bufinfo);
1819 return (0);
1820 }
Anders Persson3e95bd42010-06-17 17:22:09 -07001821 case SO_SND_COPYAVOID: {
1822 sof_instance_t *inst;
1823
1824 /*
1825 * Avoid zero-copy if there is a filter with a data_out
1826 * callback. We could let the operation succeed, but then
1827 * the filter would have to copy the data anyway.
1828 */
1829 for (inst = so->so_filter_top; inst != NULL;
1830 inst = inst->sofi_next) {
1831 if (SOF_INTERESTED(inst, data_out))
1832 return (EOPNOTSUPP);
1833 }
1834 break;
1835 }
1836
Yu Xiangning0f1702c2008-12-11 20:04:13 -08001837 default:
1838 break;
1839 }
1840
1841 /* Unknown Option */
1842 return (-1);
1843}
1844
1845void
1846socket_sonode_destroy(struct sonode *so)
1847{
1848 sonode_fini(so);
1849 kmem_cache_free(socket_cache, so);
1850}
1851
1852int
1853so_zcopy_wait(struct sonode *so)
1854{
1855 int error = 0;
1856
1857 mutex_enter(&so->so_lock);
1858 while (!(so->so_copyflag & STZCNOTIFY)) {
1859 if (so->so_state & SS_CLOSING) {
1860 mutex_exit(&so->so_lock);
1861 return (EINTR);
1862 }
1863 if (cv_wait_sig(&so->so_copy_cv, &so->so_lock) == 0) {
1864 error = EINTR;
1865 break;
1866 }
1867 }
1868 so->so_copyflag &= ~STZCNOTIFY;
1869 mutex_exit(&so->so_lock);
1870 return (error);
1871}
1872
1873void
1874so_timer_callback(void *arg)
1875{
1876 struct sonode *so = (struct sonode *)arg;
1877
1878 mutex_enter(&so->so_lock);
1879
1880 so->so_rcv_timer_tid = 0;
1881 if (so->so_rcv_queued > 0) {
1882 so_notify_data(so, so->so_rcv_queued);
1883 } else {
1884 mutex_exit(&so->so_lock);
1885 }
1886}
1887
1888#ifdef DEBUG
1889/*
1890 * Verify that the length stored in so_rcv_queued and the length of data blocks
1891 * queued is same.
1892 */
1893static boolean_t
1894so_check_length(sonode_t *so)
1895{
1896 mblk_t *mp = so->so_rcv_q_head;
1897 int len = 0;
1898
1899 ASSERT(MUTEX_HELD(&so->so_lock));
1900
1901 if (mp != NULL) {
1902 len = msgdsize(mp);
1903 while ((mp = mp->b_next) != NULL)
1904 len += msgdsize(mp);
1905 }
1906 mp = so->so_rcv_head;
1907 if (mp != NULL) {
1908 len += msgdsize(mp);
1909 while ((mp = mp->b_next) != NULL)
1910 len += msgdsize(mp);
1911 }
1912 return ((len == so->so_rcv_queued) ? B_TRUE : B_FALSE);
1913}
1914#endif
1915
1916int
1917so_get_mod_version(struct sockparams *sp)
1918{
1919 ASSERT(sp != NULL && sp->sp_smod_info != NULL);
1920 return (sp->sp_smod_info->smod_version);
1921}
1922
1923/*
1924 * so_start_fallback()
1925 *
1926 * Block new socket operations from coming in, and wait for active operations
1927 * to complete. Threads that are sleeping will be woken up so they can get
1928 * out of the way.
1929 *
1930 * The caller must be a reader on so_fallback_rwlock.
1931 */
1932static boolean_t
1933so_start_fallback(struct sonode *so)
1934{
1935 ASSERT(RW_READ_HELD(&so->so_fallback_rwlock));
1936
1937 mutex_enter(&so->so_lock);
1938 if (so->so_state & SS_FALLBACK_PENDING) {
1939 mutex_exit(&so->so_lock);
1940 return (B_FALSE);
1941 }
1942 so->so_state |= SS_FALLBACK_PENDING;
1943 /*
1944 * Poke all threads that might be sleeping. Any operation that comes
1945 * in after the cv_broadcast will observe the fallback pending flag
1946 * which cause the call to return where it would normally sleep.
1947 */
1948 cv_broadcast(&so->so_state_cv); /* threads in connect() */
1949 cv_broadcast(&so->so_rcv_cv); /* threads in recvmsg() */
1950 cv_broadcast(&so->so_snd_cv); /* threads in sendmsg() */
1951 mutex_enter(&so->so_acceptq_lock);
1952 cv_broadcast(&so->so_acceptq_cv); /* threads in accept() */
1953 mutex_exit(&so->so_acceptq_lock);
1954 mutex_exit(&so->so_lock);
1955
1956 /*
1957 * The main reason for the rw_tryupgrade call is to provide
1958 * observability during the fallback process. We want to
1959 * be able to see if there are pending operations.
1960 */
1961 if (rw_tryupgrade(&so->so_fallback_rwlock) == 0) {
1962 /*
1963 * It is safe to drop and reaquire the fallback lock, because
1964 * we are guaranteed that another fallback cannot take place.
1965 */
1966 rw_exit(&so->so_fallback_rwlock);
1967 DTRACE_PROBE1(pending__ops__wait, (struct sonode *), so);
1968 rw_enter(&so->so_fallback_rwlock, RW_WRITER);
1969 DTRACE_PROBE1(pending__ops__complete, (struct sonode *), so);
1970 }
1971
1972 return (B_TRUE);
1973}
1974
1975/*
1976 * so_end_fallback()
1977 *
1978 * Allow socket opertions back in.
1979 *
1980 * The caller must be a writer on so_fallback_rwlock.
1981 */
1982static void
1983so_end_fallback(struct sonode *so)
1984{
1985 ASSERT(RW_ISWRITER(&so->so_fallback_rwlock));
1986
1987 mutex_enter(&so->so_lock);
Anders Persson41174432009-02-12 17:35:05 -08001988 so->so_state &= ~(SS_FALLBACK_PENDING|SS_FALLBACK_DRAIN);
Yu Xiangning0f1702c2008-12-11 20:04:13 -08001989 mutex_exit(&so->so_lock);
1990
1991 rw_downgrade(&so->so_fallback_rwlock);
1992}
1993
1994/*
1995 * so_quiesced_cb()
1996 *
1997 * Callback passed to the protocol during fallback. It is called once
1998 * the endpoint is quiescent.
1999 *
2000 * No requests from the user, no notifications from the protocol, so it
2001 * is safe to synchronize the state. Data can also be moved without
2002 * risk for reordering.
2003 *
Yu Xiangning0f1702c2008-12-11 20:04:13 -08002004 * We do not need to hold so_lock, since there can be only one thread
2005 * operating on the sonode.
2006 */
Anders Persson3e95bd42010-06-17 17:22:09 -07002007static mblk_t *
2008so_quiesced_cb(sock_upper_handle_t sock_handle, sock_quiesce_arg_t *arg,
2009 struct T_capability_ack *tcap,
2010 struct sockaddr *laddr, socklen_t laddrlen,
Yu Xiangning0f1702c2008-12-11 20:04:13 -08002011 struct sockaddr *faddr, socklen_t faddrlen, short opts)
2012{
2013 struct sonode *so = (struct sonode *)sock_handle;
Anders Persson41174432009-02-12 17:35:05 -08002014 boolean_t atmark;
Anders Persson3e95bd42010-06-17 17:22:09 -07002015 mblk_t *retmp = NULL, **tailmpp = &retmp;
Yu Xiangning0f1702c2008-12-11 20:04:13 -08002016
Anders Persson3e95bd42010-06-17 17:22:09 -07002017 if (tcap != NULL)
2018 sotpi_update_state(so, tcap, laddr, laddrlen, faddr, faddrlen,
2019 opts);
Yu Xiangning0f1702c2008-12-11 20:04:13 -08002020
Anders Persson41174432009-02-12 17:35:05 -08002021 /*
2022 * Some protocols do not quiece the data path during fallback. Once
2023 * we set the SS_FALLBACK_DRAIN flag any attempt to queue data will
2024 * fail and the protocol is responsible for saving the data for later
2025 * delivery (i.e., once the fallback has completed).
2026 */
Yu Xiangning0f1702c2008-12-11 20:04:13 -08002027 mutex_enter(&so->so_lock);
Anders Persson41174432009-02-12 17:35:05 -08002028 so->so_state |= SS_FALLBACK_DRAIN;
Yu Xiangning0f1702c2008-12-11 20:04:13 -08002029 SOCKET_TIMER_CANCEL(so);
2030 mutex_exit(&so->so_lock);
Anders Persson41174432009-02-12 17:35:05 -08002031
Yu Xiangning0f1702c2008-12-11 20:04:13 -08002032 if (so->so_rcv_head != NULL) {
2033 if (so->so_rcv_q_last_head == NULL)
2034 so->so_rcv_q_head = so->so_rcv_head;
2035 else
2036 so->so_rcv_q_last_head->b_next = so->so_rcv_head;
2037 so->so_rcv_q_last_head = so->so_rcv_last_head;
2038 }
2039
Anders Persson41174432009-02-12 17:35:05 -08002040 atmark = (so->so_state & SS_RCVATMARK) != 0;
2041 /*
2042 * Clear any OOB state having to do with pending data. The TPI
2043 * code path will set the appropriate oob state when we move the
2044 * oob data to the STREAM head. We leave SS_HADOOBDATA since the oob
2045 * data has already been consumed.
2046 */
2047 so->so_state &= ~(SS_RCVATMARK|SS_OOBPEND|SS_HAVEOOBDATA);
2048
2049 ASSERT(so->so_oobmsg != NULL || so->so_oobmark <= so->so_rcv_queued);
2050
2051 /*
2052 * Move data to the STREAM head.
2053 */
Yu Xiangning0f1702c2008-12-11 20:04:13 -08002054 while (so->so_rcv_q_head != NULL) {
2055 mblk_t *mp = so->so_rcv_q_head;
2056 size_t mlen = msgdsize(mp);
2057
2058 so->so_rcv_q_head = mp->b_next;
2059 mp->b_next = NULL;
2060 mp->b_prev = NULL;
Anders Persson41174432009-02-12 17:35:05 -08002061
2062 /*
2063 * Send T_EXDATA_IND if we are at the oob mark.
2064 */
2065 if (atmark) {
2066 struct T_exdata_ind *tei;
Anders Persson3e95bd42010-06-17 17:22:09 -07002067 mblk_t *mp1 = arg->soqa_exdata_mp;
Anders Persson41174432009-02-12 17:35:05 -08002068
Anders Persson3e95bd42010-06-17 17:22:09 -07002069 arg->soqa_exdata_mp = NULL;
Anders Persson41174432009-02-12 17:35:05 -08002070 ASSERT(mp1 != NULL);
2071 mp1->b_datap->db_type = M_PROTO;
2072 tei = (struct T_exdata_ind *)mp1->b_rptr;
2073 tei->PRIM_type = T_EXDATA_IND;
2074 tei->MORE_flag = 0;
2075 mp1->b_wptr = (uchar_t *)&tei[1];
2076
2077 if (IS_SO_OOB_INLINE(so)) {
2078 mp1->b_cont = mp;
2079 } else {
2080 ASSERT(so->so_oobmsg != NULL);
2081 mp1->b_cont = so->so_oobmsg;
2082 so->so_oobmsg = NULL;
2083
2084 /* process current mp next time around */
2085 mp->b_next = so->so_rcv_q_head;
2086 so->so_rcv_q_head = mp;
2087 mlen = 0;
2088 }
2089 mp = mp1;
2090
2091 /* we have consumed the oob mark */
2092 atmark = B_FALSE;
2093 } else if (so->so_oobmark > 0) {
2094 /*
2095 * Check if the OOB mark is within the current
2096 * mblk chain. In that case we have to split it up.
2097 */
2098 if (so->so_oobmark < mlen) {
2099 mblk_t *urg_mp = mp;
2100
2101 atmark = B_TRUE;
2102 mp = NULL;
2103 mlen = so->so_oobmark;
2104
2105 /*
2106 * It is assumed that the OOB mark does
2107 * not land within a mblk.
2108 */
2109 do {
2110 so->so_oobmark -= MBLKL(urg_mp);
2111 mp = urg_mp;
2112 urg_mp = urg_mp->b_cont;
2113 } while (so->so_oobmark > 0);
2114 mp->b_cont = NULL;
2115 if (urg_mp != NULL) {
2116 urg_mp->b_next = so->so_rcv_q_head;
2117 so->so_rcv_q_head = urg_mp;
2118 }
2119 } else {
2120 so->so_oobmark -= mlen;
2121 if (so->so_oobmark == 0)
2122 atmark = B_TRUE;
2123 }
2124 }
2125
2126 /*
2127 * Queue data on the STREAM head.
2128 */
Yu Xiangning0f1702c2008-12-11 20:04:13 -08002129 so->so_rcv_queued -= mlen;
Anders Persson3e95bd42010-06-17 17:22:09 -07002130 *tailmpp = mp;
2131 tailmpp = &mp->b_next;
Yu Xiangning0f1702c2008-12-11 20:04:13 -08002132 }
Yu Xiangning0f1702c2008-12-11 20:04:13 -08002133 so->so_rcv_head = NULL;
2134 so->so_rcv_last_head = NULL;
2135 so->so_rcv_q_head = NULL;
2136 so->so_rcv_q_last_head = NULL;
2137
Anders Persson41174432009-02-12 17:35:05 -08002138 /*
2139 * Check if the oob byte is at the end of the data stream, or if the
2140 * oob byte has not yet arrived. In the latter case we have to send a
2141 * SIGURG and a mark indicator to the STREAM head. The mark indicator
2142 * is needed to guarantee correct behavior for SIOCATMARK. See block
2143 * comment in socktpi.h for more details.
2144 */
2145 if (atmark || so->so_oobmark > 0) {
2146 mblk_t *mp;
Yu Xiangning0f1702c2008-12-11 20:04:13 -08002147
Anders Persson41174432009-02-12 17:35:05 -08002148 if (atmark && so->so_oobmsg != NULL) {
2149 struct T_exdata_ind *tei;
2150
Anders Persson3e95bd42010-06-17 17:22:09 -07002151 mp = arg->soqa_exdata_mp;
2152 arg->soqa_exdata_mp = NULL;
Anders Persson41174432009-02-12 17:35:05 -08002153 ASSERT(mp != NULL);
2154 mp->b_datap->db_type = M_PROTO;
2155 tei = (struct T_exdata_ind *)mp->b_rptr;
2156 tei->PRIM_type = T_EXDATA_IND;
2157 tei->MORE_flag = 0;
2158 mp->b_wptr = (uchar_t *)&tei[1];
2159
2160 mp->b_cont = so->so_oobmsg;
2161 so->so_oobmsg = NULL;
2162
Anders Persson3e95bd42010-06-17 17:22:09 -07002163 *tailmpp = mp;
2164 tailmpp = &mp->b_next;
Anders Persson41174432009-02-12 17:35:05 -08002165 } else {
2166 /* Send up the signal */
Anders Persson3e95bd42010-06-17 17:22:09 -07002167 mp = arg->soqa_exdata_mp;
2168 arg->soqa_exdata_mp = NULL;
Anders Persson41174432009-02-12 17:35:05 -08002169 ASSERT(mp != NULL);
2170 DB_TYPE(mp) = M_PCSIG;
2171 *mp->b_wptr++ = (uchar_t)SIGURG;
Anders Persson3e95bd42010-06-17 17:22:09 -07002172 *tailmpp = mp;
2173 tailmpp = &mp->b_next;
Anders Persson41174432009-02-12 17:35:05 -08002174
2175 /* Send up the mark indicator */
Anders Persson3e95bd42010-06-17 17:22:09 -07002176 mp = arg->soqa_urgmark_mp;
2177 arg->soqa_urgmark_mp = NULL;
Anders Persson41174432009-02-12 17:35:05 -08002178 mp->b_flag = atmark ? MSGMARKNEXT : MSGNOTMARKNEXT;
Anders Persson3e95bd42010-06-17 17:22:09 -07002179 *tailmpp = mp;
2180 tailmpp = &mp->b_next;
Anders Persson41174432009-02-12 17:35:05 -08002181
2182 so->so_oobmark = 0;
2183 }
2184 }
Anders Persson41174432009-02-12 17:35:05 -08002185 ASSERT(so->so_oobmark == 0);
Yu Xiangning0f1702c2008-12-11 20:04:13 -08002186 ASSERT(so->so_rcv_queued == 0);
Anders Persson3e95bd42010-06-17 17:22:09 -07002187
2188 return (retmp);
Yu Xiangning0f1702c2008-12-11 20:04:13 -08002189}
2190
Anders Persson41174432009-02-12 17:35:05 -08002191#ifdef DEBUG
2192/*
2193 * Do an integrity check of the sonode. This should be done if a
2194 * fallback fails after sonode has initially been converted to use
2195 * TPI and subsequently have to be reverted.
2196 *
2197 * Failure to pass the integrity check will panic the system.
2198 */
2199void
2200so_integrity_check(struct sonode *cur, struct sonode *orig)
2201{
2202 VERIFY(cur->so_vnode == orig->so_vnode);
2203 VERIFY(cur->so_ops == orig->so_ops);
2204 /*
2205 * For so_state we can only VERIFY the state flags in CHECK_STATE.
2206 * The other state flags might be affected by a notification from the
2207 * protocol.
2208 */
2209#define CHECK_STATE (SS_CANTRCVMORE|SS_CANTSENDMORE|SS_NDELAY|SS_NONBLOCK| \
2210 SS_ASYNC|SS_ACCEPTCONN|SS_SAVEDEOR|SS_RCVATMARK|SS_OOBPEND| \
2211 SS_HAVEOOBDATA|SS_HADOOBDATA|SS_SENTLASTREADSIG|SS_SENTLASTWRITESIG)
2212 VERIFY((cur->so_state & (orig->so_state & CHECK_STATE)) ==
2213 (orig->so_state & CHECK_STATE));
2214 VERIFY(cur->so_mode == orig->so_mode);
2215 VERIFY(cur->so_flag == orig->so_flag);
2216 VERIFY(cur->so_count == orig->so_count);
2217 /* Cannot VERIFY so_proto_connid; proto can update it */
2218 VERIFY(cur->so_sockparams == orig->so_sockparams);
2219 /* an error might have been recorded, but it can not be lost */
2220 VERIFY(cur->so_error != 0 || orig->so_error == 0);
2221 VERIFY(cur->so_family == orig->so_family);
2222 VERIFY(cur->so_type == orig->so_type);
2223 VERIFY(cur->so_protocol == orig->so_protocol);
2224 VERIFY(cur->so_version == orig->so_version);
2225 /* New conns might have arrived, but none should have been lost */
2226 VERIFY(cur->so_acceptq_len >= orig->so_acceptq_len);
Anders Persson3e95bd42010-06-17 17:22:09 -07002227 VERIFY(list_head(&cur->so_acceptq_list) ==
2228 list_head(&orig->so_acceptq_list));
Anders Persson41174432009-02-12 17:35:05 -08002229 VERIFY(cur->so_backlog == orig->so_backlog);
2230 /* New OOB migth have arrived, but mark should not have been lost */
2231 VERIFY(cur->so_oobmark >= orig->so_oobmark);
2232 /* Cannot VERIFY so_oobmsg; the proto might have sent up a new one */
2233 VERIFY(cur->so_pgrp == orig->so_pgrp);
2234 VERIFY(cur->so_peercred == orig->so_peercred);
2235 VERIFY(cur->so_cpid == orig->so_cpid);
2236 VERIFY(cur->so_zoneid == orig->so_zoneid);
2237 /* New data migth have arrived, but none should have been lost */
2238 VERIFY(cur->so_rcv_queued >= orig->so_rcv_queued);
2239 VERIFY(cur->so_rcv_q_head == orig->so_rcv_q_head);
2240 VERIFY(cur->so_rcv_head == orig->so_rcv_head);
2241 VERIFY(cur->so_proto_handle == orig->so_proto_handle);
2242 VERIFY(cur->so_downcalls == orig->so_downcalls);
2243 /* Cannot VERIFY so_proto_props; they can be updated by proto */
2244}
2245#endif
2246
Yu Xiangning0f1702c2008-12-11 20:04:13 -08002247/*
2248 * so_tpi_fallback()
2249 *
Anders Persson41174432009-02-12 17:35:05 -08002250 * This is the fallback initation routine; things start here.
Yu Xiangning0f1702c2008-12-11 20:04:13 -08002251 *
2252 * Basic strategy:
2253 * o Block new socket operations from coming in
2254 * o Allocate/initate info needed by TPI
2255 * o Quiesce the connection, at which point we sync
2256 * state and move data
2257 * o Change operations (sonodeops) associated with the socket
2258 * o Unblock threads waiting for the fallback to finish
2259 */
2260int
2261so_tpi_fallback(struct sonode *so, struct cred *cr)
2262{
2263 int error;
2264 queue_t *q;
2265 struct sockparams *sp;
Anders Persson41174432009-02-12 17:35:05 -08002266 struct sockparams *newsp = NULL;
Yu Xiangning0f1702c2008-12-11 20:04:13 -08002267 so_proto_fallback_func_t fbfunc;
Anders Persson3e95bd42010-06-17 17:22:09 -07002268 const char *devpath;
Yu Xiangning0f1702c2008-12-11 20:04:13 -08002269 boolean_t direct;
Anders Persson41174432009-02-12 17:35:05 -08002270 struct sonode *nso;
Anders Persson3e95bd42010-06-17 17:22:09 -07002271 sock_quiesce_arg_t arg = { NULL, NULL };
Anders Persson41174432009-02-12 17:35:05 -08002272#ifdef DEBUG
2273 struct sonode origso;
2274#endif
Yu Xiangning0f1702c2008-12-11 20:04:13 -08002275 error = 0;
2276 sp = so->so_sockparams;
2277 fbfunc = sp->sp_smod_info->smod_proto_fallback_func;
2278
2279 /*
Robert Mustacchi907c2822021-02-01 13:50:02 +00002280 * Cannot fallback if the socket has active filters or a krecv callback.
Yu Xiangning0f1702c2008-12-11 20:04:13 -08002281 */
Robert Mustacchi907c2822021-02-01 13:50:02 +00002282 if (so->so_filter_active > 0 || so->so_krecv_cb != NULL)
Anders Persson3e95bd42010-06-17 17:22:09 -07002283 return (EINVAL);
2284
2285 switch (so->so_family) {
2286 case AF_INET:
2287 devpath = sp->sp_smod_info->smod_fallback_devpath_v4;
2288 break;
2289 case AF_INET6:
2290 devpath = sp->sp_smod_info->smod_fallback_devpath_v6;
2291 break;
2292 default:
2293 return (EINVAL);
2294 }
2295
2296 /*
2297 * Fallback can only happen if the socket module has a TPI device
2298 * and fallback function.
2299 */
2300 if (devpath == NULL || fbfunc == NULL)
Yu Xiangning0f1702c2008-12-11 20:04:13 -08002301 return (EINVAL);
2302
2303 /*
2304 * Initiate fallback; upon success we know that no new requests
2305 * will come in from the user.
2306 */
2307 if (!so_start_fallback(so))
2308 return (EAGAIN);
Anders Persson41174432009-02-12 17:35:05 -08002309#ifdef DEBUG
2310 /*
2311 * Make a copy of the sonode in case we need to make an integrity
2312 * check later on.
2313 */
2314 bcopy(so, &origso, sizeof (*so));
2315#endif
Yu Xiangning0f1702c2008-12-11 20:04:13 -08002316
Anders Persson7d64f412009-02-11 15:38:45 -08002317 sp->sp_stats.sps_nfallback.value.ui64++;
2318
Yu Xiangning0f1702c2008-12-11 20:04:13 -08002319 newsp = sockparams_hold_ephemeral_bydev(so->so_family, so->so_type,
Anders Persson3e95bd42010-06-17 17:22:09 -07002320 so->so_protocol, devpath, KM_SLEEP, &error);
Yu Xiangning0f1702c2008-12-11 20:04:13 -08002321 if (error != 0)
2322 goto out;
2323
2324 if (so->so_direct != NULL) {
2325 sodirect_t *sodp = so->so_direct;
Anders Perssonbbc000e2009-04-28 12:10:59 -07002326 mutex_enter(&so->so_lock);
Yu Xiangning0f1702c2008-12-11 20:04:13 -08002327
Anders Perssonbbc000e2009-04-28 12:10:59 -07002328 so->so_direct->sod_enabled = B_FALSE;
Yu Xiangning0f1702c2008-12-11 20:04:13 -08002329 so->so_state &= ~SS_SODIRECT;
2330 ASSERT(sodp->sod_uioafh == NULL);
Anders Perssonbbc000e2009-04-28 12:10:59 -07002331 mutex_exit(&so->so_lock);
Yu Xiangning0f1702c2008-12-11 20:04:13 -08002332 }
2333
2334 /* Turn sonode into a TPI socket */
Anders Persson41174432009-02-12 17:35:05 -08002335 error = sotpi_convert_sonode(so, newsp, &direct, &q, cr);
2336 if (error != 0)
Yu Xiangning0f1702c2008-12-11 20:04:13 -08002337 goto out;
Anders Persson3e95bd42010-06-17 17:22:09 -07002338 /*
2339 * When it comes to urgent data we have two cases to deal with;
2340 * (1) The oob byte has already arrived, or (2) the protocol has
2341 * notified that oob data is pending, but it has not yet arrived.
2342 *
2343 * For (1) all we need to do is send a T_EXDATA_IND to indicate were
2344 * in the byte stream the oob byte is. For (2) we have to send a
2345 * SIGURG (M_PCSIG), followed by a zero-length mblk indicating whether
2346 * the oob byte will be the next byte from the protocol.
2347 *
2348 * So in the worst case we need two mblks, one for the signal, another
2349 * for mark indication. In that case we use the exdata_mp for the sig.
2350 */
2351 arg.soqa_exdata_mp = allocb_wait(sizeof (struct T_exdata_ind),
2352 BPRI_MED, STR_NOSIG, NULL);
2353 arg.soqa_urgmark_mp = allocb_wait(0, BPRI_MED, STR_NOSIG, NULL);
Yu Xiangning0f1702c2008-12-11 20:04:13 -08002354
2355 /*
2356 * Now tell the protocol to start using TPI. so_quiesced_cb be
2357 * called once it's safe to synchronize state.
2358 */
2359 DTRACE_PROBE1(proto__fallback__begin, struct sonode *, so);
Anders Persson3e95bd42010-06-17 17:22:09 -07002360 error = (*fbfunc)(so->so_proto_handle, q, direct, so_quiesced_cb,
2361 &arg);
Yu Xiangning0f1702c2008-12-11 20:04:13 -08002362 DTRACE_PROBE1(proto__fallback__end, struct sonode *, so);
2363
Anders Persson41174432009-02-12 17:35:05 -08002364 if (error != 0) {
2365 /* protocol was unable to do a fallback, revert the sonode */
2366 sotpi_revert_sonode(so, cr);
2367 goto out;
2368 }
2369
Yu Xiangning0f1702c2008-12-11 20:04:13 -08002370 /*
Anders Persson41174432009-02-12 17:35:05 -08002371 * Walk the accept queue and notify the proto that they should
2372 * fall back to TPI. The protocol will send up the T_CONN_IND.
2373 */
Anders Persson3e95bd42010-06-17 17:22:09 -07002374 nso = list_head(&so->so_acceptq_list);
Anders Persson41174432009-02-12 17:35:05 -08002375 while (nso != NULL) {
2376 int rval;
Anders Persson3e95bd42010-06-17 17:22:09 -07002377 struct sonode *next;
2378
2379 if (arg.soqa_exdata_mp == NULL) {
2380 arg.soqa_exdata_mp =
2381 allocb_wait(sizeof (struct T_exdata_ind),
2382 BPRI_MED, STR_NOSIG, NULL);
2383 }
2384 if (arg.soqa_urgmark_mp == NULL) {
2385 arg.soqa_urgmark_mp = allocb_wait(0, BPRI_MED,
2386 STR_NOSIG, NULL);
2387 }
Anders Persson41174432009-02-12 17:35:05 -08002388
2389 DTRACE_PROBE1(proto__fallback__begin, struct sonode *, nso);
Anders Persson3e95bd42010-06-17 17:22:09 -07002390 rval = (*fbfunc)(nso->so_proto_handle, NULL, direct,
2391 so_quiesced_cb, &arg);
Anders Persson41174432009-02-12 17:35:05 -08002392 DTRACE_PROBE1(proto__fallback__end, struct sonode *, nso);
2393 if (rval != 0) {
Anders Persson3e95bd42010-06-17 17:22:09 -07002394 /* Abort the connection */
Anders Persson41174432009-02-12 17:35:05 -08002395 zcmn_err(getzoneid(), CE_WARN,
2396 "Failed to convert socket in accept queue to TPI. "
2397 "Pid = %d\n", curproc->p_pid);
Anders Persson3e95bd42010-06-17 17:22:09 -07002398 next = list_next(&so->so_acceptq_list, nso);
2399 list_remove(&so->so_acceptq_list, nso);
2400 so->so_acceptq_len--;
2401
2402 (void) socket_close(nso, 0, CRED());
2403 socket_destroy(nso);
2404 nso = next;
2405 } else {
2406 nso = list_next(&so->so_acceptq_list, nso);
Anders Persson41174432009-02-12 17:35:05 -08002407 }
Anders Persson41174432009-02-12 17:35:05 -08002408 }
2409
2410 /*
2411 * Now flush the acceptq, this will destroy all sockets. They will
2412 * be recreated in sotpi_accept().
Yu Xiangning0f1702c2008-12-11 20:04:13 -08002413 */
Anders Persson2320a8c2009-10-21 19:52:57 -07002414 so_acceptq_flush(so, B_FALSE);
Yu Xiangning0f1702c2008-12-11 20:04:13 -08002415
2416 mutex_enter(&so->so_lock);
2417 so->so_state |= SS_FALLBACK_COMP;
2418 mutex_exit(&so->so_lock);
2419
2420 /*
2421 * Swap the sonode ops. Socket opertations that come in once this
2422 * is done will proceed without blocking.
2423 */
2424 so->so_ops = &sotpi_sonodeops;
2425
2426 /*
Yu Xiangning0f1702c2008-12-11 20:04:13 -08002427 * Wake up any threads stuck in poll. This is needed since the poll
2428 * head changes when the fallback happens (moves from the sonode to
2429 * the STREAMS head).
2430 */
2431 pollwakeup(&so->so_poll_list, POLLERR);
Anders Persson3e95bd42010-06-17 17:22:09 -07002432
2433 /*
2434 * When this non-STREAM socket was created we placed an extra ref on
2435 * the associated vnode to support asynchronous close. Drop that ref
2436 * here.
2437 */
2438 ASSERT(SOTOV(so)->v_count >= 2);
2439 VN_RELE(SOTOV(so));
Yu Xiangning0f1702c2008-12-11 20:04:13 -08002440out:
2441 so_end_fallback(so);
2442
Anders Persson41174432009-02-12 17:35:05 -08002443 if (error != 0) {
2444#ifdef DEBUG
2445 so_integrity_check(so, &origso);
2446#endif
2447 zcmn_err(getzoneid(), CE_WARN,
2448 "Failed to convert socket to TPI (err=%d). Pid = %d\n",
2449 error, curproc->p_pid);
2450 if (newsp != NULL)
2451 SOCKPARAMS_DEC_REF(newsp);
2452 }
Anders Persson3e95bd42010-06-17 17:22:09 -07002453 if (arg.soqa_exdata_mp != NULL)
2454 freemsg(arg.soqa_exdata_mp);
2455 if (arg.soqa_urgmark_mp != NULL)
2456 freemsg(arg.soqa_urgmark_mp);
Anders Persson41174432009-02-12 17:35:05 -08002457
Yu Xiangning0f1702c2008-12-11 20:04:13 -08002458 return (error);
2459}
Robert Mustacchi907c2822021-02-01 13:50:02 +00002460
2461int
2462so_krecv_set(sonode_t *so, so_krecv_f cb, void *arg)
2463{
2464 int ret;
2465
2466 if (cb == NULL && arg != NULL)
2467 return (EINVAL);
2468
2469 SO_BLOCK_FALLBACK(so, so_krecv_set(so, cb, arg));
2470
2471 mutex_enter(&so->so_lock);
2472 if (so->so_state & SS_FALLBACK_COMP) {
2473 mutex_exit(&so->so_lock);
2474 SO_UNBLOCK_FALLBACK(so);
2475 return (ENOTSUP);
2476 }
2477
2478 ret = so_lock_read(so, 0);
2479