Line data Source code
1 : /*
2 : *
3 : * Copyright 2015, Google Inc.
4 : * All rights reserved.
5 : *
6 : * Redistribution and use in source and binary forms, with or without
7 : * modification, are permitted provided that the following conditions are
8 : * met:
9 : *
10 : * * Redistributions of source code must retain the above copyright
11 : * notice, this list of conditions and the following disclaimer.
12 : * * Redistributions in binary form must reproduce the above
13 : * copyright notice, this list of conditions and the following disclaimer
14 : * in the documentation and/or other materials provided with the
15 : * distribution.
16 : * * Neither the name of Google Inc. nor the names of its
17 : * contributors may be used to endorse or promote products derived from
18 : * this software without specific prior written permission.
19 : *
20 : * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
21 : * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
22 : * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
23 : * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
24 : * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
25 : * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
26 : * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
27 : * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
28 : * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
29 : * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
30 : * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
31 : *
32 : */
33 :
34 : #include <grpc/support/port_platform.h>
35 :
36 : #ifdef GPR_POSIX_SOCKET
37 :
38 : #include "src/core/iomgr/fd_posix.h"
39 :
40 : #include <assert.h>
41 : #include <sys/socket.h>
42 : #include <unistd.h>
43 :
44 : #include <grpc/support/alloc.h>
45 : #include <grpc/support/log.h>
46 : #include <grpc/support/useful.h>
47 :
48 : #define CLOSURE_NOT_READY ((grpc_closure *)0)
49 : #define CLOSURE_READY ((grpc_closure *)1)
50 :
51 : /* We need to keep a freelist not because of any concerns of malloc performance
52 : * but instead so that implementations with multiple threads in (for example)
53 : * epoll_wait deal with the race between pollset removal and incoming poll
54 : * notifications.
55 : *
56 : * The problem is that the poller ultimately holds a reference to this
57 : * object, so it is very difficult to know when is safe to free it, at least
58 : * without some expensive synchronization.
59 : *
60 : * If we keep the object freelisted, in the worst case losing this race just
61 : * becomes a spurious read notification on a reused fd.
62 : */
63 : /* TODO(klempner): We could use some form of polling generation count to know
64 : * when these are safe to free. */
65 : /* TODO(klempner): Consider disabling freelisting if we don't have multiple
66 : * threads in poll on the same fd */
67 : /* TODO(klempner): Batch these allocations to reduce fragmentation */
68 : static grpc_fd *fd_freelist = NULL;
69 : static gpr_mu fd_freelist_mu;
70 :
71 9605 : static void freelist_fd(grpc_fd *fd) {
72 9605 : gpr_mu_lock(&fd_freelist_mu);
73 9606 : fd->freelist_next = fd_freelist;
74 9606 : fd_freelist = fd;
75 9606 : grpc_iomgr_unregister_object(&fd->iomgr_object);
76 9606 : gpr_mu_unlock(&fd_freelist_mu);
77 9599 : }
78 :
79 9606 : static grpc_fd *alloc_fd(int fd) {
80 9606 : grpc_fd *r = NULL;
81 9606 : gpr_mu_lock(&fd_freelist_mu);
82 9606 : if (fd_freelist != NULL) {
83 5276 : r = fd_freelist;
84 5276 : fd_freelist = fd_freelist->freelist_next;
85 : }
86 9606 : gpr_mu_unlock(&fd_freelist_mu);
87 9606 : if (r == NULL) {
88 4330 : r = gpr_malloc(sizeof(grpc_fd));
89 4330 : gpr_mu_init(&r->mu);
90 : }
91 :
92 9606 : gpr_atm_rel_store(&r->refst, 1);
93 9606 : r->shutdown = 0;
94 9606 : r->read_closure = CLOSURE_NOT_READY;
95 9606 : r->write_closure = CLOSURE_NOT_READY;
96 9606 : r->fd = fd;
97 9606 : r->inactive_watcher_root.next = r->inactive_watcher_root.prev =
98 9606 : &r->inactive_watcher_root;
99 9606 : r->freelist_next = NULL;
100 9606 : r->read_watcher = r->write_watcher = NULL;
101 9606 : r->on_done_closure = NULL;
102 9606 : r->closed = 0;
103 9606 : return r;
104 : }
105 :
106 4330 : static void destroy(grpc_fd *fd) {
107 4330 : gpr_mu_destroy(&fd->mu);
108 4330 : gpr_free(fd);
109 4330 : }
110 :
111 : #ifdef GRPC_FD_REF_COUNT_DEBUG
112 : #define REF_BY(fd, n, reason) ref_by(fd, n, reason, __FILE__, __LINE__)
113 : #define UNREF_BY(fd, n, reason) unref_by(fd, n, reason, __FILE__, __LINE__)
114 : static void ref_by(grpc_fd *fd, int n, const char *reason, const char *file,
115 : int line) {
116 : gpr_log(GPR_DEBUG, "FD %d %p ref %d %d -> %d [%s; %s:%d]", fd->fd, fd, n,
117 : gpr_atm_no_barrier_load(&fd->refst),
118 : gpr_atm_no_barrier_load(&fd->refst) + n, reason, file, line);
119 : #else
120 : #define REF_BY(fd, n, reason) ref_by(fd, n)
121 : #define UNREF_BY(fd, n, reason) unref_by(fd, n)
122 3948923 : static void ref_by(grpc_fd *fd, int n) {
123 : #endif
124 3948923 : GPR_ASSERT(gpr_atm_no_barrier_fetch_add(&fd->refst, n) > 0);
125 3948923 : }
126 :
127 : #ifdef GRPC_FD_REF_COUNT_DEBUG
128 : static void unref_by(grpc_fd *fd, int n, const char *reason, const char *file,
129 : int line) {
130 : gpr_atm old;
131 : gpr_log(GPR_DEBUG, "FD %d %p unref %d %d -> %d [%s; %s:%d]", fd->fd, fd, n,
132 : gpr_atm_no_barrier_load(&fd->refst),
133 : gpr_atm_no_barrier_load(&fd->refst) - n, reason, file, line);
134 : #else
135 3949016 : static void unref_by(grpc_fd *fd, int n) {
136 : gpr_atm old;
137 : #endif
138 3949016 : old = gpr_atm_full_fetch_add(&fd->refst, -n);
139 3949016 : if (old == n) {
140 9603 : freelist_fd(fd);
141 : } else {
142 3939413 : GPR_ASSERT(old > n);
143 : }
144 3949012 : }
145 :
146 2506 : void grpc_fd_global_init(void) { gpr_mu_init(&fd_freelist_mu); }
147 :
148 2506 : void grpc_fd_global_shutdown(void) {
149 2506 : gpr_mu_lock(&fd_freelist_mu);
150 2506 : gpr_mu_unlock(&fd_freelist_mu);
151 9342 : while (fd_freelist != NULL) {
152 4330 : grpc_fd *fd = fd_freelist;
153 4330 : fd_freelist = fd_freelist->freelist_next;
154 4330 : destroy(fd);
155 : }
156 2506 : gpr_mu_destroy(&fd_freelist_mu);
157 2506 : }
158 :
159 9606 : grpc_fd *grpc_fd_create(int fd, const char *name) {
160 9606 : grpc_fd *r = alloc_fd(fd);
161 9606 : grpc_iomgr_register_object(&r->iomgr_object, name);
162 : #ifdef GRPC_FD_REF_COUNT_DEBUG
163 : gpr_log(GPR_DEBUG, "FD %d %p create %s", fd, r, name);
164 : #endif
165 9606 : return r;
166 : }
167 :
168 3626854 : int grpc_fd_is_orphaned(grpc_fd *fd) {
169 3626854 : return (gpr_atm_acq_load(&fd->refst) & 1) == 0;
170 : }
171 :
172 115610 : static void pollset_kick_locked(grpc_fd_watcher *watcher) {
173 115610 : gpr_mu_lock(GRPC_POLLSET_MU(watcher->pollset));
174 115614 : GPR_ASSERT(watcher->worker);
175 115614 : grpc_pollset_kick_ext(watcher->pollset, watcher->worker,
176 : GRPC_POLLSET_REEVALUATE_POLLING_ON_WAKEUP);
177 115613 : gpr_mu_unlock(GRPC_POLLSET_MU(watcher->pollset));
178 115614 : }
179 :
180 891978 : static void maybe_wake_one_watcher_locked(grpc_fd *fd) {
181 891978 : if (fd->inactive_watcher_root.next != &fd->inactive_watcher_root) {
182 115157 : pollset_kick_locked(fd->inactive_watcher_root.next);
183 776821 : } else if (fd->read_watcher) {
184 407 : pollset_kick_locked(fd->read_watcher);
185 776414 : } else if (fd->write_watcher) {
186 0 : pollset_kick_locked(fd->write_watcher);
187 : }
188 891982 : }
189 :
190 46 : static void wake_all_watchers_locked(grpc_fd *fd) {
191 : grpc_fd_watcher *watcher;
192 94 : for (watcher = fd->inactive_watcher_root.next;
193 2 : watcher != &fd->inactive_watcher_root; watcher = watcher->next) {
194 2 : pollset_kick_locked(watcher);
195 : }
196 46 : if (fd->read_watcher) {
197 44 : pollset_kick_locked(fd->read_watcher);
198 : }
199 46 : if (fd->write_watcher && fd->write_watcher != fd->read_watcher) {
200 0 : pollset_kick_locked(fd->write_watcher);
201 : }
202 46 : }
203 :
204 9652 : static int has_watchers(grpc_fd *fd) {
205 19260 : return fd->read_watcher != NULL || fd->write_watcher != NULL ||
206 9608 : fd->inactive_watcher_root.next != &fd->inactive_watcher_root;
207 : }
208 :
209 9606 : void grpc_fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd, grpc_closure *on_done,
210 : const char *reason) {
211 9606 : fd->on_done_closure = on_done;
212 9606 : shutdown(fd->fd, SHUT_RDWR);
213 9606 : gpr_mu_lock(&fd->mu);
214 9606 : REF_BY(fd, 1, reason); /* remove active status, but keep referenced */
215 9606 : if (!has_watchers(fd)) {
216 9560 : fd->closed = 1;
217 9560 : close(fd->fd);
218 9559 : grpc_exec_ctx_enqueue(exec_ctx, fd->on_done_closure, 1);
219 : } else {
220 46 : wake_all_watchers_locked(fd);
221 : }
222 9605 : gpr_mu_unlock(&fd->mu);
223 9606 : UNREF_BY(fd, 2, reason); /* drop the reference */
224 9599 : }
225 :
226 : /* increment refcount by two to avoid changing the orphan bit */
227 : #ifdef GRPC_FD_REF_COUNT_DEBUG
228 : void grpc_fd_ref(grpc_fd *fd, const char *reason, const char *file, int line) {
229 : ref_by(fd, 2, reason, file, line);
230 : }
231 :
232 : void grpc_fd_unref(grpc_fd *fd, const char *reason, const char *file,
233 : int line) {
234 : unref_by(fd, 2, reason, file, line);
235 : }
236 : #else
237 3938565 : void grpc_fd_ref(grpc_fd *fd) { ref_by(fd, 2); }
238 :
239 3938902 : void grpc_fd_unref(grpc_fd *fd) { unref_by(fd, 2); }
240 : #endif
241 :
242 1232808 : static void notify_on_locked(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
243 : grpc_closure **st, grpc_closure *closure) {
244 1232808 : if (*st == CLOSURE_NOT_READY) {
245 : /* not ready ==> switch to a waiting state by setting the closure */
246 1156712 : *st = closure;
247 76096 : } else if (*st == CLOSURE_READY) {
248 : /* already ready ==> queue the closure to run immediately */
249 76096 : *st = CLOSURE_NOT_READY;
250 76096 : grpc_exec_ctx_enqueue(exec_ctx, closure, !fd->shutdown);
251 76096 : maybe_wake_one_watcher_locked(fd);
252 : } else {
253 : /* upcallptr was set to a different closure. This is an error! */
254 0 : gpr_log(GPR_ERROR,
255 : "User called a notify_on function with a previous callback still "
256 : "pending");
257 0 : abort();
258 : }
259 1232808 : }
260 :
261 : /* returns 1 if state becomes not ready */
262 3356640 : static int set_ready_locked(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
263 : grpc_closure **st) {
264 3356640 : if (*st == CLOSURE_READY) {
265 : /* duplicate ready ==> ignore */
266 2113595 : return 0;
267 1243045 : } else if (*st == CLOSURE_NOT_READY) {
268 : /* not ready, and not waiting ==> flag ready */
269 86268 : *st = CLOSURE_READY;
270 86268 : return 0;
271 : } else {
272 : /* waiting ==> queue closure */
273 1156777 : grpc_exec_ctx_enqueue(exec_ctx, *st, !fd->shutdown);
274 1156769 : *st = CLOSURE_NOT_READY;
275 1156769 : return 1;
276 : }
277 : }
278 :
279 2565580 : static void set_ready(grpc_exec_ctx *exec_ctx, grpc_fd *fd, grpc_closure **st) {
280 : /* only one set_ready can be active at once (but there may be a racing
281 : notify_on) */
282 2565580 : gpr_mu_lock(&fd->mu);
283 2567274 : set_ready_locked(exec_ctx, fd, st);
284 2567274 : gpr_mu_unlock(&fd->mu);
285 2567113 : }
286 :
287 6102 : void grpc_fd_shutdown(grpc_exec_ctx *exec_ctx, grpc_fd *fd) {
288 6102 : gpr_mu_lock(&fd->mu);
289 6102 : GPR_ASSERT(!gpr_atm_no_barrier_load(&fd->shutdown));
290 6102 : fd->shutdown = 1;
291 6102 : set_ready_locked(exec_ctx, fd, &fd->read_closure);
292 6101 : set_ready_locked(exec_ctx, fd, &fd->write_closure);
293 6102 : gpr_mu_unlock(&fd->mu);
294 6102 : }
295 :
296 1226470 : void grpc_fd_notify_on_read(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
297 : grpc_closure *closure) {
298 1226470 : gpr_mu_lock(&fd->mu);
299 1226671 : notify_on_locked(exec_ctx, fd, &fd->read_closure, closure);
300 1226669 : gpr_mu_unlock(&fd->mu);
301 1226670 : }
302 :
303 6136 : void grpc_fd_notify_on_write(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
304 : grpc_closure *closure) {
305 6136 : gpr_mu_lock(&fd->mu);
306 6136 : notify_on_locked(exec_ctx, fd, &fd->write_closure, closure);
307 6136 : gpr_mu_unlock(&fd->mu);
308 6136 : }
309 :
310 2501401 : gpr_uint32 grpc_fd_begin_poll(grpc_fd *fd, grpc_pollset *pollset,
311 : grpc_pollset_worker *worker, gpr_uint32 read_mask,
312 : gpr_uint32 write_mask, grpc_fd_watcher *watcher) {
313 2501401 : gpr_uint32 mask = 0;
314 : grpc_closure *cur;
315 : int requested;
316 : /* keep track of pollers that have requested our events, in case they change
317 : */
318 2501401 : GRPC_FD_REF(fd, "poll");
319 :
320 2501876 : gpr_mu_lock(&fd->mu);
321 :
322 : /* if we are shutdown, then don't add to the watcher set */
323 2501672 : if (gpr_atm_no_barrier_load(&fd->shutdown)) {
324 10 : watcher->fd = NULL;
325 10 : watcher->pollset = NULL;
326 10 : watcher->worker = NULL;
327 10 : gpr_mu_unlock(&fd->mu);
328 10 : GRPC_FD_UNREF(fd, "poll");
329 10 : return 0;
330 : }
331 :
332 : /* if there is nobody polling for read, but we need to, then start doing so */
333 2501662 : cur = fd->read_closure;
334 2501662 : requested = cur != CLOSURE_READY;
335 2501662 : if (read_mask && fd->read_watcher == NULL && requested) {
336 821458 : fd->read_watcher = watcher;
337 821458 : mask |= read_mask;
338 : }
339 : /* if there is nobody polling for write, but we need to, then start doing so
340 : */
341 2501662 : cur = fd->write_closure;
342 2501662 : requested = cur != CLOSURE_READY;
343 2501662 : if (write_mask && fd->write_watcher == NULL && requested) {
344 10735 : fd->write_watcher = watcher;
345 10735 : mask |= write_mask;
346 : }
347 : /* if not polling, remember this watcher in case we need someone to later */
348 2501662 : if (mask == 0 && worker != NULL) {
349 282604 : watcher->next = &fd->inactive_watcher_root;
350 282604 : watcher->prev = watcher->next->prev;
351 282604 : watcher->next->prev = watcher->prev->next = watcher;
352 : }
353 2501662 : watcher->pollset = pollset;
354 2501662 : watcher->worker = worker;
355 2501662 : watcher->fd = fd;
356 2501662 : gpr_mu_unlock(&fd->mu);
357 :
358 2501864 : return mask;
359 : }
360 :
361 2501249 : void grpc_fd_end_poll(grpc_exec_ctx *exec_ctx, grpc_fd_watcher *watcher,
362 : int got_read, int got_write) {
363 2501249 : int was_polling = 0;
364 2501249 : int kick = 0;
365 2501249 : grpc_fd *fd = watcher->fd;
366 :
367 2501249 : if (fd == NULL) {
368 2501906 : return;
369 : }
370 :
371 2501239 : gpr_mu_lock(&fd->mu);
372 :
373 2501639 : if (watcher == fd->read_watcher) {
374 : /* remove read watcher, kick if we still need a read */
375 821455 : was_polling = 1;
376 821455 : if (!got_read) {
377 51617 : kick = 1;
378 : }
379 821455 : fd->read_watcher = NULL;
380 : }
381 2501639 : if (watcher == fd->write_watcher) {
382 : /* remove write watcher, kick if we still need a write */
383 10735 : was_polling = 1;
384 10735 : if (!got_write) {
385 3484 : kick = 1;
386 : }
387 10735 : fd->write_watcher = NULL;
388 : }
389 2501639 : if (!was_polling && watcher->worker != NULL) {
390 : /* remove from inactive list */
391 282583 : watcher->next->prev = watcher->prev;
392 282583 : watcher->prev->next = watcher->next;
393 : }
394 2501639 : if (got_read) {
395 769841 : if (set_ready_locked(exec_ctx, fd, &fd->read_closure)) {
396 764088 : kick = 1;
397 : }
398 : }
399 2501635 : if (got_write) {
400 7283 : if (set_ready_locked(exec_ctx, fd, &fd->write_closure)) {
401 2906 : kick = 1;
402 : }
403 : }
404 2501635 : if (kick) {
405 815883 : maybe_wake_one_watcher_locked(fd);
406 : }
407 2501630 : if (grpc_fd_is_orphaned(fd) && !has_watchers(fd) && !fd->closed) {
408 46 : fd->closed = 1;
409 46 : close(fd->fd);
410 46 : grpc_exec_ctx_enqueue(exec_ctx, fd->on_done_closure, 1);
411 : }
412 2501491 : gpr_mu_unlock(&fd->mu);
413 :
414 2501384 : GRPC_FD_UNREF(fd, "poll");
415 : }
416 :
417 1020963 : void grpc_fd_become_readable(grpc_exec_ctx *exec_ctx, grpc_fd *fd) {
418 1020963 : set_ready(exec_ctx, fd, &fd->read_closure);
419 1021269 : }
420 :
421 1545936 : void grpc_fd_become_writable(grpc_exec_ctx *exec_ctx, grpc_fd *fd) {
422 1545936 : set_ready(exec_ctx, fd, &fd->write_closure);
423 1546001 : }
424 :
425 : #endif
|
