Bug Summary

File:cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c
Warning:line 2474, column 7
Copies out a struct with uncleared padding (>= 3 bytes)

Annotated Source Code

1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 *
21 * Portions Copyright 2010 The FreeBSD Foundation
22 *
23 * $FreeBSD: releng/11.0/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c 296479 2016-03-08 00:43:03Z markj $
24 */
25
26/*
27 * Copyright 2008 Sun Microsystems, Inc. All rights reserved.
28 * Use is subject to license terms.
29 */
30
31/*
32 * Copyright (c) 2015, Joyent, Inc. All rights reserved.
33 */
34
35#include <sys/atomic.h>
36#include <sys/errno.h>
37#include <sys/stat.h>
38#include <sys/modctl.h>
39#include <sys/conf.h>
40#include <sys/systm.h>
41#ifdef illumos
42#include <sys/ddi.h>
43#endif
44#include <sys/sunddi.h>
45#include <sys/cpuvar.h>
46#include <sys/kmem.h>
47#ifdef illumos
48#include <sys/strsubr.h>
49#endif
50#include <sys/fasttrap.h>
51#include <sys/fasttrap_impl.h>
52#include <sys/fasttrap_isa.h>
53#include <sys/dtrace.h>
54#include <sys/dtrace_impl.h>
55#include <sys/sysmacros.h>
56#include <sys/proc.h>
57#include <sys/policy.h>
58#ifdef illumos
59#include <util/qsort.h>
60#endif
61#include <sys/mutex.h>
62#include <sys/kernel.h>
63#ifndef illumos
64#include <sys/dtrace_bsd.h>
65#include <sys/eventhandler.h>
66#include <sys/rmlock.h>
67#include <sys/sysctl.h>
68#include <sys/u8_textprep.h>
69#include <sys/user.h>
70
71#include <vm/vm.h>
72#include <vm/pmap.h>
73#include <vm/vm_map.h>
74#include <vm/vm_param.h>
75
76#include <cddl/dev/dtrace/dtrace_cddl.h>
77#endif
78
79/*
80 * User-Land Trap-Based Tracing
81 * ----------------------------
82 *
83 * The fasttrap provider allows DTrace consumers to instrument any user-level
84 * instruction to gather data; this includes probes with semantic
85 * signifigance like entry and return as well as simple offsets into the
86 * function. While the specific techniques used are very ISA specific, the
87 * methodology is generalizable to any architecture.
88 *
89 *
90 * The General Methodology
91 * -----------------------
92 *
93 * With the primary goal of tracing every user-land instruction and the
94 * limitation that we can't trust user space so don't want to rely on much
95 * information there, we begin by replacing the instructions we want to trace
96 * with trap instructions. Each instruction we overwrite is saved into a hash
97 * table keyed by process ID and pc address. When we enter the kernel due to
98 * this trap instruction, we need the effects of the replaced instruction to
99 * appear to have occurred before we proceed with the user thread's
100 * execution.
101 *
102 * Each user level thread is represented by a ulwp_t structure which is
103 * always easily accessible through a register. The most basic way to produce
104 * the effects of the instruction we replaced is to copy that instruction out
105 * to a bit of scratch space reserved in the user thread's ulwp_t structure
106 * (a sort of kernel-private thread local storage), set the PC to that
107 * scratch space and single step. When we reenter the kernel after single
108 * stepping the instruction we must then adjust the PC to point to what would
109 * normally be the next instruction. Of course, special care must be taken
110 * for branches and jumps, but these represent such a small fraction of any
111 * instruction set that writing the code to emulate these in the kernel is
112 * not too difficult.
113 *
114 * Return probes may require several tracepoints to trace every return site,
115 * and, conversely, each tracepoint may activate several probes (the entry
116 * and offset 0 probes, for example). To solve this muliplexing problem,
117 * tracepoints contain lists of probes to activate and probes contain lists
118 * of tracepoints to enable. If a probe is activated, it adds its ID to
119 * existing tracepoints or creates new ones as necessary.
120 *
121 * Most probes are activated _before_ the instruction is executed, but return
122 * probes are activated _after_ the effects of the last instruction of the
123 * function are visible. Return probes must be fired _after_ we have
124 * single-stepped the instruction whereas all other probes are fired
125 * beforehand.
126 *
127 *
128 * Lock Ordering
129 * -------------
130 *
131 * The lock ordering below -- both internally and with respect to the DTrace
132 * framework -- is a little tricky and bears some explanation. Each provider
133 * has a lock (ftp_mtx) that protects its members including reference counts
134 * for enabled probes (ftp_rcount), consumers actively creating probes
135 * (ftp_ccount) and USDT consumers (ftp_mcount); all three prevent a provider
136 * from being freed. A provider is looked up by taking the bucket lock for the
137 * provider hash table, and is returned with its lock held. The provider lock
138 * may be taken in functions invoked by the DTrace framework, but may not be
139 * held while calling functions in the DTrace framework.
140 *
141 * To ensure consistency over multiple calls to the DTrace framework, the
142 * creation lock (ftp_cmtx) should be held. Naturally, the creation lock may
143 * not be taken when holding the provider lock as that would create a cyclic
144 * lock ordering. In situations where one would naturally take the provider
145 * lock and then the creation lock, we instead up a reference count to prevent
146 * the provider from disappearing, drop the provider lock, and acquire the
147 * creation lock.
148 *
149 * Briefly:
150 * bucket lock before provider lock
151 * DTrace before provider lock
152 * creation lock before DTrace
153 * never hold the provider lock and creation lock simultaneously
154 */
155
156static d_open_t fasttrap_open;
157static d_ioctl_t fasttrap_ioctl;
158
159static struct cdevsw fasttrap_cdevsw = {
160 .d_version = D_VERSION0x17122009,
161 .d_open = fasttrap_open,
162 .d_ioctl = fasttrap_ioctl,
163 .d_name = "fasttrap",
164};
165static struct cdev *fasttrap_cdev;
166static dtrace_meta_provider_id_t fasttrap_meta_id;
167
168static struct proc *fasttrap_cleanup_proc;
169static struct mtx fasttrap_cleanup_mtx;
170static uint_t fasttrap_cleanup_work, fasttrap_cleanup_drain, fasttrap_cleanup_cv;
171
172/*
173 * Generation count on modifications to the global tracepoint lookup table.
174 */
175static volatile uint64_t fasttrap_mod_gen;
176
177/*
178 * When the fasttrap provider is loaded, fasttrap_max is set to either
179 * FASTTRAP_MAX_DEFAULT, or the value for fasttrap-max-probes in the
180 * fasttrap.conf file (Illumos), or the value provied in the loader.conf (FreeBSD).
181 * Each time a probe is created, fasttrap_total is incremented by the number
182 * of tracepoints that may be associated with that probe; fasttrap_total is capped
183 * at fasttrap_max.
184 */
185#define FASTTRAP_MAX_DEFAULT250000 250000
186static uint32_t fasttrap_max = FASTTRAP_MAX_DEFAULT250000;
187static uint32_t fasttrap_total;
188
189/*
190 * Copyright (c) 2011, Joyent, Inc. All rights reserved.
191 */
192
193#define FASTTRAP_TPOINTS_DEFAULT_SIZE0x4000 0x4000
194#define FASTTRAP_PROVIDERS_DEFAULT_SIZE0x100 0x100
195#define FASTTRAP_PROCS_DEFAULT_SIZE0x100 0x100
196
197#define FASTTRAP_PID_NAME"pid" "pid"
198
199fasttrap_hash_t fasttrap_tpoints;
200static fasttrap_hash_t fasttrap_provs;
201static fasttrap_hash_t fasttrap_procs;
202
203static uint64_t fasttrap_pid_count; /* pid ref count */
204static kmutex_t fasttrap_count_mtx; /* lock on ref count */
205
206#define FASTTRAP_ENABLE_FAIL1 1
207#define FASTTRAP_ENABLE_PARTIAL2 2
208
209static int fasttrap_tracepoint_enable(proc_t *, fasttrap_probe_t *, uint_t);
210static void fasttrap_tracepoint_disable(proc_t *, fasttrap_probe_t *, uint_t);
211
212static fasttrap_provider_t *fasttrap_provider_lookup(pid_t, const char *,
213 const dtrace_pattr_t *);
214static void fasttrap_provider_retire(pid_t, const char *, int);
215static void fasttrap_provider_free(fasttrap_provider_t *);
216
217static fasttrap_proc_t *fasttrap_proc_lookup(pid_t);
218static void fasttrap_proc_release(fasttrap_proc_t *);
219
220#ifndef illumos
221static void fasttrap_thread_dtor(void *, struct thread *);
222#endif
223
224#define FASTTRAP_PROVS_INDEX(pid, name)((fasttrap_hash_str(name) + (pid)) & fasttrap_provs.fth_mask
) \
225 ((fasttrap_hash_str(name) + (pid)) & fasttrap_provs.fth_mask)
226
227#define FASTTRAP_PROCS_INDEX(pid)((pid) & fasttrap_procs.fth_mask) ((pid) & fasttrap_procs.fth_mask)
228
229#ifndef illumos
230struct rmlock fasttrap_tp_lock;
231static eventhandler_tag fasttrap_thread_dtor_tag;
232#endif
233
234static unsigned long tpoints_hash_size = FASTTRAP_TPOINTS_DEFAULT_SIZE0x4000;
235
236#ifdef __FreeBSD__11
237SYSCTL_DECL(_kern_dtrace)extern struct sysctl_oid sysctl___kern_dtrace;
238SYSCTL_NODE(_kern_dtrace, OID_AUTO, fasttrap, CTLFLAG_RD, 0, "DTrace fasttrap parameters")struct sysctl_oid sysctl___kern_dtrace_fasttrap = { .oid_parent
= ((&(&sysctl___kern_dtrace)->oid_children)), .oid_children
= { ((void *)0) }, .oid_number = ((-1)), .oid_kind = (1|(0x80000000
)), .oid_arg1 = (((void *)0)), .oid_arg2 = (0), .oid_name = (
"fasttrap"), .oid_handler = (0), .oid_fmt = ("N"), .oid_descr
= "DTrace fasttrap parameters" }; __asm__(".globl " "__start_set_sysctl_set"
); __asm__(".globl " "__stop_set_sysctl_set"); static void const
* const __set_sysctl_set_sym_sysctl___kern_dtrace_fasttrap __attribute__
((__section__("set_" "sysctl_set"))) __attribute__((__used__)
) = &(sysctl___kern_dtrace_fasttrap); _Static_assert(((0x80000000
) & 0xf) == 0 || ((0x80000000) & 0xf) == 1, "compile-time assertion failed"
);
239SYSCTL_UINT(_kern_dtrace_fasttrap, OID_AUTO, max_probes, CTLFLAG_RWTUN, &fasttrap_max,static struct sysctl_oid sysctl___kern_dtrace_fasttrap_max_probes
= { .oid_parent = ((&(&sysctl___kern_dtrace_fasttrap
)->oid_children)), .oid_children = { ((void *)0) }, .oid_number
= ((-1)), .oid_kind = (6 | 0x00040000 | (((0x80000000|0x40000000
)|0x00080000))), .oid_arg1 = (&fasttrap_max), .oid_arg2 =
(250000), .oid_name = ("max_probes"), .oid_handler = (sysctl_handle_int
), .oid_fmt = ("IU"), .oid_descr = "Maximum number of fasttrap probes"
}; __asm__(".globl " "__start_set_sysctl_set"); __asm__(".globl "
"__stop_set_sysctl_set"); static void const * const __set_sysctl_set_sym_sysctl___kern_dtrace_fasttrap_max_probes
__attribute__((__section__("set_" "sysctl_set"))) __attribute__
((__used__)) = &(sysctl___kern_dtrace_fasttrap_max_probes
); _Static_assert((((((0x80000000|0x40000000)|0x00080000)) &
0xf) == 0 || ((((0x80000000|0x40000000)|0x00080000)) & 0xf
) == 6) && sizeof(unsigned) == sizeof(*(&fasttrap_max
)), "compile-time assertion failed")
240 FASTTRAP_MAX_DEFAULT, "Maximum number of fasttrap probes")static struct sysctl_oid sysctl___kern_dtrace_fasttrap_max_probes
= { .oid_parent = ((&(&sysctl___kern_dtrace_fasttrap
)->oid_children)), .oid_children = { ((void *)0) }, .oid_number
= ((-1)), .oid_kind = (6 | 0x00040000 | (((0x80000000|0x40000000
)|0x00080000))), .oid_arg1 = (&fasttrap_max), .oid_arg2 =
(250000), .oid_name = ("max_probes"), .oid_handler = (sysctl_handle_int
), .oid_fmt = ("IU"), .oid_descr = "Maximum number of fasttrap probes"
}; __asm__(".globl " "__start_set_sysctl_set"); __asm__(".globl "
"__stop_set_sysctl_set"); static void const * const __set_sysctl_set_sym_sysctl___kern_dtrace_fasttrap_max_probes
__attribute__((__section__("set_" "sysctl_set"))) __attribute__
((__used__)) = &(sysctl___kern_dtrace_fasttrap_max_probes
); _Static_assert((((((0x80000000|0x40000000)|0x00080000)) &
0xf) == 0 || ((((0x80000000|0x40000000)|0x00080000)) & 0xf
) == 6) && sizeof(unsigned) == sizeof(*(&fasttrap_max
)), "compile-time assertion failed");
241SYSCTL_ULONG(_kern_dtrace_fasttrap, OID_AUTO, tpoints_hash_size, CTLFLAG_RDTUN, &tpoints_hash_size,static struct sysctl_oid sysctl___kern_dtrace_fasttrap_tpoints_hash_size
= { .oid_parent = ((&(&sysctl___kern_dtrace_fasttrap
)->oid_children)), .oid_children = { ((void *)0) }, .oid_number
= ((-1)), .oid_kind = (8 | 0x00040000 | ((0x80000000|0x00080000
))), .oid_arg1 = (&tpoints_hash_size), .oid_arg2 = (0x4000
), .oid_name = ("tpoints_hash_size"), .oid_handler = (sysctl_handle_long
), .oid_fmt = ("LU"), .oid_descr = "Size of the tracepoint hash table"
}; __asm__(".globl " "__start_set_sysctl_set"); __asm__(".globl "
"__stop_set_sysctl_set"); static void const * const __set_sysctl_set_sym_sysctl___kern_dtrace_fasttrap_tpoints_hash_size
__attribute__((__section__("set_" "sysctl_set"))) __attribute__
((__used__)) = &(sysctl___kern_dtrace_fasttrap_tpoints_hash_size
); _Static_assert(((((0x80000000|0x00080000)) & 0xf) == 0
|| (((0x80000000|0x00080000)) & 0xf) == 8) && sizeof
(unsigned long) == sizeof(*(&tpoints_hash_size)), "compile-time assertion failed"
)
242 FASTTRAP_TPOINTS_DEFAULT_SIZE, "Size of the tracepoint hash table")static struct sysctl_oid sysctl___kern_dtrace_fasttrap_tpoints_hash_size
= { .oid_parent = ((&(&sysctl___kern_dtrace_fasttrap
)->oid_children)), .oid_children = { ((void *)0) }, .oid_number
= ((-1)), .oid_kind = (8 | 0x00040000 | ((0x80000000|0x00080000
))), .oid_arg1 = (&tpoints_hash_size), .oid_arg2 = (0x4000
), .oid_name = ("tpoints_hash_size"), .oid_handler = (sysctl_handle_long
), .oid_fmt = ("LU"), .oid_descr = "Size of the tracepoint hash table"
}; __asm__(".globl " "__start_set_sysctl_set"); __asm__(".globl "
"__stop_set_sysctl_set"); static void const * const __set_sysctl_set_sym_sysctl___kern_dtrace_fasttrap_tpoints_hash_size
__attribute__((__section__("set_" "sysctl_set"))) __attribute__
((__used__)) = &(sysctl___kern_dtrace_fasttrap_tpoints_hash_size
); _Static_assert(((((0x80000000|0x00080000)) & 0xf) == 0
|| (((0x80000000|0x00080000)) & 0xf) == 8) && sizeof
(unsigned long) == sizeof(*(&tpoints_hash_size)), "compile-time assertion failed"
);
243#endif
244
245static int
246fasttrap_highbit(ulong_t i)
247{
248 int h = 1;
249
250 if (i == 0)
251 return (0);
252#ifdef _LP641
253 if (i & 0xffffffff00000000ul) {
254 h += 32; i >>= 32;
255 }
256#endif
257 if (i & 0xffff0000) {
258 h += 16; i >>= 16;
259 }
260 if (i & 0xff00) {
261 h += 8; i >>= 8;
262 }
263 if (i & 0xf0) {
264 h += 4; i >>= 4;
265 }
266 if (i & 0xc) {
267 h += 2; i >>= 2;
268 }
269 if (i & 0x2) {
270 h += 1;
271 }
272 return (h);
273}
274
275static uint_t
276fasttrap_hash_str(const char *p)
277{
278 unsigned int g;
279 uint_t hval = 0;
280
281 while (*p) {
282 hval = (hval << 4) + *p++;
283 if ((g = (hval & 0xf0000000)) != 0)
284 hval ^= g >> 24;
285 hval &= ~g;
286 }
287 return (hval);
288}
289
290void
291fasttrap_sigtrap(proc_t *p, kthread_t *t, uintptr_t pc)
292{
293#ifdef illumos
294 sigqueue_t *sqp = kmem_zalloc(sizeof (sigqueue_t), KM_SLEEP)zfs_kmem_alloc((sizeof (sigqueue_t)), (0x0002) | 0x0100);
295
296 sqp->sq_info.si_signo = SIGTRAP5;
297 sqp->sq_info.si_code = TRAP_DTRACE3;
298 sqp->sq_info.si_addr = (caddr_t)pc;
299
300 mutex_enter(&p->p_lock)(void)_sx_xlock(((&p->p_lock)), 0, ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (300));
301 sigaddqa(p, t, sqp);
302 mutex_exit(&p->p_lock)_sx_xunlock(((&p->p_lock)), ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (302));
303
304 if (t != NULL((void *)0))
305 aston(t);
306#else
307 ksiginfo_t *ksi = kmem_zalloc(sizeof (ksiginfo_t), KM_SLEEP)zfs_kmem_alloc((sizeof (ksiginfo_t)), (0x0002) | 0x0100);
308
309 ksiginfo_init(ksi)do { bzero(ksi, sizeof(ksiginfo_t)); } while(0);
310 ksi->ksi_signoksi_info.si_signo = SIGTRAP5;
311 ksi->ksi_codeksi_info.si_code = TRAP_DTRACE3;
312 ksi->ksi_addrksi_info.si_addr = (caddr_t)pc;
313 PROC_LOCK(p)__mtx_lock_flags(&((((&(p)->p_mtx))))->mtx_lock
, ((0)), ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (313));
314 (void) tdsendsignal(p, t, SIGTRAP5, ksi);
315 PROC_UNLOCK(p)__mtx_unlock_flags(&((((&(p)->p_mtx))))->mtx_lock
, ((0)), ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (315));
316#endif
317}
318
319#ifndef illumos
320/*
321 * Obtain a chunk of scratch space in the address space of the target process.
322 */
323fasttrap_scrspace_t *
324fasttrap_scraddr(struct thread *td, fasttrap_proc_t *fprc)
325{
326 fasttrap_scrblock_t *scrblk;
327 fasttrap_scrspace_t *scrspc;
328 struct proc *p;
329 vm_offset_t addr;
330 int error, i;
331
332 scrspc = NULL((void *)0);
333 if (td->t_dtrace_sscrtd_dtrace->td_dtrace_sscr != NULL((void *)0)) {
334 /* If the thread already has scratch space, we're done. */
335 scrspc = (fasttrap_scrspace_t *)td->t_dtrace_sscrtd_dtrace->td_dtrace_sscr;
336 return (scrspc);
337 }
338
339 p = td->td_proc;
340
341 mutex_enter(&fprc->ftpc_mtx)(void)_sx_xlock(((&fprc->ftpc_mtx)), 0, ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (341));
342 if (LIST_EMPTY(&fprc->ftpc_fscr)((&fprc->ftpc_fscr)->lh_first == ((void *)0))) {
343 /*
344 * No scratch space is available, so we'll map a new scratch
345 * space block into the traced process' address space.
346 */
347 addr = 0;
348 error = vm_map_find(&p->p_vmspace->vm_map, NULL((void *)0), 0, &addr,
349 FASTTRAP_SCRBLOCK_SIZE(1<<12), 0, VMFS_ANY_SPACE1, VM_PROT_ALL(((vm_prot_t) 0x01)|((vm_prot_t) 0x02)|((vm_prot_t) 0x04)),
350 VM_PROT_ALL(((vm_prot_t) 0x01)|((vm_prot_t) 0x02)|((vm_prot_t) 0x04)), 0);
351 if (error != KERN_SUCCESS0)
352 goto done;
353
354 scrblk = malloc(sizeof(*scrblk), M_SOLARIS, M_WAITOK0x0002);
355 scrblk->ftsb_addr = addr;
356 LIST_INSERT_HEAD(&fprc->ftpc_scrblks, scrblk, ftsb_next)do { ; if (((((scrblk))->ftsb_next.le_next) = (((&fprc
->ftpc_scrblks))->lh_first)) != ((void *)0)) (((&fprc
->ftpc_scrblks))->lh_first)->ftsb_next.le_prev = &
(((scrblk))->ftsb_next.le_next); (((&fprc->ftpc_scrblks
))->lh_first) = (scrblk); (scrblk)->ftsb_next.le_prev =
&(((&fprc->ftpc_scrblks))->lh_first); } while (
0);
357
358 /*
359 * Carve the block up into chunks and put them on the free list.
360 */
361 for (i = 0;
362 i < FASTTRAP_SCRBLOCK_SIZE(1<<12) / FASTTRAP_SCRSPACE_SIZE64; i++) {
363 scrspc = malloc(sizeof(*scrspc), M_SOLARIS, M_WAITOK0x0002);
364 scrspc->ftss_addr = addr +
365 i * FASTTRAP_SCRSPACE_SIZE64;
366 LIST_INSERT_HEAD(&fprc->ftpc_fscr, scrspc,do { ; if (((((scrspc))->ftss_next.le_next) = (((&fprc
->ftpc_fscr))->lh_first)) != ((void *)0)) (((&fprc->
ftpc_fscr))->lh_first)->ftss_next.le_prev = &(((scrspc
))->ftss_next.le_next); (((&fprc->ftpc_fscr))->lh_first
) = (scrspc); (scrspc)->ftss_next.le_prev = &(((&fprc
->ftpc_fscr))->lh_first); } while (0)
367 ftss_next)do { ; if (((((scrspc))->ftss_next.le_next) = (((&fprc
->ftpc_fscr))->lh_first)) != ((void *)0)) (((&fprc->
ftpc_fscr))->lh_first)->ftss_next.le_prev = &(((scrspc
))->ftss_next.le_next); (((&fprc->ftpc_fscr))->lh_first
) = (scrspc); (scrspc)->ftss_next.le_prev = &(((&fprc
->ftpc_fscr))->lh_first); } while (0);
368 }
369 }
370
371 /*
372 * Take the first scratch chunk off the free list, put it on the
373 * allocated list, and return its address.
374 */
375 scrspc = LIST_FIRST(&fprc->ftpc_fscr)((&fprc->ftpc_fscr)->lh_first);
376 LIST_REMOVE(scrspc, ftss_next)do { ; ; ; ; if ((((scrspc))->ftss_next.le_next) != ((void
*)0)) (((scrspc))->ftss_next.le_next)->ftss_next.le_prev
= (scrspc)->ftss_next.le_prev; *(scrspc)->ftss_next.le_prev
= (((scrspc))->ftss_next.le_next); ; ; } while (0);
377 LIST_INSERT_HEAD(&fprc->ftpc_ascr, scrspc, ftss_next)do { ; if (((((scrspc))->ftss_next.le_next) = (((&fprc
->ftpc_ascr))->lh_first)) != ((void *)0)) (((&fprc->
ftpc_ascr))->lh_first)->ftss_next.le_prev = &(((scrspc
))->ftss_next.le_next); (((&fprc->ftpc_ascr))->lh_first
) = (scrspc); (scrspc)->ftss_next.le_prev = &(((&fprc
->ftpc_ascr))->lh_first); } while (0);
378
379 /*
380 * This scratch space is reserved for use by td until the thread exits.
381 */
382 td->t_dtrace_sscrtd_dtrace->td_dtrace_sscr = scrspc;
383
384done:
385 mutex_exit(&fprc->ftpc_mtx)_sx_xunlock(((&fprc->ftpc_mtx)), ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (385));
386
387 return (scrspc);
388}
389
390/*
391 * Return any allocated per-thread scratch space chunks back to the process'
392 * free list.
393 */
394static void
395fasttrap_thread_dtor(void *arg __unused__attribute__((__unused__)), struct thread *td)
396{
397 fasttrap_bucket_t *bucket;
398 fasttrap_proc_t *fprc;
399 fasttrap_scrspace_t *scrspc;
400 pid_t pid;
401
402 if (td->t_dtrace_sscrtd_dtrace->td_dtrace_sscr == NULL((void *)0))
403 return;
404
405 pid = td->td_proc->p_pid;
406 bucket = &fasttrap_procs.fth_table[FASTTRAP_PROCS_INDEX(pid)((pid) & fasttrap_procs.fth_mask)];
407 fprc = NULL((void *)0);
408
409 /* Look up the fasttrap process handle for this process. */
410 mutex_enter(&bucket->ftb_mtx)(void)_sx_xlock(((&bucket->ftb_mtx)), 0, ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (410));
411 for (fprc = bucket->ftb_data; fprc != NULL((void *)0); fprc = fprc->ftpc_next) {
412 if (fprc->ftpc_pid == pid) {
413 mutex_enter(&fprc->ftpc_mtx)(void)_sx_xlock(((&fprc->ftpc_mtx)), 0, ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (413));
414 mutex_exit(&bucket->ftb_mtx)_sx_xunlock(((&bucket->ftb_mtx)), ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (414));
415 break;
416 }
417 }
418 if (fprc == NULL((void *)0)) {
419 mutex_exit(&bucket->ftb_mtx)_sx_xunlock(((&bucket->ftb_mtx)), ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (419));
420 return;
421 }
422
423 scrspc = (fasttrap_scrspace_t *)td->t_dtrace_sscrtd_dtrace->td_dtrace_sscr;
424 LIST_REMOVE(scrspc, ftss_next)do { ; ; ; ; if ((((scrspc))->ftss_next.le_next) != ((void
*)0)) (((scrspc))->ftss_next.le_next)->ftss_next.le_prev
= (scrspc)->ftss_next.le_prev; *(scrspc)->ftss_next.le_prev
= (((scrspc))->ftss_next.le_next); ; ; } while (0);
425 LIST_INSERT_HEAD(&fprc->ftpc_fscr, scrspc, ftss_next)do { ; if (((((scrspc))->ftss_next.le_next) = (((&fprc
->ftpc_fscr))->lh_first)) != ((void *)0)) (((&fprc->
ftpc_fscr))->lh_first)->ftss_next.le_prev = &(((scrspc
))->ftss_next.le_next); (((&fprc->ftpc_fscr))->lh_first
) = (scrspc); (scrspc)->ftss_next.le_prev = &(((&fprc
->ftpc_fscr))->lh_first); } while (0);
426
427 mutex_exit(&fprc->ftpc_mtx)_sx_xunlock(((&fprc->ftpc_mtx)), ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (427));
428}
429#endif
430
431/*
432 * This function ensures that no threads are actively using the memory
433 * associated with probes that were formerly live.
434 */
435static void
436fasttrap_mod_barrier(uint64_t gen)
437{
438 int i;
439
440 if (gen < fasttrap_mod_gen)
441 return;
442
443 fasttrap_mod_gen++;
444
445#ifdef illumos
446 CPU_FOREACH(i)for ((i) = 0; (i) <= mp_maxid; (i)++) if (!(!((((&all_cpus
)->__bits[(((((((256)) + (((sizeof(long) * 8)) - 1)) / ((sizeof
(long) * 8)))) == 1) ? 0 : (((i)) / (sizeof(long) * 8)))] &
(1L << ((((((((256))) + (((sizeof(long) * 8)) - 1)) / (
(sizeof(long) * 8)))) == 1) ? (__size_t)(((i))) : ((((i))) % (
sizeof(long) * 8))))) != 0)))) {
447 mutex_enter(&fasttrap_cpuc_pid_lock[i])(void)_sx_xlock(((&fasttrap_cpuc_pid_lock[i])), 0, ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (447));
448 mutex_exit(&fasttrap_cpuc_pid_lock[i])_sx_xunlock(((&fasttrap_cpuc_pid_lock[i])), ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (448));
449 }
450#else
451 rm_wlock(&fasttrap_tp_lock)_rm_wlock_debug((&fasttrap_tp_lock), "/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
, 451);
452 rm_wunlock(&fasttrap_tp_lock)_rm_wunlock_debug((&fasttrap_tp_lock), "/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
, 452);
453#endif
454}
455
456/*
457 * This function performs asynchronous cleanup of fasttrap providers. The
458 * Solaris implementation of this mechanism use a timeout that's activated in
459 * fasttrap_pid_cleanup(), but this doesn't work in FreeBSD: one may sleep while
460 * holding the DTrace mutexes, but it is unsafe to sleep in a callout handler.
461 * Thus we use a dedicated process to perform the cleanup when requested.
462 */
463/*ARGSUSED*/
464static void
465fasttrap_pid_cleanup_cb(void *data)
466{
467 fasttrap_provider_t **fpp, *fp;
468 fasttrap_bucket_t *bucket;
469 dtrace_provider_id_t provid;
470 int i, later = 0, rval;
471
472 mtx_lock(&fasttrap_cleanup_mtx)__mtx_lock_flags(&((((&fasttrap_cleanup_mtx))))->mtx_lock
, ((0)), ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (472));
473 while (!fasttrap_cleanup_drain || later > 0) {
474 fasttrap_cleanup_work = 0;
475 mtx_unlock(&fasttrap_cleanup_mtx)__mtx_unlock_flags(&((((&fasttrap_cleanup_mtx))))->
mtx_lock, ((0)), ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (475));
476
477 later = 0;
478
479 /*
480 * Iterate over all the providers trying to remove the marked
481 * ones. If a provider is marked but not retired, we just
482 * have to take a crack at removing it -- it's no big deal if
483 * we can't.
484 */
485 for (i = 0; i < fasttrap_provs.fth_nent; i++) {
486 bucket = &fasttrap_provs.fth_table[i];
487 mutex_enter(&bucket->ftb_mtx)(void)_sx_xlock(((&bucket->ftb_mtx)), 0, ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (487));
488 fpp = (fasttrap_provider_t **)&bucket->ftb_data;
489
490 while ((fp = *fpp) != NULL((void *)0)) {
491 if (!fp->ftp_marked) {
492 fpp = &fp->ftp_next;
493 continue;
494 }
495
496 mutex_enter(&fp->ftp_mtx)(void)_sx_xlock(((&fp->ftp_mtx)), 0, ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (496));
497
498 /*
499 * If this provider has consumers actively
500 * creating probes (ftp_ccount) or is a USDT
501 * provider (ftp_mcount), we can't unregister
502 * or even condense.
503 */
504 if (fp->ftp_ccount != 0 ||
505 fp->ftp_mcount != 0) {
506 mutex_exit(&fp->ftp_mtx)_sx_xunlock(((&fp->ftp_mtx)), ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (506));
507 fp->ftp_marked = 0;
508 continue;
509 }
510
511 if (!fp->ftp_retired || fp->ftp_rcount != 0)
512 fp->ftp_marked = 0;
513
514 mutex_exit(&fp->ftp_mtx)_sx_xunlock(((&fp->ftp_mtx)), ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (514));
515
516 /*
517 * If we successfully unregister this
518 * provider we can remove it from the hash
519 * chain and free the memory. If our attempt
520 * to unregister fails and this is a retired
521 * provider, increment our flag to try again
522 * pretty soon. If we've consumed more than
523 * half of our total permitted number of
524 * probes call dtrace_condense() to try to
525 * clean out the unenabled probes.
526 */
527 provid = fp->ftp_provid;
528 if ((rval = dtrace_unregister(provid)) != 0) {
529 if (fasttrap_total > fasttrap_max / 2)
530 (void) dtrace_condense(provid);
531
532 if (rval == EAGAIN35)
533 fp->ftp_marked = 1;
534
535 later += fp->ftp_marked;
536 fpp = &fp->ftp_next;
537 } else {
538 *fpp = fp->ftp_next;
539 fasttrap_provider_free(fp);
540 }
541 }
542 mutex_exit(&bucket->ftb_mtx)_sx_xunlock(((&bucket->ftb_mtx)), ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (542));
543 }
544 mtx_lock(&fasttrap_cleanup_mtx)__mtx_lock_flags(&((((&fasttrap_cleanup_mtx))))->mtx_lock
, ((0)), ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (544));
545
546 /*
547 * If we were unable to retire a provider, try again after a
548 * second. This situation can occur in certain circumstances
549 * where providers cannot be unregistered even though they have
550 * no probes enabled because of an execution of dtrace -l or
551 * something similar.
552 */
553 if (later > 0 || fasttrap_cleanup_work ||
554 fasttrap_cleanup_drain) {
555 mtx_unlock(&fasttrap_cleanup_mtx)__mtx_unlock_flags(&((((&fasttrap_cleanup_mtx))))->
mtx_lock, ((0)), ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (555));
556 pause("ftclean", hz)pause_sbt(("ftclean"), tick_sbt * (hz), 0, 0x0100);
557 mtx_lock(&fasttrap_cleanup_mtx)__mtx_lock_flags(&((((&fasttrap_cleanup_mtx))))->mtx_lock
, ((0)), ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (557));
558 } else
559 mtx_sleep(&fasttrap_cleanup_cv, &fasttrap_cleanup_mtx,_sleep((&fasttrap_cleanup_cv), &(&fasttrap_cleanup_mtx
)->lock_object, (0), ("ftcl"), tick_sbt * (0), 0, 0x0100)
560 0, "ftcl", 0)_sleep((&fasttrap_cleanup_cv), &(&fasttrap_cleanup_mtx
)->lock_object, (0), ("ftcl"), tick_sbt * (0), 0, 0x0100);
561 }
562
563 /*
564 * Wake up the thread in fasttrap_unload() now that we're done.
565 */
566 wakeup(&fasttrap_cleanup_drain);
567 mtx_unlock(&fasttrap_cleanup_mtx)__mtx_unlock_flags(&((((&fasttrap_cleanup_mtx))))->
mtx_lock, ((0)), ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (567));
568
569 kthread_exit();
570}
571
572/*
573 * Activates the asynchronous cleanup mechanism.
574 */
575static void
576fasttrap_pid_cleanup(void)
577{
578
579 mtx_lock(&fasttrap_cleanup_mtx)__mtx_lock_flags(&((((&fasttrap_cleanup_mtx))))->mtx_lock
, ((0)), ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (579));
580 if (!fasttrap_cleanup_work) {
581 fasttrap_cleanup_work = 1;
582 wakeup(&fasttrap_cleanup_cv);
583 }
584 mtx_unlock(&fasttrap_cleanup_mtx)__mtx_unlock_flags(&((((&fasttrap_cleanup_mtx))))->
mtx_lock, ((0)), ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (584));
585}
586
587/*
588 * This is called from cfork() via dtrace_fasttrap_fork(). The child
589 * process's address space is (roughly) a copy of the parent process's so
590 * we have to remove all the instrumentation we had previously enabled in the
591 * parent.
592 */
593static void
594fasttrap_fork(proc_t *p, proc_t *cp)
595{
596#ifndef illumos
597 fasttrap_scrblock_t *scrblk;
598 fasttrap_proc_t *fprc = NULL((void *)0);
599#endif
600 pid_t ppid = p->p_pid;
601 int i;
602
603#ifdef illumos
604 ASSERT(curproc == p)((void)0);
605 ASSERT(p->p_proc_flag & P_PR_LOCK)((void)0);
606#else
607 PROC_LOCK_ASSERT(p, MA_OWNED)(void)0;
608#endif
609#ifdef illumos
610 ASSERT(p->p_dtrace_count > 0)((void)0);
611#else
612 if (p->p_dtrace_helpersp_dtrace->p_dtrace_helpers) {
613 /*
614 * dtrace_helpers_duplicate() allocates memory.
615 */
616 _PHOLD(cp)do { (void)0; do { } while (0); (cp)->p_lock++; if (((cp)->
p_flag & 0x10000000) == 0) faultin((cp)); } while (0);
617 PROC_UNLOCK(p)__mtx_unlock_flags(&((((&(p)->p_mtx))))->mtx_lock
, ((0)), ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (617));
618 PROC_UNLOCK(cp)__mtx_unlock_flags(&((((&(cp)->p_mtx))))->mtx_lock
, ((0)), ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (618));
619 dtrace_helpers_duplicate(p, cp);
620 PROC_LOCK(cp)__mtx_lock_flags(&((((&(cp)->p_mtx))))->mtx_lock
, ((0)), ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (620));
621 PROC_LOCK(p)__mtx_lock_flags(&((((&(p)->p_mtx))))->mtx_lock
, ((0)), ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (621));
622 _PRELE(cp)do { (void)0; do { do { } while (0); } while (0); (--(cp)->
p_lock); if (((cp)->p_flag & 0x02000) && (cp)->
p_lock == 0) wakeup(&(cp)->p_lock); } while (0);
623 }
624 /*
625 * This check is purposely here instead of in kern_fork.c because,
626 * for legal resons, we cannot include the dtrace_cddl.h header
627 * inside kern_fork.c and insert if-clause there.
628 */
629 if (p->p_dtrace_countp_dtrace->p_dtrace_count == 0)
630 return;
631#endif
632 ASSERT(cp->p_dtrace_count == 0)((void)0);
633
634 /*
635 * This would be simpler and faster if we maintained per-process
636 * hash tables of enabled tracepoints. It could, however, potentially
637 * slow down execution of a tracepoint since we'd need to go
638 * through two levels of indirection. In the future, we should
639 * consider either maintaining per-process ancillary lists of
640 * enabled tracepoints or hanging a pointer to a per-process hash
641 * table of enabled tracepoints off the proc structure.
642 */
643
644 /*
645 * We don't have to worry about the child process disappearing
646 * because we're in fork().
647 */
648#ifdef illumos
649 mtx_lock_spin(&cp->p_slock)__mtx_lock_spin_flags(&((((&cp->p_slock))))->mtx_lock
, ((0)), ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (649));
650 sprlock_proc(cp);
651 mtx_unlock_spin(&cp->p_slock)__mtx_unlock_spin_flags(&((((&cp->p_slock))))->
mtx_lock, ((0)), ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (651));
652#else
653 /*
654 * fasttrap_tracepoint_remove() expects the child process to be
655 * unlocked and the VM then expects curproc to be unlocked.
656 */
657 _PHOLD(cp)do { (void)0; do { } while (0); (cp)->p_lock++; if (((cp)->
p_flag & 0x10000000) == 0) faultin((cp)); } while (0);
658 PROC_UNLOCK(cp)__mtx_unlock_flags(&((((&(cp)->p_mtx))))->mtx_lock
, ((0)), ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (658));
659 PROC_UNLOCK(p)__mtx_unlock_flags(&((((&(p)->p_mtx))))->mtx_lock
, ((0)), ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (659));
660#endif
661
662 /*
663 * Iterate over every tracepoint looking for ones that belong to the
664 * parent process, and remove each from the child process.
665 */
666 for (i = 0; i < fasttrap_tpoints.fth_nent; i++) {
667 fasttrap_tracepoint_t *tp;
668 fasttrap_bucket_t *bucket = &fasttrap_tpoints.fth_table[i];
669
670 mutex_enter(&bucket->ftb_mtx)(void)_sx_xlock(((&bucket->ftb_mtx)), 0, ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (670));
671 for (tp = bucket->ftb_data; tp != NULL((void *)0); tp = tp->ftt_next) {
672 if (tp->ftt_pid == ppid &&
673 tp->ftt_proc->ftpc_acount != 0) {
674 int ret = fasttrap_tracepoint_remove(cp, tp);
675 ASSERT(ret == 0)((void)0);
676
677 /*
678 * The count of active providers can only be
679 * decremented (i.e. to zero) during exec,
680 * exit, and removal of a meta provider so it
681 * should be impossible to drop the count
682 * mid-fork.
683 */
684 ASSERT(tp->ftt_proc->ftpc_acount != 0)((void)0);
685#ifndef illumos
686 fprc = tp->ftt_proc;
687#endif
688 }
689 }
690 mutex_exit(&bucket->ftb_mtx)_sx_xunlock(((&bucket->ftb_mtx)), ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (690));
691
692#ifndef illumos
693 /*
694 * Unmap any scratch space inherited from the parent's address
695 * space.
696 */
697 if (fprc != NULL((void *)0)) {
698 mutex_enter(&fprc->ftpc_mtx)(void)_sx_xlock(((&fprc->ftpc_mtx)), 0, ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (698));
699 LIST_FOREACH(scrblk, &fprc->ftpc_scrblks, ftsb_next)for ((scrblk) = (((&fprc->ftpc_scrblks))->lh_first)
; (scrblk); (scrblk) = (((scrblk))->ftsb_next.le_next)) {
700 vm_map_remove(&cp->p_vmspace->vm_map,
701 scrblk->ftsb_addr,
702 scrblk->ftsb_addr + FASTTRAP_SCRBLOCK_SIZE(1<<12));
703 }
704 mutex_exit(&fprc->ftpc_mtx)_sx_xunlock(((&fprc->ftpc_mtx)), ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (704));
705 }
706#endif
707 }
708
709#ifdef illumos
710 mutex_enter(&cp->p_lock)(void)_sx_xlock(((&cp->p_lock)), 0, ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (710));
711 sprunlock(cp);
712#else
713 PROC_LOCK(p)__mtx_lock_flags(&((((&(p)->p_mtx))))->mtx_lock
, ((0)), ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (713));
714 PROC_LOCK(cp)__mtx_lock_flags(&((((&(cp)->p_mtx))))->mtx_lock
, ((0)), ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (714));
715 _PRELE(cp)do { (void)0; do { do { } while (0); } while (0); (--(cp)->
p_lock); if (((cp)->p_flag & 0x02000) && (cp)->
p_lock == 0) wakeup(&(cp)->p_lock); } while (0);
716#endif
717}
718
719/*
720 * This is called from proc_exit() or from exec_common() if p_dtrace_probes
721 * is set on the proc structure to indicate that there is a pid provider
722 * associated with this process.
723 */
724static void
725fasttrap_exec_exit(proc_t *p)
726{
727#ifndef illumos
728 struct thread *td;
729#endif
730
731#ifdef illumos
732 ASSERT(p == curproc)((void)0);
733#else
734 PROC_LOCK_ASSERT(p, MA_OWNED)(void)0;
735 _PHOLD(p)do { (void)0; do { } while (0); (p)->p_lock++; if (((p)->
p_flag & 0x10000000) == 0) faultin((p)); } while (0);
736 /*
737 * Since struct threads may be recycled, we cannot rely on t_dtrace_sscr
738 * fields to be zeroed by kdtrace_thread_ctor. Thus we must zero it
739 * ourselves when a process exits.
740 */
741 FOREACH_THREAD_IN_PROC(p, td)for (((td)) = (((&(p)->p_threads))->tqh_first); ((td
)); ((td)) = ((((td)))->td_plist.tqe_next))
742 td->t_dtrace_sscrtd_dtrace->td_dtrace_sscr = NULL((void *)0);
743 PROC_UNLOCK(p)__mtx_unlock_flags(&((((&(p)->p_mtx))))->mtx_lock
, ((0)), ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (743));
744#endif
745
746 /*
747 * We clean up the pid provider for this process here; user-land
748 * static probes are handled by the meta-provider remove entry point.
749 */
750 fasttrap_provider_retire(p->p_pid, FASTTRAP_PID_NAME"pid", 0);
751#ifndef illumos
752 if (p->p_dtrace_helpersp_dtrace->p_dtrace_helpers)
753 dtrace_helpers_destroy(p);
754 PROC_LOCK(p)__mtx_lock_flags(&((((&(p)->p_mtx))))->mtx_lock
, ((0)), ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (754));
755 _PRELE(p)do { (void)0; do { do { } while (0); } while (0); (--(p)->
p_lock); if (((p)->p_flag & 0x02000) && (p)->
p_lock == 0) wakeup(&(p)->p_lock); } while (0);
756#endif
757}
758
759
760/*ARGSUSED*/
761static void
762fasttrap_pid_provide(void *arg, dtrace_probedesc_t *desc)
763{
764 /*
765 * There are no "default" pid probes.
766 */
767}
768
769static int
770fasttrap_tracepoint_enable(proc_t *p, fasttrap_probe_t *probe, uint_t index)
771{
772 fasttrap_tracepoint_t *tp, *new_tp = NULL((void *)0);
773 fasttrap_bucket_t *bucket;
774 fasttrap_id_t *id;
775 pid_t pid;
776 uintptr_t pc;
777
778 ASSERT(index < probe->ftp_ntps)((void)0);
779
780 pid = probe->ftp_pid;
781 pc = probe->ftp_tps[index].fit_tp->ftt_pc;
782 id = &probe->ftp_tps[index].fit_id;
783
784 ASSERT(probe->ftp_tps[index].fit_tp->ftt_pid == pid)((void)0);
785
786#ifdef illumos
787 ASSERT(!(p->p_flag & SVFORK))((void)0);
788#endif
789
790 /*
791 * Before we make any modifications, make sure we've imposed a barrier
792 * on the generation in which this probe was last modified.
793 */
794 fasttrap_mod_barrier(probe->ftp_gen);
795
796 bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)(((pc) / sizeof (fasttrap_instr_t) + (pid)) & fasttrap_tpoints
.fth_mask)];
797
798 /*
799 * If the tracepoint has already been enabled, just add our id to the
800 * list of interested probes. This may be our second time through
801 * this path in which case we'll have constructed the tracepoint we'd
802 * like to install. If we can't find a match, and have an allocated
803 * tracepoint ready to go, enable that one now.
804 *
805 * A tracepoint whose process is defunct is also considered defunct.
806 */
807again:
808 mutex_enter(&bucket->ftb_mtx)(void)_sx_xlock(((&bucket->ftb_mtx)), 0, ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (808));
809 for (tp = bucket->ftb_data; tp != NULL((void *)0); tp = tp->ftt_next) {
810 /*
811 * Note that it's safe to access the active count on the
812 * associated proc structure because we know that at least one
813 * provider (this one) will still be around throughout this
814 * operation.
815 */
816 if (tp->ftt_pid != pid || tp->ftt_pc != pc ||
817 tp->ftt_proc->ftpc_acount == 0)
818 continue;
819
820 /*
821 * Now that we've found a matching tracepoint, it would be
822 * a decent idea to confirm that the tracepoint is still
823 * enabled and the trap instruction hasn't been overwritten.
824 * Since this is a little hairy, we'll punt for now.
825 */
826
827 /*
828 * This can't be the first interested probe. We don't have
829 * to worry about another thread being in the midst of
830 * deleting this tracepoint (which would be the only valid
831 * reason for a tracepoint to have no interested probes)
832 * since we're holding P_PR_LOCK for this process.
833 */
834 ASSERT(tp->ftt_ids != NULL || tp->ftt_retids != NULL)((void)0);
835
836 switch (id->fti_ptype) {
837 case DTFTP_ENTRY:
838 case DTFTP_OFFSETS:
839 case DTFTP_IS_ENABLED:
840 id->fti_next = tp->ftt_ids;
841 membar_producer();
842 tp->ftt_ids = id;
843 membar_producer();
844 break;
845
846 case DTFTP_RETURN:
847 case DTFTP_POST_OFFSETS:
848 id->fti_next = tp->ftt_retids;
849 membar_producer();
850 tp->ftt_retids = id;
851 membar_producer();
852 break;
853
854 default:
855 ASSERT(0)((void)0);
856 }
857
858 mutex_exit(&bucket->ftb_mtx)_sx_xunlock(((&bucket->ftb_mtx)), ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (858));
859
860 if (new_tp != NULL((void *)0)) {
861 new_tp->ftt_ids = NULL((void *)0);
862 new_tp->ftt_retids = NULL((void *)0);
863 }
864
865 return (0);
866 }
867
868 /*
869 * If we have a good tracepoint ready to go, install it now while
870 * we have the lock held and no one can screw with us.
871 */
872 if (new_tp != NULL((void *)0)) {
873 int rc = 0;
874
875 new_tp->ftt_next = bucket->ftb_data;
876 membar_producer();
877 bucket->ftb_data = new_tp;
878 membar_producer();
879 mutex_exit(&bucket->ftb_mtx)_sx_xunlock(((&bucket->ftb_mtx)), ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (879));
880
881 /*
882 * Activate the tracepoint in the ISA-specific manner.
883 * If this fails, we need to report the failure, but
884 * indicate that this tracepoint must still be disabled
885 * by calling fasttrap_tracepoint_disable().
886 */
887 if (fasttrap_tracepoint_install(p, new_tp) != 0)
888 rc = FASTTRAP_ENABLE_PARTIAL2;
889
890 /*
891 * Increment the count of the number of tracepoints active in
892 * the victim process.
893 */
894#ifdef illumos
895 ASSERT(p->p_proc_flag & P_PR_LOCK)((void)0);
896#endif
897 p->p_dtrace_countp_dtrace->p_dtrace_count++;
898
899 return (rc);
900 }
901
902 mutex_exit(&bucket->ftb_mtx)_sx_xunlock(((&bucket->ftb_mtx)), ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (902));
903
904 /*
905 * Initialize the tracepoint that's been preallocated with the probe.
906 */
907 new_tp = probe->ftp_tps[index].fit_tp;
908
909 ASSERT(new_tp->ftt_pid == pid)((void)0);
910 ASSERT(new_tp->ftt_pc == pc)((void)0);
911 ASSERT(new_tp->ftt_proc == probe->ftp_prov->ftp_proc)((void)0);
912 ASSERT(new_tp->ftt_ids == NULL)((void)0);
913 ASSERT(new_tp->ftt_retids == NULL)((void)0);
914
915 switch (id->fti_ptype) {
916 case DTFTP_ENTRY:
917 case DTFTP_OFFSETS:
918 case DTFTP_IS_ENABLED:
919 id->fti_next = NULL((void *)0);
920 new_tp->ftt_ids = id;
921 break;
922
923 case DTFTP_RETURN:
924 case DTFTP_POST_OFFSETS:
925 id->fti_next = NULL((void *)0);
926 new_tp->ftt_retids = id;
927 break;
928
929 default:
930 ASSERT(0)((void)0);
931 }
932
933 /*
934 * If the ISA-dependent initialization goes to plan, go back to the
935 * beginning and try to install this freshly made tracepoint.
936 */
937 if (fasttrap_tracepoint_init(p, new_tp, pc, id->fti_ptype) == 0)
938 goto again;
939
940 new_tp->ftt_ids = NULL((void *)0);
941 new_tp->ftt_retids = NULL((void *)0);
942
943 return (FASTTRAP_ENABLE_FAIL1);
944}
945
946static void
947fasttrap_tracepoint_disable(proc_t *p, fasttrap_probe_t *probe, uint_t index)
948{
949 fasttrap_bucket_t *bucket;
950 fasttrap_provider_t *provider = probe->ftp_prov;
951 fasttrap_tracepoint_t **pp, *tp;
952 fasttrap_id_t *id, **idp = NULL((void *)0);
953 pid_t pid;
954 uintptr_t pc;
955
956 ASSERT(index < probe->ftp_ntps)((void)0);
957
958 pid = probe->ftp_pid;
959 pc = probe->ftp_tps[index].fit_tp->ftt_pc;
960 id = &probe->ftp_tps[index].fit_id;
961
962 ASSERT(probe->ftp_tps[index].fit_tp->ftt_pid == pid)((void)0);
963
964 /*
965 * Find the tracepoint and make sure that our id is one of the
966 * ones registered with it.
967 */
968 bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)(((pc) / sizeof (fasttrap_instr_t) + (pid)) & fasttrap_tpoints
.fth_mask)];
969 mutex_enter(&bucket->ftb_mtx)(void)_sx_xlock(((&bucket->ftb_mtx)), 0, ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (969));
970 for (tp = bucket->ftb_data; tp != NULL((void *)0); tp = tp->ftt_next) {
971 if (tp->ftt_pid == pid && tp->ftt_pc == pc &&
972 tp->ftt_proc == provider->ftp_proc)
973 break;
974 }
975
976 /*
977 * If we somehow lost this tracepoint, we're in a world of hurt.
978 */
979 ASSERT(tp != NULL)((void)0);
980
981 switch (id->fti_ptype) {
982 case DTFTP_ENTRY:
983 case DTFTP_OFFSETS:
984 case DTFTP_IS_ENABLED:
985 ASSERT(tp->ftt_ids != NULL)((void)0);
986 idp = &tp->ftt_ids;
987 break;
988
989 case DTFTP_RETURN:
990 case DTFTP_POST_OFFSETS:
991 ASSERT(tp->ftt_retids != NULL)((void)0);
992 idp = &tp->ftt_retids;
993 break;
994
995 default:
996 ASSERT(0)((void)0);
997 }
998
999 while ((*idp)->fti_probe != probe) {
1000 idp = &(*idp)->fti_next;
1001 ASSERT(*idp != NULL)((void)0);
1002 }
1003
1004 id = *idp;
1005 *idp = id->fti_next;
1006 membar_producer();
1007
1008 ASSERT(id->fti_probe == probe)((void)0);
1009
1010 /*
1011 * If there are other registered enablings of this tracepoint, we're
1012 * all done, but if this was the last probe assocated with this
1013 * this tracepoint, we need to remove and free it.
1014 */
1015 if (tp->ftt_ids != NULL((void *)0) || tp->ftt_retids != NULL((void *)0)) {
1016
1017 /*
1018 * If the current probe's tracepoint is in use, swap it
1019 * for an unused tracepoint.
1020 */
1021 if (tp == probe->ftp_tps[index].fit_tp) {
1022 fasttrap_probe_t *tmp_probe;
1023 fasttrap_tracepoint_t **tmp_tp;
1024 uint_t tmp_index;
1025
1026 if (tp->ftt_ids != NULL((void *)0)) {
1027 tmp_probe = tp->ftt_ids->fti_probe;
1028 /* LINTED - alignment */
1029 tmp_index = FASTTRAP_ID_INDEX(tp->ftt_ids)((fasttrap_id_tp_t *)(((char *)(tp->ftt_ids) - __builtin_offsetof
(fasttrap_id_tp_t, fit_id))) - &(tp->ftt_ids)->fti_probe
->ftp_tps[0]);
1030 tmp_tp = &tmp_probe->ftp_tps[tmp_index].fit_tp;
1031 } else {
1032 tmp_probe = tp->ftt_retids->fti_probe;
1033 /* LINTED - alignment */
1034 tmp_index = FASTTRAP_ID_INDEX(tp->ftt_retids)((fasttrap_id_tp_t *)(((char *)(tp->ftt_retids) - __builtin_offsetof
(fasttrap_id_tp_t, fit_id))) - &(tp->ftt_retids)->fti_probe
->ftp_tps[0]);
1035 tmp_tp = &tmp_probe->ftp_tps[tmp_index].fit_tp;
1036 }
1037
1038 ASSERT(*tmp_tp != NULL)((void)0);
1039 ASSERT(*tmp_tp != probe->ftp_tps[index].fit_tp)((void)0);
1040 ASSERT((*tmp_tp)->ftt_ids == NULL)((void)0);
1041 ASSERT((*tmp_tp)->ftt_retids == NULL)((void)0);
1042
1043 probe->ftp_tps[index].fit_tp = *tmp_tp;
1044 *tmp_tp = tp;
1045 }
1046
1047 mutex_exit(&bucket->ftb_mtx)_sx_xunlock(((&bucket->ftb_mtx)), ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (1047));
1048
1049 /*
1050 * Tag the modified probe with the generation in which it was
1051 * changed.
1052 */
1053 probe->ftp_gen = fasttrap_mod_gen;
1054 return;
1055 }
1056
1057 mutex_exit(&bucket->ftb_mtx)_sx_xunlock(((&bucket->ftb_mtx)), ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (1057));
1058
1059 /*
1060 * We can't safely remove the tracepoint from the set of active
1061 * tracepoints until we've actually removed the fasttrap instruction
1062 * from the process's text. We can, however, operate on this
1063 * tracepoint secure in the knowledge that no other thread is going to
1064 * be looking at it since we hold P_PR_LOCK on the process if it's
1065 * live or we hold the provider lock on the process if it's dead and
1066 * gone.
1067 */
1068
1069 /*
1070 * We only need to remove the actual instruction if we're looking
1071 * at an existing process
1072 */
1073 if (p != NULL((void *)0)) {
1074 /*
1075 * If we fail to restore the instruction we need to kill
1076 * this process since it's in a completely unrecoverable
1077 * state.
1078 */
1079 if (fasttrap_tracepoint_remove(p, tp) != 0)
1080 fasttrap_sigtrap(p, NULL((void *)0), pc);
1081
1082 /*
1083 * Decrement the count of the number of tracepoints active
1084 * in the victim process.
1085 */
1086#ifdef illumos
1087 ASSERT(p->p_proc_flag & P_PR_LOCK)((void)0);
1088#endif
1089 p->p_dtrace_countp_dtrace->p_dtrace_count--;
1090 }
1091
1092 /*
1093 * Remove the probe from the hash table of active tracepoints.
1094 */
1095 mutex_enter(&bucket->ftb_mtx)(void)_sx_xlock(((&bucket->ftb_mtx)), 0, ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (1095));
1096 pp = (fasttrap_tracepoint_t **)&bucket->ftb_data;
1097 ASSERT(*pp != NULL)((void)0);
1098 while (*pp != tp) {
1099 pp = &(*pp)->ftt_next;
1100 ASSERT(*pp != NULL)((void)0);
1101 }
1102
1103 *pp = tp->ftt_next;
1104 membar_producer();
1105
1106 mutex_exit(&bucket->ftb_mtx)_sx_xunlock(((&bucket->ftb_mtx)), ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (1106));
1107
1108 /*
1109 * Tag the modified probe with the generation in which it was changed.
1110 */
1111 probe->ftp_gen = fasttrap_mod_gen;
1112}
1113
1114static void
1115fasttrap_enable_callbacks(void)
1116{
1117 /*
1118 * We don't have to play the rw lock game here because we're
1119 * providing something rather than taking something away --
1120 * we can be sure that no threads have tried to follow this
1121 * function pointer yet.
1122 */
1123 mutex_enter(&fasttrap_count_mtx)(void)_sx_xlock(((&fasttrap_count_mtx)), 0, ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (1123));
1124 if (fasttrap_pid_count == 0) {
1125 ASSERT(dtrace_pid_probe_ptr == NULL)((void)0);
1126 ASSERT(dtrace_return_probe_ptr == NULL)((void)0);
1127 dtrace_pid_probe_ptr = &fasttrap_pid_probe;
1128 dtrace_return_probe_ptr = &fasttrap_return_probe;
1129 }
1130 ASSERT(dtrace_pid_probe_ptr == &fasttrap_pid_probe)((void)0);
1131 ASSERT(dtrace_return_probe_ptr == &fasttrap_return_probe)((void)0);
1132 fasttrap_pid_count++;
1133 mutex_exit(&fasttrap_count_mtx)_sx_xunlock(((&fasttrap_count_mtx)), ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (1133));
1134}
1135
1136static void
1137fasttrap_disable_callbacks(void)
1138{
1139#ifdef illumos
1140 ASSERT(MUTEX_HELD(&cpu_lock))((void)0);
1141#endif
1142
1143
1144 mutex_enter(&fasttrap_count_mtx)(void)_sx_xlock(((&fasttrap_count_mtx)), 0, ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (1144));
1145 ASSERT(fasttrap_pid_count > 0)((void)0);
1146 fasttrap_pid_count--;
1147 if (fasttrap_pid_count == 0) {
1148#ifdef illumos
1149 cpu_tsolaris_cpu_t *cur, *cpu = CPU__extension__ ({ __typeof(((struct pcpu *)0)->pc_cpuid) __res
; struct __s { u_char __b[(((sizeof(__typeof(((struct pcpu *)
0)->pc_cpuid)))<(8))?(sizeof(__typeof(((struct pcpu *)0
)->pc_cpuid))):(8))]; } __s; if (sizeof(__res) == 1 || sizeof
(__res) == 2 || sizeof(__res) == 4 || sizeof(__res) == 8) { __asm
volatile("mov %%gs:%1,%0" : "=r" (__s) : "m" (*(struct __s *
)(__builtin_offsetof(struct pcpu, pc_cpuid)))); *(struct __s *
)(void *)&__res = __s; } else { __res = *__extension__ ({
__typeof(((struct pcpu *)0)->pc_cpuid) *__p; __asm volatile
("movq %%gs:%1,%0; addq %2,%0" : "=r" (__p) : "m" (*(struct pcpu
*)(__builtin_offsetof(struct pcpu, pc_prvspace))), "i" (__builtin_offsetof
(struct pcpu, pc_cpuid))); __p; }); } __res; });
1150
1151 for (cur = cpu->cpu_next_onln; cur != cpu;
1152 cur = cur->cpu_next_onln) {
1153 rw_enter(&cur->cpu_ft_lock, RW_WRITER);
1154 }
1155#endif
1156 dtrace_pid_probe_ptr = NULL((void *)0);
1157 dtrace_return_probe_ptr = NULL((void *)0);
1158#ifdef illumos
1159 for (cur = cpu->cpu_next_onln; cur != cpu;
1160 cur = cur->cpu_next_onln) {
1161 rw_exit(&cur->cpu_ft_lock);
1162 }
1163#endif
1164 }
1165 mutex_exit(&fasttrap_count_mtx)_sx_xunlock(((&fasttrap_count_mtx)), ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (1165));
1166}
1167
1168/*ARGSUSED*/
1169static void
1170fasttrap_pid_enable(void *arg, dtrace_id_t id, void *parg)
1171{
1172 fasttrap_probe_t *probe = parg;
1173 proc_t *p = NULL((void *)0);
1174 int i, rc;
1175
1176 ASSERT(probe != NULL)((void)0);
1177 ASSERT(!probe->ftp_enabled)((void)0);
1178 ASSERT(id == probe->ftp_id)((void)0);
1179#ifdef illumos
1180 ASSERT(MUTEX_HELD(&cpu_lock))((void)0);
1181#endif
1182
1183 /*
1184 * Increment the count of enabled probes on this probe's provider;
1185 * the provider can't go away while the probe still exists. We
1186 * must increment this even if we aren't able to properly enable
1187 * this probe.
1188 */
1189 mutex_enter(&probe->ftp_prov->ftp_mtx)(void)_sx_xlock(((&probe->ftp_prov->ftp_mtx)), 0, (
"/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (1189));
1190 probe->ftp_prov->ftp_rcount++;
1191 mutex_exit(&probe->ftp_prov->ftp_mtx)_sx_xunlock(((&probe->ftp_prov->ftp_mtx)), ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (1191));
1192
1193 /*
1194 * If this probe's provider is retired (meaning it was valid in a
1195 * previously exec'ed incarnation of this address space), bail out. The
1196 * provider can't go away while we're in this code path.
1197 */
1198 if (probe->ftp_prov->ftp_retired)
1199 return;
1200
1201 /*
1202 * If we can't find the process, it may be that we're in the context of
1203 * a fork in which the traced process is being born and we're copying
1204 * USDT probes. Otherwise, the process is gone so bail.
1205 */
1206#ifdef illumos
1207 if ((p = sprlock(probe->ftp_pid)) == NULL((void *)0)) {
1208 if ((curproc((__curthread())->td_proc)->p_flag & SFORKING) == 0)
1209 return;
1210
1211 mutex_enter(&pidlock)(void)_sx_xlock(((&pidlock)), 0, ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (1211));
1212 p = prfind(probe->ftp_pid);
1213
1214 if (p == NULL((void *)0)) {
1215 /*
1216 * So it's not that the target process is being born,
1217 * it's that it isn't there at all (and we simply
1218 * happen to be forking). Anyway, we know that the
1219 * target is definitely gone, so bail out.
1220 */
1221 mutex_exit(&pidlock)_sx_xunlock(((&pidlock)), ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (1221));
1222 return (0);
1223 }
1224
1225 /*
1226 * Confirm that curproc is indeed forking the process in which
1227 * we're trying to enable probes.
1228 */
1229 ASSERT(p->p_parent == curproc)((void)0);
1230 ASSERT(p->p_stat == SIDL)((void)0);
1231
1232 mutex_enter(&p->p_lock)(void)_sx_xlock(((&p->p_lock)), 0, ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (1232));
1233 mutex_exit(&pidlock)_sx_xunlock(((&pidlock)), ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (1233));
1234
1235 sprlock_proc(p);
1236 }
1237
1238 ASSERT(!(p->p_flag & SVFORK))((void)0);
1239 mutex_exit(&p->p_lock)_sx_xunlock(((&p->p_lock)), ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (1239));
1240#else
1241 if ((p = pfind(probe->ftp_pid)) == NULL((void *)0))
1242 return;
1243#endif
1244
1245 /*
1246 * We have to enable the trap entry point before any user threads have
1247 * the chance to execute the trap instruction we're about to place
1248 * in their process's text.
1249 */
1250#ifdef __FreeBSD__11
1251 /*
1252 * pfind() returns a locked process.
1253 */
1254 _PHOLD(p)do { (void)0; do { } while (0); (p)->p_lock++; if (((p)->
p_flag & 0x10000000) == 0) faultin((p)); } while (0);
1255 PROC_UNLOCK(p)__mtx_unlock_flags(&((((&(p)->p_mtx))))->mtx_lock
, ((0)), ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (1255));
1256#endif
1257 fasttrap_enable_callbacks();
1258
1259 /*
1260 * Enable all the tracepoints and add this probe's id to each
1261 * tracepoint's list of active probes.
1262 */
1263 for (i = 0; i < probe->ftp_ntps; i++) {
1264 if ((rc = fasttrap_tracepoint_enable(p, probe, i)) != 0) {
1265 /*
1266 * If enabling the tracepoint failed completely,
1267 * we don't have to disable it; if the failure
1268 * was only partial we must disable it.
1269 */
1270 if (rc == FASTTRAP_ENABLE_FAIL1)
1271 i--;
1272 else
1273 ASSERT(rc == FASTTRAP_ENABLE_PARTIAL)((void)0);
1274
1275 /*
1276 * Back up and pull out all the tracepoints we've
1277 * created so far for this probe.
1278 */
1279 while (i >= 0) {
1280 fasttrap_tracepoint_disable(p, probe, i);
1281 i--;
1282 }
1283
1284#ifdef illumos
1285 mutex_enter(&p->p_lock)(void)_sx_xlock(((&p->p_lock)), 0, ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (1285));
1286 sprunlock(p);
1287#else
1288 PRELE(p)do { __mtx_lock_flags(&((((&((p))->p_mtx))))->mtx_lock
, ((0)), ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (1288)); do { (void)0; do { do { } while (0); } while (0);
(--((p))->p_lock); if ((((p))->p_flag & 0x02000) &&
((p))->p_lock == 0) wakeup(&((p))->p_lock); } while
(0); __mtx_unlock_flags(&((((&((p))->p_mtx))))->
mtx_lock, ((0)), ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (1288)); } while (0);
1289#endif
1290
1291 /*
1292 * Since we're not actually enabling this probe,
1293 * drop our reference on the trap table entry.
1294 */
1295 fasttrap_disable_callbacks();
1296 return;
1297 }
1298 }
1299#ifdef illumos
1300 mutex_enter(&p->p_lock)(void)_sx_xlock(((&p->p_lock)), 0, ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (1300));
1301 sprunlock(p);
1302#else
1303 PRELE(p)do { __mtx_lock_flags(&((((&((p))->p_mtx))))->mtx_lock
, ((0)), ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (1303)); do { (void)0; do { do { } while (0); } while (0);
(--((p))->p_lock); if ((((p))->p_flag & 0x02000) &&
((p))->p_lock == 0) wakeup(&((p))->p_lock); } while
(0); __mtx_unlock_flags(&((((&((p))->p_mtx))))->
mtx_lock, ((0)), ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (1303)); } while (0);
1304#endif
1305
1306 probe->ftp_enabled = 1;
1307}
1308
1309/*ARGSUSED*/
1310static void
1311fasttrap_pid_disable(void *arg, dtrace_id_t id, void *parg)
1312{
1313 fasttrap_probe_t *probe = parg;
1314 fasttrap_provider_t *provider = probe->ftp_prov;
1315 proc_t *p;
1316 int i, whack = 0;
1317
1318 ASSERT(id == probe->ftp_id)((void)0);
1319
1320 mutex_enter(&provider->ftp_mtx)(void)_sx_xlock(((&provider->ftp_mtx)), 0, ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (1320));
1321
1322 /*
1323 * We won't be able to acquire a /proc-esque lock on the process
1324 * iff the process is dead and gone. In this case, we rely on the
1325 * provider lock as a point of mutual exclusion to prevent other
1326 * DTrace consumers from disabling this probe.
1327 */
1328 if ((p = pfind(probe->ftp_pid)) != NULL((void *)0)) {
1329#ifdef __FreeBSD__11
1330 if (p->p_flag & P_WEXIT0x02000) {
1331 PROC_UNLOCK(p)__mtx_unlock_flags(&((((&(p)->p_mtx))))->mtx_lock
, ((0)), ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (1331));
1332 p = NULL((void *)0);
1333 } else {
1334 _PHOLD(p)do { (void)0; do { } while (0); (p)->p_lock++; if (((p)->
p_flag & 0x10000000) == 0) faultin((p)); } while (0);
1335 PROC_UNLOCK(p)__mtx_unlock_flags(&((((&(p)->p_mtx))))->mtx_lock
, ((0)), ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (1335));
1336 }
1337#endif
1338 }
1339
1340 /*
1341 * Disable all the associated tracepoints (for fully enabled probes).
1342 */
1343 if (probe->ftp_enabled) {
1344 for (i = 0; i < probe->ftp_ntps; i++) {
1345 fasttrap_tracepoint_disable(p, probe, i);
1346 }
1347 }
1348
1349 ASSERT(provider->ftp_rcount > 0)((void)0);
1350 provider->ftp_rcount--;
1351
1352 if (p != NULL((void *)0)) {
1353 /*
1354 * Even though we may not be able to remove it entirely, we
1355 * mark this retired provider to get a chance to remove some
1356 * of the associated probes.
1357 */
1358 if (provider->ftp_retired && !provider->ftp_marked)
1359 whack = provider->ftp_marked = 1;
1360 mutex_exit(&provider->ftp_mtx)_sx_xunlock(((&provider->ftp_mtx)), ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (1360));
1361 } else {
1362 /*
1363 * If the process is dead, we're just waiting for the
1364 * last probe to be disabled to be able to free it.
1365 */
1366 if (provider->ftp_rcount == 0 && !provider->ftp_marked)
1367 whack = provider->ftp_marked = 1;
1368 mutex_exit(&provider->ftp_mtx)_sx_xunlock(((&provider->ftp_mtx)), ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (1368));
1369 }
1370
1371 if (whack)
1372 fasttrap_pid_cleanup();
1373
1374#ifdef __FreeBSD__11
1375 if (p != NULL((void *)0))
1376 PRELE(p)do { __mtx_lock_flags(&((((&((p))->p_mtx))))->mtx_lock
, ((0)), ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (1376)); do { (void)0; do { do { } while (0); } while (0);
(--((p))->p_lock); if ((((p))->p_flag & 0x02000) &&
((p))->p_lock == 0) wakeup(&((p))->p_lock); } while
(0); __mtx_unlock_flags(&((((&((p))->p_mtx))))->
mtx_lock, ((0)), ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (1376)); } while (0);
1377#endif
1378 if (!probe->ftp_enabled)
1379 return;
1380
1381 probe->ftp_enabled = 0;
1382
1383#ifdef illumos
1384 ASSERT(MUTEX_HELD(&cpu_lock))((void)0);
1385#endif
1386 fasttrap_disable_callbacks();
1387}
1388
1389/*ARGSUSED*/
1390static void
1391fasttrap_pid_getargdesc(void *arg, dtrace_id_t id, void *parg,
1392 dtrace_argdesc_t *desc)
1393{
1394 fasttrap_probe_t *probe = parg;
1395 char *str;
1396 int i, ndx;
1397
1398 desc->dtargd_native[0] = '\0';
1399 desc->dtargd_xlate[0] = '\0';
1400
1401 if (probe->ftp_prov->ftp_retired != 0 ||
1402 desc->dtargd_ndx >= probe->ftp_nargs) {
1403 desc->dtargd_ndx = DTRACE_ARGNONE-1;
1404 return;
1405 }
1406
1407 ndx = (probe->ftp_argmap != NULL((void *)0)) ?
1408 probe->ftp_argmap[desc->dtargd_ndx] : desc->dtargd_ndx;
1409
1410 str = probe->ftp_ntypes;
1411 for (i = 0; i < ndx; i++) {
1412 str += strlen(str) + 1;
1413 }
1414
1415 ASSERT(strlen(str + 1) < sizeof (desc->dtargd_native))((void)0);
1416 (void) strcpy(desc->dtargd_native, str);
1417
1418 if (probe->ftp_xtypes == NULL((void *)0))
1419 return;
1420
1421 str = probe->ftp_xtypes;
1422 for (i = 0; i < desc->dtargd_ndx; i++) {
1423 str += strlen(str) + 1;
1424 }
1425
1426 ASSERT(strlen(str + 1) < sizeof (desc->dtargd_xlate))((void)0);
1427 (void) strcpy(desc->dtargd_xlate, str);
1428}
1429
1430/*ARGSUSED*/
1431static void
1432fasttrap_pid_destroy(void *arg, dtrace_id_t id, void *parg)
1433{
1434 fasttrap_probe_t *probe = parg;
1435 int i;
1436 size_t size;
1437
1438 ASSERT(probe != NULL)((void)0);
1439 ASSERT(!probe->ftp_enabled)((void)0);
1440 ASSERT(fasttrap_total >= probe->ftp_ntps)((void)0);
1441
1442 atomic_add_32atomic_add_int(&fasttrap_total, -probe->ftp_ntps);
1443 size = offsetof(fasttrap_probe_t, ftp_tps[probe->ftp_ntps])__builtin_offsetof(fasttrap_probe_t, ftp_tps[probe->ftp_ntps
]);
1444
1445 if (probe->ftp_gen + 1 >= fasttrap_mod_gen)
1446 fasttrap_mod_barrier(probe->ftp_gen);
1447
1448 for (i = 0; i < probe->ftp_ntps; i++) {
1449 kmem_free(probe->ftp_tps[i].fit_tp,zfs_kmem_free((probe->ftp_tps[i].fit_tp), (sizeof (fasttrap_tracepoint_t
)))
1450 sizeof (fasttrap_tracepoint_t))zfs_kmem_free((probe->ftp_tps[i].fit_tp), (sizeof (fasttrap_tracepoint_t
)));
1451 }
1452
1453 kmem_free(probe, size)zfs_kmem_free((probe), (size));
1454}
1455
1456
1457static const dtrace_pattr_t pid_attr = {
1458{ DTRACE_STABILITY_EVOLVING5, DTRACE_STABILITY_EVOLVING5, DTRACE_CLASS_ISA4 },
1459{ DTRACE_STABILITY_PRIVATE1, DTRACE_STABILITY_PRIVATE1, DTRACE_CLASS_UNKNOWN0 },
1460{ DTRACE_STABILITY_PRIVATE1, DTRACE_STABILITY_PRIVATE1, DTRACE_CLASS_UNKNOWN0 },
1461{ DTRACE_STABILITY_EVOLVING5, DTRACE_STABILITY_EVOLVING5, DTRACE_CLASS_ISA4 },
1462{ DTRACE_STABILITY_PRIVATE1, DTRACE_STABILITY_PRIVATE1, DTRACE_CLASS_UNKNOWN0 },
1463};
1464
1465static dtrace_pops_t pid_pops = {
1466 fasttrap_pid_provide,
1467 NULL((void *)0),
1468 fasttrap_pid_enable,
1469 fasttrap_pid_disable,
1470 NULL((void *)0),
1471 NULL((void *)0),
1472 fasttrap_pid_getargdesc,
1473 fasttrap_pid_getarg,
1474 NULL((void *)0),
1475 fasttrap_pid_destroy
1476};
1477
1478static dtrace_pops_t usdt_pops = {
1479 fasttrap_pid_provide,
1480 NULL((void *)0),
1481 fasttrap_pid_enable,
1482 fasttrap_pid_disable,
1483 NULL((void *)0),
1484 NULL((void *)0),
1485 fasttrap_pid_getargdesc,
1486 fasttrap_usdt_getarg,
1487 NULL((void *)0),
1488 fasttrap_pid_destroy
1489};
1490
1491static fasttrap_proc_t *
1492fasttrap_proc_lookup(pid_t pid)
1493{
1494 fasttrap_bucket_t *bucket;
1495 fasttrap_proc_t *fprc, *new_fprc;
1496
1497
1498 bucket = &fasttrap_procs.fth_table[FASTTRAP_PROCS_INDEX(pid)((pid) & fasttrap_procs.fth_mask)];
1499 mutex_enter(&bucket->ftb_mtx)(void)_sx_xlock(((&bucket->ftb_mtx)), 0, ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (1499));
1500
1501 for (fprc = bucket->ftb_data; fprc != NULL((void *)0); fprc = fprc->ftpc_next) {
1502 if (fprc->ftpc_pid == pid && fprc->ftpc_acount != 0) {
1503 mutex_enter(&fprc->ftpc_mtx)(void)_sx_xlock(((&fprc->ftpc_mtx)), 0, ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (1503));
1504 mutex_exit(&bucket->ftb_mtx)_sx_xunlock(((&bucket->ftb_mtx)), ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (1504));
1505 fprc->ftpc_rcount++;
1506 atomic_inc_64(&fprc->ftpc_acount);
1507 ASSERT(fprc->ftpc_acount <= fprc->ftpc_rcount)((void)0);
1508 mutex_exit(&fprc->ftpc_mtx)_sx_xunlock(((&fprc->ftpc_mtx)), ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (1508));
1509
1510 return (fprc);
1511 }
1512 }
1513
1514 /*
1515 * Drop the bucket lock so we don't try to perform a sleeping
1516 * allocation under it.
1517 */
1518 mutex_exit(&bucket->ftb_mtx)_sx_xunlock(((&bucket->ftb_mtx)), ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (1518));
1519
1520 new_fprc = kmem_zalloc(sizeof (fasttrap_proc_t), KM_SLEEP)zfs_kmem_alloc((sizeof (fasttrap_proc_t)), (0x0002) | 0x0100);
1521 new_fprc->ftpc_pid = pid;
1522 new_fprc->ftpc_rcount = 1;
1523 new_fprc->ftpc_acount = 1;
1524#ifndef illumos
1525 mutex_init(&new_fprc->ftpc_mtx, "fasttrap proc mtx", MUTEX_DEFAULT,do { const char *_name; ((void)0); do { } while (0); bzero((&
new_fprc->ftpc_mtx), sizeof(struct sx)); for (_name = "&new_fprc->ftpc_mtx"
; *_name != '\0'; _name++) { if (*_name >= 'a' && *
_name <= 'z') break; } if (*_name == '\0') _name = "&new_fprc->ftpc_mtx"
; sx_init_flags((&new_fprc->ftpc_mtx), _name, (0x01 | 0x04
)); } while (0)
1526 NULL)do { const char *_name; ((void)0); do { } while (0); bzero((&
new_fprc->ftpc_mtx), sizeof(struct sx)); for (_name = "&new_fprc->ftpc_mtx"
; *_name != '\0'; _name++) { if (*_name >= 'a' && *
_name <= 'z') break; } if (*_name == '\0') _name = "&new_fprc->ftpc_mtx"
; sx_init_flags((&new_fprc->ftpc_mtx), _name, (0x01 | 0x04
)); } while (0);
1527#endif
1528
1529 mutex_enter(&bucket->ftb_mtx)(void)_sx_xlock(((&bucket->ftb_mtx)), 0, ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (1529));
1530
1531 /*
1532 * Take another lap through the list to make sure a proc hasn't
1533 * been created for this pid while we weren't under the bucket lock.
1534 */
1535 for (fprc = bucket->ftb_data; fprc != NULL((void *)0); fprc = fprc->ftpc_next) {
1536 if (fprc->ftpc_pid == pid && fprc->ftpc_acount != 0) {
1537 mutex_enter(&fprc->ftpc_mtx)(void)_sx_xlock(((&fprc->ftpc_mtx)), 0, ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (1537));
1538 mutex_exit(&bucket->ftb_mtx)_sx_xunlock(((&bucket->ftb_mtx)), ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (1538));
1539 fprc->ftpc_rcount++;
1540 atomic_inc_64(&fprc->ftpc_acount);
1541 ASSERT(fprc->ftpc_acount <= fprc->ftpc_rcount)((void)0);
1542 mutex_exit(&fprc->ftpc_mtx)_sx_xunlock(((&fprc->ftpc_mtx)), ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (1542));
1543
1544 kmem_free(new_fprc, sizeof (fasttrap_proc_t))zfs_kmem_free((new_fprc), (sizeof (fasttrap_proc_t)));
1545
1546 return (fprc);
1547 }
1548 }
1549
1550 new_fprc->ftpc_next = bucket->ftb_data;
1551 bucket->ftb_data = new_fprc;
1552
1553 mutex_exit(&bucket->ftb_mtx)_sx_xunlock(((&bucket->ftb_mtx)), ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (1553));
1554
1555 return (new_fprc);
1556}
1557
1558static void
1559fasttrap_proc_release(fasttrap_proc_t *proc)
1560{
1561 fasttrap_bucket_t *bucket;
1562 fasttrap_proc_t *fprc, **fprcp;
1563 pid_t pid = proc->ftpc_pid;
1564#ifndef illumos
1565 fasttrap_scrblock_t *scrblk, *scrblktmp;
1566 fasttrap_scrspace_t *scrspc, *scrspctmp;
1567 struct proc *p;
1568 struct thread *td;
1569#endif
1570
1571 mutex_enter(&proc->ftpc_mtx)(void)_sx_xlock(((&proc->ftpc_mtx)), 0, ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (1571));
1572
1573 ASSERT(proc->ftpc_rcount != 0)((void)0);
1574 ASSERT(proc->ftpc_acount <= proc->ftpc_rcount)((void)0);
1575
1576 if (--proc->ftpc_rcount != 0) {
1577 mutex_exit(&proc->ftpc_mtx)_sx_xunlock(((&proc->ftpc_mtx)), ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (1577));
1578 return;
1579 }
1580
1581#ifndef illumos
1582 /*
1583 * Free all structures used to manage per-thread scratch space.
1584 */
1585 LIST_FOREACH_SAFE(scrblk, &proc->ftpc_scrblks, ftsb_next,for ((scrblk) = (((&proc->ftpc_scrblks))->lh_first)
; (scrblk) && ((scrblktmp) = (((scrblk))->ftsb_next
.le_next), 1); (scrblk) = (scrblktmp))
1586 scrblktmp)for ((scrblk) = (((&proc->ftpc_scrblks))->lh_first)
; (scrblk) && ((scrblktmp) = (((scrblk))->ftsb_next
.le_next), 1); (scrblk) = (scrblktmp)) {
1587 LIST_REMOVE(scrblk, ftsb_next)do { ; ; ; ; if ((((scrblk))->ftsb_next.le_next) != ((void
*)0)) (((scrblk))->ftsb_next.le_next)->ftsb_next.le_prev
= (scrblk)->ftsb_next.le_prev; *(scrblk)->ftsb_next.le_prev
= (((scrblk))->ftsb_next.le_next); ; ; } while (0);
1588 free(scrblk, M_SOLARIS);
1589 }
1590 LIST_FOREACH_SAFE(scrspc, &proc->ftpc_fscr, ftss_next, scrspctmp)for ((scrspc) = (((&proc->ftpc_fscr))->lh_first); (
scrspc) && ((scrspctmp) = (((scrspc))->ftss_next.le_next
), 1); (scrspc) = (scrspctmp)) {
1591 LIST_REMOVE(scrspc, ftss_next)do { ; ; ; ; if ((((scrspc))->ftss_next.le_next) != ((void
*)0)) (((scrspc))->ftss_next.le_next)->ftss_next.le_prev
= (scrspc)->ftss_next.le_prev; *(scrspc)->ftss_next.le_prev
= (((scrspc))->ftss_next.le_next); ; ; } while (0);
1592 free(scrspc, M_SOLARIS);
1593 }
1594 LIST_FOREACH_SAFE(scrspc, &proc->ftpc_ascr, ftss_next, scrspctmp)for ((scrspc) = (((&proc->ftpc_ascr))->lh_first); (
scrspc) && ((scrspctmp) = (((scrspc))->ftss_next.le_next
), 1); (scrspc) = (scrspctmp)) {
1595 LIST_REMOVE(scrspc, ftss_next)do { ; ; ; ; if ((((scrspc))->ftss_next.le_next) != ((void
*)0)) (((scrspc))->ftss_next.le_next)->ftss_next.le_prev
= (scrspc)->ftss_next.le_prev; *(scrspc)->ftss_next.le_prev
= (((scrspc))->ftss_next.le_next); ; ; } while (0);
1596 free(scrspc, M_SOLARIS);
1597 }
1598
1599 if ((p = pfind(pid)) != NULL((void *)0)) {
1600 FOREACH_THREAD_IN_PROC(p, td)for (((td)) = (((&(p)->p_threads))->tqh_first); ((td
)); ((td)) = ((((td)))->td_plist.tqe_next))
1601 td->t_dtrace_sscrtd_dtrace->td_dtrace_sscr = NULL((void *)0);
1602 PROC_UNLOCK(p)__mtx_unlock_flags(&((((&(p)->p_mtx))))->mtx_lock
, ((0)), ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (1602));
1603 }
1604#endif
1605
1606 mutex_exit(&proc->ftpc_mtx)_sx_xunlock(((&proc->ftpc_mtx)), ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (1606));
1607
1608 /*
1609 * There should definitely be no live providers associated with this
1610 * process at this point.
1611 */
1612 ASSERT(proc->ftpc_acount == 0)((void)0);
1613
1614 bucket = &fasttrap_procs.fth_table[FASTTRAP_PROCS_INDEX(pid)((pid) & fasttrap_procs.fth_mask)];
1615 mutex_enter(&bucket->ftb_mtx)(void)_sx_xlock(((&bucket->ftb_mtx)), 0, ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (1615));
1616
1617 fprcp = (fasttrap_proc_t **)&bucket->ftb_data;
1618 while ((fprc = *fprcp) != NULL((void *)0)) {
1619 if (fprc == proc)
1620 break;
1621
1622 fprcp = &fprc->ftpc_next;
1623 }
1624
1625 /*
1626 * Something strange has happened if we can't find the proc.
1627 */
1628 ASSERT(fprc != NULL)((void)0);
1629
1630 *fprcp = fprc->ftpc_next;
1631
1632 mutex_exit(&bucket->ftb_mtx)_sx_xunlock(((&bucket->ftb_mtx)), ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (1632));
1633
1634 kmem_free(fprc, sizeof (fasttrap_proc_t))zfs_kmem_free((fprc), (sizeof (fasttrap_proc_t)));
1635}
1636
1637/*
1638 * Lookup a fasttrap-managed provider based on its name and associated pid.
1639 * If the pattr argument is non-NULL, this function instantiates the provider
1640 * if it doesn't exist otherwise it returns NULL. The provider is returned
1641 * with its lock held.
1642 */
1643static fasttrap_provider_t *
1644fasttrap_provider_lookup(pid_t pid, const char *name,
1645 const dtrace_pattr_t *pattr)
1646{
1647 fasttrap_provider_t *fp, *new_fp = NULL((void *)0);
1648 fasttrap_bucket_t *bucket;
1649 char provname[DTRACE_PROVNAMELEN64];
1650 proc_t *p;
1651 cred_t *cred;
1652
1653 ASSERT(strlen(name) < sizeof (fp->ftp_name))((void)0);
1654 ASSERT(pattr != NULL)((void)0);
1655
1656 bucket = &fasttrap_provs.fth_table[FASTTRAP_PROVS_INDEX(pid, name)((fasttrap_hash_str(name) + (pid)) & fasttrap_provs.fth_mask
)];
1657 mutex_enter(&bucket->ftb_mtx)(void)_sx_xlock(((&bucket->ftb_mtx)), 0, ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (1657));
1658
1659 /*
1660 * Take a lap through the list and return the match if we find it.
1661 */
1662 for (fp = bucket->ftb_data; fp != NULL((void *)0); fp = fp->ftp_next) {
1663 if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 &&
1664 !fp->ftp_retired) {
1665 mutex_enter(&fp->ftp_mtx)(void)_sx_xlock(((&fp->ftp_mtx)), 0, ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (1665));
1666 mutex_exit(&bucket->ftb_mtx)_sx_xunlock(((&bucket->ftb_mtx)), ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (1666));
1667 return (fp);
1668 }
1669 }
1670
1671 /*
1672 * Drop the bucket lock so we don't try to perform a sleeping
1673 * allocation under it.
1674 */
1675 mutex_exit(&bucket->ftb_mtx)_sx_xunlock(((&bucket->ftb_mtx)), ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (1675));
1676
1677 /*
1678 * Make sure the process exists, isn't a child created as the result
1679 * of a vfork(2), and isn't a zombie (but may be in fork).
1680 */
1681 if ((p = pfind(pid)) == NULL((void *)0))
1682 return (NULL((void *)0));
1683
1684 /*
1685 * Increment p_dtrace_probes so that the process knows to inform us
1686 * when it exits or execs. fasttrap_provider_free() decrements this
1687 * when we're done with this provider.
1688 */
1689 p->p_dtrace_probesp_dtrace->p_dtrace_probes++;
1690
1691 /*
1692 * Grab the credentials for this process so we have
1693 * something to pass to dtrace_register().
1694 */
1695 PROC_LOCK_ASSERT(p, MA_OWNED)(void)0;
1696 crhold(p->p_ucred);
1697 cred = p->p_ucred;
1698 PROC_UNLOCK(p)__mtx_unlock_flags(&((((&(p)->p_mtx))))->mtx_lock
, ((0)), ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (1698));
1699
1700 new_fp = kmem_zalloc(sizeof (fasttrap_provider_t), KM_SLEEP)zfs_kmem_alloc((sizeof (fasttrap_provider_t)), (0x0002) | 0x0100
);
1701 new_fp->ftp_pid = pid;
1702 new_fp->ftp_proc = fasttrap_proc_lookup(pid);
1703#ifndef illumos
1704 mutex_init(&new_fp->ftp_mtx, "provider mtx", MUTEX_DEFAULT, NULL)do { const char *_name; ((void)0); do { } while (0); bzero((&
new_fp->ftp_mtx), sizeof(struct sx)); for (_name = "&new_fp->ftp_mtx"
; *_name != '\0'; _name++) { if (*_name >= 'a' && *
_name <= 'z') break; } if (*_name == '\0') _name = "&new_fp->ftp_mtx"
; sx_init_flags((&new_fp->ftp_mtx), _name, (0x01 | 0x04
)); } while (0);
1705 mutex_init(&new_fp->ftp_cmtx, "lock on creating", MUTEX_DEFAULT, NULL)do { const char *_name; ((void)0); do { } while (0); bzero((&
new_fp->ftp_cmtx), sizeof(struct sx)); for (_name = "&new_fp->ftp_cmtx"
; *_name != '\0'; _name++) { if (*_name >= 'a' && *
_name <= 'z') break; } if (*_name == '\0') _name = "&new_fp->ftp_cmtx"
; sx_init_flags((&new_fp->ftp_cmtx), _name, (0x01 | 0x04
)); } while (0);
1706#endif
1707
1708 ASSERT(new_fp->ftp_proc != NULL)((void)0);
1709
1710 mutex_enter(&bucket->ftb_mtx)(void)_sx_xlock(((&bucket->ftb_mtx)), 0, ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (1710));
1711
1712 /*
1713 * Take another lap through the list to make sure a provider hasn't
1714 * been created for this pid while we weren't under the bucket lock.
1715 */
1716 for (fp = bucket->ftb_data; fp != NULL((void *)0); fp = fp->ftp_next) {
1717 if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 &&
1718 !fp->ftp_retired) {
1719 mutex_enter(&fp->ftp_mtx)(void)_sx_xlock(((&fp->ftp_mtx)), 0, ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (1719));
1720 mutex_exit(&bucket->ftb_mtx)_sx_xunlock(((&bucket->ftb_mtx)), ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (1720));
1721 fasttrap_provider_free(new_fp);
1722 crfree(cred);
1723 return (fp);
1724 }
1725 }
1726
1727 (void) strcpy(new_fp->ftp_name, name);
1728
1729 /*
1730 * Fail and return NULL if either the provider name is too long
1731 * or we fail to register this new provider with the DTrace
1732 * framework. Note that this is the only place we ever construct
1733 * the full provider name -- we keep it in pieces in the provider
1734 * structure.
1735 */
1736 if (snprintf(provname, sizeof (provname), "%s%u", name, (uint_t)pid) >=
1737 sizeof (provname) ||
1738 dtrace_register(provname, pattr,
1739 DTRACE_PRIV_PROC0x0004 | DTRACE_PRIV_OWNER0x0008 | DTRACE_PRIV_ZONEOWNER0x0010, cred,
1740 pattr == &pid_attr ? &pid_pops : &usdt_pops, new_fp,
1741 &new_fp->ftp_provid) != 0) {
1742 mutex_exit(&bucket->ftb_mtx)_sx_xunlock(((&bucket->ftb_mtx)), ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (1742));
1743 fasttrap_provider_free(new_fp);
1744 crfree(cred);
1745 return (NULL((void *)0));
1746 }
1747
1748 new_fp->ftp_next = bucket->ftb_data;
1749 bucket->ftb_data = new_fp;
1750
1751 mutex_enter(&new_fp->ftp_mtx)(void)_sx_xlock(((&new_fp->ftp_mtx)), 0, ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (1751));
1752 mutex_exit(&bucket->ftb_mtx)_sx_xunlock(((&bucket->ftb_mtx)), ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (1752));
1753
1754 crfree(cred);
1755 return (new_fp);
1756}
1757
1758static void
1759fasttrap_provider_free(fasttrap_provider_t *provider)
1760{
1761 pid_t pid = provider->ftp_pid;
1762 proc_t *p;
1763
1764 /*
1765 * There need to be no associated enabled probes, no consumers
1766 * creating probes, and no meta providers referencing this provider.
1767 */
1768 ASSERT(provider->ftp_rcount == 0)((void)0);
1769 ASSERT(provider->ftp_ccount == 0)((void)0);
1770 ASSERT(provider->ftp_mcount == 0)((void)0);
1771
1772 /*
1773 * If this provider hasn't been retired, we need to explicitly drop the
1774 * count of active providers on the associated process structure.
1775 */
1776 if (!provider->ftp_retired) {
1777 atomic_dec_64(&provider->ftp_proc->ftpc_acount);
1778 ASSERT(provider->ftp_proc->ftpc_acount <((void)0)
1779 provider->ftp_proc->ftpc_rcount)((void)0);
1780 }
1781
1782 fasttrap_proc_release(provider->ftp_proc);
1783
1784#ifndef illumos
1785 mutex_destroy(&provider->ftp_mtx)sx_destroy(&provider->ftp_mtx);
1786 mutex_destroy(&provider->ftp_cmtx)sx_destroy(&provider->ftp_cmtx);
1787#endif
1788 kmem_free(provider, sizeof (fasttrap_provider_t))zfs_kmem_free((provider), (sizeof (fasttrap_provider_t)));
1789
1790 /*
1791 * Decrement p_dtrace_probes on the process whose provider we're
1792 * freeing. We don't have to worry about clobbering somone else's
1793 * modifications to it because we have locked the bucket that
1794 * corresponds to this process's hash chain in the provider hash
1795 * table. Don't sweat it if we can't find the process.
1796 */
1797 if ((p = pfind(pid)) == NULL((void *)0)) {
1798 return;
1799 }
1800
1801 p->p_dtrace_probesp_dtrace->p_dtrace_probes--;
1802#ifndef illumos
1803 PROC_UNLOCK(p)__mtx_unlock_flags(&((((&(p)->p_mtx))))->mtx_lock
, ((0)), ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (1803));
1804#endif
1805}
1806
1807static void
1808fasttrap_provider_retire(pid_t pid, const char *name, int mprov)
1809{
1810 fasttrap_provider_t *fp;
1811 fasttrap_bucket_t *bucket;
1812 dtrace_provider_id_t provid;
1813
1814 ASSERT(strlen(name) < sizeof (fp->ftp_name))((void)0);
1815
1816 bucket = &fasttrap_provs.fth_table[FASTTRAP_PROVS_INDEX(pid, name)((fasttrap_hash_str(name) + (pid)) & fasttrap_provs.fth_mask
)];
1817 mutex_enter(&bucket->ftb_mtx)(void)_sx_xlock(((&bucket->ftb_mtx)), 0, ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (1817));
1818
1819 for (fp = bucket->ftb_data; fp != NULL((void *)0); fp = fp->ftp_next) {
1820 if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 &&
1821 !fp->ftp_retired)
1822 break;
1823 }
1824
1825 if (fp == NULL((void *)0)) {
1826 mutex_exit(&bucket->ftb_mtx)_sx_xunlock(((&bucket->ftb_mtx)), ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (1826));
1827 return;
1828 }
1829
1830 mutex_enter(&fp->ftp_mtx)(void)_sx_xlock(((&fp->ftp_mtx)), 0, ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (1830));
1831 ASSERT(!mprov || fp->ftp_mcount > 0)((void)0);
1832 if (mprov && --fp->ftp_mcount != 0) {
1833 mutex_exit(&fp->ftp_mtx)_sx_xunlock(((&fp->ftp_mtx)), ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (1833));
1834 mutex_exit(&bucket->ftb_mtx)_sx_xunlock(((&bucket->ftb_mtx)), ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (1834));
1835 return;
1836 }
1837
1838 /*
1839 * Mark the provider to be removed in our post-processing step, mark it
1840 * retired, and drop the active count on its proc. Marking it indicates
1841 * that we should try to remove it; setting the retired flag indicates
1842 * that we're done with this provider; dropping the active the proc
1843 * releases our hold, and when this reaches zero (as it will during
1844 * exit or exec) the proc and associated providers become defunct.
1845 *
1846 * We obviously need to take the bucket lock before the provider lock
1847 * to perform the lookup, but we need to drop the provider lock
1848 * before calling into the DTrace framework since we acquire the
1849 * provider lock in callbacks invoked from the DTrace framework. The
1850 * bucket lock therefore protects the integrity of the provider hash
1851 * table.
1852 */
1853 atomic_dec_64(&fp->ftp_proc->ftpc_acount);
1854 ASSERT(fp->ftp_proc->ftpc_acount < fp->ftp_proc->ftpc_rcount)((void)0);
1855
1856 fp->ftp_retired = 1;
1857 fp->ftp_marked = 1;
1858 provid = fp->ftp_provid;
1859 mutex_exit(&fp->ftp_mtx)_sx_xunlock(((&fp->ftp_mtx)), ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (1859));
1860
1861 /*
1862 * We don't have to worry about invalidating the same provider twice
1863 * since fasttrap_provider_lookup() will ignore provider that have
1864 * been marked as retired.
1865 */
1866 dtrace_invalidate(provid);
1867
1868 mutex_exit(&bucket->ftb_mtx)_sx_xunlock(((&bucket->ftb_mtx)), ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (1868));
1869
1870 fasttrap_pid_cleanup();
1871}
1872
1873static int
1874fasttrap_uint32_cmp(const void *ap, const void *bp)
1875{
1876 return (*(const uint32_t *)ap - *(const uint32_t *)bp);
1877}
1878
1879static int
1880fasttrap_uint64_cmp(const void *ap, const void *bp)
1881{
1882 return (*(const uint64_t *)ap - *(const uint64_t *)bp);
1883}
1884
1885static int
1886fasttrap_add_probe(fasttrap_probe_spec_t *pdata)
1887{
1888 fasttrap_provider_t *provider;
1889 fasttrap_probe_t *pp;
1890 fasttrap_tracepoint_t *tp;
1891 char *name;
1892 int i, aframes = 0, whack;
1893
1894 /*
1895 * There needs to be at least one desired trace point.
1896 */
1897 if (pdata->ftps_noffs == 0)
1898 return (EINVAL22);
1899
1900 switch (pdata->ftps_type) {
1901 case DTFTP_ENTRY:
1902 name = "entry";
1903 aframes = FASTTRAP_ENTRY_AFRAMES3;
1904 break;
1905 case DTFTP_RETURN:
1906 name = "return";
1907 aframes = FASTTRAP_RETURN_AFRAMES4;
1908 break;
1909 case DTFTP_OFFSETS:
1910 name = NULL((void *)0);
1911 break;
1912 default:
1913 return (EINVAL22);
1914 }
1915
1916 if ((provider = fasttrap_provider_lookup(pdata->ftps_pid,
1917 FASTTRAP_PID_NAME"pid", &pid_attr)) == NULL((void *)0))
1918 return (ESRCH3);
1919
1920 /*
1921 * Increment this reference count to indicate that a consumer is
1922 * actively adding a new probe associated with this provider. This
1923 * prevents the provider from being deleted -- we'll need to check
1924 * for pending deletions when we drop this reference count.
1925 */
1926 provider->ftp_ccount++;
1927 mutex_exit(&provider->ftp_mtx)_sx_xunlock(((&provider->ftp_mtx)), ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (1927));
1928
1929 /*
1930 * Grab the creation lock to ensure consistency between calls to
1931 * dtrace_probe_lookup() and dtrace_probe_create() in the face of
1932 * other threads creating probes. We must drop the provider lock
1933 * before taking this lock to avoid a three-way deadlock with the
1934 * DTrace framework.
1935 */
1936 mutex_enter(&provider->ftp_cmtx)(void)_sx_xlock(((&provider->ftp_cmtx)), 0, ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (1936));
1937
1938 if (name == NULL((void *)0)) {
1939 for (i = 0; i < pdata->ftps_noffs; i++) {
1940 char name_str[17];
1941
1942 (void) sprintf(name_str, "%llx",
1943 (unsigned long long)pdata->ftps_offs[i]);
1944
1945 if (dtrace_probe_lookup(provider->ftp_provid,
1946 pdata->ftps_mod, pdata->ftps_func, name_str) != 0)
1947 continue;
1948
1949 atomic_inc_32(&fasttrap_total);
1950
1951 if (fasttrap_total > fasttrap_max) {
1952 atomic_dec_32(&fasttrap_total);
1953 goto no_mem;
1954 }
1955
1956 pp = kmem_zalloc(sizeof (fasttrap_probe_t), KM_SLEEP)zfs_kmem_alloc((sizeof (fasttrap_probe_t)), (0x0002) | 0x0100
);
1957
1958 pp->ftp_prov = provider;
1959 pp->ftp_faddr = pdata->ftps_pc;
1960 pp->ftp_fsize = pdata->ftps_size;
1961 pp->ftp_pid = pdata->ftps_pid;
1962 pp->ftp_ntps = 1;
1963
1964 tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t),zfs_kmem_alloc((sizeof (fasttrap_tracepoint_t)), (0x0002) | 0x0100
)
1965 KM_SLEEP)zfs_kmem_alloc((sizeof (fasttrap_tracepoint_t)), (0x0002) | 0x0100
);
1966
1967 tp->ftt_proc = provider->ftp_proc;
1968 tp->ftt_pc = pdata->ftps_offs[i] + pdata->ftps_pc;
1969 tp->ftt_pid = pdata->ftps_pid;
1970
1971 pp->ftp_tps[0].fit_tp = tp;
1972 pp->ftp_tps[0].fit_id.fti_probe = pp;
1973 pp->ftp_tps[0].fit_id.fti_ptype = pdata->ftps_type;
1974
1975 pp->ftp_id = dtrace_probe_create(provider->ftp_provid,
1976 pdata->ftps_mod, pdata->ftps_func, name_str,
1977 FASTTRAP_OFFSET_AFRAMES3, pp);
1978 }
1979
1980 } else if (dtrace_probe_lookup(provider->ftp_provid, pdata->ftps_mod,
1981 pdata->ftps_func, name) == 0) {
1982 atomic_add_32atomic_add_int(&fasttrap_total, pdata->ftps_noffs);
1983
1984 if (fasttrap_total > fasttrap_max) {
1985 atomic_add_32atomic_add_int(&fasttrap_total, -pdata->ftps_noffs);
1986 goto no_mem;
1987 }
1988
1989 /*
1990 * Make sure all tracepoint program counter values are unique.
1991 * We later assume that each probe has exactly one tracepoint
1992 * for a given pc.
1993 */
1994 qsort(pdata->ftps_offs, pdata->ftps_noffs,
1995 sizeof (uint64_t), fasttrap_uint64_cmp);
1996 for (i = 1; i < pdata->ftps_noffs; i++) {
1997 if (pdata->ftps_offs[i] > pdata->ftps_offs[i - 1])
1998 continue;
1999
2000 atomic_add_32atomic_add_int(&fasttrap_total, -pdata->ftps_noffs);
2001 goto no_mem;
2002 }
2003
2004 ASSERT(pdata->ftps_noffs > 0)((void)0);
2005 pp = kmem_zalloc(offsetof(fasttrap_probe_t,zfs_kmem_alloc((__builtin_offsetof(fasttrap_probe_t, ftp_tps[
pdata->ftps_noffs])), (0x0002) | 0x0100)
2006 ftp_tps[pdata->ftps_noffs]), KM_SLEEP)zfs_kmem_alloc((__builtin_offsetof(fasttrap_probe_t, ftp_tps[
pdata->ftps_noffs])), (0x0002) | 0x0100);
2007
2008 pp->ftp_prov = provider;
2009 pp->ftp_faddr = pdata->ftps_pc;
2010 pp->ftp_fsize = pdata->ftps_size;
2011 pp->ftp_pid = pdata->ftps_pid;
2012 pp->ftp_ntps = pdata->ftps_noffs;
2013
2014 for (i = 0; i < pdata->ftps_noffs; i++) {
2015 tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t),zfs_kmem_alloc((sizeof (fasttrap_tracepoint_t)), (0x0002) | 0x0100
)
2016 KM_SLEEP)zfs_kmem_alloc((sizeof (fasttrap_tracepoint_t)), (0x0002) | 0x0100
);
2017
2018 tp->ftt_proc = provider->ftp_proc;
2019 tp->ftt_pc = pdata->ftps_offs[i] + pdata->ftps_pc;
2020 tp->ftt_pid = pdata->ftps_pid;
2021
2022 pp->ftp_tps[i].fit_tp = tp;
2023 pp->ftp_tps[i].fit_id.fti_probe = pp;
2024 pp->ftp_tps[i].fit_id.fti_ptype = pdata->ftps_type;
2025 }
2026
2027 pp->ftp_id = dtrace_probe_create(provider->ftp_provid,
2028 pdata->ftps_mod, pdata->ftps_func, name, aframes, pp);
2029 }
2030
2031 mutex_exit(&provider->ftp_cmtx)_sx_xunlock(((&provider->ftp_cmtx)), ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (2031));
2032
2033 /*
2034 * We know that the provider is still valid since we incremented the
2035 * creation reference count. If someone tried to clean up this provider
2036 * while we were using it (e.g. because the process called exec(2) or
2037 * exit(2)), take note of that and try to clean it up now.
2038 */
2039 mutex_enter(&provider->ftp_mtx)(void)_sx_xlock(((&provider->ftp_mtx)), 0, ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (2039));
2040 provider->ftp_ccount--;
2041 whack = provider->ftp_retired;
2042 mutex_exit(&provider->ftp_mtx)_sx_xunlock(((&provider->ftp_mtx)), ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (2042));
2043
2044 if (whack)
2045 fasttrap_pid_cleanup();
2046
2047 return (0);
2048
2049no_mem:
2050 /*
2051 * If we've exhausted the allowable resources, we'll try to remove
2052 * this provider to free some up. This is to cover the case where
2053 * the user has accidentally created many more probes than was
2054 * intended (e.g. pid123:::).
2055 */
2056 mutex_exit(&provider->ftp_cmtx)_sx_xunlock(((&provider->ftp_cmtx)), ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (2056));
2057 mutex_enter(&provider->ftp_mtx)(void)_sx_xlock(((&provider->ftp_mtx)), 0, ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (2057));
2058 provider->ftp_ccount--;
2059 provider->ftp_marked = 1;
2060 mutex_exit(&provider->ftp_mtx)_sx_xunlock(((&provider->ftp_mtx)), ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (2060));
2061
2062 fasttrap_pid_cleanup();
2063
2064 return (ENOMEM12);
2065}
2066
2067/*ARGSUSED*/
2068static void *
2069fasttrap_meta_provide(void *arg, dtrace_helper_provdesc_t *dhpv, pid_t pid)
2070{
2071 fasttrap_provider_t *provider;
2072
2073 /*
2074 * A 32-bit unsigned integer (like a pid for example) can be
2075 * expressed in 10 or fewer decimal digits. Make sure that we'll
2076 * have enough space for the provider name.
2077 */
2078 if (strlen(dhpv->dthpv_provname) + 10 >=
2079 sizeof (provider->ftp_name)) {
2080 printf("failed to instantiate provider %s: "
2081 "name too long to accomodate pid", dhpv->dthpv_provname);
2082 return (NULL((void *)0));
2083 }
2084
2085 /*
2086 * Don't let folks spoof the true pid provider.
2087 */
2088 if (strcmp(dhpv->dthpv_provname, FASTTRAP_PID_NAME"pid") == 0) {
2089 printf("failed to instantiate provider %s: "
2090 "%s is an invalid name", dhpv->dthpv_provname,
2091 FASTTRAP_PID_NAME"pid");
2092 return (NULL((void *)0));
2093 }
2094
2095 /*
2096 * The highest stability class that fasttrap supports is ISA; cap
2097 * the stability of the new provider accordingly.
2098 */
2099 if (dhpv->dthpv_pattr.dtpa_provider.dtat_class > DTRACE_CLASS_ISA4)
2100 dhpv->dthpv_pattr.dtpa_provider.dtat_class = DTRACE_CLASS_ISA4;
2101 if (dhpv->dthpv_pattr.dtpa_mod.dtat_class > DTRACE_CLASS_ISA4)
2102 dhpv->dthpv_pattr.dtpa_mod.dtat_class = DTRACE_CLASS_ISA4;
2103 if (dhpv->dthpv_pattr.dtpa_func.dtat_class > DTRACE_CLASS_ISA4)
2104 dhpv->dthpv_pattr.dtpa_func.dtat_class = DTRACE_CLASS_ISA4;
2105 if (dhpv->dthpv_pattr.dtpa_name.dtat_class > DTRACE_CLASS_ISA4)
2106 dhpv->dthpv_pattr.dtpa_name.dtat_class = DTRACE_CLASS_ISA4;
2107 if (dhpv->dthpv_pattr.dtpa_args.dtat_class > DTRACE_CLASS_ISA4)
2108 dhpv->dthpv_pattr.dtpa_args.dtat_class = DTRACE_CLASS_ISA4;
2109
2110 if ((provider = fasttrap_provider_lookup(pid, dhpv->dthpv_provname,
2111 &dhpv->dthpv_pattr)) == NULL((void *)0)) {
2112 printf("failed to instantiate provider %s for "
2113 "process %u", dhpv->dthpv_provname, (uint_t)pid);
2114 return (NULL((void *)0));
2115 }
2116
2117 /*
2118 * Up the meta provider count so this provider isn't removed until
2119 * the meta provider has been told to remove it.
2120 */
2121 provider->ftp_mcount++;
2122
2123 mutex_exit(&provider->ftp_mtx)_sx_xunlock(((&provider->ftp_mtx)), ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (2123));
2124
2125 return (provider);
2126}
2127
2128/*
2129 * We know a few things about our context here: we know that the probe being
2130 * created doesn't already exist (DTrace won't load DOF at the same address
2131 * twice, even if explicitly told to do so) and we know that we are
2132 * single-threaded with respect to the meta provider machinery. Knowing that
2133 * this is a new probe and that there is no way for us to race with another
2134 * operation on this provider allows us an important optimization: we need not
2135 * lookup a probe before adding it. Saving this lookup is important because
2136 * this code is in the fork path for processes with USDT probes, and lookups
2137 * here are potentially very expensive because of long hash conflicts on
2138 * module, function and name (DTrace doesn't hash on provider name).
2139 */
2140/*ARGSUSED*/
2141static void
2142fasttrap_meta_create_probe(void *arg, void *parg,
2143 dtrace_helper_probedesc_t *dhpb)
2144{
2145 fasttrap_provider_t *provider = parg;
2146 fasttrap_probe_t *pp;
2147 fasttrap_tracepoint_t *tp;
2148 int i, j;
2149 uint32_t ntps;
2150
2151 /*
2152 * Since the meta provider count is non-zero we don't have to worry
2153 * about this provider disappearing.
2154 */
2155 ASSERT(provider->ftp_mcount > 0)((void)0);
2156
2157 /*
2158 * The offsets must be unique.
2159 */
2160 qsort(dhpb->dthpb_offs, dhpb->dthpb_noffs, sizeof (uint32_t),
2161 fasttrap_uint32_cmp);
2162 for (i = 1; i < dhpb->dthpb_noffs; i++) {
2163 if (dhpb->dthpb_base + dhpb->dthpb_offs[i] <=
2164 dhpb->dthpb_base + dhpb->dthpb_offs[i - 1])
2165 return;
2166 }
2167
2168 qsort(dhpb->dthpb_enoffs, dhpb->dthpb_nenoffs, sizeof (uint32_t),
2169 fasttrap_uint32_cmp);
2170 for (i = 1; i < dhpb->dthpb_nenoffs; i++) {
2171 if (dhpb->dthpb_base + dhpb->dthpb_enoffs[i] <=
2172 dhpb->dthpb_base + dhpb->dthpb_enoffs[i - 1])
2173 return;
2174 }
2175
2176 ntps = dhpb->dthpb_noffs + dhpb->dthpb_nenoffs;
2177 ASSERT(ntps > 0)((void)0);
2178
2179 atomic_add_32atomic_add_int(&fasttrap_total, ntps);
2180
2181 if (fasttrap_total > fasttrap_max) {
2182 atomic_add_32atomic_add_int(&fasttrap_total, -ntps);
2183 return;
2184 }
2185
2186 pp = kmem_zalloc(offsetof(fasttrap_probe_t, ftp_tps[ntps]), KM_SLEEP)zfs_kmem_alloc((__builtin_offsetof(fasttrap_probe_t, ftp_tps[
ntps])), (0x0002) | 0x0100);
2187
2188 pp->ftp_prov = provider;
2189 pp->ftp_pid = provider->ftp_pid;
2190 pp->ftp_ntps = ntps;
2191 pp->ftp_nargs = dhpb->dthpb_xargc;
2192 pp->ftp_xtypes = dhpb->dthpb_xtypes;
2193 pp->ftp_ntypes = dhpb->dthpb_ntypes;
2194
2195 /*
2196 * First create a tracepoint for each actual point of interest.
2197 */
2198 for (i = 0; i < dhpb->dthpb_noffs; i++) {
2199 tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), KM_SLEEP)zfs_kmem_alloc((sizeof (fasttrap_tracepoint_t)), (0x0002) | 0x0100
);
2200
2201 tp->ftt_proc = provider->ftp_proc;
2202 tp->ftt_pc = dhpb->dthpb_base + dhpb->dthpb_offs[i];
2203 tp->ftt_pid = provider->ftp_pid;
2204
2205 pp->ftp_tps[i].fit_tp = tp;
2206 pp->ftp_tps[i].fit_id.fti_probe = pp;
2207#ifdef __sparc
2208 pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_POST_OFFSETS;
2209#else
2210 pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_OFFSETS;
2211#endif
2212 }
2213
2214 /*
2215 * Then create a tracepoint for each is-enabled point.
2216 */
2217 for (j = 0; i < ntps; i++, j++) {
2218 tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), KM_SLEEP)zfs_kmem_alloc((sizeof (fasttrap_tracepoint_t)), (0x0002) | 0x0100
);
2219
2220 tp->ftt_proc = provider->ftp_proc;
2221 tp->ftt_pc = dhpb->dthpb_base + dhpb->dthpb_enoffs[j];
2222 tp->ftt_pid = provider->ftp_pid;
2223
2224 pp->ftp_tps[i].fit_tp = tp;
2225 pp->ftp_tps[i].fit_id.fti_probe = pp;
2226 pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_IS_ENABLED;
2227 }
2228
2229 /*
2230 * If the arguments are shuffled around we set the argument remapping
2231 * table. Later, when the probe fires, we only remap the arguments
2232 * if the table is non-NULL.
2233 */
2234 for (i = 0; i < dhpb->dthpb_xargc; i++) {
2235 if (dhpb->dthpb_args[i] != i) {
2236 pp->ftp_argmap = dhpb->dthpb_args;
2237 break;
2238 }
2239 }
2240
2241 /*
2242 * The probe is fully constructed -- register it with DTrace.
2243 */
2244 pp->ftp_id = dtrace_probe_create(provider->ftp_provid, dhpb->dthpb_mod,
2245 dhpb->dthpb_func, dhpb->dthpb_name, FASTTRAP_OFFSET_AFRAMES3, pp);
2246}
2247
2248/*ARGSUSED*/
2249static void
2250fasttrap_meta_remove(void *arg, dtrace_helper_provdesc_t *dhpv, pid_t pid)
2251{
2252 /*
2253 * Clean up the USDT provider. There may be active consumers of the
2254 * provider busy adding probes, no damage will actually befall the
2255 * provider until that count has dropped to zero. This just puts
2256 * the provider on death row.
2257 */
2258 fasttrap_provider_retire(pid, dhpv->dthpv_provname, 1);
2259}
2260
2261static dtrace_mops_t fasttrap_mops = {
2262 fasttrap_meta_create_probe,
2263 fasttrap_meta_provide,
2264 fasttrap_meta_remove
2265};
2266
2267/*ARGSUSED*/
2268static int
2269fasttrap_open(struct cdev *dev __unused__attribute__((__unused__)), int oflags __unused__attribute__((__unused__)),
2270 int devtype __unused__attribute__((__unused__)), struct thread *td __unused__attribute__((__unused__)))
2271{
2272 return (0);
2273}
2274
2275/*ARGSUSED*/
2276static int
2277fasttrap_ioctl(struct cdev *dev, u_long cmd, caddr_t arg, int fflag,
2278 struct thread *td)
2279{
2280#ifdef notyet
2281 struct kinfo_proc kp;
2282 const cred_t *cr = td->td_ucred;
2283#endif
2284 if (!dtrace_attached())
1
Taking false branch
→
2285 return (EAGAIN35);
2286
2287 if (cmd == FASTTRAPIOC_MAKEPROBE((unsigned long) ((0x20000000) | (((0) & ((1 << 13)
- 1)) << 16) | ((('f')) << 8) | ((3))))) {
2
←
Taking false branch
→
2288 fasttrap_probe_spec_t *uprobe = *(fasttrap_probe_spec_t **)arg;
2289 fasttrap_probe_spec_t *probe;
2290 uint64_t noffs;
2291 size_t size;
2292 int ret, err;
2293
2294 if (copyin(&uprobe->ftps_noffs, &noffs,
2295 sizeof (uprobe->ftps_noffs)))
2296 return (EFAULT14);
2297
2298 /*
2299 * Probes must have at least one tracepoint.
2300 */
2301 if (noffs == 0)
2302 return (EINVAL22);
2303
2304 size = sizeof (fasttrap_probe_spec_t) +
2305 sizeof (probe->ftps_offs[0]) * (noffs - 1);
2306
2307 if (size > 1024 * 1024)
2308 return (ENOMEM12);
2309
2310 probe = kmem_alloc(size, KM_SLEEP)zfs_kmem_alloc((size), (0x0002));
2311
2312 if (copyin(uprobe, probe, size) != 0 ||
2313 probe->ftps_noffs != noffs) {
2314 kmem_free(probe, size)zfs_kmem_free((probe), (size));
2315 return (EFAULT14);
2316 }
2317
2318 /*
2319 * Verify that the function and module strings contain no
2320 * funny characters.
2321 */
2322 if (u8_validate(probe->ftps_func, strlen(probe->ftps_func),
2323 NULL((void *)0), U8_VALIDATE_ENTIRE(0x00100000), &err) < 0) {
2324 ret = EINVAL22;
2325 goto err;
2326 }
2327
2328 if (u8_validate(probe->ftps_mod, strlen(probe->ftps_mod),
2329 NULL((void *)0), U8_VALIDATE_ENTIRE(0x00100000), &err) < 0) {
2330 ret = EINVAL22;
2331 goto err;
2332 }
2333
2334#ifdef notyet
2335 if (!PRIV_POLICY_CHOICE(cr, PRIV_ALL, B_FALSE)priv_policy_choice((cr), (PRIV_ALL), (0))) {
2336 proc_t *p;
2337 pid_t pid = probe->ftps_pid;
2338
2339#ifdef illumos
2340 mutex_enter(&pidlock)(void)_sx_xlock(((&pidlock)), 0, ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (2340));
2341#endif
2342 /*
2343 * Report an error if the process doesn't exist
2344 * or is actively being birthed.
2345 */
2346 sx_slock(&proctree_lock)(void)_sx_slock(((&proctree_lock)), 0, ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (2346));
2347 p = pfind(pid);
2348 if (p)
2349 fill_kinfo_proc(p, &kp);
2350 sx_sunlock(&proctree_lock)_sx_sunlock(((&proctree_lock)), ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (2350));
2351 if (p == NULL((void *)0) || kp.ki_stat == SIDL1) {
2352#ifdef illumos
2353 mutex_exit(&pidlock)_sx_xunlock(((&pidlock)), ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (2353));
2354#endif
2355 return (ESRCH3);
2356 }
2357#ifdef illumos
2358 mutex_enter(&p->p_lock)(void)_sx_xlock(((&p->p_lock)), 0, ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (2358));
2359 mutex_exit(&pidlock)_sx_xunlock(((&pidlock)), ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (2359));
2360#else
2361 PROC_LOCK_ASSERT(p, MA_OWNED)(void)0;
2362#endif
2363
2364#ifdef notyet
2365 if ((ret = priv_proc_cred_perm(cr, p, NULL((void *)0),
2366 VREAD000000000400 | VWRITE000000000200)) != 0) {
2367#ifdef illumos
2368 mutex_exit(&p->p_lock)_sx_xunlock(((&p->p_lock)), ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (2368));
2369#else
2370 PROC_UNLOCK(p)__mtx_unlock_flags(&((((&(p)->p_mtx))))->mtx_lock
, ((0)), ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (2370));
2371#endif
2372 return (ret);
2373 }
2374#endif /* notyet */
2375#ifdef illumos
2376 mutex_exit(&p->p_lock)_sx_xunlock(((&p->p_lock)), ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (2376));
2377#else
2378 PROC_UNLOCK(p)__mtx_unlock_flags(&((((&(p)->p_mtx))))->mtx_lock
, ((0)), ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (2378));
2379#endif
2380 }
2381#endif /* notyet */
2382
2383 ret = fasttrap_add_probe(probe);
2384err:
2385 kmem_free(probe, size)zfs_kmem_free((probe), (size));
2386
2387 return (ret);
2388
2389 } else if (cmd == FASTTRAPIOC_GETINSTR((unsigned long) (((0x80000000|0x40000000)) | (((sizeof(uint8_t
)) & ((1 << 13) - 1)) << 16) | ((('f')) <<
8) | ((2))))) {
3
←
Taking true branch
→
2390 fasttrap_instr_query_t instr;
2391 fasttrap_tracepoint_t *tp;
2392 uint_t index;
2393#ifdef illumos
2394 int ret;
2395#endif
2396
2397#ifdef illumos
2398 if (copyin((void *)arg, &instr, sizeof (instr)) != 0)
2399 return (EFAULT14);
2400#endif
2401
2402#ifdef notyet
2403 if (!PRIV_POLICY_CHOICE(cr, PRIV_ALL, B_FALSE)priv_policy_choice((cr), (PRIV_ALL), (0))) {
2404 proc_t *p;
2405 pid_t pid = instr.ftiq_pid;
2406
2407#ifdef illumos
2408 mutex_enter(&pidlock)(void)_sx_xlock(((&pidlock)), 0, ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (2408));
2409#endif
2410 /*
2411 * Report an error if the process doesn't exist
2412 * or is actively being birthed.
2413 */
2414 sx_slock(&proctree_lock)(void)_sx_slock(((&proctree_lock)), 0, ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (2414));
2415 p = pfind(pid);
2416 if (p)
2417 fill_kinfo_proc(p, &kp);
2418 sx_sunlock(&proctree_lock)_sx_sunlock(((&proctree_lock)), ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (2418));
2419 if (p == NULL((void *)0) || kp.ki_stat == SIDL1) {
2420#ifdef illumos
2421 mutex_exit(&pidlock)_sx_xunlock(((&pidlock)), ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (2421));
2422#endif
2423 return (ESRCH3);
2424 }
2425#ifdef illumos
2426 mutex_enter(&p->p_lock)(void)_sx_xlock(((&p->p_lock)), 0, ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (2426));
2427 mutex_exit(&pidlock)_sx_xunlock(((&pidlock)), ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (2427));
2428#else
2429 PROC_LOCK_ASSERT(p, MA_OWNED)(void)0;
2430#endif
2431
2432#ifdef notyet
2433 if ((ret = priv_proc_cred_perm(cr, p, NULL((void *)0),
2434 VREAD000000000400)) != 0) {
2435#ifdef illumos
2436 mutex_exit(&p->p_lock)_sx_xunlock(((&p->p_lock)), ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (2436));
2437#else
2438 PROC_UNLOCK(p)__mtx_unlock_flags(&((((&(p)->p_mtx))))->mtx_lock
, ((0)), ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (2438));
2439#endif
2440 return (ret);
2441 }
2442#endif /* notyet */
2443
2444#ifdef illumos
2445 mutex_exit(&p->p_lock)_sx_xunlock(((&p->p_lock)), ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (2445));
2446#else
2447 PROC_UNLOCK(p)__mtx_unlock_flags(&((((&(p)->p_mtx))))->mtx_lock
, ((0)), ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (2447));
2448#endif
2449 }
2450#endif /* notyet */
2451
2452 index = FASTTRAP_TPOINTS_INDEX(instr.ftiq_pid, instr.ftiq_pc)(((instr.ftiq_pc) / sizeof (fasttrap_instr_t) + (instr.ftiq_pid
)) & fasttrap_tpoints.fth_mask);
2453
2454 mutex_enter(&fasttrap_tpoints.fth_table[index].ftb_mtx)(void)_sx_xlock(((&fasttrap_tpoints.fth_table[index].ftb_mtx
)), 0, ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (2454));
2455 tp = fasttrap_tpoints.fth_table[index].ftb_data;
2456 while (tp != NULL((void *)0)) {
4
←
Assuming 'tp' is not equal to null
→
5
←
Loop condition is true. Entering loop body
→
2457 if (instr.ftiq_pid == tp->ftt_pid &&
6
←
Taking true branch
→
2458 instr.ftiq_pc == tp->ftt_pc &&
2459 tp->ftt_proc->ftpc_acount != 0)
2460 break;
7
←
 Execution continues on line 2465
→
2461
2462 tp = tp->ftt_next;
2463 }
2464
2465 if (tp == NULL((void *)0)) {
8
←
Taking false branch
→
2466 mutex_exit(&fasttrap_tpoints.fth_table[index].ftb_mtx)_sx_xunlock(((&fasttrap_tpoints.fth_table[index].ftb_mtx)
), ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (2466));
2467 return (ENOENT2);
2468 }
2469
2470 bcopy(&tp->ftt_instrftt_mtp.ftmt_instr, &instr.ftiq_instr,
2471 sizeof (instr.ftiq_instr));
2472 mutex_exit(&fasttrap_tpoints.fth_table[index].ftb_mtx)_sx_xunlock(((&fasttrap_tpoints.fth_table[index].ftb_mtx)
), ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (2472));
2473
2474 if (copyout(&instr, (void *)arg, sizeof (instr)) != 0)
9
←
Copies out a struct with uncleared padding (>= 3 bytes)
2475 return (EFAULT14);
2476
2477 return (0);
2478 }
2479
2480 return (EINVAL22);
2481}
2482
2483static int
2484fasttrap_load(void)
2485{
2486 ulong_t nent;
2487 int i, ret;
2488
2489 /* Create the /dev/dtrace/fasttrap entry. */
2490 fasttrap_cdev = make_dev(&fasttrap_cdevsw, 0, UID_ROOT0, GID_WHEEL0, 0600,
2491 "dtrace/fasttrap");
2492
2493 mtx_init(&fasttrap_cleanup_mtx, "fasttrap clean", "dtrace", MTX_DEF)_mtx_init(&(&fasttrap_cleanup_mtx)->mtx_lock, "fasttrap clean"
, "dtrace", 0x00000000);
2494 mutex_init(&fasttrap_count_mtx, "fasttrap count mtx", MUTEX_DEFAULT,do { const char *_name; ((void)0); do { } while (0); bzero((&
fasttrap_count_mtx), sizeof(struct sx)); for (_name = "&fasttrap_count_mtx"
; *_name != '\0'; _name++) { if (*_name >= 'a' && *
_name <= 'z') break; } if (*_name == '\0') _name = "&fasttrap_count_mtx"
; sx_init_flags((&fasttrap_count_mtx), _name, (0x01 | 0x04
)); } while (0)
2495 NULL)do { const char *_name; ((void)0); do { } while (0); bzero((&
fasttrap_count_mtx), sizeof(struct sx)); for (_name = "&fasttrap_count_mtx"
; *_name != '\0'; _name++) { if (*_name >= 'a' && *
_name <= 'z') break; } if (*_name == '\0') _name = "&fasttrap_count_mtx"
; sx_init_flags((&fasttrap_count_mtx), _name, (0x01 | 0x04
)); } while (0);
2496
2497#ifdef illumos
2498 fasttrap_max = ddi_getprop(DDI_DEV_T_ANY, devi, DDI_PROP_DONTPASS,
2499 "fasttrap-max-probes", FASTTRAP_MAX_DEFAULT250000);
2500#endif
2501 fasttrap_total = 0;
2502
2503 /*
2504 * Conjure up the tracepoints hashtable...
2505 */
2506#ifdef illumos
2507 nent = ddi_getprop(DDI_DEV_T_ANY, devi, DDI_PROP_DONTPASS,
2508 "fasttrap-hash-size", FASTTRAP_TPOINTS_DEFAULT_SIZE0x4000);
2509#else
2510 nent = tpoints_hash_size;
2511#endif
2512
2513 if (nent == 0 || nent > 0x1000000)
2514 nent = FASTTRAP_TPOINTS_DEFAULT_SIZE0x4000;
2515
2516 tpoints_hash_size = nent;
2517
2518 if (ISP2(nent)(((nent) & ((nent) - 1)) == 0))
2519 fasttrap_tpoints.fth_nent = nent;
2520 else
2521 fasttrap_tpoints.fth_nent = 1 << fasttrap_highbit(nent);
2522 ASSERT(fasttrap_tpoints.fth_nent > 0)((void)0);
2523 fasttrap_tpoints.fth_mask = fasttrap_tpoints.fth_nent - 1;
2524 fasttrap_tpoints.fth_table = kmem_zalloc(fasttrap_tpoints.fth_nent *zfs_kmem_alloc((fasttrap_tpoints.fth_nent * sizeof (fasttrap_bucket_t
)), (0x0002) | 0x0100)
2525 sizeof (fasttrap_bucket_t), KM_SLEEP)zfs_kmem_alloc((fasttrap_tpoints.fth_nent * sizeof (fasttrap_bucket_t
)), (0x0002) | 0x0100);
2526#ifndef illumos
2527 for (i = 0; i < fasttrap_tpoints.fth_nent; i++)
2528 mutex_init(&fasttrap_tpoints.fth_table[i].ftb_mtx,do { const char *_name; ((void)0); do { } while (0); bzero((&
fasttrap_tpoints.fth_table[i].ftb_mtx), sizeof(struct sx)); for
(_name = "&fasttrap_tpoints.fth_table[i].ftb_mtx"; *_name
!= '\0'; _name++) { if (*_name >= 'a' && *_name <=
'z') break; } if (*_name == '\0') _name = "&fasttrap_tpoints.fth_table[i].ftb_mtx"
; sx_init_flags((&fasttrap_tpoints.fth_table[i].ftb_mtx),
_name, (0x01 | 0x04)); } while (0)
2529 "tracepoints bucket mtx", MUTEX_DEFAULT, NULL)do { const char *_name; ((void)0); do { } while (0); bzero((&
fasttrap_tpoints.fth_table[i].ftb_mtx), sizeof(struct sx)); for
(_name = "&fasttrap_tpoints.fth_table[i].ftb_mtx"; *_name
!= '\0'; _name++) { if (*_name >= 'a' && *_name <=
'z') break; } if (*_name == '\0') _name = "&fasttrap_tpoints.fth_table[i].ftb_mtx"
; sx_init_flags((&fasttrap_tpoints.fth_table[i].ftb_mtx),
_name, (0x01 | 0x04)); } while (0);
2530#endif
2531
2532 /*
2533 * ... and the providers hash table...
2534 */
2535 nent = FASTTRAP_PROVIDERS_DEFAULT_SIZE0x100;
2536 if (ISP2(nent)(((nent) & ((nent) - 1)) == 0))
2537 fasttrap_provs.fth_nent = nent;
2538 else
2539 fasttrap_provs.fth_nent = 1 << fasttrap_highbit(nent);
2540 ASSERT(fasttrap_provs.fth_nent > 0)((void)0);
2541 fasttrap_provs.fth_mask = fasttrap_provs.fth_nent - 1;
2542 fasttrap_provs.fth_table = kmem_zalloc(fasttrap_provs.fth_nent *zfs_kmem_alloc((fasttrap_provs.fth_nent * sizeof (fasttrap_bucket_t
)), (0x0002) | 0x0100)
2543 sizeof (fasttrap_bucket_t), KM_SLEEP)zfs_kmem_alloc((fasttrap_provs.fth_nent * sizeof (fasttrap_bucket_t
)), (0x0002) | 0x0100);
2544#ifndef illumos
2545 for (i = 0; i < fasttrap_provs.fth_nent; i++)
2546 mutex_init(&fasttrap_provs.fth_table[i].ftb_mtx,do { const char *_name; ((void)0); do { } while (0); bzero((&
fasttrap_provs.fth_table[i].ftb_mtx), sizeof(struct sx)); for
(_name = "&fasttrap_provs.fth_table[i].ftb_mtx"; *_name !=
'\0'; _name++) { if (*_name >= 'a' && *_name <=
'z') break; } if (*_name == '\0') _name = "&fasttrap_provs.fth_table[i].ftb_mtx"
; sx_init_flags((&fasttrap_provs.fth_table[i].ftb_mtx), _name
, (0x01 | 0x04)); } while (0)
2547 "providers bucket mtx", MUTEX_DEFAULT, NULL)do { const char *_name; ((void)0); do { } while (0); bzero((&
fasttrap_provs.fth_table[i].ftb_mtx), sizeof(struct sx)); for
(_name = "&fasttrap_provs.fth_table[i].ftb_mtx"; *_name !=
'\0'; _name++) { if (*_name >= 'a' && *_name <=
'z') break; } if (*_name == '\0') _name = "&fasttrap_provs.fth_table[i].ftb_mtx"
; sx_init_flags((&fasttrap_provs.fth_table[i].ftb_mtx), _name
, (0x01 | 0x04)); } while (0);
2548#endif
2549
2550 ret = kproc_create(fasttrap_pid_cleanup_cb, NULL((void *)0),
2551 &fasttrap_cleanup_proc, 0, 0, "ftcleanup");
2552 if (ret != 0) {
2553 destroy_dev(fasttrap_cdev);
2554#ifndef illumos
2555 for (i = 0; i < fasttrap_provs.fth_nent; i++)
2556 mutex_destroy(&fasttrap_provs.fth_table[i].ftb_mtx)sx_destroy(&fasttrap_provs.fth_table[i].ftb_mtx);
2557 for (i = 0; i < fasttrap_tpoints.fth_nent; i++)
2558 mutex_destroy(&fasttrap_tpoints.fth_table[i].ftb_mtx)sx_destroy(&fasttrap_tpoints.fth_table[i].ftb_mtx);
2559#endif
2560 kmem_free(fasttrap_provs.fth_table, fasttrap_provs.fth_nent *zfs_kmem_free((fasttrap_provs.fth_table), (fasttrap_provs.fth_nent
* sizeof (fasttrap_bucket_t)))
2561 sizeof (fasttrap_bucket_t))zfs_kmem_free((fasttrap_provs.fth_table), (fasttrap_provs.fth_nent
* sizeof (fasttrap_bucket_t)));
2562 mtx_destroy(&fasttrap_cleanup_mtx)_mtx_destroy(&(&fasttrap_cleanup_mtx)->mtx_lock);
2563 mutex_destroy(&fasttrap_count_mtx)sx_destroy(&fasttrap_count_mtx);
2564 return (ret);
2565 }
2566
2567
2568 /*
2569 * ... and the procs hash table.
2570 */
2571 nent = FASTTRAP_PROCS_DEFAULT_SIZE0x100;
2572 if (ISP2(nent)(((nent) & ((nent) - 1)) == 0))
2573 fasttrap_procs.fth_nent = nent;
2574 else
2575 fasttrap_procs.fth_nent = 1 << fasttrap_highbit(nent);
2576 ASSERT(fasttrap_procs.fth_nent > 0)((void)0);
2577 fasttrap_procs.fth_mask = fasttrap_procs.fth_nent - 1;
2578 fasttrap_procs.fth_table = kmem_zalloc(fasttrap_procs.fth_nent *zfs_kmem_alloc((fasttrap_procs.fth_nent * sizeof (fasttrap_bucket_t
)), (0x0002) | 0x0100)
2579 sizeof (fasttrap_bucket_t), KM_SLEEP)zfs_kmem_alloc((fasttrap_procs.fth_nent * sizeof (fasttrap_bucket_t
)), (0x0002) | 0x0100);
2580#ifndef illumos
2581 for (i = 0; i < fasttrap_procs.fth_nent; i++)
2582 mutex_init(&fasttrap_procs.fth_table[i].ftb_mtx,do { const char *_name; ((void)0); do { } while (0); bzero((&
fasttrap_procs.fth_table[i].ftb_mtx), sizeof(struct sx)); for
(_name = "&fasttrap_procs.fth_table[i].ftb_mtx"; *_name !=
'\0'; _name++) { if (*_name >= 'a' && *_name <=
'z') break; } if (*_name == '\0') _name = "&fasttrap_procs.fth_table[i].ftb_mtx"
; sx_init_flags((&fasttrap_procs.fth_table[i].ftb_mtx), _name
, (0x01 | 0x04)); } while (0)
2583 "processes bucket mtx", MUTEX_DEFAULT, NULL)do { const char *_name; ((void)0); do { } while (0); bzero((&
fasttrap_procs.fth_table[i].ftb_mtx), sizeof(struct sx)); for
(_name = "&fasttrap_procs.fth_table[i].ftb_mtx"; *_name !=
'\0'; _name++) { if (*_name >= 'a' && *_name <=
'z') break; } if (*_name == '\0') _name = "&fasttrap_procs.fth_table[i].ftb_mtx"
; sx_init_flags((&fasttrap_procs.fth_table[i].ftb_mtx), _name
, (0x01 | 0x04)); } while (0);
2584
2585 rm_init(&fasttrap_tp_lock, "fasttrap tracepoint");
2586
2587 /*
2588 * This event handler must run before kdtrace_thread_dtor() since it
2589 * accesses the thread's struct kdtrace_thread.
2590 */
2591 fasttrap_thread_dtor_tag = EVENTHANDLER_REGISTER(thread_dtor,eventhandler_register(((void *)0), "thread_dtor", fasttrap_thread_dtor
, ((void *)0), 0)
2592 fasttrap_thread_dtor, NULL, EVENTHANDLER_PRI_FIRST)eventhandler_register(((void *)0), "thread_dtor", fasttrap_thread_dtor
, ((void *)0), 0);
2593#endif
2594
2595 /*
2596 * Install our hooks into fork(2), exec(2), and exit(2).
2597 */
2598 dtrace_fasttrap_fork = &fasttrap_fork;
2599 dtrace_fasttrap_exit = &fasttrap_exec_exit;
2600 dtrace_fasttrap_exec = &fasttrap_exec_exit;
2601
2602 (void) dtrace_meta_register("fasttrap", &fasttrap_mops, NULL((void *)0),
2603 &fasttrap_meta_id);
2604
2605 return (0);
2606}
2607
2608static int
2609fasttrap_unload(void)
2610{
2611 int i, fail = 0;
2612
2613 /*
2614 * Unregister the meta-provider to make sure no new fasttrap-
2615 * managed providers come along while we're trying to close up
2616 * shop. If we fail to detach, we'll need to re-register as a
2617 * meta-provider. We can fail to unregister as a meta-provider
2618 * if providers we manage still exist.
2619 */
2620 if (fasttrap_meta_id != DTRACE_METAPROVNONE0 &&
2621 dtrace_meta_unregister(fasttrap_meta_id) != 0)
2622 return (-1);
2623
2624 /*
2625 * Iterate over all of our providers. If there's still a process
2626 * that corresponds to that pid, fail to detach.
2627 */
2628 for (i = 0; i < fasttrap_provs.fth_nent; i++) {
2629 fasttrap_provider_t **fpp, *fp;
2630 fasttrap_bucket_t *bucket = &fasttrap_provs.fth_table[i];
2631
2632 mutex_enter(&bucket->ftb_mtx)(void)_sx_xlock(((&bucket->ftb_mtx)), 0, ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (2632));
2633 fpp = (fasttrap_provider_t **)&bucket->ftb_data;
2634 while ((fp = *fpp) != NULL((void *)0)) {
2635 /*
2636 * Acquire and release the lock as a simple way of
2637 * waiting for any other consumer to finish with
2638 * this provider. A thread must first acquire the
2639 * bucket lock so there's no chance of another thread
2640 * blocking on the provider's lock.
2641 */
2642 mutex_enter(&fp->ftp_mtx)(void)_sx_xlock(((&fp->ftp_mtx)), 0, ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (2642));
2643 mutex_exit(&fp->ftp_mtx)_sx_xunlock(((&fp->ftp_mtx)), ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (2643));
2644
2645 if (dtrace_unregister(fp->ftp_provid) != 0) {
2646 fail = 1;
2647 fpp = &fp->ftp_next;
2648 } else {
2649 *fpp = fp->ftp_next;
2650 fasttrap_provider_free(fp);
2651 }
2652 }
2653
2654 mutex_exit(&bucket->ftb_mtx)_sx_xunlock(((&bucket->ftb_mtx)), ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (2654));
2655 }
2656
2657 if (fail) {
2658 (void) dtrace_meta_register("fasttrap", &fasttrap_mops, NULL((void *)0),
2659 &fasttrap_meta_id);
2660
2661 return (-1);
2662 }
2663
2664 /*
2665 * Stop new processes from entering these hooks now, before the
2666 * fasttrap_cleanup thread runs. That way all processes will hopefully
2667 * be out of these hooks before we free fasttrap_provs.fth_table
2668 */
2669 ASSERT(dtrace_fasttrap_fork == &fasttrap_fork)((void)0);
2670 dtrace_fasttrap_fork = NULL((void *)0);
2671
2672 ASSERT(dtrace_fasttrap_exec == &fasttrap_exec_exit)((void)0);
2673 dtrace_fasttrap_exec = NULL((void *)0);
2674
2675 ASSERT(dtrace_fasttrap_exit == &fasttrap_exec_exit)((void)0);
2676 dtrace_fasttrap_exit = NULL((void *)0);
2677
2678 mtx_lock(&fasttrap_cleanup_mtx)__mtx_lock_flags(&((((&fasttrap_cleanup_mtx))))->mtx_lock
, ((0)), ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (2678));
2679 fasttrap_cleanup_drain = 1;
2680 /* Wait for the cleanup thread to finish up and signal us. */
2681 wakeup(&fasttrap_cleanup_cv);
2682 mtx_sleep(&fasttrap_cleanup_drain, &fasttrap_cleanup_mtx, 0, "ftcld",_sleep((&fasttrap_cleanup_drain), &(&fasttrap_cleanup_mtx
)->lock_object, (0), ("ftcld"), tick_sbt * (0), 0, 0x0100)
2683 0)_sleep((&fasttrap_cleanup_drain), &(&fasttrap_cleanup_mtx
)->lock_object, (0), ("ftcld"), tick_sbt * (0), 0, 0x0100);
2684 fasttrap_cleanup_proc = NULL((void *)0);
2685 mtx_destroy(&fasttrap_cleanup_mtx)_mtx_destroy(&(&fasttrap_cleanup_mtx)->mtx_lock);
2686
2687#ifdef DEBUG
2688 mutex_enter(&fasttrap_count_mtx)(void)_sx_xlock(((&fasttrap_count_mtx)), 0, ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (2688));
2689 ASSERT(fasttrap_pid_count == 0)((void)0);
2690 mutex_exit(&fasttrap_count_mtx)_sx_xunlock(((&fasttrap_count_mtx)), ("/usr/src/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c"
), (2690));
2691#endif
2692
2693#ifndef illumos
2694 EVENTHANDLER_DEREGISTER(thread_dtor, fasttrap_thread_dtor_tag)do { struct eventhandler_list *_el; if ((_el = eventhandler_find_list
("thread_dtor")) != ((void *)0)) eventhandler_deregister(_el,
fasttrap_thread_dtor_tag); } while(0);
2695
2696 for (i = 0; i < fasttrap_tpoints.fth_nent; i++)
2697 mutex_destroy(&fasttrap_tpoints.fth_table[i].ftb_mtx)sx_destroy(&fasttrap_tpoints.fth_table[i].ftb_mtx);
2698 for (i = 0; i < fasttrap_provs.fth_nent; i++)
2699 mutex_destroy(&fasttrap_provs.fth_table[i].ftb_mtx)sx_destroy(&fasttrap_provs.fth_table[i].ftb_mtx);
2700 for (i = 0; i < fasttrap_procs.fth_nent; i++)
2701 mutex_destroy(&fasttrap_procs.fth_table[i].ftb_mtx)sx_destroy(&fasttrap_procs.fth_table[i].ftb_mtx);
2702#endif
2703 kmem_free(fasttrap_tpoints.fth_table,zfs_kmem_free((fasttrap_tpoints.fth_table), (fasttrap_tpoints
.fth_nent * sizeof (fasttrap_bucket_t)))
2704 fasttrap_tpoints.fth_nent * sizeof (fasttrap_bucket_t))zfs_kmem_free((fasttrap_tpoints.fth_table), (fasttrap_tpoints
.fth_nent * sizeof (fasttrap_bucket_t)));
2705 fasttrap_tpoints.fth_nent = 0;
2706
2707 kmem_free(fasttrap_provs.fth_table,zfs_kmem_free((fasttrap_provs.fth_table), (fasttrap_provs.fth_nent
* sizeof (fasttrap_bucket_t)))
2708 fasttrap_provs.fth_nent * sizeof (fasttrap_bucket_t))zfs_kmem_free((fasttrap_provs.fth_table), (fasttrap_provs.fth_nent
* sizeof (fasttrap_bucket_t)));
2709 fasttrap_provs.fth_nent = 0;
2710
2711 kmem_free(fasttrap_procs.fth_table,zfs_kmem_free((fasttrap_procs.fth_table), (fasttrap_procs.fth_nent
* sizeof (fasttrap_bucket_t)))
2712 fasttrap_procs.fth_nent * sizeof (fasttrap_bucket_t))zfs_kmem_free((fasttrap_procs.fth_table), (fasttrap_procs.fth_nent
* sizeof (fasttrap_bucket_t)));
2713 fasttrap_procs.fth_nent = 0;
2714
2715#ifndef illumos
2716 destroy_dev(fasttrap_cdev);
2717 mutex_destroy(&fasttrap_count_mtx)sx_destroy(&fasttrap_count_mtx);
2718 rm_destroy(&fasttrap_tp_lock);
2719#endif
2720
2721 return (0);
2722}
2723
2724/* ARGSUSED */
2725static int
2726fasttrap_modevent(module_t mod __unused__attribute__((__unused__)), int type, void *data __unused__attribute__((__unused__)))
2727{
2728 int error = 0;
2729
2730 switch (type) {
2731 case MOD_LOAD:
2732 break;
2733
2734 case MOD_UNLOAD:
2735 break;
2736
2737 case MOD_SHUTDOWN:
2738 break;
2739
2740 default:
2741 error = EOPNOTSUPP45;
2742 break;
2743 }
2744 return (error);
2745}
2746
2747SYSINIT(fasttrap_load, SI_SUB_DTRACE_PROVIDER, SI_ORDER_ANY, fasttrap_load,static struct sysinit fasttrap_load_sys_init = { SI_SUB_DTRACE_PROVIDER
, SI_ORDER_ANY, (sysinit_cfunc_t)(sysinit_nfunc_t)fasttrap_load
, ((void *)(((void *)0))) }; __asm__(".globl " "__start_set_sysinit_set"
); __asm__(".globl " "__stop_set_sysinit_set"); static void const
* const __set_sysinit_set_sym_fasttrap_load_sys_init __attribute__
((__section__("set_" "sysinit_set"))) __attribute__((__used__
)) = &(fasttrap_load_sys_init)
2748 NULL)static struct sysinit fasttrap_load_sys_init = { SI_SUB_DTRACE_PROVIDER
, SI_ORDER_ANY, (sysinit_cfunc_t)(sysinit_nfunc_t)fasttrap_load
, ((void *)(((void *)0))) }; __asm__(".globl " "__start_set_sysinit_set"
); __asm__(".globl " "__stop_set_sysinit_set"); static void const
* const __set_sysinit_set_sym_fasttrap_load_sys_init __attribute__
((__section__("set_" "sysinit_set"))) __attribute__((__used__
)) = &(fasttrap_load_sys_init);
2749SYSUNINIT(fasttrap_unload, SI_SUB_DTRACE_PROVIDER, SI_ORDER_ANY,static struct sysinit fasttrap_unload_sys_uninit = { SI_SUB_DTRACE_PROVIDER
, SI_ORDER_ANY, (sysinit_cfunc_t)(sysinit_nfunc_t)fasttrap_unload
, ((void *)(((void *)0))) }; __asm__(".globl " "__start_set_sysuninit_set"
); __asm__(".globl " "__stop_set_sysuninit_set"); static void
const * const __set_sysuninit_set_sym_fasttrap_unload_sys_uninit
__attribute__((__section__("set_" "sysuninit_set"))) __attribute__
((__used__)) = &(fasttrap_unload_sys_uninit)
2750 fasttrap_unload, NULL)static struct sysinit fasttrap_unload_sys_uninit = { SI_SUB_DTRACE_PROVIDER
, SI_ORDER_ANY, (sysinit_cfunc_t)(sysinit_nfunc_t)fasttrap_unload
, ((void *)(((void *)0))) }; __asm__(".globl " "__start_set_sysuninit_set"
); __asm__(".globl " "__stop_set_sysuninit_set"); static void
const * const __set_sysuninit_set_sym_fasttrap_unload_sys_uninit
__attribute__((__section__("set_" "sysuninit_set"))) __attribute__
((__used__)) = &(fasttrap_unload_sys_uninit);
2751
2752DEV_MODULE(fasttrap, fasttrap_modevent, NULL)static moduledata_t fasttrap_mod = { "fasttrap", fasttrap_modevent
, ((void *)0) }; static struct mod_depend _fasttrap_depend_on_kernel
__attribute__((__section__(".data"))) = { 1100122, 1100122, (
((((1100122)+((100000)-1))/(100000))*(100000)) - 1) }; static
struct mod_metadata _mod_metadata_md_fasttrap_on_kernel = { 1
, 1, &_fasttrap_depend_on_kernel, "kernel" }; __asm__(".globl "
"__start_set_modmetadata_set"); __asm__(".globl " "__stop_set_modmetadata_set"
); static void const * const __set_modmetadata_set_sym__mod_metadata_md_fasttrap_on_kernel
__attribute__((__section__("set_" "modmetadata_set"))) __attribute__
((__used__)) = &(_mod_metadata_md_fasttrap_on_kernel); static
struct mod_metadata _mod_metadata_md_fasttrap = { 1, 2, &
fasttrap_mod, "fasttrap" }; __asm__(".globl " "__start_set_modmetadata_set"
); __asm__(".globl " "__stop_set_modmetadata_set"); static void
const * const __set_modmetadata_set_sym__mod_metadata_md_fasttrap
__attribute__((__section__("set_" "modmetadata_set"))) __attribute__
((__used__)) = &(_mod_metadata_md_fasttrap); static struct
sysinit fasttrapmodule_sys_init = { SI_SUB_DRIVERS, SI_ORDER_MIDDLE
, (sysinit_cfunc_t)(sysinit_nfunc_t)module_register_init, ((void
*)(&fasttrap_mod)) }; __asm__(".globl " "__start_set_sysinit_set"
); __asm__(".globl " "__stop_set_sysinit_set"); static void const
* const __set_sysinit_set_sym_fasttrapmodule_sys_init __attribute__
((__section__("set_" "sysinit_set"))) __attribute__((__used__
)) = &(fasttrapmodule_sys_init); struct __hack;
2753MODULE_VERSION(fasttrap, 1)static struct mod_version _fasttrap_version __attribute__((__section__
(".data"))) = { 1 }; static struct mod_metadata _mod_metadata_fasttrap_version
= { 1, 3, &_fasttrap_version, "fasttrap" }; __asm__(".globl "
"__start_set_modmetadata_set"); __asm__(".globl " "__stop_set_modmetadata_set"
); static void const * const __set_modmetadata_set_sym__mod_metadata_fasttrap_version
__attribute__((__section__("set_" "modmetadata_set"))) __attribute__
((__used__)) = &(_mod_metadata_fasttrap_version);
2754MODULE_DEPEND(fasttrap, dtrace, 1, 1, 1)static struct mod_depend _fasttrap_depend_on_dtrace __attribute__
((__section__(".data"))) = { 1, 1, 1 }; static struct mod_metadata
_mod_metadata_md_fasttrap_on_dtrace = { 1, 1, &_fasttrap_depend_on_dtrace
, "dtrace" }; __asm__(".globl " "__start_set_modmetadata_set"
); __asm__(".globl " "__stop_set_modmetadata_set"); static void
const * const __set_modmetadata_set_sym__mod_metadata_md_fasttrap_on_dtrace
__attribute__((__section__("set_" "modmetadata_set"))) __attribute__
((__used__)) = &(_mod_metadata_md_fasttrap_on_dtrace);
2755MODULE_DEPEND(fasttrap, opensolaris, 1, 1, 1)static struct mod_depend _fasttrap_depend_on_opensolaris __attribute__
((__section__(".data"))) = { 1, 1, 1 }; static struct mod_metadata
_mod_metadata_md_fasttrap_on_opensolaris = { 1, 1, &_fasttrap_depend_on_opensolaris
, "opensolaris" }; __asm__(".globl " "__start_set_modmetadata_set"
); __asm__(".globl " "__stop_set_modmetadata_set"); static void
const * const __set_modmetadata_set_sym__mod_metadata_md_fasttrap_on_opensolaris
__attribute__((__section__("set_" "modmetadata_set"))) __attribute__
((__used__)) = &(_mod_metadata_md_fasttrap_on_opensolaris
);