| File: | cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c |
| Warning: | line 2410, column 7 Copies out a struct with uncleared padding (>= 3 bytes) |
Press '?' to see keyboard shortcuts
Keyboard shortcuts:
| 1 | /* | |||
| 2 | * CDDL HEADER START | |||
| 3 | * | |||
| 4 | * The contents of this file are subject to the terms of the | |||
| 5 | * Common Development and Distribution License (the "License"). | |||
| 6 | * You may not use this file except in compliance with the License. | |||
| 7 | * | |||
| 8 | * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE | |||
| 9 | * or http://www.opensolaris.org/os/licensing. | |||
| 10 | * See the License for the specific language governing permissions | |||
| 11 | * and limitations under the License. | |||
| 12 | * | |||
| 13 | * When distributing Covered Code, include this CDDL HEADER in each | |||
| 14 | * file and include the License file at usr/src/OPENSOLARIS.LICENSE. | |||
| 15 | * If applicable, add the following below this CDDL HEADER, with the | |||
| 16 | * fields enclosed by brackets "[]" replaced with your own identifying | |||
| 17 | * information: Portions Copyright [yyyy] [name of copyright owner] | |||
| 18 | * | |||
| 19 | * CDDL HEADER END | |||
| 20 | * | |||
| 21 | * Portions Copyright 2010 The FreeBSD Foundation | |||
| 22 | * | |||
| 23 | * $FreeBSD$ | |||
| 24 | */ | |||
| 25 | ||||
| 26 | /* | |||
| 27 | * Copyright 2008 Sun Microsystems, Inc. All rights reserved. | |||
| 28 | * Use is subject to license terms. | |||
| 29 | */ | |||
| 30 | ||||
| 31 | /* | |||
| 32 | * Copyright (c) 2015, Joyent, Inc. All rights reserved. | |||
| 33 | */ | |||
| 34 | ||||
| 35 | #include <sys/atomic.h> | |||
| 36 | #include <sys/errno.h> | |||
| 37 | #include <sys/stat.h> | |||
| 38 | #include <sys/modctl.h> | |||
| 39 | #include <sys/conf.h> | |||
| 40 | #include <sys/systm.h> | |||
| 41 | #ifdef illumos | |||
| 42 | #include <sys/ddi.h> | |||
| 43 | #endif | |||
| 44 | #include <sys/sunddi.h> | |||
| 45 | #include <sys/cpuvar.h> | |||
| 46 | #include <sys/kmem.h> | |||
| 47 | #ifdef illumos | |||
| 48 | #include <sys/strsubr.h> | |||
| 49 | #endif | |||
| 50 | #include <sys/fasttrap.h> | |||
| 51 | #include <sys/fasttrap_impl.h> | |||
| 52 | #include <sys/fasttrap_isa.h> | |||
| 53 | #include <sys/dtrace.h> | |||
| 54 | #include <sys/dtrace_impl.h> | |||
| 55 | #include <sys/sysmacros.h> | |||
| 56 | #include <sys/proc.h> | |||
| 57 | #include <sys/policy.h> | |||
| 58 | #ifdef illumos | |||
| 59 | #include <util/qsort.h> | |||
| 60 | #endif | |||
| 61 | #include <sys/mutex.h> | |||
| 62 | #include <sys/kernel.h> | |||
| 63 | #ifndef illumos | |||
| 64 | #include <sys/dtrace_bsd.h> | |||
| 65 | #include <sys/eventhandler.h> | |||
| 66 | #include <sys/rmlock.h> | |||
| 67 | #include <sys/sysent.h> | |||
| 68 | #include <sys/sysctl.h> | |||
| 69 | #include <sys/u8_textprep.h> | |||
| 70 | #include <sys/user.h> | |||
| 71 | ||||
| 72 | #include <vm/vm.h> | |||
| 73 | #include <vm/pmap.h> | |||
| 74 | #include <vm/vm_map.h> | |||
| 75 | #include <vm/vm_param.h> | |||
| 76 | ||||
| 77 | #include <cddl/dev/dtrace/dtrace_cddl.h> | |||
| 78 | #endif | |||
| 79 | ||||
| 80 | /* | |||
| 81 | * User-Land Trap-Based Tracing | |||
| 82 | * ---------------------------- | |||
| 83 | * | |||
| 84 | * The fasttrap provider allows DTrace consumers to instrument any user-level | |||
| 85 | * instruction to gather data; this includes probes with semantic | |||
| 86 | * signifigance like entry and return as well as simple offsets into the | |||
| 87 | * function. While the specific techniques used are very ISA specific, the | |||
| 88 | * methodology is generalizable to any architecture. | |||
| 89 | * | |||
| 90 | * | |||
| 91 | * The General Methodology | |||
| 92 | * ----------------------- | |||
| 93 | * | |||
| 94 | * With the primary goal of tracing every user-land instruction and the | |||
| 95 | * limitation that we can't trust user space so don't want to rely on much | |||
| 96 | * information there, we begin by replacing the instructions we want to trace | |||
| 97 | * with trap instructions. Each instruction we overwrite is saved into a hash | |||
| 98 | * table keyed by process ID and pc address. When we enter the kernel due to | |||
| 99 | * this trap instruction, we need the effects of the replaced instruction to | |||
| 100 | * appear to have occurred before we proceed with the user thread's | |||
| 101 | * execution. | |||
| 102 | * | |||
| 103 | * Each user level thread is represented by a ulwp_t structure which is | |||
| 104 | * always easily accessible through a register. The most basic way to produce | |||
| 105 | * the effects of the instruction we replaced is to copy that instruction out | |||
| 106 | * to a bit of scratch space reserved in the user thread's ulwp_t structure | |||
| 107 | * (a sort of kernel-private thread local storage), set the PC to that | |||
| 108 | * scratch space and single step. When we reenter the kernel after single | |||
| 109 | * stepping the instruction we must then adjust the PC to point to what would | |||
| 110 | * normally be the next instruction. Of course, special care must be taken | |||
| 111 | * for branches and jumps, but these represent such a small fraction of any | |||
| 112 | * instruction set that writing the code to emulate these in the kernel is | |||
| 113 | * not too difficult. | |||
| 114 | * | |||
| 115 | * Return probes may require several tracepoints to trace every return site, | |||
| 116 | * and, conversely, each tracepoint may activate several probes (the entry | |||
| 117 | * and offset 0 probes, for example). To solve this muliplexing problem, | |||
| 118 | * tracepoints contain lists of probes to activate and probes contain lists | |||
| 119 | * of tracepoints to enable. If a probe is activated, it adds its ID to | |||
| 120 | * existing tracepoints or creates new ones as necessary. | |||
| 121 | * | |||
| 122 | * Most probes are activated _before_ the instruction is executed, but return | |||
| 123 | * probes are activated _after_ the effects of the last instruction of the | |||
| 124 | * function are visible. Return probes must be fired _after_ we have | |||
| 125 | * single-stepped the instruction whereas all other probes are fired | |||
| 126 | * beforehand. | |||
| 127 | * | |||
| 128 | * | |||
| 129 | * Lock Ordering | |||
| 130 | * ------------- | |||
| 131 | * | |||
| 132 | * The lock ordering below -- both internally and with respect to the DTrace | |||
| 133 | * framework -- is a little tricky and bears some explanation. Each provider | |||
| 134 | * has a lock (ftp_mtx) that protects its members including reference counts | |||
| 135 | * for enabled probes (ftp_rcount), consumers actively creating probes | |||
| 136 | * (ftp_ccount) and USDT consumers (ftp_mcount); all three prevent a provider | |||
| 137 | * from being freed. A provider is looked up by taking the bucket lock for the | |||
| 138 | * provider hash table, and is returned with its lock held. The provider lock | |||
| 139 | * may be taken in functions invoked by the DTrace framework, but may not be | |||
| 140 | * held while calling functions in the DTrace framework. | |||
| 141 | * | |||
| 142 | * To ensure consistency over multiple calls to the DTrace framework, the | |||
| 143 | * creation lock (ftp_cmtx) should be held. Naturally, the creation lock may | |||
| 144 | * not be taken when holding the provider lock as that would create a cyclic | |||
| 145 | * lock ordering. In situations where one would naturally take the provider | |||
| 146 | * lock and then the creation lock, we instead up a reference count to prevent | |||
| 147 | * the provider from disappearing, drop the provider lock, and acquire the | |||
| 148 | * creation lock. | |||
| 149 | * | |||
| 150 | * Briefly: | |||
| 151 | * bucket lock before provider lock | |||
| 152 | * DTrace before provider lock | |||
| 153 | * creation lock before DTrace | |||
| 154 | * never hold the provider lock and creation lock simultaneously | |||
| 155 | */ | |||
| 156 | ||||
| 157 | static d_open_t fasttrap_open; | |||
| 158 | static d_ioctl_t fasttrap_ioctl; | |||
| 159 | ||||
| 160 | static struct cdevsw fasttrap_cdevsw = { | |||
| 161 | .d_version = D_VERSION0x17122009, | |||
| 162 | .d_open = fasttrap_open, | |||
| 163 | .d_ioctl = fasttrap_ioctl, | |||
| 164 | .d_name = "fasttrap", | |||
| 165 | }; | |||
| 166 | static struct cdev *fasttrap_cdev; | |||
| 167 | static dtrace_meta_provider_id_t fasttrap_meta_id; | |||
| 168 | ||||
| 169 | static struct proc *fasttrap_cleanup_proc; | |||
| 170 | static struct mtx fasttrap_cleanup_mtx; | |||
| 171 | static uint_t fasttrap_cleanup_work, fasttrap_cleanup_drain, fasttrap_cleanup_cv; | |||
| 172 | ||||
| 173 | /* | |||
| 174 | * Generation count on modifications to the global tracepoint lookup table. | |||
| 175 | */ | |||
| 176 | static volatile uint64_t fasttrap_mod_gen; | |||
| 177 | ||||
| 178 | /* | |||
| 179 | * When the fasttrap provider is loaded, fasttrap_max is set to either | |||
| 180 | * FASTTRAP_MAX_DEFAULT, or the value for fasttrap-max-probes in the | |||
| 181 | * fasttrap.conf file (Illumos), or the value provied in the loader.conf (FreeBSD). | |||
| 182 | * Each time a probe is created, fasttrap_total is incremented by the number | |||
| 183 | * of tracepoints that may be associated with that probe; fasttrap_total is capped | |||
| 184 | * at fasttrap_max. | |||
| 185 | */ | |||
| 186 | #define FASTTRAP_MAX_DEFAULT250000 250000 | |||
| 187 | static uint32_t fasttrap_max = FASTTRAP_MAX_DEFAULT250000; | |||
| 188 | static uint32_t fasttrap_total; | |||
| 189 | ||||
| 190 | /* | |||
| 191 | * Copyright (c) 2011, Joyent, Inc. All rights reserved. | |||
| 192 | */ | |||
| 193 | ||||
| 194 | #define FASTTRAP_TPOINTS_DEFAULT_SIZE0x4000 0x4000 | |||
| 195 | #define FASTTRAP_PROVIDERS_DEFAULT_SIZE0x100 0x100 | |||
| 196 | #define FASTTRAP_PROCS_DEFAULT_SIZE0x100 0x100 | |||
| 197 | ||||
| 198 | #define FASTTRAP_PID_NAME"pid" "pid" | |||
| 199 | ||||
| 200 | fasttrap_hash_t fasttrap_tpoints; | |||
| 201 | static fasttrap_hash_t fasttrap_provs; | |||
| 202 | static fasttrap_hash_t fasttrap_procs; | |||
| 203 | ||||
| 204 | static uint64_t fasttrap_pid_count; /* pid ref count */ | |||
| 205 | static kmutex_t fasttrap_count_mtx; /* lock on ref count */ | |||
| 206 | ||||
| 207 | #define FASTTRAP_ENABLE_FAIL1 1 | |||
| 208 | #define FASTTRAP_ENABLE_PARTIAL2 2 | |||
| 209 | ||||
| 210 | static int fasttrap_tracepoint_enable(proc_t *, fasttrap_probe_t *, uint_t); | |||
| 211 | static void fasttrap_tracepoint_disable(proc_t *, fasttrap_probe_t *, uint_t); | |||
| 212 | ||||
| 213 | static fasttrap_provider_t *fasttrap_provider_lookup(pid_t, const char *, | |||
| 214 | const dtrace_pattr_t *); | |||
| 215 | static void fasttrap_provider_retire(pid_t, const char *, int); | |||
| 216 | static void fasttrap_provider_free(fasttrap_provider_t *); | |||
| 217 | ||||
| 218 | static fasttrap_proc_t *fasttrap_proc_lookup(pid_t); | |||
| 219 | static void fasttrap_proc_release(fasttrap_proc_t *); | |||
| 220 | ||||
| 221 | #ifndef illumos | |||
| 222 | static void fasttrap_thread_dtor(void *, struct thread *); | |||
| 223 | #endif | |||
| 224 | ||||
| 225 | #define FASTTRAP_PROVS_INDEX(pid, name)((fasttrap_hash_str(name) + (pid)) & fasttrap_provs.fth_mask ) \ | |||
| 226 | ((fasttrap_hash_str(name) + (pid)) & fasttrap_provs.fth_mask) | |||
| 227 | ||||
| 228 | #define FASTTRAP_PROCS_INDEX(pid)((pid) & fasttrap_procs.fth_mask) ((pid) & fasttrap_procs.fth_mask) | |||
| 229 | ||||
| 230 | #ifndef illumos | |||
| 231 | struct rmlock fasttrap_tp_lock; | |||
| 232 | static eventhandler_tag fasttrap_thread_dtor_tag; | |||
| 233 | #endif | |||
| 234 | ||||
| 235 | static unsigned long tpoints_hash_size = FASTTRAP_TPOINTS_DEFAULT_SIZE0x4000; | |||
| 236 | ||||
| 237 | #ifdef __FreeBSD__11 | |||
| 238 | SYSCTL_DECL(_kern_dtrace)extern struct sysctl_oid sysctl___kern_dtrace; | |||
| 239 | SYSCTL_NODE(_kern_dtrace, OID_AUTO, fasttrap, CTLFLAG_RD, 0, "DTrace fasttrap parameters")struct sysctl_oid sysctl___kern_dtrace_fasttrap = { .oid_parent = ((&(&sysctl___kern_dtrace)->oid_children)), .oid_children = { ((void *)0) }, .oid_number = ((-1)), .oid_kind = (1|(0x80000000 )), .oid_arg1 = (((void *)0)), .oid_arg2 = (0), .oid_name = ( "fasttrap"), .oid_handler = (0), .oid_fmt = ("N"), .oid_descr = "DTrace fasttrap parameters", .oid_label = (((void *)0)), } ; __asm__(".globl " "__start_set_sysctl_set"); __asm__(".globl " "__stop_set_sysctl_set"); static void const * const __set_sysctl_set_sym_sysctl___kern_dtrace_fasttrap __attribute__((__section__("set_" "sysctl_set"))) __attribute__ ((__used__)) = &(sysctl___kern_dtrace_fasttrap); _Static_assert (((0x80000000) & 0xf) == 0 || ((0x80000000) & 0xf) == 1, "compile-time assertion failed"); | |||
| 240 | SYSCTL_UINT(_kern_dtrace_fasttrap, OID_AUTO, max_probes, CTLFLAG_RWTUN, &fasttrap_max,static struct sysctl_oid sysctl___kern_dtrace_fasttrap_max_probes = { .oid_parent = ((&(&sysctl___kern_dtrace_fasttrap )->oid_children)), .oid_children = { ((void *)0) }, .oid_number = ((-1)), .oid_kind = (6 | 0x00040000 | (((0x80000000|0x40000000 )|0x00080000))), .oid_arg1 = (&fasttrap_max), .oid_arg2 = (250000), .oid_name = ("max_probes"), .oid_handler = (sysctl_handle_int ), .oid_fmt = ("IU"), .oid_descr = "Maximum number of fasttrap probes" , .oid_label = (((void *)0)), }; __asm__(".globl " "__start_set_sysctl_set" ); __asm__(".globl " "__stop_set_sysctl_set"); static void const * const __set_sysctl_set_sym_sysctl___kern_dtrace_fasttrap_max_probes __attribute__((__section__("set_" "sysctl_set"))) __attribute__ ((__used__)) = &(sysctl___kern_dtrace_fasttrap_max_probes ); _Static_assert((((((0x80000000|0x40000000)|0x00080000)) & 0xf) == 0 || ((((0x80000000|0x40000000)|0x00080000)) & 0xf ) == 6) && sizeof(unsigned) == sizeof(*(&fasttrap_max )), "compile-time assertion failed") | |||
| 241 | FASTTRAP_MAX_DEFAULT, "Maximum number of fasttrap probes")static struct sysctl_oid sysctl___kern_dtrace_fasttrap_max_probes = { .oid_parent = ((&(&sysctl___kern_dtrace_fasttrap )->oid_children)), .oid_children = { ((void *)0) }, .oid_number = ((-1)), .oid_kind = (6 | 0x00040000 | (((0x80000000|0x40000000 )|0x00080000))), .oid_arg1 = (&fasttrap_max), .oid_arg2 = (250000), .oid_name = ("max_probes"), .oid_handler = (sysctl_handle_int ), .oid_fmt = ("IU"), .oid_descr = "Maximum number of fasttrap probes" , .oid_label = (((void *)0)), }; __asm__(".globl " "__start_set_sysctl_set" ); __asm__(".globl " "__stop_set_sysctl_set"); static void const * const __set_sysctl_set_sym_sysctl___kern_dtrace_fasttrap_max_probes __attribute__((__section__("set_" "sysctl_set"))) __attribute__ ((__used__)) = &(sysctl___kern_dtrace_fasttrap_max_probes ); _Static_assert((((((0x80000000|0x40000000)|0x00080000)) & 0xf) == 0 || ((((0x80000000|0x40000000)|0x00080000)) & 0xf ) == 6) && sizeof(unsigned) == sizeof(*(&fasttrap_max )), "compile-time assertion failed"); | |||
| 242 | SYSCTL_ULONG(_kern_dtrace_fasttrap, OID_AUTO, tpoints_hash_size, CTLFLAG_RDTUN, &tpoints_hash_size,static struct sysctl_oid sysctl___kern_dtrace_fasttrap_tpoints_hash_size = { .oid_parent = ((&(&sysctl___kern_dtrace_fasttrap )->oid_children)), .oid_children = { ((void *)0) }, .oid_number = ((-1)), .oid_kind = (8 | 0x00040000 | ((0x80000000|0x00080000 ))), .oid_arg1 = (&tpoints_hash_size), .oid_arg2 = (0x4000 ), .oid_name = ("tpoints_hash_size"), .oid_handler = (sysctl_handle_long ), .oid_fmt = ("LU"), .oid_descr = "Size of the tracepoint hash table" , .oid_label = (((void *)0)), }; __asm__(".globl " "__start_set_sysctl_set" ); __asm__(".globl " "__stop_set_sysctl_set"); static void const * const __set_sysctl_set_sym_sysctl___kern_dtrace_fasttrap_tpoints_hash_size __attribute__((__section__("set_" "sysctl_set"))) __attribute__ ((__used__)) = &(sysctl___kern_dtrace_fasttrap_tpoints_hash_size ); _Static_assert(((((0x80000000|0x00080000)) & 0xf) == 0 || (((0x80000000|0x00080000)) & 0xf) == 8) && sizeof (unsigned long) == sizeof(*(&tpoints_hash_size)), "compile-time assertion failed" ) | |||
| 243 | FASTTRAP_TPOINTS_DEFAULT_SIZE, "Size of the tracepoint hash table")static struct sysctl_oid sysctl___kern_dtrace_fasttrap_tpoints_hash_size = { .oid_parent = ((&(&sysctl___kern_dtrace_fasttrap )->oid_children)), .oid_children = { ((void *)0) }, .oid_number = ((-1)), .oid_kind = (8 | 0x00040000 | ((0x80000000|0x00080000 ))), .oid_arg1 = (&tpoints_hash_size), .oid_arg2 = (0x4000 ), .oid_name = ("tpoints_hash_size"), .oid_handler = (sysctl_handle_long ), .oid_fmt = ("LU"), .oid_descr = "Size of the tracepoint hash table" , .oid_label = (((void *)0)), }; __asm__(".globl " "__start_set_sysctl_set" ); __asm__(".globl " "__stop_set_sysctl_set"); static void const * const __set_sysctl_set_sym_sysctl___kern_dtrace_fasttrap_tpoints_hash_size __attribute__((__section__("set_" "sysctl_set"))) __attribute__ ((__used__)) = &(sysctl___kern_dtrace_fasttrap_tpoints_hash_size ); _Static_assert(((((0x80000000|0x00080000)) & 0xf) == 0 || (((0x80000000|0x00080000)) & 0xf) == 8) && sizeof (unsigned long) == sizeof(*(&tpoints_hash_size)), "compile-time assertion failed" ); | |||
| 244 | #endif | |||
| 245 | ||||
| 246 | static int | |||
| 247 | fasttrap_highbit(ulong_t i) | |||
| 248 | { | |||
| 249 | int h = 1; | |||
| 250 | ||||
| 251 | if (i == 0) | |||
| 252 | return (0); | |||
| 253 | #ifdef _LP641 | |||
| 254 | if (i & 0xffffffff00000000ul) { | |||
| 255 | h += 32; i >>= 32; | |||
| 256 | } | |||
| 257 | #endif | |||
| 258 | if (i & 0xffff0000) { | |||
| 259 | h += 16; i >>= 16; | |||
| 260 | } | |||
| 261 | if (i & 0xff00) { | |||
| 262 | h += 8; i >>= 8; | |||
| 263 | } | |||
| 264 | if (i & 0xf0) { | |||
| 265 | h += 4; i >>= 4; | |||
| 266 | } | |||
| 267 | if (i & 0xc) { | |||
| 268 | h += 2; i >>= 2; | |||
| 269 | } | |||
| 270 | if (i & 0x2) { | |||
| 271 | h += 1; | |||
| 272 | } | |||
| 273 | return (h); | |||
| 274 | } | |||
| 275 | ||||
| 276 | static uint_t | |||
| 277 | fasttrap_hash_str(const char *p) | |||
| 278 | { | |||
| 279 | unsigned int g; | |||
| 280 | uint_t hval = 0; | |||
| 281 | ||||
| 282 | while (*p) { | |||
| 283 | hval = (hval << 4) + *p++; | |||
| 284 | if ((g = (hval & 0xf0000000)) != 0) | |||
| 285 | hval ^= g >> 24; | |||
| 286 | hval &= ~g; | |||
| 287 | } | |||
| 288 | return (hval); | |||
| 289 | } | |||
| 290 | ||||
| 291 | void | |||
| 292 | fasttrap_sigtrap(proc_t *p, kthread_t *t, uintptr_t pc) | |||
| 293 | { | |||
| 294 | #ifdef illumos | |||
| 295 | sigqueue_t *sqp = kmem_zalloc(sizeof (sigqueue_t), KM_SLEEP)zfs_kmem_alloc((sizeof (sigqueue_t)), (0x0002) | 0x0100); | |||
| 296 | ||||
| 297 | sqp->sq_info.si_signo = SIGTRAP5; | |||
| 298 | sqp->sq_info.si_code = TRAP_DTRACE3; | |||
| 299 | sqp->sq_info.si_addr = (caddr_t)pc; | |||
| 300 | ||||
| 301 | mutex_enter(&p->p_lock)(void)_sx_xlock(((&p->p_lock)), 0, ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (301)); | |||
| 302 | sigaddqa(p, t, sqp); | |||
| 303 | mutex_exit(&p->p_lock)_sx_xunlock(((&p->p_lock)), ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (303)); | |||
| 304 | ||||
| 305 | if (t != NULL((void *)0)) | |||
| 306 | aston(t); | |||
| 307 | #else | |||
| 308 | ksiginfo_t *ksi = kmem_zalloc(sizeof (ksiginfo_t), KM_SLEEP)zfs_kmem_alloc((sizeof (ksiginfo_t)), (0x0002) | 0x0100); | |||
| 309 | ||||
| 310 | ksiginfo_init(ksi)do { __builtin_memset((ksi), 0, (sizeof(ksiginfo_t))); } while (0); | |||
| 311 | ksi->ksi_signoksi_info.si_signo = SIGTRAP5; | |||
| 312 | ksi->ksi_codeksi_info.si_code = TRAP_DTRACE3; | |||
| 313 | ksi->ksi_addrksi_info.si_addr = (caddr_t)pc; | |||
| 314 | PROC_LOCK(p)__mtx_lock_flags(&((((&(p)->p_mtx))))->mtx_lock , ((0)), ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (314)); | |||
| 315 | (void) tdsendsignal(p, t, SIGTRAP5, ksi); | |||
| 316 | PROC_UNLOCK(p)__mtx_unlock_flags(&((((&(p)->p_mtx))))->mtx_lock , ((0)), ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (316)); | |||
| 317 | #endif | |||
| 318 | } | |||
| 319 | ||||
| 320 | #ifndef illumos | |||
| 321 | /* | |||
| 322 | * Obtain a chunk of scratch space in the address space of the target process. | |||
| 323 | */ | |||
| 324 | fasttrap_scrspace_t * | |||
| 325 | fasttrap_scraddr(struct thread *td, fasttrap_proc_t *fprc) | |||
| 326 | { | |||
| 327 | fasttrap_scrblock_t *scrblk; | |||
| 328 | fasttrap_scrspace_t *scrspc; | |||
| 329 | struct proc *p; | |||
| 330 | vm_offset_t addr; | |||
| 331 | int error, i; | |||
| 332 | ||||
| 333 | scrspc = NULL((void *)0); | |||
| 334 | if (td->t_dtrace_sscrtd_dtrace->td_dtrace_sscr != NULL((void *)0)) { | |||
| 335 | /* If the thread already has scratch space, we're done. */ | |||
| 336 | scrspc = (fasttrap_scrspace_t *)td->t_dtrace_sscrtd_dtrace->td_dtrace_sscr; | |||
| 337 | return (scrspc); | |||
| 338 | } | |||
| 339 | ||||
| 340 | p = td->td_proc; | |||
| 341 | ||||
| 342 | mutex_enter(&fprc->ftpc_mtx)(void)_sx_xlock(((&fprc->ftpc_mtx)), 0, ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (342)); | |||
| 343 | if (LIST_EMPTY(&fprc->ftpc_fscr)((&fprc->ftpc_fscr)->lh_first == ((void *)0))) { | |||
| 344 | /* | |||
| 345 | * No scratch space is available, so we'll map a new scratch | |||
| 346 | * space block into the traced process' address space. | |||
| 347 | */ | |||
| 348 | addr = 0; | |||
| 349 | error = vm_map_find(&p->p_vmspace->vm_map, NULL((void *)0), 0, &addr, | |||
| 350 | FASTTRAP_SCRBLOCK_SIZE(1<<12), 0, VMFS_ANY_SPACE1, VM_PROT_ALL(((vm_prot_t) 0x01)|((vm_prot_t) 0x02)|((vm_prot_t) 0x04)), | |||
| 351 | VM_PROT_ALL(((vm_prot_t) 0x01)|((vm_prot_t) 0x02)|((vm_prot_t) 0x04)), 0); | |||
| 352 | if (error != KERN_SUCCESS0) | |||
| 353 | goto done; | |||
| 354 | ||||
| 355 | scrblk = malloc(sizeof(*scrblk), M_SOLARIS, M_WAITOK0x0002); | |||
| 356 | scrblk->ftsb_addr = addr; | |||
| 357 | LIST_INSERT_HEAD(&fprc->ftpc_scrblks, scrblk, ftsb_next)do { do { if (((((&fprc->ftpc_scrblks)))->lh_first) != ((void *)0) && ((((&fprc->ftpc_scrblks)))-> lh_first)->ftsb_next.le_prev != &((((&fprc->ftpc_scrblks )))->lh_first)) panic("Bad list head %p first->prev != head" , ((&fprc->ftpc_scrblks))); } while (0); if (((((scrblk ))->ftsb_next.le_next) = (((&fprc->ftpc_scrblks))-> lh_first)) != ((void *)0)) (((&fprc->ftpc_scrblks))-> lh_first)->ftsb_next.le_prev = &(((scrblk))->ftsb_next .le_next); (((&fprc->ftpc_scrblks))->lh_first) = (scrblk ); (scrblk)->ftsb_next.le_prev = &(((&fprc->ftpc_scrblks ))->lh_first); } while (0); | |||
| 358 | ||||
| 359 | /* | |||
| 360 | * Carve the block up into chunks and put them on the free list. | |||
| 361 | */ | |||
| 362 | for (i = 0; | |||
| 363 | i < FASTTRAP_SCRBLOCK_SIZE(1<<12) / FASTTRAP_SCRSPACE_SIZE64; i++) { | |||
| 364 | scrspc = malloc(sizeof(*scrspc), M_SOLARIS, M_WAITOK0x0002); | |||
| 365 | scrspc->ftss_addr = addr + | |||
| 366 | i * FASTTRAP_SCRSPACE_SIZE64; | |||
| 367 | LIST_INSERT_HEAD(&fprc->ftpc_fscr, scrspc,do { do { if (((((&fprc->ftpc_fscr)))->lh_first) != ((void *)0) && ((((&fprc->ftpc_fscr)))->lh_first )->ftss_next.le_prev != &((((&fprc->ftpc_fscr)) )->lh_first)) panic("Bad list head %p first->prev != head" , ((&fprc->ftpc_fscr))); } while (0); if (((((scrspc)) ->ftss_next.le_next) = (((&fprc->ftpc_fscr))->lh_first )) != ((void *)0)) (((&fprc->ftpc_fscr))->lh_first) ->ftss_next.le_prev = &(((scrspc))->ftss_next.le_next ); (((&fprc->ftpc_fscr))->lh_first) = (scrspc); (scrspc )->ftss_next.le_prev = &(((&fprc->ftpc_fscr))-> lh_first); } while (0) | |||
| 368 | ftss_next)do { do { if (((((&fprc->ftpc_fscr)))->lh_first) != ((void *)0) && ((((&fprc->ftpc_fscr)))->lh_first )->ftss_next.le_prev != &((((&fprc->ftpc_fscr)) )->lh_first)) panic("Bad list head %p first->prev != head" , ((&fprc->ftpc_fscr))); } while (0); if (((((scrspc)) ->ftss_next.le_next) = (((&fprc->ftpc_fscr))->lh_first )) != ((void *)0)) (((&fprc->ftpc_fscr))->lh_first) ->ftss_next.le_prev = &(((scrspc))->ftss_next.le_next ); (((&fprc->ftpc_fscr))->lh_first) = (scrspc); (scrspc )->ftss_next.le_prev = &(((&fprc->ftpc_fscr))-> lh_first); } while (0); | |||
| 369 | } | |||
| 370 | } | |||
| 371 | ||||
| 372 | /* | |||
| 373 | * Take the first scratch chunk off the free list, put it on the | |||
| 374 | * allocated list, and return its address. | |||
| 375 | */ | |||
| 376 | scrspc = LIST_FIRST(&fprc->ftpc_fscr)((&fprc->ftpc_fscr)->lh_first); | |||
| 377 | LIST_REMOVE(scrspc, ftss_next)do { ; ; do { if ((((scrspc))->ftss_next.le_next) != ((void *)0) && (((scrspc))->ftss_next.le_next)->ftss_next .le_prev != &((scrspc)->ftss_next.le_next)) panic("Bad link elm %p next->prev != elm" , (scrspc)); } while (0); do { if (*(scrspc)->ftss_next.le_prev != (scrspc)) panic("Bad link elm %p prev->next != elm", ( scrspc)); } while (0); if ((((scrspc))->ftss_next.le_next) != ((void *)0)) (((scrspc))->ftss_next.le_next)->ftss_next .le_prev = (scrspc)->ftss_next.le_prev; *(scrspc)->ftss_next .le_prev = (((scrspc))->ftss_next.le_next); ; ; } while (0 ); | |||
| 378 | LIST_INSERT_HEAD(&fprc->ftpc_ascr, scrspc, ftss_next)do { do { if (((((&fprc->ftpc_ascr)))->lh_first) != ((void *)0) && ((((&fprc->ftpc_ascr)))->lh_first )->ftss_next.le_prev != &((((&fprc->ftpc_ascr)) )->lh_first)) panic("Bad list head %p first->prev != head" , ((&fprc->ftpc_ascr))); } while (0); if (((((scrspc)) ->ftss_next.le_next) = (((&fprc->ftpc_ascr))->lh_first )) != ((void *)0)) (((&fprc->ftpc_ascr))->lh_first) ->ftss_next.le_prev = &(((scrspc))->ftss_next.le_next ); (((&fprc->ftpc_ascr))->lh_first) = (scrspc); (scrspc )->ftss_next.le_prev = &(((&fprc->ftpc_ascr))-> lh_first); } while (0); | |||
| 379 | ||||
| 380 | /* | |||
| 381 | * This scratch space is reserved for use by td until the thread exits. | |||
| 382 | */ | |||
| 383 | td->t_dtrace_sscrtd_dtrace->td_dtrace_sscr = scrspc; | |||
| 384 | ||||
| 385 | done: | |||
| 386 | mutex_exit(&fprc->ftpc_mtx)_sx_xunlock(((&fprc->ftpc_mtx)), ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (386)); | |||
| 387 | ||||
| 388 | return (scrspc); | |||
| 389 | } | |||
| 390 | ||||
| 391 | /* | |||
| 392 | * Return any allocated per-thread scratch space chunks back to the process' | |||
| 393 | * free list. | |||
| 394 | */ | |||
| 395 | static void | |||
| 396 | fasttrap_thread_dtor(void *arg __unused__attribute__((__unused__)), struct thread *td) | |||
| 397 | { | |||
| 398 | fasttrap_bucket_t *bucket; | |||
| 399 | fasttrap_proc_t *fprc; | |||
| 400 | fasttrap_scrspace_t *scrspc; | |||
| 401 | pid_t pid; | |||
| 402 | ||||
| 403 | if (td->t_dtrace_sscrtd_dtrace->td_dtrace_sscr == NULL((void *)0)) | |||
| 404 | return; | |||
| 405 | ||||
| 406 | pid = td->td_proc->p_pid; | |||
| 407 | bucket = &fasttrap_procs.fth_table[FASTTRAP_PROCS_INDEX(pid)((pid) & fasttrap_procs.fth_mask)]; | |||
| 408 | fprc = NULL((void *)0); | |||
| 409 | ||||
| 410 | /* Look up the fasttrap process handle for this process. */ | |||
| 411 | mutex_enter(&bucket->ftb_mtx)(void)_sx_xlock(((&bucket->ftb_mtx)), 0, ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (411)); | |||
| 412 | for (fprc = bucket->ftb_data; fprc != NULL((void *)0); fprc = fprc->ftpc_next) { | |||
| 413 | if (fprc->ftpc_pid == pid) { | |||
| 414 | mutex_enter(&fprc->ftpc_mtx)(void)_sx_xlock(((&fprc->ftpc_mtx)), 0, ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (414)); | |||
| 415 | mutex_exit(&bucket->ftb_mtx)_sx_xunlock(((&bucket->ftb_mtx)), ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (415)); | |||
| 416 | break; | |||
| 417 | } | |||
| 418 | } | |||
| 419 | if (fprc == NULL((void *)0)) { | |||
| 420 | mutex_exit(&bucket->ftb_mtx)_sx_xunlock(((&bucket->ftb_mtx)), ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (420)); | |||
| 421 | return; | |||
| 422 | } | |||
| 423 | ||||
| 424 | scrspc = (fasttrap_scrspace_t *)td->t_dtrace_sscrtd_dtrace->td_dtrace_sscr; | |||
| 425 | LIST_REMOVE(scrspc, ftss_next)do { ; ; do { if ((((scrspc))->ftss_next.le_next) != ((void *)0) && (((scrspc))->ftss_next.le_next)->ftss_next .le_prev != &((scrspc)->ftss_next.le_next)) panic("Bad link elm %p next->prev != elm" , (scrspc)); } while (0); do { if (*(scrspc)->ftss_next.le_prev != (scrspc)) panic("Bad link elm %p prev->next != elm", ( scrspc)); } while (0); if ((((scrspc))->ftss_next.le_next) != ((void *)0)) (((scrspc))->ftss_next.le_next)->ftss_next .le_prev = (scrspc)->ftss_next.le_prev; *(scrspc)->ftss_next .le_prev = (((scrspc))->ftss_next.le_next); ; ; } while (0 ); | |||
| 426 | LIST_INSERT_HEAD(&fprc->ftpc_fscr, scrspc, ftss_next)do { do { if (((((&fprc->ftpc_fscr)))->lh_first) != ((void *)0) && ((((&fprc->ftpc_fscr)))->lh_first )->ftss_next.le_prev != &((((&fprc->ftpc_fscr)) )->lh_first)) panic("Bad list head %p first->prev != head" , ((&fprc->ftpc_fscr))); } while (0); if (((((scrspc)) ->ftss_next.le_next) = (((&fprc->ftpc_fscr))->lh_first )) != ((void *)0)) (((&fprc->ftpc_fscr))->lh_first) ->ftss_next.le_prev = &(((scrspc))->ftss_next.le_next ); (((&fprc->ftpc_fscr))->lh_first) = (scrspc); (scrspc )->ftss_next.le_prev = &(((&fprc->ftpc_fscr))-> lh_first); } while (0); | |||
| 427 | ||||
| 428 | mutex_exit(&fprc->ftpc_mtx)_sx_xunlock(((&fprc->ftpc_mtx)), ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (428)); | |||
| 429 | } | |||
| 430 | #endif | |||
| 431 | ||||
| 432 | /* | |||
| 433 | * This function ensures that no threads are actively using the memory | |||
| 434 | * associated with probes that were formerly live. | |||
| 435 | */ | |||
| 436 | static void | |||
| 437 | fasttrap_mod_barrier(uint64_t gen) | |||
| 438 | { | |||
| 439 | int i; | |||
| 440 | ||||
| 441 | if (gen < fasttrap_mod_gen) | |||
| 442 | return; | |||
| 443 | ||||
| 444 | fasttrap_mod_gen++; | |||
| 445 | ||||
| 446 | #ifdef illumos | |||
| 447 | CPU_FOREACH(i)for ((i) = 0; (i) <= mp_maxid; (i)++) if (!(!((((&all_cpus )->__bits[(((((((256)) + (((sizeof(long) * 8)) - 1)) / ((sizeof (long) * 8)))) == 1) ? 0 : (((i)) / (sizeof(long) * 8)))] & (1L << ((((((((256))) + (((sizeof(long) * 8)) - 1)) / ( (sizeof(long) * 8)))) == 1) ? (__size_t)(((i))) : ((((i))) % ( sizeof(long) * 8))))) != 0)))) { | |||
| 448 | mutex_enter(&fasttrap_cpuc_pid_lock[i])(void)_sx_xlock(((&fasttrap_cpuc_pid_lock[i])), 0, ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (448)); | |||
| 449 | mutex_exit(&fasttrap_cpuc_pid_lock[i])_sx_xunlock(((&fasttrap_cpuc_pid_lock[i])), ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (449)); | |||
| 450 | } | |||
| 451 | #else | |||
| 452 | rm_wlock(&fasttrap_tp_lock)_rm_wlock_debug((&fasttrap_tp_lock), "/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" , 452); | |||
| 453 | rm_wunlock(&fasttrap_tp_lock)_rm_wunlock_debug((&fasttrap_tp_lock), "/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" , 453); | |||
| 454 | #endif | |||
| 455 | } | |||
| 456 | ||||
| 457 | /* | |||
| 458 | * This function performs asynchronous cleanup of fasttrap providers. The | |||
| 459 | * Solaris implementation of this mechanism use a timeout that's activated in | |||
| 460 | * fasttrap_pid_cleanup(), but this doesn't work in FreeBSD: one may sleep while | |||
| 461 | * holding the DTrace mutexes, but it is unsafe to sleep in a callout handler. | |||
| 462 | * Thus we use a dedicated process to perform the cleanup when requested. | |||
| 463 | */ | |||
| 464 | /*ARGSUSED*/ | |||
| 465 | static void | |||
| 466 | fasttrap_pid_cleanup_cb(void *data) | |||
| 467 | { | |||
| 468 | fasttrap_provider_t **fpp, *fp; | |||
| 469 | fasttrap_bucket_t *bucket; | |||
| 470 | dtrace_provider_id_t provid; | |||
| 471 | int i, later = 0, rval; | |||
| 472 | ||||
| 473 | mtx_lock(&fasttrap_cleanup_mtx)__mtx_lock_flags(&((((&fasttrap_cleanup_mtx))))->mtx_lock , ((0)), ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (473)); | |||
| 474 | while (!fasttrap_cleanup_drain || later > 0) { | |||
| 475 | fasttrap_cleanup_work = 0; | |||
| 476 | mtx_unlock(&fasttrap_cleanup_mtx)__mtx_unlock_flags(&((((&fasttrap_cleanup_mtx))))-> mtx_lock, ((0)), ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (476)); | |||
| 477 | ||||
| 478 | later = 0; | |||
| 479 | ||||
| 480 | /* | |||
| 481 | * Iterate over all the providers trying to remove the marked | |||
| 482 | * ones. If a provider is marked but not retired, we just | |||
| 483 | * have to take a crack at removing it -- it's no big deal if | |||
| 484 | * we can't. | |||
| 485 | */ | |||
| 486 | for (i = 0; i < fasttrap_provs.fth_nent; i++) { | |||
| 487 | bucket = &fasttrap_provs.fth_table[i]; | |||
| 488 | mutex_enter(&bucket->ftb_mtx)(void)_sx_xlock(((&bucket->ftb_mtx)), 0, ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (488)); | |||
| 489 | fpp = (fasttrap_provider_t **)&bucket->ftb_data; | |||
| 490 | ||||
| 491 | while ((fp = *fpp) != NULL((void *)0)) { | |||
| 492 | if (!fp->ftp_marked) { | |||
| 493 | fpp = &fp->ftp_next; | |||
| 494 | continue; | |||
| 495 | } | |||
| 496 | ||||
| 497 | mutex_enter(&fp->ftp_mtx)(void)_sx_xlock(((&fp->ftp_mtx)), 0, ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (497)); | |||
| 498 | ||||
| 499 | /* | |||
| 500 | * If this provider has consumers actively | |||
| 501 | * creating probes (ftp_ccount) or is a USDT | |||
| 502 | * provider (ftp_mcount), we can't unregister | |||
| 503 | * or even condense. | |||
| 504 | */ | |||
| 505 | if (fp->ftp_ccount != 0 || | |||
| 506 | fp->ftp_mcount != 0) { | |||
| 507 | mutex_exit(&fp->ftp_mtx)_sx_xunlock(((&fp->ftp_mtx)), ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (507)); | |||
| 508 | fp->ftp_marked = 0; | |||
| 509 | continue; | |||
| 510 | } | |||
| 511 | ||||
| 512 | if (!fp->ftp_retired || fp->ftp_rcount != 0) | |||
| 513 | fp->ftp_marked = 0; | |||
| 514 | ||||
| 515 | mutex_exit(&fp->ftp_mtx)_sx_xunlock(((&fp->ftp_mtx)), ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (515)); | |||
| 516 | ||||
| 517 | /* | |||
| 518 | * If we successfully unregister this | |||
| 519 | * provider we can remove it from the hash | |||
| 520 | * chain and free the memory. If our attempt | |||
| 521 | * to unregister fails and this is a retired | |||
| 522 | * provider, increment our flag to try again | |||
| 523 | * pretty soon. If we've consumed more than | |||
| 524 | * half of our total permitted number of | |||
| 525 | * probes call dtrace_condense() to try to | |||
| 526 | * clean out the unenabled probes. | |||
| 527 | */ | |||
| 528 | provid = fp->ftp_provid; | |||
| 529 | if ((rval = dtrace_unregister(provid)) != 0) { | |||
| 530 | if (fasttrap_total > fasttrap_max / 2) | |||
| 531 | (void) dtrace_condense(provid); | |||
| 532 | ||||
| 533 | if (rval == EAGAIN35) | |||
| 534 | fp->ftp_marked = 1; | |||
| 535 | ||||
| 536 | later += fp->ftp_marked; | |||
| 537 | fpp = &fp->ftp_next; | |||
| 538 | } else { | |||
| 539 | *fpp = fp->ftp_next; | |||
| 540 | fasttrap_provider_free(fp); | |||
| 541 | } | |||
| 542 | } | |||
| 543 | mutex_exit(&bucket->ftb_mtx)_sx_xunlock(((&bucket->ftb_mtx)), ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (543)); | |||
| 544 | } | |||
| 545 | mtx_lock(&fasttrap_cleanup_mtx)__mtx_lock_flags(&((((&fasttrap_cleanup_mtx))))->mtx_lock , ((0)), ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (545)); | |||
| 546 | ||||
| 547 | /* | |||
| 548 | * If we were unable to retire a provider, try again after a | |||
| 549 | * second. This situation can occur in certain circumstances | |||
| 550 | * where providers cannot be unregistered even though they have | |||
| 551 | * no probes enabled because of an execution of dtrace -l or | |||
| 552 | * something similar. | |||
| 553 | */ | |||
| 554 | if (later > 0 || fasttrap_cleanup_work || | |||
| 555 | fasttrap_cleanup_drain) { | |||
| 556 | mtx_unlock(&fasttrap_cleanup_mtx)__mtx_unlock_flags(&((((&fasttrap_cleanup_mtx))))-> mtx_lock, ((0)), ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (556)); | |||
| 557 | pause("ftclean", hz)pause_sbt(("ftclean"), tick_sbt * (hz), 0, 0x0100); | |||
| 558 | mtx_lock(&fasttrap_cleanup_mtx)__mtx_lock_flags(&((((&fasttrap_cleanup_mtx))))->mtx_lock , ((0)), ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (558)); | |||
| 559 | } else | |||
| 560 | mtx_sleep(&fasttrap_cleanup_cv, &fasttrap_cleanup_mtx,_sleep((&fasttrap_cleanup_cv), &(&fasttrap_cleanup_mtx )->lock_object, (0), ("ftcl"), tick_sbt * (0), 0, 0x0100) | |||
| 561 | 0, "ftcl", 0)_sleep((&fasttrap_cleanup_cv), &(&fasttrap_cleanup_mtx )->lock_object, (0), ("ftcl"), tick_sbt * (0), 0, 0x0100); | |||
| 562 | } | |||
| 563 | ||||
| 564 | /* | |||
| 565 | * Wake up the thread in fasttrap_unload() now that we're done. | |||
| 566 | */ | |||
| 567 | wakeup(&fasttrap_cleanup_drain); | |||
| 568 | mtx_unlock(&fasttrap_cleanup_mtx)__mtx_unlock_flags(&((((&fasttrap_cleanup_mtx))))-> mtx_lock, ((0)), ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (568)); | |||
| 569 | ||||
| 570 | kthread_exit(); | |||
| 571 | } | |||
| 572 | ||||
| 573 | /* | |||
| 574 | * Activates the asynchronous cleanup mechanism. | |||
| 575 | */ | |||
| 576 | static void | |||
| 577 | fasttrap_pid_cleanup(void) | |||
| 578 | { | |||
| 579 | ||||
| 580 | mtx_lock(&fasttrap_cleanup_mtx)__mtx_lock_flags(&((((&fasttrap_cleanup_mtx))))->mtx_lock , ((0)), ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (580)); | |||
| 581 | if (!fasttrap_cleanup_work) { | |||
| 582 | fasttrap_cleanup_work = 1; | |||
| 583 | wakeup(&fasttrap_cleanup_cv); | |||
| 584 | } | |||
| 585 | mtx_unlock(&fasttrap_cleanup_mtx)__mtx_unlock_flags(&((((&fasttrap_cleanup_mtx))))-> mtx_lock, ((0)), ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (585)); | |||
| 586 | } | |||
| 587 | ||||
| 588 | /* | |||
| 589 | * This is called from cfork() via dtrace_fasttrap_fork(). The child | |||
| 590 | * process's address space is (roughly) a copy of the parent process's so | |||
| 591 | * we have to remove all the instrumentation we had previously enabled in the | |||
| 592 | * parent. | |||
| 593 | */ | |||
| 594 | static void | |||
| 595 | fasttrap_fork(proc_t *p, proc_t *cp) | |||
| 596 | { | |||
| 597 | #ifndef illumos | |||
| 598 | fasttrap_scrblock_t *scrblk; | |||
| 599 | fasttrap_proc_t *fprc = NULL((void *)0); | |||
| 600 | #endif | |||
| 601 | pid_t ppid = p->p_pid; | |||
| 602 | int i; | |||
| 603 | ||||
| 604 | ASSERT(curproc == p)((void)((((__curthread())->td_proc) == p) || dtrace_assfail ("curproc == p", "/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" , 604))); | |||
| 605 | #ifdef illumos | |||
| 606 | ASSERT(p->p_proc_flag & P_PR_LOCK)((void)((p->p_proc_flag & P_PR_LOCK) || dtrace_assfail ("p->p_proc_flag & P_PR_LOCK", "/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" , 606))); | |||
| 607 | #else | |||
| 608 | PROC_LOCK_ASSERT(p, MA_OWNED)__mtx_assert(&(((&(p)->p_mtx)))->mtx_lock, (((0x00000004 ))), ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (608)); | |||
| 609 | #endif | |||
| 610 | #ifdef illumos | |||
| 611 | ASSERT(p->p_dtrace_count > 0)((void)((p->p_dtrace->p_dtrace_count > 0) || dtrace_assfail ("p->p_dtrace_count > 0", "/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" , 611))); | |||
| 612 | #else | |||
| 613 | /* | |||
| 614 | * This check is purposely here instead of in kern_fork.c because, | |||
| 615 | * for legal resons, we cannot include the dtrace_cddl.h header | |||
| 616 | * inside kern_fork.c and insert if-clause there. | |||
| 617 | */ | |||
| 618 | if (p->p_dtrace_countp_dtrace->p_dtrace_count == 0 && p->p_dtrace_helpersp_dtrace->p_dtrace_helpers == NULL((void *)0)) | |||
| 619 | return; | |||
| 620 | #endif | |||
| 621 | ||||
| 622 | ASSERT(cp->p_dtrace_count == 0)((void)((cp->p_dtrace->p_dtrace_count == 0) || dtrace_assfail ("cp->p_dtrace_count == 0", "/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" , 622))); | |||
| 623 | ||||
| 624 | /* | |||
| 625 | * This would be simpler and faster if we maintained per-process | |||
| 626 | * hash tables of enabled tracepoints. It could, however, potentially | |||
| 627 | * slow down execution of a tracepoint since we'd need to go | |||
| 628 | * through two levels of indirection. In the future, we should | |||
| 629 | * consider either maintaining per-process ancillary lists of | |||
| 630 | * enabled tracepoints or hanging a pointer to a per-process hash | |||
| 631 | * table of enabled tracepoints off the proc structure. | |||
| 632 | */ | |||
| 633 | ||||
| 634 | /* | |||
| 635 | * We don't have to worry about the child process disappearing | |||
| 636 | * because we're in fork(). | |||
| 637 | */ | |||
| 638 | #ifdef illumos | |||
| 639 | mtx_lock_spin(&cp->p_slock)__mtx_lock_spin_flags(&((((&cp->p_slock))))->mtx_lock , ((0)), ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (639)); | |||
| 640 | sprlock_proc(cp); | |||
| 641 | mtx_unlock_spin(&cp->p_slock)__mtx_unlock_spin_flags(&((((&cp->p_slock))))-> mtx_lock, ((0)), ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (641)); | |||
| 642 | #else | |||
| 643 | /* | |||
| 644 | * fasttrap_tracepoint_remove() expects the child process to be | |||
| 645 | * unlocked and the VM then expects curproc to be unlocked. | |||
| 646 | */ | |||
| 647 | _PHOLD(cp)do { __mtx_assert(&(((&((cp))->p_mtx)))->mtx_lock , (((0x00000004))), ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (647)); do { if (__builtin_expect((!(!((cp)->p_flag & 0x02000) || (cp) == ((__curthread())->td_proc))), 0)) panic ("PHOLD of exiting process %p", cp); } while (0); (cp)->p_lock ++; if (((cp)->p_flag & 0x10000000) == 0) faultin((cp) ); } while (0); | |||
| 648 | PROC_UNLOCK(cp)__mtx_unlock_flags(&((((&(cp)->p_mtx))))->mtx_lock , ((0)), ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (648)); | |||
| 649 | PROC_UNLOCK(p)__mtx_unlock_flags(&((((&(p)->p_mtx))))->mtx_lock , ((0)), ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (649)); | |||
| 650 | if (p->p_dtrace_countp_dtrace->p_dtrace_count == 0) | |||
| 651 | goto dup_helpers; | |||
| 652 | #endif | |||
| 653 | ||||
| 654 | /* | |||
| 655 | * Iterate over every tracepoint looking for ones that belong to the | |||
| 656 | * parent process, and remove each from the child process. | |||
| 657 | */ | |||
| 658 | for (i = 0; i < fasttrap_tpoints.fth_nent; i++) { | |||
| 659 | fasttrap_tracepoint_t *tp; | |||
| 660 | fasttrap_bucket_t *bucket = &fasttrap_tpoints.fth_table[i]; | |||
| 661 | ||||
| 662 | mutex_enter(&bucket->ftb_mtx)(void)_sx_xlock(((&bucket->ftb_mtx)), 0, ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (662)); | |||
| 663 | for (tp = bucket->ftb_data; tp != NULL((void *)0); tp = tp->ftt_next) { | |||
| 664 | if (tp->ftt_pid == ppid && | |||
| 665 | tp->ftt_proc->ftpc_acount != 0) { | |||
| 666 | int ret = fasttrap_tracepoint_remove(cp, tp); | |||
| 667 | ASSERT(ret == 0)((void)((ret == 0) || dtrace_assfail("ret == 0", "/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" , 667))); | |||
| 668 | ||||
| 669 | /* | |||
| 670 | * The count of active providers can only be | |||
| 671 | * decremented (i.e. to zero) during exec, | |||
| 672 | * exit, and removal of a meta provider so it | |||
| 673 | * should be impossible to drop the count | |||
| 674 | * mid-fork. | |||
| 675 | */ | |||
| 676 | ASSERT(tp->ftt_proc->ftpc_acount != 0)((void)((tp->ftt_proc->ftpc_acount != 0) || dtrace_assfail ("tp->ftt_proc->ftpc_acount != 0", "/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" , 676))); | |||
| 677 | #ifndef illumos | |||
| 678 | fprc = tp->ftt_proc; | |||
| 679 | #endif | |||
| 680 | } | |||
| 681 | } | |||
| 682 | mutex_exit(&bucket->ftb_mtx)_sx_xunlock(((&bucket->ftb_mtx)), ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (682)); | |||
| 683 | ||||
| 684 | #ifndef illumos | |||
| 685 | /* | |||
| 686 | * Unmap any scratch space inherited from the parent's address | |||
| 687 | * space. | |||
| 688 | */ | |||
| 689 | if (fprc != NULL((void *)0)) { | |||
| 690 | mutex_enter(&fprc->ftpc_mtx)(void)_sx_xlock(((&fprc->ftpc_mtx)), 0, ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (690)); | |||
| 691 | LIST_FOREACH(scrblk, &fprc->ftpc_scrblks, ftsb_next)for ((scrblk) = (((&fprc->ftpc_scrblks))->lh_first) ; (scrblk); (scrblk) = (((scrblk))->ftsb_next.le_next)) { | |||
| 692 | vm_map_remove(&cp->p_vmspace->vm_map, | |||
| 693 | scrblk->ftsb_addr, | |||
| 694 | scrblk->ftsb_addr + FASTTRAP_SCRBLOCK_SIZE(1<<12)); | |||
| 695 | } | |||
| 696 | mutex_exit(&fprc->ftpc_mtx)_sx_xunlock(((&fprc->ftpc_mtx)), ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (696)); | |||
| 697 | } | |||
| 698 | #endif | |||
| 699 | } | |||
| 700 | ||||
| 701 | #ifdef illumos | |||
| 702 | mutex_enter(&cp->p_lock)(void)_sx_xlock(((&cp->p_lock)), 0, ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (702)); | |||
| 703 | sprunlock(cp); | |||
| 704 | #else | |||
| 705 | dup_helpers: | |||
| 706 | if (p->p_dtrace_helpersp_dtrace->p_dtrace_helpers != NULL((void *)0)) | |||
| 707 | dtrace_helpers_duplicate(p, cp); | |||
| 708 | PROC_LOCK(p)__mtx_lock_flags(&((((&(p)->p_mtx))))->mtx_lock , ((0)), ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (708)); | |||
| 709 | PROC_LOCK(cp)__mtx_lock_flags(&((((&(cp)->p_mtx))))->mtx_lock , ((0)), ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (709)); | |||
| 710 | _PRELE(cp)do { __mtx_assert(&(((&((cp))->p_mtx)))->mtx_lock , (((0x00000004))), ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (710)); do { do { if (__builtin_expect((!((cp)->p_lock > 0)), 0)) panic ("process %p not held", cp); } while (0); } while (0); (--(cp)->p_lock); if (((cp)->p_flag & 0x02000 ) && (cp)->p_lock == 0) wakeup(&(cp)->p_lock ); } while (0); | |||
| 711 | #endif | |||
| 712 | } | |||
| 713 | ||||
| 714 | /* | |||
| 715 | * This is called from proc_exit() or from exec_common() if p_dtrace_probes | |||
| 716 | * is set on the proc structure to indicate that there is a pid provider | |||
| 717 | * associated with this process. | |||
| 718 | */ | |||
| 719 | static void | |||
| 720 | fasttrap_exec_exit(proc_t *p) | |||
| 721 | { | |||
| 722 | #ifndef illumos | |||
| 723 | struct thread *td; | |||
| 724 | #endif | |||
| 725 | ||||
| 726 | #ifdef illumos | |||
| 727 | ASSERT(p == curproc)((void)((p == ((__curthread())->td_proc)) || dtrace_assfail ("p == curproc", "/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" , 727))); | |||
| 728 | #else | |||
| 729 | PROC_LOCK_ASSERT(p, MA_OWNED)__mtx_assert(&(((&(p)->p_mtx)))->mtx_lock, (((0x00000004 ))), ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (729)); | |||
| 730 | _PHOLD(p)do { __mtx_assert(&(((&((p))->p_mtx)))->mtx_lock , (((0x00000004))), ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (730)); do { if (__builtin_expect((!(!((p)->p_flag & 0x02000) || (p) == ((__curthread())->td_proc))), 0)) panic ("PHOLD of exiting process %p", p); } while (0); (p)->p_lock ++; if (((p)->p_flag & 0x10000000) == 0) faultin((p)); } while (0); | |||
| 731 | /* | |||
| 732 | * Since struct threads may be recycled, we cannot rely on t_dtrace_sscr | |||
| 733 | * fields to be zeroed by kdtrace_thread_ctor. Thus we must zero it | |||
| 734 | * ourselves when a process exits. | |||
| 735 | */ | |||
| 736 | FOREACH_THREAD_IN_PROC(p, td)for (((td)) = (((&(p)->p_threads))->tqh_first); ((td )); ((td)) = ((((td)))->td_plist.tqe_next)) | |||
| 737 | td->t_dtrace_sscrtd_dtrace->td_dtrace_sscr = NULL((void *)0); | |||
| 738 | PROC_UNLOCK(p)__mtx_unlock_flags(&((((&(p)->p_mtx))))->mtx_lock , ((0)), ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (738)); | |||
| 739 | #endif | |||
| 740 | ||||
| 741 | /* | |||
| 742 | * We clean up the pid provider for this process here; user-land | |||
| 743 | * static probes are handled by the meta-provider remove entry point. | |||
| 744 | */ | |||
| 745 | fasttrap_provider_retire(p->p_pid, FASTTRAP_PID_NAME"pid", 0); | |||
| 746 | #ifndef illumos | |||
| 747 | if (p->p_dtrace_helpersp_dtrace->p_dtrace_helpers) | |||
| 748 | dtrace_helpers_destroy(p); | |||
| 749 | PROC_LOCK(p)__mtx_lock_flags(&((((&(p)->p_mtx))))->mtx_lock , ((0)), ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (749)); | |||
| 750 | _PRELE(p)do { __mtx_assert(&(((&((p))->p_mtx)))->mtx_lock , (((0x00000004))), ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (750)); do { do { if (__builtin_expect((!((p)->p_lock > 0)), 0)) panic ("process %p not held", p); } while (0); } while (0); (--(p)->p_lock); if (((p)->p_flag & 0x02000) && (p)->p_lock == 0) wakeup(&(p)->p_lock); } while (0 ); | |||
| 751 | #endif | |||
| 752 | } | |||
| 753 | ||||
| 754 | ||||
| 755 | /*ARGSUSED*/ | |||
| 756 | static void | |||
| 757 | fasttrap_pid_provide(void *arg, dtrace_probedesc_t *desc) | |||
| 758 | { | |||
| 759 | /* | |||
| 760 | * There are no "default" pid probes. | |||
| 761 | */ | |||
| 762 | } | |||
| 763 | ||||
| 764 | static int | |||
| 765 | fasttrap_tracepoint_enable(proc_t *p, fasttrap_probe_t *probe, uint_t index) | |||
| 766 | { | |||
| 767 | fasttrap_tracepoint_t *tp, *new_tp = NULL((void *)0); | |||
| 768 | fasttrap_bucket_t *bucket; | |||
| 769 | fasttrap_id_t *id; | |||
| 770 | pid_t pid; | |||
| 771 | uintptr_t pc; | |||
| 772 | ||||
| 773 | ASSERT(index < probe->ftp_ntps)((void)((index < probe->ftp_ntps) || dtrace_assfail("index < probe->ftp_ntps" , "/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" , 773))); | |||
| 774 | ||||
| 775 | pid = probe->ftp_pid; | |||
| 776 | pc = probe->ftp_tps[index].fit_tp->ftt_pc; | |||
| 777 | id = &probe->ftp_tps[index].fit_id; | |||
| 778 | ||||
| 779 | ASSERT(probe->ftp_tps[index].fit_tp->ftt_pid == pid)((void)((probe->ftp_tps[index].fit_tp->ftt_pid == pid) || dtrace_assfail("probe->ftp_tps[index].fit_tp->ftt_pid == pid" , "/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" , 779))); | |||
| 780 | ||||
| 781 | #ifdef illumos | |||
| 782 | ASSERT(!(p->p_flag & SVFORK))((void)((!(p->p_flag & SVFORK)) || dtrace_assfail("!(p->p_flag & SVFORK)" , "/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" , 782))); | |||
| 783 | #endif | |||
| 784 | ||||
| 785 | /* | |||
| 786 | * Before we make any modifications, make sure we've imposed a barrier | |||
| 787 | * on the generation in which this probe was last modified. | |||
| 788 | */ | |||
| 789 | fasttrap_mod_barrier(probe->ftp_gen); | |||
| 790 | ||||
| 791 | bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)(((pc) / sizeof (fasttrap_instr_t) + (pid)) & fasttrap_tpoints .fth_mask)]; | |||
| 792 | ||||
| 793 | /* | |||
| 794 | * If the tracepoint has already been enabled, just add our id to the | |||
| 795 | * list of interested probes. This may be our second time through | |||
| 796 | * this path in which case we'll have constructed the tracepoint we'd | |||
| 797 | * like to install. If we can't find a match, and have an allocated | |||
| 798 | * tracepoint ready to go, enable that one now. | |||
| 799 | * | |||
| 800 | * A tracepoint whose process is defunct is also considered defunct. | |||
| 801 | */ | |||
| 802 | again: | |||
| 803 | mutex_enter(&bucket->ftb_mtx)(void)_sx_xlock(((&bucket->ftb_mtx)), 0, ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (803)); | |||
| 804 | for (tp = bucket->ftb_data; tp != NULL((void *)0); tp = tp->ftt_next) { | |||
| 805 | /* | |||
| 806 | * Note that it's safe to access the active count on the | |||
| 807 | * associated proc structure because we know that at least one | |||
| 808 | * provider (this one) will still be around throughout this | |||
| 809 | * operation. | |||
| 810 | */ | |||
| 811 | if (tp->ftt_pid != pid || tp->ftt_pc != pc || | |||
| 812 | tp->ftt_proc->ftpc_acount == 0) | |||
| 813 | continue; | |||
| 814 | ||||
| 815 | /* | |||
| 816 | * Now that we've found a matching tracepoint, it would be | |||
| 817 | * a decent idea to confirm that the tracepoint is still | |||
| 818 | * enabled and the trap instruction hasn't been overwritten. | |||
| 819 | * Since this is a little hairy, we'll punt for now. | |||
| 820 | */ | |||
| 821 | ||||
| 822 | /* | |||
| 823 | * This can't be the first interested probe. We don't have | |||
| 824 | * to worry about another thread being in the midst of | |||
| 825 | * deleting this tracepoint (which would be the only valid | |||
| 826 | * reason for a tracepoint to have no interested probes) | |||
| 827 | * since we're holding P_PR_LOCK for this process. | |||
| 828 | */ | |||
| 829 | ASSERT(tp->ftt_ids != NULL || tp->ftt_retids != NULL)((void)((tp->ftt_ids != ((void *)0) || tp->ftt_retids != ((void *)0)) || dtrace_assfail("tp->ftt_ids != NULL || tp->ftt_retids != NULL" , "/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" , 829))); | |||
| 830 | ||||
| 831 | switch (id->fti_ptype) { | |||
| 832 | case DTFTP_ENTRY: | |||
| 833 | case DTFTP_OFFSETS: | |||
| 834 | case DTFTP_IS_ENABLED: | |||
| 835 | id->fti_next = tp->ftt_ids; | |||
| 836 | membar_producer(); | |||
| 837 | tp->ftt_ids = id; | |||
| 838 | membar_producer(); | |||
| 839 | break; | |||
| 840 | ||||
| 841 | case DTFTP_RETURN: | |||
| 842 | case DTFTP_POST_OFFSETS: | |||
| 843 | id->fti_next = tp->ftt_retids; | |||
| 844 | membar_producer(); | |||
| 845 | tp->ftt_retids = id; | |||
| 846 | membar_producer(); | |||
| 847 | break; | |||
| 848 | ||||
| 849 | default: | |||
| 850 | ASSERT(0)((void)((0) || dtrace_assfail("0", "/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" , 850))); | |||
| 851 | } | |||
| 852 | ||||
| 853 | mutex_exit(&bucket->ftb_mtx)_sx_xunlock(((&bucket->ftb_mtx)), ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (853)); | |||
| 854 | ||||
| 855 | if (new_tp != NULL((void *)0)) { | |||
| 856 | new_tp->ftt_ids = NULL((void *)0); | |||
| 857 | new_tp->ftt_retids = NULL((void *)0); | |||
| 858 | } | |||
| 859 | ||||
| 860 | return (0); | |||
| 861 | } | |||
| 862 | ||||
| 863 | /* | |||
| 864 | * If we have a good tracepoint ready to go, install it now while | |||
| 865 | * we have the lock held and no one can screw with us. | |||
| 866 | */ | |||
| 867 | if (new_tp != NULL((void *)0)) { | |||
| 868 | int rc = 0; | |||
| 869 | ||||
| 870 | new_tp->ftt_next = bucket->ftb_data; | |||
| 871 | membar_producer(); | |||
| 872 | bucket->ftb_data = new_tp; | |||
| 873 | membar_producer(); | |||
| 874 | mutex_exit(&bucket->ftb_mtx)_sx_xunlock(((&bucket->ftb_mtx)), ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (874)); | |||
| 875 | ||||
| 876 | /* | |||
| 877 | * Activate the tracepoint in the ISA-specific manner. | |||
| 878 | * If this fails, we need to report the failure, but | |||
| 879 | * indicate that this tracepoint must still be disabled | |||
| 880 | * by calling fasttrap_tracepoint_disable(). | |||
| 881 | */ | |||
| 882 | if (fasttrap_tracepoint_install(p, new_tp) != 0) | |||
| 883 | rc = FASTTRAP_ENABLE_PARTIAL2; | |||
| 884 | ||||
| 885 | /* | |||
| 886 | * Increment the count of the number of tracepoints active in | |||
| 887 | * the victim process. | |||
| 888 | */ | |||
| 889 | #ifdef illumos | |||
| 890 | ASSERT(p->p_proc_flag & P_PR_LOCK)((void)((p->p_proc_flag & P_PR_LOCK) || dtrace_assfail ("p->p_proc_flag & P_PR_LOCK", "/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" , 890))); | |||
| 891 | #endif | |||
| 892 | p->p_dtrace_countp_dtrace->p_dtrace_count++; | |||
| 893 | ||||
| 894 | return (rc); | |||
| 895 | } | |||
| 896 | ||||
| 897 | mutex_exit(&bucket->ftb_mtx)_sx_xunlock(((&bucket->ftb_mtx)), ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (897)); | |||
| 898 | ||||
| 899 | /* | |||
| 900 | * Initialize the tracepoint that's been preallocated with the probe. | |||
| 901 | */ | |||
| 902 | new_tp = probe->ftp_tps[index].fit_tp; | |||
| 903 | ||||
| 904 | ASSERT(new_tp->ftt_pid == pid)((void)((new_tp->ftt_pid == pid) || dtrace_assfail("new_tp->ftt_pid == pid" , "/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" , 904))); | |||
| 905 | ASSERT(new_tp->ftt_pc == pc)((void)((new_tp->ftt_pc == pc) || dtrace_assfail("new_tp->ftt_pc == pc" , "/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" , 905))); | |||
| 906 | ASSERT(new_tp->ftt_proc == probe->ftp_prov->ftp_proc)((void)((new_tp->ftt_proc == probe->ftp_prov->ftp_proc ) || dtrace_assfail("new_tp->ftt_proc == probe->ftp_prov->ftp_proc" , "/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" , 906))); | |||
| 907 | ASSERT(new_tp->ftt_ids == NULL)((void)((new_tp->ftt_ids == ((void *)0)) || dtrace_assfail ("new_tp->ftt_ids == NULL", "/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" , 907))); | |||
| 908 | ASSERT(new_tp->ftt_retids == NULL)((void)((new_tp->ftt_retids == ((void *)0)) || dtrace_assfail ("new_tp->ftt_retids == NULL", "/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" , 908))); | |||
| 909 | ||||
| 910 | switch (id->fti_ptype) { | |||
| 911 | case DTFTP_ENTRY: | |||
| 912 | case DTFTP_OFFSETS: | |||
| 913 | case DTFTP_IS_ENABLED: | |||
| 914 | id->fti_next = NULL((void *)0); | |||
| 915 | new_tp->ftt_ids = id; | |||
| 916 | break; | |||
| 917 | ||||
| 918 | case DTFTP_RETURN: | |||
| 919 | case DTFTP_POST_OFFSETS: | |||
| 920 | id->fti_next = NULL((void *)0); | |||
| 921 | new_tp->ftt_retids = id; | |||
| 922 | break; | |||
| 923 | ||||
| 924 | default: | |||
| 925 | ASSERT(0)((void)((0) || dtrace_assfail("0", "/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" , 925))); | |||
| 926 | } | |||
| 927 | ||||
| 928 | #ifdef __FreeBSD__11 | |||
| 929 | if (SV_PROC_FLAG(p, SV_LP64)((p)->p_sysent->sv_flags & (0x000200))) | |||
| 930 | p->p_modelp_dtrace->p_dtrace_model = DATAMODEL_LP640; | |||
| 931 | else | |||
| 932 | p->p_modelp_dtrace->p_dtrace_model = DATAMODEL_ILP321; | |||
| 933 | #endif | |||
| 934 | ||||
| 935 | /* | |||
| 936 | * If the ISA-dependent initialization goes to plan, go back to the | |||
| 937 | * beginning and try to install this freshly made tracepoint. | |||
| 938 | */ | |||
| 939 | if (fasttrap_tracepoint_init(p, new_tp, pc, id->fti_ptype) == 0) | |||
| 940 | goto again; | |||
| 941 | ||||
| 942 | new_tp->ftt_ids = NULL((void *)0); | |||
| 943 | new_tp->ftt_retids = NULL((void *)0); | |||
| 944 | ||||
| 945 | return (FASTTRAP_ENABLE_FAIL1); | |||
| 946 | } | |||
| 947 | ||||
| 948 | static void | |||
| 949 | fasttrap_tracepoint_disable(proc_t *p, fasttrap_probe_t *probe, uint_t index) | |||
| 950 | { | |||
| 951 | fasttrap_bucket_t *bucket; | |||
| 952 | fasttrap_provider_t *provider = probe->ftp_prov; | |||
| 953 | fasttrap_tracepoint_t **pp, *tp; | |||
| 954 | fasttrap_id_t *id, **idp = NULL((void *)0); | |||
| 955 | pid_t pid; | |||
| 956 | uintptr_t pc; | |||
| 957 | ||||
| 958 | ASSERT(index < probe->ftp_ntps)((void)((index < probe->ftp_ntps) || dtrace_assfail("index < probe->ftp_ntps" , "/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" , 958))); | |||
| 959 | ||||
| 960 | pid = probe->ftp_pid; | |||
| 961 | pc = probe->ftp_tps[index].fit_tp->ftt_pc; | |||
| 962 | id = &probe->ftp_tps[index].fit_id; | |||
| 963 | ||||
| 964 | ASSERT(probe->ftp_tps[index].fit_tp->ftt_pid == pid)((void)((probe->ftp_tps[index].fit_tp->ftt_pid == pid) || dtrace_assfail("probe->ftp_tps[index].fit_tp->ftt_pid == pid" , "/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" , 964))); | |||
| 965 | ||||
| 966 | /* | |||
| 967 | * Find the tracepoint and make sure that our id is one of the | |||
| 968 | * ones registered with it. | |||
| 969 | */ | |||
| 970 | bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)(((pc) / sizeof (fasttrap_instr_t) + (pid)) & fasttrap_tpoints .fth_mask)]; | |||
| 971 | mutex_enter(&bucket->ftb_mtx)(void)_sx_xlock(((&bucket->ftb_mtx)), 0, ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (971)); | |||
| 972 | for (tp = bucket->ftb_data; tp != NULL((void *)0); tp = tp->ftt_next) { | |||
| 973 | if (tp->ftt_pid == pid && tp->ftt_pc == pc && | |||
| 974 | tp->ftt_proc == provider->ftp_proc) | |||
| 975 | break; | |||
| 976 | } | |||
| 977 | ||||
| 978 | /* | |||
| 979 | * If we somehow lost this tracepoint, we're in a world of hurt. | |||
| 980 | */ | |||
| 981 | ASSERT(tp != NULL)((void)((tp != ((void *)0)) || dtrace_assfail("tp != NULL", "/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" , 981))); | |||
| 982 | ||||
| 983 | switch (id->fti_ptype) { | |||
| 984 | case DTFTP_ENTRY: | |||
| 985 | case DTFTP_OFFSETS: | |||
| 986 | case DTFTP_IS_ENABLED: | |||
| 987 | ASSERT(tp->ftt_ids != NULL)((void)((tp->ftt_ids != ((void *)0)) || dtrace_assfail("tp->ftt_ids != NULL" , "/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" , 987))); | |||
| 988 | idp = &tp->ftt_ids; | |||
| 989 | break; | |||
| 990 | ||||
| 991 | case DTFTP_RETURN: | |||
| 992 | case DTFTP_POST_OFFSETS: | |||
| 993 | ASSERT(tp->ftt_retids != NULL)((void)((tp->ftt_retids != ((void *)0)) || dtrace_assfail( "tp->ftt_retids != NULL", "/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" , 993))); | |||
| 994 | idp = &tp->ftt_retids; | |||
| 995 | break; | |||
| 996 | ||||
| 997 | default: | |||
| 998 | ASSERT(0)((void)((0) || dtrace_assfail("0", "/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" , 998))); | |||
| 999 | } | |||
| 1000 | ||||
| 1001 | while ((*idp)->fti_probe != probe) { | |||
| 1002 | idp = &(*idp)->fti_next; | |||
| 1003 | ASSERT(*idp != NULL)((void)((*idp != ((void *)0)) || dtrace_assfail("*idp != NULL" , "/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" , 1003))); | |||
| 1004 | } | |||
| 1005 | ||||
| 1006 | id = *idp; | |||
| 1007 | *idp = id->fti_next; | |||
| 1008 | membar_producer(); | |||
| 1009 | ||||
| 1010 | ASSERT(id->fti_probe == probe)((void)((id->fti_probe == probe) || dtrace_assfail("id->fti_probe == probe" , "/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" , 1010))); | |||
| 1011 | ||||
| 1012 | /* | |||
| 1013 | * If there are other registered enablings of this tracepoint, we're | |||
| 1014 | * all done, but if this was the last probe assocated with this | |||
| 1015 | * this tracepoint, we need to remove and free it. | |||
| 1016 | */ | |||
| 1017 | if (tp->ftt_ids != NULL((void *)0) || tp->ftt_retids != NULL((void *)0)) { | |||
| 1018 | ||||
| 1019 | /* | |||
| 1020 | * If the current probe's tracepoint is in use, swap it | |||
| 1021 | * for an unused tracepoint. | |||
| 1022 | */ | |||
| 1023 | if (tp == probe->ftp_tps[index].fit_tp) { | |||
| 1024 | fasttrap_probe_t *tmp_probe; | |||
| 1025 | fasttrap_tracepoint_t **tmp_tp; | |||
| 1026 | uint_t tmp_index; | |||
| 1027 | ||||
| 1028 | if (tp->ftt_ids != NULL((void *)0)) { | |||
| 1029 | tmp_probe = tp->ftt_ids->fti_probe; | |||
| 1030 | /* LINTED - alignment */ | |||
| 1031 | tmp_index = FASTTRAP_ID_INDEX(tp->ftt_ids)((fasttrap_id_tp_t *)(((char *)(tp->ftt_ids) - __builtin_offsetof (fasttrap_id_tp_t, fit_id))) - &(tp->ftt_ids)->fti_probe ->ftp_tps[0]); | |||
| 1032 | tmp_tp = &tmp_probe->ftp_tps[tmp_index].fit_tp; | |||
| 1033 | } else { | |||
| 1034 | tmp_probe = tp->ftt_retids->fti_probe; | |||
| 1035 | /* LINTED - alignment */ | |||
| 1036 | tmp_index = FASTTRAP_ID_INDEX(tp->ftt_retids)((fasttrap_id_tp_t *)(((char *)(tp->ftt_retids) - __builtin_offsetof (fasttrap_id_tp_t, fit_id))) - &(tp->ftt_retids)->fti_probe ->ftp_tps[0]); | |||
| 1037 | tmp_tp = &tmp_probe->ftp_tps[tmp_index].fit_tp; | |||
| 1038 | } | |||
| 1039 | ||||
| 1040 | ASSERT(*tmp_tp != NULL)((void)((*tmp_tp != ((void *)0)) || dtrace_assfail("*tmp_tp != NULL" , "/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" , 1040))); | |||
| 1041 | ASSERT(*tmp_tp != probe->ftp_tps[index].fit_tp)((void)((*tmp_tp != probe->ftp_tps[index].fit_tp) || dtrace_assfail ("*tmp_tp != probe->ftp_tps[index].fit_tp", "/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" , 1041))); | |||
| 1042 | ASSERT((*tmp_tp)->ftt_ids == NULL)((void)(((*tmp_tp)->ftt_ids == ((void *)0)) || dtrace_assfail ("(*tmp_tp)->ftt_ids == NULL", "/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" , 1042))); | |||
| 1043 | ASSERT((*tmp_tp)->ftt_retids == NULL)((void)(((*tmp_tp)->ftt_retids == ((void *)0)) || dtrace_assfail ("(*tmp_tp)->ftt_retids == NULL", "/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" , 1043))); | |||
| 1044 | ||||
| 1045 | probe->ftp_tps[index].fit_tp = *tmp_tp; | |||
| 1046 | *tmp_tp = tp; | |||
| 1047 | } | |||
| 1048 | ||||
| 1049 | mutex_exit(&bucket->ftb_mtx)_sx_xunlock(((&bucket->ftb_mtx)), ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (1049)); | |||
| 1050 | ||||
| 1051 | /* | |||
| 1052 | * Tag the modified probe with the generation in which it was | |||
| 1053 | * changed. | |||
| 1054 | */ | |||
| 1055 | probe->ftp_gen = fasttrap_mod_gen; | |||
| 1056 | return; | |||
| 1057 | } | |||
| 1058 | ||||
| 1059 | mutex_exit(&bucket->ftb_mtx)_sx_xunlock(((&bucket->ftb_mtx)), ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (1059)); | |||
| 1060 | ||||
| 1061 | /* | |||
| 1062 | * We can't safely remove the tracepoint from the set of active | |||
| 1063 | * tracepoints until we've actually removed the fasttrap instruction | |||
| 1064 | * from the process's text. We can, however, operate on this | |||
| 1065 | * tracepoint secure in the knowledge that no other thread is going to | |||
| 1066 | * be looking at it since we hold P_PR_LOCK on the process if it's | |||
| 1067 | * live or we hold the provider lock on the process if it's dead and | |||
| 1068 | * gone. | |||
| 1069 | */ | |||
| 1070 | ||||
| 1071 | /* | |||
| 1072 | * We only need to remove the actual instruction if we're looking | |||
| 1073 | * at an existing process | |||
| 1074 | */ | |||
| 1075 | if (p != NULL((void *)0)) { | |||
| 1076 | /* | |||
| 1077 | * If we fail to restore the instruction we need to kill | |||
| 1078 | * this process since it's in a completely unrecoverable | |||
| 1079 | * state. | |||
| 1080 | */ | |||
| 1081 | if (fasttrap_tracepoint_remove(p, tp) != 0) | |||
| 1082 | fasttrap_sigtrap(p, NULL((void *)0), pc); | |||
| 1083 | ||||
| 1084 | /* | |||
| 1085 | * Decrement the count of the number of tracepoints active | |||
| 1086 | * in the victim process. | |||
| 1087 | */ | |||
| 1088 | #ifdef illumos | |||
| 1089 | ASSERT(p->p_proc_flag & P_PR_LOCK)((void)((p->p_proc_flag & P_PR_LOCK) || dtrace_assfail ("p->p_proc_flag & P_PR_LOCK", "/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" , 1089))); | |||
| 1090 | #endif | |||
| 1091 | p->p_dtrace_countp_dtrace->p_dtrace_count--; | |||
| 1092 | } | |||
| 1093 | ||||
| 1094 | /* | |||
| 1095 | * Remove the probe from the hash table of active tracepoints. | |||
| 1096 | */ | |||
| 1097 | mutex_enter(&bucket->ftb_mtx)(void)_sx_xlock(((&bucket->ftb_mtx)), 0, ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (1097)); | |||
| 1098 | pp = (fasttrap_tracepoint_t **)&bucket->ftb_data; | |||
| 1099 | ASSERT(*pp != NULL)((void)((*pp != ((void *)0)) || dtrace_assfail("*pp != NULL", "/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" , 1099))); | |||
| 1100 | while (*pp != tp) { | |||
| 1101 | pp = &(*pp)->ftt_next; | |||
| 1102 | ASSERT(*pp != NULL)((void)((*pp != ((void *)0)) || dtrace_assfail("*pp != NULL", "/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" , 1102))); | |||
| 1103 | } | |||
| 1104 | ||||
| 1105 | *pp = tp->ftt_next; | |||
| 1106 | membar_producer(); | |||
| 1107 | ||||
| 1108 | mutex_exit(&bucket->ftb_mtx)_sx_xunlock(((&bucket->ftb_mtx)), ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (1108)); | |||
| 1109 | ||||
| 1110 | /* | |||
| 1111 | * Tag the modified probe with the generation in which it was changed. | |||
| 1112 | */ | |||
| 1113 | probe->ftp_gen = fasttrap_mod_gen; | |||
| 1114 | } | |||
| 1115 | ||||
| 1116 | static void | |||
| 1117 | fasttrap_enable_callbacks(void) | |||
| 1118 | { | |||
| 1119 | /* | |||
| 1120 | * We don't have to play the rw lock game here because we're | |||
| 1121 | * providing something rather than taking something away -- | |||
| 1122 | * we can be sure that no threads have tried to follow this | |||
| 1123 | * function pointer yet. | |||
| 1124 | */ | |||
| 1125 | mutex_enter(&fasttrap_count_mtx)(void)_sx_xlock(((&fasttrap_count_mtx)), 0, ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (1125)); | |||
| 1126 | if (fasttrap_pid_count == 0) { | |||
| 1127 | ASSERT(dtrace_pid_probe_ptr == NULL)((void)((dtrace_pid_probe_ptr == ((void *)0)) || dtrace_assfail ("dtrace_pid_probe_ptr == NULL", "/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" , 1127))); | |||
| 1128 | ASSERT(dtrace_return_probe_ptr == NULL)((void)((dtrace_return_probe_ptr == ((void *)0)) || dtrace_assfail ("dtrace_return_probe_ptr == NULL", "/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" , 1128))); | |||
| 1129 | dtrace_pid_probe_ptr = &fasttrap_pid_probe; | |||
| 1130 | dtrace_return_probe_ptr = &fasttrap_return_probe; | |||
| 1131 | } | |||
| 1132 | ASSERT(dtrace_pid_probe_ptr == &fasttrap_pid_probe)((void)((dtrace_pid_probe_ptr == &fasttrap_pid_probe) || dtrace_assfail ("dtrace_pid_probe_ptr == &fasttrap_pid_probe", "/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" , 1132))); | |||
| 1133 | ASSERT(dtrace_return_probe_ptr == &fasttrap_return_probe)((void)((dtrace_return_probe_ptr == &fasttrap_return_probe ) || dtrace_assfail("dtrace_return_probe_ptr == &fasttrap_return_probe" , "/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" , 1133))); | |||
| 1134 | fasttrap_pid_count++; | |||
| 1135 | mutex_exit(&fasttrap_count_mtx)_sx_xunlock(((&fasttrap_count_mtx)), ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (1135)); | |||
| 1136 | } | |||
| 1137 | ||||
| 1138 | static void | |||
| 1139 | fasttrap_disable_callbacks(void) | |||
| 1140 | { | |||
| 1141 | #ifdef illumos | |||
| 1142 | ASSERT(MUTEX_HELD(&cpu_lock))((void)((((((&cpu_lock)->sx_lock & ~((0x01 | 0x02 | 0x04 | 0x10 | 0x08) & ~0x01)) == (uintptr_t)(__curthread ())))) || dtrace_assfail("MUTEX_HELD(&cpu_lock)", "/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" , 1142))); | |||
| 1143 | #endif | |||
| 1144 | ||||
| 1145 | ||||
| 1146 | mutex_enter(&fasttrap_count_mtx)(void)_sx_xlock(((&fasttrap_count_mtx)), 0, ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (1146)); | |||
| 1147 | ASSERT(fasttrap_pid_count > 0)((void)((fasttrap_pid_count > 0) || dtrace_assfail("fasttrap_pid_count > 0" , "/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" , 1147))); | |||
| 1148 | fasttrap_pid_count--; | |||
| 1149 | if (fasttrap_pid_count == 0) { | |||
| 1150 | #ifdef illumos | |||
| 1151 | cpu_tsolaris_cpu_t *cur, *cpu = CPU__extension__ ({ __typeof(((struct pcpu *)0)->pc_cpuid) __res ; struct __s { u_char __b[(((sizeof(__typeof(((struct pcpu *) 0)->pc_cpuid)))<(8))?(sizeof(__typeof(((struct pcpu *)0 )->pc_cpuid))):(8))]; } __s; if (sizeof(__res) == 1 || sizeof (__res) == 2 || sizeof(__res) == 4 || sizeof(__res) == 8) { __asm volatile("mov %%gs:%1,%0" : "=r" (__s) : "m" (*(struct __s * )(__builtin_offsetof(struct pcpu, pc_cpuid)))); *(struct __s * )(void *)&__res = __s; } else { __res = *__extension__ ({ __typeof(((struct pcpu *)0)->pc_cpuid) *__p; __asm volatile ("movq %%gs:%1,%0; addq %2,%0" : "=r" (__p) : "m" (*(struct pcpu *)(__builtin_offsetof(struct pcpu, pc_prvspace))), "i" (__builtin_offsetof (struct pcpu, pc_cpuid))); __p; }); } __res; }); | |||
| 1152 | ||||
| 1153 | for (cur = cpu->cpu_next_onln; cur != cpu; | |||
| 1154 | cur = cur->cpu_next_onln) { | |||
| 1155 | rw_enter(&cur->cpu_ft_lock, RW_WRITER); | |||
| 1156 | } | |||
| 1157 | #endif | |||
| 1158 | dtrace_pid_probe_ptr = NULL((void *)0); | |||
| 1159 | dtrace_return_probe_ptr = NULL((void *)0); | |||
| 1160 | #ifdef illumos | |||
| 1161 | for (cur = cpu->cpu_next_onln; cur != cpu; | |||
| 1162 | cur = cur->cpu_next_onln) { | |||
| 1163 | rw_exit(&cur->cpu_ft_lock); | |||
| 1164 | } | |||
| 1165 | #endif | |||
| 1166 | } | |||
| 1167 | mutex_exit(&fasttrap_count_mtx)_sx_xunlock(((&fasttrap_count_mtx)), ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (1167)); | |||
| 1168 | } | |||
| 1169 | ||||
| 1170 | /*ARGSUSED*/ | |||
| 1171 | static void | |||
| 1172 | fasttrap_pid_enable(void *arg, dtrace_id_t id, void *parg) | |||
| 1173 | { | |||
| 1174 | fasttrap_probe_t *probe = parg; | |||
| 1175 | proc_t *p = NULL((void *)0); | |||
| 1176 | int i, rc; | |||
| 1177 | ||||
| 1178 | ASSERT(probe != NULL)((void)((probe != ((void *)0)) || dtrace_assfail("probe != NULL" , "/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" , 1178))); | |||
| 1179 | ASSERT(!probe->ftp_enabled)((void)((!probe->ftp_enabled) || dtrace_assfail("!probe->ftp_enabled" , "/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" , 1179))); | |||
| 1180 | ASSERT(id == probe->ftp_id)((void)((id == probe->ftp_id) || dtrace_assfail("id == probe->ftp_id" , "/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" , 1180))); | |||
| 1181 | #ifdef illumos | |||
| 1182 | ASSERT(MUTEX_HELD(&cpu_lock))((void)((((((&cpu_lock)->sx_lock & ~((0x01 | 0x02 | 0x04 | 0x10 | 0x08) & ~0x01)) == (uintptr_t)(__curthread ())))) || dtrace_assfail("MUTEX_HELD(&cpu_lock)", "/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" , 1182))); | |||
| 1183 | #endif | |||
| 1184 | ||||
| 1185 | /* | |||
| 1186 | * Increment the count of enabled probes on this probe's provider; | |||
| 1187 | * the provider can't go away while the probe still exists. We | |||
| 1188 | * must increment this even if we aren't able to properly enable | |||
| 1189 | * this probe. | |||
| 1190 | */ | |||
| 1191 | mutex_enter(&probe->ftp_prov->ftp_mtx)(void)_sx_xlock(((&probe->ftp_prov->ftp_mtx)), 0, ( "/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (1191)); | |||
| 1192 | probe->ftp_prov->ftp_rcount++; | |||
| 1193 | mutex_exit(&probe->ftp_prov->ftp_mtx)_sx_xunlock(((&probe->ftp_prov->ftp_mtx)), ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (1193)); | |||
| 1194 | ||||
| 1195 | /* | |||
| 1196 | * If this probe's provider is retired (meaning it was valid in a | |||
| 1197 | * previously exec'ed incarnation of this address space), bail out. The | |||
| 1198 | * provider can't go away while we're in this code path. | |||
| 1199 | */ | |||
| 1200 | if (probe->ftp_prov->ftp_retired) | |||
| 1201 | return; | |||
| 1202 | ||||
| 1203 | /* | |||
| 1204 | * If we can't find the process, it may be that we're in the context of | |||
| 1205 | * a fork in which the traced process is being born and we're copying | |||
| 1206 | * USDT probes. Otherwise, the process is gone so bail. | |||
| 1207 | */ | |||
| 1208 | #ifdef illumos | |||
| 1209 | if ((p = sprlock(probe->ftp_pid)) == NULL((void *)0)) { | |||
| 1210 | if ((curproc((__curthread())->td_proc)->p_flag & SFORKING) == 0) | |||
| 1211 | return; | |||
| 1212 | ||||
| 1213 | mutex_enter(&pidlock)(void)_sx_xlock(((&pidlock)), 0, ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (1213)); | |||
| 1214 | p = prfind(probe->ftp_pid); | |||
| 1215 | ||||
| 1216 | if (p == NULL((void *)0)) { | |||
| 1217 | /* | |||
| 1218 | * So it's not that the target process is being born, | |||
| 1219 | * it's that it isn't there at all (and we simply | |||
| 1220 | * happen to be forking). Anyway, we know that the | |||
| 1221 | * target is definitely gone, so bail out. | |||
| 1222 | */ | |||
| 1223 | mutex_exit(&pidlock)_sx_xunlock(((&pidlock)), ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (1223)); | |||
| 1224 | return (0); | |||
| 1225 | } | |||
| 1226 | ||||
| 1227 | /* | |||
| 1228 | * Confirm that curproc is indeed forking the process in which | |||
| 1229 | * we're trying to enable probes. | |||
| 1230 | */ | |||
| 1231 | ASSERT(p->p_parent == curproc)((void)((p->p_pptr == ((__curthread())->td_proc)) || dtrace_assfail ("p->p_parent == curproc", "/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" , 1231))); | |||
| 1232 | ASSERT(p->p_stat == SIDL)((void)((p->p_stat == 1) || dtrace_assfail("p->p_stat == SIDL" , "/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" , 1232))); | |||
| 1233 | ||||
| 1234 | mutex_enter(&p->p_lock)(void)_sx_xlock(((&p->p_lock)), 0, ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (1234)); | |||
| 1235 | mutex_exit(&pidlock)_sx_xunlock(((&pidlock)), ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (1235)); | |||
| 1236 | ||||
| 1237 | sprlock_proc(p); | |||
| 1238 | } | |||
| 1239 | ||||
| 1240 | ASSERT(!(p->p_flag & SVFORK))((void)((!(p->p_flag & SVFORK)) || dtrace_assfail("!(p->p_flag & SVFORK)" , "/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" , 1240))); | |||
| 1241 | mutex_exit(&p->p_lock)_sx_xunlock(((&p->p_lock)), ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (1241)); | |||
| 1242 | #else | |||
| 1243 | if (pget(probe->ftp_pid, PGET_HOLD0x00001 | PGET_NOTWEXIT0x00010, &p) != 0) | |||
| 1244 | return; | |||
| 1245 | #endif | |||
| 1246 | ||||
| 1247 | /* | |||
| 1248 | * We have to enable the trap entry point before any user threads have | |||
| 1249 | * the chance to execute the trap instruction we're about to place | |||
| 1250 | * in their process's text. | |||
| 1251 | */ | |||
| 1252 | fasttrap_enable_callbacks(); | |||
| 1253 | ||||
| 1254 | /* | |||
| 1255 | * Enable all the tracepoints and add this probe's id to each | |||
| 1256 | * tracepoint's list of active probes. | |||
| 1257 | */ | |||
| 1258 | for (i = 0; i < probe->ftp_ntps; i++) { | |||
| 1259 | if ((rc = fasttrap_tracepoint_enable(p, probe, i)) != 0) { | |||
| 1260 | /* | |||
| 1261 | * If enabling the tracepoint failed completely, | |||
| 1262 | * we don't have to disable it; if the failure | |||
| 1263 | * was only partial we must disable it. | |||
| 1264 | */ | |||
| 1265 | if (rc == FASTTRAP_ENABLE_FAIL1) | |||
| 1266 | i--; | |||
| 1267 | else | |||
| 1268 | ASSERT(rc == FASTTRAP_ENABLE_PARTIAL)((void)((rc == 2) || dtrace_assfail("rc == FASTTRAP_ENABLE_PARTIAL" , "/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" , 1268))); | |||
| 1269 | ||||
| 1270 | /* | |||
| 1271 | * Back up and pull out all the tracepoints we've | |||
| 1272 | * created so far for this probe. | |||
| 1273 | */ | |||
| 1274 | while (i >= 0) { | |||
| 1275 | fasttrap_tracepoint_disable(p, probe, i); | |||
| 1276 | i--; | |||
| 1277 | } | |||
| 1278 | ||||
| 1279 | #ifdef illumos | |||
| 1280 | mutex_enter(&p->p_lock)(void)_sx_xlock(((&p->p_lock)), 0, ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (1280)); | |||
| 1281 | sprunlock(p); | |||
| 1282 | #else | |||
| 1283 | PRELE(p)do { __mtx_lock_flags(&((((&((p))->p_mtx))))->mtx_lock , ((0)), ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (1283)); do { __mtx_assert(&(((&(((p)))->p_mtx) ))->mtx_lock, (((0x00000004))), ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (1283)); do { do { if (__builtin_expect((!(((p))->p_lock > 0)), 0)) panic ("process %p not held", (p)); } while (0 ); } while (0); (--((p))->p_lock); if ((((p))->p_flag & 0x02000) && ((p))->p_lock == 0) wakeup(&((p)) ->p_lock); } while (0); __mtx_unlock_flags(&((((&( (p))->p_mtx))))->mtx_lock, ((0)), ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (1283)); } while (0); | |||
| 1284 | #endif | |||
| 1285 | ||||
| 1286 | /* | |||
| 1287 | * Since we're not actually enabling this probe, | |||
| 1288 | * drop our reference on the trap table entry. | |||
| 1289 | */ | |||
| 1290 | fasttrap_disable_callbacks(); | |||
| 1291 | return; | |||
| 1292 | } | |||
| 1293 | } | |||
| 1294 | #ifdef illumos | |||
| 1295 | mutex_enter(&p->p_lock)(void)_sx_xlock(((&p->p_lock)), 0, ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (1295)); | |||
| 1296 | sprunlock(p); | |||
| 1297 | #else | |||
| 1298 | PRELE(p)do { __mtx_lock_flags(&((((&((p))->p_mtx))))->mtx_lock , ((0)), ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (1298)); do { __mtx_assert(&(((&(((p)))->p_mtx) ))->mtx_lock, (((0x00000004))), ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (1298)); do { do { if (__builtin_expect((!(((p))->p_lock > 0)), 0)) panic ("process %p not held", (p)); } while (0 ); } while (0); (--((p))->p_lock); if ((((p))->p_flag & 0x02000) && ((p))->p_lock == 0) wakeup(&((p)) ->p_lock); } while (0); __mtx_unlock_flags(&((((&( (p))->p_mtx))))->mtx_lock, ((0)), ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (1298)); } while (0); | |||
| 1299 | #endif | |||
| 1300 | ||||
| 1301 | probe->ftp_enabled = 1; | |||
| 1302 | } | |||
| 1303 | ||||
| 1304 | /*ARGSUSED*/ | |||
| 1305 | static void | |||
| 1306 | fasttrap_pid_disable(void *arg, dtrace_id_t id, void *parg) | |||
| 1307 | { | |||
| 1308 | fasttrap_probe_t *probe = parg; | |||
| 1309 | fasttrap_provider_t *provider = probe->ftp_prov; | |||
| 1310 | proc_t *p; | |||
| 1311 | int i, whack = 0; | |||
| 1312 | ||||
| 1313 | ASSERT(id == probe->ftp_id)((void)((id == probe->ftp_id) || dtrace_assfail("id == probe->ftp_id" , "/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" , 1313))); | |||
| 1314 | ||||
| 1315 | mutex_enter(&provider->ftp_mtx)(void)_sx_xlock(((&provider->ftp_mtx)), 0, ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (1315)); | |||
| 1316 | ||||
| 1317 | /* | |||
| 1318 | * We won't be able to acquire a /proc-esque lock on the process | |||
| 1319 | * iff the process is dead and gone. In this case, we rely on the | |||
| 1320 | * provider lock as a point of mutual exclusion to prevent other | |||
| 1321 | * DTrace consumers from disabling this probe. | |||
| 1322 | */ | |||
| 1323 | if (pget(probe->ftp_pid, PGET_HOLD0x00001 | PGET_NOTWEXIT0x00010, &p) != 0) | |||
| 1324 | p = NULL((void *)0); | |||
| 1325 | ||||
| 1326 | /* | |||
| 1327 | * Disable all the associated tracepoints (for fully enabled probes). | |||
| 1328 | */ | |||
| 1329 | if (probe->ftp_enabled) { | |||
| 1330 | for (i = 0; i < probe->ftp_ntps; i++) { | |||
| 1331 | fasttrap_tracepoint_disable(p, probe, i); | |||
| 1332 | } | |||
| 1333 | } | |||
| 1334 | ||||
| 1335 | ASSERT(provider->ftp_rcount > 0)((void)((provider->ftp_rcount > 0) || dtrace_assfail("provider->ftp_rcount > 0" , "/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" , 1335))); | |||
| 1336 | provider->ftp_rcount--; | |||
| 1337 | ||||
| 1338 | if (p != NULL((void *)0)) { | |||
| 1339 | /* | |||
| 1340 | * Even though we may not be able to remove it entirely, we | |||
| 1341 | * mark this retired provider to get a chance to remove some | |||
| 1342 | * of the associated probes. | |||
| 1343 | */ | |||
| 1344 | if (provider->ftp_retired && !provider->ftp_marked) | |||
| 1345 | whack = provider->ftp_marked = 1; | |||
| 1346 | mutex_exit(&provider->ftp_mtx)_sx_xunlock(((&provider->ftp_mtx)), ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (1346)); | |||
| 1347 | } else { | |||
| 1348 | /* | |||
| 1349 | * If the process is dead, we're just waiting for the | |||
| 1350 | * last probe to be disabled to be able to free it. | |||
| 1351 | */ | |||
| 1352 | if (provider->ftp_rcount == 0 && !provider->ftp_marked) | |||
| 1353 | whack = provider->ftp_marked = 1; | |||
| 1354 | mutex_exit(&provider->ftp_mtx)_sx_xunlock(((&provider->ftp_mtx)), ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (1354)); | |||
| 1355 | } | |||
| 1356 | ||||
| 1357 | if (whack) | |||
| 1358 | fasttrap_pid_cleanup(); | |||
| 1359 | ||||
| 1360 | #ifdef __FreeBSD__11 | |||
| 1361 | if (p != NULL((void *)0)) | |||
| 1362 | PRELE(p)do { __mtx_lock_flags(&((((&((p))->p_mtx))))->mtx_lock , ((0)), ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (1362)); do { __mtx_assert(&(((&(((p)))->p_mtx) ))->mtx_lock, (((0x00000004))), ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (1362)); do { do { if (__builtin_expect((!(((p))->p_lock > 0)), 0)) panic ("process %p not held", (p)); } while (0 ); } while (0); (--((p))->p_lock); if ((((p))->p_flag & 0x02000) && ((p))->p_lock == 0) wakeup(&((p)) ->p_lock); } while (0); __mtx_unlock_flags(&((((&( (p))->p_mtx))))->mtx_lock, ((0)), ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (1362)); } while (0); | |||
| 1363 | #endif | |||
| 1364 | if (!probe->ftp_enabled) | |||
| 1365 | return; | |||
| 1366 | ||||
| 1367 | probe->ftp_enabled = 0; | |||
| 1368 | ||||
| 1369 | #ifdef illumos | |||
| 1370 | ASSERT(MUTEX_HELD(&cpu_lock))((void)((((((&cpu_lock)->sx_lock & ~((0x01 | 0x02 | 0x04 | 0x10 | 0x08) & ~0x01)) == (uintptr_t)(__curthread ())))) || dtrace_assfail("MUTEX_HELD(&cpu_lock)", "/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" , 1370))); | |||
| 1371 | #endif | |||
| 1372 | fasttrap_disable_callbacks(); | |||
| 1373 | } | |||
| 1374 | ||||
| 1375 | /*ARGSUSED*/ | |||
| 1376 | static void | |||
| 1377 | fasttrap_pid_getargdesc(void *arg, dtrace_id_t id, void *parg, | |||
| 1378 | dtrace_argdesc_t *desc) | |||
| 1379 | { | |||
| 1380 | fasttrap_probe_t *probe = parg; | |||
| 1381 | char *str; | |||
| 1382 | int i, ndx; | |||
| 1383 | ||||
| 1384 | desc->dtargd_native[0] = '\0'; | |||
| 1385 | desc->dtargd_xlate[0] = '\0'; | |||
| 1386 | ||||
| 1387 | if (probe->ftp_prov->ftp_retired != 0 || | |||
| 1388 | desc->dtargd_ndx >= probe->ftp_nargs) { | |||
| 1389 | desc->dtargd_ndx = DTRACE_ARGNONE-1; | |||
| 1390 | return; | |||
| 1391 | } | |||
| 1392 | ||||
| 1393 | ndx = (probe->ftp_argmap != NULL((void *)0)) ? | |||
| 1394 | probe->ftp_argmap[desc->dtargd_ndx] : desc->dtargd_ndx; | |||
| 1395 | ||||
| 1396 | str = probe->ftp_ntypes; | |||
| 1397 | for (i = 0; i < ndx; i++) { | |||
| 1398 | str += strlen(str) + 1; | |||
| 1399 | } | |||
| 1400 | ||||
| 1401 | ASSERT(strlen(str + 1) < sizeof (desc->dtargd_native))((void)((strlen(str + 1) < sizeof (desc->dtargd_native) ) || dtrace_assfail("strlen(str + 1) < sizeof (desc->dtargd_native)" , "/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" , 1401))); | |||
| 1402 | (void) strcpy(desc->dtargd_native, str); | |||
| 1403 | ||||
| 1404 | if (probe->ftp_xtypes == NULL((void *)0)) | |||
| 1405 | return; | |||
| 1406 | ||||
| 1407 | str = probe->ftp_xtypes; | |||
| 1408 | for (i = 0; i < desc->dtargd_ndx; i++) { | |||
| 1409 | str += strlen(str) + 1; | |||
| 1410 | } | |||
| 1411 | ||||
| 1412 | ASSERT(strlen(str + 1) < sizeof (desc->dtargd_xlate))((void)((strlen(str + 1) < sizeof (desc->dtargd_xlate)) || dtrace_assfail("strlen(str + 1) < sizeof (desc->dtargd_xlate)" , "/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" , 1412))); | |||
| 1413 | (void) strcpy(desc->dtargd_xlate, str); | |||
| 1414 | } | |||
| 1415 | ||||
| 1416 | /*ARGSUSED*/ | |||
| 1417 | static void | |||
| 1418 | fasttrap_pid_destroy(void *arg, dtrace_id_t id, void *parg) | |||
| 1419 | { | |||
| 1420 | fasttrap_probe_t *probe = parg; | |||
| 1421 | int i; | |||
| 1422 | size_t size; | |||
| 1423 | ||||
| 1424 | ASSERT(probe != NULL)((void)((probe != ((void *)0)) || dtrace_assfail("probe != NULL" , "/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" , 1424))); | |||
| 1425 | ASSERT(!probe->ftp_enabled)((void)((!probe->ftp_enabled) || dtrace_assfail("!probe->ftp_enabled" , "/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" , 1425))); | |||
| 1426 | ASSERT(fasttrap_total >= probe->ftp_ntps)((void)((fasttrap_total >= probe->ftp_ntps) || dtrace_assfail ("fasttrap_total >= probe->ftp_ntps", "/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" , 1426))); | |||
| 1427 | ||||
| 1428 | atomic_add_32atomic_add_int(&fasttrap_total, -probe->ftp_ntps); | |||
| 1429 | size = offsetof(fasttrap_probe_t, ftp_tps[probe->ftp_ntps])__builtin_offsetof(fasttrap_probe_t, ftp_tps[probe->ftp_ntps ]); | |||
| 1430 | ||||
| 1431 | if (probe->ftp_gen + 1 >= fasttrap_mod_gen) | |||
| 1432 | fasttrap_mod_barrier(probe->ftp_gen); | |||
| 1433 | ||||
| 1434 | for (i = 0; i < probe->ftp_ntps; i++) { | |||
| 1435 | kmem_free(probe->ftp_tps[i].fit_tp,zfs_kmem_free((probe->ftp_tps[i].fit_tp), (sizeof (fasttrap_tracepoint_t ))) | |||
| 1436 | sizeof (fasttrap_tracepoint_t))zfs_kmem_free((probe->ftp_tps[i].fit_tp), (sizeof (fasttrap_tracepoint_t ))); | |||
| 1437 | } | |||
| 1438 | ||||
| 1439 | kmem_free(probe, size)zfs_kmem_free((probe), (size)); | |||
| 1440 | } | |||
| 1441 | ||||
| 1442 | ||||
| 1443 | static const dtrace_pattr_t pid_attr = { | |||
| 1444 | { DTRACE_STABILITY_EVOLVING5, DTRACE_STABILITY_EVOLVING5, DTRACE_CLASS_ISA4 }, | |||
| 1445 | { DTRACE_STABILITY_PRIVATE1, DTRACE_STABILITY_PRIVATE1, DTRACE_CLASS_UNKNOWN0 }, | |||
| 1446 | { DTRACE_STABILITY_PRIVATE1, DTRACE_STABILITY_PRIVATE1, DTRACE_CLASS_UNKNOWN0 }, | |||
| 1447 | { DTRACE_STABILITY_EVOLVING5, DTRACE_STABILITY_EVOLVING5, DTRACE_CLASS_ISA4 }, | |||
| 1448 | { DTRACE_STABILITY_PRIVATE1, DTRACE_STABILITY_PRIVATE1, DTRACE_CLASS_UNKNOWN0 }, | |||
| 1449 | }; | |||
| 1450 | ||||
| 1451 | static dtrace_pops_t pid_pops = { | |||
| 1452 | .dtps_provide = fasttrap_pid_provide, | |||
| 1453 | .dtps_provide_module = NULL((void *)0), | |||
| 1454 | .dtps_enable = fasttrap_pid_enable, | |||
| 1455 | .dtps_disable = fasttrap_pid_disable, | |||
| 1456 | .dtps_suspend = NULL((void *)0), | |||
| 1457 | .dtps_resume = NULL((void *)0), | |||
| 1458 | .dtps_getargdesc = fasttrap_pid_getargdesc, | |||
| 1459 | .dtps_getargval = fasttrap_pid_getarg, | |||
| 1460 | .dtps_usermode = NULL((void *)0), | |||
| 1461 | .dtps_destroy = fasttrap_pid_destroy | |||
| 1462 | }; | |||
| 1463 | ||||
| 1464 | static dtrace_pops_t usdt_pops = { | |||
| 1465 | .dtps_provide = fasttrap_pid_provide, | |||
| 1466 | .dtps_provide_module = NULL((void *)0), | |||
| 1467 | .dtps_enable = fasttrap_pid_enable, | |||
| 1468 | .dtps_disable = fasttrap_pid_disable, | |||
| 1469 | .dtps_suspend = NULL((void *)0), | |||
| 1470 | .dtps_resume = NULL((void *)0), | |||
| 1471 | .dtps_getargdesc = fasttrap_pid_getargdesc, | |||
| 1472 | .dtps_getargval = fasttrap_usdt_getarg, | |||
| 1473 | .dtps_usermode = NULL((void *)0), | |||
| 1474 | .dtps_destroy = fasttrap_pid_destroy | |||
| 1475 | }; | |||
| 1476 | ||||
| 1477 | static fasttrap_proc_t * | |||
| 1478 | fasttrap_proc_lookup(pid_t pid) | |||
| 1479 | { | |||
| 1480 | fasttrap_bucket_t *bucket; | |||
| 1481 | fasttrap_proc_t *fprc, *new_fprc; | |||
| 1482 | ||||
| 1483 | ||||
| 1484 | bucket = &fasttrap_procs.fth_table[FASTTRAP_PROCS_INDEX(pid)((pid) & fasttrap_procs.fth_mask)]; | |||
| 1485 | mutex_enter(&bucket->ftb_mtx)(void)_sx_xlock(((&bucket->ftb_mtx)), 0, ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (1485)); | |||
| 1486 | ||||
| 1487 | for (fprc = bucket->ftb_data; fprc != NULL((void *)0); fprc = fprc->ftpc_next) { | |||
| 1488 | if (fprc->ftpc_pid == pid && fprc->ftpc_acount != 0) { | |||
| 1489 | mutex_enter(&fprc->ftpc_mtx)(void)_sx_xlock(((&fprc->ftpc_mtx)), 0, ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (1489)); | |||
| 1490 | mutex_exit(&bucket->ftb_mtx)_sx_xunlock(((&bucket->ftb_mtx)), ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (1490)); | |||
| 1491 | fprc->ftpc_rcount++; | |||
| 1492 | atomic_inc_64(&fprc->ftpc_acount); | |||
| 1493 | ASSERT(fprc->ftpc_acount <= fprc->ftpc_rcount)((void)((fprc->ftpc_acount <= fprc->ftpc_rcount) || dtrace_assfail ("fprc->ftpc_acount <= fprc->ftpc_rcount", "/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" , 1493))); | |||
| 1494 | mutex_exit(&fprc->ftpc_mtx)_sx_xunlock(((&fprc->ftpc_mtx)), ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (1494)); | |||
| 1495 | ||||
| 1496 | return (fprc); | |||
| 1497 | } | |||
| 1498 | } | |||
| 1499 | ||||
| 1500 | /* | |||
| 1501 | * Drop the bucket lock so we don't try to perform a sleeping | |||
| 1502 | * allocation under it. | |||
| 1503 | */ | |||
| 1504 | mutex_exit(&bucket->ftb_mtx)_sx_xunlock(((&bucket->ftb_mtx)), ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (1504)); | |||
| 1505 | ||||
| 1506 | new_fprc = kmem_zalloc(sizeof (fasttrap_proc_t), KM_SLEEP)zfs_kmem_alloc((sizeof (fasttrap_proc_t)), (0x0002) | 0x0100); | |||
| 1507 | new_fprc->ftpc_pid = pid; | |||
| 1508 | new_fprc->ftpc_rcount = 1; | |||
| 1509 | new_fprc->ftpc_acount = 1; | |||
| 1510 | #ifndef illumos | |||
| 1511 | mutex_init(&new_fprc->ftpc_mtx, "fasttrap proc mtx", MUTEX_DEFAULT,do { const char *_name; ((void)(((MUTEX_DEFAULT) == 0 || (MUTEX_DEFAULT ) == MUTEX_DEFAULT) || dtrace_assfail("(MUTEX_DEFAULT) == 0 || (MUTEX_DEFAULT) == MUTEX_DEFAULT" , "/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" , 1512))); do { if (__builtin_expect((!(((&new_fprc->ftpc_mtx )->lock_object.lo_flags & (0x00010000 | 0x00020000 | 0x00040000 | 0x00080000 | 0x00100000 | 0x00200000 | 0x00400000 | 0x0f000000 | 0x10000000)) != (0x00010000 | 0x00020000 | 0x00080000 | 0x00100000 | 0x00200000 | 0x00400000 | (2 << 24)))), 0)) panic ("lock %s already initialized" , "&new_fprc->ftpc_mtx"); } while (0); for (_name = "&new_fprc->ftpc_mtx" ; *_name != '\0'; _name++) { if (*_name >= 'a' && * _name <= 'z') break; } if (*_name == '\0') _name = "&new_fprc->ftpc_mtx" ; sx_init_flags((&new_fprc->ftpc_mtx), _name, (0x01 | 0x40 | 0x04)); } while (0) | |||
| 1512 | NULL)do { const char *_name; ((void)(((MUTEX_DEFAULT) == 0 || (MUTEX_DEFAULT ) == MUTEX_DEFAULT) || dtrace_assfail("(MUTEX_DEFAULT) == 0 || (MUTEX_DEFAULT) == MUTEX_DEFAULT" , "/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" , 1512))); do { if (__builtin_expect((!(((&new_fprc->ftpc_mtx )->lock_object.lo_flags & (0x00010000 | 0x00020000 | 0x00040000 | 0x00080000 | 0x00100000 | 0x00200000 | 0x00400000 | 0x0f000000 | 0x10000000)) != (0x00010000 | 0x00020000 | 0x00080000 | 0x00100000 | 0x00200000 | 0x00400000 | (2 << 24)))), 0)) panic ("lock %s already initialized" , "&new_fprc->ftpc_mtx"); } while (0); for (_name = "&new_fprc->ftpc_mtx" ; *_name != '\0'; _name++) { if (*_name >= 'a' && * _name <= 'z') break; } if (*_name == '\0') _name = "&new_fprc->ftpc_mtx" ; sx_init_flags((&new_fprc->ftpc_mtx), _name, (0x01 | 0x40 | 0x04)); } while (0); | |||
| 1513 | #endif | |||
| 1514 | ||||
| 1515 | mutex_enter(&bucket->ftb_mtx)(void)_sx_xlock(((&bucket->ftb_mtx)), 0, ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (1515)); | |||
| 1516 | ||||
| 1517 | /* | |||
| 1518 | * Take another lap through the list to make sure a proc hasn't | |||
| 1519 | * been created for this pid while we weren't under the bucket lock. | |||
| 1520 | */ | |||
| 1521 | for (fprc = bucket->ftb_data; fprc != NULL((void *)0); fprc = fprc->ftpc_next) { | |||
| 1522 | if (fprc->ftpc_pid == pid && fprc->ftpc_acount != 0) { | |||
| 1523 | mutex_enter(&fprc->ftpc_mtx)(void)_sx_xlock(((&fprc->ftpc_mtx)), 0, ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (1523)); | |||
| 1524 | mutex_exit(&bucket->ftb_mtx)_sx_xunlock(((&bucket->ftb_mtx)), ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (1524)); | |||
| 1525 | fprc->ftpc_rcount++; | |||
| 1526 | atomic_inc_64(&fprc->ftpc_acount); | |||
| 1527 | ASSERT(fprc->ftpc_acount <= fprc->ftpc_rcount)((void)((fprc->ftpc_acount <= fprc->ftpc_rcount) || dtrace_assfail ("fprc->ftpc_acount <= fprc->ftpc_rcount", "/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" , 1527))); | |||
| 1528 | mutex_exit(&fprc->ftpc_mtx)_sx_xunlock(((&fprc->ftpc_mtx)), ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (1528)); | |||
| 1529 | ||||
| 1530 | kmem_free(new_fprc, sizeof (fasttrap_proc_t))zfs_kmem_free((new_fprc), (sizeof (fasttrap_proc_t))); | |||
| 1531 | ||||
| 1532 | return (fprc); | |||
| 1533 | } | |||
| 1534 | } | |||
| 1535 | ||||
| 1536 | new_fprc->ftpc_next = bucket->ftb_data; | |||
| 1537 | bucket->ftb_data = new_fprc; | |||
| 1538 | ||||
| 1539 | mutex_exit(&bucket->ftb_mtx)_sx_xunlock(((&bucket->ftb_mtx)), ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (1539)); | |||
| 1540 | ||||
| 1541 | return (new_fprc); | |||
| 1542 | } | |||
| 1543 | ||||
| 1544 | static void | |||
| 1545 | fasttrap_proc_release(fasttrap_proc_t *proc) | |||
| 1546 | { | |||
| 1547 | fasttrap_bucket_t *bucket; | |||
| 1548 | fasttrap_proc_t *fprc, **fprcp; | |||
| 1549 | pid_t pid = proc->ftpc_pid; | |||
| 1550 | #ifndef illumos | |||
| 1551 | fasttrap_scrblock_t *scrblk, *scrblktmp; | |||
| 1552 | fasttrap_scrspace_t *scrspc, *scrspctmp; | |||
| 1553 | struct proc *p; | |||
| 1554 | struct thread *td; | |||
| 1555 | #endif | |||
| 1556 | ||||
| 1557 | mutex_enter(&proc->ftpc_mtx)(void)_sx_xlock(((&proc->ftpc_mtx)), 0, ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (1557)); | |||
| 1558 | ||||
| 1559 | ASSERT(proc->ftpc_rcount != 0)((void)((proc->ftpc_rcount != 0) || dtrace_assfail("proc->ftpc_rcount != 0" , "/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" , 1559))); | |||
| 1560 | ASSERT(proc->ftpc_acount <= proc->ftpc_rcount)((void)((proc->ftpc_acount <= proc->ftpc_rcount) || dtrace_assfail ("proc->ftpc_acount <= proc->ftpc_rcount", "/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" , 1560))); | |||
| 1561 | ||||
| 1562 | if (--proc->ftpc_rcount != 0) { | |||
| 1563 | mutex_exit(&proc->ftpc_mtx)_sx_xunlock(((&proc->ftpc_mtx)), ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (1563)); | |||
| 1564 | return; | |||
| 1565 | } | |||
| 1566 | ||||
| 1567 | #ifndef illumos | |||
| 1568 | /* | |||
| 1569 | * Free all structures used to manage per-thread scratch space. | |||
| 1570 | */ | |||
| 1571 | LIST_FOREACH_SAFE(scrblk, &proc->ftpc_scrblks, ftsb_next,for ((scrblk) = (((&proc->ftpc_scrblks))->lh_first) ; (scrblk) && ((scrblktmp) = (((scrblk))->ftsb_next .le_next), 1); (scrblk) = (scrblktmp)) | |||
| 1572 | scrblktmp)for ((scrblk) = (((&proc->ftpc_scrblks))->lh_first) ; (scrblk) && ((scrblktmp) = (((scrblk))->ftsb_next .le_next), 1); (scrblk) = (scrblktmp)) { | |||
| 1573 | LIST_REMOVE(scrblk, ftsb_next)do { ; ; do { if ((((scrblk))->ftsb_next.le_next) != ((void *)0) && (((scrblk))->ftsb_next.le_next)->ftsb_next .le_prev != &((scrblk)->ftsb_next.le_next)) panic("Bad link elm %p next->prev != elm" , (scrblk)); } while (0); do { if (*(scrblk)->ftsb_next.le_prev != (scrblk)) panic("Bad link elm %p prev->next != elm", ( scrblk)); } while (0); if ((((scrblk))->ftsb_next.le_next) != ((void *)0)) (((scrblk))->ftsb_next.le_next)->ftsb_next .le_prev = (scrblk)->ftsb_next.le_prev; *(scrblk)->ftsb_next .le_prev = (((scrblk))->ftsb_next.le_next); ; ; } while (0 ); | |||
| 1574 | free(scrblk, M_SOLARIS); | |||
| 1575 | } | |||
| 1576 | LIST_FOREACH_SAFE(scrspc, &proc->ftpc_fscr, ftss_next, scrspctmp)for ((scrspc) = (((&proc->ftpc_fscr))->lh_first); ( scrspc) && ((scrspctmp) = (((scrspc))->ftss_next.le_next ), 1); (scrspc) = (scrspctmp)) { | |||
| 1577 | LIST_REMOVE(scrspc, ftss_next)do { ; ; do { if ((((scrspc))->ftss_next.le_next) != ((void *)0) && (((scrspc))->ftss_next.le_next)->ftss_next .le_prev != &((scrspc)->ftss_next.le_next)) panic("Bad link elm %p next->prev != elm" , (scrspc)); } while (0); do { if (*(scrspc)->ftss_next.le_prev != (scrspc)) panic("Bad link elm %p prev->next != elm", ( scrspc)); } while (0); if ((((scrspc))->ftss_next.le_next) != ((void *)0)) (((scrspc))->ftss_next.le_next)->ftss_next .le_prev = (scrspc)->ftss_next.le_prev; *(scrspc)->ftss_next .le_prev = (((scrspc))->ftss_next.le_next); ; ; } while (0 ); | |||
| 1578 | free(scrspc, M_SOLARIS); | |||
| 1579 | } | |||
| 1580 | LIST_FOREACH_SAFE(scrspc, &proc->ftpc_ascr, ftss_next, scrspctmp)for ((scrspc) = (((&proc->ftpc_ascr))->lh_first); ( scrspc) && ((scrspctmp) = (((scrspc))->ftss_next.le_next ), 1); (scrspc) = (scrspctmp)) { | |||
| 1581 | LIST_REMOVE(scrspc, ftss_next)do { ; ; do { if ((((scrspc))->ftss_next.le_next) != ((void *)0) && (((scrspc))->ftss_next.le_next)->ftss_next .le_prev != &((scrspc)->ftss_next.le_next)) panic("Bad link elm %p next->prev != elm" , (scrspc)); } while (0); do { if (*(scrspc)->ftss_next.le_prev != (scrspc)) panic("Bad link elm %p prev->next != elm", ( scrspc)); } while (0); if ((((scrspc))->ftss_next.le_next) != ((void *)0)) (((scrspc))->ftss_next.le_next)->ftss_next .le_prev = (scrspc)->ftss_next.le_prev; *(scrspc)->ftss_next .le_prev = (((scrspc))->ftss_next.le_next); ; ; } while (0 ); | |||
| 1582 | free(scrspc, M_SOLARIS); | |||
| 1583 | } | |||
| 1584 | ||||
| 1585 | if ((p = pfind(pid)) != NULL((void *)0)) { | |||
| 1586 | FOREACH_THREAD_IN_PROC(p, td)for (((td)) = (((&(p)->p_threads))->tqh_first); ((td )); ((td)) = ((((td)))->td_plist.tqe_next)) | |||
| 1587 | td->t_dtrace_sscrtd_dtrace->td_dtrace_sscr = NULL((void *)0); | |||
| 1588 | PROC_UNLOCK(p)__mtx_unlock_flags(&((((&(p)->p_mtx))))->mtx_lock , ((0)), ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (1588)); | |||
| 1589 | } | |||
| 1590 | #endif | |||
| 1591 | ||||
| 1592 | mutex_exit(&proc->ftpc_mtx)_sx_xunlock(((&proc->ftpc_mtx)), ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (1592)); | |||
| 1593 | ||||
| 1594 | /* | |||
| 1595 | * There should definitely be no live providers associated with this | |||
| 1596 | * process at this point. | |||
| 1597 | */ | |||
| 1598 | ASSERT(proc->ftpc_acount == 0)((void)((proc->ftpc_acount == 0) || dtrace_assfail("proc->ftpc_acount == 0" , "/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" , 1598))); | |||
| 1599 | ||||
| 1600 | bucket = &fasttrap_procs.fth_table[FASTTRAP_PROCS_INDEX(pid)((pid) & fasttrap_procs.fth_mask)]; | |||
| 1601 | mutex_enter(&bucket->ftb_mtx)(void)_sx_xlock(((&bucket->ftb_mtx)), 0, ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (1601)); | |||
| 1602 | ||||
| 1603 | fprcp = (fasttrap_proc_t **)&bucket->ftb_data; | |||
| 1604 | while ((fprc = *fprcp) != NULL((void *)0)) { | |||
| 1605 | if (fprc == proc) | |||
| 1606 | break; | |||
| 1607 | ||||
| 1608 | fprcp = &fprc->ftpc_next; | |||
| 1609 | } | |||
| 1610 | ||||
| 1611 | /* | |||
| 1612 | * Something strange has happened if we can't find the proc. | |||
| 1613 | */ | |||
| 1614 | ASSERT(fprc != NULL)((void)((fprc != ((void *)0)) || dtrace_assfail("fprc != NULL" , "/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" , 1614))); | |||
| 1615 | ||||
| 1616 | *fprcp = fprc->ftpc_next; | |||
| 1617 | ||||
| 1618 | mutex_exit(&bucket->ftb_mtx)_sx_xunlock(((&bucket->ftb_mtx)), ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (1618)); | |||
| 1619 | ||||
| 1620 | kmem_free(fprc, sizeof (fasttrap_proc_t))zfs_kmem_free((fprc), (sizeof (fasttrap_proc_t))); | |||
| 1621 | } | |||
| 1622 | ||||
| 1623 | /* | |||
| 1624 | * Lookup a fasttrap-managed provider based on its name and associated pid. | |||
| 1625 | * If the pattr argument is non-NULL, this function instantiates the provider | |||
| 1626 | * if it doesn't exist otherwise it returns NULL. The provider is returned | |||
| 1627 | * with its lock held. | |||
| 1628 | */ | |||
| 1629 | static fasttrap_provider_t * | |||
| 1630 | fasttrap_provider_lookup(pid_t pid, const char *name, | |||
| 1631 | const dtrace_pattr_t *pattr) | |||
| 1632 | { | |||
| 1633 | fasttrap_provider_t *fp, *new_fp = NULL((void *)0); | |||
| 1634 | fasttrap_bucket_t *bucket; | |||
| 1635 | char provname[DTRACE_PROVNAMELEN64]; | |||
| 1636 | proc_t *p; | |||
| 1637 | cred_t *cred; | |||
| 1638 | ||||
| 1639 | ASSERT(strlen(name) < sizeof (fp->ftp_name))((void)((strlen(name) < sizeof (fp->ftp_name)) || dtrace_assfail ("strlen(name) < sizeof (fp->ftp_name)", "/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" , 1639))); | |||
| 1640 | ASSERT(pattr != NULL)((void)((pattr != ((void *)0)) || dtrace_assfail("pattr != NULL" , "/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" , 1640))); | |||
| 1641 | ||||
| 1642 | bucket = &fasttrap_provs.fth_table[FASTTRAP_PROVS_INDEX(pid, name)((fasttrap_hash_str(name) + (pid)) & fasttrap_provs.fth_mask )]; | |||
| 1643 | mutex_enter(&bucket->ftb_mtx)(void)_sx_xlock(((&bucket->ftb_mtx)), 0, ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (1643)); | |||
| 1644 | ||||
| 1645 | /* | |||
| 1646 | * Take a lap through the list and return the match if we find it. | |||
| 1647 | */ | |||
| 1648 | for (fp = bucket->ftb_data; fp != NULL((void *)0); fp = fp->ftp_next) { | |||
| 1649 | if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 && | |||
| 1650 | !fp->ftp_retired) { | |||
| 1651 | mutex_enter(&fp->ftp_mtx)(void)_sx_xlock(((&fp->ftp_mtx)), 0, ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (1651)); | |||
| 1652 | mutex_exit(&bucket->ftb_mtx)_sx_xunlock(((&bucket->ftb_mtx)), ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (1652)); | |||
| 1653 | return (fp); | |||
| 1654 | } | |||
| 1655 | } | |||
| 1656 | ||||
| 1657 | /* | |||
| 1658 | * Drop the bucket lock so we don't try to perform a sleeping | |||
| 1659 | * allocation under it. | |||
| 1660 | */ | |||
| 1661 | mutex_exit(&bucket->ftb_mtx)_sx_xunlock(((&bucket->ftb_mtx)), ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (1661)); | |||
| 1662 | ||||
| 1663 | /* | |||
| 1664 | * Make sure the process exists, isn't a child created as the result | |||
| 1665 | * of a vfork(2), and isn't a zombie (but may be in fork). | |||
| 1666 | */ | |||
| 1667 | if ((p = pfind(pid)) == NULL((void *)0)) | |||
| 1668 | return (NULL((void *)0)); | |||
| 1669 | ||||
| 1670 | /* | |||
| 1671 | * Increment p_dtrace_probes so that the process knows to inform us | |||
| 1672 | * when it exits or execs. fasttrap_provider_free() decrements this | |||
| 1673 | * when we're done with this provider. | |||
| 1674 | */ | |||
| 1675 | p->p_dtrace_probesp_dtrace->p_dtrace_probes++; | |||
| 1676 | ||||
| 1677 | /* | |||
| 1678 | * Grab the credentials for this process so we have | |||
| 1679 | * something to pass to dtrace_register(). | |||
| 1680 | */ | |||
| 1681 | PROC_LOCK_ASSERT(p, MA_OWNED)__mtx_assert(&(((&(p)->p_mtx)))->mtx_lock, (((0x00000004 ))), ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (1681)); | |||
| 1682 | crhold(p->p_ucred); | |||
| 1683 | cred = p->p_ucred; | |||
| 1684 | PROC_UNLOCK(p)__mtx_unlock_flags(&((((&(p)->p_mtx))))->mtx_lock , ((0)), ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (1684)); | |||
| 1685 | ||||
| 1686 | new_fp = kmem_zalloc(sizeof (fasttrap_provider_t), KM_SLEEP)zfs_kmem_alloc((sizeof (fasttrap_provider_t)), (0x0002) | 0x0100 ); | |||
| 1687 | new_fp->ftp_pid = pid; | |||
| 1688 | new_fp->ftp_proc = fasttrap_proc_lookup(pid); | |||
| 1689 | #ifndef illumos | |||
| 1690 | mutex_init(&new_fp->ftp_mtx, "provider mtx", MUTEX_DEFAULT, NULL)do { const char *_name; ((void)(((MUTEX_DEFAULT) == 0 || (MUTEX_DEFAULT ) == MUTEX_DEFAULT) || dtrace_assfail("(MUTEX_DEFAULT) == 0 || (MUTEX_DEFAULT) == MUTEX_DEFAULT" , "/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" , 1690))); do { if (__builtin_expect((!(((&new_fp->ftp_mtx )->lock_object.lo_flags & (0x00010000 | 0x00020000 | 0x00040000 | 0x00080000 | 0x00100000 | 0x00200000 | 0x00400000 | 0x0f000000 | 0x10000000)) != (0x00010000 | 0x00020000 | 0x00080000 | 0x00100000 | 0x00200000 | 0x00400000 | (2 << 24)))), 0)) panic ("lock %s already initialized" , "&new_fp->ftp_mtx"); } while (0); for (_name = "&new_fp->ftp_mtx" ; *_name != '\0'; _name++) { if (*_name >= 'a' && * _name <= 'z') break; } if (*_name == '\0') _name = "&new_fp->ftp_mtx" ; sx_init_flags((&new_fp->ftp_mtx), _name, (0x01 | 0x40 | 0x04)); } while (0); | |||
| 1691 | mutex_init(&new_fp->ftp_cmtx, "lock on creating", MUTEX_DEFAULT, NULL)do { const char *_name; ((void)(((MUTEX_DEFAULT) == 0 || (MUTEX_DEFAULT ) == MUTEX_DEFAULT) || dtrace_assfail("(MUTEX_DEFAULT) == 0 || (MUTEX_DEFAULT) == MUTEX_DEFAULT" , "/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" , 1691))); do { if (__builtin_expect((!(((&new_fp->ftp_cmtx )->lock_object.lo_flags & (0x00010000 | 0x00020000 | 0x00040000 | 0x00080000 | 0x00100000 | 0x00200000 | 0x00400000 | 0x0f000000 | 0x10000000)) != (0x00010000 | 0x00020000 | 0x00080000 | 0x00100000 | 0x00200000 | 0x00400000 | (2 << 24)))), 0)) panic ("lock %s already initialized" , "&new_fp->ftp_cmtx"); } while (0); for (_name = "&new_fp->ftp_cmtx" ; *_name != '\0'; _name++) { if (*_name >= 'a' && * _name <= 'z') break; } if (*_name == '\0') _name = "&new_fp->ftp_cmtx" ; sx_init_flags((&new_fp->ftp_cmtx), _name, (0x01 | 0x40 | 0x04)); } while (0); | |||
| 1692 | #endif | |||
| 1693 | ||||
| 1694 | ASSERT(new_fp->ftp_proc != NULL)((void)((new_fp->ftp_proc != ((void *)0)) || dtrace_assfail ("new_fp->ftp_proc != NULL", "/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" , 1694))); | |||
| 1695 | ||||
| 1696 | mutex_enter(&bucket->ftb_mtx)(void)_sx_xlock(((&bucket->ftb_mtx)), 0, ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (1696)); | |||
| 1697 | ||||
| 1698 | /* | |||
| 1699 | * Take another lap through the list to make sure a provider hasn't | |||
| 1700 | * been created for this pid while we weren't under the bucket lock. | |||
| 1701 | */ | |||
| 1702 | for (fp = bucket->ftb_data; fp != NULL((void *)0); fp = fp->ftp_next) { | |||
| 1703 | if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 && | |||
| 1704 | !fp->ftp_retired) { | |||
| 1705 | mutex_enter(&fp->ftp_mtx)(void)_sx_xlock(((&fp->ftp_mtx)), 0, ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (1705)); | |||
| 1706 | mutex_exit(&bucket->ftb_mtx)_sx_xunlock(((&bucket->ftb_mtx)), ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (1706)); | |||
| 1707 | fasttrap_provider_free(new_fp); | |||
| 1708 | crfree(cred); | |||
| 1709 | return (fp); | |||
| 1710 | } | |||
| 1711 | } | |||
| 1712 | ||||
| 1713 | (void) strcpy(new_fp->ftp_name, name); | |||
| 1714 | ||||
| 1715 | /* | |||
| 1716 | * Fail and return NULL if either the provider name is too long | |||
| 1717 | * or we fail to register this new provider with the DTrace | |||
| 1718 | * framework. Note that this is the only place we ever construct | |||
| 1719 | * the full provider name -- we keep it in pieces in the provider | |||
| 1720 | * structure. | |||
| 1721 | */ | |||
| 1722 | if (snprintf(provname, sizeof (provname), "%s%u", name, (uint_t)pid) >= | |||
| 1723 | sizeof (provname) || | |||
| 1724 | dtrace_register(provname, pattr, | |||
| 1725 | DTRACE_PRIV_PROC0x0004 | DTRACE_PRIV_OWNER0x0008 | DTRACE_PRIV_ZONEOWNER0x0010, cred, | |||
| 1726 | pattr == &pid_attr ? &pid_pops : &usdt_pops, new_fp, | |||
| 1727 | &new_fp->ftp_provid) != 0) { | |||
| 1728 | mutex_exit(&bucket->ftb_mtx)_sx_xunlock(((&bucket->ftb_mtx)), ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (1728)); | |||
| 1729 | fasttrap_provider_free(new_fp); | |||
| 1730 | crfree(cred); | |||
| 1731 | return (NULL((void *)0)); | |||
| 1732 | } | |||
| 1733 | ||||
| 1734 | new_fp->ftp_next = bucket->ftb_data; | |||
| 1735 | bucket->ftb_data = new_fp; | |||
| 1736 | ||||
| 1737 | mutex_enter(&new_fp->ftp_mtx)(void)_sx_xlock(((&new_fp->ftp_mtx)), 0, ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (1737)); | |||
| 1738 | mutex_exit(&bucket->ftb_mtx)_sx_xunlock(((&bucket->ftb_mtx)), ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (1738)); | |||
| 1739 | ||||
| 1740 | crfree(cred); | |||
| 1741 | return (new_fp); | |||
| 1742 | } | |||
| 1743 | ||||
| 1744 | static void | |||
| 1745 | fasttrap_provider_free(fasttrap_provider_t *provider) | |||
| 1746 | { | |||
| 1747 | pid_t pid = provider->ftp_pid; | |||
| 1748 | proc_t *p; | |||
| 1749 | ||||
| 1750 | /* | |||
| 1751 | * There need to be no associated enabled probes, no consumers | |||
| 1752 | * creating probes, and no meta providers referencing this provider. | |||
| 1753 | */ | |||
| 1754 | ASSERT(provider->ftp_rcount == 0)((void)((provider->ftp_rcount == 0) || dtrace_assfail("provider->ftp_rcount == 0" , "/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" , 1754))); | |||
| 1755 | ASSERT(provider->ftp_ccount == 0)((void)((provider->ftp_ccount == 0) || dtrace_assfail("provider->ftp_ccount == 0" , "/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" , 1755))); | |||
| 1756 | ASSERT(provider->ftp_mcount == 0)((void)((provider->ftp_mcount == 0) || dtrace_assfail("provider->ftp_mcount == 0" , "/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" , 1756))); | |||
| 1757 | ||||
| 1758 | /* | |||
| 1759 | * If this provider hasn't been retired, we need to explicitly drop the | |||
| 1760 | * count of active providers on the associated process structure. | |||
| 1761 | */ | |||
| 1762 | if (!provider->ftp_retired) { | |||
| 1763 | atomic_dec_64(&provider->ftp_proc->ftpc_acount); | |||
| 1764 | ASSERT(provider->ftp_proc->ftpc_acount <((void)((provider->ftp_proc->ftpc_acount < provider-> ftp_proc->ftpc_rcount) || dtrace_assfail("provider->ftp_proc->ftpc_acount < provider->ftp_proc->ftpc_rcount" , "/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" , 1765))) | |||
| 1765 | provider->ftp_proc->ftpc_rcount)((void)((provider->ftp_proc->ftpc_acount < provider-> ftp_proc->ftpc_rcount) || dtrace_assfail("provider->ftp_proc->ftpc_acount < provider->ftp_proc->ftpc_rcount" , "/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" , 1765))); | |||
| 1766 | } | |||
| 1767 | ||||
| 1768 | fasttrap_proc_release(provider->ftp_proc); | |||
| 1769 | ||||
| 1770 | #ifndef illumos | |||
| 1771 | mutex_destroy(&provider->ftp_mtx)sx_destroy(&provider->ftp_mtx); | |||
| 1772 | mutex_destroy(&provider->ftp_cmtx)sx_destroy(&provider->ftp_cmtx); | |||
| 1773 | #endif | |||
| 1774 | kmem_free(provider, sizeof (fasttrap_provider_t))zfs_kmem_free((provider), (sizeof (fasttrap_provider_t))); | |||
| 1775 | ||||
| 1776 | /* | |||
| 1777 | * Decrement p_dtrace_probes on the process whose provider we're | |||
| 1778 | * freeing. We don't have to worry about clobbering somone else's | |||
| 1779 | * modifications to it because we have locked the bucket that | |||
| 1780 | * corresponds to this process's hash chain in the provider hash | |||
| 1781 | * table. Don't sweat it if we can't find the process. | |||
| 1782 | */ | |||
| 1783 | if ((p = pfind(pid)) == NULL((void *)0)) { | |||
| 1784 | return; | |||
| 1785 | } | |||
| 1786 | ||||
| 1787 | p->p_dtrace_probesp_dtrace->p_dtrace_probes--; | |||
| 1788 | #ifndef illumos | |||
| 1789 | PROC_UNLOCK(p)__mtx_unlock_flags(&((((&(p)->p_mtx))))->mtx_lock , ((0)), ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (1789)); | |||
| 1790 | #endif | |||
| 1791 | } | |||
| 1792 | ||||
| 1793 | static void | |||
| 1794 | fasttrap_provider_retire(pid_t pid, const char *name, int mprov) | |||
| 1795 | { | |||
| 1796 | fasttrap_provider_t *fp; | |||
| 1797 | fasttrap_bucket_t *bucket; | |||
| 1798 | dtrace_provider_id_t provid; | |||
| 1799 | ||||
| 1800 | ASSERT(strlen(name) < sizeof (fp->ftp_name))((void)((strlen(name) < sizeof (fp->ftp_name)) || dtrace_assfail ("strlen(name) < sizeof (fp->ftp_name)", "/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" , 1800))); | |||
| 1801 | ||||
| 1802 | bucket = &fasttrap_provs.fth_table[FASTTRAP_PROVS_INDEX(pid, name)((fasttrap_hash_str(name) + (pid)) & fasttrap_provs.fth_mask )]; | |||
| 1803 | mutex_enter(&bucket->ftb_mtx)(void)_sx_xlock(((&bucket->ftb_mtx)), 0, ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (1803)); | |||
| 1804 | ||||
| 1805 | for (fp = bucket->ftb_data; fp != NULL((void *)0); fp = fp->ftp_next) { | |||
| 1806 | if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 && | |||
| 1807 | !fp->ftp_retired) | |||
| 1808 | break; | |||
| 1809 | } | |||
| 1810 | ||||
| 1811 | if (fp == NULL((void *)0)) { | |||
| 1812 | mutex_exit(&bucket->ftb_mtx)_sx_xunlock(((&bucket->ftb_mtx)), ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (1812)); | |||
| 1813 | return; | |||
| 1814 | } | |||
| 1815 | ||||
| 1816 | mutex_enter(&fp->ftp_mtx)(void)_sx_xlock(((&fp->ftp_mtx)), 0, ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (1816)); | |||
| 1817 | ASSERT(!mprov || fp->ftp_mcount > 0)((void)((!mprov || fp->ftp_mcount > 0) || dtrace_assfail ("!mprov || fp->ftp_mcount > 0", "/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" , 1817))); | |||
| 1818 | if (mprov && --fp->ftp_mcount != 0) { | |||
| 1819 | mutex_exit(&fp->ftp_mtx)_sx_xunlock(((&fp->ftp_mtx)), ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (1819)); | |||
| 1820 | mutex_exit(&bucket->ftb_mtx)_sx_xunlock(((&bucket->ftb_mtx)), ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (1820)); | |||
| 1821 | return; | |||
| 1822 | } | |||
| 1823 | ||||
| 1824 | /* | |||
| 1825 | * Mark the provider to be removed in our post-processing step, mark it | |||
| 1826 | * retired, and drop the active count on its proc. Marking it indicates | |||
| 1827 | * that we should try to remove it; setting the retired flag indicates | |||
| 1828 | * that we're done with this provider; dropping the active the proc | |||
| 1829 | * releases our hold, and when this reaches zero (as it will during | |||
| 1830 | * exit or exec) the proc and associated providers become defunct. | |||
| 1831 | * | |||
| 1832 | * We obviously need to take the bucket lock before the provider lock | |||
| 1833 | * to perform the lookup, but we need to drop the provider lock | |||
| 1834 | * before calling into the DTrace framework since we acquire the | |||
| 1835 | * provider lock in callbacks invoked from the DTrace framework. The | |||
| 1836 | * bucket lock therefore protects the integrity of the provider hash | |||
| 1837 | * table. | |||
| 1838 | */ | |||
| 1839 | atomic_dec_64(&fp->ftp_proc->ftpc_acount); | |||
| 1840 | ASSERT(fp->ftp_proc->ftpc_acount < fp->ftp_proc->ftpc_rcount)((void)((fp->ftp_proc->ftpc_acount < fp->ftp_proc ->ftpc_rcount) || dtrace_assfail("fp->ftp_proc->ftpc_acount < fp->ftp_proc->ftpc_rcount" , "/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" , 1840))); | |||
| 1841 | ||||
| 1842 | fp->ftp_retired = 1; | |||
| 1843 | fp->ftp_marked = 1; | |||
| 1844 | provid = fp->ftp_provid; | |||
| 1845 | mutex_exit(&fp->ftp_mtx)_sx_xunlock(((&fp->ftp_mtx)), ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (1845)); | |||
| 1846 | ||||
| 1847 | /* | |||
| 1848 | * We don't have to worry about invalidating the same provider twice | |||
| 1849 | * since fasttrap_provider_lookup() will ignore provider that have | |||
| 1850 | * been marked as retired. | |||
| 1851 | */ | |||
| 1852 | dtrace_invalidate(provid); | |||
| 1853 | ||||
| 1854 | mutex_exit(&bucket->ftb_mtx)_sx_xunlock(((&bucket->ftb_mtx)), ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (1854)); | |||
| 1855 | ||||
| 1856 | fasttrap_pid_cleanup(); | |||
| 1857 | } | |||
| 1858 | ||||
| 1859 | static int | |||
| 1860 | fasttrap_uint32_cmp(const void *ap, const void *bp) | |||
| 1861 | { | |||
| 1862 | return (*(const uint32_t *)ap - *(const uint32_t *)bp); | |||
| 1863 | } | |||
| 1864 | ||||
| 1865 | static int | |||
| 1866 | fasttrap_uint64_cmp(const void *ap, const void *bp) | |||
| 1867 | { | |||
| 1868 | return (*(const uint64_t *)ap - *(const uint64_t *)bp); | |||
| 1869 | } | |||
| 1870 | ||||
| 1871 | static int | |||
| 1872 | fasttrap_add_probe(fasttrap_probe_spec_t *pdata) | |||
| 1873 | { | |||
| 1874 | fasttrap_provider_t *provider; | |||
| 1875 | fasttrap_probe_t *pp; | |||
| 1876 | fasttrap_tracepoint_t *tp; | |||
| 1877 | char *name; | |||
| 1878 | int i, aframes = 0, whack; | |||
| 1879 | ||||
| 1880 | /* | |||
| 1881 | * There needs to be at least one desired trace point. | |||
| 1882 | */ | |||
| 1883 | if (pdata->ftps_noffs == 0) | |||
| 1884 | return (EINVAL22); | |||
| 1885 | ||||
| 1886 | switch (pdata->ftps_type) { | |||
| 1887 | case DTFTP_ENTRY: | |||
| 1888 | name = "entry"; | |||
| 1889 | aframes = FASTTRAP_ENTRY_AFRAMES3; | |||
| 1890 | break; | |||
| 1891 | case DTFTP_RETURN: | |||
| 1892 | name = "return"; | |||
| 1893 | aframes = FASTTRAP_RETURN_AFRAMES4; | |||
| 1894 | break; | |||
| 1895 | case DTFTP_OFFSETS: | |||
| 1896 | name = NULL((void *)0); | |||
| 1897 | break; | |||
| 1898 | default: | |||
| 1899 | return (EINVAL22); | |||
| 1900 | } | |||
| 1901 | ||||
| 1902 | if ((provider = fasttrap_provider_lookup(pdata->ftps_pid, | |||
| 1903 | FASTTRAP_PID_NAME"pid", &pid_attr)) == NULL((void *)0)) | |||
| 1904 | return (ESRCH3); | |||
| 1905 | ||||
| 1906 | /* | |||
| 1907 | * Increment this reference count to indicate that a consumer is | |||
| 1908 | * actively adding a new probe associated with this provider. This | |||
| 1909 | * prevents the provider from being deleted -- we'll need to check | |||
| 1910 | * for pending deletions when we drop this reference count. | |||
| 1911 | */ | |||
| 1912 | provider->ftp_ccount++; | |||
| 1913 | mutex_exit(&provider->ftp_mtx)_sx_xunlock(((&provider->ftp_mtx)), ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (1913)); | |||
| 1914 | ||||
| 1915 | /* | |||
| 1916 | * Grab the creation lock to ensure consistency between calls to | |||
| 1917 | * dtrace_probe_lookup() and dtrace_probe_create() in the face of | |||
| 1918 | * other threads creating probes. We must drop the provider lock | |||
| 1919 | * before taking this lock to avoid a three-way deadlock with the | |||
| 1920 | * DTrace framework. | |||
| 1921 | */ | |||
| 1922 | mutex_enter(&provider->ftp_cmtx)(void)_sx_xlock(((&provider->ftp_cmtx)), 0, ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (1922)); | |||
| 1923 | ||||
| 1924 | if (name == NULL((void *)0)) { | |||
| 1925 | for (i = 0; i < pdata->ftps_noffs; i++) { | |||
| 1926 | char name_str[17]; | |||
| 1927 | ||||
| 1928 | (void) sprintf(name_str, "%llx", | |||
| 1929 | (unsigned long long)pdata->ftps_offs[i]); | |||
| 1930 | ||||
| 1931 | if (dtrace_probe_lookup(provider->ftp_provid, | |||
| 1932 | pdata->ftps_mod, pdata->ftps_func, name_str) != 0) | |||
| 1933 | continue; | |||
| 1934 | ||||
| 1935 | atomic_inc_32(&fasttrap_total); | |||
| 1936 | ||||
| 1937 | if (fasttrap_total > fasttrap_max) { | |||
| 1938 | atomic_dec_32(&fasttrap_total); | |||
| 1939 | goto no_mem; | |||
| 1940 | } | |||
| 1941 | ||||
| 1942 | pp = kmem_zalloc(sizeof (fasttrap_probe_t), KM_SLEEP)zfs_kmem_alloc((sizeof (fasttrap_probe_t)), (0x0002) | 0x0100 ); | |||
| 1943 | ||||
| 1944 | pp->ftp_prov = provider; | |||
| 1945 | pp->ftp_faddr = pdata->ftps_pc; | |||
| 1946 | pp->ftp_fsize = pdata->ftps_size; | |||
| 1947 | pp->ftp_pid = pdata->ftps_pid; | |||
| 1948 | pp->ftp_ntps = 1; | |||
| 1949 | ||||
| 1950 | tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t),zfs_kmem_alloc((sizeof (fasttrap_tracepoint_t)), (0x0002) | 0x0100 ) | |||
| 1951 | KM_SLEEP)zfs_kmem_alloc((sizeof (fasttrap_tracepoint_t)), (0x0002) | 0x0100 ); | |||
| 1952 | ||||
| 1953 | tp->ftt_proc = provider->ftp_proc; | |||
| 1954 | tp->ftt_pc = pdata->ftps_offs[i] + pdata->ftps_pc; | |||
| 1955 | tp->ftt_pid = pdata->ftps_pid; | |||
| 1956 | ||||
| 1957 | pp->ftp_tps[0].fit_tp = tp; | |||
| 1958 | pp->ftp_tps[0].fit_id.fti_probe = pp; | |||
| 1959 | pp->ftp_tps[0].fit_id.fti_ptype = pdata->ftps_type; | |||
| 1960 | ||||
| 1961 | pp->ftp_id = dtrace_probe_create(provider->ftp_provid, | |||
| 1962 | pdata->ftps_mod, pdata->ftps_func, name_str, | |||
| 1963 | FASTTRAP_OFFSET_AFRAMES3, pp); | |||
| 1964 | } | |||
| 1965 | ||||
| 1966 | } else if (dtrace_probe_lookup(provider->ftp_provid, pdata->ftps_mod, | |||
| 1967 | pdata->ftps_func, name) == 0) { | |||
| 1968 | atomic_add_32atomic_add_int(&fasttrap_total, pdata->ftps_noffs); | |||
| 1969 | ||||
| 1970 | if (fasttrap_total > fasttrap_max) { | |||
| 1971 | atomic_add_32atomic_add_int(&fasttrap_total, -pdata->ftps_noffs); | |||
| 1972 | goto no_mem; | |||
| 1973 | } | |||
| 1974 | ||||
| 1975 | /* | |||
| 1976 | * Make sure all tracepoint program counter values are unique. | |||
| 1977 | * We later assume that each probe has exactly one tracepoint | |||
| 1978 | * for a given pc. | |||
| 1979 | */ | |||
| 1980 | qsort(pdata->ftps_offs, pdata->ftps_noffs, | |||
| 1981 | sizeof (uint64_t), fasttrap_uint64_cmp); | |||
| 1982 | for (i = 1; i < pdata->ftps_noffs; i++) { | |||
| 1983 | if (pdata->ftps_offs[i] > pdata->ftps_offs[i - 1]) | |||
| 1984 | continue; | |||
| 1985 | ||||
| 1986 | atomic_add_32atomic_add_int(&fasttrap_total, -pdata->ftps_noffs); | |||
| 1987 | goto no_mem; | |||
| 1988 | } | |||
| 1989 | ||||
| 1990 | ASSERT(pdata->ftps_noffs > 0)((void)((pdata->ftps_noffs > 0) || dtrace_assfail("pdata->ftps_noffs > 0" , "/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" , 1990))); | |||
| 1991 | pp = kmem_zalloc(offsetof(fasttrap_probe_t,zfs_kmem_alloc((__builtin_offsetof(fasttrap_probe_t, ftp_tps[ pdata->ftps_noffs])), (0x0002) | 0x0100) | |||
| 1992 | ftp_tps[pdata->ftps_noffs]), KM_SLEEP)zfs_kmem_alloc((__builtin_offsetof(fasttrap_probe_t, ftp_tps[ pdata->ftps_noffs])), (0x0002) | 0x0100); | |||
| 1993 | ||||
| 1994 | pp->ftp_prov = provider; | |||
| 1995 | pp->ftp_faddr = pdata->ftps_pc; | |||
| 1996 | pp->ftp_fsize = pdata->ftps_size; | |||
| 1997 | pp->ftp_pid = pdata->ftps_pid; | |||
| 1998 | pp->ftp_ntps = pdata->ftps_noffs; | |||
| 1999 | ||||
| 2000 | for (i = 0; i < pdata->ftps_noffs; i++) { | |||
| 2001 | tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t),zfs_kmem_alloc((sizeof (fasttrap_tracepoint_t)), (0x0002) | 0x0100 ) | |||
| 2002 | KM_SLEEP)zfs_kmem_alloc((sizeof (fasttrap_tracepoint_t)), (0x0002) | 0x0100 ); | |||
| 2003 | ||||
| 2004 | tp->ftt_proc = provider->ftp_proc; | |||
| 2005 | tp->ftt_pc = pdata->ftps_offs[i] + pdata->ftps_pc; | |||
| 2006 | tp->ftt_pid = pdata->ftps_pid; | |||
| 2007 | ||||
| 2008 | pp->ftp_tps[i].fit_tp = tp; | |||
| 2009 | pp->ftp_tps[i].fit_id.fti_probe = pp; | |||
| 2010 | pp->ftp_tps[i].fit_id.fti_ptype = pdata->ftps_type; | |||
| 2011 | } | |||
| 2012 | ||||
| 2013 | pp->ftp_id = dtrace_probe_create(provider->ftp_provid, | |||
| 2014 | pdata->ftps_mod, pdata->ftps_func, name, aframes, pp); | |||
| 2015 | } | |||
| 2016 | ||||
| 2017 | mutex_exit(&provider->ftp_cmtx)_sx_xunlock(((&provider->ftp_cmtx)), ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (2017)); | |||
| 2018 | ||||
| 2019 | /* | |||
| 2020 | * We know that the provider is still valid since we incremented the | |||
| 2021 | * creation reference count. If someone tried to clean up this provider | |||
| 2022 | * while we were using it (e.g. because the process called exec(2) or | |||
| 2023 | * exit(2)), take note of that and try to clean it up now. | |||
| 2024 | */ | |||
| 2025 | mutex_enter(&provider->ftp_mtx)(void)_sx_xlock(((&provider->ftp_mtx)), 0, ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (2025)); | |||
| 2026 | provider->ftp_ccount--; | |||
| 2027 | whack = provider->ftp_retired; | |||
| 2028 | mutex_exit(&provider->ftp_mtx)_sx_xunlock(((&provider->ftp_mtx)), ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (2028)); | |||
| 2029 | ||||
| 2030 | if (whack) | |||
| 2031 | fasttrap_pid_cleanup(); | |||
| 2032 | ||||
| 2033 | return (0); | |||
| 2034 | ||||
| 2035 | no_mem: | |||
| 2036 | /* | |||
| 2037 | * If we've exhausted the allowable resources, we'll try to remove | |||
| 2038 | * this provider to free some up. This is to cover the case where | |||
| 2039 | * the user has accidentally created many more probes than was | |||
| 2040 | * intended (e.g. pid123:::). | |||
| 2041 | */ | |||
| 2042 | mutex_exit(&provider->ftp_cmtx)_sx_xunlock(((&provider->ftp_cmtx)), ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (2042)); | |||
| 2043 | mutex_enter(&provider->ftp_mtx)(void)_sx_xlock(((&provider->ftp_mtx)), 0, ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (2043)); | |||
| 2044 | provider->ftp_ccount--; | |||
| 2045 | provider->ftp_marked = 1; | |||
| 2046 | mutex_exit(&provider->ftp_mtx)_sx_xunlock(((&provider->ftp_mtx)), ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (2046)); | |||
| 2047 | ||||
| 2048 | fasttrap_pid_cleanup(); | |||
| 2049 | ||||
| 2050 | return (ENOMEM12); | |||
| 2051 | } | |||
| 2052 | ||||
| 2053 | /*ARGSUSED*/ | |||
| 2054 | static void * | |||
| 2055 | fasttrap_meta_provide(void *arg, dtrace_helper_provdesc_t *dhpv, pid_t pid) | |||
| 2056 | { | |||
| 2057 | fasttrap_provider_t *provider; | |||
| 2058 | ||||
| 2059 | /* | |||
| 2060 | * A 32-bit unsigned integer (like a pid for example) can be | |||
| 2061 | * expressed in 10 or fewer decimal digits. Make sure that we'll | |||
| 2062 | * have enough space for the provider name. | |||
| 2063 | */ | |||
| 2064 | if (strlen(dhpv->dthpv_provname) + 10 >= | |||
| 2065 | sizeof (provider->ftp_name)) { | |||
| 2066 | printf("failed to instantiate provider %s: " | |||
| 2067 | "name too long to accomodate pid", dhpv->dthpv_provname); | |||
| 2068 | return (NULL((void *)0)); | |||
| 2069 | } | |||
| 2070 | ||||
| 2071 | /* | |||
| 2072 | * Don't let folks spoof the true pid provider. | |||
| 2073 | */ | |||
| 2074 | if (strcmp(dhpv->dthpv_provname, FASTTRAP_PID_NAME"pid") == 0) { | |||
| 2075 | printf("failed to instantiate provider %s: " | |||
| 2076 | "%s is an invalid name", dhpv->dthpv_provname, | |||
| 2077 | FASTTRAP_PID_NAME"pid"); | |||
| 2078 | return (NULL((void *)0)); | |||
| 2079 | } | |||
| 2080 | ||||
| 2081 | /* | |||
| 2082 | * The highest stability class that fasttrap supports is ISA; cap | |||
| 2083 | * the stability of the new provider accordingly. | |||
| 2084 | */ | |||
| 2085 | if (dhpv->dthpv_pattr.dtpa_provider.dtat_class > DTRACE_CLASS_ISA4) | |||
| 2086 | dhpv->dthpv_pattr.dtpa_provider.dtat_class = DTRACE_CLASS_ISA4; | |||
| 2087 | if (dhpv->dthpv_pattr.dtpa_mod.dtat_class > DTRACE_CLASS_ISA4) | |||
| 2088 | dhpv->dthpv_pattr.dtpa_mod.dtat_class = DTRACE_CLASS_ISA4; | |||
| 2089 | if (dhpv->dthpv_pattr.dtpa_func.dtat_class > DTRACE_CLASS_ISA4) | |||
| 2090 | dhpv->dthpv_pattr.dtpa_func.dtat_class = DTRACE_CLASS_ISA4; | |||
| 2091 | if (dhpv->dthpv_pattr.dtpa_name.dtat_class > DTRACE_CLASS_ISA4) | |||
| 2092 | dhpv->dthpv_pattr.dtpa_name.dtat_class = DTRACE_CLASS_ISA4; | |||
| 2093 | if (dhpv->dthpv_pattr.dtpa_args.dtat_class > DTRACE_CLASS_ISA4) | |||
| 2094 | dhpv->dthpv_pattr.dtpa_args.dtat_class = DTRACE_CLASS_ISA4; | |||
| 2095 | ||||
| 2096 | if ((provider = fasttrap_provider_lookup(pid, dhpv->dthpv_provname, | |||
| 2097 | &dhpv->dthpv_pattr)) == NULL((void *)0)) { | |||
| 2098 | printf("failed to instantiate provider %s for " | |||
| 2099 | "process %u", dhpv->dthpv_provname, (uint_t)pid); | |||
| 2100 | return (NULL((void *)0)); | |||
| 2101 | } | |||
| 2102 | ||||
| 2103 | /* | |||
| 2104 | * Up the meta provider count so this provider isn't removed until | |||
| 2105 | * the meta provider has been told to remove it. | |||
| 2106 | */ | |||
| 2107 | provider->ftp_mcount++; | |||
| 2108 | ||||
| 2109 | mutex_exit(&provider->ftp_mtx)_sx_xunlock(((&provider->ftp_mtx)), ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (2109)); | |||
| 2110 | ||||
| 2111 | return (provider); | |||
| 2112 | } | |||
| 2113 | ||||
| 2114 | /* | |||
| 2115 | * We know a few things about our context here: we know that the probe being | |||
| 2116 | * created doesn't already exist (DTrace won't load DOF at the same address | |||
| 2117 | * twice, even if explicitly told to do so) and we know that we are | |||
| 2118 | * single-threaded with respect to the meta provider machinery. Knowing that | |||
| 2119 | * this is a new probe and that there is no way for us to race with another | |||
| 2120 | * operation on this provider allows us an important optimization: we need not | |||
| 2121 | * lookup a probe before adding it. Saving this lookup is important because | |||
| 2122 | * this code is in the fork path for processes with USDT probes, and lookups | |||
| 2123 | * here are potentially very expensive because of long hash conflicts on | |||
| 2124 | * module, function and name (DTrace doesn't hash on provider name). | |||
| 2125 | */ | |||
| 2126 | /*ARGSUSED*/ | |||
| 2127 | static void | |||
| 2128 | fasttrap_meta_create_probe(void *arg, void *parg, | |||
| 2129 | dtrace_helper_probedesc_t *dhpb) | |||
| 2130 | { | |||
| 2131 | fasttrap_provider_t *provider = parg; | |||
| 2132 | fasttrap_probe_t *pp; | |||
| 2133 | fasttrap_tracepoint_t *tp; | |||
| 2134 | int i, j; | |||
| 2135 | uint32_t ntps; | |||
| 2136 | ||||
| 2137 | /* | |||
| 2138 | * Since the meta provider count is non-zero we don't have to worry | |||
| 2139 | * about this provider disappearing. | |||
| 2140 | */ | |||
| 2141 | ASSERT(provider->ftp_mcount > 0)((void)((provider->ftp_mcount > 0) || dtrace_assfail("provider->ftp_mcount > 0" , "/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" , 2141))); | |||
| 2142 | ||||
| 2143 | /* | |||
| 2144 | * The offsets must be unique. | |||
| 2145 | */ | |||
| 2146 | qsort(dhpb->dthpb_offs, dhpb->dthpb_noffs, sizeof (uint32_t), | |||
| 2147 | fasttrap_uint32_cmp); | |||
| 2148 | for (i = 1; i < dhpb->dthpb_noffs; i++) { | |||
| 2149 | if (dhpb->dthpb_base + dhpb->dthpb_offs[i] <= | |||
| 2150 | dhpb->dthpb_base + dhpb->dthpb_offs[i - 1]) | |||
| 2151 | return; | |||
| 2152 | } | |||
| 2153 | ||||
| 2154 | qsort(dhpb->dthpb_enoffs, dhpb->dthpb_nenoffs, sizeof (uint32_t), | |||
| 2155 | fasttrap_uint32_cmp); | |||
| 2156 | for (i = 1; i < dhpb->dthpb_nenoffs; i++) { | |||
| 2157 | if (dhpb->dthpb_base + dhpb->dthpb_enoffs[i] <= | |||
| 2158 | dhpb->dthpb_base + dhpb->dthpb_enoffs[i - 1]) | |||
| 2159 | return; | |||
| 2160 | } | |||
| 2161 | ||||
| 2162 | ntps = dhpb->dthpb_noffs + dhpb->dthpb_nenoffs; | |||
| 2163 | ASSERT(ntps > 0)((void)((ntps > 0) || dtrace_assfail("ntps > 0", "/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" , 2163))); | |||
| 2164 | ||||
| 2165 | atomic_add_32atomic_add_int(&fasttrap_total, ntps); | |||
| 2166 | ||||
| 2167 | if (fasttrap_total > fasttrap_max) { | |||
| 2168 | atomic_add_32atomic_add_int(&fasttrap_total, -ntps); | |||
| 2169 | return; | |||
| 2170 | } | |||
| 2171 | ||||
| 2172 | pp = kmem_zalloc(offsetof(fasttrap_probe_t, ftp_tps[ntps]), KM_SLEEP)zfs_kmem_alloc((__builtin_offsetof(fasttrap_probe_t, ftp_tps[ ntps])), (0x0002) | 0x0100); | |||
| 2173 | ||||
| 2174 | pp->ftp_prov = provider; | |||
| 2175 | pp->ftp_pid = provider->ftp_pid; | |||
| 2176 | pp->ftp_ntps = ntps; | |||
| 2177 | pp->ftp_nargs = dhpb->dthpb_xargc; | |||
| 2178 | pp->ftp_xtypes = dhpb->dthpb_xtypes; | |||
| 2179 | pp->ftp_ntypes = dhpb->dthpb_ntypes; | |||
| 2180 | ||||
| 2181 | /* | |||
| 2182 | * First create a tracepoint for each actual point of interest. | |||
| 2183 | */ | |||
| 2184 | for (i = 0; i < dhpb->dthpb_noffs; i++) { | |||
| 2185 | tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), KM_SLEEP)zfs_kmem_alloc((sizeof (fasttrap_tracepoint_t)), (0x0002) | 0x0100 ); | |||
| 2186 | ||||
| 2187 | tp->ftt_proc = provider->ftp_proc; | |||
| 2188 | tp->ftt_pc = dhpb->dthpb_base + dhpb->dthpb_offs[i]; | |||
| 2189 | tp->ftt_pid = provider->ftp_pid; | |||
| 2190 | ||||
| 2191 | pp->ftp_tps[i].fit_tp = tp; | |||
| 2192 | pp->ftp_tps[i].fit_id.fti_probe = pp; | |||
| 2193 | #ifdef __sparc | |||
| 2194 | pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_POST_OFFSETS; | |||
| 2195 | #else | |||
| 2196 | pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_OFFSETS; | |||
| 2197 | #endif | |||
| 2198 | } | |||
| 2199 | ||||
| 2200 | /* | |||
| 2201 | * Then create a tracepoint for each is-enabled point. | |||
| 2202 | */ | |||
| 2203 | for (j = 0; i < ntps; i++, j++) { | |||
| 2204 | tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), KM_SLEEP)zfs_kmem_alloc((sizeof (fasttrap_tracepoint_t)), (0x0002) | 0x0100 ); | |||
| 2205 | ||||
| 2206 | tp->ftt_proc = provider->ftp_proc; | |||
| 2207 | tp->ftt_pc = dhpb->dthpb_base + dhpb->dthpb_enoffs[j]; | |||
| 2208 | tp->ftt_pid = provider->ftp_pid; | |||
| 2209 | ||||
| 2210 | pp->ftp_tps[i].fit_tp = tp; | |||
| 2211 | pp->ftp_tps[i].fit_id.fti_probe = pp; | |||
| 2212 | pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_IS_ENABLED; | |||
| 2213 | } | |||
| 2214 | ||||
| 2215 | /* | |||
| 2216 | * If the arguments are shuffled around we set the argument remapping | |||
| 2217 | * table. Later, when the probe fires, we only remap the arguments | |||
| 2218 | * if the table is non-NULL. | |||
| 2219 | */ | |||
| 2220 | for (i = 0; i < dhpb->dthpb_xargc; i++) { | |||
| 2221 | if (dhpb->dthpb_args[i] != i) { | |||
| 2222 | pp->ftp_argmap = dhpb->dthpb_args; | |||
| 2223 | break; | |||
| 2224 | } | |||
| 2225 | } | |||
| 2226 | ||||
| 2227 | /* | |||
| 2228 | * The probe is fully constructed -- register it with DTrace. | |||
| 2229 | */ | |||
| 2230 | pp->ftp_id = dtrace_probe_create(provider->ftp_provid, dhpb->dthpb_mod, | |||
| 2231 | dhpb->dthpb_func, dhpb->dthpb_name, FASTTRAP_OFFSET_AFRAMES3, pp); | |||
| 2232 | } | |||
| 2233 | ||||
| 2234 | /*ARGSUSED*/ | |||
| 2235 | static void | |||
| 2236 | fasttrap_meta_remove(void *arg, dtrace_helper_provdesc_t *dhpv, pid_t pid) | |||
| 2237 | { | |||
| 2238 | /* | |||
| 2239 | * Clean up the USDT provider. There may be active consumers of the | |||
| 2240 | * provider busy adding probes, no damage will actually befall the | |||
| 2241 | * provider until that count has dropped to zero. This just puts | |||
| 2242 | * the provider on death row. | |||
| 2243 | */ | |||
| 2244 | fasttrap_provider_retire(pid, dhpv->dthpv_provname, 1); | |||
| 2245 | } | |||
| 2246 | ||||
| 2247 | static dtrace_mops_t fasttrap_mops = { | |||
| 2248 | .dtms_create_probe = fasttrap_meta_create_probe, | |||
| 2249 | .dtms_provide_pid = fasttrap_meta_provide, | |||
| 2250 | .dtms_remove_pid = fasttrap_meta_remove | |||
| 2251 | }; | |||
| 2252 | ||||
| 2253 | /*ARGSUSED*/ | |||
| 2254 | static int | |||
| 2255 | fasttrap_open(struct cdev *dev __unused__attribute__((__unused__)), int oflags __unused__attribute__((__unused__)), | |||
| 2256 | int devtype __unused__attribute__((__unused__)), struct thread *td __unused__attribute__((__unused__))) | |||
| 2257 | { | |||
| 2258 | return (0); | |||
| 2259 | } | |||
| 2260 | ||||
| 2261 | /*ARGSUSED*/ | |||
| 2262 | static int | |||
| 2263 | fasttrap_ioctl(struct cdev *dev, u_long cmd, caddr_t arg, int fflag, | |||
| 2264 | struct thread *td) | |||
| 2265 | { | |||
| 2266 | if (!dtrace_attached()) | |||
| ||||
| 2267 | return (EAGAIN35); | |||
| 2268 | ||||
| 2269 | if (cmd == FASTTRAPIOC_MAKEPROBE((unsigned long) ((0x20000000) | (((0) & ((1 << 13) - 1)) << 16) | ((('f')) << 8) | ((3))))) { | |||
| 2270 | fasttrap_probe_spec_t *uprobe = *(fasttrap_probe_spec_t **)arg; | |||
| 2271 | fasttrap_probe_spec_t *probe; | |||
| 2272 | uint64_t noffs; | |||
| 2273 | size_t size; | |||
| 2274 | int ret, err; | |||
| 2275 | ||||
| 2276 | if (copyin(&uprobe->ftps_noffs, &noffs, | |||
| 2277 | sizeof (uprobe->ftps_noffs))) | |||
| 2278 | return (EFAULT14); | |||
| 2279 | ||||
| 2280 | /* | |||
| 2281 | * Probes must have at least one tracepoint. | |||
| 2282 | */ | |||
| 2283 | if (noffs == 0) | |||
| 2284 | return (EINVAL22); | |||
| 2285 | ||||
| 2286 | size = sizeof (fasttrap_probe_spec_t) + | |||
| 2287 | sizeof (probe->ftps_offs[0]) * (noffs - 1); | |||
| 2288 | ||||
| 2289 | if (size > 1024 * 1024) | |||
| 2290 | return (ENOMEM12); | |||
| 2291 | ||||
| 2292 | probe = kmem_alloc(size, KM_SLEEP)zfs_kmem_alloc((size), (0x0002)); | |||
| 2293 | ||||
| 2294 | if (copyin(uprobe, probe, size) != 0 || | |||
| 2295 | probe->ftps_noffs != noffs) { | |||
| 2296 | kmem_free(probe, size)zfs_kmem_free((probe), (size)); | |||
| 2297 | return (EFAULT14); | |||
| 2298 | } | |||
| 2299 | ||||
| 2300 | /* | |||
| 2301 | * Verify that the function and module strings contain no | |||
| 2302 | * funny characters. | |||
| 2303 | */ | |||
| 2304 | if (u8_validate(probe->ftps_func, strlen(probe->ftps_func), | |||
| 2305 | NULL((void *)0), U8_VALIDATE_ENTIRE(0x00100000), &err) < 0) { | |||
| 2306 | ret = EINVAL22; | |||
| 2307 | goto err; | |||
| 2308 | } | |||
| 2309 | ||||
| 2310 | if (u8_validate(probe->ftps_mod, strlen(probe->ftps_mod), | |||
| 2311 | NULL((void *)0), U8_VALIDATE_ENTIRE(0x00100000), &err) < 0) { | |||
| 2312 | ret = EINVAL22; | |||
| 2313 | goto err; | |||
| 2314 | } | |||
| 2315 | ||||
| 2316 | #ifdef notyet | |||
| 2317 | if (!PRIV_POLICY_CHOICE(cr, PRIV_ALL, B_FALSE)priv_policy_choice((cr), (PRIV_ALL), (0))) { | |||
| 2318 | proc_t *p; | |||
| 2319 | pid_t pid = probe->ftps_pid; | |||
| 2320 | ||||
| 2321 | mutex_enter(&pidlock)(void)_sx_xlock(((&pidlock)), 0, ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (2321)); | |||
| 2322 | /* | |||
| 2323 | * Report an error if the process doesn't exist | |||
| 2324 | * or is actively being birthed. | |||
| 2325 | */ | |||
| 2326 | if ((p = pfind(pid)) == NULL((void *)0) || p->p_stat == SIDL1) { | |||
| 2327 | mutex_exit(&pidlock)_sx_xunlock(((&pidlock)), ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (2327)); | |||
| 2328 | return (ESRCH3); | |||
| 2329 | } | |||
| 2330 | mutex_enter(&p->p_lock)(void)_sx_xlock(((&p->p_lock)), 0, ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (2330)); | |||
| 2331 | mutex_exit(&pidlock)_sx_xunlock(((&pidlock)), ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (2331)); | |||
| 2332 | ||||
| 2333 | if ((ret = priv_proc_cred_perm(cr, p, NULL((void *)0), | |||
| 2334 | VREAD000000000400 | VWRITE000000000200)) != 0) { | |||
| 2335 | mutex_exit(&p->p_lock)_sx_xunlock(((&p->p_lock)), ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (2335)); | |||
| 2336 | return (ret); | |||
| 2337 | } | |||
| 2338 | mutex_exit(&p->p_lock)_sx_xunlock(((&p->p_lock)), ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (2338)); | |||
| 2339 | } | |||
| 2340 | #endif /* notyet */ | |||
| 2341 | ||||
| 2342 | ret = fasttrap_add_probe(probe); | |||
| 2343 | err: | |||
| 2344 | kmem_free(probe, size)zfs_kmem_free((probe), (size)); | |||
| 2345 | ||||
| 2346 | return (ret); | |||
| 2347 | ||||
| 2348 | } else if (cmd == FASTTRAPIOC_GETINSTR((unsigned long) (((0x80000000|0x40000000)) | (((sizeof(uint8_t )) & ((1 << 13) - 1)) << 16) | ((('f')) << 8) | ((2))))) { | |||
| 2349 | fasttrap_instr_query_t instr; | |||
| 2350 | fasttrap_tracepoint_t *tp; | |||
| 2351 | uint_t index; | |||
| 2352 | #ifdef notyet | |||
| 2353 | int ret; | |||
| 2354 | #endif | |||
| 2355 | ||||
| 2356 | #ifdef illumos | |||
| 2357 | if (copyin((void *)arg, &instr, sizeof (instr)) != 0) | |||
| 2358 | return (EFAULT14); | |||
| 2359 | #endif | |||
| 2360 | ||||
| 2361 | #ifdef notyet | |||
| 2362 | if (!PRIV_POLICY_CHOICE(cr, PRIV_ALL, B_FALSE)priv_policy_choice((cr), (PRIV_ALL), (0))) { | |||
| 2363 | proc_t *p; | |||
| 2364 | pid_t pid = instr.ftiq_pid; | |||
| 2365 | ||||
| 2366 | mutex_enter(&pidlock)(void)_sx_xlock(((&pidlock)), 0, ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (2366)); | |||
| 2367 | /* | |||
| 2368 | * Report an error if the process doesn't exist | |||
| 2369 | * or is actively being birthed. | |||
| 2370 | */ | |||
| 2371 | if ((p == pfind(pid)) == NULL((void *)0) || p->p_stat == SIDL1) { | |||
| 2372 | mutex_exit(&pidlock)_sx_xunlock(((&pidlock)), ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (2372)); | |||
| 2373 | return (ESRCH3); | |||
| 2374 | } | |||
| 2375 | mutex_enter(&p->p_lock)(void)_sx_xlock(((&p->p_lock)), 0, ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (2375)); | |||
| 2376 | mutex_exit(&pidlock)_sx_xunlock(((&pidlock)), ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (2376)); | |||
| 2377 | ||||
| 2378 | if ((ret = priv_proc_cred_perm(cr, p, NULL((void *)0), | |||
| 2379 | VREAD000000000400)) != 0) { | |||
| 2380 | mutex_exit(&p->p_lock)_sx_xunlock(((&p->p_lock)), ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (2380)); | |||
| 2381 | return (ret); | |||
| 2382 | } | |||
| 2383 | ||||
| 2384 | mutex_exit(&p->p_lock)_sx_xunlock(((&p->p_lock)), ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (2384)); | |||
| 2385 | } | |||
| 2386 | #endif /* notyet */ | |||
| 2387 | ||||
| 2388 | index = FASTTRAP_TPOINTS_INDEX(instr.ftiq_pid, instr.ftiq_pc)(((instr.ftiq_pc) / sizeof (fasttrap_instr_t) + (instr.ftiq_pid )) & fasttrap_tpoints.fth_mask); | |||
| 2389 | ||||
| 2390 | mutex_enter(&fasttrap_tpoints.fth_table[index].ftb_mtx)(void)_sx_xlock(((&fasttrap_tpoints.fth_table[index].ftb_mtx )), 0, ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (2390)); | |||
| 2391 | tp = fasttrap_tpoints.fth_table[index].ftb_data; | |||
| 2392 | while (tp != NULL((void *)0)) { | |||
| 2393 | if (instr.ftiq_pid == tp->ftt_pid && | |||
| 2394 | instr.ftiq_pc == tp->ftt_pc && | |||
| 2395 | tp->ftt_proc->ftpc_acount != 0) | |||
| 2396 | break; | |||
| 2397 | ||||
| 2398 | tp = tp->ftt_next; | |||
| 2399 | } | |||
| 2400 | ||||
| 2401 | if (tp == NULL((void *)0)) { | |||
| 2402 | mutex_exit(&fasttrap_tpoints.fth_table[index].ftb_mtx)_sx_xunlock(((&fasttrap_tpoints.fth_table[index].ftb_mtx) ), ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (2402)); | |||
| 2403 | return (ENOENT2); | |||
| 2404 | } | |||
| 2405 | ||||
| 2406 | bcopy(&tp->ftt_instr, &instr.ftiq_instr,__builtin_memmove((&instr.ftiq_instr), (&tp->ftt_mtp .ftmt_instr), (sizeof (instr.ftiq_instr))) | |||
| 2407 | sizeof (instr.ftiq_instr))__builtin_memmove((&instr.ftiq_instr), (&tp->ftt_mtp .ftmt_instr), (sizeof (instr.ftiq_instr))); | |||
| 2408 | mutex_exit(&fasttrap_tpoints.fth_table[index].ftb_mtx)_sx_xunlock(((&fasttrap_tpoints.fth_table[index].ftb_mtx) ), ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (2408)); | |||
| 2409 | ||||
| 2410 | if (copyout(&instr, (void *)arg, sizeof (instr)) != 0) | |||
| ||||
| 2411 | return (EFAULT14); | |||
| 2412 | ||||
| 2413 | return (0); | |||
| 2414 | } | |||
| 2415 | ||||
| 2416 | return (EINVAL22); | |||
| 2417 | } | |||
| 2418 | ||||
| 2419 | static int | |||
| 2420 | fasttrap_load(void) | |||
| 2421 | { | |||
| 2422 | ulong_t nent; | |||
| 2423 | int i, ret; | |||
| 2424 | ||||
| 2425 | /* Create the /dev/dtrace/fasttrap entry. */ | |||
| 2426 | fasttrap_cdev = make_dev(&fasttrap_cdevsw, 0, UID_ROOT0, GID_WHEEL0, 0600, | |||
| 2427 | "dtrace/fasttrap"); | |||
| 2428 | ||||
| 2429 | mtx_init(&fasttrap_cleanup_mtx, "fasttrap clean", "dtrace", MTX_DEF)_mtx_init(&(&fasttrap_cleanup_mtx)->mtx_lock, "fasttrap clean" , "dtrace", 0x00000000); | |||
| 2430 | mutex_init(&fasttrap_count_mtx, "fasttrap count mtx", MUTEX_DEFAULT,do { const char *_name; ((void)(((MUTEX_DEFAULT) == 0 || (MUTEX_DEFAULT ) == MUTEX_DEFAULT) || dtrace_assfail("(MUTEX_DEFAULT) == 0 || (MUTEX_DEFAULT) == MUTEX_DEFAULT" , "/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" , 2431))); do { if (__builtin_expect((!(((&fasttrap_count_mtx )->lock_object.lo_flags & (0x00010000 | 0x00020000 | 0x00040000 | 0x00080000 | 0x00100000 | 0x00200000 | 0x00400000 | 0x0f000000 | 0x10000000)) != (0x00010000 | 0x00020000 | 0x00080000 | 0x00100000 | 0x00200000 | 0x00400000 | (2 << 24)))), 0)) panic ("lock %s already initialized" , "&fasttrap_count_mtx"); } while (0); for (_name = "&fasttrap_count_mtx" ; *_name != '\0'; _name++) { if (*_name >= 'a' && * _name <= 'z') break; } if (*_name == '\0') _name = "&fasttrap_count_mtx" ; sx_init_flags((&fasttrap_count_mtx), _name, (0x01 | 0x40 | 0x04)); } while (0) | |||
| 2431 | NULL)do { const char *_name; ((void)(((MUTEX_DEFAULT) == 0 || (MUTEX_DEFAULT ) == MUTEX_DEFAULT) || dtrace_assfail("(MUTEX_DEFAULT) == 0 || (MUTEX_DEFAULT) == MUTEX_DEFAULT" , "/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" , 2431))); do { if (__builtin_expect((!(((&fasttrap_count_mtx )->lock_object.lo_flags & (0x00010000 | 0x00020000 | 0x00040000 | 0x00080000 | 0x00100000 | 0x00200000 | 0x00400000 | 0x0f000000 | 0x10000000)) != (0x00010000 | 0x00020000 | 0x00080000 | 0x00100000 | 0x00200000 | 0x00400000 | (2 << 24)))), 0)) panic ("lock %s already initialized" , "&fasttrap_count_mtx"); } while (0); for (_name = "&fasttrap_count_mtx" ; *_name != '\0'; _name++) { if (*_name >= 'a' && * _name <= 'z') break; } if (*_name == '\0') _name = "&fasttrap_count_mtx" ; sx_init_flags((&fasttrap_count_mtx), _name, (0x01 | 0x40 | 0x04)); } while (0); | |||
| 2432 | ||||
| 2433 | #ifdef illumos | |||
| 2434 | fasttrap_max = ddi_getprop(DDI_DEV_T_ANY, devi, DDI_PROP_DONTPASS, | |||
| 2435 | "fasttrap-max-probes", FASTTRAP_MAX_DEFAULT250000); | |||
| 2436 | #endif | |||
| 2437 | fasttrap_total = 0; | |||
| 2438 | ||||
| 2439 | /* | |||
| 2440 | * Conjure up the tracepoints hashtable... | |||
| 2441 | */ | |||
| 2442 | #ifdef illumos | |||
| 2443 | nent = ddi_getprop(DDI_DEV_T_ANY, devi, DDI_PROP_DONTPASS, | |||
| 2444 | "fasttrap-hash-size", FASTTRAP_TPOINTS_DEFAULT_SIZE0x4000); | |||
| 2445 | #else | |||
| 2446 | nent = tpoints_hash_size; | |||
| 2447 | #endif | |||
| 2448 | ||||
| 2449 | if (nent == 0 || nent > 0x1000000) | |||
| 2450 | nent = FASTTRAP_TPOINTS_DEFAULT_SIZE0x4000; | |||
| 2451 | ||||
| 2452 | tpoints_hash_size = nent; | |||
| 2453 | ||||
| 2454 | if (ISP2(nent)(((nent) & ((nent) - 1)) == 0)) | |||
| 2455 | fasttrap_tpoints.fth_nent = nent; | |||
| 2456 | else | |||
| 2457 | fasttrap_tpoints.fth_nent = 1 << fasttrap_highbit(nent); | |||
| 2458 | ASSERT(fasttrap_tpoints.fth_nent > 0)((void)((fasttrap_tpoints.fth_nent > 0) || dtrace_assfail( "fasttrap_tpoints.fth_nent > 0", "/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" , 2458))); | |||
| 2459 | fasttrap_tpoints.fth_mask = fasttrap_tpoints.fth_nent - 1; | |||
| 2460 | fasttrap_tpoints.fth_table = kmem_zalloc(fasttrap_tpoints.fth_nent *zfs_kmem_alloc((fasttrap_tpoints.fth_nent * sizeof (fasttrap_bucket_t )), (0x0002) | 0x0100) | |||
| 2461 | sizeof (fasttrap_bucket_t), KM_SLEEP)zfs_kmem_alloc((fasttrap_tpoints.fth_nent * sizeof (fasttrap_bucket_t )), (0x0002) | 0x0100); | |||
| 2462 | #ifndef illumos | |||
| 2463 | for (i = 0; i < fasttrap_tpoints.fth_nent; i++) | |||
| 2464 | mutex_init(&fasttrap_tpoints.fth_table[i].ftb_mtx,do { const char *_name; ((void)(((MUTEX_DEFAULT) == 0 || (MUTEX_DEFAULT ) == MUTEX_DEFAULT) || dtrace_assfail("(MUTEX_DEFAULT) == 0 || (MUTEX_DEFAULT) == MUTEX_DEFAULT" , "/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" , 2465))); do { if (__builtin_expect((!(((&fasttrap_tpoints .fth_table[i].ftb_mtx)->lock_object.lo_flags & (0x00010000 | 0x00020000 | 0x00040000 | 0x00080000 | 0x00100000 | 0x00200000 | 0x00400000 | 0x0f000000 | 0x10000000)) != (0x00010000 | 0x00020000 | 0x00080000 | 0x00100000 | 0x00200000 | 0x00400000 | (2 << 24)))), 0)) panic ("lock %s already initialized", "&fasttrap_tpoints.fth_table[i].ftb_mtx" ); } while (0); for (_name = "&fasttrap_tpoints.fth_table[i].ftb_mtx" ; *_name != '\0'; _name++) { if (*_name >= 'a' && * _name <= 'z') break; } if (*_name == '\0') _name = "&fasttrap_tpoints.fth_table[i].ftb_mtx" ; sx_init_flags((&fasttrap_tpoints.fth_table[i].ftb_mtx), _name, (0x01 | 0x40 | 0x04)); } while (0) | |||
| 2465 | "tracepoints bucket mtx", MUTEX_DEFAULT, NULL)do { const char *_name; ((void)(((MUTEX_DEFAULT) == 0 || (MUTEX_DEFAULT ) == MUTEX_DEFAULT) || dtrace_assfail("(MUTEX_DEFAULT) == 0 || (MUTEX_DEFAULT) == MUTEX_DEFAULT" , "/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" , 2465))); do { if (__builtin_expect((!(((&fasttrap_tpoints .fth_table[i].ftb_mtx)->lock_object.lo_flags & (0x00010000 | 0x00020000 | 0x00040000 | 0x00080000 | 0x00100000 | 0x00200000 | 0x00400000 | 0x0f000000 | 0x10000000)) != (0x00010000 | 0x00020000 | 0x00080000 | 0x00100000 | 0x00200000 | 0x00400000 | (2 << 24)))), 0)) panic ("lock %s already initialized", "&fasttrap_tpoints.fth_table[i].ftb_mtx" ); } while (0); for (_name = "&fasttrap_tpoints.fth_table[i].ftb_mtx" ; *_name != '\0'; _name++) { if (*_name >= 'a' && * _name <= 'z') break; } if (*_name == '\0') _name = "&fasttrap_tpoints.fth_table[i].ftb_mtx" ; sx_init_flags((&fasttrap_tpoints.fth_table[i].ftb_mtx), _name, (0x01 | 0x40 | 0x04)); } while (0); | |||
| 2466 | #endif | |||
| 2467 | ||||
| 2468 | /* | |||
| 2469 | * ... and the providers hash table... | |||
| 2470 | */ | |||
| 2471 | nent = FASTTRAP_PROVIDERS_DEFAULT_SIZE0x100; | |||
| 2472 | if (ISP2(nent)(((nent) & ((nent) - 1)) == 0)) | |||
| 2473 | fasttrap_provs.fth_nent = nent; | |||
| 2474 | else | |||
| 2475 | fasttrap_provs.fth_nent = 1 << fasttrap_highbit(nent); | |||
| 2476 | ASSERT(fasttrap_provs.fth_nent > 0)((void)((fasttrap_provs.fth_nent > 0) || dtrace_assfail("fasttrap_provs.fth_nent > 0" , "/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" , 2476))); | |||
| 2477 | fasttrap_provs.fth_mask = fasttrap_provs.fth_nent - 1; | |||
| 2478 | fasttrap_provs.fth_table = kmem_zalloc(fasttrap_provs.fth_nent *zfs_kmem_alloc((fasttrap_provs.fth_nent * sizeof (fasttrap_bucket_t )), (0x0002) | 0x0100) | |||
| 2479 | sizeof (fasttrap_bucket_t), KM_SLEEP)zfs_kmem_alloc((fasttrap_provs.fth_nent * sizeof (fasttrap_bucket_t )), (0x0002) | 0x0100); | |||
| 2480 | #ifndef illumos | |||
| 2481 | for (i = 0; i < fasttrap_provs.fth_nent; i++) | |||
| 2482 | mutex_init(&fasttrap_provs.fth_table[i].ftb_mtx,do { const char *_name; ((void)(((MUTEX_DEFAULT) == 0 || (MUTEX_DEFAULT ) == MUTEX_DEFAULT) || dtrace_assfail("(MUTEX_DEFAULT) == 0 || (MUTEX_DEFAULT) == MUTEX_DEFAULT" , "/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" , 2483))); do { if (__builtin_expect((!(((&fasttrap_provs .fth_table[i].ftb_mtx)->lock_object.lo_flags & (0x00010000 | 0x00020000 | 0x00040000 | 0x00080000 | 0x00100000 | 0x00200000 | 0x00400000 | 0x0f000000 | 0x10000000)) != (0x00010000 | 0x00020000 | 0x00080000 | 0x00100000 | 0x00200000 | 0x00400000 | (2 << 24)))), 0)) panic ("lock %s already initialized", "&fasttrap_provs.fth_table[i].ftb_mtx" ); } while (0); for (_name = "&fasttrap_provs.fth_table[i].ftb_mtx" ; *_name != '\0'; _name++) { if (*_name >= 'a' && * _name <= 'z') break; } if (*_name == '\0') _name = "&fasttrap_provs.fth_table[i].ftb_mtx" ; sx_init_flags((&fasttrap_provs.fth_table[i].ftb_mtx), _name , (0x01 | 0x40 | 0x04)); } while (0) | |||
| 2483 | "providers bucket mtx", MUTEX_DEFAULT, NULL)do { const char *_name; ((void)(((MUTEX_DEFAULT) == 0 || (MUTEX_DEFAULT ) == MUTEX_DEFAULT) || dtrace_assfail("(MUTEX_DEFAULT) == 0 || (MUTEX_DEFAULT) == MUTEX_DEFAULT" , "/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" , 2483))); do { if (__builtin_expect((!(((&fasttrap_provs .fth_table[i].ftb_mtx)->lock_object.lo_flags & (0x00010000 | 0x00020000 | 0x00040000 | 0x00080000 | 0x00100000 | 0x00200000 | 0x00400000 | 0x0f000000 | 0x10000000)) != (0x00010000 | 0x00020000 | 0x00080000 | 0x00100000 | 0x00200000 | 0x00400000 | (2 << 24)))), 0)) panic ("lock %s already initialized", "&fasttrap_provs.fth_table[i].ftb_mtx" ); } while (0); for (_name = "&fasttrap_provs.fth_table[i].ftb_mtx" ; *_name != '\0'; _name++) { if (*_name >= 'a' && * _name <= 'z') break; } if (*_name == '\0') _name = "&fasttrap_provs.fth_table[i].ftb_mtx" ; sx_init_flags((&fasttrap_provs.fth_table[i].ftb_mtx), _name , (0x01 | 0x40 | 0x04)); } while (0); | |||
| 2484 | #endif | |||
| 2485 | ||||
| 2486 | ret = kproc_create(fasttrap_pid_cleanup_cb, NULL((void *)0), | |||
| 2487 | &fasttrap_cleanup_proc, 0, 0, "ftcleanup"); | |||
| 2488 | if (ret != 0) { | |||
| 2489 | destroy_dev(fasttrap_cdev); | |||
| 2490 | #ifndef illumos | |||
| 2491 | for (i = 0; i < fasttrap_provs.fth_nent; i++) | |||
| 2492 | mutex_destroy(&fasttrap_provs.fth_table[i].ftb_mtx)sx_destroy(&fasttrap_provs.fth_table[i].ftb_mtx); | |||
| 2493 | for (i = 0; i < fasttrap_tpoints.fth_nent; i++) | |||
| 2494 | mutex_destroy(&fasttrap_tpoints.fth_table[i].ftb_mtx)sx_destroy(&fasttrap_tpoints.fth_table[i].ftb_mtx); | |||
| 2495 | #endif | |||
| 2496 | kmem_free(fasttrap_provs.fth_table, fasttrap_provs.fth_nent *zfs_kmem_free((fasttrap_provs.fth_table), (fasttrap_provs.fth_nent * sizeof (fasttrap_bucket_t))) | |||
| 2497 | sizeof (fasttrap_bucket_t))zfs_kmem_free((fasttrap_provs.fth_table), (fasttrap_provs.fth_nent * sizeof (fasttrap_bucket_t))); | |||
| 2498 | mtx_destroy(&fasttrap_cleanup_mtx)_mtx_destroy(&(&fasttrap_cleanup_mtx)->mtx_lock); | |||
| 2499 | mutex_destroy(&fasttrap_count_mtx)sx_destroy(&fasttrap_count_mtx); | |||
| 2500 | return (ret); | |||
| 2501 | } | |||
| 2502 | ||||
| 2503 | ||||
| 2504 | /* | |||
| 2505 | * ... and the procs hash table. | |||
| 2506 | */ | |||
| 2507 | nent = FASTTRAP_PROCS_DEFAULT_SIZE0x100; | |||
| 2508 | if (ISP2(nent)(((nent) & ((nent) - 1)) == 0)) | |||
| 2509 | fasttrap_procs.fth_nent = nent; | |||
| 2510 | else | |||
| 2511 | fasttrap_procs.fth_nent = 1 << fasttrap_highbit(nent); | |||
| 2512 | ASSERT(fasttrap_procs.fth_nent > 0)((void)((fasttrap_procs.fth_nent > 0) || dtrace_assfail("fasttrap_procs.fth_nent > 0" , "/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" , 2512))); | |||
| 2513 | fasttrap_procs.fth_mask = fasttrap_procs.fth_nent - 1; | |||
| 2514 | fasttrap_procs.fth_table = kmem_zalloc(fasttrap_procs.fth_nent *zfs_kmem_alloc((fasttrap_procs.fth_nent * sizeof (fasttrap_bucket_t )), (0x0002) | 0x0100) | |||
| 2515 | sizeof (fasttrap_bucket_t), KM_SLEEP)zfs_kmem_alloc((fasttrap_procs.fth_nent * sizeof (fasttrap_bucket_t )), (0x0002) | 0x0100); | |||
| 2516 | #ifndef illumos | |||
| 2517 | for (i = 0; i < fasttrap_procs.fth_nent; i++) | |||
| 2518 | mutex_init(&fasttrap_procs.fth_table[i].ftb_mtx,do { const char *_name; ((void)(((MUTEX_DEFAULT) == 0 || (MUTEX_DEFAULT ) == MUTEX_DEFAULT) || dtrace_assfail("(MUTEX_DEFAULT) == 0 || (MUTEX_DEFAULT) == MUTEX_DEFAULT" , "/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" , 2519))); do { if (__builtin_expect((!(((&fasttrap_procs .fth_table[i].ftb_mtx)->lock_object.lo_flags & (0x00010000 | 0x00020000 | 0x00040000 | 0x00080000 | 0x00100000 | 0x00200000 | 0x00400000 | 0x0f000000 | 0x10000000)) != (0x00010000 | 0x00020000 | 0x00080000 | 0x00100000 | 0x00200000 | 0x00400000 | (2 << 24)))), 0)) panic ("lock %s already initialized", "&fasttrap_procs.fth_table[i].ftb_mtx" ); } while (0); for (_name = "&fasttrap_procs.fth_table[i].ftb_mtx" ; *_name != '\0'; _name++) { if (*_name >= 'a' && * _name <= 'z') break; } if (*_name == '\0') _name = "&fasttrap_procs.fth_table[i].ftb_mtx" ; sx_init_flags((&fasttrap_procs.fth_table[i].ftb_mtx), _name , (0x01 | 0x40 | 0x04)); } while (0) | |||
| 2519 | "processes bucket mtx", MUTEX_DEFAULT, NULL)do { const char *_name; ((void)(((MUTEX_DEFAULT) == 0 || (MUTEX_DEFAULT ) == MUTEX_DEFAULT) || dtrace_assfail("(MUTEX_DEFAULT) == 0 || (MUTEX_DEFAULT) == MUTEX_DEFAULT" , "/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" , 2519))); do { if (__builtin_expect((!(((&fasttrap_procs .fth_table[i].ftb_mtx)->lock_object.lo_flags & (0x00010000 | 0x00020000 | 0x00040000 | 0x00080000 | 0x00100000 | 0x00200000 | 0x00400000 | 0x0f000000 | 0x10000000)) != (0x00010000 | 0x00020000 | 0x00080000 | 0x00100000 | 0x00200000 | 0x00400000 | (2 << 24)))), 0)) panic ("lock %s already initialized", "&fasttrap_procs.fth_table[i].ftb_mtx" ); } while (0); for (_name = "&fasttrap_procs.fth_table[i].ftb_mtx" ; *_name != '\0'; _name++) { if (*_name >= 'a' && * _name <= 'z') break; } if (*_name == '\0') _name = "&fasttrap_procs.fth_table[i].ftb_mtx" ; sx_init_flags((&fasttrap_procs.fth_table[i].ftb_mtx), _name , (0x01 | 0x40 | 0x04)); } while (0); | |||
| 2520 | ||||
| 2521 | rm_init(&fasttrap_tp_lock, "fasttrap tracepoint"); | |||
| 2522 | ||||
| 2523 | /* | |||
| 2524 | * This event handler must run before kdtrace_thread_dtor() since it | |||
| 2525 | * accesses the thread's struct kdtrace_thread. | |||
| 2526 | */ | |||
| 2527 | fasttrap_thread_dtor_tag = EVENTHANDLER_REGISTER(thread_dtor,eventhandler_register(((void *)0), "thread_dtor", fasttrap_thread_dtor , ((void *)0), 0) | |||
| 2528 | fasttrap_thread_dtor, NULL, EVENTHANDLER_PRI_FIRST)eventhandler_register(((void *)0), "thread_dtor", fasttrap_thread_dtor , ((void *)0), 0); | |||
| 2529 | #endif | |||
| 2530 | ||||
| 2531 | /* | |||
| 2532 | * Install our hooks into fork(2), exec(2), and exit(2). | |||
| 2533 | */ | |||
| 2534 | dtrace_fasttrap_fork = &fasttrap_fork; | |||
| 2535 | dtrace_fasttrap_exit = &fasttrap_exec_exit; | |||
| 2536 | dtrace_fasttrap_exec = &fasttrap_exec_exit; | |||
| 2537 | ||||
| 2538 | (void) dtrace_meta_register("fasttrap", &fasttrap_mops, NULL((void *)0), | |||
| 2539 | &fasttrap_meta_id); | |||
| 2540 | ||||
| 2541 | return (0); | |||
| 2542 | } | |||
| 2543 | ||||
| 2544 | static int | |||
| 2545 | fasttrap_unload(void) | |||
| 2546 | { | |||
| 2547 | int i, fail = 0; | |||
| 2548 | ||||
| 2549 | /* | |||
| 2550 | * Unregister the meta-provider to make sure no new fasttrap- | |||
| 2551 | * managed providers come along while we're trying to close up | |||
| 2552 | * shop. If we fail to detach, we'll need to re-register as a | |||
| 2553 | * meta-provider. We can fail to unregister as a meta-provider | |||
| 2554 | * if providers we manage still exist. | |||
| 2555 | */ | |||
| 2556 | if (fasttrap_meta_id != DTRACE_METAPROVNONE0 && | |||
| 2557 | dtrace_meta_unregister(fasttrap_meta_id) != 0) | |||
| 2558 | return (-1); | |||
| 2559 | ||||
| 2560 | /* | |||
| 2561 | * Iterate over all of our providers. If there's still a process | |||
| 2562 | * that corresponds to that pid, fail to detach. | |||
| 2563 | */ | |||
| 2564 | for (i = 0; i < fasttrap_provs.fth_nent; i++) { | |||
| 2565 | fasttrap_provider_t **fpp, *fp; | |||
| 2566 | fasttrap_bucket_t *bucket = &fasttrap_provs.fth_table[i]; | |||
| 2567 | ||||
| 2568 | mutex_enter(&bucket->ftb_mtx)(void)_sx_xlock(((&bucket->ftb_mtx)), 0, ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (2568)); | |||
| 2569 | fpp = (fasttrap_provider_t **)&bucket->ftb_data; | |||
| 2570 | while ((fp = *fpp) != NULL((void *)0)) { | |||
| 2571 | /* | |||
| 2572 | * Acquire and release the lock as a simple way of | |||
| 2573 | * waiting for any other consumer to finish with | |||
| 2574 | * this provider. A thread must first acquire the | |||
| 2575 | * bucket lock so there's no chance of another thread | |||
| 2576 | * blocking on the provider's lock. | |||
| 2577 | */ | |||
| 2578 | mutex_enter(&fp->ftp_mtx)(void)_sx_xlock(((&fp->ftp_mtx)), 0, ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (2578)); | |||
| 2579 | mutex_exit(&fp->ftp_mtx)_sx_xunlock(((&fp->ftp_mtx)), ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (2579)); | |||
| 2580 | ||||
| 2581 | if (dtrace_unregister(fp->ftp_provid) != 0) { | |||
| 2582 | fail = 1; | |||
| 2583 | fpp = &fp->ftp_next; | |||
| 2584 | } else { | |||
| 2585 | *fpp = fp->ftp_next; | |||
| 2586 | fasttrap_provider_free(fp); | |||
| 2587 | } | |||
| 2588 | } | |||
| 2589 | ||||
| 2590 | mutex_exit(&bucket->ftb_mtx)_sx_xunlock(((&bucket->ftb_mtx)), ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (2590)); | |||
| 2591 | } | |||
| 2592 | ||||
| 2593 | if (fail) { | |||
| 2594 | (void) dtrace_meta_register("fasttrap", &fasttrap_mops, NULL((void *)0), | |||
| 2595 | &fasttrap_meta_id); | |||
| 2596 | ||||
| 2597 | return (-1); | |||
| 2598 | } | |||
| 2599 | ||||
| 2600 | /* | |||
| 2601 | * Stop new processes from entering these hooks now, before the | |||
| 2602 | * fasttrap_cleanup thread runs. That way all processes will hopefully | |||
| 2603 | * be out of these hooks before we free fasttrap_provs.fth_table | |||
| 2604 | */ | |||
| 2605 | ASSERT(dtrace_fasttrap_fork == &fasttrap_fork)((void)((dtrace_fasttrap_fork == &fasttrap_fork) || dtrace_assfail ("dtrace_fasttrap_fork == &fasttrap_fork", "/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" , 2605))); | |||
| 2606 | dtrace_fasttrap_fork = NULL((void *)0); | |||
| 2607 | ||||
| 2608 | ASSERT(dtrace_fasttrap_exec == &fasttrap_exec_exit)((void)((dtrace_fasttrap_exec == &fasttrap_exec_exit) || dtrace_assfail ("dtrace_fasttrap_exec == &fasttrap_exec_exit", "/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" , 2608))); | |||
| 2609 | dtrace_fasttrap_exec = NULL((void *)0); | |||
| 2610 | ||||
| 2611 | ASSERT(dtrace_fasttrap_exit == &fasttrap_exec_exit)((void)((dtrace_fasttrap_exit == &fasttrap_exec_exit) || dtrace_assfail ("dtrace_fasttrap_exit == &fasttrap_exec_exit", "/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" , 2611))); | |||
| 2612 | dtrace_fasttrap_exit = NULL((void *)0); | |||
| 2613 | ||||
| 2614 | mtx_lock(&fasttrap_cleanup_mtx)__mtx_lock_flags(&((((&fasttrap_cleanup_mtx))))->mtx_lock , ((0)), ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (2614)); | |||
| 2615 | fasttrap_cleanup_drain = 1; | |||
| 2616 | /* Wait for the cleanup thread to finish up and signal us. */ | |||
| 2617 | wakeup(&fasttrap_cleanup_cv); | |||
| 2618 | mtx_sleep(&fasttrap_cleanup_drain, &fasttrap_cleanup_mtx, 0, "ftcld",_sleep((&fasttrap_cleanup_drain), &(&fasttrap_cleanup_mtx )->lock_object, (0), ("ftcld"), tick_sbt * (0), 0, 0x0100) | |||
| 2619 | 0)_sleep((&fasttrap_cleanup_drain), &(&fasttrap_cleanup_mtx )->lock_object, (0), ("ftcld"), tick_sbt * (0), 0, 0x0100); | |||
| 2620 | fasttrap_cleanup_proc = NULL((void *)0); | |||
| 2621 | mtx_destroy(&fasttrap_cleanup_mtx)_mtx_destroy(&(&fasttrap_cleanup_mtx)->mtx_lock); | |||
| 2622 | ||||
| 2623 | #ifdef DEBUG | |||
| 2624 | mutex_enter(&fasttrap_count_mtx)(void)_sx_xlock(((&fasttrap_count_mtx)), 0, ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (2624)); | |||
| 2625 | ASSERT(fasttrap_pid_count == 0)((void)((fasttrap_pid_count == 0) || dtrace_assfail("fasttrap_pid_count == 0" , "/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" , 2625))); | |||
| 2626 | mutex_exit(&fasttrap_count_mtx)_sx_xunlock(((&fasttrap_count_mtx)), ("/root/freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c" ), (2626)); | |||
| 2627 | #endif | |||
| 2628 | ||||
| 2629 | #ifndef illumos | |||
| 2630 | EVENTHANDLER_DEREGISTER(thread_dtor, fasttrap_thread_dtor_tag)do { struct eventhandler_list *_el; if ((_el = eventhandler_find_list ("thread_dtor")) != ((void *)0)) eventhandler_deregister(_el, fasttrap_thread_dtor_tag); } while(0); | |||
| 2631 | ||||
| 2632 | for (i = 0; i < fasttrap_tpoints.fth_nent; i++) | |||
| 2633 | mutex_destroy(&fasttrap_tpoints.fth_table[i].ftb_mtx)sx_destroy(&fasttrap_tpoints.fth_table[i].ftb_mtx); | |||
| 2634 | for (i = 0; i < fasttrap_provs.fth_nent; i++) | |||
| 2635 | mutex_destroy(&fasttrap_provs.fth_table[i].ftb_mtx)sx_destroy(&fasttrap_provs.fth_table[i].ftb_mtx); | |||
| 2636 | for (i = 0; i < fasttrap_procs.fth_nent; i++) | |||
| 2637 | mutex_destroy(&fasttrap_procs.fth_table[i].ftb_mtx)sx_destroy(&fasttrap_procs.fth_table[i].ftb_mtx); | |||
| 2638 | #endif | |||
| 2639 | kmem_free(fasttrap_tpoints.fth_table,zfs_kmem_free((fasttrap_tpoints.fth_table), (fasttrap_tpoints .fth_nent * sizeof (fasttrap_bucket_t))) | |||
| 2640 | fasttrap_tpoints.fth_nent * sizeof (fasttrap_bucket_t))zfs_kmem_free((fasttrap_tpoints.fth_table), (fasttrap_tpoints .fth_nent * sizeof (fasttrap_bucket_t))); | |||
| 2641 | fasttrap_tpoints.fth_nent = 0; | |||
| 2642 | ||||
| 2643 | kmem_free(fasttrap_provs.fth_table,zfs_kmem_free((fasttrap_provs.fth_table), (fasttrap_provs.fth_nent * sizeof (fasttrap_bucket_t))) | |||
| 2644 | fasttrap_provs.fth_nent * sizeof (fasttrap_bucket_t))zfs_kmem_free((fasttrap_provs.fth_table), (fasttrap_provs.fth_nent * sizeof (fasttrap_bucket_t))); | |||
| 2645 | fasttrap_provs.fth_nent = 0; | |||
| 2646 | ||||
| 2647 | kmem_free(fasttrap_procs.fth_table,zfs_kmem_free((fasttrap_procs.fth_table), (fasttrap_procs.fth_nent * sizeof (fasttrap_bucket_t))) | |||
| 2648 | fasttrap_procs.fth_nent * sizeof (fasttrap_bucket_t))zfs_kmem_free((fasttrap_procs.fth_table), (fasttrap_procs.fth_nent * sizeof (fasttrap_bucket_t))); | |||
| 2649 | fasttrap_procs.fth_nent = 0; | |||
| 2650 | ||||
| 2651 | #ifndef illumos | |||
| 2652 | destroy_dev(fasttrap_cdev); | |||
| 2653 | mutex_destroy(&fasttrap_count_mtx)sx_destroy(&fasttrap_count_mtx); | |||
| 2654 | rm_destroy(&fasttrap_tp_lock); | |||
| 2655 | #endif | |||
| 2656 | ||||
| 2657 | return (0); | |||
| 2658 | } | |||
| 2659 | ||||
| 2660 | /* ARGSUSED */ | |||
| 2661 | static int | |||
| 2662 | fasttrap_modevent(module_t mod __unused__attribute__((__unused__)), int type, void *data __unused__attribute__((__unused__))) | |||
| 2663 | { | |||
| 2664 | int error = 0; | |||
| 2665 | ||||
| 2666 | switch (type) { | |||
| 2667 | case MOD_LOAD: | |||
| 2668 | break; | |||
| 2669 | ||||
| 2670 | case MOD_UNLOAD: | |||
| 2671 | break; | |||
| 2672 | ||||
| 2673 | case MOD_SHUTDOWN: | |||
| 2674 | break; | |||
| 2675 | ||||
| 2676 | default: | |||
| 2677 | error = EOPNOTSUPP45; | |||
| 2678 | break; | |||
| 2679 | } | |||
| 2680 | return (error); | |||
| 2681 | } | |||
| 2682 | ||||
| 2683 | SYSINIT(fasttrap_load, SI_SUB_DTRACE_PROVIDER, SI_ORDER_ANY, fasttrap_load,static struct sysinit fasttrap_load_sys_init = { SI_SUB_DTRACE_PROVIDER , SI_ORDER_ANY, (sysinit_cfunc_t)(sysinit_nfunc_t)fasttrap_load , ((void *)(((void *)0))) }; __asm__(".globl " "__start_set_sysinit_set" ); __asm__(".globl " "__stop_set_sysinit_set"); static void const * __set_sysinit_set_sym_fasttrap_load_sys_init __attribute__ ((__section__("set_" "sysinit_set"))) __attribute__((__used__ )) = &(fasttrap_load_sys_init) | |||
| 2684 | NULL)static struct sysinit fasttrap_load_sys_init = { SI_SUB_DTRACE_PROVIDER , SI_ORDER_ANY, (sysinit_cfunc_t)(sysinit_nfunc_t)fasttrap_load , ((void *)(((void *)0))) }; __asm__(".globl " "__start_set_sysinit_set" ); __asm__(".globl " "__stop_set_sysinit_set"); static void const * __set_sysinit_set_sym_fasttrap_load_sys_init __attribute__ ((__section__("set_" "sysinit_set"))) __attribute__((__used__ )) = &(fasttrap_load_sys_init); | |||
| 2685 | SYSUNINIT(fasttrap_unload, SI_SUB_DTRACE_PROVIDER, SI_ORDER_ANY,static struct sysinit fasttrap_unload_sys_uninit = { SI_SUB_DTRACE_PROVIDER , SI_ORDER_ANY, (sysinit_cfunc_t)(sysinit_nfunc_t)fasttrap_unload , ((void *)(((void *)0))) }; __asm__(".globl " "__start_set_sysuninit_set" ); __asm__(".globl " "__stop_set_sysuninit_set"); static void const * __set_sysuninit_set_sym_fasttrap_unload_sys_uninit __attribute__ ((__section__("set_" "sysuninit_set"))) __attribute__((__used__ )) = &(fasttrap_unload_sys_uninit) | |||
| 2686 | fasttrap_unload, NULL)static struct sysinit fasttrap_unload_sys_uninit = { SI_SUB_DTRACE_PROVIDER , SI_ORDER_ANY, (sysinit_cfunc_t)(sysinit_nfunc_t)fasttrap_unload , ((void *)(((void *)0))) }; __asm__(".globl " "__start_set_sysuninit_set" ); __asm__(".globl " "__stop_set_sysuninit_set"); static void const * __set_sysuninit_set_sym_fasttrap_unload_sys_uninit __attribute__ ((__section__("set_" "sysuninit_set"))) __attribute__((__used__ )) = &(fasttrap_unload_sys_uninit); | |||
| 2687 | ||||
| 2688 | DEV_MODULE(fasttrap, fasttrap_modevent, NULL)static moduledata_t fasttrap_mod = { "fasttrap", fasttrap_modevent , ((void *)0) }; static struct mod_depend _fasttrap_depend_on_kernel __attribute__((__section__(".data"))) = { 1300006, 1300006, 1300006 }; static struct mod_metadata _mod_metadata_md_fasttrap_on_kernel = { 1, 1, &_fasttrap_depend_on_kernel, "kernel" }; __asm__ (".globl " "__start_set_modmetadata_set"); __asm__(".globl " "__stop_set_modmetadata_set" ); static void const * const __set_modmetadata_set_sym__mod_metadata_md_fasttrap_on_kernel __attribute__((__section__("set_" "modmetadata_set"))) __attribute__ ((__used__)) = &(_mod_metadata_md_fasttrap_on_kernel); static struct mod_metadata _mod_metadata_md_fasttrap = { 1, 2, & fasttrap_mod, "fasttrap" }; __asm__(".globl " "__start_set_modmetadata_set" ); __asm__(".globl " "__stop_set_modmetadata_set"); static void const * const __set_modmetadata_set_sym__mod_metadata_md_fasttrap __attribute__((__section__("set_" "modmetadata_set"))) __attribute__ ((__used__)) = &(_mod_metadata_md_fasttrap); static struct sysinit fasttrapmodule_sys_init = { SI_SUB_DRIVERS, SI_ORDER_MIDDLE , (sysinit_cfunc_t)(sysinit_nfunc_t)module_register_init, ((void *)(&fasttrap_mod)) }; __asm__(".globl " "__start_set_sysinit_set" ); __asm__(".globl " "__stop_set_sysinit_set"); static void const * __set_sysinit_set_sym_fasttrapmodule_sys_init __attribute__ ((__section__("set_" "sysinit_set"))) __attribute__((__used__ )) = &(fasttrapmodule_sys_init); struct __hack; | |||
| 2689 | MODULE_VERSION(fasttrap, 1)static struct mod_version _fasttrap_version __attribute__((__section__ (".data"))) = { 1 }; static struct mod_metadata _mod_metadata_fasttrap_version = { 1, 3, &_fasttrap_version, "fasttrap" }; __asm__(".globl " "__start_set_modmetadata_set"); __asm__(".globl " "__stop_set_modmetadata_set" ); static void const * const __set_modmetadata_set_sym__mod_metadata_fasttrap_version __attribute__((__section__("set_" "modmetadata_set"))) __attribute__ ((__used__)) = &(_mod_metadata_fasttrap_version); | |||
| 2690 | MODULE_DEPEND(fasttrap, dtrace, 1, 1, 1)static struct mod_depend _fasttrap_depend_on_dtrace __attribute__ ((__section__(".data"))) = { 1, 1, 1 }; static struct mod_metadata _mod_metadata_md_fasttrap_on_dtrace = { 1, 1, &_fasttrap_depend_on_dtrace , "dtrace" }; __asm__(".globl " "__start_set_modmetadata_set" ); __asm__(".globl " "__stop_set_modmetadata_set"); static void const * const __set_modmetadata_set_sym__mod_metadata_md_fasttrap_on_dtrace __attribute__((__section__("set_" "modmetadata_set"))) __attribute__ ((__used__)) = &(_mod_metadata_md_fasttrap_on_dtrace); | |||
| 2691 | MODULE_DEPEND(fasttrap, opensolaris, 1, 1, 1)static struct mod_depend _fasttrap_depend_on_opensolaris __attribute__ ((__section__(".data"))) = { 1, 1, 1 }; static struct mod_metadata _mod_metadata_md_fasttrap_on_opensolaris = { 1, 1, &_fasttrap_depend_on_opensolaris , "opensolaris" }; __asm__(".globl " "__start_set_modmetadata_set" ); __asm__(".globl " "__stop_set_modmetadata_set"); static void const * const __set_modmetadata_set_sym__mod_metadata_md_fasttrap_on_opensolaris __attribute__((__section__("set_" "modmetadata_set"))) __attribute__ ((__used__)) = &(_mod_metadata_md_fasttrap_on_opensolaris ); |