the way to the first working release

improve hw bp support
add -k kill option
improve book keeping error handling
support for gperftools tcmalloc
aditional fallback backtracing

backend.h

@@ -87,6 +87,9 @@ int set_hw_bp(struct task *task, unsigned int n, arch_addr_t addr);
 /* remove instruction hw breakpoint */
 int reset_hw_bp(struct task *task, unsigned int n);
 
+/* remove all instruction hw breakpoints */
+int reset_all_hw_bp(struct task *task);
+
 /* save the process context (state, registers, stack pointer) */
 int fetch_context(struct task *task);
 

breakpoint.c

@@ -114,7 +114,7 @@ static int find_hw_bp_slot(struct task *leader)
 
 static void enable_hw_bp(struct task *task, struct breakpoint *bp)
 {
-	int slot = bp->hw_bp_slot;
+	unsigned int slot = bp->hw_bp_slot;
 
 	if (bp->hw_bp_slot != HW_BP_SCRATCH_SLOT)
 		assert(task->hw_bp[slot] == NULL);
@@ -127,7 +127,7 @@ static void enable_hw_bp(struct task *task, struct breakpoint *bp)
 
 void breakpoint_hw_clone(struct task *task)
 {
-	int i;
+	unsigned int i;
 	struct task *leader = task->leader;
 
 	if (leader == task)
@@ -139,15 +139,21 @@ void breakpoint_hw_clone(struct task *task)
 			continue;
 		}
 
-		assert(leader->hw_bp[i]->hw_bp_slot == i);
+		if (leader->hw_bp[i]) {
+			assert(leader->hw_bp[i]->enabled);
+			assert(leader->hw_bp[i]->hw_bp_slot == i);
 
-		enable_hw_bp(task, leader->hw_bp[i]);
+			enable_hw_bp(task, leader->hw_bp[i]);
+		}
 	}
 }
 
 static void disable_hw_bp(struct task *task, struct breakpoint *bp)
 {
-	int slot = bp->hw_bp_slot;
+	unsigned int slot = bp->hw_bp_slot;
+
+	if (!task->hw_bp[slot])
+		return;
 
 	assert(task->hw_bp[slot] == bp);
 
@@ -159,15 +165,17 @@ static void disable_hw_bp(struct task *task, struct breakpoint *bp)
 
 void breakpoint_hw_destroy(struct task *task)
 {
-	int i;
+	unsigned int i;
 
 	for(i = 0; i < HW_BREAKPOINTS; ++i) {
 		if (task->hw_bp[i]) {
 			assert(task->hw_bp[i]->hw_bp_slot == i);
 
-			disable_hw_bp(task, task->hw_bp[i]);
+			task->hw_bp[i] = NULL;
 		}
 	}
+
+	reset_all_hw_bp(task);
 }
 
 void enable_scratch_hw_bp(struct task *task, struct breakpoint *bp)
@@ -338,7 +346,7 @@ void breakpoint_delete(struct task *task, struct breakpoint *bp)
 
 #if HW_BREAKPOINTS > 0
 	if (bp->type != SW_BP) {
-		int slot = bp->hw_bp_slot;
+		unsigned int slot = bp->hw_bp_slot;
 
 		if (bp->type == HW_BP) {
 			assert(slot != HW_BP_SCRATCH_SLOT);

breakpoint.h

@@ -42,7 +42,7 @@ struct breakpoint {
 	union {
 		unsigned char orig_value[BREAKPOINT_LENGTH];
 #if HW_BREAKPOINTS > 0
-		int hw_bp_slot;
+		unsigned int hw_bp_slot;
 #endif
 	};
 };

client/process.c

@@ -522,6 +522,15 @@ static void process_dump_stack(struct process *process, struct rb_stack *this)
 	}
 }
 
+static void process_dump_collision(struct process *process, struct rb_block *this, unsigned long addr, unsigned long size, enum mt_operation operation)
+{
+	fprintf(stderr, ">>> block collision pid:%d\n new: %s=%#lx(%lu)\n old: %s=%#lx(%lu)\n",
+		process->pid,
+		str_operation(operation), addr, size,
+		str_operation(this->stack_node->stack->operation), this->addr, this->size
+	);
+}
+
 static struct rb_block *process_rb_search_range(struct rb_root *root, unsigned long addr, unsigned long size)
 {
 	struct rb_node *node = root->rb_node;
@@ -589,7 +598,7 @@ static void process_rb_delete_block(struct process *process, struct rb_block *bl
 	free(block);
 }
 
-static int process_rb_insert_block(struct process *process, unsigned long addr, unsigned long size, struct rb_stack *stack, unsigned long flags)
+static int process_rb_insert_block(struct process *process, unsigned long addr, unsigned long size, struct rb_stack *stack, unsigned long flags, enum mt_operation operation)
 {
 	struct rb_node **new = &process->block_table.rb_node, *parent = NULL;
 	struct rb_block *block;
@@ -605,8 +614,15 @@ static int process_rb_insert_block(struct process *process, unsigned long addr,
 
 		parent = *new;
 
-		if (addr <= this->addr && addr + n > this->addr)
+		if (addr <= this->addr && addr + n > this->addr) {
+			if (options.kill || options.verbose > 1) {
+				process_dump_collision(process, this, addr, size, operation);
+
+				if (options.kill)
+					abort();
+			}
 			return -1;
+		}
 
 		if (addr < this->addr)
 			new = &((*new)->rb_left);
@@ -776,8 +792,8 @@ static int process_rb_duplicate_block(struct rb_node *node, void *user)
 	struct process *process = user;
 	struct rb_stack *stack = stack_clone(process, block->stack_node);
 
-	if (process_rb_insert_block(process, block->addr, block->size, stack, block->flags))
+	if (process_rb_insert_block(process, block->addr, block->size, stack, block->flags, block->stack_node->stack->operation))
-		abort();
+		return -1;
 
 	process->bytes_used += block->size;
 
@@ -887,24 +903,41 @@ static void _process_dump(struct process *process, int (*sortby)(const struct rb
 	unsigned long i;
 	void *data;
 
-	arr = malloc(sizeof(struct rb_stack *) * process->stack_trees);
-	if (!arr)
-		return;
-
-	for(i = 0, data = rb_first(&process->stack_table); data; data = rb_next(data))
-		arr[i++] = container_of(data, struct rb_stack, node);
-
 	if (dump_init(file) == -1)
 		return;
 
 	dump_printf("Process dump %d %s\n", process->pid, process->filename ? process->filename : "<unknown>");
 
+	if (!process->stack_trees)
+		goto skip;
+
+	arr = malloc(sizeof(struct rb_stack *) * process->stack_trees);
+	if (!arr)
+		goto skip;
+
+	for(i = 0, data = rb_first(&process->stack_table); data; data = rb_next(data))
+		arr[i++] = container_of(data, struct rb_stack, node);
+
+	assert(i == process->stack_trees);
+
 	qsort(arr, process->stack_trees, sizeof(struct stack *), (void *)sortby);
 
+	if (file == stderr) {
+		unsigned long n = process->stack_trees / 2;
+		unsigned long l = process->stack_trees - 1;
+
+		for(i = 0; i < n; ++i) {
+			struct rb_stack *tmp = arr[i];
+
+			arr[i] = arr[l - i];
+			arr[l - i] = tmp;
+		}
+	}
+
 	for(i = 0; i < process->stack_trees; ++i) {
 		struct rb_stack *stack = arr[i];
 
-		if (!skipfunc(stack) && !stack->stack->ignore) {
+		if (!stack->stack->ignore && !skipfunc(stack)) {
 			if (dump_printf(
 				"Stack (%s):\n"
 				" bytes used: %llu\n"
@@ -926,6 +959,7 @@ static void _process_dump(struct process *process, int (*sortby)(const struct rb
 		}
 	}
 	free(arr);
+skip:
 	dump_flush();
 	return;
 }
@@ -1101,8 +1135,13 @@ void process_munmap(struct process *process, struct mt_msg *mt_msg, void *payloa
 			break;
 
 		if (!is_mmap(block->stack_node->stack->operation)) {
-			fprintf(stderr, ">>> block missmatch MAP<>MALLOC %#lx found\n", ptr);
+			if (options.kill || options.verbose > 1) {
-			abort();
+				fprintf(stderr, ">>> block missmatch pid:%d MAP<>MALLOC %#lx\n", process->pid, ptr);
+
+				if (options.kill)
+					abort();
+			}
+			break;
 		}
 
 		if (block->addr >= ptr) {
@@ -1136,8 +1175,8 @@ void process_munmap(struct process *process, struct mt_msg *mt_msg, void *payloa
 
 				block->size = off;
 
-				if (process_rb_insert_block(process, new_addr, new_size, block->stack_node, 0))
+				if (process_rb_insert_block(process, new_addr, new_size, block->stack_node, 0, mt_msg->operation))
-					abort();
+					break;
 
 				process->n_allocations++;
 				process->total_allocations++;
@@ -1181,14 +1220,24 @@ void process_free(struct process *process, struct mt_msg *mt_msg, void *payload)
 	block = process_rb_search(&process->block_table, ptr);
 	if (block) {
 		if (is_mmap(block->stack_node->stack->operation)) {
-			fprintf(stderr, ">>> block missmatch MAP<>MALLOC %#lx found\n", ptr);
+			if (options.kill || options.verbose > 1) {
-			abort();
+				fprintf(stderr, ">>> block missmatch pid:%d MAP<>MALLOC %#lx\n", process->pid, ptr);
+
+				if (options.kill)
+					abort();
+			}
 		}
 		process_rb_delete_block(process, block);
 	}
 	else {
-		if (!process->attached)
+		if (!process->attached) {
-			fprintf(stderr, ">>> block %#lx not found (pid=%d, tid=%d)\n", ptr, process->pid, mt_msg->tid);
+			if (options.kill || options.verbose > 1) {
+				fprintf(stderr, ">>> block %#lx not found pid:%d tid:%d\n", ptr, process->pid, mt_msg->tid);
+
+				if (options.kill)
+					abort();
+			}
+		}
 	}
 }
 
@@ -1227,16 +1276,21 @@ void process_alloc(struct process *process, struct mt_msg *mt_msg, void *payload
 
 	debug(DEBUG_FUNCTION, "ptr=%#lx size=%lu stack_size=%lu", ptr, size, stack_size);
 
-	block = process_rb_search(&process->block_table, ptr);
+	block = process_rb_search_range(&process->block_table, ptr, size);
 	if (block) {
-		fprintf(stderr, ">>> block collison %s ptr %#lx size %lu pid %d tid %d\n", str_operation(mt_msg->operation), ptr, size, process->pid, mt_msg->tid);
+		if (options.kill || options.verbose > 1) {
-		abort();
+			process_dump_collision(process, block, ptr, size, mt_msg->operation);
+
+			if (options.kill)
+				abort();
+		}
+		process_rb_delete_block(process, block);
 	}
 
 	struct rb_stack *stack = stack_add(process, process->pid, stack_data, stack_size, mt_msg->operation);
 
-	if (process_rb_insert_block(process, ptr, size, stack, 0))
+	if (process_rb_insert_block(process, ptr, size, stack, 0, mt_msg->operation))
-		abort();
+		return;
 
 	process->total_allocations++;
 	process->bytes_used += size;

dwarf.c

@@ -873,11 +873,8 @@ static int dwarf_extract_cfi_from_fde(struct dwarf_addr_space *as, void *addrp)
 	return 0;
 }
 
-static int lib_addr_match(struct library *lib, arch_addr_t ip)
+static inline int lib_addr_match(struct library *lib, arch_addr_t ip)
 {
-	if (!lib)
-		return 0;
-
 	return ip >= lib->load_addr && ip < lib->load_addr + lib->load_size;
 }
 
@@ -886,8 +883,10 @@ int dwarf_locate_map(struct dwarf_addr_space *as, arch_addr_t ip)
 	struct task *leader;
 	struct list_head *it;
 
-	if (lib_addr_match(as->cursor.lib, ip))
+	if (as->cursor.lib) {
-		return 0;
+		if (lib_addr_match(as->cursor.lib, ip))
+			return 0;
+	}
 
 	leader = as->task->leader;
 
@@ -1885,12 +1884,13 @@ int dwarf_step(struct dwarf_addr_space *as)
 	int ret;
 	struct dwarf_cursor *c = &as->cursor;
 	struct dwarf_reg_state *rs_current;
-	arch_addr_t ip;
+	arch_addr_t ip, cfa;
 
 	if (!c->valid)
 		return -DWARF_EINVAL;
 
 	ip = c->ip;
+	cfa = c->cfa;
 
 	/* The 'ip' can point either to the previous or next instruction
 	   depending on what type of frame we have: normal call or a place
@@ -1935,10 +1935,27 @@ fail:
 		debug(DEBUG_DWARF, "try arch specific step");
 
 		ret = dwarf_arch_step(as);
-		if (!ret)
+		if (!ret) {
-			ret = dwarf_locate_map(as, c->use_prev_instr ? c->ip - 1 : c->ip);
+			if (dwarf_locate_map(as, c->use_prev_instr ? c->ip - 1 : c->ip))
+				ret = -DWARF_ENOINFO;
+		}
 	}
+
 	if (ret) {
+		unsigned int i;
+
+		for(i = 0; i < 16; ++i) {
+			if (dwarf_readw(as, &cfa, &ip, as->task->is_64bit))
+				break;
+
+			if (!dwarf_locate_map(as, ip)) {
+				c->cfa = cfa;
+				c->ip = ip;
+
+				return 0;
+			}
+		}
+
 		debug(DEBUG_DWARF, "error %d", ret);
 
 		c->valid = 0;

options.c

@@ -90,6 +90,7 @@ static void usage(void)
 		" -f, --follow-child  trace forked children\n"
 		" -h, --help          display this help and exit\n"
 		" -i, --interactive   interactive client mode\n"
+		" -k, --kill          abort mtrace on unexpected error conditon\n"
 		" -l, --listen        listen on socket path or address in server mode\n"
 		" -o, --output=FILE   write the trace output to file with given name\n"
 		" -p, --pid=PID       attach to the process with the process ID pid (may be repeated)\n"
@@ -257,6 +258,7 @@ char **process_options(int argc, char **argv)
 	options.opt_b = NULL;
 	options.sort_by = -1;
 	options.debug = 0;
+	options.kill = 0;
 
 	for(;;) {
 		int c;
@@ -273,6 +275,7 @@ char **process_options(int argc, char **argv)
 			{ "follow-child", 0, 0, 'f'},
 			{ "follow-exec", 0, 0, 'e' },
 			{ "interactive", 0, 0, 'i' },
+			{ "kill", 0, 0, 'k' },
 			{ "listen", 1, 0, 'l' },
 			{ "output", 1, 0, 'o' },
 			{ "pid", 1, 0, 'p' },
@@ -286,7 +289,7 @@ char **process_options(int argc, char **argv)
 			{ 0, 0, 0, 0 }
 		};
 		c = getopt_long(argc, argv,
-				"+aefhisVvw"
+				"+aefhiksVvw"
 				"b:c:C:D:F:l:o:p:P:u:d:S:",
 				long_options,
 				&option_index);
@@ -412,6 +415,9 @@ char **process_options(int argc, char **argv)
 		case 'e':
 			options.follow_exec = 1;
 			break;
+		case 'k':
+			options.kill = 1;
+			break;
 		case 'v':
 			options.verbose++;
 			break;

options.h

@@ -65,6 +65,7 @@ struct options_t {
 	int interactive;	/* interactive mode */
 	FILE *output;		/* output to a specific file */
 	int server;		/* server mode flag */
+	int kill;		/* kill on errors */
 	char *listen;		/* server listen on socket path or address */
 	char *client;		/* connect to socket path or address */
 	char *user;		/* -u: username to run command as */

report.c

@@ -307,6 +307,13 @@ static int _report_pvalloc(struct task *task, struct library_symbol *libsym)
 	return report_alloc(task, MT_PVALLOC, ret, len, options.bt_depth);
 }
 
+static int report_ni(struct task *task, struct library_symbol *libsym)
+{
+	fprintf(stderr, "%s not implemented!!!\n", libsym->func->name);
+
+	return 0;
+}
+
 static const struct function flist[] = {
 	{ "malloc",		2,	NULL,		_report_malloc },
 	{ "free",		3,	report_free,	NULL },
@@ -320,8 +327,24 @@ static const struct function flist[] = {
 	{ "aligned_alloc",	11,	NULL,		_report_aligned_alloc },
 	{ "valloc",		12,	NULL,		_report_valloc },
 	{ "pvalloc",		13,	NULL,		_report_pvalloc },
+	{ "mremap",		14,	report_ni,	NULL },
 #if 0
 	{ "cfree",		14,	report_free,	NULL },
+	{ "vfree",		14,	report_free,	NULL },
+#endif
+#if 1
+	/*
+	 * support for Google gperftools
+	 * the c++ operators new and delete do not call malloc() and free()
+	 */
+	{ "tc_delete",			14,	report_free,	NULL },
+	{ "tc_deletearray",		14,	report_free,	NULL },
+	{ "tc_deletearray_nothrow",	14,	report_free,	NULL },
+	{ "tc_delete_nothrow",		14,	report_free,	NULL },
+	{ "tc_new",			14,	NULL,	_report_malloc },
+	{ "tc_newarray",		14,	NULL,	_report_malloc },
+	{ "tc_newarray_nothrow",	14,	NULL,	_report_malloc },
+	{ "tc_new_nothrow",		14,	NULL,	_report_malloc },
 #endif
 };
 

sysdeps/linux-gnu/trace.c

@@ -219,7 +219,7 @@ static void process_event(struct task *task, int status)
 	int stop_signal;
 	struct task *leader = task->leader;
 	struct breakpoint *bp = NULL;
-	int i;
+	unsigned int i;
 	arch_addr_t ip;
 
 	assert(leader != NULL);

sysdeps/linux-gnu/x86/arch.c

@@ -31,15 +31,37 @@
 #include "task.h"
 
 /* Breakpoint access modes */
-enum {
+#define	BP_X	1
-	BP_X = 1,
+#define BP_RW	2
-	BP_RW = 2,
+#define BP_W	4
-	BP_W = 4,
-};
 
-static int set_breakpoint_addr(struct task *task, arch_addr_t addr, int n)
+static int _apply_hw_bp(struct task *task, uint32_t dr7)
 {
-	int ret;
+	long ret;
+
+	task->arch.dr7 = dr7;
+
+	ret = ptrace(PTRACE_POKEUSER, task->pid, offsetof(struct user, u_debugreg[7]), task->arch.dr7);
+	if (ret) {
+		if (errno != ESRCH) {
+			fprintf(stderr, "PTRACE_POKEUSER u_debugreg[7] pid=%d %s\n", task->pid, strerror(errno));
+			return -1;
+		}
+	}
+	return 0;
+}
+
+static inline int apply_hw_bp(struct task *task, uint32_t dr7)
+{
+	if (dr7 == task->arch.dr7)
+		return 0;
+
+	return _apply_hw_bp(task, dr7);
+}
+
+static int set_breakpoint_addr(struct task *task, arch_addr_t addr, unsigned int n)
+{
+	long ret;
 
 #ifdef __x86_64__
 	if (!task->is_64bit)
@@ -56,9 +78,8 @@ static int set_breakpoint_addr(struct task *task, arch_addr_t addr, int n)
 	return 0;
 }
 
-static int set_breakpoint_mode(struct task *task, int n, int type, int len, int local, int global)
+static int set_breakpoint_mode(struct task *task, unsigned int n, int type, int len, int local, int global)
 {
-	long ret;
 	uint32_t mode;
 	uint32_t dr7, mask;
 
@@ -114,25 +135,15 @@ static int set_breakpoint_mode(struct task *task, int n, int type, int len, int
 	if (!(dr7 & 0b10101010))
 		dr7 &= ~(1 << 9);
 
-	if (dr7 != task->arch.dr7) {
+	return apply_hw_bp(task, dr7);
-		task->arch.dr7 = dr7;
-
-		ret = ptrace(PTRACE_POKEUSER, task->pid, offsetof(struct user, u_debugreg[7]), task->arch.dr7);
-		if (ret) {
-			if (errno != ESRCH) {
-				fprintf(stderr, "PTRACE_POKEUSER u_debugreg[7] pid=%d %s\n", task->pid, strerror(errno));
-				return -1;
-			}
-		}
-	}
-	return 0;
 }
 
 int set_hw_bp(struct task *task, unsigned int n, arch_addr_t addr)
 {
+#if 0
 	if (reset_hw_bp(task, n) == -1)
 		return -1;
-
+#endif
 	if (set_breakpoint_addr(task, addr, n) == -1)
 		return -1;
 
@@ -147,7 +158,6 @@ int set_hw_bp(struct task *task, unsigned int n, arch_addr_t addr)
 
 int reset_hw_bp(struct task *task, unsigned int n)
 {
-	long ret;
 	uint32_t dr7, mask;
 
 	mask = (0b1111 << (16 + 4 * n)) | (0b11 << (2 * n));
@@ -160,18 +170,12 @@ int reset_hw_bp(struct task *task, unsigned int n)
 	if (!(dr7 & 0b10101010))
 		dr7 &= ~(1 << 9);
 
-	if (dr7 != task->arch.dr7) {
+	return apply_hw_bp(task, dr7);
-		task->arch.dr7 = dr7;
+}
 
-		ret = ptrace(PTRACE_POKEUSER, task->pid, offsetof(struct user, u_debugreg[7]), task->arch.dr7);
+int reset_all_hw_bp(struct task *task)
-		if (ret) {
+{
-			if (errno != ESRCH) {
+	return apply_hw_bp(task, 0);
-				fprintf(stderr, "PTRACE_POKEUSER u_debugreg[7] pid=%d %s\n", task->pid, strerror(errno));
-				return -1;
-			}
-		}
-	}
-	return 0;
 }
 
 int is_64bit(struct mt_elf *mte)
@@ -181,28 +185,12 @@ int is_64bit(struct mt_elf *mte)
 
 int arch_task_init(struct task *task)
 {
-	long ret;
+	return _apply_hw_bp(task, 0);
-	int i;
-
-	ret = ptrace(PTRACE_PEEKUSER, task->pid, offsetof(struct user, u_debugreg[7]), 0);
-	if (ret == -1 && errno) {
-		if (errno != ESRCH) {
-			fatal("PTRACE_PEEKUSER u_debugreg[7] pid=%d %s\n", task->pid, strerror(errno));
-			return -1;
-		}
-		return 0;
-	}
-
-	task->arch.dr7 = ret;
-
-	for(i = 0; i < HW_BREAKPOINTS; ++i)
-		reset_hw_bp(task, i);
-
-	return 0;
 }
 
 void arch_task_destroy(struct task *task)
 {
+	apply_hw_bp(task, 0);
 }
 
 int arch_task_clone(struct task *retp, struct task *task)

task.c

@@ -43,7 +43,6 @@
 #include "library.h"
 #include "mtelf.h"
 #include "report.h"
-#include "server.h"
 #include "task.h"
 #include "trace.h"
 
@@ -199,7 +198,6 @@ static void leader_cleanup(struct task *task)
 
 static void task_destroy(struct task *task)
 {
-	breakpoint_hw_destroy(task);
 	backtrace_destroy(task);
 	arch_task_destroy(task);
 	os_task_destroy(task);
@@ -323,8 +321,7 @@ int task_clone(struct task *task, struct task *newtask)
 	if (backtrace_init(newtask) < 0)
 		goto fail;
 
-	if (server_connected())
+	breakpoint_hw_clone(newtask);
-		breakpoint_hw_clone(newtask);
 
 	return 0;
 fail: