[GitHub/mt8127/android_kernel_alcatel_ttab.git] / kernel / trace / trace_syscalls.c

#include <trace/syscall.h>
#include <trace/events/syscalls.h>
#include <linux/kernel.h>
#include <linux/ftrace.h>
#include <linux/perf_event.h>
#include <asm/syscall.h>

#include "trace_output.h"
#include "trace.h"

static DEFINE_MUTEX(syscall_trace_lock);
static int sys_refcount_enter;
static int sys_refcount_exit;
static DECLARE_BITMAP(enabled_enter_syscalls, NR_syscalls);
static DECLARE_BITMAP(enabled_exit_syscalls, NR_syscalls);

extern unsigned long __start_syscalls_metadata[];
extern unsigned long __stop_syscalls_metadata[];

static struct syscall_metadata **syscalls_metadata;

static struct syscall_metadata *find_syscall_meta(unsigned long syscall)
{
	struct syscall_metadata *start;
	struct syscall_metadata *stop;
	char str[KSYM_SYMBOL_LEN];


	start = (struct syscall_metadata *)__start_syscalls_metadata;
	stop = (struct syscall_metadata *)__stop_syscalls_metadata;
	kallsyms_lookup(syscall, NULL, NULL, NULL, str);

	for ( ; start < stop; start++) {
		/*
		 * Only compare after the "sys" prefix. Archs that use
		 * syscall wrappers may have syscalls symbols aliases prefixed
		 * with "SyS" instead of "sys", leading to an unwanted
		 * mismatch.
		 */
		if (start->name && !strcmp(start->name + 3, str + 3))
			return start;
	}
	return NULL;
}

static struct syscall_metadata *syscall_nr_to_meta(int nr)
{
	if (!syscalls_metadata || nr >= NR_syscalls || nr < 0)
		return NULL;

	return syscalls_metadata[nr];
}

enum print_line_t
print_syscall_enter(struct trace_iterator *iter, int flags)
{
	struct trace_seq *s = &iter->seq;
	struct trace_entry *ent = iter->ent;
	struct syscall_trace_enter *trace;
	struct syscall_metadata *entry;
	int i, ret, syscall;

	trace = (typeof(trace))ent;
	syscall = trace->nr;
	entry = syscall_nr_to_meta(syscall);

	if (!entry)
		goto end;

	if (entry->enter_event->id != ent->type) {
		WARN_ON_ONCE(1);
		goto end;
	}

	ret = trace_seq_printf(s, "%s(", entry->name);
	if (!ret)
		return TRACE_TYPE_PARTIAL_LINE;

	for (i = 0; i < entry->nb_args; i++) {
		/* parameter types */
		if (trace_flags & TRACE_ITER_VERBOSE) {
			ret = trace_seq_printf(s, "%s ", entry->types[i]);
			if (!ret)
				return TRACE_TYPE_PARTIAL_LINE;
		}
		/* parameter values */
		ret = trace_seq_printf(s, "%s: %lx%s", entry->args[i],
				       trace->args[i],
				       i == entry->nb_args - 1 ? "" : ", ");
		if (!ret)
			return TRACE_TYPE_PARTIAL_LINE;
	}

	ret = trace_seq_putc(s, ')');
	if (!ret)
		return TRACE_TYPE_PARTIAL_LINE;

end:
	ret =  trace_seq_putc(s, '\n');
	if (!ret)
		return TRACE_TYPE_PARTIAL_LINE;

	return TRACE_TYPE_HANDLED;
}

enum print_line_t
print_syscall_exit(struct trace_iterator *iter, int flags)
{
	struct trace_seq *s = &iter->seq;
	struct trace_entry *ent = iter->ent;
	struct syscall_trace_exit *trace;
	int syscall;
	struct syscall_metadata *entry;
	int ret;

	trace = (typeof(trace))ent;
	syscall = trace->nr;
	entry = syscall_nr_to_meta(syscall);

	if (!entry) {
		trace_seq_printf(s, "\n");
		return TRACE_TYPE_HANDLED;
	}

	if (entry->exit_event->id != ent->type) {
		WARN_ON_ONCE(1);
		return TRACE_TYPE_UNHANDLED;
	}

	ret = trace_seq_printf(s, "%s -> 0x%lx\n", entry->name,
				trace->ret);
	if (!ret)
		return TRACE_TYPE_PARTIAL_LINE;

	return TRACE_TYPE_HANDLED;
}

extern char *__bad_type_size(void);

#define SYSCALL_FIELD(type, name)					\
	sizeof(type) != sizeof(trace.name) ?				\
		__bad_type_size() :					\
		#type, #name, offsetof(typeof(trace), name),		\
		sizeof(trace.name), is_signed_type(type)

static
int  __set_enter_print_fmt(struct syscall_metadata *entry, char *buf, int len)
{
	int i;
	int pos = 0;

	/* When len=0, we just calculate the needed length */
#define LEN_OR_ZERO (len ? len - pos : 0)

	pos += snprintf(buf + pos, LEN_OR_ZERO, "\"");
	for (i = 0; i < entry->nb_args; i++) {
		pos += snprintf(buf + pos, LEN_OR_ZERO, "%s: 0x%%0%zulx%s",
				entry->args[i], sizeof(unsigned long),
				i == entry->nb_args - 1 ? "" : ", ");
	}
	pos += snprintf(buf + pos, LEN_OR_ZERO, "\"");

	for (i = 0; i < entry->nb_args; i++) {
		pos += snprintf(buf + pos, LEN_OR_ZERO,
				", ((unsigned long)(REC->%s))", entry->args[i]);
	}

#undef LEN_OR_ZERO

	/* return the length of print_fmt */
	return pos;
}

static int set_syscall_print_fmt(struct ftrace_event_call *call)
{
	char *print_fmt;
	int len;
	struct syscall_metadata *entry = call->data;

	if (entry->enter_event != call) {
		call->print_fmt = "\"0x%lx\", REC->ret";
		return 0;
	}

	/* First: called with 0 length to calculate the needed length */
	len = __set_enter_print_fmt(entry, NULL, 0);

	print_fmt = kmalloc(len + 1, GFP_KERNEL);
	if (!print_fmt)
		return -ENOMEM;

	/* Second: actually write the @print_fmt */
	__set_enter_print_fmt(entry, print_fmt, len + 1);
	call->print_fmt = print_fmt;

	return 0;
}

static void free_syscall_print_fmt(struct ftrace_event_call *call)
{
	struct syscall_metadata *entry = call->data;

	if (entry->enter_event == call)
		kfree(call->print_fmt);
}

int syscall_enter_define_fields(struct ftrace_event_call *call)
{
	struct syscall_trace_enter trace;
	struct syscall_metadata *meta = call->data;
	int ret;
	int i;
	int offset = offsetof(typeof(trace), args);

	ret = trace_define_field(call, SYSCALL_FIELD(int, nr), FILTER_OTHER);
	if (ret)
		return ret;

	for (i = 0; i < meta->nb_args; i++) {
		ret = trace_define_field(call, meta->types[i],
					 meta->args[i], offset,
					 sizeof(unsigned long), 0,
					 FILTER_OTHER);
		offset += sizeof(unsigned long);
	}

	return ret;
}

int syscall_exit_define_fields(struct ftrace_event_call *call)
{
	struct syscall_trace_exit trace;
	int ret;

	ret = trace_define_field(call, SYSCALL_FIELD(int, nr), FILTER_OTHER);
	if (ret)
		return ret;

	ret = trace_define_field(call, SYSCALL_FIELD(long, ret),
				 FILTER_OTHER);

	return ret;
}

void ftrace_syscall_enter(struct pt_regs *regs, long id)
{
	struct syscall_trace_enter *entry;
	struct syscall_metadata *sys_data;
	struct ring_buffer_event *event;
	struct ring_buffer *buffer;
	int size;
	int syscall_nr;

	syscall_nr = syscall_get_nr(current, regs);
	if (syscall_nr < 0)
		return;
	if (!test_bit(syscall_nr, enabled_enter_syscalls))
		return;

	sys_data = syscall_nr_to_meta(syscall_nr);
	if (!sys_data)
		return;

	size = sizeof(*entry) + sizeof(unsigned long) * sys_data->nb_args;

	event = trace_current_buffer_lock_reserve(&buffer,
			sys_data->enter_event->id, size, 0, 0);
	if (!event)
		return;

	entry = ring_buffer_event_data(event);
	entry->nr = syscall_nr;
	syscall_get_arguments(current, regs, 0, sys_data->nb_args, entry->args);

	if (!filter_current_check_discard(buffer, sys_data->enter_event,
					  entry, event))
		trace_current_buffer_unlock_commit(buffer, event, 0, 0);
}

void ftrace_syscall_exit(struct pt_regs *regs, long ret)
{
	struct syscall_trace_exit *entry;
	struct syscall_metadata *sys_data;
	struct ring_buffer_event *event;
	struct ring_buffer *buffer;
	int syscall_nr;

	syscall_nr = syscall_get_nr(current, regs);
	if (syscall_nr < 0)
		return;
	if (!test_bit(syscall_nr, enabled_exit_syscalls))
		return;

	sys_data = syscall_nr_to_meta(syscall_nr);
	if (!sys_data)
		return;

	event = trace_current_buffer_lock_reserve(&buffer,
			sys_data->exit_event->id, sizeof(*entry), 0, 0);
	if (!event)
		return;

	entry = ring_buffer_event_data(event);
	entry->nr = syscall_nr;
	entry->ret = syscall_get_return_value(current, regs);

	if (!filter_current_check_discard(buffer, sys_data->exit_event,
					  entry, event))
		trace_current_buffer_unlock_commit(buffer, event, 0, 0);
}

int reg_event_syscall_enter(struct ftrace_event_call *call)
{
	int ret = 0;
	int num;

	num = ((struct syscall_metadata *)call->data)->syscall_nr;
	if (num < 0 || num >= NR_syscalls)
		return -ENOSYS;
	mutex_lock(&syscall_trace_lock);
	if (!sys_refcount_enter)
		ret = register_trace_sys_enter(ftrace_syscall_enter);
	if (!ret) {
		set_bit(num, enabled_enter_syscalls);
		sys_refcount_enter++;
	}
	mutex_unlock(&syscall_trace_lock);
	return ret;
}

void unreg_event_syscall_enter(struct ftrace_event_call *call)
{
	int num;

	num = ((struct syscall_metadata *)call->data)->syscall_nr;
	if (num < 0 || num >= NR_syscalls)
		return;
	mutex_lock(&syscall_trace_lock);
	sys_refcount_enter--;
	clear_bit(num, enabled_enter_syscalls);
	if (!sys_refcount_enter)
		unregister_trace_sys_enter(ftrace_syscall_enter);
	mutex_unlock(&syscall_trace_lock);
}

int reg_event_syscall_exit(struct ftrace_event_call *call)
{
	int ret = 0;
	int num;

	num = ((struct syscall_metadata *)call->data)->syscall_nr;
	if (num < 0 || num >= NR_syscalls)
		return -ENOSYS;
	mutex_lock(&syscall_trace_lock);
	if (!sys_refcount_exit)
		ret = register_trace_sys_exit(ftrace_syscall_exit);
	if (!ret) {
		set_bit(num, enabled_exit_syscalls);
		sys_refcount_exit++;
	}
	mutex_unlock(&syscall_trace_lock);
	return ret;
}

void unreg_event_syscall_exit(struct ftrace_event_call *call)
{
	int num;

	num = ((struct syscall_metadata *)call->data)->syscall_nr;
	if (num < 0 || num >= NR_syscalls)
		return;
	mutex_lock(&syscall_trace_lock);
	sys_refcount_exit--;
	clear_bit(num, enabled_exit_syscalls);
	if (!sys_refcount_exit)
		unregister_trace_sys_exit(ftrace_syscall_exit);
	mutex_unlock(&syscall_trace_lock);
}

int init_syscall_trace(struct ftrace_event_call *call)
{
	int id;

	if (set_syscall_print_fmt(call) < 0)
		return -ENOMEM;

	id = trace_event_raw_init(call);

	if (id < 0) {
		free_syscall_print_fmt(call);
		return id;
	}

	return id;
}

int __init init_ftrace_syscalls(void)
{
	struct syscall_metadata *meta;
	unsigned long addr;
	int i;

	syscalls_metadata = kzalloc(sizeof(*syscalls_metadata) *
					NR_syscalls, GFP_KERNEL);
	if (!syscalls_metadata) {
		WARN_ON(1);
		return -ENOMEM;
	}

	for (i = 0; i < NR_syscalls; i++) {
		addr = arch_syscall_addr(i);
		meta = find_syscall_meta(addr);
		if (!meta)
			continue;

		meta->syscall_nr = i;
		syscalls_metadata[i] = meta;
	}

	return 0;
}
core_initcall(init_ftrace_syscalls);

#ifdef CONFIG_EVENT_PROFILE

static DECLARE_BITMAP(enabled_prof_enter_syscalls, NR_syscalls);
static DECLARE_BITMAP(enabled_prof_exit_syscalls, NR_syscalls);
static int sys_prof_refcount_enter;
static int sys_prof_refcount_exit;

static void prof_syscall_enter(struct pt_regs *regs, long id)
{
	struct syscall_metadata *sys_data;
	struct syscall_trace_enter *rec;
	unsigned long flags;
	char *trace_buf;
	char *raw_data;
	int syscall_nr;
	int rctx;
	int size;
	int cpu;

	syscall_nr = syscall_get_nr(current, regs);
	if (!test_bit(syscall_nr, enabled_prof_enter_syscalls))
		return;

	sys_data = syscall_nr_to_meta(syscall_nr);
	if (!sys_data)
		return;

	/* get the size after alignment with the u32 buffer size field */
	size = sizeof(unsigned long) * sys_data->nb_args + sizeof(*rec);
	size = ALIGN(size + sizeof(u32), sizeof(u64));
	size -= sizeof(u32);

	if (WARN_ONCE(size > FTRACE_MAX_PROFILE_SIZE,
		      "profile buffer not large enough"))
		return;

	/* Protect the per cpu buffer, begin the rcu read side */
	local_irq_save(flags);

	rctx = perf_swevent_get_recursion_context();
	if (rctx < 0)
		goto end_recursion;

	cpu = smp_processor_id();

	trace_buf = rcu_dereference(perf_trace_buf);

	if (!trace_buf)
		goto end;

	raw_data = per_cpu_ptr(trace_buf, cpu);

	/* zero the dead bytes from align to not leak stack to user */
	*(u64 *)(&raw_data[size - sizeof(u64)]) = 0ULL;

	rec = (struct syscall_trace_enter *) raw_data;
	tracing_generic_entry_update(&rec->ent, 0, 0);
	rec->ent.type = sys_data->enter_event->id;
	rec->nr = syscall_nr;
	syscall_get_arguments(current, regs, 0, sys_data->nb_args,
			       (unsigned long *)&rec->args);
	perf_tp_event(sys_data->enter_event->id, 0, 1, rec, size);

end:
	perf_swevent_put_recursion_context(rctx);
end_recursion:
	local_irq_restore(flags);
}

int prof_sysenter_enable(struct ftrace_event_call *call)
{
	int ret = 0;
	int num;

	num = ((struct syscall_metadata *)call->data)->syscall_nr;

	mutex_lock(&syscall_trace_lock);
	if (!sys_prof_refcount_enter)
		ret = register_trace_sys_enter(prof_syscall_enter);
	if (ret) {
		pr_info("event trace: Could not activate"
				"syscall entry trace point");
	} else {
		set_bit(num, enabled_prof_enter_syscalls);
		sys_prof_refcount_enter++;
	}
	mutex_unlock(&syscall_trace_lock);
	return ret;
}

void prof_sysenter_disable(struct ftrace_event_call *call)
{
	int num;

	num = ((struct syscall_metadata *)call->data)->syscall_nr;

	mutex_lock(&syscall_trace_lock);
	sys_prof_refcount_enter--;
	clear_bit(num, enabled_prof_enter_syscalls);
	if (!sys_prof_refcount_enter)
		unregister_trace_sys_enter(prof_syscall_enter);
	mutex_unlock(&syscall_trace_lock);
}

static void prof_syscall_exit(struct pt_regs *regs, long ret)
{
	struct syscall_metadata *sys_data;
	struct syscall_trace_exit *rec;
	unsigned long flags;
	int syscall_nr;
	char *trace_buf;
	char *raw_data;
	int rctx;
	int size;
	int cpu;

	syscall_nr = syscall_get_nr(current, regs);
	if (!test_bit(syscall_nr, enabled_prof_exit_syscalls))
		return;

	sys_data = syscall_nr_to_meta(syscall_nr);
	if (!sys_data)
		return;

	/* We can probably do that at build time */
	size = ALIGN(sizeof(*rec) + sizeof(u32), sizeof(u64));
	size -= sizeof(u32);

	/*
	 * Impossible, but be paranoid with the future
	 * How to put this check outside runtime?
	 */
	if (WARN_ONCE(size > FTRACE_MAX_PROFILE_SIZE,
		"exit event has grown above profile buffer size"))
		return;

	/* Protect the per cpu buffer, begin the rcu read side */
	local_irq_save(flags);

	rctx = perf_swevent_get_recursion_context();
	if (rctx < 0)
		goto end_recursion;

	cpu = smp_processor_id();

	trace_buf = rcu_dereference(perf_trace_buf);

	if (!trace_buf)
		goto end;

	raw_data = per_cpu_ptr(trace_buf, cpu);

	/* zero the dead bytes from align to not leak stack to user */
	*(u64 *)(&raw_data[size - sizeof(u64)]) = 0ULL;

	rec = (struct syscall_trace_exit *)raw_data;

	tracing_generic_entry_update(&rec->ent, 0, 0);
	rec->ent.type = sys_data->exit_event->id;
	rec->nr = syscall_nr;
	rec->ret = syscall_get_return_value(current, regs);

	perf_tp_event(sys_data->exit_event->id, 0, 1, rec, size);

end:
	perf_swevent_put_recursion_context(rctx);
end_recursion:
	local_irq_restore(flags);
}

int prof_sysexit_enable(struct ftrace_event_call *call)
{
	int ret = 0;
	int num;

	num = ((struct syscall_metadata *)call->data)->syscall_nr;

	mutex_lock(&syscall_trace_lock);
	if (!sys_prof_refcount_exit)
		ret = register_trace_sys_exit(prof_syscall_exit);
	if (ret) {
		pr_info("event trace: Could not activate"
				"syscall entry trace point");
	} else {
		set_bit(num, enabled_prof_exit_syscalls);
		sys_prof_refcount_exit++;
	}
	mutex_unlock(&syscall_trace_lock);
	return ret;
}

void prof_sysexit_disable(struct ftrace_event_call *call)
{
	int num;

	num = ((struct syscall_metadata *)call->data)->syscall_nr;

	mutex_lock(&syscall_trace_lock);
	sys_prof_refcount_exit--;
	clear_bit(num, enabled_prof_exit_syscalls);
	if (!sys_prof_refcount_exit)
		unregister_trace_sys_exit(prof_syscall_exit);
	mutex_unlock(&syscall_trace_lock);
}

#endif
Commit	Line	Data
47788c58	1	#include <trace/syscall.h>
1c569f02	2	#include <trace/events/syscalls.h>
ee08c6ec	3	#include <linux/kernel.h>
fb34a08c	4	#include <linux/ftrace.h>
cdd6c482	5	#include <linux/perf_event.h>
ee08c6ec FW	6	#include <asm/syscall.h>
	7
	8	#include "trace_output.h"
	9	#include "trace.h"
	10
5be71b61	11	static DEFINE_MUTEX(syscall_trace_lock);
fb34a08c JB	12	static int sys_refcount_enter;
fb34a08c JB	13	static int sys_refcount_exit;
57421dbb JB	14	static DECLARE_BITMAP(enabled_enter_syscalls, NR_syscalls);
57421dbb JB	15	static DECLARE_BITMAP(enabled_exit_syscalls, NR_syscalls);
ee08c6ec	16
c44fc770 FW	17	extern unsigned long __start_syscalls_metadata[];
	18	extern unsigned long __stop_syscalls_metadata[];
	19
	20	static struct syscall_metadata **syscalls_metadata;
	21
	22	static struct syscall_metadata *find_syscall_meta(unsigned long syscall)
	23	{
	24	struct syscall_metadata *start;
	25	struct syscall_metadata *stop;
	26	char str[KSYM_SYMBOL_LEN];
	27
	28
	29	start = (struct syscall_metadata *)__start_syscalls_metadata;
	30	stop = (struct syscall_metadata *)__stop_syscalls_metadata;
	31	kallsyms_lookup(syscall, NULL, NULL, NULL, str);
	32
	33	for ( ; start < stop; start++) {
	34	/*
	35	* Only compare after the "sys" prefix. Archs that use
	36	* syscall wrappers may have syscalls symbols aliases prefixed
	37	* with "SyS" instead of "sys", leading to an unwanted
	38	* mismatch.
	39	*/
	40	if (start->name && !strcmp(start->name + 3, str + 3))
	41	return start;
	42	}
	43	return NULL;
	44	}
	45
	46	static struct syscall_metadata *syscall_nr_to_meta(int nr)
	47	{
	48	if (!syscalls_metadata \|\| nr >= NR_syscalls \|\| nr < 0)
	49	return NULL;
	50
	51	return syscalls_metadata[nr];
	52	}
	53
bed1ffca FW	54	enum print_line_t
	55	print_syscall_enter(struct trace_iterator *iter, int flags)
	56	{
	57	struct trace_seq *s = &iter->seq;
	58	struct trace_entry *ent = iter->ent;
	59	struct syscall_trace_enter *trace;
	60	struct syscall_metadata *entry;
	61	int i, ret, syscall;
	62
64c12e04	63	trace = (typeof(trace))ent;
bed1ffca	64	syscall = trace->nr;
bed1ffca	65	entry = syscall_nr_to_meta(syscall);
64c12e04	66
bed1ffca FW	67	if (!entry)
	68	goto end;
	69
fcc19438	70	if (entry->enter_event->id != ent->type) {
64c12e04 JB	71	WARN_ON_ONCE(1);
	72	goto end;
	73	}
	74
bed1ffca FW	75	ret = trace_seq_printf(s, "%s(", entry->name);
	76	if (!ret)
	77	return TRACE_TYPE_PARTIAL_LINE;
	78
	79	for (i = 0; i < entry->nb_args; i++) {
	80	/* parameter types */
ba8b3a40	81	if (trace_flags & TRACE_ITER_VERBOSE) {
bed1ffca FW	82	ret = trace_seq_printf(s, "%s ", entry->types[i]);
	83	if (!ret)
	84	return TRACE_TYPE_PARTIAL_LINE;
	85	}
	86	/* parameter values */
4539f077	87	ret = trace_seq_printf(s, "%s: %lx%s", entry->args[i],
bed1ffca	88	trace->args[i],
4539f077	89	i == entry->nb_args - 1 ? "" : ", ");
bed1ffca FW	90	if (!ret)
	91	return TRACE_TYPE_PARTIAL_LINE;
	92	}
	93
4539f077 LZ	94	ret = trace_seq_putc(s, ')');
	95	if (!ret)
	96	return TRACE_TYPE_PARTIAL_LINE;
	97
bed1ffca	98	end:
4539f077 LZ	99	ret = trace_seq_putc(s, '\n');
	100	if (!ret)
	101	return TRACE_TYPE_PARTIAL_LINE;
	102
bed1ffca FW	103	return TRACE_TYPE_HANDLED;
	104	}
	105
	106	enum print_line_t
	107	print_syscall_exit(struct trace_iterator *iter, int flags)
	108	{
	109	struct trace_seq *s = &iter->seq;
	110	struct trace_entry *ent = iter->ent;
	111	struct syscall_trace_exit *trace;
	112	int syscall;
	113	struct syscall_metadata *entry;
	114	int ret;
	115
64c12e04	116	trace = (typeof(trace))ent;
bed1ffca	117	syscall = trace->nr;
bed1ffca	118	entry = syscall_nr_to_meta(syscall);
64c12e04	119
bed1ffca FW	120	if (!entry) {
	121	trace_seq_printf(s, "\n");
	122	return TRACE_TYPE_HANDLED;
	123	}
	124
fcc19438	125	if (entry->exit_event->id != ent->type) {
64c12e04 JB	126	WARN_ON_ONCE(1);
	127	return TRACE_TYPE_UNHANDLED;
	128	}
	129
bed1ffca FW	130	ret = trace_seq_printf(s, "%s -> 0x%lx\n", entry->name,
	131	trace->ret);
	132	if (!ret)
	133	return TRACE_TYPE_PARTIAL_LINE;
	134
	135	return TRACE_TYPE_HANDLED;
	136	}
	137
e6971969 LZ	138	extern char *__bad_type_size(void);
	139
	140	#define SYSCALL_FIELD(type, name) \
	141	sizeof(type) != sizeof(trace.name) ? \
	142	__bad_type_size() : \
26a50744 TZ	143	#type, #name, offsetof(typeof(trace), name), \
26a50744 TZ	144	sizeof(trace.name), is_signed_type(type)
e6971969	145
50307a45 LJ	146	static
	147	int __set_enter_print_fmt(struct syscall_metadata entry, char buf, int len)
	148	{
	149	int i;
	150	int pos = 0;
	151
	152	/* When len=0, we just calculate the needed length */
	153	#define LEN_OR_ZERO (len ? len - pos : 0)
	154
	155	pos += snprintf(buf + pos, LEN_OR_ZERO, "\"");
	156	for (i = 0; i < entry->nb_args; i++) {
	157	pos += snprintf(buf + pos, LEN_OR_ZERO, "%s: 0x%%0%zulx%s",
	158	entry->args[i], sizeof(unsigned long),
	159	i == entry->nb_args - 1 ? "" : ", ");
	160	}
	161	pos += snprintf(buf + pos, LEN_OR_ZERO, "\"");
	162
	163	for (i = 0; i < entry->nb_args; i++) {
	164	pos += snprintf(buf + pos, LEN_OR_ZERO,
	165	", ((unsigned long)(REC->%s))", entry->args[i]);
	166	}
	167
	168	#undef LEN_OR_ZERO
	169
	170	/* return the length of print_fmt */
	171	return pos;
	172	}
	173
	174	static int set_syscall_print_fmt(struct ftrace_event_call *call)
	175	{
	176	char *print_fmt;
	177	int len;
	178	struct syscall_metadata *entry = call->data;
	179
	180	if (entry->enter_event != call) {
	181	call->print_fmt = "\"0x%lx\", REC->ret";
	182	return 0;
	183	}
	184
	185	/* First: called with 0 length to calculate the needed length */
	186	len = __set_enter_print_fmt(entry, NULL, 0);
	187
	188	print_fmt = kmalloc(len + 1, GFP_KERNEL);
	189	if (!print_fmt)
	190	return -ENOMEM;
	191
	192	/* Second: actually write the @print_fmt */
	193	__set_enter_print_fmt(entry, print_fmt, len + 1);
	194	call->print_fmt = print_fmt;
	195
	196	return 0;
	197	}
	198
	199	static void free_syscall_print_fmt(struct ftrace_event_call *call)
	200	{
	201	struct syscall_metadata *entry = call->data;
	202
	203	if (entry->enter_event == call)
	204	kfree(call->print_fmt);
	205	}
	206
540b7b8d LZ	207	int syscall_enter_define_fields(struct ftrace_event_call *call)
	208	{
	209	struct syscall_trace_enter trace;
31c16b13	210	struct syscall_metadata *meta = call->data;
540b7b8d	211	int ret;
540b7b8d LZ	212	int i;
	213	int offset = offsetof(typeof(trace), args);
	214
0f1ef51d LJ	215	ret = trace_define_field(call, SYSCALL_FIELD(int, nr), FILTER_OTHER);
	216	if (ret)
	217	return ret;
	218
540b7b8d	219	for (i = 0; i < meta->nb_args; i++) {
aeaeae11 FW	220	ret = trace_define_field(call, meta->types[i],
aeaeae11 FW	221	meta->args[i], offset,
43b51ead LZ	222	sizeof(unsigned long), 0,
43b51ead LZ	223	FILTER_OTHER);
540b7b8d LZ	224	offset += sizeof(unsigned long);
	225	}
	226
	227	return ret;
	228	}
	229
	230	int syscall_exit_define_fields(struct ftrace_event_call *call)
	231	{
	232	struct syscall_trace_exit trace;
	233	int ret;
	234
0f1ef51d LJ	235	ret = trace_define_field(call, SYSCALL_FIELD(int, nr), FILTER_OTHER);
	236	if (ret)
	237	return ret;
	238
26a50744	239	ret = trace_define_field(call, SYSCALL_FIELD(long, ret),
43b51ead	240	FILTER_OTHER);
540b7b8d LZ	241
	242	return ret;
	243	}
	244
fb34a08c	245	void ftrace_syscall_enter(struct pt_regs *regs, long id)
ee08c6ec	246	{
bed1ffca FW	247	struct syscall_trace_enter *entry;
	248	struct syscall_metadata *sys_data;
	249	struct ring_buffer_event *event;
e77405ad	250	struct ring_buffer *buffer;
bed1ffca	251	int size;
ee08c6ec FW	252	int syscall_nr;
	253
	254	syscall_nr = syscall_get_nr(current, regs);
cd0980fc HB	255	if (syscall_nr < 0)
cd0980fc HB	256	return;
fb34a08c JB	257	if (!test_bit(syscall_nr, enabled_enter_syscalls))
fb34a08c JB	258	return;
ee08c6ec	259
bed1ffca FW	260	sys_data = syscall_nr_to_meta(syscall_nr);
	261	if (!sys_data)
	262	return;
	263
	264	size = sizeof(entry) + sizeof(unsigned long) sys_data->nb_args;
	265
fcc19438 LJ	266	event = trace_current_buffer_lock_reserve(&buffer,
fcc19438 LJ	267	sys_data->enter_event->id, size, 0, 0);
bed1ffca FW	268	if (!event)
	269	return;
	270
	271	entry = ring_buffer_event_data(event);
	272	entry->nr = syscall_nr;
	273	syscall_get_arguments(current, regs, 0, sys_data->nb_args, entry->args);
	274
e77405ad SR	275	if (!filter_current_check_discard(buffer, sys_data->enter_event,
	276	entry, event))
	277	trace_current_buffer_unlock_commit(buffer, event, 0, 0);
ee08c6ec FW	278	}
ee08c6ec FW	279
fb34a08c	280	void ftrace_syscall_exit(struct pt_regs *regs, long ret)
ee08c6ec	281	{
bed1ffca FW	282	struct syscall_trace_exit *entry;
	283	struct syscall_metadata *sys_data;
	284	struct ring_buffer_event *event;
e77405ad	285	struct ring_buffer *buffer;
ee08c6ec FW	286	int syscall_nr;
	287
	288	syscall_nr = syscall_get_nr(current, regs);
cd0980fc HB	289	if (syscall_nr < 0)
cd0980fc HB	290	return;
fb34a08c JB	291	if (!test_bit(syscall_nr, enabled_exit_syscalls))
fb34a08c JB	292	return;
ee08c6ec	293
bed1ffca FW	294	sys_data = syscall_nr_to_meta(syscall_nr);
	295	if (!sys_data)
	296	return;
	297
fcc19438 LJ	298	event = trace_current_buffer_lock_reserve(&buffer,
fcc19438 LJ	299	sys_data->exit_event->id, sizeof(*entry), 0, 0);
bed1ffca FW	300	if (!event)
	301	return;
	302
	303	entry = ring_buffer_event_data(event);
	304	entry->nr = syscall_nr;
	305	entry->ret = syscall_get_return_value(current, regs);
	306
e77405ad SR	307	if (!filter_current_check_discard(buffer, sys_data->exit_event,
	308	entry, event))
	309	trace_current_buffer_unlock_commit(buffer, event, 0, 0);
ee08c6ec FW	310	}
ee08c6ec FW	311
bd1a5c84	312	int reg_event_syscall_enter(struct ftrace_event_call *call)
ee08c6ec	313	{
fb34a08c JB	314	int ret = 0;
fb34a08c JB	315	int num;
fb34a08c	316
c252f657	317	num = ((struct syscall_metadata *)call->data)->syscall_nr;
57421dbb	318	if (num < 0 \|\| num >= NR_syscalls)
fb34a08c JB	319	return -ENOSYS;
	320	mutex_lock(&syscall_trace_lock);
	321	if (!sys_refcount_enter)
1c569f02	322	ret = register_trace_sys_enter(ftrace_syscall_enter);
3b8e4273	323	if (!ret) {
fb34a08c JB	324	set_bit(num, enabled_enter_syscalls);
	325	sys_refcount_enter++;
	326	}
	327	mutex_unlock(&syscall_trace_lock);
	328	return ret;
ee08c6ec FW	329	}
ee08c6ec FW	330
bd1a5c84	331	void unreg_event_syscall_enter(struct ftrace_event_call *call)
ee08c6ec	332	{
fb34a08c	333	int num;
ee08c6ec	334
c252f657	335	num = ((struct syscall_metadata *)call->data)->syscall_nr;
57421dbb	336	if (num < 0 \|\| num >= NR_syscalls)
fb34a08c JB	337	return;
	338	mutex_lock(&syscall_trace_lock);
	339	sys_refcount_enter--;
	340	clear_bit(num, enabled_enter_syscalls);
	341	if (!sys_refcount_enter)
1c569f02	342	unregister_trace_sys_enter(ftrace_syscall_enter);
fb34a08c JB	343	mutex_unlock(&syscall_trace_lock);
fb34a08c JB	344	}
ee08c6ec	345
bd1a5c84	346	int reg_event_syscall_exit(struct ftrace_event_call *call)
ee08c6ec	347	{
fb34a08c JB	348	int ret = 0;
fb34a08c JB	349	int num;
fb34a08c	350
c252f657	351	num = ((struct syscall_metadata *)call->data)->syscall_nr;
57421dbb	352	if (num < 0 \|\| num >= NR_syscalls)
fb34a08c JB	353	return -ENOSYS;
	354	mutex_lock(&syscall_trace_lock);
	355	if (!sys_refcount_exit)
1c569f02	356	ret = register_trace_sys_exit(ftrace_syscall_exit);
3b8e4273	357	if (!ret) {
fb34a08c JB	358	set_bit(num, enabled_exit_syscalls);
fb34a08c JB	359	sys_refcount_exit++;
ee08c6ec	360	}
fb34a08c JB	361	mutex_unlock(&syscall_trace_lock);
	362	return ret;
	363	}
ee08c6ec	364
bd1a5c84	365	void unreg_event_syscall_exit(struct ftrace_event_call *call)
fb34a08c JB	366	{
fb34a08c JB	367	int num;
ee08c6ec	368
c252f657	369	num = ((struct syscall_metadata *)call->data)->syscall_nr;
57421dbb	370	if (num < 0 \|\| num >= NR_syscalls)
fb34a08c JB	371	return;
	372	mutex_lock(&syscall_trace_lock);
	373	sys_refcount_exit--;
	374	clear_bit(num, enabled_exit_syscalls);
	375	if (!sys_refcount_exit)
1c569f02	376	unregister_trace_sys_exit(ftrace_syscall_exit);
fb34a08c	377	mutex_unlock(&syscall_trace_lock);
ee08c6ec	378	}
fb34a08c	379
a1301da0 LJ	380	int init_syscall_trace(struct ftrace_event_call *call)
	381	{
	382	int id;
	383
50307a45 LJ	384	if (set_syscall_print_fmt(call) < 0)
	385	return -ENOMEM;
	386
c7ef3a90 SR	387	id = trace_event_raw_init(call);
	388
	389	if (id < 0) {
50307a45	390	free_syscall_print_fmt(call);
c7ef3a90	391	return id;
50307a45	392	}
c7ef3a90 SR	393
c7ef3a90 SR	394	return id;
a1301da0 LJ	395	}
a1301da0 LJ	396
c44fc770 FW	397	int __init init_ftrace_syscalls(void)
	398	{
	399	struct syscall_metadata *meta;
	400	unsigned long addr;
	401	int i;
	402
	403	syscalls_metadata = kzalloc(sizeof(syscalls_metadata)
	404	NR_syscalls, GFP_KERNEL);
	405	if (!syscalls_metadata) {
	406	WARN_ON(1);
	407	return -ENOMEM;
	408	}
	409
	410	for (i = 0; i < NR_syscalls; i++) {
	411	addr = arch_syscall_addr(i);
	412	meta = find_syscall_meta(addr);
c252f657 LJ	413	if (!meta)
	414	continue;
	415
	416	meta->syscall_nr = i;
c44fc770 FW	417	syscalls_metadata[i] = meta;
	418	}
	419
	420	return 0;
	421	}
	422	core_initcall(init_ftrace_syscalls);
	423
f4b5ffcc	424	#ifdef CONFIG_EVENT_PROFILE
19007a67	425
57421dbb JB	426	static DECLARE_BITMAP(enabled_prof_enter_syscalls, NR_syscalls);
57421dbb JB	427	static DECLARE_BITMAP(enabled_prof_exit_syscalls, NR_syscalls);
f4b5ffcc JB	428	static int sys_prof_refcount_enter;
	429	static int sys_prof_refcount_exit;
	430
	431	static void prof_syscall_enter(struct pt_regs *regs, long id)
	432	{
	433	struct syscall_metadata *sys_data;
20ab4425 FW	434	struct syscall_trace_enter *rec;
20ab4425 FW	435	unsigned long flags;
ce71b9df	436	char *trace_buf;
20ab4425	437	char *raw_data;
f4b5ffcc	438	int syscall_nr;
4ed7c92d	439	int rctx;
19007a67	440	int size;
20ab4425	441	int cpu;
f4b5ffcc JB	442
	443	syscall_nr = syscall_get_nr(current, regs);
	444	if (!test_bit(syscall_nr, enabled_prof_enter_syscalls))
	445	return;
	446
	447	sys_data = syscall_nr_to_meta(syscall_nr);
	448	if (!sys_data)
	449	return;
	450
19007a67 FW	451	/* get the size after alignment with the u32 buffer size field */
	452	size = sizeof(unsigned long) * sys_data->nb_args + sizeof(*rec);
	453	size = ALIGN(size + sizeof(u32), sizeof(u64));
	454	size -= sizeof(u32);
	455
20ab4425 FW	456	if (WARN_ONCE(size > FTRACE_MAX_PROFILE_SIZE,
	457	"profile buffer not large enough"))
	458	return;
	459
	460	/* Protect the per cpu buffer, begin the rcu read side */
	461	local_irq_save(flags);
19007a67	462
4ed7c92d PZ	463	rctx = perf_swevent_get_recursion_context();
4ed7c92d PZ	464	if (rctx < 0)
ce71b9df FW	465	goto end_recursion;
ce71b9df FW	466
20ab4425 FW	467	cpu = smp_processor_id();
20ab4425 FW	468
28889bf9	469	trace_buf = rcu_dereference(perf_trace_buf);
20ab4425	470
444a2a3b	471	if (!trace_buf)
20ab4425	472	goto end;
19007a67	473
ce71b9df	474	raw_data = per_cpu_ptr(trace_buf, cpu);
20ab4425 FW	475
	476	/* zero the dead bytes from align to not leak stack to user */
	477	(u64 )(&raw_data[size - sizeof(u64)]) = 0ULL;
	478
	479	rec = (struct syscall_trace_enter *) raw_data;
	480	tracing_generic_entry_update(&rec->ent, 0, 0);
fcc19438	481	rec->ent.type = sys_data->enter_event->id;
20ab4425 FW	482	rec->nr = syscall_nr;
	483	syscall_get_arguments(current, regs, 0, sys_data->nb_args,
	484	(unsigned long *)&rec->args);
fcc19438	485	perf_tp_event(sys_data->enter_event->id, 0, 1, rec, size);
20ab4425 FW	486
20ab4425 FW	487	end:
4ed7c92d	488	perf_swevent_put_recursion_context(rctx);
ce71b9df	489	end_recursion:
20ab4425	490	local_irq_restore(flags);
f4b5ffcc JB	491	}
f4b5ffcc JB	492
3bbe84e9	493	int prof_sysenter_enable(struct ftrace_event_call *call)
f4b5ffcc JB	494	{
	495	int ret = 0;
	496	int num;
	497
3bbe84e9	498	num = ((struct syscall_metadata *)call->data)->syscall_nr;
f4b5ffcc JB	499
	500	mutex_lock(&syscall_trace_lock);
	501	if (!sys_prof_refcount_enter)
1c569f02	502	ret = register_trace_sys_enter(prof_syscall_enter);
f4b5ffcc JB	503	if (ret) {
	504	pr_info("event trace: Could not activate"
	505	"syscall entry trace point");
	506	} else {
	507	set_bit(num, enabled_prof_enter_syscalls);
	508	sys_prof_refcount_enter++;
	509	}
	510	mutex_unlock(&syscall_trace_lock);
	511	return ret;
	512	}
	513
3bbe84e9	514	void prof_sysenter_disable(struct ftrace_event_call *call)
f4b5ffcc JB	515	{
	516	int num;
	517
3bbe84e9	518	num = ((struct syscall_metadata *)call->data)->syscall_nr;
f4b5ffcc JB	519
	520	mutex_lock(&syscall_trace_lock);
	521	sys_prof_refcount_enter--;
	522	clear_bit(num, enabled_prof_enter_syscalls);
	523	if (!sys_prof_refcount_enter)
1c569f02	524	unregister_trace_sys_enter(prof_syscall_enter);
f4b5ffcc JB	525	mutex_unlock(&syscall_trace_lock);
	526	}
	527
	528	static void prof_syscall_exit(struct pt_regs *regs, long ret)
	529	{
	530	struct syscall_metadata *sys_data;
20ab4425 FW	531	struct syscall_trace_exit *rec;
20ab4425 FW	532	unsigned long flags;
f4b5ffcc	533	int syscall_nr;
ce71b9df	534	char *trace_buf;
20ab4425	535	char *raw_data;
4ed7c92d	536	int rctx;
20ab4425 FW	537	int size;
20ab4425 FW	538	int cpu;
f4b5ffcc JB	539
	540	syscall_nr = syscall_get_nr(current, regs);
	541	if (!test_bit(syscall_nr, enabled_prof_exit_syscalls))
	542	return;
	543
	544	sys_data = syscall_nr_to_meta(syscall_nr);
	545	if (!sys_data)
	546	return;
	547
20ab4425 FW	548	/* We can probably do that at build time */
	549	size = ALIGN(sizeof(*rec) + sizeof(u32), sizeof(u64));
	550	size -= sizeof(u32);
19007a67	551
20ab4425 FW	552	/*
	553	* Impossible, but be paranoid with the future
	554	* How to put this check outside runtime?
	555	*/
	556	if (WARN_ONCE(size > FTRACE_MAX_PROFILE_SIZE,
	557	"exit event has grown above profile buffer size"))
	558	return;
	559
	560	/* Protect the per cpu buffer, begin the rcu read side */
	561	local_irq_save(flags);
ce71b9df	562
4ed7c92d PZ	563	rctx = perf_swevent_get_recursion_context();
4ed7c92d PZ	564	if (rctx < 0)
ce71b9df FW	565	goto end_recursion;
ce71b9df FW	566
20ab4425 FW	567	cpu = smp_processor_id();
20ab4425 FW	568
28889bf9	569	trace_buf = rcu_dereference(perf_trace_buf);
20ab4425	570
444a2a3b	571	if (!trace_buf)
20ab4425 FW	572	goto end;
20ab4425 FW	573
ce71b9df	574	raw_data = per_cpu_ptr(trace_buf, cpu);
20ab4425 FW	575
	576	/* zero the dead bytes from align to not leak stack to user */
	577	(u64 )(&raw_data[size - sizeof(u64)]) = 0ULL;
	578
	579	rec = (struct syscall_trace_exit *)raw_data;
	580
	581	tracing_generic_entry_update(&rec->ent, 0, 0);
fcc19438	582	rec->ent.type = sys_data->exit_event->id;
20ab4425 FW	583	rec->nr = syscall_nr;
	584	rec->ret = syscall_get_return_value(current, regs);
	585
fcc19438	586	perf_tp_event(sys_data->exit_event->id, 0, 1, rec, size);
20ab4425 FW	587
20ab4425 FW	588	end:
4ed7c92d	589	perf_swevent_put_recursion_context(rctx);
ce71b9df	590	end_recursion:
20ab4425	591	local_irq_restore(flags);
f4b5ffcc JB	592	}
f4b5ffcc JB	593
3bbe84e9	594	int prof_sysexit_enable(struct ftrace_event_call *call)
f4b5ffcc JB	595	{
	596	int ret = 0;
	597	int num;
	598
3bbe84e9	599	num = ((struct syscall_metadata *)call->data)->syscall_nr;
f4b5ffcc JB	600
	601	mutex_lock(&syscall_trace_lock);
	602	if (!sys_prof_refcount_exit)
1c569f02	603	ret = register_trace_sys_exit(prof_syscall_exit);
f4b5ffcc JB	604	if (ret) {
	605	pr_info("event trace: Could not activate"
	606	"syscall entry trace point");
	607	} else {
	608	set_bit(num, enabled_prof_exit_syscalls);
	609	sys_prof_refcount_exit++;
	610	}
	611	mutex_unlock(&syscall_trace_lock);
	612	return ret;
	613	}
	614
3bbe84e9	615	void prof_sysexit_disable(struct ftrace_event_call *call)
f4b5ffcc JB	616	{
	617	int num;
	618
3bbe84e9	619	num = ((struct syscall_metadata *)call->data)->syscall_nr;
f4b5ffcc JB	620
	621	mutex_lock(&syscall_trace_lock);
	622	sys_prof_refcount_exit--;
	623	clear_bit(num, enabled_prof_exit_syscalls);
	624	if (!sys_prof_refcount_exit)
1c569f02	625	unregister_trace_sys_exit(prof_syscall_exit);
f4b5ffcc JB	626	mutex_unlock(&syscall_trace_lock);
	627	}
	628
	629	#endif
	630
	631