/**
 * Copyright (C) ARM Limited 2010-2012. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 *
 */

static unsigned long gator_protocol_version = 8;

#include <linux/slab.h>
#include <linux/cpu.h>
#include <linux/sched.h>
#include <linux/irq.h>
#include <linux/vmalloc.h>
#include <linux/hardirq.h>
#include <linux/highmem.h>
#include <linux/pagemap.h>
#include <linux/suspend.h>
#include <linux/module.h>
#include <linux/perf_event.h>
#include <asm/stacktrace.h>
#include <asm/uaccess.h>

#include "gator.h"
#include "gator_events.h"

#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 32)
#error kernels prior to 2.6.32 are not supported
#endif

#if !defined(CONFIG_GENERIC_TRACER) && !defined(CONFIG_TRACING)
#error gator requires the kernel to have CONFIG_GENERIC_TRACER or CONFIG_TRACING defined
#endif

#ifndef CONFIG_PROFILING
#error gator requires the kernel to have CONFIG_PROFILING defined
#endif

#ifndef CONFIG_HIGH_RES_TIMERS
#error gator requires the kernel to have CONFIG_HIGH_RES_TIMERS defined
#endif

#if defined(__arm__) && defined(CONFIG_SMP) && !defined(CONFIG_LOCAL_TIMERS)
#error gator requires the kernel to have CONFIG_LOCAL_TIMERS defined on SMP systems
#endif

#if (GATOR_PERF_SUPPORT) && (!(GATOR_PERF_PMU_SUPPORT))
#ifndef CONFIG_PERF_EVENTS
#warning gator requires the kernel to have CONFIG_PERF_EVENTS defined to support pmu hardware counters
#elif !defined CONFIG_HW_PERF_EVENTS
#warning gator requires the kernel to have CONFIG_HW_PERF_EVENTS defined to support pmu hardware counters
#endif
#endif

#if (!(GATOR_CPU_FREQ_SUPPORT))
#warning gator requires kernel version 2.6.38 or greater and CONFIG_CPU_FREQ defined in order to enable the CPU Freq timeline chart
#endif

/******************************************************************************
 * DEFINES
 ******************************************************************************/
#define TIMER_BUFFER_SIZE_DEFAULT	(512*1024)
#define EVENT_BUFFER_SIZE_DEFAULT	(128*1024)

#define NO_COOKIE				0UL
#define INVALID_COOKIE			~0UL

#define FRAME_HRTIMER		1
#define FRAME_EVENT			2
#define FRAME_ANNOTATE		3

#define MESSAGE_COOKIE				1
#define MESSAGE_COUNTERS			3
#define MESSAGE_START_BACKTRACE		5
#define MESSAGE_END_BACKTRACE		7
#define MESSAGE_SCHEDULER_TRACE		9
#define MESSAGE_PID_NAME			11
#define MESSAGE_GPU_TRACE           13
#define MESSAGE_OVERFLOW			127

#define MAXSIZE_PACK32		5
#define MAXSIZE_PACK64		9

#if defined(__arm__)
#define PC_REG regs->ARM_pc
#else
#define PC_REG regs->ip
#endif

enum {TIMER_BUF, EVENT_BUF, NUM_GATOR_BUFS};

/******************************************************************************
 * Globals
 ******************************************************************************/
static unsigned long gator_cpu_cores;
static unsigned long userspace_buffer_size;
static unsigned long gator_backtrace_depth;

static unsigned long gator_started;
static unsigned long gator_buffer_opened;
static unsigned long gator_timer_count;
static unsigned long gator_streaming;
static DEFINE_MUTEX(start_mutex);
static DEFINE_MUTEX(gator_buffer_mutex);

bool event_based_sampling;

static DECLARE_WAIT_QUEUE_HEAD(gator_buffer_wait);

static void buffer_check(int cpu, int buftype);

/******************************************************************************
 * Prototypes
 ******************************************************************************/
static bool buffer_check_space(int cpu, int buftype, int bytes);
static void gator_buffer_write_packed_int(int cpu, int buftype, unsigned int x);
static void gator_buffer_write_packed_int64(int cpu, int buftype, unsigned long long x);
static void gator_buffer_write_string(int cpu, int buftype, char *x);
static int  gator_write_packed_int(char *buffer, unsigned int x);
static int  gator_write_packed_int64(char *buffer, unsigned long long x);
static void gator_add_trace(int cpu, int buftype, unsigned int address);
static void gator_add_sample(int cpu, int buftype, struct pt_regs * const regs);
static uint64_t gator_get_time(void);

static uint32_t gator_buffer_size[NUM_GATOR_BUFS];
static uint32_t gator_buffer_mask[NUM_GATOR_BUFS];
static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_read);
static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_write);
static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_commit);
static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], buffer_space_available);
static DEFINE_PER_CPU(char *[NUM_GATOR_BUFS], gator_buffer);
static DEFINE_PER_CPU(uint64_t, emit_overflow);

/******************************************************************************
 * Application Includes
 ******************************************************************************/
#include "gator_hrtimer_perf.c"
#include "gator_hrtimer_gator.c"
#include "gator_cookies.c"
#include "gator_trace_sched.c"
#include "gator_trace_gpu.c"
#include "gator_backtrace.c"
#include "gator_annotate.c"
#include "gator_fs.c"
#include "gator_ebs.c"
#include "gator_pack.c"

/******************************************************************************
 * Misc
 ******************************************************************************/
#if defined(__arm__)
u32 gator_cpuid(void)
{
	u32 val;
	asm volatile("mrc p15, 0, %0, c0, c0, 0" : "=r" (val));
	return (val >> 4) & 0xfff;
}
#endif

/******************************************************************************
 * Commit interface
 ******************************************************************************/
static bool buffer_commit_ready(int* cpu, int* buftype)
{
	int cpu_x, x;
	for_each_present_cpu(cpu_x) {
		for (x = 0; x < NUM_GATOR_BUFS; x++)
			if (per_cpu(gator_buffer_commit, cpu_x)[x] != per_cpu(gator_buffer_read, cpu_x)[x]) {
				*cpu = cpu_x;
				*buftype = x;
				return true;
			}
	}
	return false;
}

/******************************************************************************
 * Buffer management
 ******************************************************************************/
static bool buffer_check_space(int cpu, int buftype, int bytes)
{
	int remaining, filled;
	
	filled = per_cpu(gator_buffer_write, cpu)[buftype] - per_cpu(gator_buffer_read, cpu)[buftype];
	if (filled < 0) {
		filled += gator_buffer_size[buftype];
	}

	remaining = gator_buffer_size[buftype] - filled;

	if (per_cpu(buffer_space_available, cpu)[buftype]) {
		// Give some extra room; also allows space to insert the overflow error packet
		remaining -= 200;
	} else {
		// Hysteresis, prevents multiple overflow messages
		remaining -= 2000;
	}

	if (remaining < bytes) {
		if (per_cpu(buffer_space_available, cpu)[buftype] == true) {
			// overflow packet to be emitted at a later time, as we may be in the middle of writing a message, e.g. counters
			per_cpu(emit_overflow, cpu) = gator_get_time();
			pr_err("overflow: remaining = %d\n", gator_buffer_size[buftype] - filled);
		}
		per_cpu(buffer_space_available, cpu)[buftype] = false;
	} else {
		per_cpu(buffer_space_available, cpu)[buftype] = true;
	}

	return per_cpu(buffer_space_available, cpu)[buftype];
}

static void gator_buffer_write_bytes(int cpu, int buftype, char *x, int len)
{
	int i;
	u32 write = per_cpu(gator_buffer_write, cpu)[buftype];
	u32 mask = gator_buffer_mask[buftype];
	char* buffer = per_cpu(gator_buffer, cpu)[buftype];

	for (i = 0; i < len; i++) {
		buffer[write] = x[i];
		write = (write + 1) & mask;
	}

	per_cpu(gator_buffer_write, cpu)[buftype] = write;
}

static void gator_buffer_write_string(int cpu, int buftype, char *x)
{
	int len = strlen(x);
	gator_buffer_write_packed_int(cpu, buftype, len);
	gator_buffer_write_bytes(cpu, buftype, x, len);
}

static void gator_buffer_header(int cpu, int buftype)
{
	int frame;

	if (buftype == TIMER_BUF)
		frame = FRAME_HRTIMER;
	else if (buftype == EVENT_BUF)
		frame = FRAME_EVENT;
	else
		frame = -1;

	gator_buffer_write_packed_int(cpu, buftype, frame);
	gator_buffer_write_packed_int(cpu, buftype, cpu);
}

static void gator_commit_buffer(int cpu, int buftype)
{
	per_cpu(gator_buffer_commit, cpu)[buftype] = per_cpu(gator_buffer_write, cpu)[buftype];
	gator_buffer_header(cpu, buftype);
	wake_up(&gator_buffer_wait);
}

static void buffer_check(int cpu, int buftype)
{
	int filled = per_cpu(gator_buffer_write, cpu)[buftype] - per_cpu(gator_buffer_commit, cpu)[buftype];
	if (filled < 0) {
		filled += gator_buffer_size[buftype];
	}
	if (filled >= ((gator_buffer_size[buftype] * 3) / 4)) {
		gator_commit_buffer(cpu, buftype);
	}
}

static void gator_add_trace(int cpu, int buftype, unsigned int address)
{
	off_t offset = 0;
	unsigned long cookie = get_address_cookie(cpu, buftype, current, address & ~1, &offset);

	if (cookie == NO_COOKIE || cookie == INVALID_COOKIE) {
		offset = address;
	}

	gator_buffer_write_packed_int(cpu, buftype, offset & ~1);
	gator_buffer_write_packed_int(cpu, buftype, cookie);
}

static void gator_add_sample(int cpu, int buftype, struct pt_regs * const regs)
{
	int inKernel = regs ? !user_mode(regs) : 1;
	unsigned long exec_cookie = inKernel ? NO_COOKIE : get_exec_cookie(cpu, buftype, current);

	if (!regs)
		return;

	gator_buffer_write_packed_int(cpu, buftype, MESSAGE_START_BACKTRACE);
	gator_buffer_write_packed_int64(cpu, buftype, gator_get_time());
	gator_buffer_write_packed_int(cpu, buftype, exec_cookie);
	gator_buffer_write_packed_int(cpu, buftype, (unsigned int)current->tgid); 
	gator_buffer_write_packed_int(cpu, buftype, (unsigned int)current->pid);
	gator_buffer_write_packed_int(cpu, buftype, inKernel);

	if (inKernel) {
		kernel_backtrace(cpu, buftype, regs);
	} else {
		// Cookie+PC
		gator_add_trace(cpu, buftype, PC_REG);

		// Backtrace
		if (gator_backtrace_depth)
			arm_backtrace_eabi(cpu, buftype, regs, gator_backtrace_depth);
	}

	gator_buffer_write_packed_int(cpu, buftype, MESSAGE_END_BACKTRACE);
}

/******************************************************************************
 * hrtimer interrupt processing
 ******************************************************************************/
static LIST_HEAD(gator_events);

static void gator_timer_interrupt(void)
{
	struct pt_regs * const regs = get_irq_regs();
	int cpu = smp_processor_id();
	int *buffer, len, i, buftype = TIMER_BUF;
	long long *buffer64;
	struct gator_interface *gi;

	// Output scheduler trace
	len = gator_trace_sched_read(&buffer64);
	if (len > 0 && buffer_check_space(cpu, buftype, len * MAXSIZE_PACK64 + 2 * MAXSIZE_PACK32)) {
		gator_buffer_write_packed_int(cpu, buftype, MESSAGE_SCHEDULER_TRACE);
		gator_buffer_write_packed_int(cpu, buftype, len);
		for (i = 0; i < len; i++) {
			gator_buffer_write_packed_int64(cpu, buftype, buffer64[i]);
		}
	}

	// Output GPU trace
	len = gator_trace_gpu_read(&buffer64);
	if (len > 0 && buffer_check_space(cpu, buftype, len * MAXSIZE_PACK64 + 2 * MAXSIZE_PACK32)) {
		gator_buffer_write_packed_int(cpu, buftype, MESSAGE_GPU_TRACE);
		gator_buffer_write_packed_int(cpu, buftype, len);
		for (i = 0; i < len; i++) {
			gator_buffer_write_packed_int64(cpu, buftype, buffer64[i]);
		}
	}

	// Output counters
	if (buffer_check_space(cpu, buftype, MAXSIZE_PACK32 * 2 + MAXSIZE_PACK64)) {
		gator_buffer_write_packed_int(cpu, buftype, MESSAGE_COUNTERS);
		gator_buffer_write_packed_int64(cpu, buftype, gator_get_time());
		list_for_each_entry(gi, &gator_events, list) {
			if (gi->read) {
				len = gi->read(&buffer);
				if (len > 0 && buffer_check_space(cpu, buftype, len * MAXSIZE_PACK32 + MAXSIZE_PACK32)) {
					gator_buffer_write_packed_int(cpu, buftype, len);
					for (i = 0; i < len; i++) {
						gator_buffer_write_packed_int(cpu, buftype, buffer[i]);
					}
				}
			} else if (gi->read64) {
				len = gi->read64(&buffer64);
				if (len > 0 && buffer_check_space(cpu, buftype, len * MAXSIZE_PACK64 + MAXSIZE_PACK32)) {
					gator_buffer_write_packed_int(cpu, buftype, len);
					for (i = 0; i < len; i++) {
						gator_buffer_write_packed_int64(cpu, buftype, buffer64[i]);
					}
				}
			}
		}
		gator_buffer_write_packed_int(cpu, buftype, 0);
	}

	// Output backtrace
	if (!event_based_sampling && buffer_check_space(cpu, buftype, gator_backtrace_depth * 2 * MAXSIZE_PACK32))
		gator_add_sample(cpu, buftype, regs);

	// Overflow message
	if (per_cpu(emit_overflow, cpu)) {
		gator_buffer_write_packed_int(cpu, buftype, MESSAGE_OVERFLOW);
		gator_buffer_write_packed_int64(cpu, buftype, per_cpu(emit_overflow, cpu));
		per_cpu(emit_overflow, cpu) = 0;
	}

	// Check and commit; generally, commit is set to occur once per second
	buffer_check(cpu, buftype);
}

DEFINE_PER_CPU(int, hrtimer_is_active);
static int hrtimer_running;

// This function runs in interrupt context and on the appropriate core
static void gator_timer_offline(void* unused)
{
	int i, len, cpu = smp_processor_id();
	int* buffer;
	long long* buffer64;

	if (per_cpu(hrtimer_is_active, cpu)) {
		struct gator_interface *gi;
		gator_hrtimer_offline(cpu);
		per_cpu(hrtimer_is_active, cpu) = 0;

		// Output scheduler trace
		len = gator_trace_sched_offline(&buffer64);
		if (len > 0 && buffer_check_space(cpu, TIMER_BUF, len * MAXSIZE_PACK64 + 2 * MAXSIZE_PACK32)) {
			gator_buffer_write_packed_int(cpu, TIMER_BUF, MESSAGE_SCHEDULER_TRACE);
			gator_buffer_write_packed_int(cpu, TIMER_BUF, len);
			for (i = 0; i < len; i++) {
				gator_buffer_write_packed_int64(cpu, TIMER_BUF, buffer64[i]);
			}
		}

		// Output GPU trace
		len = gator_trace_gpu_offline(&buffer64);
		if (len > 0 && buffer_check_space(cpu, TIMER_BUF, len * MAXSIZE_PACK64 + 2 * MAXSIZE_PACK32)) {
			gator_buffer_write_packed_int(cpu, TIMER_BUF, MESSAGE_GPU_TRACE);
			gator_buffer_write_packed_int(cpu, TIMER_BUF, len);
			for (i = 0; i < len; i++) {
				gator_buffer_write_packed_int64(cpu, TIMER_BUF, buffer64[i]);
			}
		}

		// offline any events and output counters
		gator_buffer_write_packed_int(cpu, TIMER_BUF, MESSAGE_COUNTERS);
		gator_buffer_write_packed_int64(cpu, TIMER_BUF, gator_get_time());
		list_for_each_entry(gi, &gator_events, list) {
			if (gi->offline) {
				len = gi->offline(&buffer);
				if (len > 0 && buffer_check_space(cpu, TIMER_BUF, len * MAXSIZE_PACK32 + MAXSIZE_PACK32)) {
					gator_buffer_write_packed_int(cpu, TIMER_BUF, len);
					for (i = 0; i < len; i++)
						gator_buffer_write_packed_int(cpu, TIMER_BUF, buffer[i]);
				}
			}
		}
		gator_buffer_write_packed_int(cpu, TIMER_BUF, 0);

		gator_commit_buffer(cpu, TIMER_BUF);
	}

	if (event_based_sampling) {
		gator_commit_buffer(cpu, EVENT_BUF);
	}
}

// This function runs in interrupt context and may be running on a core other than core 'cpu'
static void gator_timer_offline_dispatch(int cpu)
{
	struct gator_interface *gi;

	list_for_each_entry(gi, &gator_events, list)
		if (gi->offline_dispatch)
			gi->offline_dispatch(cpu);

	gator_event_sampling_offline_dispatch(cpu);
}

static void gator_timer_stop(void)
{
	int cpu;

	if (hrtimer_running) {
		on_each_cpu(gator_timer_offline, NULL, 1);
		for_each_online_cpu(cpu) {
			gator_timer_offline_dispatch(cpu);
		}

		hrtimer_running = 0;
		gator_hrtimer_shutdown();
	}
}

// This function runs in interrupt context and on the appropriate core
static void gator_timer_online(void* unused)
{
	int i, len, cpu = smp_processor_id();
	int* buffer;

	if (!per_cpu(hrtimer_is_active, cpu)) {
		struct gator_interface *gi;

		// online any events and output counters
		gator_buffer_write_packed_int(cpu, TIMER_BUF, MESSAGE_COUNTERS);
		gator_buffer_write_packed_int64(cpu, TIMER_BUF, gator_get_time());
		list_for_each_entry(gi, &gator_events, list) {
			if (gi->online) {
				len = gi->online(&buffer);
				if (len > 0 && buffer_check_space(cpu, TIMER_BUF, len * MAXSIZE_PACK32 + MAXSIZE_PACK32)) {
					gator_buffer_write_packed_int(cpu, TIMER_BUF, len);
					for (i = 0; i < len; i++)
						gator_buffer_write_packed_int(cpu, TIMER_BUF, buffer[i]);
				}
			}
		}
		gator_buffer_write_packed_int(cpu, TIMER_BUF, 0);

		gator_event_sampling_online();

		gator_hrtimer_online(cpu);
		per_cpu(hrtimer_is_active, cpu) = 1;
	}
}

// This function runs in interrupt context and may be running on a core other than core 'cpu'
static void gator_timer_online_dispatch(int cpu)
{
	struct gator_interface *gi;

	list_for_each_entry(gi, &gator_events, list)
		if (gi->online_dispatch)
			gi->online_dispatch(cpu);

	gator_event_sampling_online_dispatch(cpu);
}

int gator_timer_start(unsigned long setup)
{
	int cpu;

	if (!setup) {
		pr_err("gator: cannot start due to a system tick value of zero\n");
		return -1;
	} else if (hrtimer_running) {
		pr_notice("gator: high res timer already running\n");
		return 0;
	}

	hrtimer_running = 1;

	if (gator_hrtimer_init(setup, gator_timer_interrupt) == -1)
		return -1;

	for_each_online_cpu(cpu) {
		gator_timer_online_dispatch(cpu);
	}
	on_each_cpu(gator_timer_online, NULL, 1);

	return 0;
}

static uint64_t gator_get_time(void)
{
	struct timespec ts;
	uint64_t timestamp;

	getnstimeofday(&ts);
	timestamp = timespec_to_ns(&ts);

	return timestamp;
}

/******************************************************************************
 * cpu hotplug and pm notifiers
 ******************************************************************************/
static int __cpuinit gator_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu)
{
	long cpu = (long)hcpu;

	switch (action) {
		case CPU_DOWN_PREPARE:
		case CPU_DOWN_PREPARE_FROZEN:
			smp_call_function_single(cpu, gator_timer_offline, NULL, 1);
			gator_timer_offline_dispatch(cpu);
			break;
		case CPU_ONLINE:
		case CPU_ONLINE_FROZEN:
			gator_timer_online_dispatch(cpu);
			smp_call_function_single(cpu, gator_timer_online, NULL, 1);
			break;
	}

	return NOTIFY_OK;
}

static struct notifier_block __refdata gator_cpu_notifier = {
	.notifier_call = gator_cpu_notify,
};

// n.b. calling "on_each_cpu" only runs on those that are online
// Registered linux events are not disabled, so their counters will continue to collect
static int gator_pm_notify(struct notifier_block *nb, unsigned long event, void *dummy)
{
	int cpu;

	switch (event) {
		case PM_HIBERNATION_PREPARE:
		case PM_SUSPEND_PREPARE:
			unregister_hotcpu_notifier(&gator_cpu_notifier);
			unregister_scheduler_tracepoints();
			on_each_cpu(gator_timer_offline, NULL, 1);
			for_each_online_cpu(cpu) {
				gator_timer_offline_dispatch(cpu);
			}
			break;
		case PM_POST_HIBERNATION:
		case PM_POST_SUSPEND:
			for_each_online_cpu(cpu) {
				gator_timer_online_dispatch(cpu);
			}
			on_each_cpu(gator_timer_online, NULL, 1);
			register_scheduler_tracepoints();
			register_hotcpu_notifier(&gator_cpu_notifier);
			break;
	}

	return NOTIFY_OK;
}

static struct notifier_block gator_pm_notifier = {
	.notifier_call = gator_pm_notify,
};

static int gator_notifier_start(void)
{
	int retval;
	retval = register_hotcpu_notifier(&gator_cpu_notifier);
	if (retval == 0)
		retval = register_pm_notifier(&gator_pm_notifier);
	return retval;
}

static void gator_notifier_stop(void)
{
	unregister_pm_notifier(&gator_pm_notifier);
	unregister_hotcpu_notifier(&gator_cpu_notifier);
}

/******************************************************************************
 * Main
 ******************************************************************************/
int gator_events_install(struct gator_interface *interface)
{
	list_add_tail(&interface->list, &gator_events);

	return 0;
}

int gator_events_get_key(void)
{
	static int key;

	return key++;
}

static int gator_init(void)
{
	int i;

	if (gator_annotate_init())
		return -1;

	// events sources (gator_events.h, generated by gator_events.sh)
	for (i = 0; i < ARRAY_SIZE(gator_events_list); i++)
		if (gator_events_list[i])
			gator_events_list[i]();

	return 0;
}

static int gator_start(void)
{
	struct gator_interface *gi;

	// start all events
	list_for_each_entry(gi, &gator_events, list) {
		if (gi->start && gi->start() != 0) {
			struct list_head *ptr = gi->list.prev;

			while (ptr != &gator_events) {
				gi = list_entry(ptr, struct gator_interface, list);

				if (gi->stop)
					gi->stop();

				ptr = ptr->prev;
			}
			goto events_failure;
		}
	}

	// cookies shall be initialized before trace_sched_start() and gator_timer_start()
	if (cookies_initialize())
		goto cookies_failure;
	if (gator_annotate_start())
		goto annotate_failure;
	if (gator_trace_sched_start())
		goto sched_failure;
	if (gator_trace_gpu_start())
		goto gpu_failure;
	if (gator_event_sampling_start())
		goto event_sampling_failure;
	if (gator_timer_start(gator_timer_count))
		goto timer_failure;
	if (gator_notifier_start())
		goto notifier_failure;

	return 0;

notifier_failure:
	gator_timer_stop();
timer_failure:
	gator_event_sampling_stop();
event_sampling_failure:
	gator_trace_gpu_stop();
gpu_failure:
	gator_trace_sched_stop();
sched_failure:
	gator_annotate_stop();
annotate_failure:
	cookies_release();
cookies_failure:
	// stop all events
	list_for_each_entry(gi, &gator_events, list)
		if (gi->stop)
			gi->stop();
events_failure:

	return -1;
}

static void gator_stop(void)
{
	struct gator_interface *gi;

	// stop all events
	list_for_each_entry(gi, &gator_events, list)
		if (gi->stop)
			gi->stop();

	gator_annotate_stop();
	gator_trace_sched_stop();
	gator_trace_gpu_stop();
	gator_event_sampling_stop();

	// stop all interrupt callback reads before tearing down other interfaces
	gator_notifier_stop(); // should be called before gator_timer_stop to avoid re-enabling the hrtimer after it has been offlined
	gator_timer_stop();
}

static void gator_exit(void)
{
	gator_annotate_exit();
}

/******************************************************************************
 * Filesystem
 ******************************************************************************/
/* fopen("buffer") */
static int gator_op_setup(void)
{
	int err = 0;
	int cpu, i;

	mutex_lock(&start_mutex);

	gator_buffer_size[TIMER_BUF] = userspace_buffer_size;
	gator_buffer_mask[TIMER_BUF] = userspace_buffer_size - 1;

	// must be a power of 2
	if (gator_buffer_size[TIMER_BUF] & (gator_buffer_size[TIMER_BUF] - 1)) {
		err = -ENOEXEC;
		goto setup_error;
	}

	gator_buffer_size[EVENT_BUF] = EVENT_BUFFER_SIZE_DEFAULT;
	gator_buffer_mask[EVENT_BUF] = gator_buffer_size[EVENT_BUF] - 1;

	// Initialize percpu per buffer variables
	for (i = 0; i < NUM_GATOR_BUFS; i++) {
		for_each_present_cpu(cpu) {
			per_cpu(gator_buffer, cpu)[i] = vmalloc(gator_buffer_size[i]);
			if (!per_cpu(gator_buffer, cpu)[i]) {
				err = -ENOMEM;
				goto setup_error;
			}

			per_cpu(gator_buffer_read, cpu)[i] = 0;
			per_cpu(gator_buffer_write, cpu)[i] = 0;
			per_cpu(gator_buffer_commit, cpu)[i] = 0;
			per_cpu(buffer_space_available, cpu)[i] = true;
			per_cpu(emit_overflow, cpu) = 0;
			gator_buffer_header(cpu, i);
		}
	}

setup_error:
	mutex_unlock(&start_mutex);
	return err;
}

/* Actually start profiling (echo 1>/dev/gator/driver/enable) */
static int gator_op_start(void)
{
	int err = 0;

	mutex_lock(&start_mutex);

	if (gator_started || gator_start())
		err = -EINVAL;
	else
		gator_started = 1;

	mutex_unlock(&start_mutex);

	return err;
}

/* echo 0>/dev/gator/driver/enable */
static void gator_op_stop(void)
{
	mutex_lock(&start_mutex);

	if (gator_started) {
		gator_stop();

		mutex_lock(&gator_buffer_mutex);

		gator_started = 0;
		cookies_release();
		wake_up(&gator_buffer_wait);

		mutex_unlock(&gator_buffer_mutex);
	}

	mutex_unlock(&start_mutex);
}

static void gator_shutdown(void)
{
	int cpu, i;

	mutex_lock(&start_mutex);

	gator_annotate_shutdown();

	for_each_present_cpu(cpu) {
		mutex_lock(&gator_buffer_mutex);
		for (i = 0; i < NUM_GATOR_BUFS; i++) {
			vfree(per_cpu(gator_buffer, cpu)[i]);
			per_cpu(gator_buffer, cpu)[i] = NULL;
			per_cpu(gator_buffer_read, cpu)[i] = 0;
			per_cpu(gator_buffer_write, cpu)[i] = 0;
			per_cpu(gator_buffer_commit, cpu)[i] = 0;
			per_cpu(buffer_space_available, cpu)[i] = true;
			per_cpu(emit_overflow, cpu) = 0;
		}
		mutex_unlock(&gator_buffer_mutex);
	}

	mutex_unlock(&start_mutex);
}

static int gator_set_backtrace(unsigned long val)
{
	int err = 0;

	mutex_lock(&start_mutex);

	if (gator_started)
		err = -EBUSY;
	else
		gator_backtrace_depth = val;

	mutex_unlock(&start_mutex);

	return err;
}

static ssize_t enable_read(struct file *file, char __user *buf, size_t count, loff_t *offset)
{
	return gatorfs_ulong_to_user(gator_started, buf, count, offset);
}

static ssize_t enable_write(struct file *file, char const __user *buf, size_t count, loff_t *offset)
{
	unsigned long val;
	int retval;

	if (*offset)
		return -EINVAL;

	retval = gatorfs_ulong_from_user(&val, buf, count);
	if (retval)
		return retval;

	if (val)
		retval = gator_op_start();
	else
		gator_op_stop();

	if (retval)
		return retval;
	return count;
}

static const struct file_operations enable_fops = {
	.read		= enable_read,
	.write		= enable_write,
};

static int userspace_buffer_open(struct inode *inode, struct file *file)
{
	int err = -EPERM;

	if (!capable(CAP_SYS_ADMIN))
		return -EPERM;

	if (test_and_set_bit_lock(0, &gator_buffer_opened))
		return -EBUSY;

	if ((err = gator_op_setup()))
		goto fail;

	/* NB: the actual start happens from userspace
	 * echo 1 >/dev/gator/driver/enable
	 */

	return 0;

fail:
	__clear_bit_unlock(0, &gator_buffer_opened);
	return err;
}

static int userspace_buffer_release(struct inode *inode, struct file *file)
{
	gator_op_stop();
	gator_shutdown();
	__clear_bit_unlock(0, &gator_buffer_opened);
	return 0;
}

static ssize_t userspace_buffer_read(struct file *file, char __user *buf,
				 size_t count, loff_t *offset)
{
	int retval = -EINVAL;
	int commit = 0, length1, length2, read;
	char *buffer1;
	char *buffer2 = NULL;
	int cpu, buftype;

	/* do not handle partial reads */
	if (count != userspace_buffer_size || *offset)
		return -EINVAL;

	// sleep until the condition is true or a signal is received
	// the condition is checked each time gator_buffer_wait is woken up
	buftype = cpu = -1;
	wait_event_interruptible(gator_buffer_wait, buffer_commit_ready(&cpu, &buftype) || gator_annotate_ready() || !gator_started);

	if (signal_pending(current))
		return -EINTR;

	length2 = 0;
	retval = -EFAULT;

	mutex_lock(&gator_buffer_mutex);

	if (buftype != -1 && cpu != -1) {
		read = per_cpu(gator_buffer_read, cpu)[buftype];
		commit = per_cpu(gator_buffer_commit, cpu)[buftype];

		/* May happen if the buffer is freed during pending reads. */
		if (!per_cpu(gator_buffer, cpu)[buftype]) {
			retval = -EFAULT;
			goto out;
		}

		/* determine the size of two halves */
		length1 = commit - read;
		buffer1 = &(per_cpu(gator_buffer, cpu)[buftype][read]);
		buffer2 = &(per_cpu(gator_buffer, cpu)[buftype][0]);
		if (length1 < 0) {
			length1 = gator_buffer_size[buftype] - read;
			length2 = commit;
		}
	} else if (gator_annotate_ready()) {
		length1 = gator_annotate_read(&buffer1);
		if (!length1)
			goto out;
	} else {
		retval = 0;
		goto out;
	}

	/* start, middle or end */
	if (length1 > 0) {
		if (copy_to_user(&buf[0], buffer1, length1)) {
			goto out;
		}
	}

	/* possible wrap around */
	if (length2 > 0) {
		if (copy_to_user(&buf[length1], buffer2, length2)) {
			goto out;
		}
	}

	if (buftype != -1 && cpu != -1)
		per_cpu(gator_buffer_read, cpu)[buftype] = commit;

	retval = length1 + length2;

	/* kick just in case we've lost an SMP event */
	wake_up(&gator_buffer_wait);

out:
	mutex_unlock(&gator_buffer_mutex);
	return retval;
}

const struct file_operations gator_event_buffer_fops = {
	.open		= userspace_buffer_open,
	.release	= userspace_buffer_release,
	.read		= userspace_buffer_read,
};

static ssize_t depth_read(struct file *file, char __user *buf, size_t count, loff_t *offset)
{
	return gatorfs_ulong_to_user(gator_backtrace_depth, buf, count,
					offset);
}

static ssize_t depth_write(struct file *file, char const __user *buf, size_t count, loff_t *offset)
{
	unsigned long val;
	int retval;

	if (*offset)
		return -EINVAL;

	retval = gatorfs_ulong_from_user(&val, buf, count);
	if (retval)
		return retval;

	retval = gator_set_backtrace(val);

	if (retval)
		return retval;
	return count;
}

static const struct file_operations depth_fops = {
	.read		= depth_read,
	.write		= depth_write
};

void gator_op_create_files(struct super_block *sb, struct dentry *root)
{
	struct dentry *dir;
	struct gator_interface *gi;
	int cpu;

	/* reinitialize default values */
	gator_cpu_cores = 0;
	for_each_present_cpu(cpu) {
		gator_cpu_cores++;
	}
	userspace_buffer_size =	TIMER_BUFFER_SIZE_DEFAULT;
	gator_streaming = 1;

	gatorfs_create_file(sb, root, "enable", &enable_fops);
	gatorfs_create_file(sb, root, "buffer", &gator_event_buffer_fops);
	gatorfs_create_file(sb, root, "backtrace_depth", &depth_fops);
	gatorfs_create_ulong(sb, root, "cpu_cores", &gator_cpu_cores);
	gatorfs_create_ulong(sb, root, "buffer_size", &userspace_buffer_size);
	gatorfs_create_ulong(sb, root, "tick", &gator_timer_count);
	gatorfs_create_ulong(sb, root, "streaming", &gator_streaming);
	gatorfs_create_ro_ulong(sb, root, "version", &gator_protocol_version);

	// Annotate interface
	gator_annotate_create_files(sb, root);

	// Linux Events
	dir = gatorfs_mkdir(sb, root, "events");
	list_for_each_entry(gi, &gator_events, list)
		if (gi->create_files)
			gi->create_files(sb, dir);
}

/******************************************************************************
 * Module
 ******************************************************************************/
static int __init gator_module_init(void)
{
	if (gatorfs_register()) {
		return -1;
	}

	if (gator_init()) {
		gatorfs_unregister();
		return -1;
	}

	return 0;
}

static void __exit gator_module_exit(void)
{
	tracepoint_synchronize_unregister();
	gatorfs_unregister();
	gator_exit();
}

module_init(gator_module_init);
module_exit(gator_module_exit);

MODULE_LICENSE("GPL");
MODULE_AUTHOR("ARM Ltd");
MODULE_DESCRIPTION("Gator system profiler");
