Files
OrangePi_CM5/kernel/trace/trace_irqsoff.c
T

491 lines
11 KiB
C
Raw Normal View History

2008-05-12 21:20:42 +02:00
/*
* trace irqs off critical timings
2008-05-12 21:20:42 +02:00
*
* Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
* Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
*
* From code in the latency_tracer, that is:
*
* Copyright (C) 2004-2006 Ingo Molnar
* Copyright (C) 2004 William Lee Irwin III
*/
#include <linux/kallsyms.h>
#include <linux/debugfs.h>
#include <linux/uaccess.h>
#include <linux/module.h>
#include <linux/ftrace.h>
#include <linux/fs.h>
#include "trace.h"
static struct trace_array *irqsoff_trace __read_mostly;
static int tracer_enabled __read_mostly;
2008-05-12 21:20:42 +02:00
static DEFINE_PER_CPU(int, tracing_cpu);
2008-05-12 21:20:44 +02:00
static DEFINE_SPINLOCK(max_trace_lock);
2008-05-12 21:20:42 +02:00
enum {
TRACER_IRQS_OFF = (1 << 1),
TRACER_PREEMPT_OFF = (1 << 2),
};
static int trace_type __read_mostly;
static int save_lat_flag;
2008-05-12 21:20:42 +02:00
#ifdef CONFIG_PREEMPT_TRACER
2008-05-12 21:20:51 +02:00
static inline int
2008-05-12 21:20:42 +02:00
preempt_trace(void)
{
return ((trace_type & TRACER_PREEMPT_OFF) && preempt_count());
}
#else
# define preempt_trace() (0)
#endif
#ifdef CONFIG_IRQSOFF_TRACER
2008-05-12 21:20:51 +02:00
static inline int
2008-05-12 21:20:42 +02:00
irq_trace(void)
{
return ((trace_type & TRACER_IRQS_OFF) &&
irqs_disabled());
}
#else
# define irq_trace() (0)
#endif
2008-05-12 21:20:42 +02:00
/*
* Sequence count - we record it when starting a measurement and
* skip the latency if the sequence has changed - some other section
* did a maximum and could disturb our measurement with serial console
* printouts, etc. Truly coinciding maximum latencies should be rare
* and what happens together happens separately as well, so this doesnt
* decrease the validity of the maximum found:
*/
static __cacheline_aligned_in_smp unsigned long max_sequence;
2008-10-06 19:06:12 -04:00
#ifdef CONFIG_FUNCTION_TRACER
2008-05-12 21:20:42 +02:00
/*
* irqsoff uses its own tracer function to keep the overhead down:
*/
2008-05-12 21:20:51 +02:00
static void
2008-05-12 21:20:42 +02:00
irqsoff_tracer_call(unsigned long ip, unsigned long parent_ip)
{
struct trace_array *tr = irqsoff_trace;
struct trace_array_cpu *data;
unsigned long flags;
long disabled;
int cpu;
2008-05-12 21:20:44 +02:00
/*
* Does not matter if we preempt. We test the flags
* afterward, to see if irqs are disabled or not.
* If we preempt and get a false positive, the flags
* test will fail.
*/
cpu = raw_smp_processor_id();
if (likely(!per_cpu(tracing_cpu, cpu)))
2008-05-12 21:20:42 +02:00
return;
local_save_flags(flags);
2008-05-12 21:20:44 +02:00
/* slight chance to get a false positive on tracing_cpu */
if (!irqs_disabled_flags(flags))
return;
2008-05-12 21:20:42 +02:00
data = tr->data[cpu];
disabled = atomic_inc_return(&data->disabled);
if (likely(disabled == 1))
trace_function(tr, ip, parent_ip, flags, preempt_count());
2008-05-12 21:20:42 +02:00
atomic_dec(&data->disabled);
}
static struct ftrace_ops trace_ops __read_mostly =
{
.func = irqsoff_tracer_call,
};
2008-10-06 19:06:12 -04:00
#endif /* CONFIG_FUNCTION_TRACER */
2008-05-12 21:20:42 +02:00
/*
* Should this new latency be reported/recorded?
*/
2008-05-12 21:20:51 +02:00
static int report_latency(cycle_t delta)
2008-05-12 21:20:42 +02:00
{
if (tracing_thresh) {
if (delta < tracing_thresh)
return 0;
} else {
if (delta <= tracing_max_latency)
return 0;
}
return 1;
}
2008-05-12 21:20:51 +02:00
static void
2008-05-12 21:20:42 +02:00
check_critical_timing(struct trace_array *tr,
struct trace_array_cpu *data,
unsigned long parent_ip,
int cpu)
{
unsigned long latency, t0, t1;
2008-05-12 21:20:44 +02:00
cycle_t T0, T1, delta;
2008-05-12 21:20:42 +02:00
unsigned long flags;
int pc;
2008-05-12 21:20:42 +02:00
/*
* usecs conversion is slow so we try to delay the conversion
* as long as possible:
*/
T0 = data->preempt_timestamp;
2008-05-12 21:20:46 +02:00
T1 = ftrace_now(cpu);
2008-05-12 21:20:42 +02:00
delta = T1-T0;
local_save_flags(flags);
2008-10-02 19:23:04 -04:00
pc = preempt_count();
2008-05-12 21:20:42 +02:00
if (!report_latency(delta))
goto out;
2008-05-12 21:20:45 +02:00
spin_lock_irqsave(&max_trace_lock, flags);
2008-05-12 21:20:42 +02:00
2008-05-12 21:20:44 +02:00
/* check if we are still the max latency */
if (!report_latency(delta))
goto out_unlock;
trace_function(tr, CALLER_ADDR0, parent_ip, flags, pc);
2008-05-12 21:20:42 +02:00
latency = nsecs_to_usecs(delta);
if (data->critical_sequence != max_sequence)
2008-05-12 21:20:44 +02:00
goto out_unlock;
2008-05-12 21:20:42 +02:00
tracing_max_latency = delta;
t0 = nsecs_to_usecs(T0);
t1 = nsecs_to_usecs(T1);
data->critical_end = parent_ip;
update_max_tr_single(tr, current, cpu);
max_sequence++;
2008-05-12 21:20:44 +02:00
out_unlock:
2008-05-12 21:20:45 +02:00
spin_unlock_irqrestore(&max_trace_lock, flags);
2008-05-12 21:20:44 +02:00
2008-05-12 21:20:42 +02:00
out:
data->critical_sequence = max_sequence;
2008-05-12 21:20:46 +02:00
data->preempt_timestamp = ftrace_now(cpu);
2008-09-29 23:02:41 -04:00
tracing_reset(tr, cpu);
trace_function(tr, CALLER_ADDR0, parent_ip, flags, pc);
2008-05-12 21:20:42 +02:00
}
2008-05-12 21:20:51 +02:00
static inline void
2008-05-12 21:20:42 +02:00
start_critical_timing(unsigned long ip, unsigned long parent_ip)
{
int cpu;
struct trace_array *tr = irqsoff_trace;
struct trace_array_cpu *data;
unsigned long flags;
if (likely(!tracer_enabled))
return;
2008-05-12 21:20:55 +02:00
cpu = raw_smp_processor_id();
if (per_cpu(tracing_cpu, cpu))
2008-05-12 21:20:42 +02:00
return;
2008-05-12 21:20:42 +02:00
data = tr->data[cpu];
2008-05-12 21:20:55 +02:00
if (unlikely(!data) || atomic_read(&data->disabled))
2008-05-12 21:20:42 +02:00
return;
atomic_inc(&data->disabled);
data->critical_sequence = max_sequence;
2008-05-12 21:20:46 +02:00
data->preempt_timestamp = ftrace_now(cpu);
2008-05-12 21:20:42 +02:00
data->critical_start = parent_ip ? : ip;
2008-09-29 23:02:41 -04:00
tracing_reset(tr, cpu);
2008-05-12 21:20:42 +02:00
local_save_flags(flags);
2008-05-12 21:20:42 +02:00
trace_function(tr, ip, parent_ip, flags, preempt_count());
2008-05-12 21:20:42 +02:00
2008-05-12 21:20:55 +02:00
per_cpu(tracing_cpu, cpu) = 1;
2008-05-12 21:20:42 +02:00
2008-05-12 21:20:42 +02:00
atomic_dec(&data->disabled);
}
2008-05-12 21:20:51 +02:00
static inline void
2008-05-12 21:20:42 +02:00
stop_critical_timing(unsigned long ip, unsigned long parent_ip)
{
int cpu;
struct trace_array *tr = irqsoff_trace;
struct trace_array_cpu *data;
unsigned long flags;
2008-05-12 21:20:55 +02:00
cpu = raw_smp_processor_id();
2008-05-12 21:20:42 +02:00
/* Always clear the tracing cpu on stopping the trace */
2008-05-12 21:20:55 +02:00
if (unlikely(per_cpu(tracing_cpu, cpu)))
per_cpu(tracing_cpu, cpu) = 0;
2008-05-12 21:20:42 +02:00
else
return;
if (!tracer_enabled)
2008-05-12 21:20:42 +02:00
return;
data = tr->data[cpu];
2008-09-29 23:02:41 -04:00
if (unlikely(!data) ||
2008-05-12 21:20:42 +02:00
!data->critical_start || atomic_read(&data->disabled))
return;
atomic_inc(&data->disabled);
2008-05-12 21:20:55 +02:00
2008-05-12 21:20:42 +02:00
local_save_flags(flags);
trace_function(tr, ip, parent_ip, flags, preempt_count());
2008-05-12 21:20:42 +02:00
check_critical_timing(tr, data, parent_ip ? : ip, cpu);
2008-05-12 21:20:42 +02:00
data->critical_start = 0;
atomic_dec(&data->disabled);
}
2008-05-12 21:20:42 +02:00
/* start and stop critical timings used to for stoppage (in idle) */
2008-05-12 21:20:51 +02:00
void start_critical_timings(void)
2008-05-12 21:20:42 +02:00
{
2008-05-12 21:20:42 +02:00
if (preempt_trace() || irq_trace())
2008-05-12 21:20:42 +02:00
start_critical_timing(CALLER_ADDR0, CALLER_ADDR1);
}
2008-07-26 15:09:47 +02:00
EXPORT_SYMBOL_GPL(start_critical_timings);
2008-05-12 21:20:42 +02:00
2008-05-12 21:20:51 +02:00
void stop_critical_timings(void)
2008-05-12 21:20:42 +02:00
{
2008-05-12 21:20:42 +02:00
if (preempt_trace() || irq_trace())
2008-05-12 21:20:42 +02:00
stop_critical_timing(CALLER_ADDR0, CALLER_ADDR1);
}
2008-07-26 15:09:47 +02:00
EXPORT_SYMBOL_GPL(stop_critical_timings);
2008-05-12 21:20:42 +02:00
2008-05-12 21:20:42 +02:00
#ifdef CONFIG_IRQSOFF_TRACER
2008-05-12 21:20:42 +02:00
#ifdef CONFIG_PROVE_LOCKING
2008-05-12 21:20:51 +02:00
void time_hardirqs_on(unsigned long a0, unsigned long a1)
2008-05-12 21:20:42 +02:00
{
2008-05-12 21:20:42 +02:00
if (!preempt_trace() && irq_trace())
2008-05-12 21:20:42 +02:00
stop_critical_timing(a0, a1);
}
2008-05-12 21:20:51 +02:00
void time_hardirqs_off(unsigned long a0, unsigned long a1)
2008-05-12 21:20:42 +02:00
{
2008-05-12 21:20:42 +02:00
if (!preempt_trace() && irq_trace())
2008-05-12 21:20:42 +02:00
start_critical_timing(a0, a1);
}
#else /* !CONFIG_PROVE_LOCKING */
/*
* Stubs:
*/
void early_boot_irqs_off(void)
{
}
void early_boot_irqs_on(void)
{
}
void trace_softirqs_on(unsigned long ip)
{
}
void trace_softirqs_off(unsigned long ip)
{
}
2008-05-12 21:20:51 +02:00
inline void print_irqtrace_events(struct task_struct *curr)
2008-05-12 21:20:42 +02:00
{
}
/*
* We are only interested in hardirq on/off events:
*/
2008-05-12 21:20:51 +02:00
void trace_hardirqs_on(void)
2008-05-12 21:20:42 +02:00
{
2008-05-12 21:20:42 +02:00
if (!preempt_trace() && irq_trace())
2008-05-12 21:20:42 +02:00
stop_critical_timing(CALLER_ADDR0, CALLER_ADDR1);
}
EXPORT_SYMBOL(trace_hardirqs_on);
2008-05-12 21:20:51 +02:00
void trace_hardirqs_off(void)
2008-05-12 21:20:42 +02:00
{
2008-05-12 21:20:42 +02:00
if (!preempt_trace() && irq_trace())
2008-05-12 21:20:42 +02:00
start_critical_timing(CALLER_ADDR0, CALLER_ADDR1);
}
EXPORT_SYMBOL(trace_hardirqs_off);
2008-05-12 21:20:51 +02:00
void trace_hardirqs_on_caller(unsigned long caller_addr)
2008-05-12 21:20:42 +02:00
{
2008-05-12 21:20:42 +02:00
if (!preempt_trace() && irq_trace())
2008-05-12 21:20:42 +02:00
stop_critical_timing(CALLER_ADDR0, caller_addr);
}
EXPORT_SYMBOL(trace_hardirqs_on_caller);
2008-05-12 21:20:51 +02:00
void trace_hardirqs_off_caller(unsigned long caller_addr)
2008-05-12 21:20:42 +02:00
{
2008-05-12 21:20:42 +02:00
if (!preempt_trace() && irq_trace())
2008-05-12 21:20:42 +02:00
start_critical_timing(CALLER_ADDR0, caller_addr);
}
EXPORT_SYMBOL(trace_hardirqs_off_caller);
#endif /* CONFIG_PROVE_LOCKING */
2008-05-12 21:20:42 +02:00
#endif /* CONFIG_IRQSOFF_TRACER */
#ifdef CONFIG_PREEMPT_TRACER
2008-05-12 21:20:51 +02:00
void trace_preempt_on(unsigned long a0, unsigned long a1)
2008-05-12 21:20:42 +02:00
{
if (preempt_trace())
stop_critical_timing(a0, a1);
2008-05-12 21:20:42 +02:00
}
2008-05-12 21:20:51 +02:00
void trace_preempt_off(unsigned long a0, unsigned long a1)
2008-05-12 21:20:42 +02:00
{
if (preempt_trace())
start_critical_timing(a0, a1);
2008-05-12 21:20:42 +02:00
}
#endif /* CONFIG_PREEMPT_TRACER */
2008-05-12 21:20:42 +02:00
static void start_irqsoff_tracer(struct trace_array *tr)
{
register_ftrace_function(&trace_ops);
if (tracing_is_enabled())
tracer_enabled = 1;
else
tracer_enabled = 0;
2008-05-12 21:20:42 +02:00
}
static void stop_irqsoff_tracer(struct trace_array *tr)
{
tracer_enabled = 0;
2008-05-12 21:20:44 +02:00
unregister_ftrace_function(&trace_ops);
2008-05-12 21:20:42 +02:00
}
2008-05-12 21:20:42 +02:00
static void __irqsoff_tracer_init(struct trace_array *tr)
2008-05-12 21:20:42 +02:00
{
save_lat_flag = trace_flags & TRACE_ITER_LATENCY_FMT;
trace_flags |= TRACE_ITER_LATENCY_FMT;
tracing_max_latency = 0;
2008-05-12 21:20:42 +02:00
irqsoff_trace = tr;
2008-05-12 21:20:55 +02:00
/* make sure that the tracer is visible */
2008-05-12 21:20:42 +02:00
smp_wmb();
2008-11-07 22:36:02 -05:00
start_irqsoff_tracer(tr);
2008-05-12 21:20:42 +02:00
}
static void irqsoff_tracer_reset(struct trace_array *tr)
{
2008-11-07 22:36:02 -05:00
stop_irqsoff_tracer(tr);
if (!save_lat_flag)
trace_flags &= ~TRACE_ITER_LATENCY_FMT;
2008-05-12 21:20:42 +02:00
}
static void irqsoff_tracer_start(struct trace_array *tr)
{
tracer_enabled = 1;
}
static void irqsoff_tracer_stop(struct trace_array *tr)
{
tracer_enabled = 0;
2008-05-12 21:20:42 +02:00
}
2008-05-12 21:20:42 +02:00
#ifdef CONFIG_IRQSOFF_TRACER
static int irqsoff_tracer_init(struct trace_array *tr)
2008-05-12 21:20:42 +02:00
{
trace_type = TRACER_IRQS_OFF;
__irqsoff_tracer_init(tr);
return 0;
2008-05-12 21:20:42 +02:00
}
2008-05-12 21:20:42 +02:00
static struct tracer irqsoff_tracer __read_mostly =
{
.name = "irqsoff",
.init = irqsoff_tracer_init,
.reset = irqsoff_tracer_reset,
.start = irqsoff_tracer_start,
.stop = irqsoff_tracer_stop,
2008-05-12 21:20:42 +02:00
.print_max = 1,
2008-05-12 21:20:44 +02:00
#ifdef CONFIG_FTRACE_SELFTEST
.selftest = trace_selftest_startup_irqsoff,
#endif
2008-05-12 21:20:42 +02:00
};
2008-05-12 21:20:42 +02:00
# define register_irqsoff(trace) register_tracer(&trace)
#else
# define register_irqsoff(trace) do { } while (0)
#endif
#ifdef CONFIG_PREEMPT_TRACER
static int preemptoff_tracer_init(struct trace_array *tr)
2008-05-12 21:20:42 +02:00
{
trace_type = TRACER_PREEMPT_OFF;
__irqsoff_tracer_init(tr);
return 0;
2008-05-12 21:20:42 +02:00
}
static struct tracer preemptoff_tracer __read_mostly =
{
.name = "preemptoff",
.init = preemptoff_tracer_init,
.reset = irqsoff_tracer_reset,
.start = irqsoff_tracer_start,
.stop = irqsoff_tracer_stop,
2008-05-12 21:20:42 +02:00
.print_max = 1,
2008-05-12 21:20:44 +02:00
#ifdef CONFIG_FTRACE_SELFTEST
.selftest = trace_selftest_startup_preemptoff,
#endif
2008-05-12 21:20:42 +02:00
};
# define register_preemptoff(trace) register_tracer(&trace)
#else
# define register_preemptoff(trace) do { } while (0)
#endif
#if defined(CONFIG_IRQSOFF_TRACER) && \
defined(CONFIG_PREEMPT_TRACER)
static int preemptirqsoff_tracer_init(struct trace_array *tr)
2008-05-12 21:20:42 +02:00
{
trace_type = TRACER_IRQS_OFF | TRACER_PREEMPT_OFF;
__irqsoff_tracer_init(tr);
return 0;
2008-05-12 21:20:42 +02:00
}
static struct tracer preemptirqsoff_tracer __read_mostly =
{
.name = "preemptirqsoff",
.init = preemptirqsoff_tracer_init,
.reset = irqsoff_tracer_reset,
.start = irqsoff_tracer_start,
.stop = irqsoff_tracer_stop,
2008-05-12 21:20:42 +02:00
.print_max = 1,
2008-05-12 21:20:44 +02:00
#ifdef CONFIG_FTRACE_SELFTEST
.selftest = trace_selftest_startup_preemptirqsoff,
#endif
2008-05-12 21:20:42 +02:00
};
# define register_preemptirqsoff(trace) register_tracer(&trace)
#else
# define register_preemptirqsoff(trace) do { } while (0)
#endif
2008-05-12 21:20:42 +02:00
__init static int init_irqsoff_tracer(void)
{
2008-05-12 21:20:42 +02:00
register_irqsoff(irqsoff_tracer);
register_preemptoff(preemptoff_tracer);
register_preemptirqsoff(preemptirqsoff_tracer);
2008-05-12 21:20:42 +02:00
return 0;
}
device_initcall(init_irqsoff_tracer);