#include <linux/kernel.h>

#include <linux/blkdev.h>

#include <linux/blktrace_api.h>

#include <linux/percpu.h>

#include <linux/init.h>

#include <linux/mutex.h>

#include <linux/slab.h>

#include <linux/debugfs.h>

#include <linux/export.h>

#include <linux/time.h>

#include <linux/uaccess.h>

#include <trace/events/block.h>

#include "trace_output.h"

#ifdef CONFIG_BLK_DEV_IO_TRACE

static unsigned int blktrace_seq __read_mostly = 1;

static struct trace_array *blk_tr;

static bool blk_tracer_enabled __read_mostly;

#define TRACE_BLK_OPT_CLASSIC 0x1

static struct tracer_opt blk_tracer_opts[] = {

/* Default disable the minimalistic output */

{ TRACER_OPT(blk_classic, TRACE_BLK_OPT_CLASSIC) },

{ }

};

static struct tracer_flags blk_tracer_flags = {

.val = 0,

.opts = blk_tracer_opts,

};

static atomic_t blk_probes_ref = ATOMIC_INIT(0);

static void blk_register_tracepoints(void);

static void blk_unregister_tracepoints(void);

static void trace_note(struct blk_trace *bt, pid_t pid, int action,

const void *data, size_t len)

{

struct blk_io_trace *t;

struct ring_buffer_event *event = NULL;

struct ring_buffer *buffer = NULL;

int pc = 0;

int cpu = smp_processor_id();

bool blk_tracer = blk_tracer_enabled;

if (blk_tracer) {

buffer = blk_tr->buffer;

pc = preempt_count();

event = trace_buffer_lock_reserve(buffer, TRACE_BLK,

sizeof(*t) + len,

0, pc);

if (!event)

return;

t = ring_buffer_event_data(event);

goto record_it;

}

if (!bt->rchan)

return;

t = relay_reserve(bt->rchan, sizeof(*t) + len);

if (t) {

t->magic = BLK_IO_TRACE_MAGIC | BLK_IO_TRACE_VERSION;

t->time = ktime_to_ns(ktime_get());

record_it:

t->device = bt->dev;

t->action = action;

t->pid = pid;

t->cpu = cpu;

t->pdu_len = len;

memcpy((void *) t + sizeof(*t), data, len);

if (blk_tracer)

trace_buffer_unlock_commit(buffer, event, 0, pc);

}

}

static void trace_note_tsk(struct blk_trace *bt, struct task_struct *tsk)

{

tsk->btrace_seq = blktrace_seq;

trace_note(bt, tsk->pid, BLK_TN_PROCESS, tsk->comm, sizeof(tsk->comm));

}

static void trace_note_time(struct blk_trace *bt)

{

struct timespec now;

unsigned long flags;

u32 words[2];

getnstimeofday(&now);

words[0] = now.tv_sec;

words[1] = now.tv_nsec;

local_irq_save(flags);

trace_note(bt, 0, BLK_TN_TIMESTAMP, words, sizeof(words));

local_irq_restore(flags);

}

void __trace_note_message(struct blk_trace *bt, const char *fmt, ...)

{

int n;

va_list args;

unsigned long flags;

char *buf;

if (unlikely(bt->trace_state != Blktrace_running &&

!blk_tracer_enabled))

return;

/*

if (!(bt->act_mask & BLK_TC_NOTIFY))

return;

local_irq_save(flags);

buf = per_cpu_ptr(bt->msg_data, smp_processor_id());

va_start(args, fmt);

n = vscnprintf(buf, BLK_TN_MAX_MSG, fmt, args);

va_end(args);

trace_note(bt, 0, BLK_TN_MESSAGE, buf, n);

local_irq_restore(flags);

}

EXPORT_SYMBOL_GPL(__trace_note_message);

static int act_log_check(struct blk_trace *bt, u32 what, sector_t sector,

pid_t pid)

{

if (((bt->act_mask << BLK_TC_SHIFT) & what) == 0)

return 1;

if (sector && (sector < bt->start_lba || sector > bt->end_lba))

return 1;

if (bt->pid && pid != bt->pid)

return 1;

return 0;

}

static const u32 ddir_act[2] = { BLK_TC_ACT(BLK_TC_READ),

BLK_TC_ACT(BLK_TC_WRITE) };

#define BLK_TC_RAHEAD BLK_TC_AHEAD

#define MASK_TC_BIT(rw, __name) ((rw & REQ_ ## __name) << \

(ilog2(BLK_TC_ ## __name) + BLK_TC_SHIFT - __REQ_ ## __name))

static void __blk_add_trace(struct blk_trace *bt, sector_t sector, int bytes,

int rw, u32 what, int error, int pdu_len, void *pdu_data)

{

struct task_struct *tsk = current;

struct ring_buffer_event *event = NULL;

struct ring_buffer *buffer = NULL;

struct blk_io_trace *t;

unsigned long flags = 0;

unsigned long *sequence;

pid_t pid;

int cpu, pc = 0;

bool blk_tracer = blk_tracer_enabled;

if (unlikely(bt->trace_state != Blktrace_running && !blk_tracer))

return;

what |= ddir_act[rw & WRITE];

what |= MASK_TC_BIT(rw, SYNC);

what |= MASK_TC_BIT(rw, RAHEAD);

what |= MASK_TC_BIT(rw, META);

what |= MASK_TC_BIT(rw, DISCARD);

what |= MASK_TC_BIT(rw, FLUSH);

what |= MASK_TC_BIT(rw, FUA);

pid = tsk->pid;

if (act_log_check(bt, what, sector, pid))

return;

cpu = raw_smp_processor_id();

if (blk_tracer) {

tracing_record_cmdline(current);

buffer = blk_tr->buffer;

pc = preempt_count();

event = trace_buffer_lock_reserve(buffer, TRACE_BLK,

sizeof(*t) + pdu_len,

0, pc);

if (!event)

return;

t = ring_buffer_event_data(event);

goto record_it;

}

/*

local_irq_save(flags);

if (unlikely(tsk->btrace_seq != blktrace_seq))

trace_note_tsk(bt, tsk);

t = relay_reserve(bt->rchan, sizeof(*t) + pdu_len);

if (t) {

sequence = per_cpu_ptr(bt->sequence, cpu);

t->magic = BLK_IO_TRACE_MAGIC | BLK_IO_TRACE_VERSION;

t->sequence = ++(*sequence);

t->time = ktime_to_ns(ktime_get());

record_it:

/*

t->cpu = cpu;

t->pid = pid;

t->sector = sector;

t->bytes = bytes;

t->action = what;

t->device = bt->dev;

t->error = error;

t->pdu_len = pdu_len;

if (pdu_len)

memcpy((void *) t + sizeof(*t), pdu_data, pdu_len);

if (blk_tracer) {

trace_buffer_unlock_commit(buffer, event, 0, pc);

return;

}

}

local_irq_restore(flags);

}

static struct dentry *blk_tree_root;

static DEFINE_MUTEX(blk_tree_mutex);

static void blk_trace_free(struct blk_trace *bt)

{

debugfs_remove(bt->msg_file);

debugfs_remove(bt->dropped_file);

relay_close(bt->rchan);

debugfs_remove(bt->dir);

free_percpu(bt->sequence);

free_percpu(bt->msg_data);

kfree(bt);

}

static void blk_trace_cleanup(struct blk_trace *bt)

{

blk_trace_free(bt);

if (atomic_dec_and_test(&blk_probes_ref))

blk_unregister_tracepoints();

}

int blk_trace_remove(struct request_queue *q)

{

struct blk_trace *bt;

bt = xchg(&q->blk_trace, NULL);

if (!bt)

return -EINVAL;

if (bt->trace_state != Blktrace_running)

blk_trace_cleanup(bt);

return 0;

}

EXPORT_SYMBOL_GPL(blk_trace_remove);

static ssize_t blk_dropped_read(struct file *filp, char __user *buffer,

size_t count, loff_t *ppos)

{

struct blk_trace *bt = filp->private_data;

char buf[16];

snprintf(buf, sizeof(buf), "%u\n", atomic_read(&bt->dropped));

return simple_read_from_buffer(buffer, count, ppos, buf, strlen(buf));

}

static const struct file_operations blk_dropped_fops = {

.owner = THIS_MODULE,

.open = simple_open,

.read = blk_dropped_read,

.llseek = default_llseek,

};

static ssize_t blk_msg_write(struct file *filp, const char __user *buffer,

size_t count, loff_t *ppos)

{

char *msg;

struct blk_trace *bt;

if (count >= BLK_TN_MAX_MSG)

return -EINVAL;

msg = kmalloc(count + 1, GFP_KERNEL);

if (msg == NULL)

return -ENOMEM;

if (copy_from_user(msg, buffer, count)) {

kfree(msg);

return -EFAULT;

}

msg[count] = '\0';

bt = filp->private_data;

__trace_note_message(bt, "%s", msg);

kfree(msg);

return count;

}

static const struct file_operations blk_msg_fops = {

.owner = THIS_MODULE,

.open = simple_open,

.write = blk_msg_write,

.llseek = noop_llseek,

};

static int blk_subbuf_start_callback(struct rchan_buf *buf, void *subbuf,

void *prev_subbuf, size_t prev_padding)

{

struct blk_trace *bt;

if (!relay_buf_full(buf))

return 1;

bt = buf->chan->private_data;

atomic_inc(&bt->dropped);

return 0;

}

static int blk_remove_buf_file_callback(struct dentry *dentry)

{

debugfs_remove(dentry);

return 0;

}

static struct dentry *blk_create_buf_file_callback(const char *filename,

int *is_global)

{

return debugfs_create_file(filename, mode, parent, buf,

&relay_file_operations);

}

static struct rchan_callbacks blk_relay_callbacks = {

.subbuf_start = blk_subbuf_start_callback,

.create_buf_file = blk_create_buf_file_callback,

.remove_buf_file = blk_remove_buf_file_callback,

};

static void blk_trace_setup_lba(struct blk_trace *bt,

struct block_device *bdev)

{

struct hd_struct *part = NULL;

if (bdev)

part = bdev->bd_part;

if (part) {

bt->start_lba = part->start_sect;

bt->end_lba = part->start_sect + part->nr_sects;

} else {

bt->start_lba = 0;

bt->end_lba = -1ULL;

}

}

int do_blk_trace_setup(struct request_queue *q, char *name, dev_t dev,

struct block_device *bdev,

struct blk_user_trace_setup *buts)

{

struct blk_trace *old_bt, *bt = NULL;

struct dentry *dir = NULL;

int ret, i;

if (!buts->buf_size || !buts->buf_nr)

return -EINVAL;

strncpy(buts->name, name, BLKTRACE_BDEV_SIZE);

buts->name[BLKTRACE_BDEV_SIZE - 1] = '\0';

/*

for (i = 0; i < strlen(buts->name); i++)

if (buts->name[i] == '/')

buts->name[i] = '_';

bt = kzalloc(sizeof(*bt), GFP_KERNEL);

if (!bt)

return -ENOMEM;

ret = -ENOMEM;

bt->sequence = alloc_percpu(unsigned long);

if (!bt->sequence)

goto err;

bt->msg_data = __alloc_percpu(BLK_TN_MAX_MSG, __alignof__(char));

if (!bt->msg_data)

goto err;

ret = -ENOENT;

mutex_lock(&blk_tree_mutex);

if (!blk_tree_root) {

blk_tree_root = debugfs_create_dir("block", NULL);

if (!blk_tree_root) {

mutex_unlock(&blk_tree_mutex);

goto err;

}

}

mutex_unlock(&blk_tree_mutex);

dir = debugfs_create_dir(buts->name, blk_tree_root);

if (!dir)

goto err;

bt->dir = dir;

bt->dev = dev;

atomic_set(&bt->dropped, 0);

ret = -EIO;

bt->dropped_file = debugfs_create_file("dropped", 0444, dir, bt,

&blk_dropped_fops);

if (!bt->dropped_file)

goto err;

bt->msg_file = debugfs_create_file("msg", 0222, dir, bt, &blk_msg_fops);

if (!bt->msg_file)

goto err;

bt->rchan = relay_open("trace", dir, buts->buf_size,

buts->buf_nr, &blk_relay_callbacks, bt);

if (!bt->rchan)

goto err;

bt->act_mask = buts->act_mask;

if (!bt->act_mask)

bt->act_mask = (u16) -1;

blk_trace_setup_lba(bt, bdev);

/* overwrite with user settings */

if (buts->start_lba)

bt->start_lba = buts->start_lba;

if (buts->end_lba)

bt->end_lba = buts->end_lba;

bt->pid = buts->pid;

bt->trace_state = Blktrace_setup;

ret = -EBUSY;

old_bt = xchg(&q->blk_trace, bt);

if (old_bt) {

(void) xchg(&q->blk_trace, old_bt);

goto err;

}

if (atomic_inc_return(&blk_probes_ref) == 1)

blk_register_tracepoints();

return 0;

err:

blk_trace_free(bt);

return ret;

}

int blk_trace_setup(struct request_queue *q, char *name, dev_t dev,

struct block_device *bdev,

char __user *arg)

{

struct blk_user_trace_setup buts;

int ret;

ret = copy_from_user(&buts, arg, sizeof(buts));

if (ret)

return -EFAULT;

ret = do_blk_trace_setup(q, name, dev, bdev, &buts);

if (ret)

return ret;

if (copy_to_user(arg, &buts, sizeof(buts))) {

blk_trace_remove(q);

return -EFAULT;

}

return 0;

}

EXPORT_SYMBOL_GPL(blk_trace_setup);

#if defined(CONFIG_COMPAT) && defined(CONFIG_X86_64)

static int compat_blk_trace_setup(struct request_queue *q, char *name,

dev_t dev, struct block_device *bdev,

char __user *arg)

{

struct blk_user_trace_setup buts;

struct compat_blk_user_trace_setup cbuts;

int ret;

if (copy_from_user(&cbuts, arg, sizeof(cbuts)))

return -EFAULT;

buts = (struct blk_user_trace_setup) {

.act_mask = cbuts.act_mask,

.buf_size = cbuts.buf_size,

.buf_nr = cbuts.buf_nr,

.start_lba = cbuts.start_lba,

.end_lba = cbuts.end_lba,

.pid = cbuts.pid,

};

memcpy(&buts.name, &cbuts.name, 32);

ret = do_blk_trace_setup(q, name, dev, bdev, &buts);

if (ret)

return ret;

if (copy_to_user(arg, &buts.name, 32)) {

blk_trace_remove(q);

return -EFAULT;

}

return 0;

}

int blk_trace_startstop(struct request_queue *q, int start)

{

int ret;

struct blk_trace *bt = q->blk_trace;

if (bt == NULL)

return -EINVAL;

/*

ret = -EINVAL;

if (start) {

if (bt->trace_state == Blktrace_setup ||

bt->trace_state == Blktrace_stopped) {

blktrace_seq++;

smp_mb();

bt->trace_state = Blktrace_running;

trace_note_time(bt);

ret = 0;

}

} else {

if (bt->trace_state == Blktrace_running) {

bt->trace_state = Blktrace_stopped;

relay_flush(bt->rchan);

ret = 0;

}

}

return ret;

}

EXPORT_SYMBOL_GPL(blk_trace_startstop);

int blk_trace_ioctl(struct block_device *bdev, unsigned cmd, char __user *arg)

{

struct request_queue *q;

int ret, start = 0;

char b[BDEVNAME_SIZE];

q = bdev_get_queue(bdev);

if (!q)

return -ENXIO;

mutex_lock(&bdev->bd_mutex);

switch (cmd) {

case BLKTRACESETUP:

bdevname(bdev, b);

ret = blk_trace_setup(q, b, bdev->bd_dev, bdev, arg);

break;

#if defined(CONFIG_COMPAT) && defined(CONFIG_X86_64)

case BLKTRACESETUP32:

bdevname(bdev, b);

ret = compat_blk_trace_setup(q, b, bdev->bd_dev, bdev, arg);

break;

case BLKTRACESTART:

start = 1;

case BLKTRACESTOP:

ret = blk_trace_startstop(q, start);

break;

case BLKTRACETEARDOWN:

ret = blk_trace_remove(q);

break;

default:

ret = -ENOTTY;

break;

}

mutex_unlock(&bdev->bd_mutex);

return ret;

}

void blk_trace_shutdown(struct request_queue *q)

{

if (q->blk_trace) {

blk_trace_startstop(q, 0);

blk_trace_remove(q);

}

}

static void blk_add_trace_rq(struct request_queue *q, struct request *rq,

u32 what)

{

struct blk_trace *bt = q->blk_trace;

if (likely(!bt))

return;

if (rq->cmd_type == REQ_TYPE_BLOCK_PC) {

what |= BLK_TC_ACT(BLK_TC_PC);

__blk_add_trace(bt, 0, blk_rq_bytes(rq), rq->cmd_flags,

what, rq->errors, rq->cmd_len, rq->cmd);

} else {

what |= BLK_TC_ACT(BLK_TC_FS);

__blk_add_trace(bt, blk_rq_pos(rq), blk_rq_bytes(rq),

rq->cmd_flags, what, rq->errors, 0, NULL);

}

}

static void blk_add_trace_rq_abort(void *ignore,

struct request_queue *q, struct request *rq)

{

blk_add_trace_rq(q, rq, BLK_TA_ABORT);

}

static void blk_add_trace_rq_insert(void *ignore,

struct request_queue *q, struct request *rq)

{

blk_add_trace_rq(q, rq, BLK_TA_INSERT);

}

static void blk_add_trace_rq_issue(void *ignore,

struct request_queue *q, struct request *rq)

{

blk_add_trace_rq(q, rq, BLK_TA_ISSUE);

}

static void blk_add_trace_rq_requeue(void *ignore,

struct request_queue *q,

struct request *rq)

{

blk_add_trace_rq(q, rq, BLK_TA_REQUEUE);

}

static void blk_add_trace_rq_complete(void *ignore,

struct request_queue *q,

struct request *rq)

{

blk_add_trace_rq(q, rq, BLK_TA_COMPLETE);

}

static void blk_add_trace_bio(struct request_queue *q, struct bio *bio,

u32 what, int error)

{

struct blk_trace *bt = q->blk_trace;

if (likely(!bt))

return;

if (!error && !bio_flagged(bio, BIO_UPTODATE))

error = EIO;

__blk_add_trace(bt, bio->bi_sector, bio->bi_size, bio->bi_rw, what,

error, 0, NULL);

}

static void blk_add_trace_bio_bounce(void *ignore,

struct request_queue *q, struct bio *bio)

{

blk_add_trace_bio(q, bio, BLK_TA_BOUNCE, 0);

}

static void blk_add_trace_bio_complete(void *ignore,

struct request_queue *q, struct bio *bio,

int error)

{

blk_add_trace_bio(q, bio, BLK_TA_COMPLETE, error);

}

static void blk_add_trace_bio_backmerge(void *ignore,

struct request_queue *q,

struct bio *bio)

{

blk_add_trace_bio(q, bio, BLK_TA_BACKMERGE, 0);

}

static void blk_add_trace_bio_frontmerge(void *ignore,

struct request_queue *q,

struct bio *bio)

{

blk_add_trace_bio(q, bio, BLK_TA_FRONTMERGE, 0);

}

static void blk_add_trace_bio_queue(void *ignore,

struct request_queue *q, struct bio *bio)

{

blk_add_trace_bio(q, bio, BLK_TA_QUEUE, 0);

}

static void blk_add_trace_getrq(void *ignore,

struct request_queue *q,

struct bio *bio, int rw)

{

if (bio)

blk_add_trace_bio(q, bio, BLK_TA_GETRQ, 0);

else {

struct blk_trace *bt = q->blk_trace;

if (bt)

__blk_add_trace(bt, 0, 0, rw, BLK_TA_GETRQ, 0, 0, NULL);

}

}

static void blk_add_trace_sleeprq(void *ignore,

struct request_queue *q,

struct bio *bio, int rw)

{

if (bio)

blk_add_trace_bio(q, bio, BLK_TA_SLEEPRQ, 0);

else {

struct blk_trace *bt = q->blk_trace;

if (bt)

__blk_add_trace(bt, 0, 0, rw, BLK_TA_SLEEPRQ,

0, 0, NULL);

}

}

static void blk_add_trace_plug(void *ignore, struct request_queue *q)

{

struct blk_trace *bt = q->blk_trace;

if (bt)

__blk_add_trace(bt, 0, 0, 0, BLK_TA_PLUG, 0, 0, NULL);

}

static void blk_add_trace_unplug(void *ignore, struct request_queue *q,

unsigned int depth, bool explicit)

{

struct blk_trace *bt = q->blk_trace;

if (bt) {

__be64 rpdu = cpu_to_be64(depth);

u32 what;

if (explicit)

what = BLK_TA_UNPLUG_IO;

else

what = BLK_TA_UNPLUG_TIMER;

__blk_add_trace(bt, 0, 0, 0, what, 0, sizeof(rpdu), &rpdu);

}

}

static void blk_add_trace_split(void *ignore,

struct request_queue *q, struct bio *bio,

unsigned int pdu)

{

struct blk_trace *bt = q->blk_trace;

if (bt) {

__be64 rpdu = cpu_to_be64(pdu);

__blk_add_trace(bt, bio->bi_sector, bio->bi_size, bio->bi_rw,

BLK_TA_SPLIT, !bio_flagged(bio, BIO_UPTODATE),

sizeof(rpdu), &rpdu);

}

}

static void blk_add_trace_bio_remap(void *ignore,

struct request_queue *q, struct bio *bio,

dev_t dev, sector_t from)

{

struct blk_trace *bt = q->blk_trace;

struct blk_io_trace_remap r;

if (likely(!bt))

return;

r.device_from = cpu_to_be32(dev);

r.device_to = cpu_to_be32(bio->bi_bdev->bd_dev);

r.sector_from = cpu_to_be64(from);

__blk_add_trace(bt, bio->bi_sector, bio->bi_size, bio->bi_rw,

BLK_TA_REMAP, !bio_flagged(bio, BIO_UPTODATE),

sizeof(r), &r);

}

static void blk_add_trace_rq_remap(void *ignore,

struct request_queue *q,

struct request *rq, dev_t dev,

sector_t from)

{

struct blk_trace *bt = q->blk_trace;

struct blk_io_trace_remap r;

if (likely(!bt))

return;

r.device_from = cpu_to_be32(dev);

r.device_to = cpu_to_be32(disk_devt(rq->rq_disk));

r.sector_from = cpu_to_be64(from);

__blk_add_trace(bt, blk_rq_pos(rq), blk_rq_bytes(rq),

rq_data_dir(rq), BLK_TA_REMAP, !!rq->errors,

sizeof(r), &r);

}

void blk_add_driver_data(struct request_queue *q,

struct request *rq,

void *data, size_t len)

{

struct blk_trace *bt = q->blk_trace;

if (likely(!bt))

return;

if (rq->cmd_type == REQ_TYPE_BLOCK_PC)

__blk_add_trace(bt, 0, blk_rq_bytes(rq), 0,

BLK_TA_DRV_DATA, rq->errors, len, data);

else

__blk_add_trace(bt, blk_rq_pos(rq), blk_rq_bytes(rq), 0,

BLK_TA_DRV_DATA, rq->errors, len, data);

}

EXPORT_SYMBOL_GPL(blk_add_driver_data);

static void blk_register_tracepoints(void)

{

int ret;

ret = register_trace_block_rq_abort(blk_add_trace_rq_abort, NULL);

WARN_ON(ret);

ret = register_trace_block_rq_insert(blk_add_trace_rq_insert, NULL);

WARN_ON(ret);

ret = register_trace_block_rq_issue(blk_add_trace_rq_issue, NULL);

WARN_ON(ret);

ret = register_trace_block_rq_requeue(blk_add_trace_rq_requeue, NULL);

WARN_ON(ret);

ret = register_trace_block_rq_complete(blk_add_trace_rq_complete, NULL);

WARN_ON(ret);

ret = register_trace_block_bio_bounce(blk_add_trace_bio_bounce, NULL);

WARN_ON(ret);

ret = register_trace_block_bio_complete(blk_add_trace_bio_complete, NULL);

WARN_ON(ret);

ret = register_trace_block_bio_backmerge(blk_add_trace_bio_backmerge, NULL);

WARN_ON(ret);

ret = register_trace_block_bio_frontmerge(blk_add_trace_bio_frontmerge, NULL);

WARN_ON(ret);

ret = register_trace_block_bio_queue(blk_add_trace_bio_queue, NULL);

WARN_ON(ret);

ret = register_trace_block_getrq(blk_add_trace_getrq, NULL);

WARN_ON(ret);

ret = register_trace_block_sleeprq(blk_add_trace_sleeprq, NULL);

WARN_ON(ret);

ret = register_trace_block_plug(blk_add_trace_plug, NULL);

WARN_ON(ret);

ret = register_trace_block_unplug(blk_add_trace_unplug, NULL);

WARN_ON(ret);