fio-post-process

#!/usr/bin/perl
## -*- mode: perl; indent-tabs-mode: nil; perl-indent-level: 4 -*-
## vim: autoindent tabstop=4 shiftwidth=4 expandtab softtabstop=4 filetype=perl

use strict;
use warnings;
use JSON::XS;
use Data::Dumper;
BEGIN {
    if (!(exists $ENV{'TOOLBOX_HOME'} && -d "$ENV{'TOOLBOX_HOME'}/perl")) {
    print "This script requires libraries that are provided by the toolbox project.\n";
    print "Toolbox can be acquired from https://github.com/perftool-incubator/toolbox and\n";
    print "then use 'export TOOLBOX_HOME=/path/to/toolbox' so that it can be located.\n";
    exit 1;
    }
}
use lib "$ENV{'TOOLBOX_HOME'}/perl";
use toolbox::json;
use toolbox::metrics;

opendir(my $dh, ".");
for my $log_file (sort readdir($dh)) {
    if ($log_file =~ /fio_([a-z]+)\.(\d+)\.log(\.xz){0,1}$/) {
        printf "found file %s\n", $log_file;
        my @logfile_metrics;
        my $type = $1;
        my $job_id = $2;
        my %names = ('job' => $job_id);
        my %desc = ();
        if ($type eq "iops" ) {
             %desc = ('source' => 'fio', 'class' => 'throughput', 'type' => 'iops');
        } elsif ($type eq "bw") {
             %desc = ('source' => 'fio', 'class' => 'throughput', 'type' => 'bw-KiBs');
        } elsif ($type eq "lat") {
             %desc = ('source' => 'fio', 'class' => 'count', 'type' => 'latency-usec');
        } elsif ($type eq "clat") {
             %desc = ('source' => 'fio', 'class' => 'count', 'type' => 'completion-latency-usec');
        } elsif ($type eq "slat") {
             %desc = ('source' => 'fio', 'class' => 'count', 'type' => 'submission-latency-usec');
        } else {
            printf "fio log file %s not recognized, skippping\n", $type;
        }
        if (defined $desc{'type'}) {
            #open(my $log_fh, $log_file) || die "[ERROR]could not open file " . $log_file;
            (my $rc, my $log_fh) = open_read_text_file($log_file);
            die "[ERROR]could not open file " . $log_file if ($rc > 0);
            while (<$log_fh>) {
                if ( /^(\d+),\s+(\d+),\s+(\d+),\s+(\d+)/ ) {
                    my $timestamp_ms = $1;
                    my $value = $2;
                    my $io_oper = $3;
                    continue if ($io_oper > 2);
                    s/0/Read/, s/1/Write/, s/2/Trim/ for $io_oper;
                    $names{'cmd'} = $io_oper;
                    my %s = ('end' => int $timestamp_ms, 'value' => $value);
                    log_sample("0", \%desc, \%names, \%s);
                }
            }
            close($log_fh);
        }
    } else {
        printf "Not considering %s for processing\n", $log_file;
    }
}
my $metric_data_name = finish_samples();
# In any benchmark post-process script, the metrics generated need to be attributed to a
# time-period (AKA benchmark-phase).  The period which is used to report and offical
# result for the benchmark is the 'measurement' period.  Ohter periods thay may exist
# could be "warm-up", "prep", etc.
my %sample;
my @periods;
my %period = ('name' => 'measurement');
my @metric_files = ( $metric_data_name );
$period{'metric-files'} = \@metric_files;
push(@periods, \%period);
$sample{'primary-metric'} = "iops";
$sample{'primary-period'} = "measurement";
$sample{'benchmark'} = "fio";
$sample{'periods'} = \@periods;
$sample{'rickshaw-bench-metric'}{'schema'}{'version'} = "2021.04.12";
closedir $dh;
my $coder = JSON::XS->new;
open(JSON_FH, ">post-process-data.json") || die("Could not open file post-process-data.json for writing\n");
print JSON_FH $coder->encode(\%sample);
close JSON_FH;