Changeset 327

Timestamp:

12/03/06 05:44:08 (6 years ago)

Author:

lincoln

Message:

lies. damn lies. and statistics

Files:

• util/server_daily_maint (modified) (16 diffs)

util/server_daily_maint

@@ -14 +14 @@
 my $store_dir = $ENV{HOME} . "/whuffy.com/dev/stats";    # where we retrieve our feedback
 my $graph_dir = $ENV{HOME} . "/whuffy.com/dev/feedback"; # where we store our graphs
-my $debug = 1;
+my $debug = 0;
 
 use strict;
@@ -235 +235 @@
 sub record_duration
 {
-        my ($where, $dur) = @_;
-        $where->{duration} += $dur;
-        $where->{duration_max} = $dur if (!defined $where->{duration_max});
-        $where->{duration_max} = $dur if ($dur > $where->{duration_max});
-        $where->{duration_min} = $dur if (!defined $where->{duration_min});
-        $where->{duration_min} = $dur if ($dur < $where->{duration_max});
+        my ($where, $suffix, $dur) = @_;
+        $suffix .= "_" if ($suffix ne "");
+
+        $where->{$suffix."duration"} += $dur;
+        $where->{$suffix."duration_count"} ++;
+        $where->{$suffix."duration_max"} = $dur if (!defined $where->{$suffix."duration_max"});
+        $where->{$suffix."duration_max"} = $dur if ($dur > $where->{$suffix."duration_max"});
+        $where->{$suffix."duration_min"} = $dur if (!defined $where->{$suffix."duration_min"});
+        $where->{$suffix."duration_min"} = $dur if ($dur < $where->{$suffix."duration_min"});
 }
 
@@ -291 +294 @@
                 printf " - gathering statistics from $store_dir/$file ($file_modtime)\n" if ($debug);
 
+                my $parsed = 0;
+                $store->{per_hour}->[0]->{visits}++;             # total # of visits seen
+                $store->{total}->{visits}++;
+
                 open(F,"<$store_dir/$file") || die "could not open $store_dir/$file: $!\n";
                 while(<F>) {
@@ -297 +304 @@
                                 # shepherd status response
                                 my ($sysid, $user_starttime, $duration, $gmt_offset, $components_used) = ($1,$2,$3,$4,$5);
+                                $parsed = 1;
 
                                 #
                                 # usage statistics
                                 #
-                                $store->{per_hour}->[0]->{visits}++;             # total # of visits seen
-                                $store->{total}->{visits}++;
 
                                 # shepherd's duration (total count, min, max, average from duration/visits)
-                                &record_duration($store->{per_hour}->[0], $duration);
-                                &record_duration($store->{total}, $duration);
+                                &record_duration($store->{per_hour}->[0], "", $duration);
+                                &record_duration($store->{total}, "", $duration);
 
                                 # list of sysid's seen (used to derive # unique users)
@@ -332 +338 @@
 
                                 my $g = "c_".$c."_success";
-                                $store->{per_hour}->[0]->{$g}->{counter}++;
-                                $store->{total}->{$g}->{counter}++;
+                                $store->{per_hour}->[0]->{$g."_counter"}++;
+                                $store->{total}->{$g."_counter"}++;
 
                                 # duration (total count, min, max, average from duration/counter)
-                                &record_duration($store->{per_hour}->[0]->{$g}, $c_dur);
-                                &record_duration($store->{total}->{$g}, $c_dur);
+                                &record_duration($store->{per_hour}->[0], $g, $c_dur);
+                                &record_duration($store->{total}, $g, $c_dur);
 
                                 # version count
-                                $store->{per_hour}->[0]->{$g}->{version_list}->{$c_ver}++;
+                                $store->{per_hour}->[0]->{$g."_version_list"}->{$c_ver}++;
                         } elsif ($_ =~ /^(\S+)\tFAIL\t(\S+)\t(\S+)\t(\S+)\t(\S+)\t(\S+)/) {
                                 # fail line
@@ -346 +352 @@
 
                                 my $g = "c_".$c."_fail";
-                                $store->{per_hour}->[0]->{$g}->{counter}++;
-                                $store->{total}->{$g}->{counter}++;
+                                $store->{per_hour}->[0]->{$g."_counter"}++;
+                                $store->{total}->{$g."_counter"}++;
 
                                 # duration (total count, min, max, average from duration/counter)
-                                &record_duration($store->{per_hour}->[0]->{$g}, $c_dur);
-                                &record_duration($store->{total}->{$g}, $c_dur);
+                                &record_duration($store->{per_hour}->[0], $g, $c_dur);
+                                &record_duration($store->{total}, $g, $c_dur);
 
                                 # version count
-                                $store->{per_hour}->[0]->{$g}->{version_list}->{$c_ver}++;
+                                $store->{per_hour}->[0]->{$g."_version_list"}->{$c_ver}++;
                         } elsif ($_ =~ /^(\S+)\tstats\t(.*)$/) {
                                 # component statistics line
@@ -361 +367 @@
                 }
                 close(F);
+
+                if (!$parsed) {
+                        # anonymous, old version or no details
+                        $store->{per_hour}->[0]->{visits_with_no_details}++;
+                        $store->{total}->{visits_with_no_details}++;
+                }
 
                 # rename processed files so we don't re-process them
@@ -390 +402 @@
 sub report_count_data
 {
-        my ($field) = @_;
+        my ($field, $reptype) = @_;
+        $reptype = "all" if (!defined $reptype);
+
         my $r = "";
 
-        $r .= sprintf "<li>in last 60 minutes: <b>%s</b> (%s previous)<br>\n",
-                numvar($store->{per_hour}->[0]->{$field},$field), numvar($store->{per_hour}->[1]->{$field},$field);
+        $r .= sprintf "<li>in last 60 minutes: <b>%s</b> (previous %s)<br>\n",
+                numvar($store->{per_hour}->[0]->{$field},$field), numvar($store->{per_hour}->[1]->{$field},$field)
+                if ($reptype eq "all");
+
         $r .= sprintf "<li>in last 24 hours: <b>%s</b> (previous %s)<br>\n",
                 numvar($store->{per_day}->[0]->{$field},$field), numvar($store->{per_day}->[1]->{$field},$field);
+
         $r .= sprintf "<li>in last 7 days: <b>%s</b> (previous %s)<br>\n",
                 numvar($store->{per_week}->[0]->{$field},$field), numvar($store->{per_week}->[1]->{$field},$field);
+
         $r .= sprintf "<li>in last 30 days: <b>%s</b> (previous %s)<br>\n", 
-                numvar($store->{per_month}->[0]->{$field},$field), numvar($store->{per_month}->[1]->{$field},$field);
+                numvar($store->{per_month}->[0]->{$field},$field), numvar($store->{per_month}->[1]->{$field},$field)
+                if ($reptype eq "all");
+
         $r .= sprintf "<li><b>TOTAL SEEN: <u>%s</u></b><br>\n", numvar($store->{total}->{$field},$field);
 
@@ -412 +432 @@
         my $sec = shift;
 
-        return sprintf "%dh %02dm %02ds",
+        return sprintf "%dh %02dm %02ds (%ds)",
                 int($sec / 3600),
                 int(($sec % 3600) / 60),
-                $sec % 60;
+                $sec % 60,
+                $sec;
 }
 
@@ -446 +467 @@
                 $max = $data{$k} if (!defined $max);
                 $min = $data{$k} if ($data{$k} < $min);
-                $max = $data{$k} if ($data{$k} > $min);
+                $max = $data{$k} if ($data{$k} > $max);
         }
 
@@ -457 +478 @@
 
         print "Generating bar graph $gname ..\n" if ($debug);
-
         my $graph = SVG::TT::Graph::Bar->new({
                 'show_graph_title' => 1,
@@ -487 +507 @@
 sub graph_one_count_data
 {
-        my ($gname, $data, $title, $time_format, $timediv) = @_;
+        my ($gname, $title, $time_format, $timediv, $data) = @_;
         my $gheight = 120;
         my $gwidth = 250;
 
         print "Generating graph $gname ..\n" if ($debug);
-
         # work out scale..
-        my $min, my $max, my $valnum = 0;
-        foreach my $val (@$data) {
-                $valnum++;
-                next if ($valnum % 2 == 1);
-
-                $min = $val if (!defined $min);
-                $max = $val if (!defined $max);
-                $min = $val if ($val < $min);
-                $max = $val if ($val > $min);
-        }
-        my $scalediv = ($max - $min) / 3;
+        my $min, my $max;
+        foreach my $dset (@$data) {
+                my $valnum = 0;
+                foreach my $val (@$dset) {
+                        $valnum++;
+                        next if ($valnum % 2 == 1);
+
+                        $min = $val if (!defined $min);
+                        $max = $val if (!defined $max);
+                        $min = $val if ($val < $min);
+                        $max = $val if ($val > $max);
+                }
+        }
+        my $scalediv = ($max-$min)/3;
         $scalediv = 1 if ($scalediv < 1);
 
@@ -524 +546 @@
                 'timescale_divisions' => $timediv,
                 });
-        $graph->add_data({
-                'data' => $data,
-                });
+
+        foreach my $dset (@$data) {
+                $graph->add_data({
+                        'data' => \@$dset,
+                        });
+        }
 
         open(SVG,">$graph_dir/$gname".".svg") || die "could not create $graph_dir/$gname".".svg: $!\n";
@@ -540 +565 @@
 sub graph_count_data
 {
-        my ($field, $max_hrs, $max_days, $max_weeks, $max_months) = @_;
+        my ($max_hrs, $max_days, $max_weeks, $max_months, @fields) = @_;
         my $r = "";
 
         # hours
-        my @d1;
-        for (my $i = ($max_hrs-1); $i >= 0; $i--) {
-                my $val = numvar($store->{per_hour}->[$i]->{$field},$field,0);
-                my $time_offset = $starttime + $store->{tzdiff} - (60*60*$i);
-                push(@d1, strftime("%Y-%m-%d %H:%M:%S", localtime($time_offset)), $val);
+        if ($max_hrs > 0) {
+                my $fieldnum = 0, my $d, my $fielddesc = "";
+                foreach my $field (@fields) {
+                        for (my $i = ($max_hrs-1); $i >= 0; $i--) {
+                                my $val = numvar($store->{per_hour}->[$i]->{$field},$field,0);
+                                $val /= numvar($store->{per_hour}->[$i]->{duration_count},$field,1) if ($field =~ /duration$/);
+                                my $time_offset = $starttime + $store->{tzdiff} - (60*60*$i);
+                                push(@{$d->[$fieldnum]}, strftime("%Y-%m-%d %H:%M:%S", localtime($time_offset)), $val);
+                        }
+                        $fieldnum++;
+                        $fielddesc .= $field."_";
+                }
+                $r .= &graph_one_count_data($fielddesc."hr", "Last $max_hrs hours", "%a %H:%M", "6 hours", $d);
         }
 
         # days
-        my @d2;
-        for (my $i = ($max_days-1); $i >= 0; $i--) {
-                my $val = numvar($store->{per_day}->[$i]->{$field},$field,0);
-                my $time_offset = $starttime + $store->{tzdiff} - (60*60*24*$i);
-                push(@d2, strftime("%Y-%m-%d %H:%M:%S", localtime($time_offset)), $val);
+        if ($max_days > 0) {
+                my $fieldnum = 0, my $d, my $fielddesc = "";
+                foreach my $field (@fields) {
+                        for (my $i = ($max_days-1); $i >= 0; $i--) {
+                                my $val = numvar($store->{per_day}->[$i]->{$field},$field,0);
+                                $val /= numvar($store->{per_hour}->[$i]->{duration_count},$field,1) if ($field =~ /^duration$/);
+                                my $time_offset = $starttime + $store->{tzdiff} - (60*60*24*$i);
+                                push(@{$d->[$fieldnum]}, strftime("%Y-%m-%d %H:%M:%S", localtime($time_offset)), $val);
+                        }
+                        $fieldnum++;
+                        $fielddesc .= $field."_";
+                }
+                $r .= &graph_one_count_data($fielddesc."day", "Last $max_days days", "%a %e %b", "5 days", $d);
         }
 
         # weeks
-        my @d3;
-        for (my $i = ($max_weeks-1); $i >= 0; $i--) {
-                my $val = numvar($store->{per_week}->[$i]->{$field},$field,0);
-                my $time_offset = $starttime + $store->{tzdiff} - (60*60*24*7*$i);
-                push(@d3, strftime("%Y-%m-%d %H:%M:%S", localtime($time_offset)), $val);
+        if ($max_weeks > 0) {
+                my $fieldnum = 0, my $d, my $fielddesc = "";
+                foreach my $field (@fields) {
+                        for (my $i = ($max_weeks-1); $i >= 0; $i--) {
+                                my $val = numvar($store->{per_week}->[$i]->{$field},$field,0);
+                                $val /= numvar($store->{per_hour}->[$i]->{duration_count},$field,1) if ($field =~ /^duration$/);
+                                my $time_offset = $starttime + $store->{tzdiff} - (60*60*24*7*$i);
+                                push(@{$d->[$fieldnum]}, strftime("%Y-%m-%d %H:%M:%S", localtime($time_offset)), $val);
+                        }
+                        $fieldnum++;
+                        $fielddesc .= $field."_";
+                }
+                $r .= &graph_one_count_data($fielddesc."wk", "Last $max_weeks weeks", "%e %b %y", "3 weeks", $d);
         }
 
         # months
-        my @d4;
-        for (my $i = ($max_months-1); $i >= 0; $i--) {
-                my $val = numvar($store->{per_week}->[$i]->{$field},$field,0);
-                my $time_offset = $starttime + $store->{tzdiff} - (60*60*24*30*$i);
-                push(@d4, strftime("%Y-%m-%d %H:%M:%S", localtime($time_offset)), $val);
-        }
-
-        $r .= &graph_one_count_data($field."_hr", \@d1, "Last $max_hrs hours", "%a %H:%M", "6 hours") if (defined $max_hrs);
-        $r .= &graph_one_count_data($field."_day", \@d2, "Last $max_days days", "%a %e %b", "5 days") if (defined $max_days);
-        $r .= &graph_one_count_data($field."_wk", \@d3, "Last $max_weeks weeks", "%e %b %y", "3 weeks") if (defined $max_weeks);
-        $r .= &graph_one_count_data($field."_mn", \@d4, "Last $max_months months", "%b %y", "4 months") if (defined $max_months);
+        if ($max_months > 0) {
+                my $fieldnum = 0, my $d, my $fielddesc = "";
+                foreach my $field (@fields) {
+                        for (my $i = ($max_months-1); $i >= 0; $i--) {
+                                my $val = numvar($store->{per_week}->[$i]->{$field},$field,0);
+                                $val /= numvar($store->{per_hour}->[$i]->{duration_count},$field,1) if ($field =~ /^duration$/);
+                                my $time_offset = $starttime + $store->{tzdiff} - (60*60*24*30*$i);
+                                push(@{$d->[$fieldnum]}, strftime("%Y-%m-%d %H:%M:%S", localtime($time_offset)), $val);
+                        }
+                        $fieldnum++;
+                        $fielddesc .= $field."_";
+                }
+                $r .= &graph_one_count_data($fielddesc."mn", "Last $max_months months", "%b %y", "4 months", $d);
+        }
+
         return $r;
 }
@@ -586 +639 @@
 sub generate_reports
 {
-        open(F,">$graph_dir/main.html") || die "couldn't open $graph_dir for writing: $!\n";
-
-        my $now = localtime($starttime);
-
-        print F <<EOF
+        my $hdr = "
 <html>
 <head>
@@ -603 +652 @@
 body.home h2 { font: normal 24px/.8em Slyphaen, Times, serif; color: #DFB020; padding: 0; clear: both; margin: 20 0 10 0; }
 body.home p { clear: both; }
+table { border-top: 1px dotted; padding: 3px; cellpadding: 3px; }
+td { border-bottom: 1px dotted; }
 input { font: bold 12px Arial; }
 </style>
 </head>
-
-<body class="home">
-<h1>Shepherd Overall statistics <font size=-1>(as of $now)</font></h1>
-<table style='border: 1px dotted; border-collapse: separate; border-spacing: 1px dotted; padding: 3px; cellpadding: 3px;' align=left>
-<tr>
-EOF
-;
-        print F "<tr'><td align=left valign=top nowrap><h2>Shepherd usage</h2>\n";
-        print F "Number of times shepherd reported in.<br>\n";
-        print F report_count_data("visits");
-        print F "</td><td nowrap colspan=4><br>";
-        print F graph_count_data("visits",24,21,12,24);
+<body class='home'>";
+
+        my $now = localtime($starttime);
+        open(F,">$graph_dir/index.html") || die "couldn't open $graph_dir/index.html for writing: $!\n";
+        print F "$hdr<h1>Shepherd Overall statistics <font size=-1>(as of $now)</font></h1>\n";
+        print F "<table align=left>\n";
+        print F "<tr><td align=left valign=top nowrap><h2>Shepherd usage</h2>\n";
+        print F "<b>Number of times shepherd reported in:</b> <font color=pink><i>(red)</i></font><br>\n";
+        print F report_count_data("visits", "all");
+        print F "<br><b>Reported in with no data:</b> <font color=lightblue><i>(blue)</i></font>\n";
+        print F report_count_data("visits_with_no_details", "brief");
+        print F "</td><td nowrap><br>";
+        print F graph_count_data(24,21,12,24, "visits", "visits_with_no_details");
         print F "</td></tr>\n";
 
         print F "<tr><td align=left valign=top nowrap><h2>Unique users</h2>\n";
-        print F "Number of unique systems seen.<br>\n";
+        print F "<b>Number of unique systems seen:</b> <font color=pink><i>(red)</i></font><br>\n";
         print F report_count_data("seen_sysid_list");
-        print F "</td><td nowrap colspan=4><br>";
-        print F graph_count_data("seen_sysid_list",24,21,12,24);
+        print F "</td><td nowrap><br>";
+        print F graph_count_data(24,21,12,24,"seen_sysid_list");
         print F "</td></tr>\n";
 
         print F "<tr><td align=left valign=top nowrap><h2>Start time</h2>\n";
-        print F "When shepherd checked in.<br>\n";
+        print F "<b>When shepherd checked in:</b> <font color=pink><i>(red)</i></font><br>\n";
+        print F "<i>(when shepherd is being run)</i><p>";
         print F "(in 15 minute groups, time in GMT+10)";
-        print F "</td><td nowrap colspan=2><br>";
+        print F "</td><td nowrap><br>";
         print F graph_hash_data("when_run_list", "per_day", 0, 7, "Last 7 days", 120, 500);
-        print F "</td><td nowrap colspan=2><br>";
         print F graph_hash_data("when_run_list", "per_month", 0, 1, "Last 30 days", 120, 500);
         print F "</td></tr>\n";
 
         print F "<tr><td align=left valign=top nowrap><h2>Duration</h2>\n";
-        print F "How long shepherd took to run.<br>\n";
-        print F "</td><td nowrap align=left valign=top><br>";
-        printf F "<li>in last 60 minutes:</b><ul><li>%s average (mean)<li>%s min<li>%s max</ul>\n",
-                sec2hms(numvar($store->{per_hour}->[0]->{duration},"",0) / $store->{per_hour}->[0]->{visits}),
-                sec2hms(numvar($store->{per_hour}->[0]->{duration_min},"",0)),
-                sec2hms(numvar($store->{per_hour}->[0]->{duration_max},"",0))
-                if ((defined $store->{per_hour}->[0]->{visits}) && ($store->{per_hour}->[0]->{visits} > 0));
-        print F "</td><td nowrap align=left valign=top><br>";
-        printf F "<li>in last 7 days:</b><ul><li>%s average (mean)<li>%s min<li>%s max</ul>\n",
-                sec2hms(numvar($store->{per_day}->[0]->{duration},"",0) / $store->{per_day}->[0]->{visits}),
+        print F "<b>How long shepherd took to run:</b><br>\n";
+        printf F "<li><b>in last 24 hours:</b><br>".
+                "&nbsp;&nbsp;&nbsp;%s average <i><font color=pink>(red)</font></i><br>".
+                "&nbsp;&nbsp;&nbsp;%s min <i><font color=lightblue>(blue)</font></i><br>".
+                "&nbsp;&nbsp;&nbsp;%s max <i><font color=lightgreen>(green)</font></i><br>\n",
+                sec2hms(numvar($store->{per_day}->[0]->{duration},"",0) / $store->{per_hour}->[0]->{duration_count}),
                 sec2hms(numvar($store->{per_day}->[0]->{duration_min},"",0)),
                 sec2hms(numvar($store->{per_day}->[0]->{duration_max},"",0))
-                if ((defined $store->{per_day}->[0]->{visits}) && ($store->{per_day}->[0]->{visits} > 0));
-        print F "</td><td nowrap align=left valign=top><br>";
-        printf F "<li>in last 30 days:</b><ul><li>%s average (mean)<li>%s min<li>%s max</ul>\n",
-                sec2hms(numvar($store->{per_month}->[0]->{duration},"",0) / $store->{per_month}->[0]->{visits}),
+                if ((defined $store->{per_hour}->[0]->{duration_count}) && ($store->{per_hour}->[0]->{duration_count} > 0));
+        printf F "<li><b>in last month:</b><br>".
+                "&nbsp;&nbsp;&nbsp;%s average <i><font color=pink>(red)</font></i><br>".
+                "&nbsp;&nbsp;&nbsp;%s min <i><font color=lightblue>(blue)</font></i><br>".
+                "&nbsp;&nbsp;&nbsp;%s max <i><font color=lightgreen>(green)</font></i><br>\n",
+                sec2hms(numvar($store->{per_month}->[0]->{duration},"",0) / $store->{per_month}->[0]->{duration_count}),
                 sec2hms(numvar($store->{per_month}->[0]->{duration_min},"",0)),
                 sec2hms(numvar($store->{per_month}->[0]->{duration_max},"",0))
-                if ((defined $store->{per_month}->[0]->{visits}) && ($store->{per_month}->[0]->{visits} > 0));
-        print F "</td><td nowrap align=left valign=top><br>";
-        printf F "<li>OVERALL:</b><ul><li>%s average (mean)<li>%s min<li>%s max</ul>\n",
-                sec2hms(numvar($store->{total}->{duration},"",0) / $store->{total}->{visits}),
-                sec2hms(numvar($store->{total}->{duration_min},"",0)),
-                sec2hms(numvar($store->{total}->{duration_max},"",0))
-                if ((defined $store->{total}->{visits}) && ($store->{total}->{visits} > 0));
+                if ((defined $store->{per_month}->[0]->{duration_count}) && ($store->{per_month}->[0]->{duration_count} > 0));
+        print F "</td><td nowrap align=left>";
+        print F graph_count_data(24,21,12,24,"duration", "duration_min", "duration_max");
         print F "</td></tr>\n";
 
         print F "<tr><td align=left valign=top nowrap><h2>Timezones</h2>\n";
-        print F "Timezone setting on hosts.<br>\n";
-        print F "</td><td nowrap colspan=4><br>";
+        print F "<b>Where shepherd is being run:</b> <font color=pink><i>(red)</i></font><br>\n";
+        print F "<i>(timezone setting of hosts)</i><p>";
+        print F "</td><td nowrap><br>";
         print F graph_hash_data("gmt_offset_list", "per_day", 0, 7, "Last 7 days", 120, 500);
         print F graph_hash_data("gmt_offset_list", "per_month", 0, 1, "Last 30 days", 120, 500);
         print F "</td></tr>\n";
 
-
-# SHEPHERD
-# what components were used (and in what order?)
-# $store->{per_hour}->[0]->{components_used_order_list}->{$components_used}++;
-
-
-# PER-COMPONENT
-# my $g = "c_".$c."_success";
-# my $g = "c_".$c."_fail";
-# $store->{per_hour}->[0]->{$g}->{counter}++;
-# $store->{total}->{$g}->{counter}++;
-
-# duration (total count, min, max, average from duration/counter)
-# &record_duration($store->{per_hour}->[0]->{$g}, $c_dur);
-# &record_duration($store->{total}->{$g}, $c_dur);
-
-# # version count
-# $store->{per_hour}->[0]->{$g}->{version_list}->{$c_ver}++;
-
-        print F "</table><br clear=all>\n";
-
-        print F "</body></html>\n";
-
-}
-
-###############################################################################
-
-
+        print F "<tr><td align=left valign=top nowrap colspan=2><h2>Components Used</h2>\n";
+        #print F "<b>Components (and versions) being used</b><br>\n";
+        printf F "<li><b>in last 24 hours:</b><br>";
+        foreach my $c (
+                sort { $store->{per_day}->[0]->{components_used_order_list}->{$b} <=> $store->{per_day}->[0]->{components_used_order_list}->{$a} }
+                keys %{($store->{per_day}->[0]->{components_used_order_list})}) {
+                my $val = $store->{per_day}->[0]->{components_used_order_list}->{$c};
+                printf F "&nbsp;&nbsp;&nbsp;<b>%d time%s</b>: %s<br>\n", $val, ($val != 1 ? "s" : ""), $c;
+        }
+        printf F "<li><b>in last month:</b><br>";
+        foreach my $c (
+                sort { $store->{per_month}->[0]->{components_used_order_list}->{$b} <=> $store->{per_month}->[0]->{components_used_order_list}->{$a} }
+                keys %{($store->{per_month}->[0]->{components_used_order_list})}) {
+                my $val = $store->{per_month}->[0]->{components_used_order_list}->{$c};
+                printf F "&nbsp;&nbsp;&nbsp;<b>%d time%s</b>: %s<br>\n", $val, ($val != 1 ? "s" : ""), $c;
+        }
+
+        # build a component list
+        my %clist;
+        foreach my $c (sort keys %{($store->{total})}) {
+                if (($c =~ /^c_(.*)_success_counter$/) ||
+                    ($c =~ /^c_(.*)_fail_counter$/)) {
+                        # skip if any '/' chars in name
+                        next if ($1 =~ /\//);
+                        $clist{$1}++;
+                }
+        }
+
+        print F "<tr><td align=left valign=top nowrap colspan=2><h2>Detailed Component Statistics</h2>\n";
+        print F "<p><table align=center>";
+        my $ccount = 0;
+        foreach my $c (sort keys %clist) {
+                print F "</tr><tr>\n" if (($ccount % 4 == 0) && ($ccount > 0));
+                print F "<td>\n";
+                printf F "<b><a href='%s.html'>%s</a></b><br>\n", $c, $c;
+                print F "<b>Successful runs:</b><br>\n" . report_count_data("c_".$c."_success_counter", "brief");
+                print F "<b>Failed runs:</b><br>\n" . report_count_data("c_".$c."_fail_counter", "brief");
+                print F "</td>\n";
+                $ccount++;
+        }
+        print F "</tr></table></td></tr></table><br clear=all></body></html>\n";
+        close(F);
+
+        #
+        # iterate through every component we have seen, sorted alphabetically
+        # generate per-component statistics
+        #
+
+        foreach my $c (sort keys %clist) {
+                open(F,">$graph_dir/$c.html") || die "couldn't open $graph_dir/$c for writing: $!\n";
+                print F "$hdr<h1>Component statistics: $c</h1>\n";
+                print F "<table align=left>\n";
+
+                print F "<tr><td align=left valign=top nowrap><h2>Success/Failure</h2>\n";
+                print F "<i>(# of times called and return status)</i><br>\n";
+                print F "<b>Successful runs:</b> <font color=pink><i>(red)</i></font><br>\n";
+                print F report_count_data("c_".$c."_success_counter", "all");
+                print F "<br><b>Failed runs:</b> <font color=lightblue><i>(blue)</i></font>\n";
+                print F report_count_data("c_".$c."_fail_counter", "brief");
+                print F "</td><td nowrap><br>";
+                print F graph_count_data(24,21,12,24, "c_".$c."_success_counter", "c_".$c."_fail_counter");
+                print F "</td></tr>\n";
+
+                foreach my $w ("success", "fail") {
+                        my $g = "c_".$c."_".$w;
+
+                        print F "<tr><td align=left valign=top nowrap colspan=2><h2>Versions ($w)</h2>\n";
+                        printf F "<li><b>in last 24 hours:</b><br>";
+                        foreach my $c (
+                                sort { $store->{per_day}->[0]->{$g."_version_list"}->{$b} <=> $store->{per_day}->[0]->{$g."_version_list"}->{$a} }
+                                keys %{($store->{per_day}->[0]->{$g."_version_list"})}) {
+                                my $val = $store->{per_day}->[0]->{$g."_version_list"}->{$c};
+                                printf F "&nbsp;&nbsp;&nbsp;<b>%d time%s</b>: %s<br>\n", $val, ($val != 1 ? "s" : ""), $c;
+                        }
+                        printf F "<li><b>in last month:</b><br>";
+                        foreach my $c (
+                                sort { $store->{per_month}->[0]->{$g."_version_list"}->{$b} <=> $store->{per_month}->[0]->{$g."_version_list"}->{$a} }
+                                keys %{($store->{per_month}->[0]->{$g."_version_list"})}) {
+                                my $val = $store->{per_month}->[0]->{$g."_version_list"}->{$c};
+                                printf F "&nbsp;&nbsp;&nbsp;<b>%d time%s</b>: %s<br>\n", $val, ($val != 1 ? "s" : ""), $c;
+                        }
+
+                        print F "<tr><td align=left valign=top nowrap><h2>Duration ($w)</h2>\n";
+                        print F "<b>How long $c took to run ($w):</b><br>\n";
+                        printf F "<li><b>in last 24 hours:</b><br>".
+                                "&nbsp;&nbsp;&nbsp;%s average <i><font color=pink>(red)</font></i><br>".
+                                "&nbsp;&nbsp;&nbsp;%s min <i><font color=lightblue>(blue)</font></i><br>".
+                                "&nbsp;&nbsp;&nbsp;%s max <i><font color=lightgreen>(green)</font></i><br>\n",
+                                sec2hms(numvar($store->{per_day}->[0]->{$g."_duration"},"",0) / $store->{per_hour}->[0]->{$g."_duration_count"}),
+                                sec2hms(numvar($store->{per_day}->[0]->{$g."_duration_min"},"",0)),
+                                sec2hms(numvar($store->{per_day}->[0]->{$g."_duration_max"},"",0))
+                                if ((defined $store->{per_hour}->[0]->{$g."_duration_count"}) && ($store->{per_hour}->[0]->{$g."_duration_count"} > 0));
+                        printf F "<li><b>in last month:</b><br>".
+                                "&nbsp;&nbsp;&nbsp;%s average <i><font color=pink>(red)</font></i><br>".
+                                "&nbsp;&nbsp;&nbsp;%s min <i><font color=lightblue>(blue)</font></i><br>".
+                                "&nbsp;&nbsp;&nbsp;%s max <i><font color=lightgreen>(green)</font></i><br>\n",
+                                sec2hms(numvar($store->{per_month}->[0]->{$g."_duration"},"",0) / $store->{per_month}->[0]->{$g."_duration_count"}),
+                                sec2hms(numvar($store->{per_month}->[0]->{$g."_duration_min"},"",0)),
+                                sec2hms(numvar($store->{per_month}->[0]->{$g."_duration_max"},"",0))
+                                if ((defined $store->{per_month}->[0]->{$g."_duration_count"}) && ($store->{per_month}->[0]->{$g."_duration_count"} > 0));
+                        print F "</td><td nowrap align=left>&nbsp;<br>";
+                        print F graph_count_data(24,21,12,24,$g."_duration", $g."_duration_min", $g."_duration_max");
+                        print F "</td></tr>\n";
+                }
+
+                print F "</table><br clear=all></body></html>\n";
+                close(F);
+        }
+}
+
+###############################################################################
+
+