Changeset 326

Timestamp:

12/02/06 05:25:08 (6 years ago)

Author:

lincoln

Message:

drop RRD from server daily script and use our own home-grown graphing

Files:

• util/server_daily_maint (modified) (5 diffs)

util/server_daily_maint

@@ -1 +1 @@
 #!/usr/bin/perl
 
-# server-side daily maintenance script
+# server-side maintenance script
+#  * can be run daily or multiple times per day
+#  * best if run (say) every 15 or 30 minutes (more granular data)
+#
 #  - deletes old feedback
-#  - updates RRD of some common statistics (users, failures, duration, time-of-day) etc.
-
+#  - updates store of some common statistics (users, failures, duration, time-of-day) etc.
+#  - generates pretty graphs and html
+
+
+# settings
 my $days_to_keep = 14;          # keep 14 days
-my $store_dir = $ENV{HOME} . "/whuffy.com/dev/stats";    # where we store our feedback
+my $store_dir = $ENV{HOME} . "/whuffy.com/dev/stats";    # where we retrieve our feedback
 my $graph_dir = $ENV{HOME} . "/whuffy.com/dev/feedback"; # where we store our graphs
-
-my $debug = 0;
+my $debug = 1;
 
 use strict;
-use RRD::Simple;
 use Data::Dumper;
+use POSIX qw(strftime);
+use SVG::TT::Graph::TimeSeries;
+use SVG::TT::Graph::Bar;
+
+my $stats_file = $store_dir . "/stats.config";
+my $store = { };
+my $starttime = time;
 
 print "Running at " . localtime() . ".\n";
 
-&delete_old_files;
-
-my $rrd = RRD::Simple->new();
-my $last_updated = 0;
-
-&check_rrd;
-&update_stats;
-&generate_graphs;
+&set_initial_values;
+&read_config;           # read in our existing stats
+
+print "Last run " . localtime($store->{last_run}) . ".\n";
+
+&rollover_statistics;   # rotate statistics
+
+&delete_old_files;      # delete old feedback files
+&parse_feedback;        # parse any (new) feedback files
+
+&aggregate_stats;       # use last 7 days to build last week, etc.
+
+&generate_reports;      # generate reports and graphs
+
+$store->{last_run} = $starttime; # update our last_run time
+&write_config;          # write out our updated stats
 
 print "Done.\n";
@@ -41 +60 @@
         foreach my $file (@files) {
                 next if (!-f $file);                    # only work on normal files
-                next if ($file !~ /^\d+$/);     # filename should be all digits
+                next if ($file !~ /^\d+\.done$/);       # only delete processed files
 
                 if ((-M $file) > $days_to_keep) {
@@ -56 +75 @@
 
 ###############################################################################
-
-sub check_rrd
-{
-        # create usage RRD if it doesn't exist
-        if (!-e "$graph_dir/usage.rrd") {
-                $rrd->create("$graph_dir/usage.rrd", "3years",
-                        "visits" => "ABSOLUTE:86400",
-                        "unique_visits" => "ABSOLUTE:86400",
-                        "success_total" => "ABSOLUTE:86400",
-                        "fail_total" => "ABSOLUTE:86400",
-                        "duration" => "ABSOLUTE:86400");
+# read in our saved statistics
+
+sub read_config
+{
+        if (-r $stats_file) {
+                local (@ARGV, $/) = ($stats_file);
+                no warnings 'all'; eval <>; die "$@" if $@;
         } else {
-                # set $last_updated based on last update time of usage.rrd
-                $last_updated = $rrd->last("$graph_dir/usage.rrd");
-        }
-
-        # create time-of-day-usage if it doesn't exist
-        if (!-e "$graph_dir/timeofday.rrd") {
-                $rrd->create("$graph_dir/timeofday.rrd", "3years",
-                        "starttime" => "ABSOLUTE:86400",
-                        "duration" => "ABSOLUTE:86400");
-        }
-
-        printf "Gathering statistics for data since ".localtime($last_updated)."\n" if ($debug);
-}
-
-###############################################################################
-
-sub update_stats
-{
-        my %daily_stats;
-        $daily_stats{visits} = 0;
-        $daily_stats{unique_visits} = 0;
-        $daily_stats{success_total} = 0;
-        $daily_stats{fail_total} = 0;
-        $daily_stats{duration} = 0;
-
-        my %seen_sysid_list;
-        my %timeofday_start;
-        my %timeofday_duration;
-
+                printf "WARNING: no config file $stats_file - ok if this is the first time running!\n";
+
+                # try to write to it - if we can't then this will cause us to barf
+                &write_config;
+        }
+}
+
+###############################################################################
+# write out our saved statistics
+
+sub write_config
+{
+        $Data::Dumper::Sortkeys = 1;
+        $Data::Dumper::Indent = 1;
+        open(F,">$stats_file") || die "can't write to $stats_file: $!\n";
+        print F Data::Dumper->Dump([$store], ["store"]);
+        close F;
+}
+
+###############################################################################
+# define some initial settings to basic values
+# reading the config file may override these
+
+sub set_initial_values
+{
+        $store->{last_run} = 0;
+
+        $store->{num_hours} = 1;
+        $store->{num_days} = 0;
+        $store->{num_weeks} = 0;
+        $store->{num_months} = 0;
+
+        $store->{max_hours} = 72;       # keep per-hour stats for 72 hours (need min 24)
+        $store->{max_days} = 30;        # keep per-day stats for 30 days (need min 30)
+        $store->{max_weeks} = 12;       # keep per-week stats for 12 weeks
+        $store->{max_months} = 36;      # keep per-month stats for 36 months
+}
+
+###############################################################################
+# rollover 'daily' / 'weekly' / 'monthly' statistics if day/week/month is 
+# different from the last time we were run
+
+sub rollover_statistics
+{
+        # NOTE: this logic is working on GMT+10 time not whatever timezone
+        # this server is running in!
+        
+        # 1. work out GMT offset
+        my $tzstring = strftime("%z", localtime(time));
+        $store->{gmt_offset} = (60*60) * int(substr($tzstring,1,2));     # hr
+        $store->{gmt_offset} += (60 * int(substr($tzstring,3,2)));       # min
+        $store->{gmt_offset} *= -1 if (substr($tzstring,0,1) eq "-");    # +/-
+
+        # 2. work out difference from this timezone to GMT+10
+        $store->{tzdiff} = (10*60*60) - $store->{gmt_offset};
+
+        # 3. correct starttime and last_run times
+        my $gmt10_starttime = $starttime + $store->{tzdiff};
+        my $gmt10_last_run = $store->{last_run} + $store->{tzdiff};
+
+        # 4. new month
+        if (strftime("%m", localtime($gmt10_last_run)) != strftime("%m", localtime($gmt10_starttime))) {
+                for (my $i = $store->{num_months}; $i > 0; $i--) {
+                        $store->{per_month}->[$i+1] = undef;
+                        $store->{per_month}->[$i+1] = $store->{per_month}->[$i]
+                                if (defined $store->{per_month}->[$i])
+                }
+                if ($store->{num_months} >= $store->{max_months}) {
+                        delete $store->{per_month}->[($store->{num_months})];
+                } else {
+                        $store->{num_months}++;
+                }
+        }
+
+        # 5. new week
+        if (strftime("%U", localtime($gmt10_last_run)) != strftime("%U", localtime($gmt10_starttime))) {
+                for (my $i = $store->{num_weeks}; $i > 0; $i--) {
+                        $store->{per_week}->[$i+1] = undef;
+                        $store->{per_week}->[$i+1] = $store->{per_week}->[$i]
+                                if (defined $store->{per_week}->[$i]);
+                }
+                if ($store->{num_weeks} >= $store->{max_weeks}) {
+                        delete $store->{per_week}->[($store->{num_weeks})];
+                } else {
+                        $store->{num_weeks}++;
+                }
+        }
+
+        # 6. new day
+        if (strftime("%w", localtime($gmt10_last_run)) != strftime("%w", localtime($gmt10_starttime))) {
+                for (my $i = $store->{num_days}; $i > 0; $i--) {
+                        $store->{per_day}->[$i+1] = undef;
+                        $store->{per_day}->[$i+1] = $store->{per_day}->[$i]
+                                if (defined $store->{per_day}->[$i]);
+                }
+                if ($store->{num_days} >= $store->{max_days}) {
+                        delete $store->{per_day}->[($store->{num_days})];
+                } else {
+                        $store->{num_days}++;
+                }
+        }
+
+        # 7. new hour
+        if (strftime("%H", localtime($gmt10_last_run)) != strftime("%H", localtime($gmt10_starttime))) {
+                for (my $i = $store->{num_hours}; $i > 0; $i--) {
+                        $store->{per_hour}->[$i+1] = undef;
+                        $store->{per_hour}->[$i+1] = $store->{per_hour}->[$i]
+                                if (defined $store->{per_hour}->[$i]);
+                }
+                if ($store->{num_hours} >= $store->{max_hours}) {
+                        delete $store->{per_hour}->[($store->{num_hours})];
+                } else {
+                        $store->{num_hours}++;
+                }
+        }
+
+}
+
+###############################################################################
+# aggregate stats:
+#  - last 24 hours = day 0 statistics,
+#  - last 7 days = week 0 statistics,
+#  - last 30 days = month 0 statistics
+
+sub aggregate_stats
+{
+        # build day 0 based on last 24 hours
+        delete $store->{per_day}->[0];
+        for (my $i = 0; $i < 24; $i++) {
+                next if (!defined $store->{per_hour}->[$i]);
+                $store->{per_day}->[0] = &merge_in_stats($store->{per_day}->[0], $store->{per_hour}->[$i]);
+        }
+
+        # build week 0 based on last 7 days
+        delete $store->{per_week}->[0];
+        for (my $i = 0; $i < 7; $i++) {
+                next if (!defined $store->{per_day}->[$i]);
+                $store->{per_week}->[0] = &merge_in_stats($store->{per_week}->[0], $store->{per_day}->[$i]);
+        }
+
+        # build month 0 based on last 30 days
+        delete $store->{per_month}->[0];
+        for (my $i = 0; $i < 30; $i++) {
+                next if (!defined $store->{per_day}->[$i]);
+                $store->{per_month}->[0] = &merge_in_stats($store->{per_month}->[0], $store->{per_day}->[$i]);
+        }
+}
+
+###############################################################################
+
+sub record_duration
+{
+        my ($where, $dur) = @_;
+        $where->{duration} += $dur;
+        $where->{duration_max} = $dur if (!defined $where->{duration_max});
+        $where->{duration_max} = $dur if ($dur > $where->{duration_max});
+        $where->{duration_min} = $dur if (!defined $where->{duration_min});
+        $where->{duration_min} = $dur if ($dur < $where->{duration_max});
+}
+
+###############################################################################
+# merge statistics in an intelligent manner
+#  - can recursively descend HASH lists
+#  - accumulates totals
+#  - fields with _min/_max suffix take the minimum/maximum value
+
+sub merge_in_stats
+{
+        my ($to, $from) = @_;
+
+        foreach my $field (keys %{($from)}) {
+                if (ref($from->{$field}) eq "HASH") {
+                        $to->{$field} = &merge_in_stats($to->{$field}, $from->{$field});
+                } elsif (defined $from->{$field}) {
+                        if (!defined $to->{$field}) {
+                                $to->{$field} = $from->{$field};
+                        } else {
+                                # fields ending in _min and _max are special
+                                if ($field =~ /_min$/) {
+                                        $to->{$field} = $from->{$field} if ($from->{$field} < $to->{$field});
+                                } elsif ($field =~ /_max$/) {
+                                        $to->{$field} = $from->{$field} if ($from->{$field} > $to->{$field});
+                                } else {
+                                        $to->{$field} += $from->{$field};
+                                }
+                        }
+                }
+        }
+        return $to;
+}
+
+###############################################################################
+
+sub parse_feedback
+{
         chdir($store_dir) || die "can't chdir $store_dir: $!\n";
         opendir(DIR, $store_dir) || die "can't opendir $store_dir: $!\n";
@@ -104 +285 @@
         foreach my $file (@files) {
                 next if (!-f $file);                    # only work on normal files
-                next if ($file !~ /^\d+$/);     
-                next if ((stat(_))[9] < $last_updated); # only look at newly modified files
-
-                my $starttime = (stat(_))[9];
-                printf " - gathering statistics from $store_dir/$file ($starttime)\n" if ($debug);
+                next if ($file !~ /^\d+$/);             # only look at non-processed files
+                next if ((stat(_))[9] < $store->{last_updated}); # only look at newly modified files
+
+                my $file_modtime = (stat(_))[9];
+                printf " - gathering statistics from $store_dir/$file ($file_modtime)\n" if ($debug);
 
                 open(F,"<$store_dir/$file") || die "could not open $store_dir/$file: $!\n";
@@ -120 +301 @@
                                 # usage statistics
                                 #
-                                $daily_stats{visits}++;
-                                $daily_stats{duration} += ($duration/60);               # convert to minutes
-                                if (!defined $seen_sysid_list{$sysid}) {
-                                        $seen_sysid_list{$sysid} = 1;
-                                        $daily_stats{unique_visits}++;
-                                }
-
-                                #
-                                # time of day statistics
-                                # (note: we're using starttime based on FILE MODIFICATION time and not
-                                #  what the user tells us. this is because RRD can only add values)
-                                $timeofday_start{$starttime}++;
-                                $timeofday_duration{$starttime} += ($duration/60);      # convert to minutes
-
-                        } elsif ($_ =~ /^(\S+)\t(\S+)\t/) {
-                                my ($component, $status) = ($1, $2);
-
-                                #
-                                # success/fail usage statistics
-                                #
-                                if ($status eq "SUCCESS") {
-                                        $daily_stats{success_total}++;
-                                } elsif ($status eq "FAIL") {
-                                        $daily_stats{fail_total}++;
-                                }
+                                $store->{per_hour}->[0]->{visits}++;             # total # of visits seen
+                                $store->{total}->{visits}++;
+
+                                # shepherd's duration (total count, min, max, average from duration/visits)
+                                &record_duration($store->{per_hour}->[0], $duration);
+                                &record_duration($store->{total}, $duration);
+
+                                # list of sysid's seen (used to derive # unique users)
+                                $store->{per_hour}->[0]->{seen_sysid_list}->{$sysid}++;
+                                $store->{total}->{seen_sysid_list}->{$sysid}++;
+
+                                # where user is
+                                my $gmt_offset_string = sprintf "%s%02d%02d",
+                                        ($gmt_offset < 0 ? "-" : "+"),
+                                        int($gmt_offset / 3600),
+                                        int(($gmt_offset / 3600) / 60);
+                                $store->{per_hour}->[0]->{gmt_offset_list}->{$gmt_offset_string}++;
+
+                                # when shepherd was run (in GMT+10), normalized to 15 minute buckets
+                                my $when_run = strftime("%H:%M", localtime(($file_modtime-($file_modtime%(15*60))) + $store->{tzdiff}));
+                                $store->{per_hour}->[0]->{when_run_list}->{"$when_run"}++;
+                                $store->{total}->{when_run_list}->{"$when_run"}++;
+
+                                # what components were used (and in what order?)
+                                $store->{per_hour}->[0]->{components_used_order_list}->{$components_used}++;
+
+                        } elsif ($_ =~ /^(\S+)\tSUCCESS\t(\S+)\t(\S+)\t(\S+)\t(\S+)/) {
+                                # success line
+                                my ($c, $retcode, $c_start, $c_dur, $c_ver) = ($1, $2, $3, $4, $5);
+
+                                my $g = "c_".$c."_success";
+                                $store->{per_hour}->[0]->{$g}->{counter}++;
+                                $store->{total}->{$g}->{counter}++;
+
+                                # duration (total count, min, max, average from duration/counter)
+                                &record_duration($store->{per_hour}->[0]->{$g}, $c_dur);
+                                &record_duration($store->{total}->{$g}, $c_dur);
+
+                                # version count
+                                $store->{per_hour}->[0]->{$g}->{version_list}->{$c_ver}++;
+                        } elsif ($_ =~ /^(\S+)\tFAIL\t(\S+)\t(\S+)\t(\S+)\t(\S+)\t(\S+)/) {
+                                # fail line
+                                my ($c, $retcode, $c_start, $c_dur, $c_ver, $num_failures) = ($1, $2, $3, $4, $5, $6);
+
+                                my $g = "c_".$c."_fail";
+                                $store->{per_hour}->[0]->{$g}->{counter}++;
+                                $store->{total}->{$g}->{counter}++;
+
+                                # duration (total count, min, max, average from duration/counter)
+                                &record_duration($store->{per_hour}->[0]->{$g}, $c_dur);
+                                &record_duration($store->{total}->{$g}, $c_dur);
+
+                                # version count
+                                $store->{per_hour}->[0]->{$g}->{version_list}->{$c_ver}++;
+                        } elsif ($_ =~ /^(\S+)\tstats\t(.*)$/) {
+                                # component statistics line
+                                ; # ignore for now
                         }
                 }
                 close(F);
-        }
-
-        # average
-        if ($daily_stats{visits} > 0) {
-                $daily_stats{duration} /= $daily_stats{visits};
-                $daily_stats{success_total} /= $daily_stats{visits};
-                $daily_stats{fail_total} /= $daily_stats{visits};
-        }
-
-        # update usage statistics
-        $rrd->update("$graph_dir/usage.rrd", time,
-                "visits" => $daily_stats{visits},
-                "unique_visits" => $daily_stats{unique_visits},
-                "success_total" => $daily_stats{success_total},
-                "fail_total" => $daily_stats{fail_total},
-                "duration" => $daily_stats{duration});
-        printf "updated usage statistics:\n".Dumper(%daily_stats) if ($debug);
-
-        # update time-of-day statistics
-        foreach my $epoch (sort keys %timeofday_start) {
-                $timeofday_duration{$epoch} /= $timeofday_start{$epoch};
-                $rrd->update("$graph_dir/timeofday.rrd", $epoch,
-                        "starttime" => $timeofday_start{$epoch},
-                        "duration" => $timeofday_duration{$epoch});
-                printf "updated time-of-day statistics: start=%d, count=%d, duration=%d\n",
-                  $epoch, $timeofday_start{$epoch}, $timeofday_duration{$epoch} if ($debug);
-        }
-}
-
-###############################################################################
-
-sub generate_graphs
-{
-        # usage-daily.png, usage-weekly.png,
-        # usage-monthly.png, usage-annual.png
-        $rrd->graph("$graph_dir/usage.rrd",
-                destination => "$graph_dir",
-                basename => "usage",
-                extended_legend => 1);
-
-        # timeofday-daily.png, timeofday-weekly.png,
-        # timeofday-monthly.png, timeofday-annual.png
-        $rrd->graph("$graph_dir/timeofday.rrd",
-                destination => "$graph_dir",
-                basename => "timeofday");
-}
-
-###############################################################################
-
-
+
+                # rename processed files so we don't re-process them
+                rename("$file", "$file.done") if (!$debug);
+        }
+}
+
+###############################################################################
+
+sub numvar
+{
+        my ($var,$field,$notdef) = @_;
+        $notdef = "n/a" if (!defined $notdef);
+        $field = "" if (!defined $field);
+
+        return $notdef if (!defined $var);
+
+        if ($field =~ /_list$/) {
+                # fields which end in _list are HASHes - return # of keys
+                my $count = keys %{$var};
+                return $count;
+        } else {
+                return $var;
+        }
+}
+
+###############################################################################
+
+sub report_count_data
+{
+        my ($field) = @_;
+        my $r = "";
+
+        $r .= sprintf "<li>in last 60 minutes: <b>%s</b> (%s previous)<br>\n",
+                numvar($store->{per_hour}->[0]->{$field},$field), numvar($store->{per_hour}->[1]->{$field},$field);
+        $r .= sprintf "<li>in last 24 hours: <b>%s</b> (previous %s)<br>\n",
+                numvar($store->{per_day}->[0]->{$field},$field), numvar($store->{per_day}->[1]->{$field},$field);
+        $r .= sprintf "<li>in last 7 days: <b>%s</b> (previous %s)<br>\n",
+                numvar($store->{per_week}->[0]->{$field},$field), numvar($store->{per_week}->[1]->{$field},$field);
+        $r .= sprintf "<li>in last 30 days: <b>%s</b> (previous %s)<br>\n", 
+                numvar($store->{per_month}->[0]->{$field},$field), numvar($store->{per_month}->[1]->{$field},$field);
+        $r .= sprintf "<li><b>TOTAL SEEN: <u>%s</u></b><br>\n", numvar($store->{total}->{$field},$field);
+
+        return $r;
+}
+
+###############################################################################
+
+sub sec2hms
+{
+        my $sec = shift;
+
+        return sprintf "%dh %02dm %02ds",
+                int($sec / 3600),
+                int(($sec % 3600) / 60),
+                $sec % 60;
+}
+
+###############################################################################
+
+sub graph_hash_data
+{
+        my ($field, $when, $startwhen, $endwhen, $title, $gheight, $gwidth) = @_;
+
+        my $gname = $field."_".$when."_".$startwhen."_".$endwhen;
+
+        # gather up data
+        my %data;
+        for (my $i = $startwhen; $i < $endwhen; $i++) {
+                if (defined $store->{$when}->[$i]->{$field}) {
+                        foreach my $k (keys %{($store->{$when}->[$i]->{$field})}) {
+                                $data{$k} += $store->{$when}->[$i]->{$field}->{$k};
+                        }
+                }
+        }
+
+        # sort it into key order
+        my $min, my $max;
+        my @sorted_keys, my @sorted_data;
+        foreach my $k (sort keys %data) {
+                push(@sorted_keys,$k);
+                push(@sorted_data,$data{$k});
+
+                $min = $data{$k} if (!defined $min);
+                $max = $data{$k} if (!defined $max);
+                $min = $data{$k} if ($data{$k} < $min);
+                $max = $data{$k} if ($data{$k} > $min);
+        }
+
+        # work out our scaling
+        my $scalediv = ($max - $min) / 3;
+        $scalediv = $max if ($scalediv < 1);
+
+        # only graph if we have some data
+        return if (($min == 0) && ($max == 0));
+
+        print "Generating bar graph $gname ..\n" if ($debug);
+
+        my $graph = SVG::TT::Graph::Bar->new({
+                'show_graph_title' => 1,
+                'graph_title' => $title,
+                'bar_gap' => 0,
+                'show_data_values' => 0,
+                'height' => $gheight,
+                'width' => $gwidth,
+                'stagger_x_labels' => 1,
+                'show_x_labels' => 1,
+                'bar_gap' => 0,
+                'show_data_values' => 1,
+                'scale_divisions' => $scalediv,
+                'fields' => \@sorted_keys,
+                });
+        $graph->add_data({
+                'data' => \@sorted_data,
+                });
+
+        open(SVG,">$graph_dir/$gname".".svg") || die "could not create $graph_dir/$gname".".svg: $!\n";
+        print SVG $graph->burn();
+        close(SVG);
+
+        return sprintf "<object type='image/svg+xml' data='%s.svg' height='%d' width='%d'></object>\n",$gname,$gheight,$gwidth;
+}
+
+###############################################################################
+
+sub graph_one_count_data
+{
+        my ($gname, $data, $title, $time_format, $timediv) = @_;
+        my $gheight = 120;
+        my $gwidth = 250;
+
+        print "Generating graph $gname ..\n" if ($debug);
+
+        # work out scale..
+        my $min, my $max, my $valnum = 0;
+        foreach my $val (@$data) {
+                $valnum++;
+                next if ($valnum % 2 == 1);
+
+                $min = $val if (!defined $min);
+                $max = $val if (!defined $max);
+                $min = $val if ($val < $min);
+                $max = $val if ($val > $min);
+        }
+        my $scalediv = ($max - $min) / 3;
+        $scalediv = 1 if ($scalediv < 1);
+
+        return if (($min == 0) && ($max == 0));
+
+        my $graph = SVG::TT::Graph::TimeSeries->new({
+                'show_graph_title' => 1,
+                'graph_title' => $title,
+                'bar_gap' => 0,
+                'show_data_values' => 0,
+                'height' => $gheight,
+                'width' => $gwidth,
+                'stagger_x_labels' => 1,
+                'area_fill' => 1,
+                'rollover_values' => 1,
+                'show_data_values' => 1,
+                'x_label_format' => $time_format,
+                'scale_divisions' => $scalediv,
+                'timescale_divisions' => $timediv,
+                });
+        $graph->add_data({
+                'data' => $data,
+                });
+
+        open(SVG,">$graph_dir/$gname".".svg") || die "could not create $graph_dir/$gname".".svg: $!\n";
+        print SVG $graph->burn();
+        close(SVG);
+
+        return sprintf "<object type='image/svg+xml' data='%s.svg' height='%d' width='%d'></object>\n",$gname,$gheight,$gwidth;
+}
+
+###############################################################################
+# produces up to 4 sets of graph
+
+sub graph_count_data
+{
+        my ($field, $max_hrs, $max_days, $max_weeks, $max_months) = @_;
+        my $r = "";
+
+        # hours
+        my @d1;
+        for (my $i = ($max_hrs-1); $i >= 0; $i--) {
+                my $val = numvar($store->{per_hour}->[$i]->{$field},$field,0);
+                my $time_offset = $starttime + $store->{tzdiff} - (60*60*$i);
+                push(@d1, strftime("%Y-%m-%d %H:%M:%S", localtime($time_offset)), $val);
+        }
+
+        # days
+        my @d2;
+        for (my $i = ($max_days-1); $i >= 0; $i--) {
+                my $val = numvar($store->{per_day}->[$i]->{$field},$field,0);
+                my $time_offset = $starttime + $store->{tzdiff} - (60*60*24*$i);
+                push(@d2, strftime("%Y-%m-%d %H:%M:%S", localtime($time_offset)), $val);
+        }
+
+        # weeks
+        my @d3;
+        for (my $i = ($max_weeks-1); $i >= 0; $i--) {
+                my $val = numvar($store->{per_week}->[$i]->{$field},$field,0);
+                my $time_offset = $starttime + $store->{tzdiff} - (60*60*24*7*$i);
+                push(@d3, strftime("%Y-%m-%d %H:%M:%S", localtime($time_offset)), $val);
+        }
+
+        # months
+        my @d4;
+        for (my $i = ($max_months-1); $i >= 0; $i--) {
+                my $val = numvar($store->{per_week}->[$i]->{$field},$field,0);
+                my $time_offset = $starttime + $store->{tzdiff} - (60*60*24*30*$i);
+                push(@d4, strftime("%Y-%m-%d %H:%M:%S", localtime($time_offset)), $val);
+        }
+
+        $r .= &graph_one_count_data($field."_hr", \@d1, "Last $max_hrs hours", "%a %H:%M", "6 hours") if (defined $max_hrs);
+        $r .= &graph_one_count_data($field."_day", \@d2, "Last $max_days days", "%a %e %b", "5 days") if (defined $max_days);
+        $r .= &graph_one_count_data($field."_wk", \@d3, "Last $max_weeks weeks", "%e %b %y", "3 weeks") if (defined $max_weeks);
+        $r .= &graph_one_count_data($field."_mn", \@d4, "Last $max_months months", "%b %y", "4 months") if (defined $max_months);
+        return $r;
+}
+
+###############################################################################
+
+sub generate_reports
+{
+        open(F,">$graph_dir/main.html") || die "couldn't open $graph_dir for writing: $!\n";
+
+        my $now = localtime($starttime);
+
+        print F <<EOF
+<html>
+<head>
+<title>Shepherd</title>
+<style>
+a { color: #F6B620; text-decoration: none; border-bottom: 1px dotted #F6B620; }
+a:hover { border: 1px dotted #FFEE40; }
+html, body, table { margin: 0; padding: 0; color: #c0bcb0; font: normal 12px Verdana; }
+body { text-align: left; margin: 1em 10 10 20; }
+body.home { background: #474642; }
+body.home h1 { font: bold 36px/.8em Slyphaen, Times, serif; color: #F6B620; border-bottom: 3px solid #F6B620; padding: 0; clear: both; text-align: left; }
+body.home h2 { font: normal 24px/.8em Slyphaen, Times, serif; color: #DFB020; padding: 0; clear: both; margin: 20 0 10 0; }
+body.home p { clear: both; }
+input { font: bold 12px Arial; }
+</style>
+</head>
+
+<body class="home">
+<h1>Shepherd Overall statistics <font size=-1>(as of $now)</font></h1>
+<table style='border: 1px dotted; border-collapse: separate; border-spacing: 1px dotted; padding: 3px; cellpadding: 3px;' align=left>
+<tr>
+EOF
+;
+        print F "<tr'><td align=left valign=top nowrap><h2>Shepherd usage</h2>\n";
+        print F "Number of times shepherd reported in.<br>\n";
+        print F report_count_data("visits");
+        print F "</td><td nowrap colspan=4><br>";
+        print F graph_count_data("visits",24,21,12,24);
+        print F "</td></tr>\n";
+
+        print F "<tr><td align=left valign=top nowrap><h2>Unique users</h2>\n";
+        print F "Number of unique systems seen.<br>\n";
+        print F report_count_data("seen_sysid_list");
+        print F "</td><td nowrap colspan=4><br>";
+        print F graph_count_data("seen_sysid_list",24,21,12,24);
+        print F "</td></tr>\n";
+
+        print F "<tr><td align=left valign=top nowrap><h2>Start time</h2>\n";
+        print F "When shepherd checked in.<br>\n";
+        print F "(in 15 minute groups, time in GMT+10)";
+        print F "</td><td nowrap colspan=2><br>";
+        print F graph_hash_data("when_run_list", "per_day", 0, 7, "Last 7 days", 120, 500);
+        print F "</td><td nowrap colspan=2><br>";
+        print F graph_hash_data("when_run_list", "per_month", 0, 1, "Last 30 days", 120, 500);
+        print F "</td></tr>\n";
+
+        print F "<tr><td align=left valign=top nowrap><h2>Duration</h2>\n";
+        print F "How long shepherd took to run.<br>\n";
+        print F "</td><td nowrap align=left valign=top><br>";
+        printf F "<li>in last 60 minutes:</b><ul><li>%s average (mean)<li>%s min<li>%s max</ul>\n",
+                sec2hms(numvar($store->{per_hour}->[0]->{duration},"",0) / $store->{per_hour}->[0]->{visits}),
+                sec2hms(numvar($store->{per_hour}->[0]->{duration_min},"",0)),
+                sec2hms(numvar($store->{per_hour}->[0]->{duration_max},"",0))
+                if ((defined $store->{per_hour}->[0]->{visits}) && ($store->{per_hour}->[0]->{visits} > 0));
+        print F "</td><td nowrap align=left valign=top><br>";
+        printf F "<li>in last 7 days:</b><ul><li>%s average (mean)<li>%s min<li>%s max</ul>\n",
+                sec2hms(numvar($store->{per_day}->[0]->{duration},"",0) / $store->{per_day}->[0]->{visits}),
+                sec2hms(numvar($store->{per_day}->[0]->{duration_min},"",0)),
+                sec2hms(numvar($store->{per_day}->[0]->{duration_max},"",0))
+                if ((defined $store->{per_day}->[0]->{visits}) && ($store->{per_day}->[0]->{visits} > 0));
+        print F "</td><td nowrap align=left valign=top><br>";
+        printf F "<li>in last 30 days:</b><ul><li>%s average (mean)<li>%s min<li>%s max</ul>\n",
+                sec2hms(numvar($store->{per_month}->[0]->{duration},"",0) / $store->{per_month}->[0]->{visits}),
+                sec2hms(numvar($store->{per_month}->[0]->{duration_min},"",0)),
+                sec2hms(numvar($store->{per_month}->[0]->{duration_max},"",0))
+                if ((defined $store->{per_month}->[0]->{visits}) && ($store->{per_month}->[0]->{visits} > 0));
+        print F "</td><td nowrap align=left valign=top><br>";
+        printf F "<li>OVERALL:</b><ul><li>%s average (mean)<li>%s min<li>%s max</ul>\n",
+                sec2hms(numvar($store->{total}->{duration},"",0) / $store->{total}->{visits}),
+                sec2hms(numvar($store->{total}->{duration_min},"",0)),
+                sec2hms(numvar($store->{total}->{duration_max},"",0))
+                if ((defined $store->{total}->{visits}) && ($store->{total}->{visits} > 0));
+        print F "</td></tr>\n";
+
+        print F "<tr><td align=left valign=top nowrap><h2>Timezones</h2>\n";
+        print F "Timezone setting on hosts.<br>\n";
+        print F "</td><td nowrap colspan=4><br>";
+        print F graph_hash_data("gmt_offset_list", "per_day", 0, 7, "Last 7 days", 120, 500);
+        print F graph_hash_data("gmt_offset_list", "per_month", 0, 1, "Last 30 days", 120, 500);
+        print F "</td></tr>\n";
+
+
+# SHEPHERD
+# what components were used (and in what order?)
+# $store->{per_hour}->[0]->{components_used_order_list}->{$components_used}++;
+
+
+# PER-COMPONENT
+# my $g = "c_".$c."_success";
+# my $g = "c_".$c."_fail";
+# $store->{per_hour}->[0]->{$g}->{counter}++;
+# $store->{total}->{$g}->{counter}++;
+
+# duration (total count, min, max, average from duration/counter)
+# &record_duration($store->{per_hour}->[0]->{$g}, $c_dur);
+# &record_duration($store->{total}->{$g}, $c_dur);
+
+# # version count
+# $store->{per_hour}->[0]->{$g}->{version_list}->{$c_ver}++;
+
+        print F "</table><br clear=all>\n";
+
+        print F "</body></html>\n";
+
+}
+
+###############################################################################
+
+