| | 789 | |
| | 790 | # Remove any MISSING_DATA from Shepherd we don't bother reporting. |
| | 791 | if ($pending_messages->{$progname}->{MISSING_DATA}) |
| | 792 | { |
| | 793 | # We don't care about Day 6 or later |
| | 794 | my $stats_limit = $policy{starttime} - $policy{first_bucket_offset} + (6 * 86400); |
| | 795 | &log(1, "SHEPHERD: Not reporting Shepherd missing data later than " . localtime($stats_limit) . ".\n"); |
| | 796 | |
| | 797 | $pending_messages->{$progname}->{MISSING_DATA} =~ s/(\d+)-(\d+)/$1 >= $stats_limit ? '' : "$1-$2"/eg; |
| | 798 | |
| | 799 | # Clean up: drop duplicate commas, empty channel text |
| | 800 | $pending_messages->{$progname}->{MISSING_DATA} =~ s/(?<!\d),+|,+(?!\d)//g; |
| | 801 | $pending_messages->{$progname}->{MISSING_DATA} =~ s/[ \w]+:\t?(?!\d)//g; |
| | 802 | |
| | 803 | # Anything left? |
| | 804 | unless ($pending_messages->{$progname}->{MISSING_DATA} =~ /\d{6,}/) |
| | 805 | { |
| | 806 | delete $pending_messages->{$progname}->{MISSING_DATA}; |
| | 807 | } |
| | 808 | } |
| | 809 | |
| 4194 | | # $plugin_data -> 'missing_all_epoch' records MISSING_DATA for stats |
| 4195 | | # reporting. We don't want to be bothered by data holes that are: |
| 4196 | | # (a) 'Noncritical', and we already know this grabber tends |
| 4197 | | # to have those (eg yahoo7widget); or |
| 4198 | | # (b) In Shepherd and Day 6 or higher. (This allows Shepherd to attempt |
| 4199 | | # to fill a longer schedule -- e.g. 14 days -- but not throw a lot |
| 4200 | | # of warnings if it fails to.) |
| 4201 | | unless |
| 4202 | | (( &query_config($proggy, 'has_noncritical_gaps') and &is_noncritical_gap($d->{already_missing_epoch})) |
| 4203 | | # or |
| 4204 | | # ( $proggy eq $progname and $d->{already_missing_epoch} =~ /(\d+)-/ and $1 > $policy{starttime} + (5*86400))) |
| 4205 | | ) { |
| | 4215 | # Don't report noncritical data holes in grabbers we know have those. |
| | 4216 | # |
| | 4217 | # Two things to note here: |
| | 4218 | # 1. We can only do this for individual grabbers, not Shepherd overall; |
| | 4219 | # $plugin_data -> 'missing_all_epoch' is used for further analysis |
| | 4220 | # at the Shepherd & channel levels, not just stats reporting. |
| | 4221 | # 2. Normally we flag data as '$ignore_missing++' in &analyse_plugin_data, |
| | 4222 | # but that loops through individual buckets: it knows whether each |
| | 4223 | # bucket is filled or not but not how large each gap is. |
| | 4224 | unless (&query_config($proggy, 'has_noncritical_gaps') and &is_noncritical_gap($d->{already_missing_epoch})) |
| | 4225 | { |
