mark: A photo of Mark kneeling on top of the Taal Volcano in the Philippines. It was a long hike. (Default)
Mark Smith ([staff profile] mark) wrote in [site community profile] changelog2011-07-29 12:08 am

[dw-ops] Sync outstanding Nagios and Puppet changes

[commit: http://hg.dwscoalition.org/dw-ops/rev/969c631efb1a]

Sync outstanding Nagios and Puppet changes

Patch by [staff profile] mark.

Files modified:
  • nagios/conf.d/config/commands.cfg
  • nagios/conf.d/config/hostgroups.cfg
  • nagios/conf.d/hosts/sb-admin01-auto.cfg
  • nagios/conf.d/hosts/sb-db01-auto.cfg
  • nagios/conf.d/hosts/sb-db02-auto.cfg
  • nagios/conf.d/hosts/sb-jobs01-auto.cfg
  • nagios/conf.d/hosts/sb-jobs02-auto.cfg
  • nagios/conf.d/hosts/sb-lb01-auto.cfg
  • nagios/conf.d/hosts/sb-lb02-auto.cfg
  • nagios/conf.d/hosts/sb-search01-auto.cfg
  • nagios/conf.d/hosts/sb-web01-auto.cfg
  • nagios/conf.d/hosts/sb-web02-auto.cfg
  • nagios/conf.d/hosts/sb-web03-auto.cfg
  • nagios/conf.d/services/disks-web.cfg
  • nagios/custom/check_smart.pl
  • puppet/manifests/site.pp
  • puppet/modules/iptables/templates/spamhaus-drop.erb
--------------------------------------------------------------------------------
diff -r fae923426ebf -r 969c631efb1a nagios/conf.d/config/commands.cfg
--- a/nagios/conf.d/config/commands.cfg	Fri Apr 23 20:12:34 2010 +0000
+++ b/nagios/conf.d/config/commands.cfg	Fri Jul 29 00:08:09 2011 +0000
@@ -7,7 +7,7 @@
 #      Mark Smith <mark@dreamwidth.org>
 #      Robby Griffin <alierak@gmail.com>
 #
-# Copyright (c) 2009 by Dreamwidth Studios, LLC.
+# Copyright (c) 2009-2010 by Dreamwidth Studios, LLC.
 #
 # This program is free software; you may redistribute it and/or modify it under
 # the same terms as Perl itself.  For a copy of the license, please reference
@@ -95,3 +95,17 @@
     command_name  dw_check_ssl_create
     command_line  $USER1$/check_http -H www.dreamwidth.org -I $HOSTADDRESS$ -u /create -S -f critical
 }
+
+
+# check sda in a two disk configuration; this is a remote check
+define command {
+    command_name  dw_ssh_check_disk_sda
+    command_line  $USER1$/check_by_ssh -t 30 -l root -H $HOSTADDRESS$ "$USER1$/custom/check_smart.pl -d /dev/sda -i ata"
+}
+
+
+# check sdb in a two disk configuration; this is a remote check
+define command {
+    command_name  dw_ssh_check_disk_sdb
+    command_line  $USER1$/check_by_ssh -t 30 -l root -H $HOSTADDRESS$ "$USER1$/custom/check_smart.pl -d /dev/sdb -i ata"
+}
diff -r fae923426ebf -r 969c631efb1a nagios/conf.d/config/hostgroups.cfg
--- a/nagios/conf.d/config/hostgroups.cfg	Fri Apr 23 20:12:34 2010 +0000
+++ b/nagios/conf.d/config/hostgroups.cfg	Fri Jul 29 00:08:09 2011 +0000
@@ -121,3 +121,10 @@
     hostgroup_name  search
     alias           Search Servers
 }
+
+
+# hosts that have two disks with no raid
+define hostgroup {
+    hostgroup_name  disks_web
+    alias           Two Disks, No RAID
+}
diff -r fae923426ebf -r 969c631efb1a nagios/conf.d/hosts/sb-admin01-auto.cfg
--- a/nagios/conf.d/hosts/sb-admin01-auto.cfg	Fri Apr 23 20:12:34 2010 +0000
+++ b/nagios/conf.d/hosts/sb-admin01-auto.cfg	Fri Jul 29 00:08:09 2011 +0000
@@ -1,4 +1,4 @@
-# Auto-generated at Thu Aug 20 19:14:29 2009
+# Auto-generated at Fri Jun 18 00:21:04 2010
 # 172.20.6.176 sb-admin01
 
 define host {
diff -r fae923426ebf -r 969c631efb1a nagios/conf.d/hosts/sb-db01-auto.cfg
--- a/nagios/conf.d/hosts/sb-db01-auto.cfg	Fri Apr 23 20:12:34 2010 +0000
+++ b/nagios/conf.d/hosts/sb-db01-auto.cfg	Fri Jul 29 00:08:09 2011 +0000
@@ -1,10 +1,10 @@
-# Auto-generated at Thu Aug 20 19:14:29 2009
+# Auto-generated at Fri Jun 18 00:21:04 2010
 # 172.20.6.169 sb-db01
 
 define host {
     host_name  sb-db01
     alias      sb-db01
-    hostgroups db,db_schwartz,mogilefsd,memc
+    hostgroups db,db_slave,mogilefsd,memc
     address    172.20.6.169
     use        generic-host
 }
diff -r fae923426ebf -r 969c631efb1a nagios/conf.d/hosts/sb-db02-auto.cfg
--- a/nagios/conf.d/hosts/sb-db02-auto.cfg	Fri Apr 23 20:12:34 2010 +0000
+++ b/nagios/conf.d/hosts/sb-db02-auto.cfg	Fri Jul 29 00:08:09 2011 +0000
@@ -1,10 +1,10 @@
-# Auto-generated at Thu Aug 20 19:14:29 2009
+# Auto-generated at Fri Jun 18 00:21:04 2010
 # 172.20.6.175 sb-db02
 
 define host {
     host_name  sb-db02
     alias      sb-db02
-    hostgroups db,db_slave,mogilefsd,memc
+    hostgroups db,db_schwartz,mogilefsd,memc
     address    172.20.6.175
     use        generic-host
 }
diff -r fae923426ebf -r 969c631efb1a nagios/conf.d/hosts/sb-jobs01-auto.cfg
--- a/nagios/conf.d/hosts/sb-jobs01-auto.cfg	Fri Apr 23 20:12:34 2010 +0000
+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
@@ -1,10 +0,0 @@
-# Auto-generated at Thu Aug 20 19:14:29 2009
-# 172.20.6.166 sb-jobs01
-
-define host {
-    host_name  sb-jobs01
-    alias      sb-jobs01
-    hostgroups jobs,mogstored
-    address    172.20.6.166
-    use        generic-host
-}
diff -r fae923426ebf -r 969c631efb1a nagios/conf.d/hosts/sb-jobs02-auto.cfg
--- a/nagios/conf.d/hosts/sb-jobs02-auto.cfg	Fri Apr 23 20:12:34 2010 +0000
+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
@@ -1,10 +0,0 @@
-# Auto-generated at Thu Aug 20 19:14:29 2009
-# 172.20.6.167 sb-jobs02
-
-define host {
-    host_name  sb-jobs02
-    alias      sb-jobs02
-    hostgroups jobs,mogstored
-    address    172.20.6.167
-    use        generic-host
-}
diff -r fae923426ebf -r 969c631efb1a nagios/conf.d/hosts/sb-lb01-auto.cfg
--- a/nagios/conf.d/hosts/sb-lb01-auto.cfg	Fri Apr 23 20:12:34 2010 +0000
+++ b/nagios/conf.d/hosts/sb-lb01-auto.cfg	Fri Jul 29 00:08:09 2011 +0000
@@ -1,10 +1,10 @@
-# Auto-generated at Thu Aug 20 19:14:29 2009
+# Auto-generated at Fri Jun 18 00:21:04 2010
 # 172.20.6.168 sb-lb01
 
 define host {
     host_name  sb-lb01
     alias      sb-lb01
-    hostgroups lb,mogstored
+    hostgroups lb,mogstored,disks_web
     address    172.20.6.168
     use        generic-host
 }
diff -r fae923426ebf -r 969c631efb1a nagios/conf.d/hosts/sb-lb02-auto.cfg
--- a/nagios/conf.d/hosts/sb-lb02-auto.cfg	Fri Apr 23 20:12:34 2010 +0000
+++ b/nagios/conf.d/hosts/sb-lb02-auto.cfg	Fri Jul 29 00:08:09 2011 +0000
@@ -1,10 +1,10 @@
-# Auto-generated at Thu Aug 20 19:14:29 2009
+# Auto-generated at Fri Jun 18 00:21:04 2010
 # 172.20.6.172 sb-lb02
 
 define host {
     host_name  sb-lb02
     alias      sb-lb02
-    hostgroups lb,mogstored,mail
+    hostgroups lb,mogstored,disks_web,mail
     address    172.20.6.172
     use        generic-host
 }
diff -r fae923426ebf -r 969c631efb1a nagios/conf.d/hosts/sb-search01-auto.cfg
--- a/nagios/conf.d/hosts/sb-search01-auto.cfg	Fri Apr 23 20:12:34 2010 +0000
+++ b/nagios/conf.d/hosts/sb-search01-auto.cfg	Fri Jul 29 00:08:09 2011 +0000
@@ -1,10 +1,10 @@
-# Auto-generated at Thu Aug 20 19:14:29 2009
+# Auto-generated at Fri Jun 18 00:21:04 2010
 # 172.20.6.171 sb-search01
 
 define host {
     host_name  sb-search01
     alias      sb-search01
-    hostgroups search
+    hostgroups search,disks_web
     address    172.20.6.171
     use        generic-host
 }
diff -r fae923426ebf -r 969c631efb1a nagios/conf.d/hosts/sb-web01-auto.cfg
--- a/nagios/conf.d/hosts/sb-web01-auto.cfg	Fri Apr 23 20:12:34 2010 +0000
+++ b/nagios/conf.d/hosts/sb-web01-auto.cfg	Fri Jul 29 00:08:09 2011 +0000
@@ -1,10 +1,10 @@
-# Auto-generated at Thu Aug 20 19:14:29 2009
+# Auto-generated at Fri Jun 18 00:21:04 2010
 # 172.20.6.173 sb-web01
 
 define host {
     host_name  sb-web01
     alias      sb-web01
-    hostgroups web,mogstored
+    hostgroups web,mogstored,jobs,disks_web
     address    172.20.6.173
     use        generic-host
 }
diff -r fae923426ebf -r 969c631efb1a nagios/conf.d/hosts/sb-web02-auto.cfg
--- a/nagios/conf.d/hosts/sb-web02-auto.cfg	Fri Apr 23 20:12:34 2010 +0000
+++ b/nagios/conf.d/hosts/sb-web02-auto.cfg	Fri Jul 29 00:08:09 2011 +0000
@@ -1,10 +1,10 @@
-# Auto-generated at Thu Aug 20 19:14:29 2009
+# Auto-generated at Fri Jun 18 00:21:04 2010
 # 172.20.6.174 sb-web02
 
 define host {
     host_name  sb-web02
     alias      sb-web02
-    hostgroups web,mogstored
+    hostgroups web,mogstored,jobs,disks_web
     address    172.20.6.174
     use        generic-host
 }
diff -r fae923426ebf -r 969c631efb1a nagios/conf.d/hosts/sb-web03-auto.cfg
--- a/nagios/conf.d/hosts/sb-web03-auto.cfg	Fri Apr 23 20:12:34 2010 +0000
+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
@@ -1,10 +0,0 @@
-# Auto-generated at Thu Aug 20 19:14:29 2009
-# 172.20.6.170 sb-web03
-
-define host {
-    host_name  sb-web03
-    alias      sb-web03
-    hostgroups web,mogstored
-    address    172.20.6.170
-    use        generic-host
-}
diff -r fae923426ebf -r 969c631efb1a nagios/conf.d/services/disks-web.cfg
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/nagios/conf.d/services/disks-web.cfg	Fri Jul 29 00:08:09 2011 +0000
@@ -0,0 +1,33 @@
+#
+# disks-web.cfg
+#
+# Configure monitoring of the two-disk configuration we use in webservers and
+# similar machines.  Two separate ATA disks /dev/sda and /dev/sdb.
+#
+# Authors:
+#      Mark Smith <mark@dreamwidth.org>
+#
+# Copyright (c) 2010 by Dreamwidth Studios, LLC.
+#
+# This program is free software; you may redistribute it and/or modify it under
+# the same terms as Perl itself.  For a copy of the license, please reference
+# 'perldoc perlartistic' or 'perldoc perlgpl'.
+#
+
+
+# verify that sda is doing okay
+define service {
+    hostgroup_name                  disks_web
+    service_description             Disk Health sda
+    check_command                   dw_ssh_check_disk_sda
+    use                             generic-service
+}
+
+
+# verify that sdb is doing okay
+define service {
+    hostgroup_name                  disks_web
+    service_description             Disk Health sdb
+    check_command                   dw_ssh_check_disk_sdb
+    use                             generic-service
+}
diff -r fae923426ebf -r 969c631efb1a nagios/custom/check_smart.pl
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/nagios/custom/check_smart.pl	Fri Jul 29 00:08:09 2011 +0000
@@ -0,0 +1,314 @@
+#!/usr/bin/perl -w
+# Check SMART status of ATA/SCSI disks, returning any usable metrics as perfdata.
+# For usage information, run ./check_smart -h
+#
+# This script was created under contract for the US Government and is therefore Public Domain
+#
+# Changes and Modifications
+# =========================
+# Feb 3, 2009: Kurt Yoder - initial version of script
+
+use strict;
+use Getopt::Long;
+
+use File::Basename qw(basename);
+my $basename = basename($0);
+
+my $revision = '$Revision: 1.0 $';
+
+use lib '/usr/lib/nagios/plugins/';
+use utils qw(%ERRORS &print_revision &support &usage);
+
+$ENV{'PATH'}='/bin:/usr/bin:/sbin:/usr/sbin';
+$ENV{'BASH_ENV'}=''; 
+$ENV{'ENV'}='';
+
+use vars qw($opt_d $opt_debug $opt_h $opt_i $opt_v);
+Getopt::Long::Configure('bundling');
+GetOptions(
+	                  "debug"       => \$opt_debug,
+	"d=s" => \$opt_d, "device=s"    => \$opt_d,
+	"h"   => \$opt_h, "help"        => \$opt_h,
+	"i=s" => \$opt_i, "interface=s" => \$opt_i,
+	"v"   => \$opt_v, "version"     => \$opt_v,
+);
+
+if ($opt_v) {
+	print_revision($basename,$revision);
+	exit $ERRORS{'OK'};
+}
+
+if ($opt_h) {
+	print_help(); 
+	exit $ERRORS{'OK'};
+}
+
+my ($device, $interface) = qw//;
+if ($opt_d) {
+	unless($opt_i){
+		print "must specify an interface for $opt_d using -i/--interface!\n\n";
+		print_help();
+		exit $ERRORS{'UNKNOWN'};
+	}
+
+    $device = $opt_d;
+    $interface = $opt_i;
+
+#	if (-b $opt_d){
+#		$device = $opt_d;
+#	}
+#	else{
+#		print "$opt_d is not a valid block device!\n\n";
+#		print_help();
+#		exit $ERRORS{'UNKNOWN'};
+#	}
+#
+#	if(grep {$opt_i eq $_} ('ata', 'scsi')){
+#		$interface = $opt_i;
+#	}
+#	else{
+#		print "invalid interface $opt_i for $opt_d!\n\n";
+#		print_help();
+#		exit $ERRORS{'UNKNOWN'};
+#	}
+}
+else{
+	print "must specify a device!\n\n";
+	print_help();
+	exit $ERRORS{'UNKNOWN'};
+}
+
+my $smart_command = '/usr/bin/sudo /usr/sbin/smartctl';
+my @error_messages = qw//;
+my $exit_status = 'OK';
+
+
+warn "###########################################################\n" if $opt_debug;
+warn "(debug) CHECK 1: getting overall SMART health status\n" if $opt_debug;
+warn "###########################################################\n\n\n" if $opt_debug;
+
+my $full_command = "$smart_command -d $interface -H $device";
+warn "(debug) executing:\n$full_command\n\n" if $opt_debug;
+
+my @output = `$full_command`;
+warn "(debug) output:\n@output\n\n" if $opt_debug;
+
+# parse ata output, looking for "health status: passed"
+my $found_status = 0;
+my $line_str = 'SMART overall-health self-assessment test result: '; # ATA SMART line
+my $ok_str = 'PASSED'; # ATA SMART OK string
+
+if ($interface eq 'scsi'){
+	$line_str = 'SMART Health Status: '; # SCSI SMART line
+	$ok_str = 'OK'; #SCSI SMART OK string
+}
+
+foreach my $line (@output){
+	if($line =~ /$line_str(.+)/){
+		$found_status = 1;
+		warn "(debug) parsing line:\n$line\n\n" if $opt_debug;
+		if ($1 eq $ok_str) {
+			warn "(debug) found string '$ok_str'; status OK\n\n" if $opt_debug;
+		}
+		else {
+			warn "(debug) no '$ok_str' status; failing\n\n" if $opt_debug;
+			push(@error_messages, "Health status: $1");
+			escalate_status('CRITICAL');
+		}
+	}
+}
+
+unless ($found_status) {
+	push(@error_messages, 'No health status line found');
+	escalate_status('UNKNOWN');
+}
+
+
+warn "###########################################################\n" if $opt_debug;
+warn "(debug) CHECK 2: getting silent SMART health check\n" if $opt_debug;
+warn "###########################################################\n\n\n" if $opt_debug;
+
+$full_command = "$smart_command -d $interface -q silent -A $device";
+warn "(debug) executing:\n$full_command\n\n" if $opt_debug;
+
+system($full_command);
+my $return_code = $?;
+warn "(debug) exit code:\n$return_code\n\n" if $opt_debug;
+
+if ($return_code & 0x01) {
+	push(@error_messages, 'Commandline parse failure');
+	escalate_status('UNKNOWN');
+}
+if ($return_code & 0x02) {
+	push(@error_messages, 'Device could not be opened');
+	escalate_status('UNKNOWN');
+}
+if ($return_code & 0x04) {
+	push(@error_messages, 'Checksum failure');
+	escalate_status('WARNING');
+}
+if ($return_code & 0x08) {
+	push(@error_messages, 'Disk is failing');
+	escalate_status('CRITICAL');
+}
+if ($return_code & 0x10) {
+	push(@error_messages, 'Disk is in prefail');
+	escalate_status('WARNING');
+}
+if ($return_code & 0x20) {
+	push(@error_messages, 'Disk may be close to failure');
+	escalate_status('WARNING');
+}
+if ($return_code & 0x40) {
+	push(@error_messages, 'Error log contains errors');
+	escalate_status('WARNING');
+}
+if ($return_code & 0x80) {
+	push(@error_messages, 'Self-test log contains errors');
+	escalate_status('WARNING');
+}
+if ($return_code && !$exit_status) {
+	push(@error_messages, 'Unknown return code');
+	escalate_status('CRITICAL');
+}
+
+if ($return_code) {
+	warn "(debug) non-zero exit code, generating error condition\n\n" if $opt_debug;
+}
+else {
+	warn "(debug) zero exit code, status OK\n\n" if $opt_debug;
+}
+
+
+warn "###########################################################\n" if $opt_debug;
+warn "(debug) CHECK 3: getting detailed statistics\n" if $opt_debug;
+warn "(debug) information contains a few more potential trouble spots\n" if $opt_debug;
+warn "(debug) plus, we can also use the information for perfdata/graphing\n" if $opt_debug;
+warn "###########################################################\n\n\n" if $opt_debug;
+
+$full_command = "$smart_command -d $interface -A $device";
+warn "(debug) executing:\n$full_command\n\n" if $opt_debug;
+@output = `$full_command`;
+warn "(debug) output:\n@output\n\n" if $opt_debug;
+my @perfdata = qw//;
+
+# separate metric-gathering and output analysis for ATA vs SCSI SMART output
+if ($interface eq 'ata'){
+	foreach my $line(@output){
+		# get lines that look like this:
+		#    9 Power_On_Minutes        0x0032   241   241   000    Old_age   Always       -       113h+12m
+		next unless $line =~ /^\s*\d+\s(\S+)\s+(?:\S+\s+){6}(\S+)\s+(\d+)/;
+		my ($attribute_name, $when_failed, $raw_value) = ($1, $2, $3);
+		if ($when_failed ne '-'){
+			push(@error_messages, "Attribute $attribute_name failed at $when_failed");
+			escalate_status('WARNING');
+			warn "(debug) parsed SMART attribute $attribute_name with error condition:\n$when_failed\n\n" if $opt_debug;
+		}
+		# some attributes produce questionable data; no need to graph them
+		if (grep {$_ eq $attribute_name} ('Unknown_Attribute', 'Power_On_Minutes') ){
+			next;
+		}
+		push (@perfdata, "$attribute_name=$raw_value");
+
+		# do some manual checks
+		if ( ($attribute_name eq 'Current_Pending_Sector') && $raw_value ) {
+			push(@error_messages, "Sectors pending re-allocation");
+			escalate_status('WARNING');
+			warn "(debug) Current_Pending_Sector is non-zero ($raw_value)\n\n" if $opt_debug;
+		}
+	}
+}
+else{
+	my ($current_temperature, $max_temperature, $current_start_stop, $max_start_stop) = qw//;
+	foreach my $line(@output){
+		if ($line =~ /Current Drive Temperature:\s+(\d+)/){
+			$current_temperature = $1;
+		}
+		elsif ($line =~ /Drive Trip Temperature:\s+(\d+)/){
+			$max_temperature = $1;
+		}
+		elsif ($line =~ /Current start stop count:\s+(\d+)/){
+			$current_start_stop = $1;
+		}
+		elsif ($line =~ /Recommended maximum start stop count:\s+(\d+)/){
+			$max_start_stop = $1;
+		}
+		elsif ($line =~ /Elements in grown defect list:\s+(\d+)/){
+			push (@perfdata, "defect_list=$1");
+		}
+		elsif ($line =~ /Blocks sent to initiator =\s+(\d+)/){
+			push (@perfdata, "sent_blocks=$1");
+		}
+	}
+	if($current_temperature){
+		if($max_temperature){
+			push (@perfdata, "temperature=$current_temperature;;$max_temperature");
+			if($current_temperature > $max_temperature){
+				warn "(debug) Disk temperature is greater than max ($current_temperature > $max_temperature)\n\n" if $opt_debug;
+				push(@error_messages, 'Disk temperature is higher than maximum');
+				escalate_status('CRITICAL');
+			}
+		}
+		else{
+			push (@perfdata, "temperature=$current_temperature");
+		}
+	}
+	if($current_start_stop){
+		if($max_start_stop){
+			push (@perfdata, "start_stop=$current_start_stop;$max_start_stop");
+			if($current_start_stop > $max_start_stop){
+				warn "(debug) Disk start_stop is greater than max ($current_start_stop > $max_start_stop)\n\n" if $opt_debug;
+				push(@error_messages, 'Disk start_stop is higher than maximum');
+				escalate_status('WARNING');
+			}
+		}
+		else{
+			push (@perfdata, "start_stop=$current_start_stop");
+		}
+	}
+}
+warn "(debug) gathered perfdata:\n@perfdata\n\n" if $opt_debug;
+my $perf_string = join(' ', @perfdata);
+
+warn "###########################################################\n" if $opt_debug;
+warn "(debug) FINAL STATUS: $exit_status\n" if $opt_debug;
+warn "###########################################################\n\n\n" if $opt_debug;
+
+warn "(debug) final status/output:\n" if $opt_debug;
+
+my $status_string = '';
+
+if($exit_status ne 'OK'){
+	$status_string = "$exit_status: ".join(', ', @error_messages);
+}
+else {
+	$status_string = "OK: no SMART errors detected";
+}
+
+print "$status_string|$perf_string\n";
+exit $ERRORS{$exit_status};
+
+sub print_help {
+	print_revision($basename,$revision);
+	print "Usage: $basename (--device=<SMART device> --interface=(ata|scsi)|-h|-v) [--debug]\n";
+	print "  --debug: show debugging information\n";
+	print "  -d/--device: a device to be SMART monitored, eg /dev/sda\n";
+	print "  -i/--interface: ata or scsi, depending upon the device's interface type\n";
+	print "  -h/--help: this help\n";
+	print "  -v/--version: Version number\n";
+	support();
+}
+
+# escalate an exit status IFF it's more severe than the previous exit status
+sub escalate_status {
+	my $requested_status = shift;
+	# no test for 'CRITICAL'; automatically escalates upwards
+	if ($requested_status eq 'WARNING') {
+		return if $exit_status eq 'CRITICAL';
+	}
+	if ($requested_status eq 'UNKNOWN') {
+		return if $exit_status eq 'WARNING';
+		return if $exit_status eq 'CRITICAL';
+	}
+	$exit_status = $requested_status;
+}
diff -r fae923426ebf -r 969c631efb1a puppet/manifests/site.pp
--- a/puppet/manifests/site.pp	Fri Apr 23 20:12:34 2010 +0000
+++ b/puppet/manifests/site.pp	Fri Jul 29 00:08:09 2011 +0000
@@ -60,11 +60,13 @@
 # node nodename { include serverclass::nonwebserver }
 
 node sb-lb02    { include serverclass::mail }
-node sb-lb01    { include serverclass::perlbal }
+#node sb-lb01    { include serverclass::perlbal }
 #node sb-lb02    { include serverclass::perlbal }
+node sb-lb01    { include serverclass::webserver }
 node sb-web01   { include serverclass::webserver }
 node sb-web02   { include serverclass::webserver }
 node sb-web03   { include serverclass::webserver }
+node sb-search01 { include serverclass::webserver }
 node sb-admin01 { include serverclass::admin }
 
 # A special case exists for the node that will run the Puppet master service.
diff -r fae923426ebf -r 969c631efb1a puppet/modules/iptables/templates/spamhaus-drop.erb
--- a/puppet/modules/iptables/templates/spamhaus-drop.erb	Fri Apr 23 20:12:34 2010 +0000
+++ b/puppet/modules/iptables/templates/spamhaus-drop.erb	Fri Jul 29 00:08:09 2011 +0000
@@ -1,46 +1,85 @@
+109.196.140.0/24
 109.94.212.0/22
-109.95.112.0/22
-110.44.0.0/20
+110.232.160.0/20
+110.44.128.0/20
+113.20.160.0/19
 116.199.128.0/19
-117.104.168.0/22
-119.42.144.0/21
+116.68.136.0/21
 121.46.64.0/18
+122.202.96.0/19
 128.168.0.0/16
 128.199.0.0/16
+129.76.64.0/18
+130.201.0.0/16
+130.222.0.0/16
+132.145.0.0/16
 132.232.0.0/16
 132.240.0.0/16
+134.127.0.0/16
+134.172.0.0/16
+134.209.0.0/16
+134.23.0.0/16
 134.33.0.0/16
-138.252.0.0/16
+136.228.0.0/16
 138.43.0.0/16
 139.167.0.0/16
+14.1.96.0/19
+14.102.160.0/19
 140.170.0.0/16
 143.135.0.0/16
 143.49.0.0/16
+143.95.0.0/16
+148.105.0.0/16
 148.178.0.0/16
 148.248.0.0/16
 150.141.0.0/16
 150.230.0.0/16
+151.123.0.0/16
 152.147.0.0/16
 155.190.0.0/16
+157.226.0.0/16
+157.232.0.0/17
+159.223.0.0/16
+162.125.0.0/16
+167.224.0.0/19
 167.28.0.0/16
 167.97.0.0/16
 168.151.0.0/16
+169.131.128.0/17
+170.106.0.0/16
+170.120.0.0/16
+170.197.0.0/16
 170.67.0.0/16
+170.75.0.0/16
+173.225.112.0/20
+174.138.144.0/20
+178.218.128.0/20
 188.130.250.0/23
 188.170.192.0/21
-188.170.216.0/21
-188.170.243.0/24
-188.210.240.0/20
-188.213.44.0/23
-188.240.0.0/20
-188.241.192.0/20
-188.241.194.0/23
-188.241.200.0/23
-188.241.202.0/23
-188.241.204.0/23
-190.112.0.0/19
+188.212.0.0/20
+188.229.13.0/24
+188.229.35.0/24
+188.229.88.0/23
+188.229.90.0/23
+188.229.92.0/24
+188.229.93.0/24
+188.229.94.0/24
+188.229.97.0/24
+192.100.5.0/24
+192.101.177.0/24
+192.101.200.0/21
+192.101.240.0/21
+192.101.248.0/23
+192.112.112.0/20
 192.160.44.0/24
+192.171.64.0/19
+192.197.87.0/24
+192.219.120.0/21
+192.219.128.0/18
+192.219.192.0/20
+192.219.208.0/21
 192.223.64.0/18
+192.229.32.0/19
 192.26.25.0/24
 192.31.212.0/23
 192.43.153.0/24
@@ -52,79 +91,174 @@
 192.43.184.0/24
 192.67.16.0/24
 192.86.85.0/24
-193.104.106.0/24
 193.104.110.0/24
 193.104.12.0/24
-193.104.153.0/24
+193.104.146.0/24
 193.104.176.0/24
-193.104.22.0/24
+193.104.224.0/22
 193.104.253.0/24
-193.104.27.0/24
+193.104.34.0/24
 193.104.41.0/24
 193.104.94.0/24
-193.105.0.0/24
+193.105.121.0/24
+193.105.132.0/24
 193.105.141.0/24
+193.105.174.0/24
 193.105.184.0/24
+193.105.207.0/24
+193.105.245.0/24
+193.106.32.0/22
 193.110.136.0/24
-193.138.172.0/22
+193.111.235.0/24
 193.142.244.0/24
+193.148.47.0/24
 193.16.100.0/24
-193.169.234.0/23
+193.16.213.0/24
 193.169.250.0/23
-193.238.36.0/22
+193.178.172.0/24
+193.186.9.0/24
+193.201.192.0/23
+193.227.240.0/23
+193.228.145.0/24
+193.23.126.0/24
+193.238.0.0/22
+193.27.232.0/23
 193.27.246.0/23
+193.41.38.0/24
 193.43.134.0/24
 193.46.211.0/24
+194.0.221.0/24
+194.0.245.0/24
+194.1.220.0/23
 194.110.160.0/22
 194.116.146.0/23
 194.126.193.0/24
+194.126.251.0/24
+194.140.229.0/24
 194.143.130.0/23
 194.146.204.0/22
 194.165.4.0/23
+194.247.58.0/24
+194.28.112.0/22
+194.28.44.0/22
+194.54.156.0/22
+194.60.205.0/24
+194.63.144.0/22
+194.8.250.0/23
 194.8.74.0/23
 195.114.8.0/23
+195.14.112.0/23
+195.149.88.0/24
+195.162.6.0/23
 195.182.57.0/24
+195.190.157.0/24
+195.191.102.0/23
+195.2.212.0/23
 195.225.176.0/22
+195.226.197.0/24
+195.226.220.0/24
 195.234.159.0/24
 195.238.242.0/24
-195.5.168.0/24
+195.28.10.0/23
+195.5.161.0/24
+195.54.170.0/23
 195.74.88.0/23
-195.78.122.0/23
+195.78.108.0/23
+195.80.148.0/22
+195.85.204.0/24
+195.88.144.0/23
 195.88.190.0/23
 195.88.226.0/23
-195.88.32.0/23
 195.93.184.0/23
 195.93.208.0/23
 195.95.151.0/24
 195.95.155.0/24
 196.1.176.0/20
 196.32.216.0/21
+198.12.32.0/19
+198.13.0.0/20
+198.143.128.0/19
 198.151.152.0/22
+198.162.208.0/20
+198.181.64.0/19
+198.183.32.0/19
 198.186.16.0/20
 198.186.25.0/24
+198.20.16.0/20
 198.204.0.0/21
+198.205.64.0/19
+198.23.32.0/20
+198.252.64.0/18
+198.45.32.0/20
+198.57.64.0/20
+198.96.224.0/20
 199.120.163.0/24
+199.165.32.0/19
 199.166.200.0/22
+199.196.192.0/19
+199.198.160.0/20
+199.198.176.0/21
+199.198.184.0/23
+199.198.188.0/22
+199.200.64.0/19
+199.230.64.0/19
+199.230.96.0/21
 199.245.138.0/24
+199.246.137.0/24
+199.246.213.0/24
+199.246.215.0/24
+199.248.64.0/18
+199.33.145.0/24
+199.34.128.0/18
+199.46.32.0/19
+199.5.152.0/23
 199.60.102.0/24
+199.84.64.0/19
+199.88.32.0/20
+199.88.48.0/22
 200.106.128.0/20
-200.115.112.0/21
-200.115.96.0/20
-200.123.224.0/20
-200.124.160.0/21
+200.115.112.0/20
+200.125.168.0/21
 200.22.0.0/16
+200.3.128.0/20
 200.50.192.0/19
+200.63.40.0/21
 201.71.0.0/20
-203.19.101.0/24
+202.59.236.0/24
+202.61.127.0/24
 203.31.88.0/23
-203.34.205.0/24
 203.34.70.0/23
 203.34.71.0/24
-204.13.32.0/21
+204.106.128.0/18
+204.106.192.0/19
+204.107.208.0/24
+204.126.244.0/23
+204.130.167.0/24
+204.187.155.0/24
+204.187.156.0/22
+204.187.160.0/19
+204.187.224.0/20
+204.187.240.0/21
+204.187.248.0/22
+204.187.252.0/23
+204.187.254.0/24
+204.194.184.0/21
+204.225.159.0/24
+204.225.210.0/24
 204.236.0.0/19
+204.28.104.0/21
+204.48.64.0/19
 204.52.255.0/24
+204.63.144.0/21
 204.89.224.0/24
-205.210.137.0/24
+205.142.104.0/22
+205.144.0.0/20
+205.145.192.0/19
+205.159.180.0/24
+205.172.244.0/22
+205.175.160.0/19
+205.203.224.0/19
+205.214.128.0/19
 205.235.64.0/20
 205.236.189.0/24
 206.197.175.0/24
@@ -132,59 +266,111 @@
 206.197.177.0/24
 206.197.28.0/24
 206.197.29.0/24
-208.77.224.0/21
+206.201.48.0/20
+206.203.64.0/18
+206.224.160.0/19
+206.227.64.0/18
+206.81.0.0/19
+207.183.192.0/19
+207.189.0.0/19
+207.22.192.0/18
 208.81.136.0/21
-208.82.136.0/21
-208.84.96.0/21
-208.87.152.0/21
+208.85.32.0/21
 208.90.0.0/21
-209.165.224.0/20
+209.145.0.0/19
+209.182.64.0/19
+209.198.176.0/20
+209.205.192.0/19
+209.205.224.0/20
 209.213.48.0/20
-213.109.176.0/20
+209.51.32.0/20
+209.95.192.0/19
 213.109.208.0/20
 213.109.96.0/22
-216.243.240.0/20
+213.247.0.0/19
+216.151.192.0/20
+216.212.192.0/19
+216.245.64.0/18
+31.222.200.0/21
 41.221.112.0/20
+46.161.28.0/23
+46.161.29.0/24
+46.252.128.0/23
+46.252.130.0/23
+46.252.132.0/23
+46.51.100.0/24
+46.51.101.0/24
+46.51.102.0/23
 58.83.12.0/22
 58.83.8.0/22
 62.122.32.0/21
+62.122.72.0/23
+62.18.252.0/24
 62.182.152.0/21
+62.19.0.0/18
+62.19.248.0/21
+62.19.249.0/24
+62.19.250.0/24
+62.19.64.0/22
+64.112.0.0/17
+64.112.128.0/18
 64.15.0.0/20
+64.250.96.0/20
 64.28.176.0/20
-66.206.32.0/22
+64.44.0.0/16
+66.231.64.0/20
 67.210.0.0/20
 67.211.208.0/20
 67.213.128.0/20
 67.218.208.0/20
-69.8.176.0/20
 72.13.16.0/20
 72.2.176.0/20
 72.50.192.0/19
 78.155.220.0/23
-78.157.128.0/19
 78.31.184.0/21
+79.110.144.0/20
 79.110.16.0/20
 79.110.160.0/20
 79.110.176.0/20
 79.110.48.0/20
+79.174.0.0/19
+81.22.152.0/23
+85.121.39.0/24
+85.202.160.0/20
 85.202.192.0/20
 85.255.112.0/20
-86.105.230.0/24
-88.135.64.0/21
+86.55.140.0/24
+86.55.210.0/23
+86.55.243.0/24
 88.135.64.0/20
 88.214.211.0/24
-89.35.0.0/23
+89.114.9.0/24
+89.114.97.0/24
+89.45.14.0/24
+91.193.192.0/22
+91.194.0.0/23
 91.196.232.0/22
+91.197.96.0/22
 91.198.109.0/24
+91.198.125.0/24
+91.198.127.0/24
+91.198.217.0/24
 91.199.112.0/24
+91.199.123.0/24
+91.199.157.0/24
 91.200.164.0/22
+91.200.188.0/22
+91.200.240.0/22
 91.200.248.0/22
 91.201.124.0/22
 91.201.196.0/22
+91.201.212.0/22
+91.201.236.0/22
 91.201.4.0/22
 91.203.92.0/22
+91.204.40.0/21
+91.204.48.0/22
 91.205.40.0/22
-91.206.200.0/23
 91.207.116.0/23
 91.208.0.0/24
 91.208.162.0/24
@@ -193,52 +379,84 @@
 91.209.183.0/24
 91.209.184.0/24
 91.209.186.0/24
+91.209.238.0/24
 91.209.48.0/24
 91.209.58.0/24
 91.210.172.0/22
-91.211.224.0/22
 91.211.64.0/22
 91.211.88.0/22
 91.212.107.0/24
 91.212.123.0/24
+91.212.127.0/24
 91.212.132.0/24
+91.212.135.0/24
 91.212.163.0/24
+91.212.198.0/24
 91.212.201.0/24
 91.212.220.0/24
 91.212.41.0/24
 91.212.45.0/24
 91.212.65.0/24
 91.213.121.0/24
-91.213.126.0/24
+91.213.157.0/24
 91.213.174.0/24
+91.213.217.0/24
 91.213.29.0/24
-91.213.33.0/24
 91.213.72.0/24
 91.213.75.0/24
 91.213.93.0/24
 91.213.94.0/24
-93.118.0.0/20
-93.118.128.0/18
-93.118.96.0/20
+91.216.11.0/24
+91.216.122.0/24
+91.216.141.0/24
+91.216.190.0/24
+91.216.232.0/24
+91.216.3.0/24
+91.216.73.0/24
+91.217.153.0/24
+91.217.162.0/24
+91.217.249.0/24
+91.220.110.0/24
+91.220.62.0/24
+91.220.90.0/24
+91.223.201.0/24
 93.120.32.0/19
 93.168.18.0/23
 93.168.20.0/23
 93.168.22.0/23
 93.168.24.0/23
+93.174.164.0/24
 93.175.240.0/20
 93.188.160.0/21
+93.188.162.0/23
 94.126.176.0/21
 94.130.0.0/15
-94.154.0.0/18
 94.154.128.0/18
-94.154.64.0/18
 94.158.240.0/20
 94.232.248.0/21
-94.48.0.0/18
-95.129.144.0/23
-95.129.146.0/24
-95.177.128.0/18
-95.177.192.0/19
-95.177.224.0/20
-95.215.192.0/22
+94.60.121.0/24
+94.60.122.0/23
+94.63.146.0/24
+94.63.149.0/24
+94.63.150.0/23
+94.63.240.0/24
+94.63.241.0/24
+94.63.243.0/24
+94.63.244.0/23
+94.63.245.0/24
+94.63.246.0/24
+94.63.247.0/24
+95.215.140.0/22
 95.216.0.0/15
+95.64.11.0/24
+95.64.13.0/24
+95.64.37.0/24
+95.64.38.0/24
+95.64.39.0/24
+95.64.44.0/24
+95.64.45.0/24
+95.64.55.0/24
+95.64.59.0/24
+95.64.61.0/24
+95.64.9.0/24
+95.64.98.0/23
--------------------------------------------------------------------------------