mark: A photo of Mark kneeling on top of the Taal Volcano in the Philippines. It was a long hike. (Default)
Mark Smith ([staff profile] mark) wrote in [site community profile] changelog2009-08-05 06:15 pm
  • Add Memory
  • Share This Entry

[dw-ops] Update replication check to be more lenient, add monitoring of dfw-search01, spamhaus updat

[commit: http://hg.dwscoalition.org/dw-ops/rev/b352fd4d49fb]

Update replication check to be more lenient, add monitoring of dfw-search01,
spamhaus update.

Patch by [staff profile] mark.

Files modified:
  • nagios/conf.d/config/hostgroups.cfg
  • nagios/conf.d/hosts/dfw-admin01-auto.cfg
  • nagios/conf.d/hosts/dfw-db-a01-auto.cfg
  • nagios/conf.d/hosts/dfw-db-a02-auto.cfg
  • nagios/conf.d/hosts/dfw-db-b01-auto.cfg
  • nagios/conf.d/hosts/dfw-db-b02-auto.cfg
  • nagios/conf.d/hosts/dfw-db-c01-auto.cfg
  • nagios/conf.d/hosts/dfw-db-c02-auto.cfg
  • nagios/conf.d/hosts/dfw-db01-auto.cfg
  • nagios/conf.d/hosts/dfw-db02-auto.cfg
  • nagios/conf.d/hosts/dfw-jobs01-auto.cfg
  • nagios/conf.d/hosts/dfw-jobs02-auto.cfg
  • nagios/conf.d/hosts/dfw-lb01-auto.cfg
  • nagios/conf.d/hosts/dfw-lb02-auto.cfg
  • nagios/conf.d/hosts/dfw-mail01-auto.cfg
  • nagios/conf.d/hosts/dfw-memc01-auto.cfg
  • nagios/conf.d/hosts/dfw-memc02-auto.cfg
  • nagios/conf.d/hosts/dfw-mog01-auto.cfg
  • nagios/conf.d/hosts/dfw-mog02-auto.cfg
  • nagios/conf.d/hosts/dfw-mog03-auto.cfg
  • nagios/conf.d/hosts/dfw-search01-auto.cfg
  • nagios/conf.d/hosts/dfw-web01-auto.cfg
  • nagios/conf.d/hosts/dfw-web02-auto.cfg
  • nagios/conf.d/hosts/dfw-web03-auto.cfg
  • nagios/conf.d/hosts/dfw-web04-auto.cfg
  • nagios/conf.d/hosts/dfw-web05-auto.cfg
  • nagios/custom/check_mysql_replication.pl
  • puppet/modules/iptables/templates/spamhaus-drop.erb
--------------------------------------------------------------------------------
diff -r b09e9825f6fc -r b352fd4d49fb nagios/conf.d/config/hostgroups.cfg
--- a/nagios/conf.d/config/hostgroups.cfg	Thu Jul 23 23:50:41 2009 +0000
+++ b/nagios/conf.d/config/hostgroups.cfg	Wed Aug 05 18:15:17 2009 +0000
@@ -106,3 +106,10 @@ define hostgroup {
     hostgroup_name  admin
     alias           Admin Servers
 }
+
+
+# search
+define hostgroup {
+    hostgroup_name  search
+    alias           Search Servers
+}
diff -r b09e9825f6fc -r b352fd4d49fb nagios/conf.d/hosts/dfw-admin01-auto.cfg
--- a/nagios/conf.d/hosts/dfw-admin01-auto.cfg	Thu Jul 23 23:50:41 2009 +0000
+++ b/nagios/conf.d/hosts/dfw-admin01-auto.cfg	Wed Aug 05 18:15:17 2009 +0000
@@ -1,4 +1,4 @@
-# Auto-generated at Fri May  1 21:03:32 2009
+# Auto-generated at Fri Jul 24 16:08:51 2009
 # 10.176.64.124 dfw-admin01
 
 define host {
diff -r b09e9825f6fc -r b352fd4d49fb nagios/conf.d/hosts/dfw-db-a01-auto.cfg
--- a/nagios/conf.d/hosts/dfw-db-a01-auto.cfg	Thu Jul 23 23:50:41 2009 +0000
+++ b/nagios/conf.d/hosts/dfw-db-a01-auto.cfg	Wed Aug 05 18:15:17 2009 +0000
@@ -1,4 +1,4 @@
-# Auto-generated at Fri May  1 21:03:32 2009
+# Auto-generated at Fri Jul 24 16:08:51 2009
 # 10.176.71.86 dfw-db-a01
 
 define host {
diff -r b09e9825f6fc -r b352fd4d49fb nagios/conf.d/hosts/dfw-db-a02-auto.cfg
--- a/nagios/conf.d/hosts/dfw-db-a02-auto.cfg	Thu Jul 23 23:50:41 2009 +0000
+++ b/nagios/conf.d/hosts/dfw-db-a02-auto.cfg	Wed Aug 05 18:15:17 2009 +0000
@@ -1,4 +1,4 @@
-# Auto-generated at Fri May  1 21:03:32 2009
+# Auto-generated at Fri Jul 24 16:08:51 2009
 # 10.176.71.88 dfw-db-a02
 
 define host {
diff -r b09e9825f6fc -r b352fd4d49fb nagios/conf.d/hosts/dfw-db-b01-auto.cfg
--- a/nagios/conf.d/hosts/dfw-db-b01-auto.cfg	Thu Jul 23 23:50:41 2009 +0000
+++ b/nagios/conf.d/hosts/dfw-db-b01-auto.cfg	Wed Aug 05 18:15:17 2009 +0000
@@ -1,4 +1,4 @@
-# Auto-generated at Fri May  1 21:03:32 2009
+# Auto-generated at Fri Jul 24 16:08:51 2009
 # 10.176.71.89 dfw-db-b01
 
 define host {
diff -r b09e9825f6fc -r b352fd4d49fb nagios/conf.d/hosts/dfw-db-b02-auto.cfg
--- a/nagios/conf.d/hosts/dfw-db-b02-auto.cfg	Thu Jul 23 23:50:41 2009 +0000
+++ b/nagios/conf.d/hosts/dfw-db-b02-auto.cfg	Wed Aug 05 18:15:17 2009 +0000
@@ -1,4 +1,4 @@
-# Auto-generated at Fri May  1 21:03:32 2009
+# Auto-generated at Fri Jul 24 16:08:51 2009
 # 10.176.71.90 dfw-db-b02
 
 define host {
diff -r b09e9825f6fc -r b352fd4d49fb nagios/conf.d/hosts/dfw-db-c01-auto.cfg
--- a/nagios/conf.d/hosts/dfw-db-c01-auto.cfg	Thu Jul 23 23:50:41 2009 +0000
+++ b/nagios/conf.d/hosts/dfw-db-c01-auto.cfg	Wed Aug 05 18:15:17 2009 +0000
@@ -1,4 +1,4 @@
-# Auto-generated at Fri May  1 21:03:32 2009
+# Auto-generated at Fri Jul 24 16:08:51 2009
 # 10.176.74.51 dfw-db-c01
 
 define host {
diff -r b09e9825f6fc -r b352fd4d49fb nagios/conf.d/hosts/dfw-db-c02-auto.cfg
--- a/nagios/conf.d/hosts/dfw-db-c02-auto.cfg	Thu Jul 23 23:50:41 2009 +0000
+++ b/nagios/conf.d/hosts/dfw-db-c02-auto.cfg	Wed Aug 05 18:15:17 2009 +0000
@@ -1,4 +1,4 @@
-# Auto-generated at Fri May  1 21:03:32 2009
+# Auto-generated at Fri Jul 24 16:08:51 2009
 # 10.176.74.52 dfw-db-c02
 
 define host {
diff -r b09e9825f6fc -r b352fd4d49fb nagios/conf.d/hosts/dfw-db01-auto.cfg
--- a/nagios/conf.d/hosts/dfw-db01-auto.cfg	Thu Jul 23 23:50:41 2009 +0000
+++ b/nagios/conf.d/hosts/dfw-db01-auto.cfg	Wed Aug 05 18:15:17 2009 +0000
@@ -1,4 +1,4 @@
-# Auto-generated at Fri May  1 21:03:32 2009
+# Auto-generated at Fri Jul 24 16:08:51 2009
 # 10.176.64.134 dfw-db01
 
 define host {
diff -r b09e9825f6fc -r b352fd4d49fb nagios/conf.d/hosts/dfw-db02-auto.cfg
--- a/nagios/conf.d/hosts/dfw-db02-auto.cfg	Thu Jul 23 23:50:41 2009 +0000
+++ b/nagios/conf.d/hosts/dfw-db02-auto.cfg	Wed Aug 05 18:15:17 2009 +0000
@@ -1,4 +1,4 @@
-# Auto-generated at Fri May  1 21:03:32 2009
+# Auto-generated at Fri Jul 24 16:08:51 2009
 # 10.176.64.135 dfw-db02
 
 define host {
diff -r b09e9825f6fc -r b352fd4d49fb nagios/conf.d/hosts/dfw-jobs01-auto.cfg
--- a/nagios/conf.d/hosts/dfw-jobs01-auto.cfg	Thu Jul 23 23:50:41 2009 +0000
+++ b/nagios/conf.d/hosts/dfw-jobs01-auto.cfg	Wed Aug 05 18:15:17 2009 +0000
@@ -1,4 +1,4 @@
-# Auto-generated at Fri May  1 21:03:32 2009
+# Auto-generated at Fri Jul 24 16:08:51 2009
 # 10.176.64.133 dfw-jobs01
 
 define host {
diff -r b09e9825f6fc -r b352fd4d49fb nagios/conf.d/hosts/dfw-jobs02-auto.cfg
--- a/nagios/conf.d/hosts/dfw-jobs02-auto.cfg	Thu Jul 23 23:50:41 2009 +0000
+++ b/nagios/conf.d/hosts/dfw-jobs02-auto.cfg	Wed Aug 05 18:15:17 2009 +0000
@@ -1,4 +1,4 @@
-# Auto-generated at Fri May  1 21:03:32 2009
+# Auto-generated at Fri Jul 24 16:08:51 2009
 # 10.176.68.212 dfw-jobs02
 
 define host {
diff -r b09e9825f6fc -r b352fd4d49fb nagios/conf.d/hosts/dfw-lb01-auto.cfg
--- a/nagios/conf.d/hosts/dfw-lb01-auto.cfg	Thu Jul 23 23:50:41 2009 +0000
+++ b/nagios/conf.d/hosts/dfw-lb01-auto.cfg	Wed Aug 05 18:15:17 2009 +0000
@@ -1,4 +1,4 @@
-# Auto-generated at Fri May  1 21:03:32 2009
+# Auto-generated at Fri Jul 24 16:08:51 2009
 # 10.176.64.125 dfw-lb01
 
 define host {
diff -r b09e9825f6fc -r b352fd4d49fb nagios/conf.d/hosts/dfw-lb02-auto.cfg
--- a/nagios/conf.d/hosts/dfw-lb02-auto.cfg	Thu Jul 23 23:50:41 2009 +0000
+++ b/nagios/conf.d/hosts/dfw-lb02-auto.cfg	Wed Aug 05 18:15:17 2009 +0000
@@ -1,4 +1,4 @@
-# Auto-generated at Fri May  1 21:03:32 2009
+# Auto-generated at Fri Jul 24 16:08:51 2009
 # 10.176.64.126 dfw-lb02
 
 define host {
diff -r b09e9825f6fc -r b352fd4d49fb nagios/conf.d/hosts/dfw-mail01-auto.cfg
--- a/nagios/conf.d/hosts/dfw-mail01-auto.cfg	Thu Jul 23 23:50:41 2009 +0000
+++ b/nagios/conf.d/hosts/dfw-mail01-auto.cfg	Wed Aug 05 18:15:17 2009 +0000
@@ -1,4 +1,4 @@
-# Auto-generated at Fri May  1 21:03:32 2009
+# Auto-generated at Fri Jul 24 16:08:51 2009
 # 10.176.64.137 dfw-mail01
 
 define host {
diff -r b09e9825f6fc -r b352fd4d49fb nagios/conf.d/hosts/dfw-memc01-auto.cfg
--- a/nagios/conf.d/hosts/dfw-memc01-auto.cfg	Thu Jul 23 23:50:41 2009 +0000
+++ b/nagios/conf.d/hosts/dfw-memc01-auto.cfg	Wed Aug 05 18:15:17 2009 +0000
@@ -1,4 +1,4 @@
-# Auto-generated at Fri May  1 21:03:32 2009
+# Auto-generated at Fri Jul 24 16:08:51 2009
 # 10.176.64.127 dfw-memc01
 
 define host {
diff -r b09e9825f6fc -r b352fd4d49fb nagios/conf.d/hosts/dfw-memc02-auto.cfg
--- a/nagios/conf.d/hosts/dfw-memc02-auto.cfg	Thu Jul 23 23:50:41 2009 +0000
+++ b/nagios/conf.d/hosts/dfw-memc02-auto.cfg	Wed Aug 05 18:15:17 2009 +0000
@@ -1,4 +1,4 @@
-# Auto-generated at Fri May  1 21:03:32 2009
+# Auto-generated at Fri Jul 24 16:08:51 2009
 # 10.176.64.128 dfw-memc02
 
 define host {
diff -r b09e9825f6fc -r b352fd4d49fb nagios/conf.d/hosts/dfw-mog01-auto.cfg
--- a/nagios/conf.d/hosts/dfw-mog01-auto.cfg	Thu Jul 23 23:50:41 2009 +0000
+++ b/nagios/conf.d/hosts/dfw-mog01-auto.cfg	Wed Aug 05 18:15:17 2009 +0000
@@ -1,4 +1,4 @@
-# Auto-generated at Fri May  1 21:03:32 2009
+# Auto-generated at Fri Jul 24 16:08:51 2009
 # 10.176.64.129 dfw-mog01
 
 define host {
diff -r b09e9825f6fc -r b352fd4d49fb nagios/conf.d/hosts/dfw-mog02-auto.cfg
--- a/nagios/conf.d/hosts/dfw-mog02-auto.cfg	Thu Jul 23 23:50:41 2009 +0000
+++ b/nagios/conf.d/hosts/dfw-mog02-auto.cfg	Wed Aug 05 18:15:17 2009 +0000
@@ -1,4 +1,4 @@
-# Auto-generated at Fri May  1 21:03:32 2009
+# Auto-generated at Fri Jul 24 16:08:51 2009
 # 10.176.64.130 dfw-mog02
 
 define host {
diff -r b09e9825f6fc -r b352fd4d49fb nagios/conf.d/hosts/dfw-mog03-auto.cfg
--- a/nagios/conf.d/hosts/dfw-mog03-auto.cfg	Thu Jul 23 23:50:41 2009 +0000
+++ b/nagios/conf.d/hosts/dfw-mog03-auto.cfg	Wed Aug 05 18:15:17 2009 +0000
@@ -1,4 +1,4 @@
-# Auto-generated at Fri May  1 21:03:32 2009
+# Auto-generated at Fri Jul 24 16:08:51 2009
 # 10.176.74.58 dfw-mog03
 
 define host {
diff -r b09e9825f6fc -r b352fd4d49fb nagios/conf.d/hosts/dfw-search01-auto.cfg
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/nagios/conf.d/hosts/dfw-search01-auto.cfg	Wed Aug 05 18:15:17 2009 +0000
@@ -0,0 +1,10 @@
+# Auto-generated at Fri Jul 24 16:08:51 2009
+# 10.176.77.95 dfw-search01
+
+define host {
+    host_name  dfw-search01
+    alias      dfw-search01
+    hostgroups search
+    address    10.176.77.95
+    use        generic-host
+}
diff -r b09e9825f6fc -r b352fd4d49fb nagios/conf.d/hosts/dfw-web01-auto.cfg
--- a/nagios/conf.d/hosts/dfw-web01-auto.cfg	Thu Jul 23 23:50:41 2009 +0000
+++ b/nagios/conf.d/hosts/dfw-web01-auto.cfg	Wed Aug 05 18:15:17 2009 +0000
@@ -1,4 +1,4 @@
-# Auto-generated at Fri May  1 21:03:32 2009
+# Auto-generated at Fri Jul 24 16:08:51 2009
 # 10.176.64.131 dfw-web01
 
 define host {
diff -r b09e9825f6fc -r b352fd4d49fb nagios/conf.d/hosts/dfw-web02-auto.cfg
--- a/nagios/conf.d/hosts/dfw-web02-auto.cfg	Thu Jul 23 23:50:41 2009 +0000
+++ b/nagios/conf.d/hosts/dfw-web02-auto.cfg	Wed Aug 05 18:15:17 2009 +0000
@@ -1,4 +1,4 @@
-# Auto-generated at Fri May  1 21:03:32 2009
+# Auto-generated at Fri Jul 24 16:08:51 2009
 # 10.176.64.132 dfw-web02
 
 define host {
diff -r b09e9825f6fc -r b352fd4d49fb nagios/conf.d/hosts/dfw-web03-auto.cfg
--- a/nagios/conf.d/hosts/dfw-web03-auto.cfg	Thu Jul 23 23:50:41 2009 +0000
+++ b/nagios/conf.d/hosts/dfw-web03-auto.cfg	Wed Aug 05 18:15:17 2009 +0000
@@ -1,4 +1,4 @@
-# Auto-generated at Fri May  1 21:03:32 2009
+# Auto-generated at Fri Jul 24 16:08:51 2009
 # 10.176.74.62 dfw-web03
 
 define host {
diff -r b09e9825f6fc -r b352fd4d49fb nagios/conf.d/hosts/dfw-web04-auto.cfg
--- a/nagios/conf.d/hosts/dfw-web04-auto.cfg	Thu Jul 23 23:50:41 2009 +0000
+++ b/nagios/conf.d/hosts/dfw-web04-auto.cfg	Wed Aug 05 18:15:17 2009 +0000
@@ -1,4 +1,4 @@
-# Auto-generated at Fri May  1 21:03:32 2009
+# Auto-generated at Fri Jul 24 16:08:51 2009
 # 10.176.74.63 dfw-web04
 
 define host {
diff -r b09e9825f6fc -r b352fd4d49fb nagios/conf.d/hosts/dfw-web05-auto.cfg
--- a/nagios/conf.d/hosts/dfw-web05-auto.cfg	Thu Jul 23 23:50:41 2009 +0000
+++ b/nagios/conf.d/hosts/dfw-web05-auto.cfg	Wed Aug 05 18:15:17 2009 +0000
@@ -1,4 +1,4 @@
-# Auto-generated at Fri May  1 21:03:32 2009
+# Auto-generated at Fri Jul 24 16:08:51 2009
 # 10.176.74.64 dfw-web05
 
 define host {
diff -r b09e9825f6fc -r b352fd4d49fb nagios/custom/check_mysql_replication.pl
--- a/nagios/custom/check_mysql_replication.pl	Thu Jul 23 23:50:41 2009 +0000
+++ b/nagios/custom/check_mysql_replication.pl	Wed Aug 05 18:15:17 2009 +0000
@@ -24,11 +24,13 @@ crit( "Replication stopped: IO=$rep->{Sl
     unless $rep->{Slave_IO_Running} eq 'Yes' &&
            $rep->{Slave_SQL_Running} eq 'Yes';
 
-# very tight limits
+# loose limits because we don't actively use slaves for anything
+# that we care about for realtime serving.  also, the codebase knows
+# how to avoid a slave that is falling behind.
 crit( "Replication very far behind: $rep->{Seconds_Behind_Master}." )
-    if $rep->{Seconds_Behind_Master} > 60;
+    if $rep->{Seconds_Behind_Master} > 3600;
 _warn( "Replication behind: $rep->{Seconds_Behind_Master}." )
-    if $rep->{Seconds_Behind_Master} > 15;
+    if $rep->{Seconds_Behind_Master} > 1800;
 
 ok( 'Replication okay.' );
 
diff -r b09e9825f6fc -r b352fd4d49fb puppet/modules/iptables/templates/spamhaus-drop.erb
--- a/puppet/modules/iptables/templates/spamhaus-drop.erb	Thu Jul 23 23:50:41 2009 +0000
+++ b/puppet/modules/iptables/templates/spamhaus-drop.erb	Wed Aug 05 18:15:17 2009 +0000
@@ -83,7 +83,6 @@ 206.197.28.0/24
 206.197.28.0/24
 206.197.29.0/24
 206.51.240.0/21
-208.64.44.0/22
 208.66.192.0/22
 208.72.168.0/21
 208.73.88.0/21
--------------------------------------------------------------------------------