From 850359711cb8eaf0ee7de05325e392095bdf42d4 Mon Sep 17 00:00:00 2001 From: Adam Amstrong Date: Fri, 27 Apr 2012 17:18:26 +0000 Subject: [PATCH] add memcached application graphing support. move applications to <<>> for easier detection. add connection tracking to mysql application. move logout to /logout/ not /?logout=yes. git-svn-id: http://www.observium.org/svn/observer/trunk@3096 61d68cd4-352d-0410-923a-c4978735b2b8 --- html/includes/authenticate.inc.php | 3 +- .../graphs/application/memcached.inc.php | 7 +++ .../graphs/application/memcached_bits.inc.php | 13 +++++ .../application/memcached_commands.inc.php | 34 +++++++++++++ .../application/memcached_uptime.inc.php | 19 +++++++ html/includes/header.inc.php | 2 +- html/index.php | 4 +- html/pages/device/apps.inc.php | 20 +++++++- html/pages/device/apps/memcached.inc.php | 32 ++++++++++++ includes/functions.php | 4 +- includes/polling/applications/apache.inc.php | 4 +- includes/polling/applications/drbd.inc.php | 2 +- .../polling/applications/memcached.inc.php | 44 +++++++++++++++++ includes/polling/applications/mysql.inc.php | 49 ++++++++++++++++--- includes/polling/functions.inc.php | 7 ++- includes/polling/unix-agent.inc.php | 44 ++++++++++++++--- includes/static-config.php | 6 ++- poller.php | 2 +- scripts/agent-local/memcached | 13 +++++ scripts/agent-local/mysql | 3 ++ sql-schema/015.sql | 2 + 21 files changed, 286 insertions(+), 28 deletions(-) create mode 100644 html/includes/graphs/application/memcached.inc.php create mode 100755 html/includes/graphs/application/memcached_bits.inc.php create mode 100644 html/includes/graphs/application/memcached_commands.inc.php create mode 100644 html/includes/graphs/application/memcached_uptime.inc.php create mode 100644 html/pages/device/apps/memcached.inc.php create mode 100644 includes/polling/applications/memcached.inc.php create mode 100755 scripts/agent-local/memcached create mode 100644 sql-schema/015.sql diff --git a/html/includes/authenticate.inc.php b/html/includes/authenticate.inc.php index 13bd8e8b9..5ca574a9f 100644 --- a/html/includes/authenticate.inc.php +++ b/html/includes/authenticate.inc.php @@ -20,7 +20,7 @@ if (!is_writable($config['temp_dir'])) echo("
Temp Directory is not writable ({$config['tmp_dir']}). Graphing may fail.
"); } -if (isset($_GET['logout']) && $_SESSION['authenticated']) +if ($vars['page'] == "logout" && $_SESSION['authenticated']) { dbInsert(array('user' => $_SESSION['username'], 'address' => $_SERVER["REMOTE_ADDR"], 'result' => 'Logged Out'), 'authlog'); unset($_SESSION); @@ -78,7 +78,6 @@ if (isset($_SESSION['username'])) setcookie("password", $_SESSION['password'], time()+60*60*24*100, "/"); } $permissions = permissions_cache($_SESSION['user_id']); - } elseif (isset($_SESSION['username'])) { diff --git a/html/includes/graphs/application/memcached.inc.php b/html/includes/graphs/application/memcached.inc.php new file mode 100644 index 000000000..8afd6bd1c --- /dev/null +++ b/html/includes/graphs/application/memcached.inc.php @@ -0,0 +1,7 @@ + diff --git a/html/includes/graphs/application/memcached_bits.inc.php b/html/includes/graphs/application/memcached_bits.inc.php new file mode 100755 index 000000000..2fadf3e16 --- /dev/null +++ b/html/includes/graphs/application/memcached_bits.inc.php @@ -0,0 +1,13 @@ + diff --git a/html/includes/graphs/application/memcached_commands.inc.php b/html/includes/graphs/application/memcached_commands.inc.php new file mode 100644 index 000000000..5f756cb2e --- /dev/null +++ b/html/includes/graphs/application/memcached_commands.inc.php @@ -0,0 +1,34 @@ + 'Set', + 'cmd_get' => 'Get', +); + +$i = 0; +if (is_file($rrd_filename)) +{ + foreach ($array as $ds => $vars) + { + $rrd_list[$i]['filename'] = $rrd_filename; + if (is_array($vars)) + { + $rrd_list[$i]['descr'] = $vars['descr']; + } else { + $rrd_list[$i]['descr'] = $vars; + } + $rrd_list[$i]['ds'] = $ds; + $i++; + } +} else { echo("file missing: $file"); } + +$colours = "mixed"; +$nototal = 0; +$unit_text = ""; + +include("includes/graphs/generic_multi_simplex_seperated.inc.php"); + +?> diff --git a/html/includes/graphs/application/memcached_uptime.inc.php b/html/includes/graphs/application/memcached_uptime.inc.php new file mode 100644 index 000000000..116c593c1 --- /dev/null +++ b/html/includes/graphs/application/memcached_uptime.inc.php @@ -0,0 +1,19 @@ + diff --git a/html/includes/header.inc.php b/html/includes/header.inc.php index f5511546f..12ab3a193 100644 --- a/html/includes/header.inc.php +++ b/html/includes/header.inc.php @@ -24,7 +24,7 @@ if($_SESSION['widescreen'] === 1) if ($_SESSION['authenticated']) { - echo("Logged in as ".$_SESSION['username']." (Logout)"); + echo("Logged in as ".$_SESSION['username']." (Logout)"); } else { echo("Not logged in!"); } diff --git a/html/index.php b/html/index.php index 9555f4240..3a9a4357b 100755 --- a/html/index.php +++ b/html/index.php @@ -29,8 +29,6 @@ if (strpos($_SERVER['PATH_INFO'], "debug")) ini_set('error_reporting', 0); } -include("includes/authenticate.inc.php"); - foreach ($_GET as $key=>$get_var) { if (strstr($key, "opt")) @@ -97,6 +95,8 @@ foreach ($_POST as $name => $value) $vars[$name] = $value; } +include("includes/authenticate.inc.php"); + if (strstr($_SERVER['REQUEST_URI'], 'widescreen=yes')) { $_SESSION['widescreen'] = 1; } if (strstr($_SERVER['REQUEST_URI'], 'widescreen=no')) { unset($_SESSION['widescreen']); } diff --git a/html/pages/device/apps.inc.php b/html/pages/device/apps.inc.php index 14a73083c..5eca3710a 100644 --- a/html/pages/device/apps.inc.php +++ b/html/pages/device/apps.inc.php @@ -23,14 +23,30 @@ foreach (dbFetchRows("SELECT * FROM `applications` WHERE `device_id` = ?", array } else { #echo(''); } - echo(generate_link(ucfirst($app['app_type']),$link_array,array('app'=>$app['app_type']))); + $link_add = array('app'=>$app['app_type']); + $text = ucfirst($app['app_type']); + if(!empty($app['app_instance'])) + { + $text .= "(".$app['app_instance'].")"; + $link_add['instance'] = $app['app_id']; + } + echo(generate_link($text,$link_array,$link_add)); if ($vars['app'] == $app['app_type']) { echo(""); } $sep = " | "; } print_optionbar_end(); -$app = dbFetchRow("SELECT * FROM `applications` WHERE `device_id` = ? AND `app_type` = ?", array($device['device_id'], $vars['app'])); +$where_array = array($device['device_id'], $vars['app']); +if($vars['instance']) +{ + $where = " AND `app_id` = ?"; + $where_array[] = $vars['instance']; +} + +$app = dbFetchRow("SELECT * FROM `applications` WHERE `device_id` = ? AND `app_type` = ?".$where, $where_array); + +#print_r($app); if (is_file("pages/device/apps/".mres($vars['app']).".inc.php")) { diff --git a/html/pages/device/apps/memcached.inc.php b/html/pages/device/apps/memcached.inc.php new file mode 100644 index 000000000..aab74021e --- /dev/null +++ b/html/pages/device/apps/memcached.inc.php @@ -0,0 +1,32 @@ + 'Traffic', + 'memcached_commands' => 'Commands', + 'memcached_data' => 'Data Size', + 'memcached_items' => 'Items', + 'memcached_uptime' => 'Uptime', + 'memcached_threads' => 'Threads', + +); + +foreach ($graphs as $key => $text) +{ + + $graph_array['height'] = "100"; + $graph_array['width'] = "215"; + $graph_array['to'] = $now; + $graph_array['id'] = $app['app_id']; + $graph_array['type'] = "application_".$key; + + echo('

'.$text.'

'); + + echo(""); + + include("includes/print-graphrow.inc.php"); + + echo(""); +} + +?> diff --git a/includes/functions.php b/includes/functions.php index 273337742..e9cc821a6 100755 --- a/includes/functions.php +++ b/includes/functions.php @@ -533,8 +533,8 @@ function notify($device,$title,$message) $emails = parse_email($email); if ($emails) { - $message_header = $config['page_title_prefix']."\n"; - $message_footer = "E-mail sent to: "; + $message_header = $config['page_title_prefix']."\n\n"; + $message_footer = "\n\nE-mail sent to: "; $i = 0; $count = count($emails); foreach ($emails as $email => $email_name) diff --git a/includes/polling/applications/apache.inc.php b/includes/polling/applications/apache.inc.php index 8d0ee7697..d4bc663db 100644 --- a/includes/polling/applications/apache.inc.php +++ b/includes/polling/applications/apache.inc.php @@ -2,9 +2,9 @@ ## Polls Apache statistics from script via SNMP -if (!empty($agent_data['apache'])) +if (!empty($agent_data['app']['apache'])) { - $apache = $agent_data['apache']; + $apache = $agent_data['app']['apache']; } else { $options = "-O qv"; $oid = "nsExtendOutputFull.6.97.112.97.99.104.101"; diff --git a/includes/polling/applications/drbd.inc.php b/includes/polling/applications/drbd.inc.php index ab29c0b7b..d676f64bc 100644 --- a/includes/polling/applications/drbd.inc.php +++ b/includes/polling/applications/drbd.inc.php @@ -1,7 +1,7 @@ diff --git a/includes/polling/applications/mysql.inc.php b/includes/polling/applications/mysql.inc.php index 59de2d8f2..efbebdfd6 100644 --- a/includes/polling/applications/mysql.inc.php +++ b/includes/polling/applications/mysql.inc.php @@ -1,9 +1,9 @@ 'cr', 'IBLFh' => 'ct', @@ -115,9 +117,6 @@ foreach ($mapping as $key) } $string = implode(':', $values); -#$data = explode("\n", $mysql); -#$nstring = trim(implode(':',$data),':'); - if (!is_file($mysql_rrd)) { rrdtool_create ($mysql_rrd, "--step 300 \ @@ -203,6 +202,44 @@ if (!is_file($mysql_rrd)) } rrdtool_update($mysql_rrd, "N:$string"); -echo("done "); + +/// Process state statistics + +$mysql_status_rrd = $config['rrd_dir'] . "/" . $device['hostname'] . "/app-mysql-".$app['app_id']."-status.rrd"; + +$mapping_status = array( + 'State_closing_tables' => 'd2', + 'State_copying_to_tmp_table' => 'd3', + 'State_end' => 'd4', + 'State_freeing_items' => 'd5', + 'State_init' => 'd6', + 'State_locked' => 'd7', + 'State_login' => 'd8', + 'State_preparing' => 'd9', + 'State_reading_from_net' => 'da', + 'State_sending_data' => 'db', + 'State_sorting_result' => 'dc', + 'State_statistics' => 'dd', + 'State_updating' => 'de', + 'State_writing_to_net' => 'df', + 'State_none' => 'dg', + 'State_other' => 'dh' +); + +$values = array(); $rrd_create = ""; +foreach ($mapping_status as $desc => $id) +{ + $values[] = isset($map[$id]) ? $map[$id] : -1; + $rrd_create .= " DS:".$id.":GAUGE:600:0:125000000000"; +} +$string = implode(':', $values); + +if (!is_file($mysql_status_rrd)) +{ + rrdtool_create ($mysql_status_rrd, "--step 300 ".$rrd_create." ".$config['rrd_rra']); +} + +rrdtool_update($mysql_status_rrd, "N:$string"); + ?> diff --git a/includes/polling/functions.inc.php b/includes/polling/functions.inc.php index 5022063c3..3f4fb2b3c 100644 --- a/includes/polling/functions.inc.php +++ b/includes/polling/functions.inc.php @@ -134,9 +134,12 @@ function poll_device($device, $options) if ($options['m']) { - if (is_file("includes/polling/".$options['m'].".inc.php")) + foreach(explode(",", $options['m']) as $module) { - include("includes/polling/".$options['m'].".inc.php"); + if (is_file("includes/polling/".$module.".inc.php")) + { + include("includes/polling/".$module.".inc.php"); + } } } else { foreach ($config['poller_modules'] as $module => $module_status) diff --git a/includes/polling/unix-agent.inc.php b/includes/polling/unix-agent.inc.php index 800b16d5c..3e46d69f2 100755 --- a/includes/polling/unix-agent.inc.php +++ b/includes/polling/unix-agent.inc.php @@ -5,9 +5,12 @@ if($device['os_group'] == "unix") echo("Observium UNIX Agent: "); - $port='6556'; + ### FIXME - this should be in config and overridable in database - $agent = fsockopen($device['hostname'], $port, $errno, $errstr, 10); + $agent_port='6556'; + + $agent_start = utime(); + $agent = fsockopen($device['hostname'], $agent_port, $errno, $errstr, 10); if (!$agent) { @@ -18,14 +21,29 @@ if($device['os_group'] == "unix") $agent_raw .= fgets($agent, 128); } } + $agent_end = utime(); $agent_time = round(($agent_end - $agent_start) * 1000); if (!empty($agent_raw)) { + echo("execution time: ".$agent_time."ms"); + $agent_rrd = $config['rrd_dir'] . "/" . $device['hostname'] . "/agent.rrd"; + if (!is_file($agent_rrd)) + { + rrdtool_create ($agent_rrd, "DS:time:GAUGE:600:0:U ".$config['rrd_rra']); + } + rrdtool_update($agent_rrd, "N:".$agent_time); + $graphs['agent'] = TRUE; + foreach (explode("<<<", $agent_raw) as $section) { list($section, $data) = explode(">>>", $section); list($sa, $sb) = explode("-", $section, 2); + + if($section == "apache") { $sa = "app"; $sb = "apache"; } + if($section == "mysql") { $sa = "app"; $sb = "mysql"; } + if($section == "drbd") { $sa = "app"; $sb = "drbd"; } + if(!empty($sa) && !empty($sb)) { $agent_data[$sa][$sb] = trim($data); @@ -34,7 +52,7 @@ if($device['os_group'] == "unix") } } - #print_r($agent_data); +# print_r($agent_data); include("unix-agent/packages.inc.php"); include("unix-agent/munin-plugins.inc.php"); @@ -53,7 +71,7 @@ if($device['os_group'] == "unix") } ### Apache - if (!empty($agent_data['apache'])) + if (!empty($agent_data['app']['apache'])) { $app_found['apache'] = TRUE; if (dbFetchCell("SELECT COUNT(*) FROM `applications` WHERE `device_id` = ? AND `app_type` = ?", array($device['device_id'], 'apache')) == "0") @@ -63,8 +81,22 @@ if($device['os_group'] == "unix") } } + ### memcached + if (!empty($agent_data['app']['memcached'])) + { + $agent_data['app']['memcached'] = unserialize($agent_data['app']['memcached']); + foreach ($agent_data['app']['memcached'] as $memcached_host => $memcached_data) + { + if (dbFetchCell("SELECT COUNT(*) FROM `applications` WHERE `device_id` = ? AND `app_type` = ? AND `app_instance` = ?", array($device['device_id'], 'memcached', $memcached_host)) == "0") + { + echo("Found new application 'Memcached' $memcached_host\n"); + dbInsert(array('device_id' => $device['device_id'], 'app_type' => 'memcached', 'app_instance' => $memcached_host), 'applications'); + } + } + } + ### MySQL - if (!empty($agent_data['mysql'])) + if (!empty($agent_data['app']['mysql'])) { $app_found['mysql'] = TRUE; if (dbFetchCell("SELECT COUNT(*) FROM `applications` WHERE `device_id` = ? AND `app_type` = ?", array($device['device_id'], 'mysql')) == "0") @@ -75,7 +107,7 @@ if($device['os_group'] == "unix") } ### DRBD - if (!empty($agent_data['drbd'])) + if (!empty($agent_data['app']['drbd'])) { $agent_data['drbd_raw'] = $agent_data['drbd']; $agent_data['drbd'] = array(); diff --git a/includes/static-config.php b/includes/static-config.php index 69dacc122..c990c53fb 100644 --- a/includes/static-config.php +++ b/includes/static-config.php @@ -896,12 +896,16 @@ foreach ($config['os'] as $this_os => $blah) ### Graph Types -$config['graph_sections'] = array('system', 'firewall', 'netstats', 'wireless', 'storage', 'vpdn', 'load balancer'); +$config['graph_sections'] = array('general', 'system', 'firewall', 'netstats', 'wireless', 'storage', 'vpdn', 'load balancer'); $config['graph_types']['device']['wifi_clients']['section'] = 'wireless'; $config['graph_types']['device']['wifi_clients']['order'] = '0'; $config['graph_types']['device']['wifi_clients']['descr'] = 'Wireless Clients'; +$config['graph_types']['device']['agent']['section'] = 'system'; +$config['graph_types']['device']['agent']['order'] = '0'; +$config['graph_types']['device']['agent']['descr'] = 'Agent Execution Time'; + $config['graph_types']['device']['cipsec_flow_bits']['section'] = 'firewall'; $config['graph_types']['device']['cipsec_flow_bits']['order'] = '0'; $config['graph_types']['device']['cipsec_flow_bits']['descr'] = 'IPSec Tunnel Traffic Volume'; diff --git a/poller.php b/poller.php index c81bc741b..c75d13749 100755 --- a/poller.php +++ b/poller.php @@ -65,7 +65,7 @@ if (!$where) echo(" Instances start at 0. 0-3 for -n 4\n\n"); echo("Debugging and testing options:\n"); echo("-d Enable debugging output\n"); - echo("-m Specify single module to be run\n"); + echo("-m Specify module(s) to be run\n"); echo("\n"); echo("No polling type specified!\n"); exit; diff --git a/scripts/agent-local/memcached b/scripts/agent-local/memcached new file mode 100755 index 000000000..b4ed626f8 --- /dev/null +++ b/scripts/agent-local/memcached @@ -0,0 +1,13 @@ +#!/usr/bin/php +addServer('localhost', 11211); +$stats = $m->getStats(); + +if(is_array($stats)) +{ + echo("<<>>\n"); + echo(serialize($m->getStats())); + echo("\n"); +} +?> diff --git a/scripts/agent-local/mysql b/scripts/agent-local/mysql index a0ca1b5ec..3d31757c3 100755 --- a/scripts/agent-local/mysql +++ b/scripts/agent-local/mysql @@ -650,6 +650,7 @@ function ss_get_mysql_stats( $options ) { 'binary_log_space' => 'cz', 'innodb_locked_tables' => 'd0', 'innodb_lock_structs' => 'd1', + 'State_closing_tables' => 'd2', 'State_copying_to_tmp_table' => 'd3', 'State_end' => 'd4', @@ -666,6 +667,7 @@ function ss_get_mysql_stats( $options ) { 'State_writing_to_net' => 'df', 'State_none' => 'dg', 'State_other' => 'dh', + 'Handler_commit' => 'di', 'Handler_delete' => 'dj', 'Handler_discover' => 'dk', @@ -681,6 +683,7 @@ function ss_get_mysql_stats( $options ) { 'Handler_savepoint_rollback' => 'du', 'Handler_update' => 'dv', 'Handler_write' => 'dw', + # Some InnoDB stats added later... 'innodb_tables_in_use' => 'dx', 'innodb_lock_wait_secs' => 'dy', diff --git a/sql-schema/015.sql b/sql-schema/015.sql new file mode 100644 index 000000000..577b526e8 --- /dev/null +++ b/sql-schema/015.sql @@ -0,0 +1,2 @@ +ALTER TABLE `munin_plugins_ds` CHANGE `ds_cdef` `ds_cdef` VARCHAR( 255 ) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL; +ALTER TABLE `applications` ADD `app_state` TEXT CHARACTER SET utf8 COLLATE utf8_bin NOT NULL;