From b54ed80bfcd6cfb581c9f794da48409b56e83e4d Mon Sep 17 00:00:00 2001 From: laf Date: Mon, 7 Dec 2015 01:51:26 +0000 Subject: [PATCH 01/29] Stop realtime graph page from auto refreshing --- html/pages/device/port/realtime.inc.php | 2 ++ 1 file changed, 2 insertions(+) diff --git a/html/pages/device/port/realtime.inc.php b/html/pages/device/port/realtime.inc.php index 41311917f..7b282066e 100644 --- a/html/pages/device/port/realtime.inc.php +++ b/html/pages/device/port/realtime.inc.php @@ -1,5 +1,7 @@ Date: Mon, 7 Dec 2015 02:22:44 +0000 Subject: [PATCH 02/29] Updated device_by_id_cache() to convert IP column --- includes/common.php | 1 + 1 file changed, 1 insertion(+) diff --git a/includes/common.php b/includes/common.php index d74a512a9..04c681aaf 100644 --- a/includes/common.php +++ b/includes/common.php @@ -315,6 +315,7 @@ function device_by_id_cache($device_id, $refresh = '0') { } else { $device = dbFetchRow("SELECT * FROM `devices` WHERE `device_id` = ?", array($device_id)); + $device['ip'] = inet6_ntop($device['ip']); $cache['devices']['id'][$device_id] = $device; } return $device; From b831d096a91d19e987aea3b283e0f71c33f856f1 Mon Sep 17 00:00:00 2001 From: laf Date: Mon, 7 Dec 2015 03:02:06 +0000 Subject: [PATCH 03/29] For entity-sensor, changed variable name again --- includes/discovery/sensors/entity-sensor.inc.php | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/includes/discovery/sensors/entity-sensor.inc.php b/includes/discovery/sensors/entity-sensor.inc.php index 3a7de2e6a..0e5eb383a 100644 --- a/includes/discovery/sensors/entity-sensor.inc.php +++ b/includes/discovery/sensors/entity-sensor.inc.php @@ -50,7 +50,7 @@ if (is_array($oids)) { $descr = rewrite_entity_descr($descr); } - $valid = check_entity_sensor($descr, $device); + $valid_sensor = check_entity_sensor($descr, $device); $type = $entitysensor[$entry['entPhySensorType']]; @@ -103,16 +103,16 @@ if (is_array($oids)) { if ($type == 'temperature') { if ($current > '200') { - $valid = false; + $valid_sensor = false; } $descr = preg_replace('/[T|t]emperature[|s]/', '', $descr); } // echo($descr . "|" . $index . "|" .$current . "|" . $multiplier . "|" . $divisor ."|" . $entry['entPhySensorScale'] . "|" . $entry['entPhySensorPrecision'] . "\n"); if ($current == '-127') { - $valid = false; + $valid_sensor = false; } - if ($valid && dbFetchCell("SELECT COUNT(*) FROM `sensors` WHERE device_id = ? AND `sensor_class` = ? AND `sensor_type` = 'cisco-entity-sensor' AND `sensor_index` = ?", array($device['device_id'], $type, $index)) == '0') { + if ($valid_sensor && dbFetchCell("SELECT COUNT(*) FROM `sensors` WHERE device_id = ? AND `sensor_class` = ? AND `sensor_type` = 'cisco-entity-sensor' AND `sensor_index` = ?", array($device['device_id'], $type, $index)) == '0') { // Check to make sure we've not already seen this sensor via cisco's entity sensor mib discover_sensor($valid['sensor'], $type, $device, $oid, $index, 'entity-sensor', $descr, $divisor, $multiplier, null, null, null, null, $current); } From 92a7e5796d6d1fde4354d256e2541f5d8b8868e4 Mon Sep 17 00:00:00 2001 From: KHobbits Date: Mon, 1 Feb 2016 23:29:37 +0000 Subject: [PATCH 04/29] Add ./validate.php -m rrdcheck This will loop through rrd files and highlight any corrupt files. --- includes/functions.php | 62 ++++++++++++++++++++++++++++++++++++++++++ validate.php | 35 ++++++++++++++++++++++++ 2 files changed, 97 insertions(+) diff --git a/includes/functions.php b/includes/functions.php index e399a10ae..c49857bca 100644 --- a/includes/functions.php +++ b/includes/functions.php @@ -1366,3 +1366,65 @@ function dnslookup($device,$type=false,$return=false) { $record = dns_get_record($device['hostname'],$type); return $record[0][$return]; }//end dnslookup + + +/** + * Reursive Filter Iterator to iterate directories and locate .rrd files. + * +**/ + +class RRDReursiveFilterIterator extends \RecursiveFilterIterator { + public function accept() { + $filename = $this->current()->getFilename(); + if ($filename[0] === '.') { + // Ignore hidden files and directories + return false; + } + if ($this->isDir()) { + // We want to search into directories + return true; + } + // Matches files with .rrd in the filename. + // We are only searching rrd folder, but there could be other files and we don't want to cause a stink. + return strpos($filename, '.rrd') !== false; + } +} + +/** + * Run rrdtool info on a file path + * + * @param string $path Path to pass to rrdtool info + * @param string $stdOutput Variable to recieve the output of STDOUT + * @param string $stdError Variable to recieve the output of STDERR + * + * @return int exit code + * +**/ + +function rrdtest($path, &$stdOutput, &$stdError) { + //rrdtool info + $command = $config['rrdtool'].' info '.escapeshellarg($path); + $process = proc_open( + $command, + array ( + 0 => array('pipe', 'r'), + 1 => array('pipe', 'w'), + 2 => array('pipe', 'w'), + ), + $pipes + ); + + if (!is_resource($process)) { + throw new \RuntimeException('Could not create a valid process'); + } + + $status = proc_get_status($process); + while($status['running']) { + $status = proc_get_status($process); + } + + $stdOutput = stream_get_contents($pipes[1]); + $stdError = stream_get_contents($pipes[2]); + proc_close($process); + return $status['exitcode']; +} diff --git a/validate.php b/validate.php index 13db33efd..8d64262f7 100755 --- a/validate.php +++ b/validate.php @@ -24,6 +24,7 @@ if (isset($options['h'])) { -m Any sub modules you want to run, comma separated: - mail: this will test your email settings (uses default_mail option even if default_only is not set). - dist-poller: this will test for the install running as a distributed poller. + - rrdcheck: this will check to see if your rrd files are corrupt Example: ./validate.php -m mail. @@ -267,6 +268,40 @@ foreach ($modules as $module) { } } } + break; + case 'rrdcheck': + + $directory = new RecursiveDirectoryIterator($config['rrd_dir']); + $filter = new RRDReursiveFilterIterator($directory); + $iterator = new RecursiveIteratorIterator($filter); + $total = iterator_count($iterator); + $iterator->rewind(); + + echo 'Scanning '.$total.' rrd files...'.PHP_EOL; + + $count = 0; + $screenpad = 0; + + foreach ($iterator as $filename => $file) { + + $testResult = rrdtest($filename, $output, $error); + + $count++; + if (($count % 100) == 0 ) { + echo "\033[".$screenpad.'D'; + $echo = 'Status: '.$count.'/'.$total; + echo $echo; + $screenpad = strlen($echo); + } + + if ($testResult > 0) { + echo "\033[".$screenpad.'D'; + echo PHP_EOL.'Error parsing "'.$filename.'" '.$error.PHP_EOL; + $screenpad = 0; + } + + } + break; }//end switch }//end foreach From 5ebe6b44c08a452548b318d52199fa806c0dd360 Mon Sep 17 00:00:00 2001 From: KHobbits Date: Mon, 1 Feb 2016 23:46:28 +0000 Subject: [PATCH 05/29] Code clean up --- includes/functions.php | 3 ++- validate.php | 44 ++++++++++++++++++++++++------------------ 2 files changed, 27 insertions(+), 20 deletions(-) diff --git a/includes/functions.php b/includes/functions.php index c49857bca..d3dad3911 100644 --- a/includes/functions.php +++ b/includes/functions.php @@ -1373,7 +1373,7 @@ function dnslookup($device,$type=false,$return=false) { * **/ -class RRDReursiveFilterIterator extends \RecursiveFilterIterator { +class RRDRecursiveFilterIterator extends \RecursiveFilterIterator { public function accept() { $filename = $this->current()->getFilename(); if ($filename[0] === '.') { @@ -1402,6 +1402,7 @@ class RRDReursiveFilterIterator extends \RecursiveFilterIterator { **/ function rrdtest($path, &$stdOutput, &$stdError) { + global $config; //rrdtool info $command = $config['rrdtool'].' info '.escapeshellarg($path); $process = proc_open( diff --git a/validate.php b/validate.php index 8d64262f7..f2fe619e3 100755 --- a/validate.php +++ b/validate.php @@ -271,36 +271,42 @@ foreach ($modules as $module) { break; case 'rrdcheck': - $directory = new RecursiveDirectoryIterator($config['rrd_dir']); - $filter = new RRDReursiveFilterIterator($directory); - $iterator = new RecursiveIteratorIterator($filter); - $total = iterator_count($iterator); - $iterator->rewind(); + // Loop through the rrd_dir + $rrd_directory = new RecursiveDirectoryIterator($config['rrd_dir']); + // Filter out any non rrd files + $rrd_directory_filter = new RRDRecursiveFilterIterator($rrd_directory); + $rrd_iterator = new RecursiveIteratorIterator($rrd_directory_filter); + $rrd_total = iterator_count($rrd_iterator); + $rrd_iterator->rewind(); // Rewind iterator in case iterator_count left iterator in unknown state - echo 'Scanning '.$total.' rrd files...'.PHP_EOL; + echo "\nScanning ".$rrd_total." rrd files in ".$config['rrd_dir']."...\n"; - $count = 0; + // Count loops so we can push status to the user + $loopcount = 0; $screenpad = 0; - foreach ($iterator as $filename => $file) { + foreach ($rrd_iterator as $filename => $file) { - $testResult = rrdtest($filename, $output, $error); + $rrd_test_result = rrdtest($filename, $output, $error); - $count++; - if (($count % 100) == 0 ) { - echo "\033[".$screenpad.'D'; - $echo = 'Status: '.$count.'/'.$total; - echo $echo; - $screenpad = strlen($echo); + $loopcount++; + if (($loopcount % 50) == 0 ) { + //This lets us update the previous status update without spamming in most consoles + echo "\033[".$screenpad."D"; + $test_status = 'Status: '.$loopcount.'/'.$rrd_total; + echo $test_status; + $screenpad = strlen($test_status); } - if ($testResult > 0) { - echo "\033[".$screenpad.'D'; - echo PHP_EOL.'Error parsing "'.$filename.'" '.$error.PHP_EOL; + // A non zero result means there was some kind of error + if ($rrd_test_result > 0) { + echo "\033[".$screenpad."D"; + print_fail('Error parsing "'.$filename.'" RRD '.trim($error)); $screenpad = 0; } - } + echo "\033[".$screenpad."D"; + echo "Status: ".$loopcount."/".$rrd_total." - Complete\n"; break; }//end switch From f2bad9816b920dccf57b602a10d1a2fc5935d5ff Mon Sep 17 00:00:00 2001 From: KHobbits Date: Tue, 2 Feb 2016 01:05:29 +0000 Subject: [PATCH 06/29] Attempt to scare away scrutinizer issues. --- includes/functions.php | 3 +++ 1 file changed, 3 insertions(+) diff --git a/includes/functions.php b/includes/functions.php index d3dad3911..827f9345f 100644 --- a/includes/functions.php +++ b/includes/functions.php @@ -1371,9 +1371,12 @@ function dnslookup($device,$type=false,$return=false) { /** * Reursive Filter Iterator to iterate directories and locate .rrd files. * + * @method boolean isDir() + * **/ class RRDRecursiveFilterIterator extends \RecursiveFilterIterator { + public function accept() { $filename = $this->current()->getFilename(); if ($filename[0] === '.') { From 028b4300ba8c41ede3faadce7555a3f81c910f9f Mon Sep 17 00:00:00 2001 From: "Robert (KHobbits)" Date: Tue, 2 Feb 2016 11:36:49 +0000 Subject: [PATCH 07/29] Slow polling loop --- includes/functions.php | 1 + 1 file changed, 1 insertion(+) diff --git a/includes/functions.php b/includes/functions.php index 827f9345f..e54710054 100644 --- a/includes/functions.php +++ b/includes/functions.php @@ -1424,6 +1424,7 @@ function rrdtest($path, &$stdOutput, &$stdError) { $status = proc_get_status($process); while($status['running']) { + usleep(2000); // Sleep 2000 microseconds or 2 milliseconds $status = proc_get_status($process); } From dc5f286b9a1c679c9a33701df80cb7d5a589e29f Mon Sep 17 00:00:00 2001 From: laf Date: Tue, 2 Feb 2016 13:47:13 +0000 Subject: [PATCH 08/29] Added ifOperStatus and ifAdminStatus _prev values to db. --- includes/polling/ports.inc.php | 3 +++ sql-schema/099.sql | 2 ++ 2 files changed, 5 insertions(+) create mode 100644 sql-schema/099.sql diff --git a/includes/polling/ports.inc.php b/includes/polling/ports.inc.php index 553069110..aabad8a2a 100644 --- a/includes/polling/ports.inc.php +++ b/includes/polling/ports.inc.php @@ -410,6 +410,9 @@ foreach ($ports as $port) { } } $port['update'][$oid] = $this_port[$oid]; + if ($oid == 'ifOperStatus' || $oid == 'ifAdminStatus') { + $port['update'][$oid.'_prev'] = $port[$oid]; + } log_event($oid.': '.$port[$oid].' -> '.$this_port[$oid], $device, 'interface', $port['port_id']); if ($debug) { d_echo($oid.': '.$port[$oid].' -> '.$this_port[$oid].' '); diff --git a/sql-schema/099.sql b/sql-schema/099.sql new file mode 100644 index 000000000..814d2abc9 --- /dev/null +++ b/sql-schema/099.sql @@ -0,0 +1,2 @@ +ALTER TABLE `ports` ADD `ifOperStatus_prev` VARCHAR( 16 ) NULL AFTER `ifOperStatus` ; +ALTER TABLE `ports` ADD `ifAdminStatus_prev` VARCHAR( 16 ) NULL AFTER `ifAdminStatus` ; From 44cd86d24e1bbbfbb2a96779353e77c0e8c9e661 Mon Sep 17 00:00:00 2001 From: Tony Murray Date: Tue, 2 Feb 2016 20:23:08 -0600 Subject: [PATCH 09/29] Add systemd unit file for the python poller service. Move poller-service init scripts into to scripts directory and update the documentation. --- doc/Extensions/Poller-Service.md | 8 ++++++-- .../librenms-poller-service.conf | 0 .../librenms-poller-service.init | 6 +++--- scripts/librenms-poller-service.service | 14 ++++++++++++++ 4 files changed, 23 insertions(+), 5 deletions(-) rename poller-service.conf => scripts/librenms-poller-service.conf (100%) rename poller-service.init => scripts/librenms-poller-service.init (94%) create mode 100644 scripts/librenms-poller-service.service diff --git a/doc/Extensions/Poller-Service.md b/doc/Extensions/Poller-Service.md index b3ec67a1d..e07dfab33 100644 --- a/doc/Extensions/Poller-Service.md +++ b/doc/Extensions/Poller-Service.md @@ -32,6 +32,10 @@ Because locks are not replicated in Multi-Master MySQL configurations, if you ar If you are running MariaDB 10.2 or newer, you can tell poller-service to use a single mysql connection for managing locks by setting `$config['poller_service_single_connection']` to `true`. *DO NOT* configure this for any version of MariaDB less than 10.2 or any version of MySQL. ## Service Installation -An upstart configuration `poller-service.conf` is provided. To install run `ln -s /opt/librenms/poller-service.conf /etc/init/poller-service.conf`. The service will start on boot and can be started manually by running `start poller-service`. If you receive an error that the service does not exist, run `initctl reload-configuration`. The service is configured to run as the user `librenms` and will fail if that user does not exist. +### Upstart +An upstart configuration file can be found in `scripts/librenms-poller-service.conf`. To install run `ln -s /opt/librenms/scripts/librenms-poller-service.conf /etc/init/librenms-poller-service.conf`. The service will start on boot and can be started manually by running `start librenms-poller-service`. If you receive an error that the service does not exist, run `initctl reload-configuration`. The service is configured to run as the user `librenms` and will fail if that user does not exist. +### LSB +An LSB init script can be found in `scripts/librenms-poller-service.init`. To install run `ln -s /opt/librenms/scripts/librenms-poller-service.init /etc/init.d/librenms-poller-service && update-rc.d librenms-poller-service defaults`. +### systemd +A systemd unit file can be found in `scripts/librenms-poller-service.init`. To install run `ln -s /opt/librenms/scripts/librenms-poller-service.service /etc/systemd/system/librenms-poller-service.service && systemctl enable librenms-poller-service.service`. -An LSB init script `poller-service.init` is also provided. To install run `ln -s /opt/librenms/poller-service.init /etc/init.d/poller-service && update-rc.d poller-service defaults`. diff --git a/poller-service.conf b/scripts/librenms-poller-service.conf similarity index 100% rename from poller-service.conf rename to scripts/librenms-poller-service.conf diff --git a/poller-service.init b/scripts/librenms-poller-service.init similarity index 94% rename from poller-service.init rename to scripts/librenms-poller-service.init index 6e870a6db..a080372bc 100755 --- a/poller-service.init +++ b/scripts/librenms-poller-service.init @@ -1,5 +1,5 @@ ### BEGIN INIT INFO -# Provides: poller-service +# Provides: librenms-poller-service # Required-Start: networking # Required-Stop: networking # Default-Start: 2 3 4 5 @@ -11,13 +11,13 @@ . /lib/lsb/init-functions -NAME=poller-service +NAME=librenms-poller-service DAEMON=/opt/librenms/poller-service.py USER=librenms -PIDFILE=/var/run/poller-service.pid +PIDFILE=/var/run/librenms-poller-service.pid test -x $DAEMON || exit 5 diff --git a/scripts/librenms-poller-service.service b/scripts/librenms-poller-service.service new file mode 100644 index 000000000..86f918807 --- /dev/null +++ b/scripts/librenms-poller-service.service @@ -0,0 +1,14 @@ +[Unit] +Description=LibreNMS SNMP Poller Service +After=network.target + +[Service] +ExecStart=/opt/librenms/poller-service.py +WorkingDirectory=/opt/librenms +User=librenms +Group=librenms +RestartSec=2 +Restart=always + +[Install] +WantedBy=multi-user.target From 350345b611fa3f7e334bcf38e2270d58a6bc30b0 Mon Sep 17 00:00:00 2001 From: laf Date: Wed, 3 Feb 2016 07:52:35 +0000 Subject: [PATCH 10/29] Updated parsing of alert rules to allow | --- html/includes/forms/parse-alert-rule.inc.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/html/includes/forms/parse-alert-rule.inc.php b/html/includes/forms/parse-alert-rule.inc.php index aa3120424..286fb35c8 100644 --- a/html/includes/forms/parse-alert-rule.inc.php +++ b/html/includes/forms/parse-alert-rule.inc.php @@ -20,7 +20,7 @@ $alert_id = $_POST['alert_id']; if (is_numeric($alert_id) && $alert_id > 0) { $rule = dbFetchRow('SELECT * FROM `alert_rules` WHERE `id` = ? LIMIT 1', array($alert_id)); - $rule_split = preg_split('/([a-zA-Z0-9_\-\.\=\%\<\>\ \"\'\!\~\(\)\*\/\@]+[&&\|\|]+)/', $rule['rule'], -1, (PREG_SPLIT_DELIM_CAPTURE | PREG_SPLIT_NO_EMPTY)); + $rule_split = preg_split('/([a-zA-Z0-9_\-\.\=\%\<\>\ \"\'\!\~\(\)\*\/\@\|]+[&&|\|\|]{2})/', $rule['rule'], -1, (PREG_SPLIT_DELIM_CAPTURE | PREG_SPLIT_NO_EMPTY)); $count = (count($rule_split) - 1); $rule_split[$count] = $rule_split[$count].' &&'; $output = array( From aa43f418717ab4df69fa6e918867c960a682d4c2 Mon Sep 17 00:00:00 2001 From: Tony Murray Date: Wed, 3 Feb 2016 08:17:19 -0600 Subject: [PATCH 11/29] Change the poller-service docs from ln -s to cp to prevent us from breaking user's installs. --- doc/Extensions/Poller-Service.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/Extensions/Poller-Service.md b/doc/Extensions/Poller-Service.md index e07dfab33..f183e0f10 100644 --- a/doc/Extensions/Poller-Service.md +++ b/doc/Extensions/Poller-Service.md @@ -33,9 +33,9 @@ If you are running MariaDB 10.2 or newer, you can tell poller-service to use a s ## Service Installation ### Upstart -An upstart configuration file can be found in `scripts/librenms-poller-service.conf`. To install run `ln -s /opt/librenms/scripts/librenms-poller-service.conf /etc/init/librenms-poller-service.conf`. The service will start on boot and can be started manually by running `start librenms-poller-service`. If you receive an error that the service does not exist, run `initctl reload-configuration`. The service is configured to run as the user `librenms` and will fail if that user does not exist. +An upstart configuration file can be found in `scripts/librenms-poller-service.conf`. To install run `cp /opt/librenms/scripts/librenms-poller-service.conf /etc/init/librenms-poller-service.conf`. The service will start on boot and can be started manually by running `start librenms-poller-service`. If you receive an error that the service does not exist, run `initctl reload-configuration`. The service is configured to run as the user `librenms` and will fail if that user does not exist. ### LSB -An LSB init script can be found in `scripts/librenms-poller-service.init`. To install run `ln -s /opt/librenms/scripts/librenms-poller-service.init /etc/init.d/librenms-poller-service && update-rc.d librenms-poller-service defaults`. +An LSB init script can be found in `scripts/librenms-poller-service.init`. To install run `cp /opt/librenms/scripts/librenms-poller-service.init /etc/init.d/librenms-poller-service && update-rc.d librenms-poller-service defaults`. ### systemd -A systemd unit file can be found in `scripts/librenms-poller-service.init`. To install run `ln -s /opt/librenms/scripts/librenms-poller-service.service /etc/systemd/system/librenms-poller-service.service && systemctl enable librenms-poller-service.service`. +A systemd unit file can be found in `scripts/librenms-poller-service.init`. To install run `cp /opt/librenms/scripts/librenms-poller-service.service /etc/systemd/system/librenms-poller-service.service && systemctl enable librenms-poller-service.service`. From 79eab54fd98fd93a89fd4b0202734109e4e73f9d Mon Sep 17 00:00:00 2001 From: Maximilian Wilhelm Date: Wed, 3 Feb 2016 20:40:05 +0100 Subject: [PATCH 12/29] Allow ignoring unmapable interfaces in poller. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When using the ifName or ifDescr as means to map discovered ports to known ports in the DB (think of port association mode) it's possible that we're facing the problem that the ifName or ifDescr polled from the device is unset or an empty string (like when querying some ubnt devices...). If this happends we have no way to map this port to any port found in the database. As reported this situation may occur for the time of one poll and might resolve automagically before the next poller run happens. Without this special case this would lead to new ports added to the database each time this situation occurs. To give the user the choice between »a lot of new ports« and »some poll runs are missed but ports stay stable« the 'ignore_unmapable_port' option has been added to configure this behaviour. To skip the port in this loop is sufficient as the next loop is looping only over ports found in the database and "maps back". As we did not add a new port to the DB here, there's no port to be mapped to. I'm using the in_array() check here, as I'm not sure if an "ifIndex" can be legally set to 0, which would yield True when checking if the value is empty(). Signed-off-by: Maximilian Wilhelm --- includes/defaults.inc.php | 3 +++ includes/polling/ports.inc.php | 25 +++++++++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/includes/defaults.inc.php b/includes/defaults.inc.php index 8dfa46c8d..bed0b577d 100644 --- a/includes/defaults.inc.php +++ b/includes/defaults.inc.php @@ -858,3 +858,6 @@ $config['update_channel'] = 'master'; // Default port association mode $config['default_port_association_mode'] = 'ifIndex'; +// Ignore ports which can't be mapped using a devices port_association_mode +// See include/polling/ports.inc.php for a lenghty explanation. +$config['ignore_unmapable_port'] = False; diff --git a/includes/polling/ports.inc.php b/includes/polling/ports.inc.php index 553069110..03bf4693d 100644 --- a/includes/polling/ports.inc.php +++ b/includes/polling/ports.inc.php @@ -238,6 +238,31 @@ foreach ($port_stats as $ifIndex => $port) { // Port newly discovered? if (! $ports[$port_id]) { + /** + * When using the ifName or ifDescr as means to map discovered ports to + * known ports in the DB (think of port association mode) it's possible + * that we're facing the problem that the ifName or ifDescr polled from + * the device is unset or an empty string (like when querying some ubnt + * devices...). If this happends we have no way to map this port to any + * port found in the database. As reported this situation may occur for + * the time of one poll and might resolve automagically before the next + * poller run happens. Without this special case this would lead to new + * ports added to the database each time this situation occurs. To give + * the user the choice between »a lot of new ports« and »some poll runs + * are missed but ports stay stable« the 'ignore_unmapable_port' option + * has been added to configure this behaviour. To skip the port in this + * loop is sufficient as the next loop is looping only over ports found + * in the database and "maps back". As we did not add a new port to the + * DB here, there's no port to be mapped to. + * + * I'm using the in_array() check here, as I'm not sure if an "ifIndex" + * can be legally set to 0, which would yield True when checking if the + * value is empty(). + */ + if ($config['ignore_unmapable_port'] === True and in_array ($port[$port_association_mode], array ('', Null))) { + continue; + } + $port_id = dbInsert(array('device_id' => $device['device_id'], 'ifIndex' => $ifIndex, 'ifName' => $ifName), 'ports'); dbInsert(array('port_id' => $port_id), 'ports_statistics'); $ports[$port_id] = dbFetchRow('SELECT * FROM `ports` WHERE `port_id` = ?', array($port_id)); From 5521002b379bb831080d4db1ceb2af971421d946 Mon Sep 17 00:00:00 2001 From: Maximilian Wilhelm Date: Wed, 3 Feb 2016 21:05:42 +0100 Subject: [PATCH 13/29] Fix port association for adsl, etherlike and poe port information. In 2c9df26bbf22af94c354bcf445b55a56eaece81a I broke the association of ports in the DB and ports just polled via SNMP for the adsl, etherlike and poe submodules and no one noticed. This fixes this issue. Sorry. Signed-off-by: Maximilian Wilhelm --- includes/polling/port-adsl.inc.php | 4 ++-- includes/polling/port-etherlike.inc.php | 6 +++--- includes/polling/port-poe.inc.php | 6 +++--- includes/polling/ports.inc.php | 17 ++++++++++++----- 4 files changed, 20 insertions(+), 13 deletions(-) diff --git a/includes/polling/port-adsl.inc.php b/includes/polling/port-adsl.inc.php index e3567e26b..901fb6d0b 100644 --- a/includes/polling/port-adsl.inc.php +++ b/includes/polling/port-adsl.inc.php @@ -39,9 +39,9 @@ // adslAturPerfESs.1 = 0 seconds // adslAturPerfValidIntervals.1 = 0 // adslAturPerfInvalidIntervals.1 = 0 -if (isset($port_stats[$port_id]['adslLineCoding'])) { +if (isset($port_stats[$ifIndex]['adslLineCoding'])) { // Check to make sure Port data is cached. - $this_port = &$port_stats[$port_id]; + $this_port = &$port_stats[$ifIndex]; $rrdfile = get_port_rrdfile_path ($device['hostname'], $port_id, 'adsl'); diff --git a/includes/polling/port-etherlike.inc.php b/includes/polling/port-etherlike.inc.php index bf947b92d..ea8b044c2 100644 --- a/includes/polling/port-etherlike.inc.php +++ b/includes/polling/port-etherlike.inc.php @@ -1,10 +1,10 @@ $port) { dbInsert(array('port_id' => $port_id), 'ports_statistics'); } - // Assure stable mapping - $port_stats[$ifIndex]['port_id'] = $port_id; + /** Assure stable bidirectional port mapping between DB and polled data + * + * Store the *current* ifIndex in the port info array containing all port information + * fetched from the database, as this is the only means we have to map ports_stats we + * just polled from the device to a port in $ports. All code below an includeed below + * will and has to map a port using it's ifIndex. + */ $ports[$port_id]['ifIndex'] = $ifIndex; + $port_stats[$ifIndex]['port_id'] = $port_id; } // Port vanished (mark as deleted) @@ -299,11 +305,12 @@ echo "\n"; // Loop ports in the DB and update where necessary foreach ($ports as $port) { $port_id = $port['port_id']; + $ifIndex = $port['ifIndex']; - echo 'Port ' . $port['ifName'] . ': ' . $port['ifDescr'] . '(' . $port['ifIndex'] . ') '; - if ($port_stats[$port['ifIndex']] && $port['disabled'] != '1') { + echo 'Port ' . $port['ifName'] . ': ' . $port['ifDescr'] . '(' . $ifIndex . ') '; + if ($port_stats[$ifIndex] && $port['disabled'] != '1') { // Check to make sure Port data is cached. - $this_port = &$port_stats[$port['ifIndex']]; + $this_port = &$port_stats[$ifIndex]; if ($device['os'] == 'vmware' && preg_match('/Device ([a-z0-9]+) at .*/', $this_port['ifDescr'], $matches)) { $this_port['ifDescr'] = $matches[1]; From ec5d0dfd89e23eced28ffe0cbecb400ab9e25eb5 Mon Sep 17 00:00:00 2001 From: Maximilian Wilhelm Date: Wed, 3 Feb 2016 21:12:46 +0100 Subject: [PATCH 14/29] Remove redundant code from port-{adsl,etherlike,poe} polling submodules. $this_port already gets defined within include/polling/ports.inc.php so there's no need to redefine it again in each submodule. Signed-off-by: Maximilian Wilhelm --- includes/polling/port-adsl.inc.php | 5 +---- includes/polling/port-etherlike.inc.php | 7 +------ includes/polling/port-poe.inc.php | 7 +------ 3 files changed, 3 insertions(+), 16 deletions(-) diff --git a/includes/polling/port-adsl.inc.php b/includes/polling/port-adsl.inc.php index 901fb6d0b..42b986428 100644 --- a/includes/polling/port-adsl.inc.php +++ b/includes/polling/port-adsl.inc.php @@ -39,10 +39,7 @@ // adslAturPerfESs.1 = 0 seconds // adslAturPerfValidIntervals.1 = 0 // adslAturPerfInvalidIntervals.1 = 0 -if (isset($port_stats[$ifIndex]['adslLineCoding'])) { - // Check to make sure Port data is cached. - $this_port = &$port_stats[$ifIndex]; - +if (isset($this_port['adslLineCoding'])) { $rrdfile = get_port_rrdfile_path ($device['hostname'], $port_id, 'adsl'); $rrd_create = ' --step 300'; diff --git a/includes/polling/port-etherlike.inc.php b/includes/polling/port-etherlike.inc.php index ea8b044c2..729990162 100644 --- a/includes/polling/port-etherlike.inc.php +++ b/includes/polling/port-etherlike.inc.php @@ -1,11 +1,6 @@ Date: Wed, 3 Feb 2016 22:48:18 +0100 Subject: [PATCH 15/29] Fix detection of deleted interfaces for ifName/ifDescr port association mode. The previous ifIndex based port mapping schema did detect deleted ports by checking ports in the DB against the ports currently polled via SNMP which breaks when using another port mapping schema. This commit fixes problem by checking known and found ports by their port_id. Signed-off-by: Maximilian Wilhelm --- includes/polling/ports.inc.php | 40 +++++++++++++++++++++++----------- 1 file changed, 27 insertions(+), 13 deletions(-) diff --git a/includes/polling/ports.inc.php b/includes/polling/ports.inc.php index 6a4606861..3bbcf5ff3 100644 --- a/includes/polling/ports.inc.php +++ b/includes/polling/ports.inc.php @@ -224,6 +224,7 @@ foreach ($ports_mapped['maps']['ifIndex'] as $ifIndex => $port_id) { } +$ports_found = array (); // New interface detection foreach ($port_stats as $ifIndex => $port) { // Store ifIndex in port entry and prefetch ifName as we'll need it multiple times @@ -289,6 +290,9 @@ foreach ($port_stats as $ifIndex => $port) { */ $ports[$port_id]['ifIndex'] = $ifIndex; $port_stats[$ifIndex]['port_id'] = $port_id; + + /* Build a list of all ports, identified by their port_id, found within this poller run. */ + $ports_found[] = $port_id; } // Port vanished (mark as deleted) @@ -307,8 +311,28 @@ foreach ($ports as $port) { $port_id = $port['port_id']; $ifIndex = $port['ifIndex']; - echo 'Port ' . $port['ifName'] . ': ' . $port['ifDescr'] . '(' . $ifIndex . ') '; - if ($port_stats[$ifIndex] && $port['disabled'] != '1') { + $port_info_string = 'Port ' . $port['ifName'] . ': ' . $port['ifDescr'] . " ($ifIndex / #$port_id) "; + + /* We don't care for disabled ports, go on */ + if ($port['disabled'] == 1) { + echo "$port_info_string disabled.\n"; + continue; + } + + /** + * If this port did not show up in $port_stats before it has been deleted + * since the last poller run. Mark it deleted in the database and go on. + */ + if (! in_array ($port_id, $ports_found)) { + if ($port['deleted'] != '1') { + dbUpdate(array('deleted' => '1'), 'ports', '`device_id` = ? AND `port_id` = ?', array($device['device_id'], $port_id)); + echo "$port_info_string deleted.\n"; + } + continue; + } + + echo $port_info_string; + if ($port_stats[$ifIndex]) { // Check to make sure Port data is cached. $this_port = &$port_stats[$ifIndex]; @@ -654,19 +678,8 @@ foreach ($ports as $port) { $updated += dbUpdate($port['update_extended'], 'ports_statistics', '`port_id` = ?', array($port_id)); d_echo("$updated updated"); } - // End Update Database } - else if ($port['disabled'] != '1') { - echo 'Port Deleted'; - // Port missing from SNMP cache. - if ($port['deleted'] != '1') { - dbUpdate(array('deleted' => '1'), 'ports', '`device_id` = ? AND `port_id` = ?', array($device['device_id'], $port_id)); - } - } - else { - echo 'Port Disabled.'; - }//end if echo "\n"; @@ -676,3 +689,4 @@ foreach ($ports as $port) { // Clear Variables Here unset($port_stats); +unset($ports_found); From 078c323d1a89a3da0f2e26a769cd605d53508f5e Mon Sep 17 00:00:00 2001 From: "Robert (KHobbits)" Date: Thu, 4 Feb 2016 00:21:20 +0000 Subject: [PATCH 16/29] Update validation docs --- doc/Support/Install Validation.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/Support/Install Validation.md b/doc/Support/Install Validation.md index ba90879de..2ed6a2ce0 100644 --- a/doc/Support/Install Validation.md +++ b/doc/Support/Install Validation.md @@ -17,7 +17,9 @@ So, to try and help with some of the general issues people come across we've put Optionally you can also pass -m and a module name for that to be tested. Current modules are: - - mail. This will validate your mail transport configuration. + - mail - This will validate your mail transport configuration. + - dist-poller - This will test your distributed poller configuration. + - rrdcheck - This will test your rrd files to see if they are unreadable or corrupted (source of broken graphs). Output, this is color coded to try and make things a little easier: From 4ff92481e46a2d13a06016e9445e260466f3c7c1 Mon Sep 17 00:00:00 2001 From: Aaron Daniels Date: Thu, 4 Feb 2016 16:34:56 +1000 Subject: [PATCH 17/29] - Quote the SNMP v2c community, v3 is already quoted. - Change the snmpver to double quotes for consistency. --- includes/snmp.inc.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/includes/snmp.inc.php b/includes/snmp.inc.php index 7f806d867..d2d485998 100644 --- a/includes/snmp.inc.php +++ b/includes/snmp.inc.php @@ -776,8 +776,8 @@ function snmp_gen_auth(&$device) { } } else if ($device['snmpver'] === 'v2c' or $device['snmpver'] === 'v1') { - $cmd = ' -'.$device['snmpver']; - $cmd .= ' -c '.$device['community']; + $cmd = " -".$device['snmpver']; + $cmd .= " -c '".$device['community']."'"; } else { if ($debug) { From e397860257fad8dc79405c4315129e9a63fc8c7b Mon Sep 17 00:00:00 2001 From: pblasquez Date: Thu, 4 Feb 2016 18:09:09 -0800 Subject: [PATCH 18/29] Update ajax_dash.php --- html/ajax_dash.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/html/ajax_dash.php b/html/ajax_dash.php index d2e2b0a62..0ffc58067 100644 --- a/html/ajax_dash.php +++ b/html/ajax_dash.php @@ -55,5 +55,5 @@ $response = array( 'html' => $output, 'title' => $title, ); - +header('Content-type: application/json'); echo _json_encode($response); From ab6a608bdb29d711dc1e1ef398d7bb6ac9047524 Mon Sep 17 00:00:00 2001 From: pblasquez Date: Thu, 4 Feb 2016 18:09:47 -0800 Subject: [PATCH 19/29] Update ajax_table.php --- html/ajax_table.php | 1 + 1 file changed, 1 insertion(+) diff --git a/html/ajax_table.php b/html/ajax_table.php index f16cc2da9..c3513a68a 100644 --- a/html/ajax_table.php +++ b/html/ajax_table.php @@ -36,6 +36,7 @@ $response = array(); if (isset($id)) { if (file_exists("includes/table/$id.inc.php")) { + header('Content-type: application/json'); include_once "includes/table/$id.inc.php"; } } From d62ad2eb029dfc095fb0f443ad1e33f561104f23 Mon Sep 17 00:00:00 2001 From: pblasquez Date: Thu, 4 Feb 2016 18:16:36 -0800 Subject: [PATCH 20/29] Update ajax_search.php --- html/ajax_search.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/html/ajax_search.php b/html/ajax_search.php index e9ddad36f..cb8d012a3 100644 --- a/html/ajax_search.php +++ b/html/ajax_search.php @@ -19,7 +19,7 @@ $bgp = array(); if (isset($_REQUEST['search'])) { $search = mres($_REQUEST['search']); - + header('Content-type: application/json'); if (strlen($search) > 0) { $found = 0; From 1b4dfd2fc6d7db200c55c66ce0585fb3c5a6d59d Mon Sep 17 00:00:00 2001 From: pblasquez Date: Thu, 4 Feb 2016 18:18:40 -0800 Subject: [PATCH 21/29] Update ajax_rulesuggest.php --- html/ajax_rulesuggest.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/html/ajax_rulesuggest.php b/html/ajax_rulesuggest.php index cee773a2f..850658939 100644 --- a/html/ajax_rulesuggest.php +++ b/html/ajax_rulesuggest.php @@ -62,7 +62,7 @@ function levsort($base, $obj) { } - +header('Content-type: application/json'); $obj = array(array('name' => 'Error: No suggestions found.')); $term = array(); $current = false; From e8f773b0101fddc7f8985edad62d3d3cda1342da Mon Sep 17 00:00:00 2001 From: pblasquez Date: Thu, 4 Feb 2016 18:23:49 -0800 Subject: [PATCH 22/29] Update ajax_form.php --- html/ajax_form.php | 1 + 1 file changed, 1 insertion(+) diff --git a/html/ajax_form.php b/html/ajax_form.php index 999a43a4d..3851d5b4f 100644 --- a/html/ajax_form.php +++ b/html/ajax_form.php @@ -29,6 +29,7 @@ if (!$_SESSION['authenticated']) { if (preg_match('/^[a-zA-Z0-9\-]+$/', $_POST['type']) == 1) { if (file_exists('includes/forms/'.$_POST['type'].'.inc.php')) { + header('Content-type: application/json'); include_once 'includes/forms/'.$_POST['type'].'.inc.php'; } } From 6ef1cf9e43a3c7a50f6831fb51a73e844321eccc Mon Sep 17 00:00:00 2001 From: Maximilian Wilhelm Date: Fri, 5 Feb 2016 11:40:37 +0100 Subject: [PATCH 23/29] Add ability to map ports based on their ifAlias. Signed-off-by: Maximilian Wilhelm --- includes/common.php | 2 +- sql-schema/099.sql | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 sql-schema/099.sql diff --git a/includes/common.php b/includes/common.php index d74a512a9..943feb930 100644 --- a/includes/common.php +++ b/includes/common.php @@ -1248,7 +1248,7 @@ function get_port_id ($ports_mapped, $port, $port_association_mode) { */ $maps = $ports_mapped['maps']; - if (in_array ($port_association_mode, array ('ifIndex', 'ifName', 'ifDescr'))) { + if (in_array ($port_association_mode, array ('ifIndex', 'ifName', 'ifDescr', 'ifAlias'))) { $port_id = $maps[$port_association_mode][$port[$port_association_mode]]; } diff --git a/sql-schema/099.sql b/sql-schema/099.sql new file mode 100644 index 000000000..646e4c4e1 --- /dev/null +++ b/sql-schema/099.sql @@ -0,0 +1 @@ +INSERT INTO port_association_mode (name) values ('ifAlias'); From be225148ca9f4a48e6f07b4c8857c8652b8591dd Mon Sep 17 00:00:00 2001 From: Mark Schouten Date: Fri, 5 Feb 2016 14:50:27 +0100 Subject: [PATCH 24/29] There was one more include that screwed up the Ceph graphs --- html/pages/device/apps/ceph.inc.php | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/html/pages/device/apps/ceph.inc.php b/html/pages/device/apps/ceph.inc.php index 50dedc8c4..9ad753651 100644 --- a/html/pages/device/apps/ceph.inc.php +++ b/html/pages/device/apps/ceph.inc.php @@ -17,9 +17,9 @@ foreach ($graphs as $key => $text) { if ($key == "ceph_poolstats") { foreach (glob($rrddir."/app-ceph-".$app['app_id']."-pool-*") as $rrd_filename) { - $graph_array['to'] = $config['time']['now']; - $graph_array['id'] = $app['app_id']; if (preg_match("/.*-pool-(.+)\.rrd$/", $rrd_filename, $pools)) { + $graph_array['to'] = $config['time']['now']; + $graph_array['id'] = $app['app_id']; $pool = $pools[1]; echo '

'.$pool.' Reads/Writes

'; $graph_array['type'] = 'application_ceph_pool_io'; @@ -29,6 +29,8 @@ foreach ($graphs as $key => $text) { include 'includes/print-graphrow.inc.php'; echo ''; + $graph_array['to'] = $config['time']['now']; + $graph_array['id'] = $app['app_id']; echo '

'.$pool.' IOPS

'; $graph_array['type'] = 'application_ceph_pool_iops'; $graph_array['pool'] = $pool; From 6e35b97ea9c4fa56a660fee0502fc959ff880aa2 Mon Sep 17 00:00:00 2001 From: Joe Passavanti Date: Sun, 7 Feb 2016 05:08:37 -0800 Subject: [PATCH 25/29] add Dell Networking sysObjectId This is for Dell Networking N2048 switch --- includes/discovery/os/dnos.inc.php | 3 +++ 1 file changed, 3 insertions(+) diff --git a/includes/discovery/os/dnos.inc.php b/includes/discovery/os/dnos.inc.php index b83c92a91..a1e4ac6e8 100644 --- a/includes/discovery/os/dnos.inc.php +++ b/includes/discovery/os/dnos.inc.php @@ -7,4 +7,7 @@ if (!$os) { if (strstr($sysObjectId, '.1.3.6.1.4.1.674.10895.3042')) { $os = 'dnos'; } + if (strstr($sysObjectId, '.1.3.6.1.4.1.674.10895.3054')) { + $os = 'dnos'; + } } From ff91e97b169ef9af6b0ee537a9b32188c291a5a7 Mon Sep 17 00:00:00 2001 From: Daniel Preussker Date: Sun, 7 Feb 2016 15:26:16 +0000 Subject: [PATCH 26/29] Updated changelog --- doc/General/Changelog.md | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/doc/General/Changelog.md b/doc/General/Changelog.md index 54a97dacc..98f4c0207 100644 --- a/doc/General/Changelog.md +++ b/doc/General/Changelog.md @@ -1,3 +1,34 @@ +### February 2016 + +#### Bug fixes + - Discovery / Polling: + - Quote snmp v2c community (PR2927) + - For entity-sensor, changed variable name again (PR2948) + - Fix some issues with/introduced by port association mode configuration (PR2923) + - WebUI: + - Fix ceph graps (PR2909, PR2942) + - BGP Overlib (PR2915) + - Added `application/json` headers where json is returned (PR2936) + - Stop realtime graph page from auto refreshing (PR2939) + - Updated parsing of alert rules to allow `|` (PR2917) + - Misc: + - Updated `device_by_id_cache()` to convert IP column (PR2940) + - Documentation: + - Removed devloping doc as none of the info is current (PR2911) + +#### Improvements + - Discovery / Polling: + - Added ability to ignore device sensors from entity mib (PR2862) + - Added `ifOperStatus_prev` and `ifAdminStatus_prev` values to db (PR2912) + - Added detection for: + - Dell Networking N2048 (PR2949) + - Misc: + - Added check for rrd vadility (PR2908) + - Add systemd unit file for the python poller service (PR2913) + - Documentation: + - Added description of AD configuration options (PR2910) + - Add description to mibbases polling (PR2919) + ### January 2016 #### Bug fixes From f940c0c888192852874f311bb8782813b912fdcb Mon Sep 17 00:00:00 2001 From: Daniel Preussker Date: Sun, 7 Feb 2016 15:39:17 +0000 Subject: [PATCH 27/29] Fix IP Display --- html/includes/dev-overview-data.inc.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/html/includes/dev-overview-data.inc.php b/html/includes/dev-overview-data.inc.php index 2e7ed141e..856b50086 100644 --- a/html/includes/dev-overview-data.inc.php +++ b/html/includes/dev-overview-data.inc.php @@ -30,7 +30,7 @@ echo ' '.$device['sysName'].' '; -if ($ip = inet6_ntop($device['ip'])) { +if (!empty($device['ip'])) { echo ' Resolved IP '.$ip.' From b0bc7b778ae07738a8b833e9a9215f24e277455b Mon Sep 17 00:00:00 2001 From: Daniel Preussker Date: Sun, 7 Feb 2016 15:40:00 +0000 Subject: [PATCH 28/29] Typo --- html/includes/dev-overview-data.inc.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/html/includes/dev-overview-data.inc.php b/html/includes/dev-overview-data.inc.php index 856b50086..04c4de3dd 100644 --- a/html/includes/dev-overview-data.inc.php +++ b/html/includes/dev-overview-data.inc.php @@ -33,7 +33,7 @@ echo ' if (!empty($device['ip'])) { echo ' Resolved IP - '.$ip.' + '.$device['ip'].' '; } From 9c0cc8ad0cf3230f88c2d27c22bb5ac9e60a5721 Mon Sep 17 00:00:00 2001 From: Daniel Preussker Date: Sun, 7 Feb 2016 15:47:14 +0000 Subject: [PATCH 29/29] Updated#2 --- doc/General/Changelog.md | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/General/Changelog.md b/doc/General/Changelog.md index 98f4c0207..a60beb1aa 100644 --- a/doc/General/Changelog.md +++ b/doc/General/Changelog.md @@ -11,6 +11,7 @@ - Added `application/json` headers where json is returned (PR2936) - Stop realtime graph page from auto refreshing (PR2939) - Updated parsing of alert rules to allow `|` (PR2917) + - Fix IP Display (PR2951) - Misc: - Updated `device_by_id_cache()` to convert IP column (PR2940) - Documentation: