From d1f44ae1ec7096c4b06fef21b870f5de784b3ee6 Mon Sep 17 00:00:00 2001 From: Tony Murray Date: Fri, 26 Feb 2016 10:19:22 -0600 Subject: [PATCH 1/7] Simplify Cisco ios,iosxe,catos log processing. This should cover the examples I could find on the web, but we could be missing some corner cases. If we fail, the entire message is in msg, instead of being lost. --- includes/syslog.php | 38 ++++++++------------------------------ 1 file changed, 8 insertions(+), 30 deletions(-) diff --git a/includes/syslog.php b/includes/syslog.php index 4cecba55fa..7952a50ab1 100644 --- a/includes/syslog.php +++ b/includes/syslog.php @@ -53,38 +53,16 @@ function process_syslog($entry, $update) { if (in_array($os, array('ios', 'iosxe', 'catos'))) { $matches = array(); - // if (preg_match('#%(?P.*):( ?)(?P.*)#', $entry['msg'], $matches)) { - // $entry['msg'] = $matches['msg']; - // $entry['program'] = $matches['program']; - // } - // unset($matches); - if (strstr($entry['msg'], '%')) { - $entry['msg'] = preg_replace('/^%(.+?):\ /', '\\1||', $entry['msg']); - list(,$entry['msg']) = explode(': %', $entry['msg']); - $entry['msg'] = '%'.$entry['msg']; - $entry['msg'] = preg_replace('/^%(.+?):\ /', '\\1||', $entry['msg']); - } - else { - $entry['msg'] = preg_replace('/^.*[0-9]:/', '', $entry['msg']); - $entry['msg'] = preg_replace('/^[0-9][0-9]\ [A-Z]{3}:/', '', $entry['msg']); - $entry['msg'] = preg_replace('/^(.+?):\ /', '\\1||', $entry['msg']); + /* Split the following examples, or fallback to leaving everything in msg + * %LINK-3-UPDOWN: Interface GigabitEthernet0/2, changed state to up + * %SYS-5-MOD_OK:Module 1 is online + */ + if (preg_match('/^%([A-Z\d\-_]+): ?(.+)/', $entry['msg'], $matches)) { + $entry['program'] = $matches[1]; + $entry['msg'] = $matches[2]; } - $entry['msg'] = preg_replace('/^.+\.[0-9]{3}:/', '', $entry['msg']); - $entry['msg'] = preg_replace('/^.+-Traceback=/', 'Traceback||', $entry['msg']); - - list($entry['program'], $entry['msg']) = explode('||', $entry['msg']); - $entry['msg'] = preg_replace('/^[0-9]+:/', '', $entry['msg']); - - if (!$entry['program']) { - $entry['msg'] = preg_replace('/^([0-9A-Z\-]+?):\ /', '\\1||', $entry['msg']); - list($entry['program'], $entry['msg']) = explode('||', $entry['msg']); - } - - if (!$entry['msg']) { - $entry['msg'] = $entry['program']; - unset($entry['program']); - } + unset($matches); } else if ($os == 'linux' and get_cache($entry['host'], 'version') == 'Point') { // Cisco WAP200 and similar From 71d88d6975374a3e23183c1e9de3c074f0920be2 Mon Sep 17 00:00:00 2001 From: Tony Murray Date: Sat, 27 Feb 2016 16:00:06 -0600 Subject: [PATCH 2/7] This should process all of the messages on these sites and if it fails, there should be no data loss http://www.cisco.com/c/en/us/support/docs/switches/catalyst-6000-series-switches/29804-186.html http://www.cisco.com/c/en/us/td/docs/ios/system/messages/guide/consol_smg/sm_cnovr.html --- includes/syslog.php | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/includes/syslog.php b/includes/syslog.php index 7952a50ab1..8d6a902c43 100644 --- a/includes/syslog.php +++ b/includes/syslog.php @@ -52,17 +52,26 @@ function process_syslog($entry, $update) { $os = get_cache($entry['host'], 'os'); if (in_array($os, array('ios', 'iosxe', 'catos'))) { - $matches = array(); - /* Split the following examples, or fallback to leaving everything in msg - * %LINK-3-UPDOWN: Interface GigabitEthernet0/2, changed state to up - * %SYS-5-MOD_OK:Module 1 is online - */ - if (preg_match('/^%([A-Z\d\-_]+): ?(.+)/', $entry['msg'], $matches)) { - $entry['program'] = $matches[1]; - $entry['msg'] = $matches[2]; + // multipart message + if(strpos($entry['msg'], ':') !== false) { + /* Split the following examples + * %CARD-SEVERITY-MSG:SLOT %FACILITY-SEVERITY-MNEMONIC: Message-text + * %FACILITY-SUBFACILITY-SEVERITY-MNEMONIC: Message-text + */ + $matches = array(); + if(preg_match('/^%?(?[A-Za-z\d\-_]+(:[A-Z]* %[A-Z\d\-_]+)?): ?(?.*)/', $entry['msg'], $matches)) { + $entry['program'] = $matches['program']; + $entry['msg'] = $matches['msg']; + } + unset($matches); + } + else { + // if this looks like a program (no groups of 2 or more lowercase letters), move it to program + if (preg_match('/[^(a-z)]{2,}/', $entry['msg'])) { + $entry['program'] = $entry['msg']; + unset($entry['msg']); + } } - - unset($matches); } else if ($os == 'linux' and get_cache($entry['host'], 'version') == 'Point') { // Cisco WAP200 and similar From ea491c4a068c9addb71b15b8b7f7f264776fcf42 Mon Sep 17 00:00:00 2001 From: Tony Murray Date: Wed, 2 Mar 2016 23:48:07 -0600 Subject: [PATCH 3/7] move the negation outside of the regular expression for more accurate results --- includes/syslog.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/includes/syslog.php b/includes/syslog.php index 8d6a902c43..0c8ade0cd4 100644 --- a/includes/syslog.php +++ b/includes/syslog.php @@ -67,7 +67,7 @@ function process_syslog($entry, $update) { } else { // if this looks like a program (no groups of 2 or more lowercase letters), move it to program - if (preg_match('/[^(a-z)]{2,}/', $entry['msg'])) { + if (!preg_match('/[(a-z)]{2,}/', $entry['msg'])) { $entry['program'] = $entry['msg']; unset($entry['msg']); } From f52954f8a4d1ea5f5b1ed9887181ce24056867c0 Mon Sep 17 00:00:00 2001 From: Tony Murray Date: Thu, 3 Mar 2016 00:03:17 -0600 Subject: [PATCH 4/7] The old load included the leading %. Stick with that behavior. --- includes/syslog.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/includes/syslog.php b/includes/syslog.php index 0c8ade0cd4..a84a79a499 100644 --- a/includes/syslog.php +++ b/includes/syslog.php @@ -59,7 +59,7 @@ function process_syslog($entry, $update) { * %FACILITY-SUBFACILITY-SEVERITY-MNEMONIC: Message-text */ $matches = array(); - if(preg_match('/^%?(?[A-Za-z\d\-_]+(:[A-Z]* %[A-Z\d\-_]+)?): ?(?.*)/', $entry['msg'], $matches)) { + if(preg_match('/^(?%?[A-Za-z\d\-_]+(:[A-Z]* %[A-Z\d\-_]+)?): ?(?.*)/', $entry['msg'], $matches)) { $entry['program'] = $matches['program']; $entry['msg'] = $matches['msg']; } From 734d1bcd9d7a6c4b69f8955763c4a9c504e3a7a0 Mon Sep 17 00:00:00 2001 From: Tony Murray Date: Thu, 3 Mar 2016 00:06:53 -0600 Subject: [PATCH 5/7] PHPunit tests for Cisco syslog parsing. --- tests/SyslogTest.php | 90 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 tests/SyslogTest.php diff --git a/tests/SyslogTest.php b/tests/SyslogTest.php new file mode 100644 index 0000000000..e6e2d5b753 --- /dev/null +++ b/tests/SyslogTest.php @@ -0,0 +1,90 @@ + +fillLine($line); + $data = array(); + $data['input'] = $entry; + unset($entry['msg']); // empty msg + $data['result'] = array_merge($entry, $resultDelta); + return $data; + } + + public function testCiscoSyslog() + { + // populate fake $dev_cache and $config + global $config, $dev_cache; + $dev_cache['1.1.1.1'] = ['device_id' => 1, 'os' => 'ios', 'version' => 1];; + $confg = array(); + $config['syslog_filter'] = array(); + + // populate test data + $testdata = array(); + + // ---- IOS ---- + $testdata[] = $this->createData( + "1.1.1.1||user||info||info||0e||2016-02-28 00:23:34||%CARD-SEVERITY-MSG:SLOT %FACILITY-SEVERITY-MNEMONIC: Message-text||", + ['device_id'=>1, 'program'=>'%CARD-SEVERITY-MSG:SLOT %FACILITY-SEVERITY-MNEMONIC', 'msg'=>'Message-text'] + ); + $testdata[] = $this->createData( + "1.1.1.1||user||info||info||0e||2016-02-28 00:23:34||%FACILITY-SUBFACILITY-SEVERITY-MNEMONIC: Message-text||", + ['device_id'=>1, 'program'=>'%FACILITY-SUBFACILITY-SEVERITY-MNEMONIC', 'msg'=>'Message-text'] + ); + + // ---- CatOS ---- + $testdata[] = $this->createData( + "1.1.1.1||user||info||info||0e||2016-02-28 00:23:34||%IP-3-UDP_SOCKOVFL:UDP socket overflow||", + ['device_id'=>1, 'program'=>'%IP-3-UDP_SOCKOVFL', 'msg'=>'UDP socket overflow'] + ); + $testdata[] = $this->createData( + "1.1.1.1||user||info||info||0e||2016-02-28 00:23:34||DTP-1-ILGLCFG: Illegal config (on, isl--on,dot1q) on Port [mod/port]||", + ['device_id'=>1, 'program'=>'DTP-1-ILGLCFG', 'msg'=>'Illegal config (on, isl--on,dot1q) on Port [mod/port]'] + ); + $testdata[] = $this->createData( + "1.1.1.1||user||info||info||0e||2016-02-28 00:23:34||Cannot enable text mode config if ACL config is cleared from nvram||", + ['device_id'=>1, 'program'=>'', 'msg'=>'Cannot enable text mode config if ACL config is cleared from nvram'] + ); + $testdata[] = $this->createData( + "1.1.1.1||user||info||info||0e||2016-02-28 00:23:34||%PAGP-5-PORTFROMSTP / %PAGP-5-PORTTOSTP||", + ['device_id'=>1, 'program'=>'%PAGP-5-PORTFROMSTP / %PAGP-5-PORTTOSTP'] + ); + $testdata[] = $this->createData( + "1.1.1.1||user||info||info||0e||2016-02-28 00:23:34||%SYS-3-EOBC_CHANNELREINIT||", + ['device_id'=>1, 'program'=>'%SYS-3-EOBC_CHANNELREINIT'] + ); + $testdata[] = $this->createData( + "1.1.1.1||user||info||info||0e||2016-02-28 00:23:34||%SYS-4-MODHPRESET:||", + ['device_id'=>1, 'program'=>'%SYS-4-MODHPRESET', 'msg'=>''] + ); + $testdata[] = $this->createData( + "1.1.1.1||user||info||info||0e||2016-02-28 00:23:34||InbandPingProcessFailure:Module x not responding over inband||", + ['device_id'=>1, 'program'=>'INBANDPINGPROCESSFAILURE', 'msg'=>'Module x not responding over inband'] + ); + $testdata[] = $this->createData( + "1.1.1.1||user||info||info||0e||2016-02-28 00:23:34||RxSBIF_SEQ_NUM_ERROR:slot=x||", + ['device_id'=>1, 'program'=>'RXSBIF_SEQ_NUM_ERROR', 'msg'=>'slot=x'] + ); + + + // run tests + foreach($testdata as $data) { + $res = process_syslog($data['input'], 0); + $this->assertEquals($data['result'], $res); + } + } +} + From 2af1b517e527f91e1e908cc80abcf6e24f5234cb Mon Sep 17 00:00:00 2001 From: Tony Murray Date: Thu, 3 Mar 2016 06:46:39 -0600 Subject: [PATCH 6/7] Notifcation and tab->spaces fix. --- misc/notifications.rss | 5 +++++ tests/SyslogTest.php | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/misc/notifications.rss b/misc/notifications.rss index f4baa96ca4..5d094945b1 100644 --- a/misc/notifications.rss +++ b/misc/notifications.rss @@ -12,5 +12,10 @@ This is the first notification. We will post these whenever an upcoming major change is about to happen. Tue, 19 Jan 2016 12:00:00 +0000 + + Cisco syslog parsing changes + We have overhauled the Cisco syslog parsing. Please monitor your syslog entries from Cisco IOS, IOSXR, and CatOS devices. If you notice any issues please open a new issue on GitHub and include the original syslog message. + Thu, 03 Mar 2016 12:00:00 +0000 + diff --git a/tests/SyslogTest.php b/tests/SyslogTest.php index e6e2d5b753..02d6e22dab 100644 --- a/tests/SyslogTest.php +++ b/tests/SyslogTest.php @@ -27,7 +27,7 @@ class SyslogTest extends \PHPUnit_Framework_TestCase public function testCiscoSyslog() { // populate fake $dev_cache and $config - global $config, $dev_cache; + global $config, $dev_cache; $dev_cache['1.1.1.1'] = ['device_id' => 1, 'os' => 'ios', 'version' => 1];; $confg = array(); $config['syslog_filter'] = array(); From 985ea9bea684f09da0ec5f5ef7d002e9d7f05b49 Mon Sep 17 00:00:00 2001 From: Tony Murray Date: Thu, 3 Mar 2016 11:56:29 -0600 Subject: [PATCH 7/7] Enable phpunit testing in Travis CI. --- .gitignore | 2 +- .travis.yml | 11 +++++++++++ phpunit.xml | 8 ++++++++ 3 files changed, 20 insertions(+), 1 deletion(-) create mode 100644 .travis.yml create mode 100644 phpunit.xml diff --git a/.gitignore b/.gitignore index 0a6dbb3cb2..28e3f36ab6 100644 --- a/.gitignore +++ b/.gitignore @@ -4,7 +4,7 @@ !/.gitignore !/.editorconfig !/.scrutinizer.yml - +!/.travis.yml # Others # ########## diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000000..1ab38ae3fb --- /dev/null +++ b/.travis.yml @@ -0,0 +1,11 @@ +language: php php: + - '5.3' + - '5.4' + - '5.5' + - '5.6' + - '7.0' + - hhvm + +matrix: + allow_failures: + - php: 7.0 diff --git a/phpunit.xml b/phpunit.xml new file mode 100644 index 0000000000..26bc37ed3b --- /dev/null +++ b/phpunit.xml @@ -0,0 +1,8 @@ + + + + + ./tests/ + + +