diff --git a/LibreNMS/Poller.php b/LibreNMS/Poller.php deleted file mode 100644 index 516d1971ff..0000000000 --- a/LibreNMS/Poller.php +++ /dev/null @@ -1,322 +0,0 @@ -. - * - * @link https://www.librenms.org - * - * @copyright 2021 Tony Murray - * @author Tony Murray - */ - -namespace LibreNMS; - -use App\Events\DevicePolled; -use App\Events\PollingDevice; -use App\Models\Device; -use App\Models\Eventlog; -use App\Polling\Measure\Measurement; -use App\Polling\Measure\MeasurementManager; -use Carbon\Carbon; -use Illuminate\Support\Str; -use LibreNMS\Enum\Severity; -use LibreNMS\Polling\ConnectivityHelper; -use LibreNMS\Polling\Result; -use LibreNMS\RRD\RrdDefinition; -use LibreNMS\Util\Debug; -use LibreNMS\Util\Dns; -use LibreNMS\Util\Module; -use LibreNMS\Util\Version; -use Psr\Log\LoggerInterface; -use Throwable; - -class Poller -{ - /** @var string */ - private $device_spec; - /** @var array */ - private $module_override; - - /** - * @var Device - */ - private $device; - /** - * @var array - */ - private $deviceArray; - /** - * @var \LibreNMS\OS|\LibreNMS\OS\Generic - */ - private $os; - /** - * @var LoggerInterface - */ - private $logger; - - public function __construct(string $device_spec, array $module_override, LoggerInterface $logger) - { - $this->device_spec = $device_spec; - $this->module_override = $module_override; - $this->logger = $logger; - $this->parseModules(); - } - - public function poll(): Result - { - $results = new Result; - $this->printHeader(); - - if (Debug::isEnabled() && ! defined('PHPUNIT_RUNNING')) { - \LibreNMS\Util\OS::updateCache(true); // Force update of OS Cache - } - - $this->logger->info("Starting polling run:\n"); - - foreach (Device::whereDeviceSpec($this->device_spec)->pluck('device_id') as $device_id) { - $results->markAttempted(); - $this->initDevice($device_id); - PollingDevice::dispatch($this->device); - $this->os = OS::make($this->deviceArray); - - $measurement = Measurement::start('poll'); - $measurement->manager()->checkpoint(); // don't count previous stats - - $helper = new ConnectivityHelper($this->device); - $helper->saveMetrics(); - $helper->isUp(); // check and save status - - $this->pollModules(); - - $measurement->end(); - - // if modules are not overridden, record performance - if (empty($this->module_override)) { - if ($this->device->status) { - $this->recordPerformance($measurement); - } - - if ($helper->canPing()) { - $this->os->enableGraph('ping_perf'); - } - - $this->os->persistGraphs($this->device->status); // save graphs but don't delete any if device is down - $this->logger->info(sprintf("Enabled graphs (%s): %s\n\n", - $this->device->graphs->count(), - $this->device->graphs->pluck('graph')->implode(' ') - )); - } - - // finalize the device poll - $this->device->save(); - $results->markCompleted($this->device->status); - DevicePolled::dispatch($this->device); - - $this->logger->info(sprintf("\n>>> Polled %s (%s) in %0.3f seconds <<<", - $this->device->displayName(), - $this->device->device_id, - $measurement->getDuration())); - \Log::channel('single')->alert(sprintf('INFO: device:poll %s (%s) polled in %0.3fs', - $this->device->hostname, - $this->device->device_id, - $measurement->getDuration())); - - // check if the poll took too long and log an event - if ($measurement->getDuration() > Config::get('rrd.step')) { - Eventlog::log('Polling took longer than ' . round(Config::get('rrd.step') / 60, 2) . - ' minutes! This will cause gaps in graphs.', $this->device, 'system', Severity::Error); - } - } - - return $results; - } - - private function pollModules(): void - { - // update $device array status - $this->deviceArray['status'] = $this->device->status; - $this->deviceArray['status_reason'] = $this->device->status_reason; - - // import legacy garbage - include_once base_path('includes/functions.php'); - include_once base_path('includes/common.php'); - include_once base_path('includes/polling/functions.inc.php'); - include_once base_path('includes/snmp.inc.php'); - - $datastore = app('Datastore'); - - foreach (array_keys(Config::get('poller_modules')) as $module) { - $module_status = Module::pollingStatus($module, $this->device, $this->isModuleManuallyEnabled($module)); - $should_poll = false; - $start_memory = memory_get_usage(); - $module_start = microtime(true); - - try { - $instance = Module::fromName($module); - $should_poll = $instance->shouldPoll($this->os, $module_status); - - if ($should_poll) { - $this->logger->info("#### Load poller module $module ####\n"); - $this->logger->debug($module_status); - - $instance->poll($this->os, $datastore); - } - } catch (Throwable $e) { - // isolate module exceptions so they don't disrupt the polling process - $this->logger->error("%rError polling $module module for {$this->device->hostname}.%n $e", ['color' => true]); - Eventlog::log("Error polling $module module. Check log file for more details.", $this->device, 'poller', Severity::Error); - report($e); - } - - if ($should_poll) { - $this->logger->info(''); - app(MeasurementManager::class)->printChangedStats(); - $this->saveModulePerformance($module, $module_start, $start_memory); - $this->logger->info("#### Unload poller module $module ####\n"); - } - } - } - - private function saveModulePerformance(string $module, float $start_time, int $start_memory): void - { - $module_time = microtime(true) - $start_time; - $module_mem = (memory_get_usage() - $start_memory); - - $this->logger->info(sprintf(">> Runtime for poller module '%s': %.4f seconds with %s bytes", $module, $module_time, $module_mem)); - - app('Datastore')->put($this->deviceArray, 'poller-perf', [ - 'module' => $module, - 'rrd_def' => RrdDefinition::make()->addDataset('poller', 'GAUGE', 0), - 'rrd_name' => ['poller-perf', $module], - ], [ - 'poller' => $module_time, - ]); - $this->os->enableGraph('poller_modules_perf'); - } - - private function isModuleManuallyEnabled(string $module): ?bool - { - if (empty($this->module_override)) { - return null; - } - - foreach ($this->module_override as $override) { - [$override_module] = explode('/', $override); - if ($module == $override_module) { - return true; - } - } - - return false; - } - - private function initDevice(int $device_id): void - { - \DeviceCache::setPrimary($device_id); - $this->device = \DeviceCache::getPrimary(); - $this->device->ip = $this->device->overwrite_ip ?: Dns::lookupIp($this->device) ?: $this->device->ip; - - $this->deviceArray = $this->device->toArray(); - if ($os_group = Config::get("os.{$this->device->os}.group")) { - $this->deviceArray['os_group'] = $os_group; - } - - $this->printDeviceInfo($os_group); - $this->initRrdDirectory(); - } - - private function initRrdDirectory(): void - { - $host_rrd = \Rrd::name($this->device->hostname, '', ''); - if (Config::get('rrd.enable', true) && ! is_dir($host_rrd)) { - try { - mkdir($host_rrd); - $this->logger->info("Created directory : $host_rrd"); - } catch (\ErrorException $e) { - Eventlog::log("Failed to create rrd directory: $host_rrd", $this->device); - $this->logger->error($e); - } - } - } - - private function parseModules(): void - { - foreach ($this->module_override as $index => $module) { - // parse submodules (only supported by some modules) - if (Str::contains($module, '/')) { - [$module, $submodule] = explode('/', $module, 2); - $existing_submodules = Config::get("poller_submodules.$module", []); - $existing_submodules[] = $submodule; - Config::set("poller_submodules.$module", $existing_submodules); - } - - if (! Module::exists($module) && ! Module::legacyPollingExists($module)) { - unset($this->module_override[$index]); - continue; - } - - Config::set("poller_modules.$module", 1); - } - - $this->printModules(); - } - - private function printDeviceInfo(?string $group): void - { - $this->logger->info(sprintf(<<<'EOH' -Hostname: %s %s -ID: %s -OS: %s -IP: %s - -EOH, $this->device->hostname, $group ? " ($group)" : '', $this->device->device_id, $this->device->os, $this->device->ip)); - } - - private function printModules(): void - { - $modules = array_map(function ($module) { - $submodules = Config::get("poller_submodules.$module"); - - return $module . ($submodules ? '(' . implode(',', $submodules) . ')' : ''); - }, array_keys(Config::get('poller_modules', []))); - - $this->logger->debug('Override poller modules: ' . implode(', ', $modules)); - } - - private function printHeader(): void - { - if (Debug::isEnabled() || Debug::isVerbose()) { - $this->logger->info(Version::get()->header()); - } - } - - private function recordPerformance(Measurement $measurement): void - { - $measurement->manager()->record('device', $measurement); - $this->device->last_polled = Carbon::now(); - $this->device->last_polled_timetaken = $measurement->getDuration(); - - app('Datastore')->put($this->deviceArray, 'poller-perf', [ - 'rrd_def' => RrdDefinition::make()->addDataset('poller', 'GAUGE', 0), - 'module' => 'ALL', - ], [ - 'poller' => $this->device->last_polled_timetaken, - ]); - - $this->os->enableGraph('poller_perf'); - } -} diff --git a/LibreNMS/Util/Module.php b/LibreNMS/Util/Module.php index 0aabf6497d..99bed6ec40 100644 --- a/LibreNMS/Util/Module.php +++ b/LibreNMS/Util/Module.php @@ -34,7 +34,11 @@ class Module { public static function exists(string $module_name): bool { - return class_exists(StringHelpers::toClass($module_name, '\\LibreNMS\\Modules\\')); + if (class_exists(StringHelpers::toClass($module_name, '\\LibreNMS\\Modules\\'))) { + return true; + } + + return Config::has('discovery_modules.' . $module_name) || Config::has('poller_modules.' . $module_name); } public static function fromName(string $module_name): \LibreNMS\Interfaces\Module @@ -57,10 +61,27 @@ class Module public static function pollingStatus(string $module_name, Device $device, ?bool $manual = null): ModuleStatus { return new ModuleStatus( - Config::get("poller_modules.$module_name"), + Config::get("poller_modules.$module_name", false), Config::get("os.{$device->os}.poller_modules.$module_name"), $device->getAttrib("poll_$module_name"), $manual, ); } + + public static function parseUserOverrides(array $overrides): array + { + $modules = []; + + foreach ($overrides as $index => $module) { + // parse submodules (only supported by some modules) + if (str_contains($module, '/')) { + [$module, $submodule] = explode('/', $module, 2); + $modules[$module][] = $submodule; + } elseif (self::exists($module)) { + $modules[$module] = true; + } + } + + return $modules; + } } diff --git a/LibreNMS/Util/ModuleTestHelper.php b/LibreNMS/Util/ModuleTestHelper.php index d70c924374..1d90aff08d 100644 --- a/LibreNMS/Util/ModuleTestHelper.php +++ b/LibreNMS/Util/ModuleTestHelper.php @@ -26,6 +26,7 @@ namespace LibreNMS\Util; use App\Actions\Device\ValidateDeviceAndCreate; +use App\Jobs\PollDevice; use App\Models\Device; use DeviceCache; use Illuminate\Database\Eloquent\Model; @@ -35,7 +36,6 @@ use LibreNMS\Config; use LibreNMS\Data\Source\SnmpResponse; use LibreNMS\Exceptions\FileNotFoundException; use LibreNMS\Exceptions\InvalidModuleException; -use LibreNMS\Poller; class ModuleTestHelper { @@ -178,8 +178,7 @@ class ModuleTestHelper Debug::set(); Debug::setVerbose(); discover_device($device, $this->parseArgs('discovery')); - $poller = app(Poller::class, ['device_spec' => $device_id, 'module_override' => $this->modules]); - $poller->poll(); + (new PollDevice($device_id, $this->modules))->handle(); Debug::set($save_debug); Debug::setVerbose($save_vdebug); $collection_output = ob_get_contents(); @@ -312,7 +311,7 @@ class ModuleTestHelper * Probably needs to be more robust * * @param array $modules - * @return array + * @return array * * @throws InvalidModuleException */ @@ -320,17 +319,22 @@ class ModuleTestHelper { // generate a full list of modules $full_list = []; - foreach ($modules as $module) { + foreach ($modules as $index => $module) { + $module = is_string($index) ? $index : $module; + // only allow valid modules - if (! (Config::has("poller_modules.$module") || Config::has("discovery_modules.$module"))) { + if (! Module::exists($module)) { throw new InvalidModuleException("Invalid module name: $module"); } - $full_list = array_merge($full_list, Module::fromName($module)->dependencies()); - $full_list[] = $module; + foreach (Module::fromName($module)->dependencies() as $dependency) { + $full_list[$dependency] = true; + } + + $full_list[$module] = true; } - return array_unique($full_list); + return $full_list; } private function parseArgs($type) @@ -339,7 +343,7 @@ class ModuleTestHelper return false; } - return parse_modules($type, ['m' => implode(',', $this->modules)]); + return parse_modules($type, ['m' => implode(',', array_keys($this->modules))]); } private function qPrint($var) @@ -627,8 +631,7 @@ class ModuleTestHelper ob_start(); \Log::setDefaultDriver('console'); - $poller = app(Poller::class, ['device_spec' => $device_id, 'module_override' => $this->modules]); - $poller->poll(); + (new PollDevice($device_id, $this->modules))->handle(); $this->poller_output = ob_get_contents(); if ($this->quiet) { diff --git a/app/Console/Commands/DevicePoll.php b/app/Console/Commands/DevicePoll.php index f484f86721..5d38cbee54 100644 --- a/app/Console/Commands/DevicePoll.php +++ b/app/Console/Commands/DevicePoll.php @@ -3,16 +3,24 @@ namespace App\Console\Commands; use App\Console\LnmsCommand; +use App\Events\DevicePolled; +use App\Jobs\PollDevice; +use App\Models\Device; use App\Polling\Measure\MeasurementManager; use Illuminate\Database\QueryException; +use Illuminate\Support\Facades\Event; +use Illuminate\Support\Facades\Log; use LibreNMS\Config; -use LibreNMS\Poller; +use LibreNMS\Polling\Result; +use LibreNMS\Util\Module; +use LibreNMS\Util\Version; use Symfony\Component\Console\Input\InputArgument; use Symfony\Component\Console\Input\InputOption; class DevicePoll extends LnmsCommand { protected $name = 'device:poll'; + private ?int $current_device_id = null; /** * Create a new command instance. @@ -25,10 +33,15 @@ class DevicePoll extends LnmsCommand $this->addArgument('device spec', InputArgument::REQUIRED); $this->addOption('modules', 'm', InputOption::VALUE_REQUIRED); $this->addOption('no-data', 'x', InputOption::VALUE_NONE); + $this->addOption('dispatch', 'd', InputOption::VALUE_NONE); } public function handle(MeasurementManager $measurements): int { + if ($this->option('dispatch')) { + return $this->dispatchWork(); + } + $this->configureOutputOptions(); if ($this->option('no-data')) { @@ -40,9 +53,30 @@ class DevicePoll extends LnmsCommand } try { - /** @var \LibreNMS\Poller $poller */ - $poller = app(Poller::class, ['device_spec' => $this->argument('device spec'), 'module_override' => explode(',', $this->option('modules') ?? '')]); - $result = $poller->poll(); + if ($this->getOutput()->isVerbose()) { + Log::debug(Version::get()->header()); + \LibreNMS\Util\OS::updateCache(true); // Force update of OS Cache + } + + $module_overrides = Module::parseUserOverrides(explode(',', $this->option('modules') ?? '')); + $this->printModules($module_overrides); + + $result = new Result; + + $this->line("Starting polling run:\n"); + + // listen for the device polled events to mark the device completed + Event::listen(function (DevicePolled $event) use ($result) { + if ($event->device->device_id == $this->current_device_id) { + $result->markCompleted($event->device->status); + } + }); + + foreach (Device::whereDeviceSpec($this->argument('device spec'))->pluck('device_id') as $device_id) { + $this->current_device_id = $device_id; + $result->markAttempted(); + PollDevice::dispatchSync($device_id, $module_overrides); + } if ($result->hasAnyCompleted()) { if (! $this->output->isQuiet()) { @@ -92,4 +126,35 @@ class DevicePoll extends LnmsCommand return 1; // failed to poll } + + private function dispatchWork() + { + \Log::setDefaultDriver('stack'); + $module_overrides = Module::parseUserOverrides(explode(',', $this->option('modules') ?? '')); + $devices = Device::whereDeviceSpec($this->argument('device spec'))->pluck('device_id'); + + if (\config('queue.default') == 'sync') { + $this->error('Queue driver is sync, work will run in process.'); + sleep(1); + } + + foreach ($devices as $device_id) { + PollDevice::dispatch($device_id, $module_overrides); + } + + $this->line('Submitted work for ' . $devices->count() . ' devices'); + + return 0; + } + + private function printModules(array $module_overrides): void + { + if (! empty($module_overrides)) { + $modules = array_map(function ($module, $status) { + return $module . (is_array($status) ? '(' . implode(',', $status) . ')' : ''); + }, array_keys($module_overrides), array_values($module_overrides)); + + Log::debug('Override poller modules: ' . implode(', ', $modules)); + } + } } diff --git a/app/Jobs/PollDevice.php b/app/Jobs/PollDevice.php new file mode 100644 index 0000000000..3633119dc4 --- /dev/null +++ b/app/Jobs/PollDevice.php @@ -0,0 +1,247 @@ + $module_overrides + */ + public function __construct( + public int $device_id, + public array $module_overrides = [], + ) { + } + + /** + * Execute the job. + */ + public function handle() + { + $this->initDevice(); + PollingDevice::dispatch($this->device); + $this->os = OS::make($this->deviceArray); + + $measurement = Measurement::start('poll'); + $measurement->manager()->checkpoint(); // don't count previous stats + + $helper = new ConnectivityHelper($this->device); + $helper->saveMetrics(); + $helper->isUp(); // check and save status + + $this->pollModules(); + + $measurement->end(); + + // if modules are not overridden, record performance + if (empty($this->modules)) { + if ($this->device->status) { + $this->recordPerformance($measurement); + } + + if ($helper->canPing()) { + $this->os->enableGraph('ping_perf'); + } + + $this->os->persistGraphs($this->device->status); // save graphs but don't delete any if device is down + Log::info(sprintf("Enabled graphs (%s): %s\n\n", + $this->device->graphs->count(), + $this->device->graphs->pluck('graph')->implode(' ') + )); + } + + // finalize the device poll + $this->device->save(); + + Log::info(sprintf("\n>>> Polled %s (%s) in %0.3f seconds <<<", + $this->device->displayName(), + $this->device->device_id, + $measurement->getDuration())); + + // add log file line, this is used by the simple python dispatcher watchdog + Log::channel('single')->alert(sprintf('INFO: device:poll %s (%s) polled in %0.3fs', + $this->device->hostname, + $this->device->device_id, + $measurement->getDuration())); + + // check if the poll took too long and log an event + if ($measurement->getDuration() > Config::get('rrd.step')) { + Eventlog::log('Polling took longer than ' . round(Config::get('rrd.step') / 60, 2) . + ' minutes! This will cause gaps in graphs.', $this->device, 'system', Severity::Error); + } + + DevicePolled::dispatch($this->device); + } + + private function pollModules(): void + { + // update $device array status + $this->deviceArray['status'] = $this->device->status; + $this->deviceArray['status_reason'] = $this->device->status_reason; + + // import legacy garbage + include_once base_path('includes/functions.php'); + include_once base_path('includes/common.php'); + include_once base_path('includes/polling/functions.inc.php'); + include_once base_path('includes/snmp.inc.php'); + + $datastore = app('Datastore'); + + foreach ($this->getModules() as $module => $status) { + $module_status = Module::pollingStatus($module, $this->device, $this->isModuleManuallyEnabled($module)); + $should_poll = false; + $start_memory = memory_get_usage(); + $module_start = microtime(true); + + try { + $instance = Module::fromName($module); + $should_poll = $instance->shouldPoll($this->os, $module_status); + + if ($should_poll) { + Log::info("#### Load poller module $module ####\n"); + Log::debug($module_status); + + if (is_array($status)) { + Config::set('poller_submodules.' . $module, $status); + } + + $instance->poll($this->os, $datastore); + } + } catch (Throwable $e) { + // isolate module exceptions so they don't disrupt the polling process + Log::error("%rError polling $module module for {$this->device->hostname}.%n $e", ['color' => true]); + Eventlog::log("Error polling $module module. Check log file for more details.", $this->device, 'poller', Severity::Error); + report($e); + } + + if ($should_poll) { + Log::info(''); + app(MeasurementManager::class)->printChangedStats(); + $this->saveModulePerformance($module, $module_start, $start_memory); + Log::info("#### Unload poller module $module ####\n"); + } + } + } + + private function saveModulePerformance(string $module, float $start_time, int $start_memory): void + { + $module_time = microtime(true) - $start_time; + $module_mem = (memory_get_usage() - $start_memory); + + Log::info(sprintf(">> Runtime for poller module '%s': %.4f seconds with %s bytes", $module, $module_time, $module_mem)); + + app('Datastore')->put($this->deviceArray, 'poller-perf', [ + 'module' => $module, + 'rrd_def' => RrdDefinition::make()->addDataset('poller', 'GAUGE', 0), + 'rrd_name' => ['poller-perf', $module], + ], [ + 'poller' => $module_time, + ]); + $this->os->enableGraph('poller_modules_perf'); + } + + private function initDevice(): void + { + \DeviceCache::setPrimary($this->device_id); + $this->device = \DeviceCache::getPrimary(); + $this->device->ip = $this->device->overwrite_ip ?: Dns::lookupIp($this->device) ?: $this->device->ip; + + $this->deviceArray = $this->device->toArray(); + if ($os_group = Config::get("os.{$this->device->os}.group")) { + $this->deviceArray['os_group'] = $os_group; + } + + $this->printDeviceInfo($os_group); + $this->initRrdDirectory(); + } + + private function initRrdDirectory(): void + { + $host_rrd = \Rrd::name($this->device->hostname, '', ''); + if (Config::get('rrd.enable', true) && ! is_dir($host_rrd)) { + try { + mkdir($host_rrd); + Log::info("Created directory : $host_rrd"); + } catch (\ErrorException $e) { + Eventlog::log("Failed to create rrd directory: $host_rrd", $this->device); + Log::error($e); + } + } + } + + private function printDeviceInfo(?string $group): void + { + Log::info(sprintf(<<<'EOH' +Hostname: %s %s +ID: %s +OS: %s +IP: %s + +EOH, $this->device->hostname, $group ? " ($group)" : '', $this->device->device_id, $this->device->os, $this->device->ip)); + } + + private function recordPerformance(Measurement $measurement): void + { + $measurement->manager()->record('device', $measurement); + $this->device->last_polled = Carbon::now(); + $this->device->last_polled_timetaken = $measurement->getDuration(); + + app('Datastore')->put($this->deviceArray, 'poller-perf', [ + 'rrd_def' => RrdDefinition::make()->addDataset('poller', 'GAUGE', 0), + 'module' => 'ALL', + ], [ + 'poller' => $this->device->last_polled_timetaken, + ]); + + $this->os->enableGraph('poller_perf'); + } + + private function getModules(): array + { + if (! empty($this->module_overrides)) { + return $this->module_overrides; + } + + return \LibreNMS\Config::get('poller_modules', []); + } + + private function isModuleManuallyEnabled(string $module): ?bool + { + if (empty($this->module_overrides)) { + return null; + } + + return isset($this->module_overrides[$module]); + } +}