Add Supervisor services admin panel and notification.

This commit is contained in:
Buster Neece 2022-11-01 13:24:07 -05:00
parent 71ab777d31
commit fa3d607784
No known key found for this signature in database
GPG Key ID: F1D2E64A0005E80E
11 changed files with 458 additions and 148 deletions

View File

@ -12,6 +12,9 @@ release channel, you can take advantage of these new features and fixes.
## Code Quality/Technical Changes
- Because both our Docker and Ansible installations are managed by Supervisor now, we can view the realtime status of
all essential application services, and even restart them directly from the web interface.
## Bug Fixes
- Fixed a bug where if a station only had "Allowed IPs", it wouldn't be enforced.

View File

@ -169,6 +169,10 @@ return function (CallableEventDispatcherInterface $dispatcher) {
Event\GetNotifications::class,
App\Notification\Check\ProfilerAdvisorCheck::class
);
$dispatcher->addCallableListener(
Event\GetNotifications::class,
App\Notification\Check\ServiceCheck::class
);
$dispatcher->addCallableListener(
Event\Media\GetAlbumArt::class,

View File

@ -58,6 +58,18 @@ return static function (RouteCollectorProxy $group) {
$group->get('/server/stats', Controller\Api\Admin\ServerStatsAction::class)
->setName('api:admin:server:stats');
$group->get(
'/services',
Controller\Api\Admin\ServiceControlController::class . ':getAction'
)->setName('api:admin:services')
->add(new Middleware\Permissions(GlobalPermissions::View));
$group->post(
'/services/restart/{service}',
Controller\Api\Admin\ServiceControlController::class . ':restartAction'
)->setName('api:admin:services:restart')
->add(new Middleware\Permissions(GlobalPermissions::All));
$group->get('/permissions', Controller\Api\Admin\PermissionsAction::class)
->add(new Middleware\Permissions(GlobalPermissions::All));

View File

@ -26,11 +26,91 @@
</b-col>
</b-row>
<h2 class="outside-card-header mb-1">
<translate key="lang_hdr_server_status">Server Status</translate>
</h2>
<b-row>
<b-col sm="12" lg="6" xl="6" class="mb-4">
<b-card no-body>
<b-card-header header-bg-variant="primary-dark" class="d-flex align-items-center">
<div class="flex-fill">
<h2 class="card-title">
<translate key="lang_hdr_memory">Memory</translate>
</h2>
</div>
<div class="flex-shrink-0">
<b-button variant="outline-light" size="sm" class="py-2"
@click.prevent="showMemoryStatsHelpModal">
<icon icon="help_outline"></icon>
</b-button>
</div>
</b-card-header>
<b-card-body>
<h6 class="mb-1 text-center">
<translate key="lang_disk_header">Total RAM</translate>
:
{{ stats.memory.readable.total }}
</h6>
<b-progress :max="stats.memory.bytes.total" :label="stats.memory.readable.used"
class="h-20 mb-3 mt-2">
<b-progress-bar variant="primary" :value="stats.memory.bytes.used"></b-progress-bar>
<b-progress-bar variant="warning"
:value="stats.memory.bytes.cached"></b-progress-bar>
</b-progress>
<b-row>
<b-col>
<b-badge pill variant="primary">&nbsp;&nbsp;</b-badge>&nbsp;
<translate key="lang_memory_used">Used</translate>
: {{ stats.memory.readable.used }}
</b-col>
<b-col>
<b-badge pill variant="warning">&nbsp;&nbsp;</b-badge>&nbsp;
<translate key="lang_memory_cached">Cached</translate>
: {{ stats.memory.readable.cached }}
</b-col>
</b-row>
</b-card-body>
</b-card>
</b-col>
<b-col sm="12" lg="6" xl="6" class="mb-4">
<b-card no-body>
<b-card-header header-bg-variant="primary-dark">
<h2 class="card-title">
<translate key="lang_hdr_disk_space">Disk Space</translate>
</h2>
</b-card-header>
<b-card-body>
<h6 class="mb-1 text-center">
<translate key="lang_total_disk_space">Total Disk Space</translate>
:
{{ stats.disk.readable.total }}
</h6>
<b-progress :max="stats.disk.bytes.total" :label="stats.disk.readable.used"
class="h-20 mb-3 mt-2">
<b-progress-bar variant="primary" :value="stats.disk.bytes.used"></b-progress-bar>
</b-progress>
<b-row>
<b-col>
<b-badge pill variant="primary">&nbsp;&nbsp;</b-badge>&nbsp;
<translate key="lang_used_disk_space">Used</translate>
:
{{ stats.disk.readable.used }}
</b-col>
</b-row>
</b-card-body>
</b-card>
</b-col>
</b-row>
<b-row>
<b-col sm="12" lg="8" xl="6" class="mb-4">
<b-card no-body>
@ -127,87 +207,55 @@
</b-col>
<b-col sm="12" lg="4" xl="6" class="mb-4">
<b-row class="mb-4">
<b-col>
<b-card no-body>
<b-card-header header-bg-variant="primary-dark" class="d-flex align-items-center">
<div class="flex-fill">
<h2 class="card-title">
<translate key="lang_hdr_memory">Memory</translate>
</h2>
</div>
<b-card no-body>
<b-card-header header-bg-variant="primary-dark" class="d-flex align-items-center">
<div class="flex-fill">
<h2 class="card-title">
<translate key="lang_hdr_services">Services</translate>
</h2>
</div>
</b-card-header>
<div class="flex-shrink-0">
<b-button variant="outline-light" size="sm" class="py-2"
@click.prevent="showMemoryStatsHelpModal">
<icon icon="help_outline"></icon>
<table class="table table-sm table-striped table-responsive mb-0">
<colgroup>
<col style="width: 5%;">
<col style="width: 75%;">
<col style="width: 20%;">
</colgroup>
<tbody>
<tr class="align-middle" v-for="service in services">
<td class="text-center pr-2">
<template v-if="service.running">
<b-badge pill variant="success" :title="langServiceRunning">
&nbsp;&nbsp;
<span class="sr-only">{{ langServiceRunning }}</span>
</b-badge>
</template>
<template v-else>
<b-badge pill variant="danger" :title="langServiceStopped">
&nbsp;&nbsp;
<span class="sr-only">{{ langServiceStopped }}</span>
</b-badge>
</template>
</td>
<td class="pl-2">
<h6 class="mb-0">
{{ service.name }}<br>
<small>{{ service.description }}</small>
</h6>
</td>
<td>
<b-button-group size="sm" v-if="service.links.restart">
<b-button size="sm" :variant="service.running ? 'bg' : 'danger'"
@click.prevent="doRestart(service.links.restart)">
<translate key="lang_btn_restart">Restart</translate>
</b-button>
</div>
</b-card-header>
<b-card-body>
<h6 class="mb-1 text-center">
<translate key="lang_disk_header">Total RAM</translate>
:
{{ stats.memory.readable.total }}
</h6>
<b-progress :max="stats.memory.bytes.total" :label="stats.memory.readable.used"
class="h-20 mb-3 mt-2">
<b-progress-bar variant="primary" :value="stats.memory.bytes.used"></b-progress-bar>
<b-progress-bar variant="warning"
:value="stats.memory.bytes.cached"></b-progress-bar>
</b-progress>
<b-row>
<b-col>
<b-badge pill variant="primary">&nbsp;&nbsp;</b-badge>&nbsp;
<translate key="lang_memory_used">Used</translate>
: {{ stats.memory.readable.used }}
</b-col>
<b-col>
<b-badge pill variant="warning">&nbsp;&nbsp;</b-badge>&nbsp;
<translate key="lang_memory_cached">Cached</translate>
: {{ stats.memory.readable.cached }}
</b-col>
</b-row>
</b-card-body>
</b-card>
</b-col>
</b-row>
<b-row>
<b-col>
<b-card no-body>
<b-card-header header-bg-variant="primary-dark">
<h2 class="card-title">
<translate key="lang_hdr_disk_space">Disk Space</translate>
</h2>
</b-card-header>
<b-card-body>
<h6 class="mb-1 text-center">
<translate key="lang_total_disk_space">Total Disk Space</translate>
:
{{ stats.disk.readable.total }}
</h6>
<b-progress :max="stats.disk.bytes.total" :label="stats.disk.readable.used"
class="h-20 mb-3 mt-2">
<b-progress-bar variant="primary" :value="stats.disk.bytes.used"></b-progress-bar>
</b-progress>
<b-row>
<b-col>
<b-badge pill variant="primary">&nbsp;&nbsp;</b-badge>&nbsp;
<translate key="lang_used_disk_space">Used</translate>
:
{{ stats.disk.readable.used }}
</b-col>
</b-row>
</b-card-body>
</b-card>
</b-col>
</b-row>
</b-button-group>
</td>
</tr>
</tbody>
</table>
</b-card>
</b-col>
</b-row>
@ -267,6 +315,7 @@ export default {
props: {
adminPanels: Object,
statsUrl: String,
servicesUrl: String
},
data() {
return {
@ -308,11 +357,21 @@ export default {
}
},
network: []
}
},
services: []
};
},
created() {
this.updateStats();
this.updateServices();
},
computed: {
langServiceRunning() {
return this.$gettext('Service Running');
},
langServiceStopped() {
return this.$gettext('Service Stopped');
}
},
methods: {
formatCpuName(cpuName) {
@ -360,6 +419,22 @@ export default {
}
});
},
updateServices() {
this.axios.get(this.servicesUrl).then((response) => {
this.services = response.data;
setTimeout(this.updateServices, (!document.hidden) ? 5000 : 15000);
}).catch((error) => {
if (!error.response || error.response.data.code !== 403) {
setTimeout(this.updateServices, (!document.hidden) ? 15000 : 30000);
}
});
},
doRestart(serviceUrl) {
this.axios.post(serviceUrl).then((resp) => {
this.$notifySuccess(resp.data.message);
});
},
showCpuStatsHelpModal() {
this.$refs.cpuStatsHelpModal.create();
},

View File

@ -31,6 +31,7 @@ final class IndexAction
props: [
'adminPanels' => $viewData['admin_panels'] ?? [],
'statsUrl' => (string)$router->named('api:admin:server:stats'),
'servicesUrl' => (string)$router->named('api:admin:services'),
]
);
}

View File

@ -0,0 +1,57 @@
<?php
declare(strict_types=1);
namespace App\Controller\Api\Admin;
use App\Entity\Api\Status;
use App\Enums\GlobalPermissions;
use App\Http\Response;
use App\Http\ServerRequest;
use App\Service\ServiceControl;
use Psr\Http\Message\ResponseInterface;
final class ServiceControlController
{
public function __construct(
private readonly ServiceControl $serviceControl
) {
}
public function getAction(
ServerRequest $request,
Response $response
): ResponseInterface {
$router = $request->getRouter();
$canRestart = $request->getAcl()->isAllowed(GlobalPermissions::All);
$result = [];
foreach ($this->serviceControl->getServices() as $service) {
$row = $service->toArray();
$row['links'] = [];
if ($canRestart) {
$row['links']['restart'] = (string)$router->fromHere(
'api:admin:services:restart',
['service' => $service->name]
);
}
$result[] = $row;
}
return $response->withJson($result);
}
public function restartAction(
ServerRequest $request,
Response $response,
string $service
): ResponseInterface {
$this->serviceControl->restart($service);
return $response->withJson(Status::success());
}
}

View File

@ -5,7 +5,76 @@ declare(strict_types=1);
namespace App\Exception;
use App\Exception;
use App\Exception\Supervisor\AlreadyRunningException;
use App\Exception\Supervisor\BadNameException;
use App\Exception\Supervisor\NotRunningException;
use Supervisor\Exception\Fault\AlreadyStartedException;
use Supervisor\Exception\Fault\BadNameException as SupervisorBadNameException;
use Supervisor\Exception\Fault\NotRunningException as SupervisorNotRunningException;
use Supervisor\Exception\SupervisorException as SupervisorLibException;
class SupervisorException extends Exception
{
public static function fromSupervisorLibException(
SupervisorLibException $e,
string $processName
): self {
if ($e instanceof SupervisorBadNameException) {
$headline = sprintf(
__('%s is not recognized as a service.'),
$processName
);
$body = __('It may not be registered with Supervisor yet. Restarting broadcasting may help.');
$eNew = new BadNameException(
$headline . '; ' . $body,
$e->getCode(),
$e
);
} elseif ($e instanceof AlreadyStartedException) {
$headline = sprintf(
__('%s cannot start'),
$processName
);
$body = __('It is already running.');
$eNew = new AlreadyRunningException(
$headline . '; ' . $body,
$e->getCode(),
$e
);
} elseif ($e instanceof SupervisorNotRunningException) {
$headline = sprintf(
__('%s cannot stop'),
$processName
);
$body = __('It is not running.');
$eNew = new NotRunningException(
$headline . '; ' . $body,
$e->getCode(),
$e
);
} else {
$classParts = explode('\\', $e::class);
$className = array_pop($classParts);
$headline = sprintf(
__('%s encountered an error: %s'),
$processName,
$className
);
$body = __('Check the log for details.');
$eNew = new self(
$headline,
$e->getCode(),
$e
);
}
$eNew->setFormattedMessage('<b>' . $headline . '</b><br>' . $body);
return $eNew;
}
}

View File

@ -0,0 +1,50 @@
<?php
declare(strict_types=1);
namespace App\Notification\Check;
use App\Entity\Api\Notification;
use App\Enums\GlobalPermissions;
use App\Event\GetNotifications;
use App\Service\ServiceControl;
use App\Session\Flash;
final class ServiceCheck
{
public function __construct(
private readonly ServiceControl $serviceControl
) {
}
public function __invoke(GetNotifications $event): void
{
// This notification is for full administrators only.
$request = $event->getRequest();
$acl = $request->getAcl();
if (!$acl->isAllowed(GlobalPermissions::View)) {
return;
}
$services = $this->serviceControl->getServices();
foreach ($services as $service) {
if (!$service->running) {
// phpcs:disable Generic.Files.LineLength
$notification = new Notification();
$notification->title = sprintf(__('Service Not Running: %s'), $service->name);
$notification->body = __(
'One of the essential services on this installation is not currently running. Visit the system administration and check the system logs to find the cause of this issue.'
);
$notification->type = Flash::ERROR;
$router = $request->getRouter();
$notification->actionLabel = __('Administration');
$notification->actionUrl = (string)$router->named('admin:index:index');
// phpcs:enable
$event->addNotification($notification);
}
}
}
}

View File

@ -7,7 +7,6 @@ namespace App\Radio;
use App\Entity;
use App\Environment;
use App\Exception\Supervisor\AlreadyRunningException;
use App\Exception\Supervisor\BadNameException;
use App\Exception\Supervisor\NotRunningException;
use App\Exception\SupervisorException;
use App\Http\Router;
@ -231,9 +230,6 @@ abstract class AbstractLocalAdapter
* @param string $program_name
* @param Entity\Station $station
*
* @throws AlreadyRunningException
* @throws BadNameException
* @throws NotRunningException
* @throws SupervisorException
*/
protected function handleSupervisorException(
@ -241,70 +237,11 @@ abstract class AbstractLocalAdapter
string $program_name,
Entity\Station $station
): void {
$class_parts = explode('\\', static::class);
$class_name = array_pop($class_parts);
$eNew = SupervisorException::fromSupervisorLibException($e, $program_name);
$eNew->addLoggingContext('station_id', $station->getId());
$eNew->addLoggingContext('station_name', $station->getName());
if ($e instanceof Fault\BadNameException) {
$e_headline = sprintf(
__('%s is not recognized as a service.'),
$class_name
);
$e_body = __('It may not be registered with Supervisor yet. Restarting broadcasting may help.');
$app_e = new BadNameException(
$e_headline . '; ' . $e_body,
$e->getCode(),
$e
);
} elseif ($e instanceof Fault\AlreadyStartedException) {
$e_headline = sprintf(
__('%s cannot start'),
$class_name
);
$e_body = __('It is already running.');
$app_e = new AlreadyRunningException(
$e_headline . '; ' . $e_body,
$e->getCode(),
$e
);
} elseif ($e instanceof Fault\NotRunningException) {
$e_headline = sprintf(
__('%s cannot stop'),
$class_name
);
$e_body = __('It is not running.');
$app_e = new NotRunningException(
$e_headline . '; ' . $e_body,
$e->getCode(),
$e
);
} else {
$e_headline = sprintf(
__('%s encountered an error'),
$class_name
);
// Get more detailed information for more significant errors.
$process_log = $this->supervisor->tailProcessStdoutLog($program_name, 0, 500);
$process_log = array_values(array_filter(explode("\n", $process_log[0])));
$process_log = array_slice($process_log, -6);
$e_body = (!empty($process_log))
? implode('<br>', $process_log)
: __('Check the log for details.');
$app_e = new SupervisorException($e_headline, $e->getCode(), $e);
$app_e->addExtraData('supervisor_log', $process_log);
$app_e->addExtraData('supervisor_process_info', $this->supervisor->getProcessInfo($program_name));
}
$app_e->setFormattedMessage('<b>' . $e_headline . '</b><br>' . $e_body);
$app_e->addLoggingContext('station_id', $station->getId());
$app_e->addLoggingContext('station_name', $station->getName());
throw $app_e;
throw $eNew;
}
/**

View File

@ -0,0 +1,78 @@
<?php
declare(strict_types=1);
namespace App\Service;
use App\Exception\SupervisorException;
use App\Service\ServiceControl\ServiceData;
use Supervisor\Exception\Fault\BadNameException;
use Supervisor\Exception\Fault\NotRunningException;
use Supervisor\Exception\SupervisorException as SupervisorLibException;
use Supervisor\SupervisorInterface;
final class ServiceControl
{
public function __construct(
private readonly SupervisorInterface $supervisor
) {
}
/** @return ServiceData[] */
public function getServices(): array
{
$services = [];
foreach (self::getServiceNames() as $name => $description) {
try {
$isRunning = $this->supervisor->getProcess($name)->isRunning();
} catch (BadNameException) {
$isRunning = false;
}
$services[] = new ServiceData(
$name,
$description,
$isRunning
);
}
return $services;
}
public function restart(string $service): void
{
$serviceNames = self::getServiceNames();
if (!isset($serviceNames[$service])) {
throw new \InvalidArgumentException(
sprintf('Service "%s" is not managed by AzuraCast.', $service)
);
}
try {
$this->supervisor->stopProcess($service);
} catch (NotRunningException) {
}
try {
$this->supervisor->startProcess($service);
} catch (SupervisorLibException $e) {
throw SupervisorException::fromSupervisorLibException($e, $service);
}
}
public static function getServiceNames(): array
{
return [
'beanstalkd' => __('Message queue delivery service'),
'cron' => __('Runs routine synchronized tasks'),
'mariadb' => __('Database'),
'nginx' => __('Web server'),
'php-fpm' => __('PHP FastCGI Process Manager'),
'php-nowplaying' => __('Now Playing manager service'),
'php-worker' => __('PHP queue processing worker'),
'redis' => __('Cache'),
'sftpgo' => __('SFTP service'),
];
}
}

View File

@ -0,0 +1,24 @@
<?php
declare(strict_types=1);
namespace App\Service\ServiceControl;
final class ServiceData
{
public function __construct(
public readonly string $name,
public readonly string $description,
public readonly bool $running
) {
}
public function toArray(): array
{
return [
'name' => $this->name,
'description' => $this->description,
'running' => $this->running,
];
}
}