ershisan99/coolify
1
0
Fork 0
mirror of https://github.com/ershisan99/coolify.git synced 2026-02-04 21:02:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

Update server status check and notifications

Browse Source
This commit is contained in:
Andras Bacsai
2023-11-16 11:53:37 +01:00
parent f5de21a343
commit 7cec6330cf
13 changed files with 254 additions and 96 deletions
Download Patch File Download Diff File Expand all files Collapse all files

71
app/Jobs/ContainerStatusJob.php
View File

@@ -41,76 +41,7 @@ class ContainerStatusJob implements ShouldQueue, ShouldBeEncrypted
{
// ray("checking server status for {$this->server->id}");
try {
// ray()->clearAll();
$serverUptimeCheckNumber = $this->server->unreachable_count;
$serverUptimeCheckNumberMax = 3;
// ray('checking # ' . $serverUptimeCheckNumber);
if ($serverUptimeCheckNumber >= $serverUptimeCheckNumberMax) {
if ($this->server->unreachable_email_sent === false) {
ray('Server unreachable, sending notification...');
$this->server->team->notify(new Unreachable($this->server));
$this->server->update(['unreachable_email_sent' => true]);
}
$this->server->settings()->update([
'is_reachable' => false,
]);
$this->server->update([
'unreachable_count' => 0,
]);
// Update all applications, databases and services to exited
foreach ($this->server->applications() as $application) {
$application->update(['status' => 'exited']);
}
foreach ($this->server->databases() as $database) {
$database->update(['status' => 'exited']);
}
foreach ($this->server->services() as $service) {
$apps = $service->applications()->get();
$dbs = $service->databases()->get();
foreach ($apps as $app) {
$app->update(['status' => 'exited']);
}
foreach ($dbs as $db) {
$db->update(['status' => 'exited']);
}
}
return;
}
$result = $this->server->validateConnection();
if ($result) {
$this->server->settings()->update([
'is_reachable' => true,
]);
$this->server->update([
'unreachable_count' => 0,
]);
} else {
$serverUptimeCheckNumber++;
$this->server->settings()->update([
'is_reachable' => false,
]);
$this->server->update([
'unreachable_count' => $serverUptimeCheckNumber,
]);
return;
}
if (data_get($this->server, 'unreachable_email_sent') === true) {
ray('Server is reachable again, sending notification...');
$this->server->team->notify(new Revived($this->server));
$this->server->update(['unreachable_email_sent' => false]);
}
if (
data_get($this->server, 'settings.is_reachable') === false ||
data_get($this->server, 'settings.is_usable') === false
) {
$this->server->settings()->update([
'is_reachable' => true,
'is_usable' => true
]);
}
// $this->server->validateDockerEngine(true);
$this->server->checkServerRediness();
$containers = instant_remote_process(["docker container ls -q"], $this->server);
if (!$containers) {
return;

32
app/Jobs/DockerCleanupJob.php
View File

@@ -3,6 +3,7 @@
namespace App\Jobs;
use App\Models\Server;
use App\Notifications\Server\HighDiskUsage;
use Exception;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldBeEncrypted;
@@ -18,7 +19,6 @@ class DockerCleanupJob implements ShouldQueue, ShouldBeEncrypted
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
public $timeout = 300;
public ?string $dockerRootFilesystem = null;
public ?int $usageBefore = null;
public function __construct(public Server $server)
@@ -26,28 +26,27 @@ class DockerCleanupJob implements ShouldQueue, ShouldBeEncrypted
}
public function handle(): void
{
$isInprogress = false;
$this->server->applications()->each(function ($application) use (&$isInprogress) {
if ($application->isDeploymentInprogress()) {
$isInprogress = true;
return;
}
});
if ($isInprogress) {
throw new Exception('DockerCleanupJob: ApplicationDeploymentQueue is not empty, skipping...');
}
try {
$isInprogress = false;
$this->server->applications()->each(function ($application) use (&$isInprogress) {
if ($application->isDeploymentInprogress()) {
$isInprogress = true;
return;
}
});
if ($isInprogress) {
throw new Exception('DockerCleanupJob: ApplicationDeploymentQueue is not empty, skipping...');
}
if (!$this->server->isFunctional()) {
return;
}
$this->dockerRootFilesystem = "/";
$this->usageBefore = $this->getFilesystemUsage();
if ($this->usageBefore >= $this->server->settings->cleanup_after_percentage) {
ray('Cleaning up ' . $this->server->name);
instant_remote_process(['docker image prune -af'], $this->server);
instant_remote_process(['docker container prune -f --filter "label=coolify.managed=true"'], $this->server);
instant_remote_process(['docker builder prune -af'], $this->server);
$usageAfter = $this->getFilesystemUsage();
$usageAfter = $this->server->getDiskUsage();
if ($usageAfter < $this->usageBefore) {
ray('Saved ' . ($this->usageBefore - $usageAfter) . '% disk space on ' . $this->server->name);
send_internal_notification('DockerCleanupJob done: Saved ' . ($this->usageBefore - $usageAfter) . '% disk space on ' . $this->server->name);
@@ -65,9 +64,4 @@ class DockerCleanupJob implements ShouldQueue, ShouldBeEncrypted
throw $e;
}
}
private function getFilesystemUsage()
{
return instant_remote_process(["df '{$this->dockerRootFilesystem}'| tail -1 | awk '{ print $5}' | sed 's/%//g'"], $this->server, false);
}
}

54
app/Jobs/ServerStatusJob.php Normal file
View File

@@ -0,0 +1,54 @@
<?php
namespace App\Jobs;
use App\Models\Server;
use App\Notifications\Server\HighDiskUsage;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldBeEncrypted;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\Middleware\WithoutOverlapping;
use Illuminate\Queue\SerializesModels;
class ServerStatusJob implements ShouldQueue, ShouldBeEncrypted
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
public function __construct(public Server $server)
{
}
public function middleware(): array
{
return [(new WithoutOverlapping($this->server->id))->dontRelease()];
}
public function uniqueId(): int
{
return $this->server->id;
}
public function handle(): void
{
ray("checking server status for {$this->server->id}");
try {
$this->server->checkServerRediness();
$disk_usage = $this->server->getDiskUsage();
ray($this->server->settings->cleanup_after_percentage);
if ($disk_usage >= $this->server->settings->cleanup_after_percentage) {
$this->server->high_disk_usage_notification_sent = true;
$this->server->save();
$this->server->team->notify(new HighDiskUsage($this->server, $disk_usage, $this->server->settings->cleanup_after_percentage));
} else {
$this->server->high_disk_usage_notification_sent = false;
$this->server->save();
}
} catch (\Throwable $e) {
send_internal_notification('ServerStatusJob failed with: ' . $e->getMessage());
ray($e->getMessage());
handleError($e);
}
}
}