Merge branch 'next' into improve-cleanup

This commit is contained in:
Andras Bacsai
2024-09-27 16:40:48 +02:00
committed by GitHub
208 changed files with 3593 additions and 2772 deletions

View File

@@ -27,6 +27,7 @@ use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Collection;
use Illuminate\Support\Facades\Process;
use Illuminate\Support\Sleep;
use Illuminate\Support\Str;
use RuntimeException;
@@ -210,7 +211,6 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue
}
ray('New container name: ', $this->container_name)->green();
savePrivateKeyToFs($this->server);
$this->saved_outputs = collect();
// Set preview fqdn
@@ -1456,10 +1456,10 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue
executeInDocker($this->deployment_uuid, 'chmod 600 /root/.ssh/id_rsa'),
],
[
executeInDocker($this->deployment_uuid, "GIT_SSH_COMMAND=\"ssh -o ConnectTimeout=30 -p {$this->customPort} -o Port={$this->customPort} -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i /root/.ssh/id_rsa\" git ls-remote {$this->fullRepoUrl} {$local_branch}"),
executeInDocker($this->deployment_uuid, "GIT_SSH_COMMAND=\"ssh -o ConnectTimeout=30 -p {$this->customPort} -o Port={$this->customPort} -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null\" git ls-remote {$this->fullRepoUrl} {$local_branch}"),
'hidden' => true,
'save' => 'git_commit_sha',
],
]
);
} else {
$this->execute_remote_command(
@@ -2211,20 +2211,40 @@ COPY ./nginx.conf /etc/nginx/conf.d/default.conf");
$this->application_deployment_queue->addLogEntry('Building docker image completed.');
}
/**
* @param int $timeout in seconds
*/
private function graceful_shutdown_container(string $containerName, int $timeout = 30)
private function graceful_shutdown_container(string $containerName, int $timeout = 300)
{
try {
$this->execute_remote_command(
["docker stop --time=$timeout $containerName", 'hidden' => true, 'ignore_errors' => true],
["docker rm $containerName", 'hidden' => true, 'ignore_errors' => true]
);
$process = Process::timeout($timeout)->start("docker stop --time=$timeout $containerName");
$startTime = time();
while ($process->running()) {
if (time() - $startTime >= $timeout) {
$this->execute_remote_command(
["docker kill $containerName", 'hidden' => true, 'ignore_errors' => true]
);
break;
}
usleep(100000);
}
$isRunning = $this->execute_remote_command(
["docker inspect -f '{{.State.Running}}' $containerName", 'hidden' => true, 'ignore_errors' => true]
) === 'true';
if ($isRunning) {
$this->execute_remote_command(
["docker kill $containerName", 'hidden' => true, 'ignore_errors' => true]
);
}
} catch (\Exception $error) {
// report error if needed
$this->application_deployment_queue->addLogEntry("Error stopping container $containerName: ".$error->getMessage(), 'stderr');
}
$this->remove_container($containerName);
}
private function remove_container(string $containerName)
{
$this->execute_remote_command(
["docker rm -f $containerName", 'hidden' => true, 'ignore_errors' => true]
);

View File

@@ -9,8 +9,8 @@ use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Facades\Http;
use Illuminate\Support\Facades\File;
use Illuminate\Support\Facades\Http;
class CheckForUpdatesJob implements ShouldBeEncrypted, ShouldQueue
{

View File

@@ -21,11 +21,10 @@ class CleanupHelperContainersJob implements ShouldBeEncrypted, ShouldBeUnique, S
{
try {
ray('Cleaning up helper containers on '.$this->server->name);
$containers = instant_remote_process(['docker container ps --filter "ancestor=ghcr.io/coollabsio/coolify-helper:next" --filter "ancestor=ghcr.io/coollabsio/coolify-helper:latest" --format \'{{json .}}\''], $this->server, false);
$containers = format_docker_command_output_to_json($containers);
if ($containers->count() > 0) {
foreach ($containers as $container) {
$containerId = data_get($container, 'ID');
$containers = instant_remote_process(['docker container ps --format \'{{json .}}\' | jq -s \'map(select(.Image | contains("ghcr.io/coollabsio/coolify-helper")))\''], $this->server, false);
$containerIds = collect(json_decode($containers))->pluck('ID');
if ($containerIds->count() > 0) {
foreach ($containerIds as $containerId) {
ray('Removing container '.$containerId);
instant_remote_process(['docker container rm -f '.$containerId], $this->server, false);
}

View File

@@ -3,12 +3,14 @@
namespace App\Jobs;
use App\Models\Server;
use Carbon\Carbon;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Facades\Process;
use Illuminate\Support\Facades\Storage;
class CleanupStaleMultiplexedConnections implements ShouldQueue
{
@@ -16,22 +18,65 @@ class CleanupStaleMultiplexedConnections implements ShouldQueue
public function handle()
{
Server::chunk(100, function ($servers) {
foreach ($servers as $server) {
$this->cleanupStaleConnection($server);
}
});
$this->cleanupStaleConnections();
$this->cleanupNonExistentServerConnections();
}
private function cleanupStaleConnection(Server $server)
private function cleanupStaleConnections()
{
$muxSocket = "/tmp/mux_{$server->id}";
$checkCommand = "ssh -O check -o ControlPath=$muxSocket {$server->user}@{$server->ip} 2>/dev/null";
$checkProcess = Process::run($checkCommand);
$muxFiles = Storage::disk('ssh-mux')->files();
if ($checkProcess->exitCode() !== 0) {
$closeCommand = "ssh -O exit -o ControlPath=$muxSocket {$server->user}@{$server->ip} 2>/dev/null";
Process::run($closeCommand);
foreach ($muxFiles as $muxFile) {
$serverUuid = $this->extractServerUuidFromMuxFile($muxFile);
$server = Server::where('uuid', $serverUuid)->first();
if (! $server) {
$this->removeMultiplexFile($muxFile);
continue;
}
$muxSocket = "/var/www/html/storage/app/ssh/mux/{$muxFile}";
$checkCommand = "ssh -O check -o ControlPath={$muxSocket} {$server->user}@{$server->ip} 2>/dev/null";
$checkProcess = Process::run($checkCommand);
if ($checkProcess->exitCode() !== 0) {
$this->removeMultiplexFile($muxFile);
} else {
$muxContent = Storage::disk('ssh-mux')->get($muxFile);
$establishedAt = Carbon::parse(substr($muxContent, 37));
$expirationTime = $establishedAt->addSeconds(config('constants.ssh.mux_persist_time'));
if (Carbon::now()->isAfter($expirationTime)) {
$this->removeMultiplexFile($muxFile);
}
}
}
}
private function cleanupNonExistentServerConnections()
{
$muxFiles = Storage::disk('ssh-mux')->files();
$existingServerUuids = Server::pluck('uuid')->toArray();
foreach ($muxFiles as $muxFile) {
$serverUuid = $this->extractServerUuidFromMuxFile($muxFile);
if (! in_array($serverUuid, $existingServerUuids)) {
$this->removeMultiplexFile($muxFile);
}
}
}
private function extractServerUuidFromMuxFile($muxFile)
{
return substr($muxFile, 4);
}
private function removeMultiplexFile($muxFile)
{
$muxSocket = "/var/www/html/storage/app/ssh/mux/{$muxFile}";
$closeCommand = "ssh -O exit -o ControlPath={$muxSocket} localhost 2>/dev/null";
Process::run($closeCommand);
Storage::disk('ssh-mux')->delete($muxFile);
}
}

View File

@@ -9,7 +9,6 @@ use Illuminate\Contracts\Queue\ShouldBeEncrypted;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\Middleware\WithoutOverlapping;
use Illuminate\Queue\SerializesModels;
class ContainerStatusJob implements ShouldBeEncrypted, ShouldQueue
@@ -25,16 +24,6 @@ class ContainerStatusJob implements ShouldBeEncrypted, ShouldQueue
public function __construct(public Server $server) {}
public function middleware(): array
{
return [(new WithoutOverlapping($this->server->uuid))];
}
public function uniqueId(): int
{
return $this->server->uuid;
}
public function handle()
{
GetContainersStatus::run($this->server);

View File

@@ -23,7 +23,6 @@ use Illuminate\Contracts\Queue\ShouldBeEncrypted;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\Middleware\WithoutOverlapping;
use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Str;
use Visus\Cuid2\Cuid2;
@@ -80,16 +79,6 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
}
}
public function middleware(): array
{
return [new WithoutOverlapping($this->backup->id)];
}
public function uniqueId(): int
{
return $this->backup->id;
}
public function handle(): void
{
try {

View File

@@ -1,62 +0,0 @@
<?php
namespace App\Jobs;
use App\Models\ScheduledDatabaseBackup;
use App\Models\Team;
use App\Notifications\Database\DailyBackup;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldBeEncrypted;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
class DatabaseBackupStatusJob implements ShouldBeEncrypted, ShouldQueue
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
public $tries = 1;
public function __construct() {}
public function handle()
{
// $teams = Team::all();
// foreach ($teams as $team) {
// $scheduled_backups = $team->scheduledDatabaseBackups()->get();
// if ($scheduled_backups->isEmpty()) {
// continue;
// }
// foreach ($scheduled_backups as $scheduled_backup) {
// $last_days_backups = $scheduled_backup->get_last_days_backup_status();
// if ($last_days_backups->isEmpty()) {
// continue;
// }
// $failed = $last_days_backups->where('status', 'failed');
// }
// }
// $scheduled_backups = ScheduledDatabaseBackup::all();
// $databases = collect();
// $teams = collect();
// foreach ($scheduled_backups as $scheduled_backup) {
// $last_days_backups = $scheduled_backup->get_last_days_backup_status();
// if ($last_days_backups->isEmpty()) {
// continue;
// }
// $failed = $last_days_backups->where('status', 'failed');
// $database = $scheduled_backup->database;
// $team = $database->team();
// $teams->put($team->id, $team);
// $databases->put("{$team->id}:{$database->name}", [
// 'failed_count' => $failed->count(),
// ]);
// }
// foreach ($databases as $name => $database) {
// [$team_id, $name] = explode(':', $name);
// $team = $teams->get($team_id);
// $team?->notify(new DailyBackup($databases));
// }
}
}

View File

@@ -4,6 +4,7 @@ namespace App\Jobs;
use App\Actions\Application\StopApplication;
use App\Actions\Database\StopDatabase;
use App\Actions\Server\CleanupDocker;
use App\Actions\Service\DeleteService;
use App\Actions\Service\StopService;
use App\Models\Application;
@@ -30,8 +31,11 @@ class DeleteResourceJob implements ShouldBeEncrypted, ShouldQueue
public function __construct(
public Application|Service|StandalonePostgresql|StandaloneRedis|StandaloneMongodb|StandaloneMysql|StandaloneMariadb|StandaloneKeydb|StandaloneDragonfly|StandaloneClickhouse $resource,
public bool $deleteConfigurations = false,
public bool $deleteVolumes = false) {}
public bool $deleteConfigurations,
public bool $deleteVolumes,
public bool $dockerCleanup,
public bool $deleteConnectedNetworks
) {}
public function handle()
{
@@ -51,11 +55,11 @@ class DeleteResourceJob implements ShouldBeEncrypted, ShouldQueue
case 'standalone-dragonfly':
case 'standalone-clickhouse':
$persistentStorages = $this->resource?->persistentStorages()?->get();
StopDatabase::run($this->resource);
StopDatabase::run($this->resource, true);
break;
case 'service':
StopService::run($this->resource);
DeleteService::run($this->resource);
StopService::run($this->resource, true);
DeleteService::run($this->resource, $this->deleteConfigurations, $this->deleteVolumes, $this->dockerCleanup, $this->deleteConnectedNetworks);
break;
}
@@ -65,12 +69,31 @@ class DeleteResourceJob implements ShouldBeEncrypted, ShouldQueue
if ($this->deleteConfigurations) {
$this->resource?->delete_configurations();
}
$isDatabase = $this->resource instanceof StandalonePostgresql
|| $this->resource instanceof StandaloneRedis
|| $this->resource instanceof StandaloneMongodb
|| $this->resource instanceof StandaloneMysql
|| $this->resource instanceof StandaloneMariadb
|| $this->resource instanceof StandaloneKeydb
|| $this->resource instanceof StandaloneDragonfly
|| $this->resource instanceof StandaloneClickhouse;
$server = data_get($this->resource, 'server') ?? data_get($this->resource, 'destination.server');
if (($this->dockerCleanup || $isDatabase) && $server) {
CleanupDocker::dispatch($server, true);
}
if ($this->deleteConnectedNetworks && ! $isDatabase) {
$this->resource?->delete_connected_networks($this->resource->uuid);
}
} catch (\Throwable $e) {
ray($e->getMessage());
send_internal_notification('ContainerStoppingJob failed with: '.$e->getMessage());
throw $e;
} finally {
$this->resource->forceDelete();
if ($this->dockerCleanup) {
CleanupDocker::dispatch($server, true);
}
Artisan::queue('cleanup:stucked-resources');
}
}

View File

@@ -10,7 +10,6 @@ use Illuminate\Contracts\Queue\ShouldBeEncrypted;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\Middleware\WithoutOverlapping;
use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Facades\Log;
@@ -26,16 +25,6 @@ class DockerCleanupJob implements ShouldBeEncrypted, ShouldQueue
public function __construct(public Server $server, public bool $manualCleanup = false) {}
public function middleware(): array
{
return [new WithoutOverlapping($this->server->id)];
}
public function uniqueId(): int
{
return $this->server->id;
}
public function handle(): void
{
try {

View File

@@ -8,7 +8,6 @@ use Illuminate\Contracts\Queue\ShouldBeEncrypted;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\Middleware\WithoutOverlapping;
use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Facades\Http;
@@ -25,16 +24,6 @@ class GithubAppPermissionJob implements ShouldBeEncrypted, ShouldQueue
public function __construct(public GithubApp $github_app) {}
public function middleware(): array
{
return [(new WithoutOverlapping($this->github_app->uuid))];
}
public function uniqueId(): int
{
return $this->github_app->uuid;
}
public function handle()
{
try {

View File

@@ -9,7 +9,6 @@ use Illuminate\Contracts\Queue\ShouldBeEncrypted;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\Middleware\WithoutOverlapping;
use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Facades\Http;
@@ -19,17 +18,7 @@ class PullHelperImageJob implements ShouldBeEncrypted, ShouldQueue
public $timeout = 1000;
public function middleware(): array
{
return [(new WithoutOverlapping($this->server->uuid))];
}
public function uniqueId(): string
{
return $this->server->uuid;
}
public function __construct(public Server $server) {}
public function __construct() {}
public function handle(): void
{

View File

@@ -9,7 +9,6 @@ use Illuminate\Contracts\Queue\ShouldBeEncrypted;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\Middleware\WithoutOverlapping;
use Illuminate\Queue\SerializesModels;
class PullSentinelImageJob implements ShouldBeEncrypted, ShouldQueue
@@ -18,16 +17,6 @@ class PullSentinelImageJob implements ShouldBeEncrypted, ShouldQueue
public $timeout = 1000;
public function middleware(): array
{
return [(new WithoutOverlapping($this->server->uuid))];
}
public function uniqueId(): string
{
return $this->server->uuid;
}
public function __construct(public Server $server) {}
public function handle(): void

View File

@@ -13,7 +13,6 @@ use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\Middleware\WithoutOverlapping;
use Illuminate\Queue\SerializesModels;
class ScheduledTaskJob implements ShouldQueue
@@ -56,24 +55,17 @@ class ScheduledTaskJob implements ShouldQueue
{
if ($this->resource instanceof Application) {
$timezone = $this->resource->destination->server->settings->server_timezone;
return $timezone;
} elseif ($this->resource instanceof Service) {
$timezone = $this->resource->server->settings->server_timezone;
return $timezone;
}
return 'UTC';
}
public function middleware(): array
{
return [new WithoutOverlapping($this->task->id)];
}
public function uniqueId(): int
{
return $this->task->id;
}
public function handle(): void
{
@@ -94,12 +86,12 @@ class ScheduledTaskJob implements ShouldQueue
} elseif ($this->resource->type() == 'service') {
$this->resource->applications()->get()->each(function ($application) {
if (str(data_get($application, 'status'))->contains('running')) {
$this->containers[] = data_get($application, 'name') . '-' . data_get($this->resource, 'uuid');
$this->containers[] = data_get($application, 'name').'-'.data_get($this->resource, 'uuid');
}
});
$this->resource->databases()->get()->each(function ($database) {
if (str(data_get($database, 'status'))->contains('running')) {
$this->containers[] = data_get($database, 'name') . '-' . data_get($this->resource, 'uuid');
$this->containers[] = data_get($database, 'name').'-'.data_get($this->resource, 'uuid');
}
});
}
@@ -112,8 +104,8 @@ class ScheduledTaskJob implements ShouldQueue
}
foreach ($this->containers as $containerName) {
if (count($this->containers) == 1 || str_starts_with($containerName, $this->task->container . '-' . $this->resource->uuid)) {
$cmd = "sh -c '" . str_replace("'", "'\''", $this->task->command) . "'";
if (count($this->containers) == 1 || str_starts_with($containerName, $this->task->container.'-'.$this->resource->uuid)) {
$cmd = "sh -c '".str_replace("'", "'\''", $this->task->command)."'";
$exec = "docker exec {$containerName} {$cmd}";
$this->task_output = instant_remote_process([$exec], $this->server, true);
$this->task_log->update([

View File

@@ -16,7 +16,6 @@ use Illuminate\Contracts\Queue\ShouldBeEncrypted;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\Middleware\WithoutOverlapping;
use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Arr;
@@ -24,7 +23,7 @@ class ServerCheckJob implements ShouldBeEncrypted, ShouldQueue
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
public $tries = 3;
public $tries = 1;
public $timeout = 60;
@@ -45,16 +44,6 @@ class ServerCheckJob implements ShouldBeEncrypted, ShouldQueue
public function __construct(public Server $server) {}
public function middleware(): array
{
return [(new WithoutOverlapping($this->server->id))];
}
public function uniqueId(): int
{
return $this->server->id;
}
public function handle()
{
try {
@@ -80,7 +69,9 @@ class ServerCheckJob implements ShouldBeEncrypted, ShouldQueue
return 'No containers found.';
}
GetContainersStatus::run($this->server, $this->containers, $containerReplicates);
$this->checkLogDrainContainer();
if ($this->server->isLogDrainEnabled()) {
$this->checkLogDrainContainer();
}
}
} catch (\Throwable $e) {
@@ -93,7 +84,7 @@ class ServerCheckJob implements ShouldBeEncrypted, ShouldQueue
private function serverStatus()
{
['uptime' => $uptime] = $this->server->validateConnection();
['uptime' => $uptime] = $this->server->validateConnection(false);
if ($uptime) {
if ($this->server->unreachable_notification_sent === true) {
$this->server->update(['unreachable_notification_sent' => false]);
@@ -126,9 +117,6 @@ class ServerCheckJob implements ShouldBeEncrypted, ShouldQueue
private function checkLogDrainContainer()
{
if (! $this->server->isLogDrainEnabled()) {
return;
}
$foundLogDrainContainer = $this->containers->filter(function ($value, $key) {
return data_get($value, 'Name') === '/coolify-log-drain';
})->first();

View File

@@ -10,7 +10,7 @@ use Illuminate\Contracts\Queue\ShouldBeEncrypted;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\Middleware\WithoutOverlapping;
use Illuminate\Queue\Middleware\;
use Illuminate\Queue\SerializesModels;
class ServerLimitCheckJob implements ShouldBeEncrypted, ShouldQueue
@@ -26,16 +26,6 @@ class ServerLimitCheckJob implements ShouldBeEncrypted, ShouldQueue
public function __construct(public Team $team) {}
public function middleware(): array
{
return [(new WithoutOverlapping($this->team->uuid))];
}
public function uniqueId(): int
{
return $this->team->uuid;
}
public function handle()
{
try {

View File

@@ -8,7 +8,6 @@ use Illuminate\Contracts\Queue\ShouldBeEncrypted;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\Middleware\WithoutOverlapping;
use Illuminate\Queue\SerializesModels;
class ServerStatusJob implements ShouldBeEncrypted, ShouldQueue
@@ -26,16 +25,6 @@ class ServerStatusJob implements ShouldBeEncrypted, ShouldQueue
public function __construct(public Server $server) {}
public function middleware(): array
{
return [(new WithoutOverlapping($this->server->uuid))];
}
public function uniqueId(): int
{
return $this->server->uuid;
}
public function handle()
{
if (! $this->server->isServerReady($this->tries)) {