From 144508218e6339e8828d8a5001548b6e7e1b373b Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Tue, 17 Sep 2024 12:26:11 +0200 Subject: [PATCH] Fix: SSH multiplexing --- app/Actions/CoolifyTask/RunRemoteProcess.php | 3 +- app/Helpers/SshMultiplexingHelper.php | 126 +++++++++++-------- app/Jobs/ServerCheckJob.php | 7 +- app/Livewire/Project/Shared/GetLogs.php | 9 +- app/Models/Server.php | 7 +- app/Traits/ExecuteRemoteCommand.php | 3 +- bootstrap/helpers/remoteProcess.php | 26 +--- 7 files changed, 92 insertions(+), 89 deletions(-) diff --git a/app/Actions/CoolifyTask/RunRemoteProcess.php b/app/Actions/CoolifyTask/RunRemoteProcess.php index 63e3afe2f..a5dfa9226 100644 --- a/app/Actions/CoolifyTask/RunRemoteProcess.php +++ b/app/Actions/CoolifyTask/RunRemoteProcess.php @@ -10,6 +10,7 @@ use Illuminate\Process\ProcessResult; use Illuminate\Support\Facades\DB; use Illuminate\Support\Facades\Process; use Spatie\Activitylog\Models\Activity; +use App\Helpers\SshMultiplexingHelper; class RunRemoteProcess { @@ -137,7 +138,7 @@ class RunRemoteProcess $command = $this->activity->getExtraProperty('command'); $server = Server::whereUuid($server_uuid)->firstOrFail(); - return generateSshCommand($server, $command); + return SshMultiplexingHelper::generateSshCommand($server, $command); } protected function handleOutput(string $type, string $output) diff --git a/app/Helpers/SshMultiplexingHelper.php b/app/Helpers/SshMultiplexingHelper.php index 8b39d61e8..077bd68db 100644 --- a/app/Helpers/SshMultiplexingHelper.php +++ b/app/Helpers/SshMultiplexingHelper.php @@ -3,11 +3,10 @@ namespace App\Helpers; use App\Models\Server; +use App\Models\PrivateKey; use Illuminate\Support\Facades\Process; use Illuminate\Support\Facades\Storage; use Illuminate\Support\Facades\Hash; -use Illuminate\Support\Str; -use App\Models\PrivateKey; class SshMultiplexingHelper { @@ -15,7 +14,8 @@ class SshMultiplexingHelper public static function serverSshConfiguration(Server $server) { - $sshKeyLocation = $server->privateKey->getKeyLocation(); + $privateKey = PrivateKey::findOrFail($server->private_key_id); + $sshKeyLocation = $privateKey->getKeyLocation(); $muxFilename = '/var/www/html/storage/app/ssh/mux/' . $server->muxFilename(); return [ @@ -26,15 +26,21 @@ class SshMultiplexingHelper public static function ensureMultiplexedConnection(Server $server) { + if (!self::isMultiplexingEnabled()) { + ray('Multiplexing is disabled'); + return; + } + + ray('Ensuring multiplexed connection for server: ' . $server->id); + $sshConfig = self::serverSshConfiguration($server); $muxSocket = $sshConfig['muxFilename']; $sshKeyLocation = $sshConfig['sshKeyLocation']; - if (!file_exists($sshKeyLocation)) { - throw new \RuntimeException("SSH key file not accessible: $sshKeyLocation"); - } + self::validateSshKey($sshKeyLocation); if (isset(self::$ensuredConnections[$server->id]) && !self::shouldResetMultiplexedConnection($server)) { + ray('Existing connection is still valid'); return; } @@ -42,6 +48,7 @@ class SshMultiplexingHelper $fileCheckProcess = Process::run($checkFileCommand); if ($fileCheckProcess->exitCode() !== 0) { + ray('Mux socket file not found, establishing new connection'); self::establishNewMultiplexedConnection($server); return; } @@ -49,19 +56,22 @@ class SshMultiplexingHelper $checkCommand = "ssh -O check -o ControlPath=$muxSocket {$server->user}@{$server->ip}"; $process = Process::run($checkCommand); - if ($process->exitCode() === 0) { + if ($process->exitCode() !== 0) { + ray('Existing connection check failed, establishing new connection'); + self::establishNewMultiplexedConnection($server); + } else { + ray('Existing connection is valid'); self::$ensuredConnections[$server->id] = [ 'timestamp' => now(), 'muxSocket' => $muxSocket, ]; - return; } - - self::establishNewMultiplexedConnection($server); } public static function establishNewMultiplexedConnection(Server $server) { + ray('Establishing new multiplexed connection for server: ' . $server->id); + $sshConfig = self::serverSshConfiguration($server); $sshKeyLocation = $sshConfig['sshKeyLocation']; $muxSocket = $sshConfig['muxFilename']; @@ -84,9 +94,12 @@ class SshMultiplexingHelper $establishProcess = Process::run($establishCommand); if ($establishProcess->exitCode() !== 0) { + ray('Failed to establish multiplexed connection', $establishProcess->errorOutput()); throw new \RuntimeException('Failed to establish multiplexed connection: ' . $establishProcess->errorOutput()); } + ray('Multiplexed connection established successfully'); + $muxContent = "Multiplexed connection established at " . now()->toDateTimeString(); Storage::disk('ssh-mux')->put(basename($muxSocket), $muxContent); @@ -99,6 +112,7 @@ class SshMultiplexingHelper public static function shouldResetMultiplexedConnection(Server $server) { if (!(config('constants.ssh.mux_enabled') && config('coolify.is_windows_docker_desktop') == false)) { + ray('Multiplexing is disabled or running on Windows Docker Desktop'); return false; } @@ -110,7 +124,9 @@ class SshMultiplexingHelper $muxPersistTime = config('constants.ssh.mux_persist_time'); $resetInterval = strtotime($muxPersistTime) - time(); - return $lastEnsured->addSeconds($resetInterval)->isPast(); + $shouldReset = $lastEnsured->addSeconds($resetInterval)->isPast(); + ray('Should reset multiplexed connection', ['server_id' => $server->id, 'should_reset' => $shouldReset]); + return $shouldReset; } public static function removeMuxFile(Server $server) @@ -130,38 +146,22 @@ class SshMultiplexingHelper $sshKeyLocation = $sshConfig['sshKeyLocation']; $muxSocket = $sshConfig['muxFilename']; - $user = $server->user; - $port = $server->port; $timeout = config('constants.ssh.command_timeout'); $connectionTimeout = config('constants.ssh.connection_timeout'); $serverInterval = config('constants.ssh.server_interval'); - $muxPersistTime = config('constants.ssh.mux_persist_time'); $scp_command = "timeout $timeout scp "; - $muxEnabled = config('constants.ssh.mux_enabled', true) && config('coolify.is_windows_docker_desktop') == false; - ray('SSH Multiplexing Enabled:', $muxEnabled)->blue(); - if ($muxEnabled) { + if (self::isMultiplexingEnabled()) { + $muxPersistTime = config('constants.ssh.mux_persist_time'); $scp_command .= "-o ControlMaster=auto -o ControlPath=$muxSocket -o ControlPersist={$muxPersistTime} "; self::ensureMultiplexedConnection($server); - ray('Using SSH Multiplexing')->green(); - } else { - ray('Not using SSH Multiplexing')->red(); } - if (data_get($server, 'settings.is_cloudflare_tunnel')) { - $scp_command .= '-o ProxyCommand="/usr/local/bin/cloudflared access ssh --hostname %h" '; - } - $scp_command .= "-i {$sshKeyLocation} " - .'-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null ' - .'-o PasswordAuthentication=no ' - ."-o ConnectTimeout=$connectionTimeout " - ."-o ServerAliveInterval=$serverInterval " - .'-o RequestTTY=no ' - .'-o LogLevel=ERROR ' - ."-P {$port} " - ."{$source} " - ."{$user}@{$server->ip}:{$dest}"; + self::addCloudflareProxyCommand($scp_command, $server); + + $scp_command .= self::getCommonSshOptions($server, $sshKeyLocation, $connectionTimeout, $serverInterval); + $scp_command .= "{$source} {$server->user}@{$server->ip}:{$dest}"; return $scp_command; } @@ -179,45 +179,61 @@ class SshMultiplexingHelper $timeout = config('constants.ssh.command_timeout'); $connectionTimeout = config('constants.ssh.connection_timeout'); $serverInterval = config('constants.ssh.server_interval'); - $muxPersistTime = config('constants.ssh.mux_persist_time'); - $muxEnabled = config('constants.ssh.mux_enabled') && !config('coolify.is_windows_docker_desktop'); - ray('Config MUX Enabled:', config('constants.ssh.mux_enabled')); - ray('Config Windows Docker Desktop:', config('coolify.is_windows_docker_desktop')); - ray('MUX Enabled:', $muxEnabled); $ssh_command = "timeout $timeout ssh "; - ray('SSH Multiplexing Enabled:', $muxEnabled)->blue(); - - if ($muxEnabled) { + if (self::isMultiplexingEnabled()) { + $muxPersistTime = config('constants.ssh.mux_persist_time'); $ssh_command .= "-o ControlMaster=auto -o ControlPath=$muxSocket -o ControlPersist={$muxPersistTime} "; self::ensureMultiplexedConnection($server); - ray('Using SSH Multiplexing')->green(); - } else { - ray('Not using SSH Multiplexing')->red(); } - if (data_get($server, 'settings.is_cloudflare_tunnel')) { - $ssh_command .= '-o ProxyCommand="/usr/local/bin/cloudflared access ssh --hostname %h" '; - } + self::addCloudflareProxyCommand($ssh_command, $server); + + $ssh_command .= self::getCommonSshOptions($server, $sshKeyLocation, $connectionTimeout, $serverInterval); $command = "PATH=\$PATH:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/host/usr/local/sbin:/host/usr/local/bin:/host/usr/sbin:/host/usr/bin:/host/sbin:/host/bin && $command"; $delimiter = Hash::make($command); $command = str_replace($delimiter, '', $command); - $ssh_command .= "-i {$sshKeyLocation} " + $ssh_command .= "{$server->user}@{$server->ip} 'bash -se' << \\$delimiter".PHP_EOL + .$command.PHP_EOL + .$delimiter; + + return $ssh_command; + } + + private static function isMultiplexingEnabled(): bool + { + return config('constants.ssh.mux_enabled') && !config('coolify.is_windows_docker_desktop'); + } + + private static function validateSshKey(string $sshKeyLocation): void + { + $checkKeyCommand = "ls $sshKeyLocation 2>/dev/null"; + $keyCheckProcess = Process::run($checkKeyCommand); + + if ($keyCheckProcess->exitCode() !== 0) { + throw new \RuntimeException("SSH key file not accessible: $sshKeyLocation"); + } + } + + private static function addCloudflareProxyCommand(string &$command, Server $server): void + { + if (data_get($server, 'settings.is_cloudflare_tunnel')) { + $command .= '-o ProxyCommand="/usr/local/bin/cloudflared access ssh --hostname %h" '; + } + } + + private static function getCommonSshOptions(Server $server, string $sshKeyLocation, int $connectionTimeout, int $serverInterval): string + { + return "-i {$sshKeyLocation} " .'-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null ' .'-o PasswordAuthentication=no ' ."-o ConnectTimeout=$connectionTimeout " ."-o ServerAliveInterval=$serverInterval " .'-o RequestTTY=no ' .'-o LogLevel=ERROR ' - ."-p {$server->port} " - ."{$server->user}@{$server->ip} " - ." 'bash -se' << \\$delimiter".PHP_EOL - .$command.PHP_EOL - .$delimiter; - - return $ssh_command; + ."-p {$server->port} "; } } \ No newline at end of file diff --git a/app/Jobs/ServerCheckJob.php b/app/Jobs/ServerCheckJob.php index 540085385..ed31a8c8f 100644 --- a/app/Jobs/ServerCheckJob.php +++ b/app/Jobs/ServerCheckJob.php @@ -43,7 +43,7 @@ class ServerCheckJob implements ShouldBeEncrypted, ShouldQueue return isDev() ? 1 : 3; } - public function __construct(public Server $server) {} + public function __construct(public Server $server, public bool $isManualCheck = false) {} public function middleware(): array { @@ -58,6 +58,9 @@ class ServerCheckJob implements ShouldBeEncrypted, ShouldQueue public function handle() { try { + // Enable SSH multiplexing for autonomous checks, disable for manual checks + config()->set('constants.ssh.mux_enabled', !$this->isManualCheck); + $this->applications = $this->server->applications(); $this->databases = $this->server->databases(); $this->services = $this->server->services()->get(); @@ -93,7 +96,7 @@ class ServerCheckJob implements ShouldBeEncrypted, ShouldQueue private function serverStatus() { - ['uptime' => $uptime] = $this->server->validateConnection(); + ['uptime' => $uptime] = $this->server->validateConnection($this->isManualCheck); if ($uptime) { if ($this->server->unreachable_notification_sent === true) { $this->server->update(['unreachable_notification_sent' => false]); diff --git a/app/Livewire/Project/Shared/GetLogs.php b/app/Livewire/Project/Shared/GetLogs.php index deccc875c..b48ee7e23 100644 --- a/app/Livewire/Project/Shared/GetLogs.php +++ b/app/Livewire/Project/Shared/GetLogs.php @@ -17,6 +17,7 @@ use App\Models\StandalonePostgresql; use App\Models\StandaloneRedis; use Illuminate\Support\Facades\Process; use Livewire\Component; +use App\Helpers\SshMultiplexingHelper; class GetLogs extends Component { @@ -108,14 +109,14 @@ class GetLogs extends Component $command = parseCommandsByLineForSudo(collect($command), $this->server); $command = $command[0]; } - $sshCommand = generateSshCommand($this->server, $command); + $sshCommand = SshMultiplexingHelper::generateSshCommand($this->server, $command); } else { $command = "docker logs -n {$this->numberOfLines} -t {$this->container}"; if ($this->server->isNonRoot()) { $command = parseCommandsByLineForSudo(collect($command), $this->server); $command = $command[0]; } - $sshCommand = generateSshCommand($this->server, $command); + $sshCommand = SshMultiplexingHelper::generateSshCommand($this->server, $command); } } else { if ($this->server->isSwarm()) { @@ -124,14 +125,14 @@ class GetLogs extends Component $command = parseCommandsByLineForSudo(collect($command), $this->server); $command = $command[0]; } - $sshCommand = generateSshCommand($this->server, $command); + $sshCommand = SshMultiplexingHelper::generateSshCommand($this->server, $command); } else { $command = "docker logs -n {$this->numberOfLines} {$this->container}"; if ($this->server->isNonRoot()) { $command = parseCommandsByLineForSudo(collect($command), $this->server); $command = $command[0]; } - $sshCommand = generateSshCommand($this->server, $command); + $sshCommand = SshMultiplexingHelper::generateSshCommand($this->server, $command); } } if ($refresh) { diff --git a/app/Models/Server.php b/app/Models/Server.php index cae910257..6eb9acf55 100644 --- a/app/Models/Server.php +++ b/app/Models/Server.php @@ -967,9 +967,10 @@ $schema://$host { return data_get($this, 'settings.is_swarm_worker'); } - public function validateConnection() + public function validateConnection($isManualCheck = true) { - config()->set('constants.ssh.mux_enabled', false); + // Set mux_enabled to true for automatic checks, false for manual checks + config()->set('constants.ssh.mux_enabled', !$isManualCheck); $server = Server::find($this->id); if (! $server) { @@ -979,7 +980,6 @@ $schema://$host { return ['uptime' => false, 'error' => 'Server skipped.']; } try { - // EC2 does not have `uptime` command, lol instant_remote_process(['ls /'], $server); $server->settings()->update([ 'is_reachable' => true, @@ -988,7 +988,6 @@ $schema://$host { 'unreachable_count' => 0, ]); if (data_get($server, 'unreachable_notification_sent') === true) { - // $server->team?->notify(new Revived($server)); $server->update(['unreachable_notification_sent' => false]); } diff --git a/app/Traits/ExecuteRemoteCommand.php b/app/Traits/ExecuteRemoteCommand.php index 9b58882eb..03726f095 100644 --- a/app/Traits/ExecuteRemoteCommand.php +++ b/app/Traits/ExecuteRemoteCommand.php @@ -7,6 +7,7 @@ use App\Models\Server; use Carbon\Carbon; use Illuminate\Support\Collection; use Illuminate\Support\Facades\Process; +use App\Helpers\SshMultiplexingHelper; trait ExecuteRemoteCommand { @@ -42,7 +43,7 @@ trait ExecuteRemoteCommand $command = parseLineForSudo($command, $this->server); } } - $remote_command = generateSshCommand($this->server, $command); + $remote_command = SshMultiplexingHelper::generateSshCommand($this->server, $command); $process = Process::timeout(3600)->idleTimeout(3600)->start($remote_command, function (string $type, string $output) use ($command, $hidden, $customType, $append) { $output = str($output)->trim(); if ($output->startsWith('╔')) { diff --git a/bootstrap/helpers/remoteProcess.php b/bootstrap/helpers/remoteProcess.php index 988deaee3..5263ea970 100644 --- a/bootstrap/helpers/remoteProcess.php +++ b/bootstrap/helpers/remoteProcess.php @@ -35,8 +35,8 @@ function remote_process( $command_string = implode("\n", $command); - if (auth()->user()) { - $teams = auth()->user()->teams->pluck('id'); + if (Auth::check()) { + $teams = Auth::user()->teams->pluck('id'); if (!$teams->contains($server->team_id) && !$teams->contains(0)) { throw new \Exception('User is not part of the team that owns this server'); } @@ -58,15 +58,10 @@ function remote_process( ])(); } -function generateScpCommand(Server $server, string $source, string $dest) -{ - return SshMultiplexingHelper::generateScpCommand($server, $source, $dest); -} - function instant_scp(string $source, string $dest, Server $server, $throwError = true) { $timeout = config('constants.ssh.command_timeout'); - $scp_command = generateScpCommand($server, $source, $dest); + $scp_command = SshMultiplexingHelper::generateScpCommand($server, $source, $dest); $process = Process::timeout($timeout)->run($scp_command); $output = trim($process->output()); $exitCode = $process->exitCode(); @@ -84,16 +79,8 @@ function instant_scp(string $source, string $dest, Server $server, $throwError = return $output; } -function generateSshCommand(Server $server, string $command) -{ - return SshMultiplexingHelper::generateSshCommand($server, $command); -} - function instant_remote_process(Collection|array $command, Server $server, bool $throwError = true, bool $no_sudo = false): ?string { - static $processCount = 0; - $processCount++; - $timeout = config('constants.ssh.command_timeout'); if ($command instanceof Collection) { $command = $command->toArray(); @@ -104,7 +91,7 @@ function instant_remote_process(Collection|array $command, Server $server, bool $command_string = implode("\n", $command); $start_time = microtime(true); - $sshCommand = generateSshCommand($server, $command_string); + $sshCommand = SshMultiplexingHelper::generateSshCommand($server, $command_string); $process = Process::timeout($timeout)->run($sshCommand); $end_time = microtime(true); @@ -222,11 +209,6 @@ function remove_iip($text) return preg_replace('/\x1b\[[0-9;]*m/', '', $text); } -function remove_mux_file(Server $server) -{ - SshMultiplexingHelper::removeMuxFile($server); -} - function refresh_server_connection(?PrivateKey $private_key = null) { if (is_null($private_key)) {