Compare commits

...

10 Commits

Author SHA1 Message Date
Andras Bacsai
e8cadc176b Merge pull request #1468 from coollabsio/next
v4.0.0-beta.143
2023-11-17 15:21:29 +01:00
Andras Bacsai
b0c96e64c9 Fix server unreachable notification count 2023-11-17 15:18:08 +01:00
Andras Bacsai
9ce3b43e09 Add Team model and merge servers with own servers 2023-11-17 15:11:29 +01:00
Andras Bacsai
4c2b3df861 Update server runtime and comments 2023-11-17 14:56:39 +01:00
Andras Bacsai
467471f54a Fix server readiness check in ContainerStatusJob and ServerStatusJob 2023-11-17 14:46:04 +01:00
Andras Bacsai
60171093c5 Update version to 4.0.0-beta.143 2023-11-17 14:43:57 +01:00
Andras Bacsai
38f2a2dac7 Merge pull request #1467 from coollabsio/next
v4.0.0-beta.142
2023-11-17 14:32:32 +01:00
Andras Bacsai
307ee52ac0 wtf 2023-11-17 14:29:37 +01:00
Andras Bacsai
b66c9835b7 Fix server status check and add new job 2023-11-17 14:22:05 +01:00
Andras Bacsai
d38d50dca2 Fix server readiness check and update version
number
2023-11-17 14:14:13 +01:00
12 changed files with 54 additions and 51 deletions

View File

@@ -11,6 +11,7 @@ use App\Jobs\ServerStatusJob;
use App\Models\InstanceSettings;
use App\Models\ScheduledDatabaseBackup;
use App\Models\Server;
use App\Models\Team;
use Illuminate\Console\Scheduling\Schedule;
use Illuminate\Foundation\Console\Kernel as ConsoleKernel;
@@ -52,6 +53,8 @@ class Kernel extends ConsoleKernel
{
if (isCloud()) {
$servers = Server::all()->whereNotNull('team.subscription')->where('team.subscription.stripe_trial_already_ended', false)->where('ip', '!=', '1.2.3.4');
$own = Team::find(0)->servers;
$servers = $servers->merge($own);
} else {
$servers = Server::all()->where('ip', '!=', '1.2.3.4');
}

View File

@@ -4,6 +4,7 @@ namespace App\Http\Livewire\Project\Application;
use App\Actions\Application\StopApplication;
use App\Jobs\ContainerStatusJob;
use App\Jobs\ServerStatusJob;
use App\Models\Application;
use Livewire\Component;
use Visus\Cuid2\Cuid2;
@@ -28,6 +29,8 @@ class Heading extends Component
$this->application->previews->each(function ($preview) {
$preview->refresh();
});
} else {
dispatch(new ServerStatusJob($this->application->destination->server));
}
}

View File

@@ -42,28 +42,31 @@ class Rollback extends Component
{
try {
$image = $this->application->uuid;
$output = instant_remote_process([
"docker inspect --format='{{.Config.Image}}' {$this->application->uuid}",
], $this->application->destination->server, throwError: false);
$current_tag = Str::of($output)->trim()->explode(":");
$this->current = data_get($current_tag, 1);
if ($this->application->destination->server->isFunctional()) {
$output = instant_remote_process([
"docker inspect --format='{{.Config.Image}}' {$this->application->uuid}",
], $this->application->destination->server, throwError: false);
$current_tag = Str::of($output)->trim()->explode(":");
$this->current = data_get($current_tag, 1);
$output = instant_remote_process([
"docker images --format '{{.Repository}}#{{.Tag}}#{{.CreatedAt}}'",
], $this->application->destination->server);
$this->images = Str::of($output)->trim()->explode("\n")->filter(function ($item) use ($image) {
return Str::of($item)->contains($image);
})->map(function ($item) {
$item = Str::of($item)->explode('#');
if ($item[1] === $this->current) {
// $is_current = true;
}
return [
'tag' => $item[1],
'created_at' => $item[2],
'is_current' => $is_current ?? null,
];
})->toArray();
$output = instant_remote_process([
"docker images --format '{{.Repository}}#{{.Tag}}#{{.CreatedAt}}'",
], $this->application->destination->server);
$this->images = Str::of($output)->trim()->explode("\n")->filter(function ($item) use ($image) {
return Str::of($item)->contains($image);
})->map(function ($item) {
$item = Str::of($item)->explode('#');
if ($item[1] === $this->current) {
// $is_current = true;
}
return [
'tag' => $item[1],
'created_at' => $item[2],
'is_current' => $is_current ?? null,
];
})->toArray();
}
return [];
} catch (\Throwable $e) {
return handleError($e, $this);
}

View File

@@ -1022,10 +1022,6 @@ COPY ./nginx.conf /etc/nginx/conf.d/default.conf");
listen [::]:80;
server_name localhost;
// real_ip_header X-Forwarded-For;
// proxy_set_header X-Real-IP \$remote_addr;
// proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for;
location / {
root /usr/share/nginx/html;
index index.html;

View File

@@ -23,7 +23,7 @@ class ContainerStatusJob implements ShouldQueue, ShouldBeEncrypted
public function __construct(public Server $server)
{
// $this->handle();
$this->handle();
}
public function middleware(): array
{
@@ -37,11 +37,11 @@ class ContainerStatusJob implements ShouldQueue, ShouldBeEncrypted
public function handle(): void
{
// ray("checking server status for {$this->server->id}");
ray("checking container statuses for {$this->server->id}");
try {
if (!$this->server->checkServerRediness()) {
if (!$this->server->isServerReady()) {
return;
}
};
$containers = instant_remote_process(["docker container ls -q"], $this->server);
if (!$containers) {
return;
@@ -268,7 +268,7 @@ class ContainerStatusJob implements ShouldQueue, ShouldBeEncrypted
$this->server->team->notify(new ContainerStopped($containerName, $this->server, $url));
}
} catch (\Throwable $e) {
send_internal_notification('ContainerStatusJob failed with: ' . $e->getMessage());
send_internal_notification("ContainerStatusJob failed on ({$this->server->id}) with: " . $e->getMessage());
ray($e->getMessage());
handleError($e);
}

View File

@@ -34,11 +34,9 @@ class ServerStatusJob implements ShouldQueue, ShouldBeEncrypted
{
ray("checking server status for {$this->server->id}");
try {
if (!$this->server->checkServerRediness()) {
ray('server not ready');
return;
if ($this->server->isServerReady()) {
$this->cleanup(notify: false);
}
$this->cleanup(notify: false);
} catch (\Throwable $e) {
send_internal_notification('ServerStatusJob failed with: ' . $e->getMessage());
ray($e->getMessage());

View File

@@ -128,18 +128,17 @@ class Server extends BaseModel
}
return false;
}
public function checkServerRediness()
public function isServerReady()
{
if ($this->skipServer()) {
return false;
}
$serverUptimeCheckNumber = $this->unreachable_count;
$serverUptimeCheckNumberMax = 5;
$serverUptimeCheckNumberMax = 3;
$currentTime = now()->timestamp;
$runtime5Minutes = 1 * 60;
// Run for 1 minutes max and check every 5 seconds
while ($currentTime + $runtime5Minutes > now()->timestamp) {
$runtime = 30;
$isReady = false;
// Run for 30 seconds max and check every 5 seconds for 3 times
while ($currentTime + $runtime > now()->timestamp) {
if ($serverUptimeCheckNumber >= $serverUptimeCheckNumberMax) {
if ($this->unreachable_notification_sent === false) {
ray('Server unreachable, sending notification...');
@@ -168,7 +167,8 @@ class Server extends BaseModel
$db->update(['status' => 'exited']);
}
}
return false;
$isReady = false;
break;
}
$result = $this->validateConnection();
ray('validateConnection: ' . $result);
@@ -180,9 +180,10 @@ class Server extends BaseModel
Sleep::for(5)->seconds();
return;
}
$isReady = true;
break;
}
return true;
return $isReady;
}
public function getDiskUsage()
{

View File

@@ -52,7 +52,6 @@ class User extends Authenticatable implements SendsEmail
}
public function createToken(string $name, array $abilities = ['*'], DateTimeInterface $expiresAt = null)
{
ray('asd');
$plainTextToken = sprintf(
'%s%s%s',
config('sanctum.token_prefix', ''),

View File

@@ -43,7 +43,7 @@ class Unreachable extends Notification implements ShouldQueue
public function toMail(): MailMessage
{
$mail = new MailMessage();
$mail->subject("Coolify: Server ({$this->server->name}) is unreachable after trying to connect to it 5 times");
$mail->subject("Coolify: Server ({$this->server->name}) is unreachable after trying to connect to it 3 times");
$mail->view('emails.server-lost-connection', [
'name' => $this->server->name,
]);
@@ -52,13 +52,13 @@ class Unreachable extends Notification implements ShouldQueue
public function toDiscord(): string
{
$message = "Coolify: Server '{$this->server->name}' is unreachable after trying to connect to it 5 times. All automations & integrations are turned off! Please check your server! IMPORTANT: We automatically try to revive your server. If your server is back online, we will automatically turn on all automations & integrations.";
$message = "Coolify: Server '{$this->server->name}' is unreachable after trying to connect to it 3 times. All automations & integrations are turned off! Please check your server! IMPORTANT: We automatically try to revive your server. If your server is back online, we will automatically turn on all automations & integrations.";
return $message;
}
public function toTelegram(): array
{
return [
"message" => "Coolify: Server '{$this->server->name}' is unreachable after trying to connect to it 5 times. All automations & integrations are turned off! Please check your server! IMPORTANT: We automatically try to revive your server. If your server is back online, we will automatically turn on all automations & integrations."
"message" => "Coolify: Server '{$this->server->name}' is unreachable after trying to connect to it 3 times. All automations & integrations are turned off! Please check your server! IMPORTANT: We automatically try to revive your server. If your server is back online, we will automatically turn on all automations & integrations."
];
}
}

View File

@@ -7,7 +7,7 @@ return [
// The release version of your application
// Example with dynamic git hash: trim(exec('git --git-dir ' . base_path('.git') . ' log --pretty="%h" -n1 HEAD'))
'release' => '4.0.0-beta.141',
'release' => '4.0.0-beta.143',
// When left empty or `null` the Laravel environment will be used
'environment' => config('app.env'),

View File

@@ -1,3 +1,3 @@
<?php
return '4.0.0-beta.141';
return '4.0.0-beta.143';

View File

@@ -4,7 +4,7 @@
"version": "3.12.36"
},
"v4": {
"version": "4.0.0-beta.141"
"version": "4.0.0-beta.143"
}
}
}