feat: rolling update

This commit is contained in:
Andras Bacsai
2023-08-21 18:00:12 +02:00
parent a3f3470137
commit bed959f1cd
19 changed files with 263 additions and 118 deletions

View File

@@ -0,0 +1,53 @@
<?php
namespace App\Jobs;
use App\Models\Application;
use App\Models\ApplicationPreview;
use App\Notifications\Application\StatusChanged;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldBeUnique;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
class ApplicationContainerStatusJob implements ShouldQueue, ShouldBeUnique
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
public string $containerName;
public function __construct(
public Application $application,
public int $pullRequestId = 0)
{
$this->containerName = generateApplicationContainerName($application->uuid, $pullRequestId);
}
public function uniqueId(): string
{
return $this->containerName;
}
public function handle(): void
{
try {
$status = getApplicationContainerStatus(application: $this->application);
if ($this->application->status === 'running' && $status !== 'running') {
$this->application->environment->project->team->notify(new StatusChanged($this->application));
}
if ($this->pullRequestId !== 0) {
$preview = ApplicationPreview::findPreviewByApplicationAndPullId($this->application->id, $this->pullRequestId);
$preview->status = $status;
$preview->save();
} else {
$this->application->status = $status;
$this->application->save();
}
} catch (\Exception $e) {
ray($e->getMessage());
}
}
}

View File

@@ -50,6 +50,7 @@ class ApplicationDeploymentJob implements ShouldQueue
private ApplicationPreview|null $preview = null;
private string $container_name;
private string|null $currently_running_container_name = null;
private string $workdir;
private string $configuration_dir;
private string $build_workdir;
@@ -86,7 +87,7 @@ class ApplicationDeploymentJob implements ShouldQueue
$this->build_workdir = "{$this->workdir}" . rtrim($this->application->base_directory, '/');
$this->is_debug_enabled = $this->application->settings->is_debug_enabled;
$this->container_name = generate_container_name($this->application->uuid, $this->pull_request_id);
$this->container_name = generateApplicationContainerName($this->application->uuid, $this->pull_request_id);
$this->private_key_location = save_private_key_for_server($this->server);
$this->saved_outputs = collect();
@@ -113,6 +114,10 @@ class ApplicationDeploymentJob implements ShouldQueue
public function handle(): void
{
// ray()->measure();
$containers = getCurrentApplicationContainerStatus($this->server, $this->application->id);
if ($containers->count() > 0) {
$this->currently_running_container_name = data_get($containers[0], 'Names');
}
$this->application_deployment_queue->update([
'status' => ApplicationDeploymentStatus::IN_PROGRESS->value,
]);
@@ -175,9 +180,9 @@ class ApplicationDeploymentJob implements ShouldQueue
$this->generate_build_env_variables();
$this->add_build_env_variables_to_dockerfile();
$this->build_image();
$this->stop_running_container();
$this->start_by_compose_file();
$this->rolling_update();
}
private function deploy()
{
$this->execute_remote_command(
@@ -206,8 +211,7 @@ class ApplicationDeploymentJob implements ShouldQueue
"echo 'Docker Image found locally with the same Git Commit SHA {$this->application->uuid}:{$this->commit}. Build step skipped...'"
]);
$this->generate_compose_file();
$this->stop_running_container();
$this->start_by_compose_file();
$this->rolling_update();
return;
}
}
@@ -219,8 +223,54 @@ class ApplicationDeploymentJob implements ShouldQueue
$this->generate_build_env_variables();
$this->add_build_env_variables_to_dockerfile();
$this->build_image();
$this->stop_running_container();
$this->rolling_update();
}
private function rolling_update()
{
$this->start_by_compose_file();
$this->health_check();
$this->stop_running_container();
}
private function health_check()
{
ray('New container name: ',$this->container_name);
if ($this->container_name) {
$counter = 0;
$this->execute_remote_command(
[
"echo 'Waiting for health check to pass on the new version of your application.'"
],
);
while ($counter < $this->application->health_check_retries) {
$this->execute_remote_command(
[
"echo 'Attempt {$counter} of {$this->application->health_check_retries}'"
],
[
"docker inspect --format='{{json .State.Health.Status}}' {$this->container_name}",
"hidden" => true,
"save" => "health_check"
],
);
$this->execute_remote_command(
[
"echo 'New application version health check status: {$this->saved_outputs->get('health_check')}'"
],
);
if (Str::of($this->saved_outputs->get('health_check'))->contains('healthy')) {
$this->execute_remote_command(
[
"echo 'Rolling update completed.'"
],
);
break;
}
$counter++;
sleep($this->application->health_check_interval);
}
}
}
private function deploy_pull_request()
{
@@ -241,8 +291,7 @@ class ApplicationDeploymentJob implements ShouldQueue
// $this->generate_build_env_variables();
// $this->add_build_env_variables_to_dockerfile();
$this->build_image();
$this->stop_running_container();
$this->start_by_compose_file();
$this->rolling_update();
}
private function prepare_builder_image()
@@ -409,7 +458,7 @@ class ApplicationDeploymentJob implements ShouldQueue
$this->container_name => [
'image' => $this->production_image_name,
'container_name' => $this->container_name,
'restart' => 'always',
'restart' => RESTART_MODE,
'environment' => $environment_variables,
'labels' => $this->set_labels_for_applications(),
'expose' => $ports,
@@ -539,8 +588,8 @@ class ApplicationDeploymentJob implements ShouldQueue
$schema = $url->getScheme();
$slug = Str::slug($host . $path);
$http_label = "{$this->application->uuid}-{$slug}-http";
$https_label = "{$this->application->uuid}-{$slug}-https";
$http_label = "{$this->container_name}-{$slug}-http";
$https_label = "{$this->container_name}-{$slug}-https";
if ($schema === 'https') {
// Set labels for https
@@ -647,23 +696,22 @@ COPY ./nginx.conf /etc/nginx/conf.d/default.conf");
private function stop_running_container()
{
$this->execute_remote_command(
["echo -n 'Removing old running application.'"],
[$this->execute_in_builder("docker rm -f $this->container_name >/dev/null 2>&1"), "hidden" => true],
);
if ($this->currently_running_container_name) {
$this->execute_remote_command(
["echo -n 'Removing old application version.'"],
[$this->execute_in_builder("docker rm -f $this->currently_running_container_name >/dev/null 2>&1"), "hidden" => true],
);
}
}
private function start_by_compose_file()
{
$this->execute_remote_command(
["echo -n 'Starting new application... '"],
["echo -n 'Rolling update started.'"],
[$this->execute_in_builder("docker compose --project-directory {$this->workdir} up -d >/dev/null"), "hidden" => true],
["echo 'Done. 🎉'"],
);
}
private function generate_build_env_variables()
{
$this->build_args = collect(["--build-arg SOURCE_COMMIT={$this->commit}"]);

View File

@@ -1,54 +0,0 @@
<?php
namespace App\Jobs;
use App\Models\ApplicationPreview;
use App\Notifications\Application\StatusChanged;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldBeUnique;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
class ContainerStatusJob implements ShouldQueue, ShouldBeUnique
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
public string $container_name;
public string|null $pull_request_id;
public $resource;
public function __construct($resource, string $container_name, string|null $pull_request_id = null)
{
$this->resource = $resource;
$this->container_name = $container_name;
$this->pull_request_id = $pull_request_id;
}
public function uniqueId(): string
{
return $this->container_name;
}
public function handle(): void
{
try {
$status = get_container_status(server: $this->resource->destination->server, container_id: $this->container_name, throwError: false);
if ($this->resource->status === 'running' && $status !== 'running') {
$this->resource->environment->project->team->notify(new StatusChanged($this->resource));
}
if ($this->pull_request_id) {
$preview = ApplicationPreview::findPreviewByApplicationAndPullId($this->resource->id, $this->pull_request_id);
$preview->status = $status;
$preview->save();
} else {
$this->resource->status = $status;
$this->resource->save();
}
} catch (\Exception $e) {
ray($e->getMessage());
}
}
}

View File

@@ -0,0 +1,52 @@
<?php
namespace App\Jobs;
use App\Models\ApplicationPreview;
use App\Models\StandalonePostgresql;
use App\Notifications\Application\StatusChanged;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldBeUnique;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
class DatabaseContainerStatusJob implements ShouldQueue, ShouldBeUnique
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
public string $containerName;
public function __construct(
public StandalonePostgresql $database,
) {
$this->containerName = $database->uuid;
}
public function uniqueId(): string
{
return $this->containerName;
}
public function handle(): void
{
try {
$status = getContainerStatus(
server: $this->database->destination->server,
container_id: $this->containerName,
throwError: false
);
if ($this->database->status === 'running' && $status !== 'running') {
$this->database->environment->project->team->notify(new StatusChanged($this->database));
}
if ($this->database->status !== $status) {
$this->database->status = $status;
$this->database->save();
}
} catch (\Exception $e) {
ray($e->getMessage());
}
}
}

View File

@@ -24,7 +24,7 @@ class ProxyCheckJob implements ShouldQueue
$container_name = 'coolify-proxy';
$servers = Server::isUsable()->whereNotNull('proxy')->get();
foreach ($servers as $server) {
$status = get_container_status(server: $server, container_id: $container_name);
$status = getContainerStatus(server: $server, container_id: $container_name);
if ($status === 'running') {
continue;
}

View File

@@ -39,7 +39,7 @@ class ProxyContainerStatusJob implements ShouldQueue, ShouldBeUnique
public function handle(): void
{
try {
$container = get_container_status(server: $this->server, all_data: true, container_id: 'coolify-proxy', throwError: true);
$container = getContainerStatus(server: $this->server, all_data: true, container_id: 'coolify-proxy', throwError: true);
$status = $container['State']['Status'];
if ($this->server->proxy->status !== $status) {
$this->server->proxy->status = $status;

View File

@@ -23,7 +23,7 @@ class ProxyStartJob implements ShouldQueue
try {
$container_name = 'coolify-proxy';
ray('Starting proxy for server: ' . $this->server->name);
$status = get_container_status(server: $this->server, container_id: $container_name);
$status = getContainerStatus(server: $this->server, container_id: $container_name);
if ($status === 'running') {
return;
}

View File

@@ -3,6 +3,7 @@
namespace App\Jobs;
use App\Models\Application;
use App\Models\StandalonePostgresql;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldBeUnique;
use Illuminate\Contracts\Queue\ShouldQueue;
@@ -15,19 +16,25 @@ class ResourceStatusJob implements ShouldQueue, ShouldBeUnique
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
public $applications;
public $postgresqls;
public function __construct()
{
$this->applications = Application::all();
$this->postgresqls = StandalonePostgresql::all();
}
public function handle(): void
{
try {
foreach ($this->applications as $application) {
dispatch(new ContainerStatusJob(
resource: $application,
container_name: generate_container_name($application->uuid),
dispatch(new ApplicationContainerStatusJob(
application: $application,
));
}
foreach ($this->postgresqls as $postgresql) {
dispatch(new DatabaseContainerStatusJob(
database: $postgresql,
));
}
} catch (\Exception $e) {