feat: New update process (#115)

fix: Docker Engine bugs

docker-compose.yaml

@@ -2,7 +2,7 @@ version: '3.8'
 
 services:
   coolify:
-    image: coollabsio/coolify:latest
+    image: coollabsio/coolify:${TAG:-latest}
     restart: always
     container_name: coolify
     ports:

package.json

@@ -1,7 +1,7 @@
 {
 	"name": "coolify",
 	"description": "An open-source & self-hostable Heroku / Netlify alternative.",
-	"version": "2.0.0",
+	"version": "2.0.4",
 	"license": "AGPL-3.0",
 	"scripts": {
 		"dev": "docker-compose -f docker-compose-dev.yaml up -d && NODE_ENV=development svelte-kit dev --host 0.0.0.0",
@@ -16,6 +16,7 @@
 		"db:generate": "prisma generate",
 		"db:push": "prisma db push && prisma generate",
 		"db:seed": "prisma db seed",
+		"stagrelease": "cross-var docker build -t coollabsio/coolify:$npm_package_version . && docker push coollabsio/coolify:$npm_package_version",
 		"prerelease": "cross-var docker build -t coollabsio/coolify:$npm_package_version -t coollabsio/coolify:latest .",
 		"release:coolify": "cross-var yarn prerelease && docker push coollabsio/coolify:$npm_package_version && docker image push coollabsio/coolify:$npm_package_version && docker push coollabsio/coolify:latest",
 		"release:haproxy": "docker build -f haproxy.Dockerfile -t coollabsio/coolify-haproxy-alpine:1.0.0 -t coollabsio/coolify-haproxy-alpine:latest . && docker image push --all-tags coollabsio/coolify-haproxy-alpine",

src/lib/database/common.ts

@@ -37,6 +37,9 @@ if (dev) {
 export const prisma = new PrismaClient(prismaOptions);
 
 export function PrismaErrorHandler(e) {
+	if (e! instanceof Error) {
+		e = new Error(e.toString());
+	}
 	sentry.captureException(e);
 	const payload = {
 		status: e.status || 500,

src/lib/haproxy/index.ts

@@ -481,8 +481,12 @@ export async function startTcpProxy(destinationDocker, id, publicPort, privatePo
 
 	try {
 		if (foundDB && !found) {
+			const { stdout: Config } = await asyncExecShell(
+				`DOCKER_HOST="${host}" docker network inspect bridge --format '{{json .IPAM.Config }}'`
+			);
+			const ip = JSON.parse(Config)[0].Gateway;
 			return await asyncExecShell(
-				`DOCKER_HOST=${host} docker run --restart always -e PORT=${publicPort} -e APP=${id} -e PRIVATE_PORT=${privatePort} --add-host 'host.docker.internal:host-gateway' --network ${network} -p ${publicPort}:${publicPort} --name ${containerName} -d coollabsio/${defaultProxyImageTcp}`
+				`DOCKER_HOST=${host} docker run --restart always -e PORT=${publicPort} -e APP=${id} -e PRIVATE_PORT=${privatePort} --add-host 'host.docker.internal:host-gateway' --add-host 'host.docker.internal:${ip}' --network ${network} -p ${publicPort}:${publicPort} --name ${containerName} -d coollabsio/${defaultProxyImageTcp}`
 			);
 		}
 	} catch (error) {
@@ -499,8 +503,12 @@ export async function startHttpProxy(destinationDocker, id, publicPort, privateP
 
 	try {
 		if (foundDB && !found) {
+			const { stdout: Config } = await asyncExecShell(
+				`DOCKER_HOST="${host}" docker network inspect bridge --format '{{json .IPAM.Config }}'`
+			);
+			const ip = JSON.parse(Config)[0].Gateway;
 			return await asyncExecShell(
-				`DOCKER_HOST=${host} docker run --restart always -e PORT=${publicPort} -e APP=${id} -e PRIVATE_PORT=${privatePort} --add-host 'host.docker.internal:host-gateway' --network ${network} -p ${publicPort}:${publicPort} --name ${containerName} -d coollabsio/${defaultProxyImageHttp}`
+				`DOCKER_HOST=${host} docker run --restart always -e PORT=${publicPort} -e APP=${id} -e PRIVATE_PORT=${privatePort} --add-host 'host.docker.internal:host-gateway' --add-host 'host.docker.internal:${ip}' --network ${network} -p ${publicPort}:${publicPort} --name ${containerName} -d coollabsio/${defaultProxyImageHttp}`
 			);
 		}
 	} catch (error) {
@@ -512,8 +520,12 @@ export async function startCoolifyProxy(engine) {
 	const found = await checkContainer(engine, 'coolify-haproxy');
 	const { proxyPassword, proxyUser } = await db.listSettings();
 	if (!found) {
+		const { stdout: Config } = await asyncExecShell(
+			`DOCKER_HOST="${host}" docker network inspect bridge --format '{{json .IPAM.Config }}'`
+		);
+		const ip = JSON.parse(Config)[0].Gateway;
 		await asyncExecShell(
-			`DOCKER_HOST="${host}" docker run -e HAPROXY_USERNAME=${proxyUser} -e HAPROXY_PASSWORD=${proxyPassword} --restart always --add-host 'host.docker.internal:host-gateway' -v coolify-ssl-certs:/usr/local/etc/haproxy/ssl --network coolify-infra -p "80:80" -p "443:443" -p "8404:8404" -p "5555:5555" -p "5000:5000" --name coolify-haproxy -d coollabsio/${defaultProxyImage}`
+			`DOCKER_HOST="${host}" docker run -e HAPROXY_USERNAME=${proxyUser} -e HAPROXY_PASSWORD=${proxyPassword} --restart always --add-host 'host.docker.internal:host-gateway' --add-host 'host.docker.internal:${ip}' -v coolify-ssl-certs:/usr/local/etc/haproxy/ssl --network coolify-infra -p "80:80" -p "443:443" -p "8404:8404" -p "5555:5555" -p "5000:5000" --name coolify-haproxy -d coollabsio/${defaultProxyImage}`
 		);
 	}
 	await configureNetworkCoolifyProxy(engine);

src/lib/queues/builder.ts

@@ -1,4 +1,5 @@
 import crypto from 'crypto';
+import fs from 'fs/promises';
 import * as buildpacks from '../buildPacks';
 import * as importers from '../importers';
 import { dockerInstance } from '../docker';
@@ -208,10 +209,11 @@ export default async function (job) {
 		if (secrets.length > 0) {
 			secrets.forEach((secret) => {
 				if (!secret.isBuildSecret) {
-					envs.push(`--env ${secret.name}=${secret.value}`);
+					envs.push(`${secret.name}=${secret.value}`);
 				}
 			});
 		}
+		await fs.writeFile(`${workdir}/.env`, envs.join('\n'));
 		const labels = makeLabelForStandaloneApplication({
 			applicationId,
 			fqdn,
@@ -230,14 +232,20 @@ export default async function (job) {
 			baseDirectory,
 			publishDirectory
 		});
+		try {
 			saveBuildLog({ line: 'Deployment started.', buildId, applicationId });
 			const { stderr } = await asyncExecShell(
-			`DOCKER_HOST=${host} docker run ${envs.join()} ${labels.join(
+				`DOCKER_HOST=${host} docker run --env-file=${workdir}/.env ${labels.join(
 					' '
-			)} --name ${imageId} --network ${docker.network} --restart always -d ${applicationId}:${tag}`
+				)} --name ${imageId} --network ${
+					docker.network
+				} --restart always -d ${applicationId}:${tag}`
 			);
 			if (stderr) console.log(stderr);
 			saveBuildLog({ line: 'Deployment successful!', buildId, applicationId });
+		} catch (error) {
+			throw new Error(error);
+		}
 
 		if (destinationDockerId && destinationDocker.isCoolifyProxyUsed) {
 			saveBuildLog({ line: 'Proxy configuration started!', buildId, applicationId });

src/lib/queues/cleanup.ts

@@ -8,13 +8,13 @@ export default async function () {
 		for (const destinationDocker of destinationDockers) {
 			const host = getEngine(destinationDocker.engine);
 			try {
-				await asyncExecShell(`DOCKER_HOST=${host} docker container prune -f`);
+				// await asyncExecShell(`DOCKER_HOST=${host} docker container prune -f`);
 			} catch (error) {
 				//
 				console.log(error);
 			}
 			try {
-				await asyncExecShell(`DOCKER_HOST=${host} docker image prune -f`);
+				// await asyncExecShell(`DOCKER_HOST=${host} docker image prune -f`);
 			} catch (error) {
 				//
 				console.log(error);

src/lib/queues/index.ts

@@ -120,7 +120,11 @@ buildWorker.on('completed', async (job: Bullmq.Job) => {
 	} catch (err) {
 		console.log(err);
 	} finally {
-		await asyncExecShell(`rm -fr ${job.data.workdir}`);
+		const workdir = `/tmp/build-sources/${job.data.repository}/${job.data.build_id}`;
+		await asyncExecShell(`rm -fr ${workdir}`);
+		await asyncExecShell(
+			`test -f /tmp/build-sources/${job.data.repository}/id.rsa && rm /tmp/build-sources/${job.data.repository}/id.rsa`
+		);
 	}
 	return;
 });
@@ -132,7 +136,11 @@ buildWorker.on('failed', async (job: Bullmq.Job, failedReason) => {
 	} catch (error) {
 		console.log(error);
 	} finally {
-		await asyncExecShell(`rm -fr ${job.data.workdir}`);
+		const workdir = `/tmp/build-sources/${job.data.repository}/${job.data.build_id}`;
+		await asyncExecShell(`rm -fr ${workdir}`);
+		await asyncExecShell(
+			`test -f /tmp/build-sources/${job.data.repository}/id.rsa && rm /tmp/build-sources/${job.data.repository}/id.rsa`
+		);
 	}
 	saveBuildLog({ line: 'Failed build!', buildId: job.data.build_id, applicationId: job.data.id });
 	saveBuildLog({

src/routes/__layout.svelte

@@ -39,8 +39,7 @@
 	import { errorNotification } from '$lib/form';
 	import { asyncSleep } from '$lib/components/common';
 	import { del, get, post } from '$lib/api';
-	import { dev } from '$app/env';
+	import { browser } from '$app/env';
-	import Loading from '$lib/components/Loading.svelte';
 
 	let isUpdateAvailable = false;
 	let updateStatus = {
@@ -48,9 +47,10 @@
 		checking: false,
 		success: null
 	};
-
+	let latestVersion = 'latest';
 	onMount(async () => {
 		if ($session.uid) {
+			const overrideVersion = browser && window.localStorage.getItem('latestVersion');
 			try {
 				await get(`/login.json`);
 			} catch ({ error }) {
@@ -62,10 +62,11 @@
 				updateStatus.checking = true;
 				try {
 					const data = await get(`/update.json`);
-					if (data?.isUpdateAvailable) {
+					if (overrideVersion || data?.isUpdateAvailable) {
-						await post(`/update.json`, { type: 'pull' });
+						latestVersion = overrideVersion || data.latestVersion;
+						isUpdateAvailable = overrideVersion ? true : data?.isUpdateAvailable;
+						await post(`/update.json`, { type: 'pull', latestVersion });
 					}
-					isUpdateAvailable = data?.isUpdateAvailable;
 				} catch (error) {
 				} finally {
 					updateStatus.checking = false;
@@ -95,9 +96,9 @@
 
 	async function update() {
 		updateStatus.loading = true;
-		if (!dev) {
+		// if (!dev) {
 		try {
-				await post(`/update.json`, { type: 'update' });
+			await post(`/update.json`, { type: 'update', latestVersion });
 			toast.push('Update completed. Waiting for the new version to start...');
 			let reachable = false;
 			let tries = 0;
@@ -123,26 +124,25 @@
 			updateStatus.success = false;
 			updateStatus.loading = false;
 		}
-		} else {
+		// } else {
-			let reachable = false;
+		// 	let reachable = false;
-			let tries = 0;
+		// 	let tries = 0;
-			do {
+		// 	do {
-				await asyncSleep(1000);
+		// 		await asyncSleep(1000);
-				try {
+		// 		try {
-					await get(`/undead.json`);
+		// 			await get(`/undead.json`);
-					reachable = true;
+		// 			reachable = true;
-				} catch (error) {
+		// 		} catch (error) {
-					console.log(error);
+		// 			console.log(error);
-					reachable = false;
+		// 			reachable = false;
-				}
+		// 		}
-				console.log(reachable);
+		// 		if (reachable) break;
-				if (reachable) break;
+		// 		tries++;
-				tries++;
+		// 	} while (!reachable || tries < 120);
-			} while (!reachable || tries < 120);
+		// 	toast.push('New version reachable. Reloading...');
-			toast.push('New version reachable. Reloading...');
+		// 	await asyncSleep(2000);
-			await asyncSleep(2000);
+		// 	window.location.reload();
-			window.location.reload();
+		// }
-		}
 	}
 </script>
 

src/routes/applications/[id]/logs/build/build.json.ts

@@ -14,12 +14,12 @@ export const get: RequestHandler = async (event) => {
 			where: { buildId, time: { gt: sequence } },
 			orderBy: { time: 'asc' }
 		});
-		const { status } = await db.prisma.build.findFirst({ where: { id: buildId } });
+		const data = await db.prisma.build.findFirst({ where: { id: buildId } });
 
 		return {
 			body: {
 				logs,
-				status
+				status: data?.status || 'running'
 			}
 		};
 	} catch (error) {

src/routes/update.json.ts

@@ -12,7 +12,7 @@ export const get: RequestHandler = async () => {
 		const versions = await got
 			.get(`https://get.coollabs.io/versions.json?appId=${process.env['COOLIFY_APP_ID']}`)
 			.json();
-		const latestVersion = versions['coolify'].main.version;
+		const latestVersion = dev ? '10.0.0' : versions['coolify'].main.version;
 		const isUpdateAvailable = compare(latestVersion, currentVersion);
 		return {
 			body: {
@@ -26,17 +26,17 @@ export const get: RequestHandler = async () => {
 };
 
 export const post: RequestHandler = async (event) => {
-	const { type } = await event.request.json();
+	const { type, latestVersion } = await event.request.json();
 	if (type === 'pull') {
 		try {
 			if (!dev) {
-				await asyncExecShell(`env | grep COOLIFY > .env`);
+				await asyncExecShell(`docker pull coollabsio/coolify:${latestVersion}`);
-				await asyncExecShell(`docker compose pull`);
 				return {
 					status: 200,
 					body: {}
 				};
 			} else {
+				await asyncExecShell(`docker pull coollabsio/coolify:${latestVersion}`);
 				await asyncSleep(2000);
 				return {
 					status: 200,
@@ -49,14 +49,16 @@ export const post: RequestHandler = async (event) => {
 	} else if (type === 'update') {
 		try {
 			if (!dev) {
+				await asyncExecShell(`env | grep COOLIFY > .env`);
 				await asyncExecShell(
-					`docker run --rm -tid --env-file .env -v /var/run/docker.sock:/var/run/docker.sock -v coolify-db coollabsio/coolify:latest /bin/sh -c "env | grep COOLIFY > .env && docker stop -t 0 coolify && docker stop -t 0 coolify-redis && docker compose up -d --force-recreate"`
+					`docker run --rm -tid --env-file .env -v /var/run/docker.sock:/var/run/docker.sock -v coolify-db coollabsio/coolify:${latestVersion} /bin/sh -c "env | grep COOLIFY > .env && echo 'TAG=${latestVersion}' >> .env && docker stop -t 0 coolify coolify-redis && docker rm coolify coolify-redis && docker compose up -d --force-recreate"`
 				);
 				return {
 					status: 200,
 					body: {}
 				};
 			} else {
+				console.log(latestVersion);
 				await asyncSleep(2000);
 				return {
 					status: 200,