Merge pull request #882 from coollabsio/next

v3.12.18

.github/workflows/pocketbase-release.yml

@@ -5,7 +5,9 @@ on:
     paths:
       - "others/pocketbase/*"
       - ".github/workflows/pocketbase-release.yml"
-
+    branches:
+      - next
+      - main
 jobs:
   arm64:
     runs-on: [self-hosted, arm64]

.github/workflows/production-release.yml

@@ -54,7 +54,7 @@ jobs:
           context: .
           platforms: linux/amd64
           push: true
-          tags: coollabsio/coolify:${{steps.package-version.outputs.current-version}}-amd64
+          tags: coollabsio/coolify:${{steps.package-version.outputs.current-version}}
           cache-from: type=registry,ref=coollabsio/coolify:buildcache-amd64
           cache-to: type=registry,ref=coollabsio/coolify:buildcache-amd64,mode=max
   aarch64:
@@ -103,10 +103,10 @@ jobs:
         id: package-version
       - name: Create & publish manifest
         run: |
-          docker manifest create coollabsio/coolify:${{steps.package-version.outputs.current-version}} --amend coollabsio/coolify:${{steps.package-version.outputs.current-version}}-amd64 --amend coollabsio/coolify:${{steps.package-version.outputs.current-version}}-arm64 --amend coollabsio/coolify:${{steps.package-version.outputs.current-version}}-aarch64
-          docker manifest create coollabsio/coolify:latest --amend coollabsio/coolify:${{steps.package-version.outputs.current-version}}-amd64 --amend coollabsio/coolify:${{steps.package-version.outputs.current-version}}-arm64 --amend coollabsio/coolify:${{steps.package-version.outputs.current-version}}-aarch64
-          docker manifest push coollabsio/coolify:${{steps.package-version.outputs.current-version}}
-           docker manifest push coollabsio/coolify:latest
+          docker buildx imagetools create --append coollabsio/coolify:${{steps.package-version.outputs.current-version}}-arm64 --append coollabsio/coolify:${{steps.package-version.outputs.current-version}}-aarch64 --tag coollabsio/coolify:${{steps.package-version.outputs.current-version}}
+          docker tag coollabsio/coolify:${{steps.package-version.outputs.current-version}} coollabsio/coolify:latest
+          docker push coollabsio/coolify:latest
+          docker buildx imagetools create --append coollabsio/coolify:${{steps.package-version.outputs.current-version}}-arm64 --append coollabsio/coolify:${{steps.package-version.outputs.current-version}}-aarch64 --tag coollabsio/coolify:latest
       - uses: sarisia/actions-status-discord@v1
         if: always()
         with:

.github/workflows/staging-release.yml

@@ -65,7 +65,7 @@ jobs:
           context: .
           platforms: linux/amd64
           push: true
-          tags: coollabsio/coolify:next-amd64
+          tags: coollabsio/coolify:next
           cache-from: type=registry,ref=coollabsio/coolify:buildcache-next-amd64
           cache-to: type=registry,ref=coollabsio/coolify:buildcache-next-amd64,mode=max
   merge-manifest:
@@ -85,8 +85,7 @@ jobs:
           password: ${{ secrets.DOCKERHUB_TOKEN }}
       - name: Create & publish manifest
         run: |
-          docker manifest create coollabsio/coolify:next --amend coollabsio/coolify:next-amd64 --amend coollabsio/coolify:next-arm64
-          docker manifest push coollabsio/coolify:next
+          docker buildx imagetools create --append coollabsio/coolify:next-arm64 --tag coollabsio/coolify:next
       - uses: sarisia/actions-status-discord@v1
         if: always()
         with:

apps/api/src/index.ts

@@ -171,6 +171,11 @@ const host = '0.0.0.0';
 			await cleanupStorage();
 		}, 60000 * 15);
 
+		// Cleanup stucked containers (not defined in Coolify, but still running and managed by Coolify)
+		setInterval(async () => {
+			await cleanupStuckedContainers();
+		}, 60000);
+
 		// checkProxies, checkFluentBit & refresh templates
 		setInterval(async () => {
 			await checkProxies();
@@ -197,7 +202,13 @@ const host = '0.0.0.0';
 			await copySSLCertificates();
 		}, 10000);
 
-		await Promise.all([getTagsTemplates(), getArch(), getIPAddress(), configureRemoteDockers()]);
+		await Promise.all([
+			getTagsTemplates(),
+			getArch(),
+			getIPAddress(),
+			configureRemoteDockers()
+			// cleanupStuckedContainers()
+		]);
 	} catch (error) {
 		console.error(error);
 		process.exit(1);
@@ -311,6 +322,49 @@ async function getArch() {
 	} catch (error) {}
 }
 
+async function cleanupStuckedContainers() {
+	try {
+		const destinationDockers = await prisma.destinationDocker.findMany();
+		let enginesDone = new Set();
+		for (const destination of destinationDockers) {
+			if (enginesDone.has(destination.engine) || enginesDone.has(destination.remoteIpAddress))
+				return;
+			if (destination.engine) {
+				enginesDone.add(destination.engine);
+			}
+			if (destination.remoteIpAddress) {
+				if (!destination.remoteVerified) continue;
+				enginesDone.add(destination.remoteIpAddress);
+			}
+			const { stdout: containers } = await executeCommand({
+				dockerId: destination.id,
+				command: `docker container ps -a --filter "label=coolify.managed=true" --format '{{ .Names}}'`
+			});
+			if (containers) {
+				const containersArray = containers.trim().split('\n');
+				if (containersArray.length > 0) {
+					for (const container of containersArray) {
+						const containerId = container.split('-')[0];
+						const application = await prisma.application.findFirst({
+							where: { id: { startsWith: containerId } }
+						});
+						const service = await prisma.service.findFirst({
+							where: { id: { startsWith: containerId } }
+						});
+						const database = await prisma.database.findFirst({
+							where: { id: { startsWith: containerId } }
+						});
+						if (!application && !service && !database) {
+							await executeCommand({ command: `docker container rm -f ${container}` });
+						}
+					}
+				}
+			}
+		}
+	} catch (error) {
+		console.log(error);
+	}
+}
 async function configureRemoteDockers() {
 	try {
 		const remoteDocker = await prisma.destinationDocker.findMany({
@@ -543,9 +597,13 @@ async function cleanupStorage() {
 	let enginesDone = new Set();
 	for (const destination of destinationDockers) {
 		if (enginesDone.has(destination.engine) || enginesDone.has(destination.remoteIpAddress)) return;
-		if (destination.engine) enginesDone.add(destination.engine);
-		if (destination.remoteIpAddress) enginesDone.add(destination.remoteIpAddress);
-		let force = false;
+		if (destination.engine) {
+			enginesDone.add(destination.engine);
+		}
+		if (destination.remoteIpAddress) {
+			if (!destination.remoteVerified) continue;
+			enginesDone.add(destination.remoteIpAddress);
+		}
 		let lowDiskSpace = false;
 		try {
 			let stdout = null;
@@ -591,6 +649,8 @@ async function cleanupStorage() {
 				}
 			}
 		} catch (error) {}
-		await cleanupDockerStorage(destination.id, lowDiskSpace, force);
+		if (lowDiskSpace) {
+			await cleanupDockerStorage(destination.id);
+		}
 	}
 }

apps/api/src/lib/buildPacks/compose.ts

@@ -66,10 +66,12 @@ export default async function (data) {
 				}
 			}
 		}
+    if (build.length > 0 || buildArgs.length > 0 ) {
 		value['build'] = {
 			...build,
 			args: finalArgs
 		};
+    }
 
 		value['labels'] = labels;
 		// TODO: If we support separated volume for each service, we need to add it here
@@ -116,7 +118,6 @@ export default async function (data) {
 	dockerComposeYaml['networks'] = Object.assign({ ...networks }, { [network]: { external: true } });
 
 	await fs.writeFile(fileYaml, yaml.dump(dockerComposeYaml));
-	console.log(yaml.dump(dockerComposeYaml));
 	await executeCommand({
 		debug,
 		buildId,

apps/api/src/lib/common.ts

@@ -19,7 +19,7 @@ import { saveBuildLog, saveDockerRegistryCredentials } from './buildPacks/common
 import { scheduler } from './scheduler';
 import type { ExecaChildProcess } from 'execa';
 
-export const version = '3.12.13';
+export const version = '3.12.18';
 export const isDev = process.env.NODE_ENV === 'development';
 export const sentryDSN =
 	'https://409f09bcb7af47928d3e0f46b78987f3@o1082494.ingest.sentry.io/4504236622217216';
@@ -1714,78 +1714,24 @@ export function convertTolOldVolumeNames(type) {
 	}
 }
 
-export async function cleanupDockerStorage(dockerId, lowDiskSpace, force) {
-	// Cleanup old coolify images
+export async function cleanupDockerStorage(dockerId) {
+	// Cleanup images that are not used by any container
 	try {
-		let { stdout: images } = await executeCommand({
-			dockerId,
-			command: `docker images coollabsio/coolify --filter before="coollabsio/coolify:${version}" -q | xargs -r`,
-			shell: true
-		});
-
-		images = images.trim();
-		if (images) {
-			await executeCommand({
-				dockerId,
-				command: `docker rmi -f ${images}" -q | xargs -r`,
-				shell: true
-			});
-		}
+		await executeCommand({ dockerId, command: `docker image prune -af` });
 	} catch (error) {}
-	if (lowDiskSpace || force) {
-		// Cleanup images that are not used
-		try {
-			await executeCommand({ dockerId, command: `docker image prune -f` });
-		} catch (error) {}
 
-		const { numberOfDockerImagesKeptLocally } = await prisma.setting.findUnique({
-			where: { id: '0' }
-		});
-		const { stdout: images } = await executeCommand({
+	// Prune coolify managed containers
+	try {
+		await executeCommand({
 			dockerId,
-			command: `docker images|grep -v "<none>"|grep -v REPOSITORY|awk '{print $1, $2}'`,
-			shell: true
+			command: `docker container prune -f --filter "label=coolify.managed=true"`
 		});
-		const imagesArray = images.trim().replaceAll(' ', ':').split('\n');
-		const imagesSet = new Set(imagesArray.map((image) => image.split(':')[0]));
-		let deleteImage = [];
-		for (const image of imagesSet) {
-			let keepImage = [];
-			for (const image2 of imagesArray) {
-				if (image2.startsWith(image)) {
-					if (force) {
-						deleteImage.push(image2);
-						continue;
-					}
-					if (keepImage.length >= numberOfDockerImagesKeptLocally) {
-						deleteImage.push(image2);
-					} else {
-						keepImage.push(image2);
-					}
-				}
-			}
-		}
-		for (const image of deleteImage) {
-			try {
-				await executeCommand({ dockerId, command: `docker image rm -f ${image}` });
-			} catch (error) {
-				console.log(error);
-			}
-		}
+	} catch (error) {}
 
-		// Prune coolify managed containers
-		try {
-			await executeCommand({
-				dockerId,
-				command: `docker container prune -f --filter "label=coolify.managed=true"`
-			});
-		} catch (error) {}
-
-		// Cleanup build caches
-		try {
-			await executeCommand({ dockerId, command: `docker builder prune -a -f` });
-		} catch (error) {}
-	}
+	// Cleanup build caches
+	try {
+		await executeCommand({ dockerId, command: `docker builder prune -af` });
+	} catch (error) {}
 }
 
 export function persistentVolumes(id, persistentStorage, config) {

apps/api/src/routes/api/v1/applications/handlers.ts

@@ -732,14 +732,15 @@ export async function deleteApplication(
 ) {
 	try {
 		const { id } = request.params;
-		const { force } = request.body;
-
 		const { teamId } = request.user;
 		const application = await prisma.application.findUnique({
 			where: { id },
-			include: { destinationDocker: true }
+			include: { destinationDocker: true, teams: true }
 		});
-		if (!force && application?.destinationDockerId && application.destinationDocker?.network) {
+		if (!application.teams.find((team) => team.id === teamId) || teamId !== '0') {
+			throw { status: 403, message: 'You are not allowed to delete this application.' };
+		}
+		if (application?.destinationDocker?.id && application.destinationDocker?.network) {
 			const { stdout: containers } = await executeCommand({
 				dockerId: application.destinationDocker.id,
 				command: `docker ps -a --filter network=${application.destinationDocker.network} --filter name=${id} --format '{{json .}}'`

apps/api/src/routes/api/v1/databases/handlers.ts

@@ -427,19 +427,15 @@ export async function deleteDatabase(request: FastifyRequest<DeleteDatabase>) {
 	try {
 		const teamId = request.user.teamId;
 		const { id } = request.params;
-		const { force } = request.body;
 		const database = await prisma.database.findFirst({
 			where: { id, teams: { some: { id: teamId === '0' ? undefined : teamId } } },
 			include: { destinationDocker: true, settings: true }
 		});
-		if (!force) {
-			if (database.dbUserPassword) database.dbUserPassword = decrypt(database.dbUserPassword);
-			if (database.rootUserPassword) database.rootUserPassword = decrypt(database.rootUserPassword);
-			if (database.destinationDockerId) {
-				const everStarted = await stopDatabaseContainer(database);
-				if (everStarted)
-					await stopTcpHttpProxy(id, database.destinationDocker, database.publicPort);
-			}
+		if (database.dbUserPassword) database.dbUserPassword = decrypt(database.dbUserPassword);
+		if (database.rootUserPassword) database.rootUserPassword = decrypt(database.rootUserPassword);
+		if (database.destinationDockerId) {
+			const everStarted = await stopDatabaseContainer(database);
+			if (everStarted) await stopTcpHttpProxy(id, database.destinationDocker, database.publicPort);
 		}
 		await prisma.databaseSettings.deleteMany({ where: { databaseId: id } });
 		await prisma.databaseSecret.deleteMany({ where: { databaseId: id } });

apps/api/src/routes/api/v1/databases/types.ts

@@ -4,7 +4,7 @@ export interface SaveDatabaseType extends OnlyId {
     Body: { type: string }
 }
 export interface DeleteDatabase extends OnlyId {
-    Body: { force: string }
+    Body: { }
 }
 export interface SaveVersion extends OnlyId {
     Body: {

apps/api/src/routes/api/v1/handlers.ts

@@ -59,7 +59,7 @@ export async function cleanupManually(request: FastifyRequest) {
 		const destination = await prisma.destinationDocker.findUnique({
 			where: { id: serverId }
 		});
-		await cleanupDockerStorage(destination.id, true, true);
+		await cleanupDockerStorage(destination.id);
 		return {};
 	} catch ({ status, message }) {
 		return errorHandler({ status, message });

apps/api/src/routes/api/v1/services/handlers.ts

@@ -617,6 +617,29 @@ export async function getServiceLogs(request: FastifyRequest<GetServiceLogs>) {
 export async function deleteService(request: FastifyRequest<OnlyId>) {
 	try {
 		const { id } = request.params;
+		const teamId = request.user.teamId;
+		const { destinationDockerId } = await getServiceFromDB({ id, teamId });
+		if (destinationDockerId) {
+			const { stdout: containers } = await executeCommand({
+				dockerId: destinationDockerId,
+				command: `docker ps -a --filter 'label=com.docker.compose.project=${id}' --format {{.ID}}`
+			});
+			if (containers) {
+				const containerArray = containers.split('\n');
+				if (containerArray.length > 0) {
+					for (const container of containerArray) {
+						await executeCommand({
+							dockerId: destinationDockerId,
+							command: `docker stop -t 0 ${container}`
+						});
+						await executeCommand({
+							dockerId: destinationDockerId,
+							command: `docker rm --force ${container}`
+						});
+					}
+				}
+			}
+		}
 		await removeService({ id });
 		return {};
 	} catch ({ status, message }) {

apps/api/src/routes/webhooks/traefik/handlers.ts

@@ -2,6 +2,7 @@ import { FastifyRequest } from 'fastify';
 import { errorHandler, getDomain, isDev, prisma, executeCommand } from '../../../lib/common';
 import { getTemplates } from '../../../lib/services';
 import { OnlyId } from '../../../types';
+import { parseAndFindServiceTemplates } from '../../api/v1/services/handlers';
 
 function generateServices(serviceId, containerId, port, isHttp2 = false, isHttps = false) {
 	if (isHttp2) {
@@ -50,6 +51,9 @@ function generateRouters(
 	isHttp2 = false
 ) {
 	let rule = `Host(\`${nakedDomain}\`)${pathPrefix ? ` && PathPrefix(\`${pathPrefix}\`)` : ''}`;
+	let ruleWWW = `Host(\`www.${nakedDomain}\`)${
+		pathPrefix ? ` && PathPrefix(\`${pathPrefix}\`)` : ''
+	}`;
 	let http: any = {
 		entrypoints: ['web'],
 		rule,
@@ -69,14 +73,14 @@ function generateRouters(
 	};
 	let httpWWW: any = {
 		entrypoints: ['web'],
-		rule,
+		rule: ruleWWW,
 		service: `${serviceId}`,
 		priority: 2,
 		middlewares: []
 	};
 	let httpsWWW: any = {
 		entrypoints: ['websecure'],
-		rule,
+		rule: ruleWWW,
 		service: `${serviceId}`,
 		priority: 2,
 		tls: {
@@ -533,10 +537,11 @@ export async function proxyConfiguration(request: FastifyRequest<OnlyId>, remote
 					}
 					found = JSON.parse(JSON.stringify(found).replaceAll('$$id', id));
 					for (const oneService of Object.keys(found.services)) {
-						const isDomainConfiguration =
-							found?.services[oneService]?.proxy?.filter((p) => p.domain) ?? [];
-						if (isDomainConfiguration.length > 0) {
-							const { proxy } = found.services[oneService];
+						const isDomainAndProxyConfiguration =
+							found?.services[oneService]?.proxy?.filter((p) => p.port) ?? [];
+						if (isDomainAndProxyConfiguration.length > 0) {
+							const template: any = await parseAndFindServiceTemplates(service, null, true);
+							const { proxy } = template.services[oneService] || found.services[oneService];
 							for (let configuration of proxy) {
 								if (configuration.domain) {
 									const setting = serviceSetting.find(

apps/api/src/routes/webhooks/traefik/index.ts

@@ -4,9 +4,11 @@ import { proxyConfiguration, otherProxyConfiguration } from './handlers';
 import { OtherProxyConfiguration } from './types';
 
 const root: FastifyPluginAsync = async (fastify): Promise<void> => {
-    fastify.get<OnlyId>('/main.json', async (request, reply) => proxyConfiguration(request, false));
-    fastify.get<OnlyId>('/remote/:id', async (request) => proxyConfiguration(request, true));
-    fastify.get<OtherProxyConfiguration>('/other.json', async (request, reply) => otherProxyConfiguration(request));
+	fastify.get<OnlyId>('/main.json', async (request, reply) => proxyConfiguration(request, false));
+	fastify.get<OnlyId>('/remote/:id', async (request) => proxyConfiguration(request, true));
+	fastify.get<OtherProxyConfiguration>('/other.json', async (request, reply) =>
+		otherProxyConfiguration(request)
+	);
 };
 
 export default root;