From d4798a3b2270bcde01559802e08f0ace37463d86 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Wed, 24 May 2023 20:34:40 +0200 Subject: [PATCH 01/23] fix: more aggressive cleanup --- apps/api/src/index.ts | 93 +++++++++++++++++++------------------- apps/api/src/lib/common.ts | 2 +- package.json | 4 +- 3 files changed, 50 insertions(+), 49 deletions(-) diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts index 18d2e6fc4..e0f57be5d 100644 --- a/apps/api/src/index.ts +++ b/apps/api/src/index.ts @@ -604,53 +604,54 @@ async function cleanupStorage() { if (!destination.remoteVerified) continue; enginesDone.add(destination.remoteIpAddress); } - let lowDiskSpace = false; - try { - let stdout = null; - if (!isDev) { - const output = await executeCommand({ - dockerId: destination.id, - command: `CONTAINER=$(docker ps -lq | head -1) && docker exec $CONTAINER sh -c 'df -kPT /'`, - shell: true - }); - stdout = output.stdout; - } else { - const output = await executeCommand({ - command: `df -kPT /` - }); - stdout = output.stdout; - } - let lines = stdout.trim().split('\n'); - let header = lines[0]; - let regex = - /^Filesystem\s+|Type\s+|1024-blocks|\s+Used|\s+Available|\s+Capacity|\s+Mounted on\s*$/g; - const boundaries = []; - let match; + await cleanupDockerStorage(destination.id); + // let lowDiskSpace = false; + // try { + // let stdout = null; + // if (!isDev) { + // const output = await executeCommand({ + // dockerId: destination.id, + // command: `CONTAINER=$(docker ps -lq | head -1) && docker exec $CONTAINER sh -c 'df -kPT /'`, + // shell: true + // }); + // stdout = output.stdout; + // } else { + // const output = await executeCommand({ + // command: `df -kPT /` + // }); + // stdout = output.stdout; + // } + // let lines = stdout.trim().split('\n'); + // let header = lines[0]; + // let regex = + // /^Filesystem\s+|Type\s+|1024-blocks|\s+Used|\s+Available|\s+Capacity|\s+Mounted on\s*$/g; + // const boundaries = []; + // let match; - while ((match = regex.exec(header))) { - boundaries.push(match[0].length); - } + // while ((match = regex.exec(header))) { + // boundaries.push(match[0].length); + // } - boundaries[boundaries.length - 1] = -1; - const data = lines.slice(1).map((line) => { - const cl = boundaries.map((boundary) => { - const column = boundary > 0 ? line.slice(0, boundary) : line; - line = line.slice(boundary); - return column.trim(); - }); - return { - capacity: Number.parseInt(cl[5], 10) / 100 - }; - }); - if (data.length > 0) { - const { capacity } = data[0]; - if (capacity > 0.8) { - lowDiskSpace = true; - } - } - } catch (error) {} - if (lowDiskSpace) { - await cleanupDockerStorage(destination.id); - } + // boundaries[boundaries.length - 1] = -1; + // const data = lines.slice(1).map((line) => { + // const cl = boundaries.map((boundary) => { + // const column = boundary > 0 ? line.slice(0, boundary) : line; + // line = line.slice(boundary); + // return column.trim(); + // }); + // return { + // capacity: Number.parseInt(cl[5], 10) / 100 + // }; + // }); + // if (data.length > 0) { + // const { capacity } = data[0]; + // if (capacity > 0.8) { + // lowDiskSpace = true; + // } + // } + // } catch (error) {} + // if (lowDiskSpace) { + // await cleanupDockerStorage(destination.id); + // } } } diff --git a/apps/api/src/lib/common.ts b/apps/api/src/lib/common.ts index dd17f1c05..a781bc0cb 100644 --- a/apps/api/src/lib/common.ts +++ b/apps/api/src/lib/common.ts @@ -19,7 +19,7 @@ import { saveBuildLog } from './buildPacks/common'; import { scheduler } from './scheduler'; import type { ExecaChildProcess } from 'execa'; -export const version = '3.12.31'; +export const version = '3.12.32'; export const isDev = process.env.NODE_ENV === 'development'; export const proxyPort = process.env.COOLIFY_PROXY_PORT; export const proxySecurePort = process.env.COOLIFY_PROXY_SECURE_PORT; diff --git a/package.json b/package.json index c4ce72c9d..691c42a48 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "coolify", "description": "An open-source & self-hostable Heroku / Netlify alternative.", - "version": "3.12.31", + "version": "3.12.32", "license": "Apache-2.0", "repository": "github:coollabsio/coolify", "scripts": { @@ -50,4 +50,4 @@ "open-source", "coolify" ] -} +} \ No newline at end of file From f30f23af59fa95f4ac4549282f04771e05bf507d Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Wed, 24 May 2023 20:51:33 +0200 Subject: [PATCH 02/23] fix: restart storage volumes --- apps/api/src/routes/api/v1/applications/handlers.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/apps/api/src/routes/api/v1/applications/handlers.ts b/apps/api/src/routes/api/v1/applications/handlers.ts index 8621a93ca..da10e119b 100644 --- a/apps/api/src/routes/api/v1/applications/handlers.ts +++ b/apps/api/src/routes/api/v1/applications/handlers.ts @@ -640,8 +640,7 @@ export async function restartApplication( const volumes = persistentStorage?.map((storage) => { - return `${applicationId}${storage.path.replace(/\//gi, '-')}:${buildPack !== 'docker' ? '/app' : '' - }${storage.path}`; + return `${applicationId}${storage.path.replace(/\//gi, '-')}:${storage.path}`; }) || []; const composeVolumes = volumes.map((volume) => { return { From e6063fb93ac0a09477c23cc624ef83604d84038b Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Wed, 24 May 2023 21:55:24 +0200 Subject: [PATCH 03/23] fix: force delete stucked destinations --- apps/api/src/lib/common.ts | 230 ++++++++++-------- .../routes/api/v1/destinations/handlers.ts | 31 +++ .../src/routes/api/v1/destinations/index.ts | 3 +- .../routes/destinations/[id]/__layout.svelte | 31 ++- 4 files changed, 185 insertions(+), 110 deletions(-) diff --git a/apps/api/src/lib/common.ts b/apps/api/src/lib/common.ts index a781bc0cb..366cce212 100644 --- a/apps/api/src/lib/common.ts +++ b/apps/api/src/lib/common.ts @@ -579,7 +579,8 @@ export async function executeCommand({ stream = false, buildId, applicationId, - debug + debug, + timeout = 0 }: { command: string; sshCommand?: boolean; @@ -589,6 +590,7 @@ export async function executeCommand({ buildId?: string; applicationId?: string; debug?: boolean; + timeout?: number; }): Promise> { const { execa, execaCommand } = await import('execa'); const { parse } = await import('shell-quote'); @@ -613,20 +615,26 @@ export async function executeCommand({ } if (sshCommand) { if (shell) { - return execaCommand(`ssh ${remoteIpAddress}-remote ${command}`); + return execaCommand(`ssh ${remoteIpAddress}-remote ${command}`, { + timeout + }); } - return await execa('ssh', [`${remoteIpAddress}-remote`, dockerCommand, ...dockerArgs]); + return await execa('ssh', [`${remoteIpAddress}-remote`, dockerCommand, ...dockerArgs], { + timeout + }); } if (stream) { return await new Promise(async (resolve, reject) => { let subprocess = null; if (shell) { subprocess = execaCommand(command, { - env: { DOCKER_BUILDKIT: '1', DOCKER_HOST: engine } + env: { DOCKER_BUILDKIT: '1', DOCKER_HOST: engine }, + timeout }); } else { subprocess = execa(dockerCommand, dockerArgs, { - env: { DOCKER_BUILDKIT: '1', DOCKER_HOST: engine } + env: { DOCKER_BUILDKIT: '1', DOCKER_HOST: engine }, + timeout }); } const logs = []; @@ -680,19 +688,26 @@ export async function executeCommand({ } else { if (shell) { return await execaCommand(command, { - env: { DOCKER_BUILDKIT: '1', DOCKER_HOST: engine } + env: { DOCKER_BUILDKIT: '1', DOCKER_HOST: engine }, + timeout }); } else { return await execa(dockerCommand, dockerArgs, { - env: { DOCKER_BUILDKIT: '1', DOCKER_HOST: engine } + env: { DOCKER_BUILDKIT: '1', DOCKER_HOST: engine }, + timeout }); } } } else { if (shell) { - return execaCommand(command, { shell: true }); + return execaCommand(command, { + shell: true, + timeout + }); } - return await execa(dockerCommand, dockerArgs); + return await execa(dockerCommand, dockerArgs, { + timeout + }); } } @@ -849,97 +864,97 @@ export function generatePassword({ type DatabaseConfiguration = | { - volume: string; - image: string; - command?: string; - ulimits: Record; - privatePort: number; - environmentVariables: { - MYSQL_DATABASE: string; - MYSQL_PASSWORD: string; - MYSQL_ROOT_USER: string; - MYSQL_USER: string; - MYSQL_ROOT_PASSWORD: string; - }; - } + volume: string; + image: string; + command?: string; + ulimits: Record; + privatePort: number; + environmentVariables: { + MYSQL_DATABASE: string; + MYSQL_PASSWORD: string; + MYSQL_ROOT_USER: string; + MYSQL_USER: string; + MYSQL_ROOT_PASSWORD: string; + }; + } | { - volume: string; - image: string; - command?: string; - ulimits: Record; - privatePort: number; - environmentVariables: { - MONGO_INITDB_ROOT_USERNAME?: string; - MONGO_INITDB_ROOT_PASSWORD?: string; - MONGODB_ROOT_USER?: string; - MONGODB_ROOT_PASSWORD?: string; - }; - } + volume: string; + image: string; + command?: string; + ulimits: Record; + privatePort: number; + environmentVariables: { + MONGO_INITDB_ROOT_USERNAME?: string; + MONGO_INITDB_ROOT_PASSWORD?: string; + MONGODB_ROOT_USER?: string; + MONGODB_ROOT_PASSWORD?: string; + }; + } | { - volume: string; - image: string; - command?: string; - ulimits: Record; - privatePort: number; - environmentVariables: { - MARIADB_ROOT_USER: string; - MARIADB_ROOT_PASSWORD: string; - MARIADB_USER: string; - MARIADB_PASSWORD: string; - MARIADB_DATABASE: string; - }; - } + volume: string; + image: string; + command?: string; + ulimits: Record; + privatePort: number; + environmentVariables: { + MARIADB_ROOT_USER: string; + MARIADB_ROOT_PASSWORD: string; + MARIADB_USER: string; + MARIADB_PASSWORD: string; + MARIADB_DATABASE: string; + }; + } | { - volume: string; - image: string; - command?: string; - ulimits: Record; - privatePort: number; - environmentVariables: { - POSTGRES_PASSWORD?: string; - POSTGRES_USER?: string; - POSTGRES_DB?: string; - POSTGRESQL_POSTGRES_PASSWORD?: string; - POSTGRESQL_USERNAME?: string; - POSTGRESQL_PASSWORD?: string; - POSTGRESQL_DATABASE?: string; - }; - } + volume: string; + image: string; + command?: string; + ulimits: Record; + privatePort: number; + environmentVariables: { + POSTGRES_PASSWORD?: string; + POSTGRES_USER?: string; + POSTGRES_DB?: string; + POSTGRESQL_POSTGRES_PASSWORD?: string; + POSTGRESQL_USERNAME?: string; + POSTGRESQL_PASSWORD?: string; + POSTGRESQL_DATABASE?: string; + }; + } | { - volume: string; - image: string; - command?: string; - ulimits: Record; - privatePort: number; - environmentVariables: { - REDIS_AOF_ENABLED: string; - REDIS_PASSWORD: string; - }; - } + volume: string; + image: string; + command?: string; + ulimits: Record; + privatePort: number; + environmentVariables: { + REDIS_AOF_ENABLED: string; + REDIS_PASSWORD: string; + }; + } | { - volume: string; - image: string; - command?: string; - ulimits: Record; - privatePort: number; - environmentVariables: { - COUCHDB_PASSWORD: string; - COUCHDB_USER: string; - }; - } + volume: string; + image: string; + command?: string; + ulimits: Record; + privatePort: number; + environmentVariables: { + COUCHDB_PASSWORD: string; + COUCHDB_USER: string; + }; + } | { - volume: string; - image: string; - command?: string; - ulimits: Record; - privatePort: number; - environmentVariables: { - EDGEDB_SERVER_PASSWORD: string; - EDGEDB_SERVER_USER: string; - EDGEDB_SERVER_DATABASE: string; - EDGEDB_SERVER_TLS_CERT_MODE: string; - }; - }; + volume: string; + image: string; + command?: string; + ulimits: Record; + privatePort: number; + environmentVariables: { + EDGEDB_SERVER_PASSWORD: string; + EDGEDB_SERVER_USER: string; + EDGEDB_SERVER_DATABASE: string; + EDGEDB_SERVER_TLS_CERT_MODE: string; + }; + }; export function generateDatabaseConfiguration(database: any): DatabaseConfiguration { const { id, dbUser, dbUserPassword, rootUser, rootUserPassword, defaultDatabase, version, type } = database; @@ -1038,9 +1053,8 @@ export function generateDatabaseConfiguration(database: any): DatabaseConfigurat }; if (isARM()) { configuration.volume = `${id}-${type}-data:/data`; - configuration.command = `/usr/local/bin/redis-server --appendonly ${ - appendOnly ? 'yes' : 'no' - } --requirepass ${dbUserPassword}`; + configuration.command = `/usr/local/bin/redis-server --appendonly ${appendOnly ? 'yes' : 'no' + } --requirepass ${dbUserPassword}`; } return configuration; } else if (type === 'couchdb') { @@ -1125,12 +1139,12 @@ export type ComposeFileService = { command?: string; ports?: string[]; build?: - | { - context: string; - dockerfile: string; - args?: Record; - } - | string; + | { + context: string; + dockerfile: string; + args?: Record; + } + | string; deploy?: { restart_policy?: { condition?: string; @@ -1201,7 +1215,7 @@ export const createDirectories = async ({ let workdirFound = false; try { workdirFound = !!(await fs.stat(workdir)); - } catch (error) {} + } catch (error) { } if (workdirFound) { await executeCommand({ command: `rm -fr ${workdir}` }); } @@ -1728,7 +1742,7 @@ export async function stopBuild(buildId, applicationId) { } } count++; - } catch (error) {} + } catch (error) { } }, 100); }); } @@ -1751,7 +1765,7 @@ export async function cleanupDockerStorage(dockerId) { // Cleanup images that are not used by any container try { await executeCommand({ dockerId, command: `docker image prune -af` }); - } catch (error) {} + } catch (error) { } // Prune coolify managed containers try { @@ -1759,12 +1773,12 @@ export async function cleanupDockerStorage(dockerId) { dockerId, command: `docker container prune -f --filter "label=coolify.managed=true"` }); - } catch (error) {} + } catch (error) { } // Cleanup build caches try { await executeCommand({ dockerId, command: `docker builder prune -af` }); - } catch (error) {} + } catch (error) { } } export function persistentVolumes(id, persistentStorage, config) { diff --git a/apps/api/src/routes/api/v1/destinations/handlers.ts b/apps/api/src/routes/api/v1/destinations/handlers.ts index e64ac211d..e72b46b01 100644 --- a/apps/api/src/routes/api/v1/destinations/handlers.ts +++ b/apps/api/src/routes/api/v1/destinations/handlers.ts @@ -18,6 +18,7 @@ import type { Proxy, SaveDestinationSettings } from './types'; +import { removeService } from '../../../../lib/services/common'; export async function listDestinations(request: FastifyRequest) { try { @@ -143,6 +144,35 @@ export async function newDestination(request: FastifyRequest, re return errorHandler({ status, message }); } } +export async function forceDeleteDestination(request: FastifyRequest) { + try { + const { id } = request.params; + const services = await prisma.service.findMany({ where: { destinationDockerId: id } }); + for (const service of services) { + await removeService({ id: service.id }); + } + const applications = await prisma.application.findMany({ where: { destinationDockerId: id } }); + for (const application of applications) { + await prisma.applicationSettings.deleteMany({ where: { application: { id: application.id } } }); + await prisma.buildLog.deleteMany({ where: { applicationId: application.id } }); + await prisma.build.deleteMany({ where: { applicationId: application.id } }); + await prisma.secret.deleteMany({ where: { applicationId: application.id } }); + await prisma.applicationPersistentStorage.deleteMany({ where: { applicationId: application.id } }); + await prisma.applicationConnectedDatabase.deleteMany({ where: { applicationId: application.id } }); + await prisma.previewApplication.deleteMany({ where: { applicationId: application.id } }); + } + const databases = await prisma.database.findMany({ where: { destinationDockerId: id } }); + for (const database of databases) { + await prisma.databaseSettings.deleteMany({ where: { databaseId: database.id } }); + await prisma.databaseSecret.deleteMany({ where: { databaseId: database.id } }); + await prisma.database.delete({ where: { id: database.id } }); + } + await prisma.destinationDocker.delete({ where: { id } }); + return {}; + } catch ({ status, message }) { + return errorHandler({ status, message }); + } +} export async function deleteDestination(request: FastifyRequest) { try { const { id } = request.params; @@ -318,6 +348,7 @@ export async function verifyRemoteDockerEngineFn(id: string) { } await prisma.destinationDocker.update({ where: { id }, data: { remoteVerified: true } }); } catch (error) { + console.log(error) throw new Error('Error while verifying remote docker engine'); } } diff --git a/apps/api/src/routes/api/v1/destinations/index.ts b/apps/api/src/routes/api/v1/destinations/index.ts index 774afa285..704c2aa79 100644 --- a/apps/api/src/routes/api/v1/destinations/index.ts +++ b/apps/api/src/routes/api/v1/destinations/index.ts @@ -1,5 +1,5 @@ import { FastifyPluginAsync } from 'fastify'; -import { assignSSHKey, checkDestination, deleteDestination, getDestination, getDestinationStatus, listDestinations, newDestination, restartProxy, saveDestinationSettings, startProxy, stopProxy, verifyRemoteDockerEngine } from './handlers'; +import { assignSSHKey, checkDestination, deleteDestination, forceDeleteDestination, getDestination, getDestinationStatus, listDestinations, newDestination, restartProxy, saveDestinationSettings, startProxy, stopProxy, verifyRemoteDockerEngine } from './handlers'; import type { OnlyId } from '../../../../types'; import type { CheckDestination, ListDestinations, NewDestination, Proxy, SaveDestinationSettings } from './types'; @@ -14,6 +14,7 @@ const root: FastifyPluginAsync = async (fastify): Promise => { fastify.get('/:id', async (request) => await getDestination(request)); fastify.post('/:id', async (request, reply) => await newDestination(request, reply)); fastify.delete('/:id', async (request) => await deleteDestination(request)); + fastify.delete('/:id/force', async (request) => await forceDeleteDestination(request)); fastify.get('/:id/status', async (request) => await getDestinationStatus(request)); fastify.post('/:id/settings', async (request) => await saveDestinationSettings(request)); diff --git a/apps/ui/src/routes/destinations/[id]/__layout.svelte b/apps/ui/src/routes/destinations/[id]/__layout.svelte index 475dbc342..a0f80137f 100644 --- a/apps/ui/src/routes/destinations/[id]/__layout.svelte +++ b/apps/ui/src/routes/destinations/[id]/__layout.svelte @@ -75,6 +75,25 @@ } } } + async function forceDeleteDestination(destination: any) { + let sure = confirm($t('application.confirm_to_delete', { name: destination.name })); + if (sure) { + sure = confirm( + 'Are you REALLY sure? This will delete all resources associated with this destination, but not on the destination (server) itself. You will have manually delete everything on the server afterwards.' + ); + if (sure) { + sure = confirm('REALLY?'); + if (sure) { + try { + await del(`/destinations/${destination.id}/force`, { id: destination.id }); + return await goto('/', { replaceState: true }); + } catch (error) { + return errorNotification(error); + } + } + } + } + } function deletable() { if (!isDestinationDeletable) { return 'Please delete all resources before deleting this.'; @@ -88,7 +107,7 @@ {#if $page.params.id !== 'new'} -