Merge pull request #801 from coollabsio/next

v3.12.3

.dockerignore

@@ -1,5 +1,6 @@
 .DS_Store
 node_modules
+.pnpm-store
 build
 .svelte-kit
 package
@@ -7,6 +8,9 @@ package
 .env.*
 !.env.example
 dist
-client
 apps/api/db/*.db
-local-serve
+local-serve
+apps/api/db/migration.db-journal
+apps/api/core*
+logs
+others/certificates

.github/ISSUE_TEMPLATE/--bug-report.yaml

@@ -9,13 +9,21 @@ body:
 - type: markdown
   attributes:
     value: |
-      Thanks for taking the time to fill out this bug report! Please fill the form in English
+      Thanks for taking the time to fill out this bug report! Please fill the form in English.
 - type: checkboxes
   attributes:
     label: Is there an existing issue for this?
     options:
     - label: I have searched the existing issues
       required: true
+- type: input
+  id: repository
+  attributes:
+    label: Example public repository
+    description: "An example public git repository to reproduce the issue easily (if applicable)."
+    placeholder: "ex: https://github.com/coollabsio/coolify"
+  validations:
+    required: false
 - type: textarea
   attributes:
     label: Description

.github/workflows/fluent-bit-release.yml

@@ -0,0 +1,93 @@
+name: fluent-bit-release
+
+on:
+  push:
+    paths:
+      - "others/fluentbit"
+      - ".github/workflows/fluent-bit-release.yml"
+    branches:    
+      - next
+
+jobs:
+  arm64:
+    runs-on: [self-hosted, arm64]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      - name: Login to DockerHub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Build and push
+        uses: docker/build-push-action@v2
+        with:
+          context: others/fluentbit/
+          platforms: linux/arm64
+          push: true
+          tags: coollabsio/coolify-fluent-bit:1.0.0-arm64
+  amd64:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+      - name: Login to DockerHub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Build and push
+        uses: docker/build-push-action@v3
+        with:
+          context: others/fluentbit/
+          platforms: linux/amd64
+          push: true
+          tags: coollabsio/coolify-fluent-bit:1.0.0-amd64
+  aarch64:
+    runs-on: [self-hosted, arm64]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      - name: Login to DockerHub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Build and push
+        uses: docker/build-push-action@v2
+        with:
+          context: others/fluentbit/
+          platforms: linux/aarch64
+          push: true
+          tags: coollabsio/coolify-fluent-bit:1.0.0-aarch64
+  merge-manifest:
+    runs-on: ubuntu-latest
+    needs: [amd64, arm64, aarch64]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+      - name: Login to DockerHub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Create & publish manifest
+        run: |
+          docker manifest create coollabsio/coolify-fluent-bit:1.0.0 --amend coollabsio/coolify-fluent-bit:1.0.0-amd64 --amend coollabsio/coolify-fluent-bit:1.0.0-arm64 --amend coollabsio/coolify-fluent-bit:1.0.0-aarch64
+          docker manifest push coollabsio/coolify-fluent-bit:1.0.0

.github/workflows/pocketbase-release.yml

@@ -0,0 +1,93 @@
+name: pocketbase-release
+
+on:
+  push:
+    paths:
+      - "others/pocketbase"
+      - ".github/workflows/pocketbase-release.yml"
+    branches:
+      - next
+
+jobs:
+  arm64:
+    runs-on: [self-hosted, arm64]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      - name: Login to DockerHub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Build and push
+        uses: docker/build-push-action@v2
+        with:
+          context: others/pocketbase/
+          platforms: linux/arm64
+          push: true
+          tags: coollabsio/pocketbase:0.8.0-arm64
+  amd64:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+      - name: Login to DockerHub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Build and push
+        uses: docker/build-push-action@v3
+        with:
+          context: others/pocketbase/
+          platforms: linux/amd64
+          push: true
+          tags: coollabsio/pocketbase:0.8.0-amd64
+  aarch64:
+    runs-on: [self-hosted, arm64]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      - name: Login to DockerHub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Build and push
+        uses: docker/build-push-action@v2
+        with:
+          context: others/pocketbase/
+          platforms: linux/aarch64
+          push: true
+          tags: coollabsio/pocketbase:0.8.0-aarch64
+  merge-manifest:
+    runs-on: ubuntu-latest
+    needs: [amd64, arm64, aarch64]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+      - name: Login to DockerHub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Create & publish manifest
+        run: |
+          docker manifest create coollabsio/pocketbase:0.8.0 --amend coollabsio/pocketbase:0.8.0-amd64 --amend coollabsio/pocketbase:0.8.0-arm64 --amend coollabsio/pocketbase:0.8.0-aarch64
+          docker manifest push coollabsio/pocketbase:0.8.0

.github/workflows/production-release.yml

@@ -5,7 +5,7 @@ on:
     types: [released]
 
 jobs:
-  arm64-build:
+  arm64:
     runs-on: [self-hosted, arm64]
     steps:
       - name: Checkout
@@ -31,7 +31,7 @@ jobs:
           tags: coollabsio/coolify:${{steps.package-version.outputs.current-version}}-arm64
           cache-from: type=registry,ref=coollabsio/coolify:buildcache-arm64
           cache-to: type=registry,ref=coollabsio/coolify:buildcache-arm64,mode=max
-  amd64-build:
+  amd64:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
@@ -57,9 +57,35 @@ jobs:
           tags: coollabsio/coolify:${{steps.package-version.outputs.current-version}}-amd64
           cache-from: type=registry,ref=coollabsio/coolify:buildcache-amd64
           cache-to: type=registry,ref=coollabsio/coolify:buildcache-amd64,mode=max
+  aarch64:
+    runs-on: [self-hosted, arm64]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      - name: Login to DockerHub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Get current package version
+        uses: martinbeentjes/npm-get-version-action@v1.2.3
+        id: package-version
+      - name: Build and push
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          platforms: linux/aarch64
+          push: true
+          tags: coollabsio/coolify:${{steps.package-version.outputs.current-version}}-aarch64
+          cache-from: type=registry,ref=coollabsio/coolify:buildcache-aarch64
+          cache-to: type=registry,ref=coollabsio/coolify:buildcache-aarch64,mode=max
   merge-manifest:
     runs-on: ubuntu-latest
-    needs: [amd64-build, arm64-build]
+    needs: [amd64, arm64, aarch64]
     steps:
       - name: Checkout
         uses: actions/checkout@v3
@@ -77,7 +103,7 @@ jobs:
         id: package-version
       - name: Create & publish manifest
         run: |
-          docker manifest create coollabsio/coolify:${{steps.package-version.outputs.current-version}} --amend coollabsio/coolify:${{steps.package-version.outputs.current-version}}-amd64 --amend coollabsio/coolify:${{steps.package-version.outputs.current-version}}-arm64
+          docker manifest create coollabsio/coolify:${{steps.package-version.outputs.current-version}} --amend coollabsio/coolify:${{steps.package-version.outputs.current-version}}-amd64 --amend coollabsio/coolify:${{steps.package-version.outputs.current-version}}-arm64 --amend coollabsio/coolify:${{steps.package-version.outputs.current-version}}-aarch64
           docker manifest push coollabsio/coolify:${{steps.package-version.outputs.current-version}}
       - uses: sarisia/actions-status-discord@v1
         if: always()

.github/workflows/staging-release.yml

@@ -2,11 +2,17 @@ name: staging-release
 
 on:
   push:
+    paths:
+      - "**"
+      - "!others/fluentbit"
+      - "!others/pocketbase"
+      - "!.github/workflows/fluent-bit-release.yml"
+      - "!.github/workflows/pocketbase-release.yml"
     branches:
       - next
 
 jobs:
-  arm64-making-something-cool:
+  arm64:
     runs-on: [self-hosted, arm64]
     steps:
       - name: Checkout
@@ -34,7 +40,7 @@ jobs:
           tags: coollabsio/coolify:next-arm64
           cache-from: type=registry,ref=coollabsio/coolify:buildcache-next-arm64
           cache-to: type=registry,ref=coollabsio/coolify:buildcache-next-arm64,mode=max
-  amd64-making-something-cool:
+  amd64:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
@@ -59,12 +65,12 @@ jobs:
           context: .
           platforms: linux/amd64
           push: true
-          tags: coollabsio/coolify:next-amd64,coollabsio/coolify:next-test
+          tags: coollabsio/coolify:next-amd64
           cache-from: type=registry,ref=coollabsio/coolify:buildcache-next-amd64
           cache-to: type=registry,ref=coollabsio/coolify:buildcache-next-amd64,mode=max
-  merge-manifest-to-be-cool:
+  merge-manifest:
     runs-on: ubuntu-latest
-    needs: [arm64-making-something-cool, amd64-making-something-cool]
+    needs: [arm64, amd64]
     steps:
       - name: Checkout
         uses: actions/checkout@v3

.gitignore

@@ -8,8 +8,16 @@ package
 .env.*
 !.env.example
 dist
-client
 apps/api/db/*.db
-local-serve
 apps/api/db/migration.db-journal
-apps/api/core*
+apps/api/core*
+apps/backup/backups/*
+!apps/backup/backups/.gitkeep
+logs
+others/certificates
+backups/*
+!backups/.gitkeep
+
+# Trpc
+apps/server/db/*.db
+apps/server/db/*.db-journal

.vscode/settings.json

@@ -1,11 +1,22 @@
 {
-	"i18n-ally.localesPaths": ["src/lib/locales"],
+	"i18n-ally.localesPaths": [
+		"src/lib/locales"
+	],
 	"i18n-ally.keystyle": "nested",
 	"i18n-ally.extract.ignoredByFiles": {
-		"src\\routes\\__layout.svelte": ["Coolify", "coolLabs logo"]
+		"src\\routes\\__layout.svelte": [
+			"Coolify",
+			"coolLabs logo"
+		]
 	},
 	"i18n-ally.sourceLanguage": "en",
-	"i18n-ally.enabledFrameworks": ["svelte"],
-	"i18n-ally.enabledParsers": ["js", "ts", "json"],
+	"i18n-ally.enabledFrameworks": [
+		"svelte"
+	],
+	"i18n-ally.enabledParsers": [
+		"js",
+		"ts",
+		"json"
+	],
 	"i18n-ally.extract.autoDetect": true
-}
+}

CONTRIBUTING.md

@@ -1,256 +0,0 @@
-# 👋 Welcome
-
-First of all, thank you for considering contributing to my project! It means a lot 💜.
-
-
-## 🙋 Want to help?
-
-If you begin in GitHub contribution, you can find the [first contribution](https://github.com/firstcontributions/first-contributions) and follow this guide.
-
-Follow the [introduction](#introduction) to get started then start contributing!
-
-This is a little list of what you can do to help the project:
-
-- [🧑‍💻 Develop your own ideas](#developer-contribution)
-- [🌐 Translate the project](#translation)
-
-## 👋 Introduction
-
-### Setup with Github codespaces
-
-If you have github codespaces enabled then you can just create a codespace and run `pnpm dev` to run your the dev environment. All the required dependencies and packages has been configured for you already.
-
-### Setup with Gitpod
-
-If you have a [Gitpod](https://gitpod.io), you can just create a workspace from this repository, run `pnpm install && pnpm db:push && pnpm db:seed` and then `pnpm dev`. All the required dependencies and packages has been configured for you already.
-
-### Setup locally in your machine
-
-> 🔴 At the moment, Coolify **doesn't support Windows**. You must use Linux or MacOS. Consider using Gitpod or Github Codespaces.
-
-#### Recommended Pull Request Guideline
-
-- Fork the project
-- Clone your fork repo to local
-- Create a new branch
-- Push to your fork repo
-- Create a pull request: https://github.com/coollabsio/coolify/compare
-- Write a proper description
-- Open the pull request to review against `next` branch
-
----
-
-# 🧑‍💻 Developer contribution
-## Technical skills required
-
-- **Languages**: Node.js / Javascript / Typescript
-- **Framework JS/TS**: [SvelteKit](https://kit.svelte.dev/) & [Fastify](https://www.fastify.io/)
-- **Database ORM**: [Prisma.io](https://www.prisma.io/)
-- **Docker Engine API**
-
----
-
-## How to start after you set up your local fork?
-
-### Prerequisites
-1. Due to the lock file, this repository is best with [pnpm](https://pnpm.io). I recommend you try and use `pnpm` because it is cool and efficient!
-
-2. You need to have [Docker Engine](https://docs.docker.com/engine/install/) installed locally.
-3. You need to have [Docker Compose Plugin](https://docs.docker.com/compose/install/compose-plugin/) installed locally.
-4. You need to have [GIT LFS Support](https://git-lfs.github.com/) installed locally.
-
-Optional:
-
-4. To test Heroku buildpacks, you need [pack](https://github.com/buildpacks/pack) binary installed locally.
-
-### Steps for local setup
-
-1. Copy `apps/api/.env.template` to `apps/api/.env.template` and set the `COOLIFY_APP_ID` environment variable to something cool.
-2. Install dependencies with `pnpm install`.
-3. Need to create a local SQlite database with `pnpm db:push`.
-
-   This will apply all migrations at `db/dev.db`.
-
-4. Seed the database with base entities with `pnpm db:seed`
-5. You can start coding after starting `pnpm dev`.
-
----
-
-## Database migrations
-
-During development, if you change the database layout, you need to run `pnpm db:push` to migrate the database and create types for Prisma. You also need to restart the development process.
-
-If the schema is finalized, you need to create a migration file with `pnpm db:migrate <nameOfMigration>` where `nameOfMigration` is given by you. Make it sense. :)
-
----
-
-## How to add new services
-
-You can add any open-source and self-hostable software (service/application) to Coolify if the following statements are true:
-
-- Self-hostable (obviously)
-- Open-source
-- Maintained (I do not want to add software full of bugs)
-
-## Backend
-
-There are 5 steps you should make on the backend side.
-
-1. Create Prisma / database schema for the new service.
-2. Add supported versions of the service.
-3. Update global functions.
-4. Create API endpoints.
-5. Define automatically generated variables.
-
-> I will use [Umami](https://umami.is/) as an example service.
-
-### Create Prisma / Database schema for the new service.
-
-You only need to do this if you store passwords or any persistent configuration. Mostly it is required by all services, but there are some exceptions, like NocoDB.
-
-Update Prisma schema in [prisma/schema.prisma](prisma/schema.prisma).
-
-- Add new model with the new service name.
-- Make a relationship with `Service` model.
-- In the `Service` model, the name of the new field should be with low-capital.
-- If the service needs a database, define a `publicPort` field to be able to make it's database public, example field name in case of PostgreSQL: `postgresqlPublicPort`. It should be a optional field.
-
-If you are finished with the Prisma schema, you should update the database schema with `pnpm db:push` command.
-
-> You must restart the running development environment to be able to use the new model
-
-> If you use VSCode/TLS, you probably need to restart the `Typescript Language Server` to get the new types loaded in the running environment.
-
-### Add supported versions
-
-Supported versions are hardcoded into Coolify (for now).
-
-You need to update `supportedServiceTypesAndVersions` function at [apps/api/src/lib/services/supportedVersions.ts](apps/api/src/lib/services/supportedVersions.ts). Example JSON:
-
-```js
-     {
-       // Name used to identify the service internally
-       name: 'umami',
-       // Fancier name to show to the user
-       fancyName: 'Umami',
-       // Docker base image for the service
-       baseImage: 'ghcr.io/mikecao/umami',
-       // Optional: If there is any dependent image, you should list it here
-       images: [],
-       // Usable tags
-       versions: ['postgresql-latest'],
-       // Which tag is the recommended
-       recommendedVersion: 'postgresql-latest',
-       // Application's default port, Umami listens on 3000
-       ports: {
-         main: 3000
-       }
-     }
-```
-
-### Add required functions/properties
-
-1. Add the new service to the `includeServices` variable in [apps/api/src/lib/services/common.ts](apps/api/src/lib/services/common.ts), so it will be included in all places in the database queries where it is required.
-
-```js
-const include: any = {
-	destinationDocker: true,
-	persistentStorage: true,
-	serviceSecret: true,
-	minio: true,
-	plausibleAnalytics: true,
-	vscodeserver: true,
-	wordpress: true,
-	ghost: true,
-	meiliSearch: true,
-	umami: true // This line!
-};
-```
-
-2. Update the database update query with the new service type to `configureServiceType` function in [apps/api/src/lib/services/common.ts](apps/api/src/lib/services/common.ts). This function defines the automatically generated variables (passwords, users, etc.) and it's encryption process (if applicable).
-
-```js
-[...]
-else if (type === 'umami') {
-		const postgresqlUser = cuid();
-		const postgresqlPassword = encrypt(generatePassword());
-		const postgresqlDatabase = 'umami';
-		const hashSalt = encrypt(generatePassword(64));
-		await prisma.service.update({
-			where: { id },
-			data: {
-				type,
-				umami: {
-					create: {
-						postgresqlDatabase,
-						postgresqlPassword,
-						postgresqlUser,
-						hashSalt,
-					}
-				}
-			}
-		});
-	}
-```
-
-3. Add field details to [apps/api/src/lib/services/serviceFields.ts](apps/api/src/lib/services/serviceFields.ts), so every component will know what to do with the values (decrypt/show it by default/readonly)
-
-```js
-export const umami = [{
-	name: 'postgresqlUser',
-	isEditable: false,
-	isLowerCase: false,
-	isNumber: false,
-	isBoolean: false,
-	isEncrypted: false
-}]
-```
-
-4. Add service deletion query to `removeService` function in [apps/api/src/lib/services/common.ts](apps/api/src/lib/services/common.ts)
-
-5. Add start process for the new service in [apps/api/src/lib/services/handlers.ts](apps/api/src/lib/services/handlers.ts)
-
-> See startUmamiService() function as example.
-
-6. Add the newly added start process to `startService` in [apps/api/src/routes/api/v1/services/handlers.ts](apps/api/src/routes/api/v1/services/handlers.ts)
-
-7. You need to add a custom logo at [apps/ui/src/lib/components/svg/services](apps/ui/src/lib/components/svg/services) as a svelte component and export it in [apps/ui/src/lib/components/svg/services/index.ts](apps/ui/src/lib/components/svg/services/index.ts)
-
-   SVG is recommended, but you can use PNG as well. It should have the `isAbsolute` variable with the suitable CSS classes, primarily for sizing and positioning.
-
-8. You need to include it the logo at:
-
-- [apps/ui/src/lib/components/svg/services/ServiceIcons.svelte](apps/ui/src/lib/components/svg/services/ServiceIcons.svelte) with `isAbsolute`.
-- [apps/ui/src/routes/services/[id]/_ServiceLinks.svelte](apps/ui/src/routes/services/[id]/_ServiceLinks.svelte) with the link to the docs/main site of the service
-
-9. By default the URL and the name frontend forms are included in [apps/ui/src/routes/services/[id]/_Services/_Services.svelte](apps/ui/src/routes/services/[id]/_Services/_Services.svelte).
-
-   If you need to show more details on the frontend, such as users/passwords, you need to add Svelte component to [apps/ui/src/routes/services/[id]/_Services](apps/ui/src/routes/services/[id]/_Services) with an underscore. 
-   
-   > For example, see other [here](apps/ui/src/routes/services/[id]/_Services/_Umami.svelte).
-
-
-Good job! 👏
-
-<!-- # 🌐 Translate the project
-
-The project use [sveltekit-i18n](https://github.com/sveltekit-i18n/lib) to translate the project.
-It follows the [ISO 639-1](https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes) to name languages.
-
-### Installation
-
-You must have gone throw all the [intro](#introduction) steps before you can start translating.
-
-It's only an advice, but I recommend you to use:
-
-- Visual Studio Code
-- [i18n Ally for Visual Studio Code](https://marketplace.visualstudio.com/items?itemName=Lokalise.i18n-ally): ideal to see the progress of the translation.
-- [Svelte for VS Code](https://marketplace.visualstudio.com/items?itemName=svelte.svelte-vscode): to get the syntax color for the project
-
-### Adding a language
-
-If your language doesn't appear in the [locales folder list](src/lib/locales/), follow the step below:
-
-1.  In `src/lib/locales/`, Copy paste `en.json` and rename it with your language (eg: `cz.json`).
-2.  In the [lang.json](src/lib/lang.json) file, add a line after the first bracket (`{`) with `"ISO of your language": "Language",` (eg: `"cz": "Czech",`).
-3.  Have fun translating! -->

CONTRIBUTION.md

@@ -0,0 +1,48 @@
+# Contributing
+
+> "First, thanks for considering to contribute to my project. 
+  It really means a lot! 😁" - [@andrasbacsai](https://github.com/andrasbacsai)
+
+You can ask for guidance anytime on our 
+[Discord server](https://coollabs.io/discord) in the `#contribution` channel.
+
+You'll need a set of skills to [get started](docs/contribution/GettingStarted.md).
+
+## 1) Setup your development environment
+
+- 🌟 [Container based](docs/dev_setup/Container.md) ← *Recomended*
+- 📦 [DockerContainer](docs/dev_setup/DockerContiner.md) *WIP
+- 🐙 [Github Codespaces](docs/dev_setup/GithubCodespaces.md)
+- ☁️ [GitPod](docs/dev_setup/GitPod.md)
+- 🍏 [Local Mac](docs/dev_setup/Mac.md)
+
+## 2) Basic requirements
+
+- [Install Pnpm](https://pnpm.io/installation)
+- [Install Docker Engine](https://docs.docker.com/engine/install/)
+- [Setup Docker Compose Plugin](https://docs.docker.com/compose/install/compose-plugin/)
+- [Setup GIT LFS Support](https://git-lfs.github.com/)
+
+## 3) Setup Coolify
+
+- Copy `apps/api/.env.example` to `apps/api/.env` 
+- Edit `apps/api/.env`, set the `COOLIFY_APP_ID` environment variable to something cool.
+- Run `pnpm install` to install dependencies.
+- Run `pnpm db:push` to o create a local SQlite database. This will apply all migrations at `db/dev.db`.
+- Run `pnpm db:seed` seed the database.
+- Run `pnpm dev` start coding.
+
+```sh
+# Or... Copy and paste commands bellow:
+cp apps/api/.env.example apps/api/.env
+pnpm install
+pnpm db:push
+pnpm db:seed
+pnpm dev
+```
+
+## 4) Start Coding
+
+You should be able to access `http://localhost:3000`.
+
+1. Click `Register` and setup your first user.

CONTRIBUTION_NEW.md

@@ -1,108 +0,0 @@
----
-head:
-  - - meta
-    - name: description
-      content: Coolify - Databases
-  - - meta
-    - name: keywords
-      content: databases coollabs coolify 
-  - - meta
-    - name: twitter:card
-      content: summary_large_image
-  - - meta
-    - name: twitter:site
-      content: '@andrasbacsai'
-  - - meta
-    - name: twitter:title
-      content: Coolify
-  - - meta
-    - name: twitter:description
-      content: An open-source & self-hostable Heroku / Netlify alternative.
-  - - meta
-    - name: twitter:image
-      content: https://cdn.coollabs.io/assets/coollabs/og-image-databases.png
-  - - meta
-    - property: og:type
-      content: website
-  - - meta
-    - property: og:url
-      content: https://coolify.io
-  - - meta
-    - property: og:title
-      content: Coolify
-  - - meta
-    - property: og:description
-      content: An open-source & self-hostable Heroku / Netlify alternative.
-  - - meta
-    - property: og:site_name
-      content: Coolify
-  - - meta
-    - property: og:image
-      content: https://cdn.coollabs.io/assets/coollabs/og-image-databases.png
----
-# Contribution
-
-First, thanks for considering to contribute to my project. It really means a lot! :)
-
-You can ask for guidance anytime on our Discord server in the #contribution channel.
-
-## Setup your development environment
-### Github codespaces
-
-If you have github codespaces enabled then you can just create a codespace and run `pnpm dev` to run your the dev environment. All the required dependencies and packages has been configured for you already.
-
-### Gitpod
-
-If you have a [Gitpod](https://gitpod.io), you can just create a workspace from this repository, run `pnpm install && pnpm db:push && pnpm db:seed` and then `pnpm dev`. All the required dependencies and packages has been configured for you already.
-
-### Local Machine
-> At the moment, Coolify `doesn't support Windows`. You must use `Linux` or `MacOS` or consider using Gitpod or Github Codespaces.
-
-- Due to the lock file, this repository is best with [pnpm](https://pnpm.io). I recommend you try and use `pnpm` because it is cool and efficient!
-
-- You need to have [Docker Engine](https://docs.docker.com/engine/install/) installed locally.
-- You need to have [Docker Compose Plugin](https://docs.docker.com/compose/install/compose-plugin/) installed locally.
-- You need to have [GIT LFS Support](https://git-lfs.github.com/) installed locally.
-
-Optional:
-- To test Heroku buildpacks, you need [pack](https://github.com/buildpacks/pack) binary installed locally.
-
-### Inside a Docker container
-`WIP`
-
-## Setup Coolify
-- Copy `apps/api/.env.template` to `apps/api/.env.template` and set the `COOLIFY_APP_ID` environment variable to something cool.
-- `pnpm install` to install dependencies.
-- `pnpm db:push` to o create a local SQlite database.
-
-   This will apply all migrations at `db/dev.db`.
-
-- `pnpm db:seed` seed the database.
-- `pnpm dev` start coding.
-
-## Technical skills required
-
-- **Languages**: Node.js / Javascript / Typescript
-- **Framework JS/TS**: [SvelteKit](https://kit.svelte.dev/) & [Fastify](https://www.fastify.io/)
-- **Database ORM**: [Prisma.io](https://www.prisma.io/)
-- **Docker Engine API**
-
-## Add a new service
-### Which service is eligable to add to Coolify?
-The following statements needs to be true:
-
-- Self-hostable
-- Open-source
-- Maintained (I do not want to add software full of bugs)
-
-### Create Prisma / Database schema for the new service.
-All data that needs to be persist for a service should be saved to the database in `cleartext` or `encrypted`.
-
-very password/api key/passphrase needs to be encrypted. If you are not sure, whether it should be encrypted or not, just encrypt it.
-
-Update Prisma schema in [src/api/prisma/schema.prisma](https://github.com/coollabsio/coolify/blob/main/apps/api/prisma/schema.prisma).
-
-- Add new model with the new service name.
-- Make a relationship with `Service` model.
-- In the `Service` model, the name of the new field should be with low-capital.
-- If the service needs a database, define a `publicPort` field to be able to make it's database public, example field name in case of PostgreSQL: `postgresqlPublicPort`. It should be a optional field.

Dockerfile

@@ -1,5 +1,4 @@
 ARG PNPM_VERSION=7.11.0
-ARG NPM_VERSION=8.19.1
 
 FROM node:18-slim as build
 WORKDIR /app
@@ -17,26 +16,35 @@ WORKDIR /app
 ENV NODE_ENV production
 ARG TARGETPLATFORM
 
+# https://download.docker.com/linux/static/stable/
+ARG DOCKER_VERSION=20.10.18
+# https://github.com/docker/compose/releases
+# Reverted to 2.6.1 because of this https://github.com/docker/compose/issues/9704. 2.9.0 still has a bug.
+ARG DOCKER_COMPOSE_VERSION=2.6.1
+# https://github.com/buildpacks/pack/releases
+ARG PACK_VERSION=v0.27.0
+
 RUN apt update && apt -y install --no-install-recommends ca-certificates git git-lfs openssh-client curl jq cmake sqlite3 openssl psmisc python3
 RUN apt-get clean autoclean && apt-get autoremove --yes && rm -rf /var/lib/{apt,dpkg,cache,log}/
 RUN npm --no-update-notifier --no-fund --global install pnpm@${PNPM_VERSION}
 RUN npm install -g npm@${PNPM_VERSION}
 
 RUN mkdir -p ~/.docker/cli-plugins/
-# https://download.docker.com/linux/static/stable/
-RUN curl -SL https://cdn.coollabs.io/bin/$TARGETPLATFORM/docker-20.10.9 -o /usr/bin/docker
-# https://github.com/docker/compose/releases
-# Reverted to 2.6.1 because of this https://github.com/docker/compose/issues/9704. 2.9.0 still has a bug.
-RUN curl -SL https://cdn.coollabs.io/bin/$TARGETPLATFORM/docker-compose-linux-2.6.1 -o ~/.docker/cli-plugins/docker-compose
-RUN chmod +x ~/.docker/cli-plugins/docker-compose /usr/bin/docker
 
-RUN (curl -sSL "https://github.com/buildpacks/pack/releases/download/v0.27.0/pack-v0.27.0-linux.tgz" | tar -C /usr/local/bin/ --no-same-owner -xzv pack)
+RUN curl -SL https://cdn.coollabs.io/bin/$TARGETPLATFORM/docker-$DOCKER_VERSION -o /usr/bin/docker
+RUN curl -SL https://cdn.coollabs.io/bin/$TARGETPLATFORM/docker-compose-linux-$DOCKER_COMPOSE_VERSION -o ~/.docker/cli-plugins/docker-compose
+RUN curl -SL https://cdn.coollabs.io/bin/$TARGETPLATFORM/pack-$PACK_VERSION -o /usr/local/bin/pack 
+
+RUN chmod +x ~/.docker/cli-plugins/docker-compose /usr/bin/docker /usr/local/bin/pack
 
 COPY --from=build /app/apps/api/build/ .
+COPY --from=build /app/others/fluentbit/ ./fluentbit
 COPY --from=build /app/apps/ui/build/ ./public
 COPY --from=build /app/apps/api/prisma/ ./prisma
 COPY --from=build /app/apps/api/package.json .
 COPY --from=build /app/docker-compose.yaml .
+COPY --from=build /app/apps/api/tags.json .
+COPY --from=build /app/apps/api/templates.json .
 
 RUN pnpm install -p
 

Dockerfile-dev

@@ -0,0 +1,31 @@
+FROM node:18-slim
+ENV NODE_ENV development
+ARG TARGETPLATFORM
+ARG PNPM_VERSION=7.11.0
+ARG NPM_VERSION=8.19.1
+# https://download.docker.com/linux/static/stable/
+ARG DOCKER_VERSION=20.10.18
+# https://github.com/docker/compose/releases
+# Reverted to 2.6.1 because of this https://github.com/docker/compose/issues/9704. 2.9.0 still has a bug.
+ARG DOCKER_COMPOSE_VERSION=2.6.1
+# https://github.com/buildpacks/pack/releases
+ARG PACK_VERSION=v0.27.0
+WORKDIR /app
+
+RUN npm --no-update-notifier --no-fund --global install pnpm@${PNPM_VERSION}
+
+RUN apt update && apt -y install --no-install-recommends ca-certificates git git-lfs openssh-client curl jq cmake sqlite3 openssl psmisc python3
+RUN apt-get clean autoclean && apt-get autoremove --yes && rm -rf /var/lib/{apt,dpkg,cache,log}/
+RUN npm --no-update-notifier --no-fund --global install pnpm@${PNPM_VERSION}
+RUN npm install -g npm@${PNPM_VERSION}
+
+RUN mkdir -p ~/.docker/cli-plugins/
+
+RUN curl -SL https://cdn.coollabs.io/bin/$TARGETPLATFORM/docker-$DOCKER_VERSION -o /usr/bin/docker
+RUN curl -SL https://cdn.coollabs.io/bin/$TARGETPLATFORM/docker-compose-linux-$DOCKER_COMPOSE_VERSION -o ~/.docker/cli-plugins/docker-compose
+RUN curl -SL https://cdn.coollabs.io/bin/$TARGETPLATFORM/pack-$PACK_VERSION -o /usr/local/bin/pack
+
+RUN chmod +x ~/.docker/cli-plugins/docker-compose /usr/bin/docker /usr/local/bin/pack
+
+EXPOSE 3000
+ENV CHECKPOINT_DISABLE=1

README.md

@@ -77,6 +77,7 @@ Deploy your resource to:
 <a href="https://redis.io"><svg style="width:40px;height:40px" viewBox="0 0 32 32" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" ><defs ><path id="a" d="m45.536 38.764c-2.013 1.05-12.44 5.337-14.66 6.494s-3.453 1.146-5.207.308-12.85-5.32-14.85-6.276c-1-.478-1.524-.88-1.524-1.26v-3.813s14.447-3.145 16.78-3.982 3.14-.867 5.126-.14 13.853 2.868 15.814 3.587v3.76c0 .377-.452.8-1.477 1.324z" /><path id="b" d="m45.536 28.733c-2.013 1.05-12.44 5.337-14.66 6.494s-3.453 1.146-5.207.308-12.85-5.32-14.85-6.276-2.04-1.613-.077-2.382l15.332-5.935c2.332-.837 3.14-.867 5.126-.14s12.35 4.853 14.312 5.57 2.037 1.31.024 2.36z" /></defs ><g transform="matrix(.848327 0 0 .848327 -7.883573 -9.449691)" ><use fill="#a41e11" xlink:href="#a" /><path d="m45.536 34.95c-2.013 1.05-12.44 5.337-14.66 6.494s-3.453 1.146-5.207.308-12.85-5.32-14.85-6.276-2.04-1.613-.077-2.382l15.332-5.936c2.332-.836 3.14-.867 5.126-.14s12.35 4.852 14.31 5.582 2.037 1.31.024 2.36z" fill="#d82c20" /><use fill="#a41e11" xlink:href="#a" y="-6.218" /><use fill="#d82c20" xlink:href="#b" /><path d="m45.536 26.098c-2.013 1.05-12.44 5.337-14.66 6.495s-3.453 1.146-5.207.308-12.85-5.32-14.85-6.276c-1-.478-1.524-.88-1.524-1.26v-3.815s14.447-3.145 16.78-3.982 3.14-.867 5.126-.14 13.853 2.868 15.814 3.587v3.76c0 .377-.452.8-1.477 1.324z" fill="#a41e11" /><use fill="#d82c20" xlink:href="#b" y="-6.449" /><g fill="#fff" ><path d="m29.096 20.712-1.182-1.965-3.774-.34 2.816-1.016-.845-1.56 2.636 1.03 2.486-.814-.672 1.612 2.534.95-3.268.34zm-6.296 3.912 8.74-1.342-2.64 3.872z" /><ellipse cx="20.444" cy="21.402" rx="4.672" ry="1.811" /></g ><path d="m42.132 21.138-5.17 2.042-.004-4.087z" fill="#7a0c00" /><path d="m36.963 23.18-.56.22-5.166-2.042 5.723-2.264z" fill="#ad2115" /></g ></svg ></a>
 
 ### Services
+
 - [Appwrite](https://appwrite.io)
 - [WordPress](https://docs.coollabs.io/coolify/services/wordpress)
 - [Ghost](https://ghost.org)
@@ -93,19 +94,39 @@ Deploy your resource to:
 - [Fider](https://fider.io)
 - [Hasura](https://hasura.io)
 - [GlitchTip](https://glitchtip.com)
-
-## Migration from v1
-
-A fresh installation is necessary. v2 and v3 are not compatible with v1.
+- And more...
 
 ## Support
 
-- Twitter: [@andrasbacsai](https://twitter.com/andrasbacsai)
+- Mastodon: [@andrasbacsai@fosstodon.org](https://fosstodon.org/@andrasbacsai)
 - Telegram: [@andrasbacsai](https://t.me/andrasbacsai)
+- Twitter: [@andrasbacsai](https://twitter.com/andrasbacsai)
 - Email: [andras@coollabs.io](mailto:andras@coollabs.io)
 - Discord: [Invitation](https://coollabs.io/discord)
 
-## Financial Contributors
+---
+
+## ⚗️ Expertise Contributions
+
+Coolify is developed under the [Apache License](./LICENSE) and you can help to make it grow.
+Our community will be glad to have you on board!
+
+Learn how to contribute to Coolify as as ...
+
+&rarr; [👩🏾‍💻 Software developer](./CONTRIBUTION.md)
+
+&rarr; [🧑🏻‍🏫 Translator](./docs/contribution/Translating.md)
+
+<!-- 
+&rarr; 🧑🏽‍🎨 Designer
+&rarr; 🙋‍♀️ Community Managemer
+&rarr; 🧙🏻‍♂️ Text Content Creator
+&rarr; 👨🏼‍🎤 Video Content Creator
+-->
+
+---
+
+## 💰 Financial Contributors
 
 Become a financial contributor and help us sustain our community. [[Contribute](https://opencollective.com/coollabsio/contribute)]
 

apps/api/.env.example

@@ -1,10 +1,9 @@
-COOLIFY_APP_ID=local-dev
-# 32 bits long secret key
-COOLIFY_SECRET_KEY=12341234123412341234123412341234
-COOLIFY_DATABASE_URL=file:../db/dev.db
-COOLIFY_SENTRY_DSN=
-
-COOLIFY_IS_ON=docker
-COOLIFY_WHITE_LABELED=false
-COOLIFY_WHITE_LABELED_ICON=
-COOLIFY_AUTO_UPDATE=
+COOLIFY_APP_ID=local-dev
+# 32 bits long secret key
+COOLIFY_SECRET_KEY=12341234123412341234123412341234
+COOLIFY_DATABASE_URL=file:../db/dev.db
+
+COOLIFY_IS_ON=docker
+COOLIFY_WHITE_LABELED=false
+COOLIFY_WHITE_LABELED_ICON=
+COOLIFY_AUTO_UPDATE=

apps/api/nodemon.json

@@ -1,7 +1,11 @@
 {
-    "watch": ["src"],
-    "ignore": ["src/**/*.test.ts"],
-    "ext": "ts,mjs,json,graphql",
-    "exec": "rimraf build && esbuild `find src \\( -name '*.ts' \\)` --minify=true --platform=node --outdir=build --format=cjs  && node build",
-    "legacyWatch": true
-  }
+  "watch": [
+    "src"
+  ],
+  "ignore": [
+    "src/**/*.test.ts"
+  ],
+  "ext": "ts,mjs,json,graphql",
+  "exec": "rimraf build && esbuild `find src \\( -name '*.ts' \\)` --platform=node --outdir=build --format=cjs && node build",
+  "legacyWatch": true
+}

apps/api/package.json

@@ -1,73 +1,85 @@
 {
-  "name": "api",
-  "description": "Coolify's Fastify API",
-  "license": "Apache-2.0",
-  "scripts": {
-    "db:push": "prisma db push && prisma generate",
-    "db:seed": "prisma db seed",
-    "db:studio": "prisma studio",
-    "db:migrate": "COOLIFY_DATABASE_URL=file:../db/migration.db prisma migrate dev --skip-seed --name",
-    "dev": "nodemon",
-    "build": "rimraf build && esbuild `find src \\( -name '*.ts' \\)| grep -v client/` --platform=node --outdir=build --format=cjs",
-    "format": "prettier --write 'src/**/*.{js,ts,json,md}'",
-    "lint": "prettier --check 'src/**/*.{js,ts,json,md}' && eslint --ignore-path .eslintignore .",
-    "start": "NODE_ENV=production npx -y prisma migrate deploy && npx prisma generate && npx prisma db seed && node index.js"
-  },
-  "dependencies": {
-    "@breejs/ts-worker": "2.0.0",
-    "@fastify/autoload": "5.3.1",
-    "@fastify/cookie": "8.1.0",
-    "@fastify/cors": "8.1.0",
-    "@fastify/env": "4.1.0",
-    "@fastify/jwt": "6.3.2",
-    "@fastify/static": "6.5.0",
-    "@iarna/toml": "2.2.5",
-    "@ladjs/graceful": "3.0.2",
-    "@prisma/client": "4.3.1",
-    "axios": "0.27.2",
-    "bcryptjs": "2.4.3",
-    "bree": "9.1.2",
-    "cabin": "9.1.2",
-    "compare-versions": "5.0.1",
-    "cuid": "2.1.8",
-    "dayjs": "1.11.5",
-    "dockerode": "3.3.4",
-    "dotenv-extended": "2.9.0",
-    "execa": "6.1.0",
-    "fastify": "4.5.3",
-    "fastify-plugin": "4.2.1",
-    "generate-password": "1.7.0",
-    "got": "12.4.1",
-    "is-ip": "5.0.0",
-    "is-port-reachable": "4.0.0",
-    "js-yaml": "4.1.0",
-    "jsonwebtoken": "8.5.1",
-    "node-forge": "1.3.1",
-    "node-os-utils": "1.3.7",
-    "p-all": "4.0.0",
-    "p-throttle": "5.0.0",
-    "public-ip": "6.0.1",
-    "ssh-config": "4.1.6",
-    "strip-ansi": "7.0.1",
-    "unique-names-generator": "4.7.1"
-  },
-  "devDependencies": {
-    "@types/node": "18.7.15",
-    "@types/node-os-utils": "1.3.0",
-    "@typescript-eslint/eslint-plugin": "5.36.2",
-    "@typescript-eslint/parser": "5.36.2",
-    "esbuild": "0.15.7",
-    "eslint": "8.23.0",
-    "eslint-config-prettier": "8.5.0",
-    "eslint-plugin-prettier": "4.2.1",
-    "nodemon": "2.0.19",
-    "prettier": "2.7.1",
-    "prisma": "4.3.1",
-    "rimraf": "3.0.2",
-    "tsconfig-paths": "4.1.0",
-    "typescript": "4.8.2"
-  },
-  "prisma": {
-    "seed": "node prisma/seed.js"
-  }
-}
+	"name": "api",
+	"description": "Coolify's Fastify API",
+	"license": "Apache-2.0",
+	"scripts": {
+		"db:generate": "prisma generate",
+		"db:push": "prisma db push && prisma generate",
+		"db:seed": "prisma db seed",
+		"db:studio": "prisma studio",
+		"db:migrate": "COOLIFY_DATABASE_URL=file:../db/migration.db prisma migrate dev --skip-seed --name",
+		"dev": "nodemon",
+		"build": "rimraf build && esbuild `find src \\( -name '*.ts' \\)| grep -v client/` --platform=node --outdir=build --format=cjs",
+		"format": "prettier --write 'src/**/*.{js,ts,json,md}'",
+		"lint": "prettier --check 'src/**/*.{js,ts,json,md}' && eslint --ignore-path .eslintignore .",
+		"start": "NODE_ENV=production pnpm prisma migrate deploy && pnpm prisma generate && pnpm prisma db seed && node index.js"
+	},
+	"dependencies": {
+		"@breejs/ts-worker": "2.0.0",
+		"@fastify/autoload": "5.5.0",
+		"@fastify/cookie": "8.3.0",
+		"@fastify/cors": "8.2.0",
+		"@fastify/env": "4.1.0",
+		"@fastify/jwt": "6.3.3",
+		"@fastify/multipart": "7.3.0",
+		"@fastify/static": "6.5.1",
+		"@iarna/toml": "2.2.5",
+		"@ladjs/graceful": "3.0.2",
+		"@prisma/client": "4.6.1",
+		"@sentry/node": "7.21.1",
+		"@sentry/tracing": "7.21.1",
+		"axe": "11.0.0",
+		"bcryptjs": "2.4.3",
+		"bree": "9.1.2",
+		"cabin": "11.0.1",
+		"compare-versions": "5.0.1",
+		"csv-parse": "5.3.2",
+		"csvtojson": "2.0.10",
+		"cuid": "2.1.8",
+		"dayjs": "1.11.6",
+		"dockerode": "3.3.4",
+		"dotenv-extended": "2.9.0",
+		"execa": "6.1.0",
+		"fastify": "4.10.2",
+		"fastify-plugin": "4.3.0",
+		"fastify-socket.io": "4.0.0",
+		"generate-password": "1.7.0",
+		"got": "12.5.3",
+		"is-ip": "5.0.0",
+		"is-port-reachable": "4.0.0",
+		"js-yaml": "4.1.0",
+		"jsonwebtoken": "8.5.1",
+		"minimist": "^1.2.7",
+		"node-forge": "1.3.1",
+		"node-os-utils": "1.3.7",
+		"p-all": "4.0.0",
+		"p-throttle": "5.0.0",
+		"prisma": "4.6.1",
+		"public-ip": "6.0.1",
+		"pump": "3.0.0",
+		"shell-quote": "^1.7.4",
+		"socket.io": "4.5.3",
+		"ssh-config": "4.1.6",
+		"strip-ansi": "7.0.1",
+		"unique-names-generator": "4.7.1"
+	},
+	"devDependencies": {
+		"@types/node": "18.11.9",
+		"@types/node-os-utils": "1.3.0",
+		"@typescript-eslint/eslint-plugin": "5.44.0",
+		"@typescript-eslint/parser": "5.44.0",
+		"esbuild": "0.15.15",
+		"eslint": "8.28.0",
+		"eslint-config-prettier": "8.5.0",
+		"eslint-plugin-prettier": "4.2.1",
+		"nodemon": "2.0.20",
+		"prettier": "2.7.1",
+		"rimraf": "3.0.2",
+		"tsconfig-paths": "4.1.0",
+		"types-fastify-socket.io": "0.0.1",
+		"typescript": "4.9.3"
+	},
+	"prisma": {
+		"seed": "node prisma/seed.js"
+	}
+}

apps/api/prisma/migrations/20220913092100_preview_applications/migration.sql

@@ -0,0 +1,18 @@
+-- AlterTable
+ALTER TABLE "Build" ADD COLUMN "previewApplicationId" TEXT;
+
+-- CreateTable
+CREATE TABLE "PreviewApplication" (
+    "id" TEXT NOT NULL PRIMARY KEY,
+    "pullmergeRequestId" TEXT NOT NULL,
+    "sourceBranch" TEXT NOT NULL,
+    "isRandomDomain" BOOLEAN NOT NULL DEFAULT false,
+    "customDomain" TEXT,
+    "applicationId" TEXT NOT NULL,
+    "createdAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" DATETIME NOT NULL,
+    CONSTRAINT "PreviewApplication_applicationId_fkey" FOREIGN KEY ("applicationId") REFERENCES "Application" ("id") ON DELETE RESTRICT ON UPDATE CASCADE
+);
+
+-- CreateIndex
+CREATE UNIQUE INDEX "PreviewApplication_applicationId_key" ON "PreviewApplication"("applicationId");

apps/api/prisma/migrations/20220922064605_custom_certificates/migration.sql

@@ -0,0 +1,10 @@
+-- CreateTable
+CREATE TABLE "Certificate" (
+    "id" TEXT NOT NULL PRIMARY KEY,
+    "key" TEXT NOT NULL,
+    "cert" TEXT NOT NULL,
+    "createdAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" DATETIME NOT NULL,
+    "teamId" TEXT,
+    CONSTRAINT "Certificate_teamId_fkey" FOREIGN KEY ("teamId") REFERENCES "Team" ("id") ON DELETE SET NULL ON UPDATE CASCADE
+);

apps/api/prisma/migrations/20220923122227_custom_ssl_for_applications/migration.sql

@@ -0,0 +1,23 @@
+-- RedefineTables
+PRAGMA foreign_keys=OFF;
+CREATE TABLE "new_ApplicationSettings" (
+    "id" TEXT NOT NULL PRIMARY KEY,
+    "applicationId" TEXT NOT NULL,
+    "dualCerts" BOOLEAN NOT NULL DEFAULT false,
+    "debug" BOOLEAN NOT NULL DEFAULT false,
+    "previews" BOOLEAN NOT NULL DEFAULT false,
+    "autodeploy" BOOLEAN NOT NULL DEFAULT true,
+    "isBot" BOOLEAN NOT NULL DEFAULT false,
+    "isPublicRepository" BOOLEAN NOT NULL DEFAULT false,
+    "isDBBranching" BOOLEAN NOT NULL DEFAULT false,
+    "isCustomSSL" BOOLEAN NOT NULL DEFAULT false,
+    "createdAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" DATETIME NOT NULL,
+    CONSTRAINT "ApplicationSettings_applicationId_fkey" FOREIGN KEY ("applicationId") REFERENCES "Application" ("id") ON DELETE RESTRICT ON UPDATE CASCADE
+);
+INSERT INTO "new_ApplicationSettings" ("applicationId", "autodeploy", "createdAt", "debug", "dualCerts", "id", "isBot", "isDBBranching", "isPublicRepository", "previews", "updatedAt") SELECT "applicationId", "autodeploy", "createdAt", "debug", "dualCerts", "id", "isBot", "isDBBranching", "isPublicRepository", "previews", "updatedAt" FROM "ApplicationSettings";
+DROP TABLE "ApplicationSettings";
+ALTER TABLE "new_ApplicationSettings" RENAME TO "ApplicationSettings";
+CREATE UNIQUE INDEX "ApplicationSettings_applicationId_key" ON "ApplicationSettings"("applicationId");
+PRAGMA foreign_key_check;
+PRAGMA foreign_keys=ON;

apps/api/prisma/migrations/20220928083348_system_wide_git_sources/migration.sql

@@ -0,0 +1,26 @@
+-- RedefineTables
+PRAGMA foreign_keys=OFF;
+CREATE TABLE "new_GitSource" (
+    "id" TEXT NOT NULL PRIMARY KEY,
+    "name" TEXT NOT NULL,
+    "forPublic" BOOLEAN NOT NULL DEFAULT false,
+    "type" TEXT,
+    "apiUrl" TEXT,
+    "htmlUrl" TEXT,
+    "customPort" INTEGER NOT NULL DEFAULT 22,
+    "organization" TEXT,
+    "createdAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" DATETIME NOT NULL,
+    "githubAppId" TEXT,
+    "gitlabAppId" TEXT,
+    "isSystemWide" BOOLEAN NOT NULL DEFAULT false,
+    CONSTRAINT "GitSource_gitlabAppId_fkey" FOREIGN KEY ("gitlabAppId") REFERENCES "GitlabApp" ("id") ON DELETE SET NULL ON UPDATE CASCADE,
+    CONSTRAINT "GitSource_githubAppId_fkey" FOREIGN KEY ("githubAppId") REFERENCES "GithubApp" ("id") ON DELETE SET NULL ON UPDATE CASCADE
+);
+INSERT INTO "new_GitSource" ("apiUrl", "createdAt", "customPort", "forPublic", "githubAppId", "gitlabAppId", "htmlUrl", "id", "name", "organization", "type", "updatedAt") SELECT "apiUrl", "createdAt", "customPort", "forPublic", "githubAppId", "gitlabAppId", "htmlUrl", "id", "name", "organization", "type", "updatedAt" FROM "GitSource";
+DROP TABLE "GitSource";
+ALTER TABLE "new_GitSource" RENAME TO "GitSource";
+CREATE UNIQUE INDEX "GitSource_githubAppId_key" ON "GitSource"("githubAppId");
+CREATE UNIQUE INDEX "GitSource_gitlabAppId_key" ON "GitSource"("gitlabAppId");
+PRAGMA foreign_key_check;
+PRAGMA foreign_keys=ON;

apps/api/prisma/migrations/20221002084246_no_unique_appid_on_previews/migration.sql

@@ -0,0 +1,2 @@
+-- DropIndex
+DROP INDEX "PreviewApplication_applicationId_key";

apps/api/prisma/migrations/20221002091630_forked_previews/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "Build" ADD COLUMN "sourceRepository" TEXT;

apps/api/prisma/migrations/20221005120323_initial_docker_compose/migration.sql

@@ -0,0 +1,3 @@
+-- AlterTable
+ALTER TABLE "Application" ADD COLUMN "dockerComposeFile" TEXT;
+ALTER TABLE "Application" ADD COLUMN "dockerComposeFileLocation" TEXT;

apps/api/prisma/migrations/20221005132352_docker_compose_configuration/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "Application" ADD COLUMN "dockerComposeConfiguration" TEXT;

apps/api/prisma/migrations/20221017134342_standardized_service_configs/migration.sql

@@ -0,0 +1,13 @@
+-- CreateTable
+CREATE TABLE "ServiceSetting" (
+    "id" TEXT NOT NULL PRIMARY KEY,
+    "serviceId" TEXT NOT NULL,
+    "name" TEXT NOT NULL,
+    "value" TEXT NOT NULL,
+    "createdAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" DATETIME NOT NULL,
+    CONSTRAINT "ServiceSetting_serviceId_fkey" FOREIGN KEY ("serviceId") REFERENCES "Service" ("id") ON DELETE RESTRICT ON UPDATE CASCADE
+);
+
+-- CreateIndex
+CREATE UNIQUE INDEX "ServiceSetting_serviceId_name_key" ON "ServiceSetting"("serviceId", "name");

apps/api/prisma/migrations/20221018090939_service_peristent_volumes_predefined/migration.sql

@@ -0,0 +1,19 @@
+-- RedefineTables
+PRAGMA foreign_keys=OFF;
+CREATE TABLE "new_ServicePersistentStorage" (
+    "id" TEXT NOT NULL PRIMARY KEY,
+    "serviceId" TEXT NOT NULL,
+    "path" TEXT NOT NULL,
+    "volumeName" TEXT,
+    "predefined" BOOLEAN NOT NULL DEFAULT false,
+    "containerId" TEXT,
+    "createdAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" DATETIME NOT NULL,
+    CONSTRAINT "ServicePersistentStorage_serviceId_fkey" FOREIGN KEY ("serviceId") REFERENCES "Service" ("id") ON DELETE RESTRICT ON UPDATE CASCADE
+);
+INSERT INTO "new_ServicePersistentStorage" ("createdAt", "id", "path", "serviceId", "updatedAt") SELECT "createdAt", "id", "path", "serviceId", "updatedAt" FROM "ServicePersistentStorage";
+DROP TABLE "ServicePersistentStorage";
+ALTER TABLE "new_ServicePersistentStorage" RENAME TO "ServicePersistentStorage";
+CREATE UNIQUE INDEX "ServicePersistentStorage_serviceId_path_key" ON "ServicePersistentStorage"("serviceId", "path");
+PRAGMA foreign_key_check;
+PRAGMA foreign_keys=ON;

apps/api/prisma/migrations/20221021112429_serivce_settings_extended/migration.sql

@@ -0,0 +1,24 @@
+/*
+  Warnings:
+
+  - Added the required column `variableName` to the `ServiceSetting` table without a default value. This is not possible if the table is not empty.
+
+*/
+-- RedefineTables
+PRAGMA foreign_keys=OFF;
+CREATE TABLE "new_ServiceSetting" (
+    "id" TEXT NOT NULL PRIMARY KEY,
+    "serviceId" TEXT NOT NULL,
+    "name" TEXT NOT NULL,
+    "value" TEXT NOT NULL,
+    "variableName" TEXT NOT NULL,
+    "createdAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" DATETIME NOT NULL,
+    CONSTRAINT "ServiceSetting_serviceId_fkey" FOREIGN KEY ("serviceId") REFERENCES "Service" ("id") ON DELETE RESTRICT ON UPDATE CASCADE
+);
+INSERT INTO "new_ServiceSetting" ("createdAt", "id", "name", "serviceId", "updatedAt", "value") SELECT "createdAt", "id", "name", "serviceId", "updatedAt", "value" FROM "ServiceSetting";
+DROP TABLE "ServiceSetting";
+ALTER TABLE "new_ServiceSetting" RENAME TO "ServiceSetting";
+CREATE UNIQUE INDEX "ServiceSetting_serviceId_name_key" ON "ServiceSetting"("serviceId", "name");
+PRAGMA foreign_key_check;
+PRAGMA foreign_keys=ON;

apps/api/prisma/migrations/20221021185630_service_template_version/migration.sql

@@ -0,0 +1,21 @@
+-- RedefineTables
+PRAGMA foreign_keys=OFF;
+CREATE TABLE "new_Service" (
+    "id" TEXT NOT NULL PRIMARY KEY,
+    "name" TEXT NOT NULL,
+    "fqdn" TEXT,
+    "exposePort" INTEGER,
+    "dualCerts" BOOLEAN NOT NULL DEFAULT false,
+    "type" TEXT,
+    "version" TEXT,
+    "templateVersion" TEXT NOT NULL DEFAULT '0.0.0',
+    "destinationDockerId" TEXT,
+    "createdAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" DATETIME NOT NULL,
+    CONSTRAINT "Service_destinationDockerId_fkey" FOREIGN KEY ("destinationDockerId") REFERENCES "DestinationDocker" ("id") ON DELETE SET NULL ON UPDATE CASCADE
+);
+INSERT INTO "new_Service" ("createdAt", "destinationDockerId", "dualCerts", "exposePort", "fqdn", "id", "name", "type", "updatedAt", "version") SELECT "createdAt", "destinationDockerId", "dualCerts", "exposePort", "fqdn", "id", "name", "type", "updatedAt", "version" FROM "Service";
+DROP TABLE "Service";
+ALTER TABLE "new_Service" RENAME TO "Service";
+PRAGMA foreign_key_check;
+PRAGMA foreign_keys=ON;

apps/api/prisma/migrations/20221026115123_service_persistent_unique/migration.sql

@@ -0,0 +1,11 @@
+/*
+  Warnings:
+
+  - A unique constraint covering the columns `[serviceId,containerId,path]` on the table `ServicePersistentStorage` will be added. If there are existing duplicate values, this will fail.
+
+*/
+-- DropIndex
+DROP INDEX "ServicePersistentStorage_serviceId_path_key";
+
+-- CreateIndex
+CREATE UNIQUE INDEX "ServicePersistentStorage_serviceId_containerId_path_key" ON "ServicePersistentStorage"("serviceId", "containerId", "path");

apps/api/prisma/migrations/20221028074301_wordpress_optional_fields/migration.sql

@@ -0,0 +1,32 @@
+-- RedefineTables
+PRAGMA foreign_keys=OFF;
+CREATE TABLE "new_Wordpress" (
+    "id" TEXT NOT NULL PRIMARY KEY,
+    "extraConfig" TEXT,
+    "tablePrefix" TEXT,
+    "ownMysql" BOOLEAN NOT NULL DEFAULT false,
+    "mysqlHost" TEXT,
+    "mysqlPort" INTEGER,
+    "mysqlUser" TEXT,
+    "mysqlPassword" TEXT,
+    "mysqlRootUser" TEXT,
+    "mysqlRootUserPassword" TEXT,
+    "mysqlDatabase" TEXT,
+    "mysqlPublicPort" INTEGER,
+    "ftpEnabled" BOOLEAN NOT NULL DEFAULT false,
+    "ftpUser" TEXT,
+    "ftpPassword" TEXT,
+    "ftpPublicPort" INTEGER,
+    "ftpHostKey" TEXT,
+    "ftpHostKeyPrivate" TEXT,
+    "serviceId" TEXT NOT NULL,
+    "createdAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" DATETIME NOT NULL,
+    CONSTRAINT "Wordpress_serviceId_fkey" FOREIGN KEY ("serviceId") REFERENCES "Service" ("id") ON DELETE RESTRICT ON UPDATE CASCADE
+);
+INSERT INTO "new_Wordpress" ("createdAt", "extraConfig", "ftpEnabled", "ftpHostKey", "ftpHostKeyPrivate", "ftpPassword", "ftpPublicPort", "ftpUser", "id", "mysqlDatabase", "mysqlHost", "mysqlPassword", "mysqlPort", "mysqlPublicPort", "mysqlRootUser", "mysqlRootUserPassword", "mysqlUser", "ownMysql", "serviceId", "tablePrefix", "updatedAt") SELECT "createdAt", "extraConfig", "ftpEnabled", "ftpHostKey", "ftpHostKeyPrivate", "ftpPassword", "ftpPublicPort", "ftpUser", "id", "mysqlDatabase", "mysqlHost", "mysqlPassword", "mysqlPort", "mysqlPublicPort", "mysqlRootUser", "mysqlRootUserPassword", "mysqlUser", "ownMysql", "serviceId", "tablePrefix", "updatedAt" FROM "Wordpress";
+DROP TABLE "Wordpress";
+ALTER TABLE "new_Wordpress" RENAME TO "Wordpress";
+CREATE UNIQUE INDEX "Wordpress_serviceId_key" ON "Wordpress"("serviceId");
+PRAGMA foreign_key_check;
+PRAGMA foreign_keys=ON;

apps/api/prisma/migrations/20221104092223_default_redirect_proxy/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "Setting" ADD COLUMN "proxyDefaultRedirect" TEXT;

apps/api/prisma/migrations/20221114093217_application_storage_path_migration/migration.sql

@@ -0,0 +1,45 @@
+-- RedefineTables
+PRAGMA foreign_keys=OFF;
+CREATE TABLE "new_Setting" (
+    "id" TEXT NOT NULL PRIMARY KEY,
+    "fqdn" TEXT,
+    "isAPIDebuggingEnabled" BOOLEAN DEFAULT false,
+    "isRegistrationEnabled" BOOLEAN NOT NULL DEFAULT false,
+    "dualCerts" BOOLEAN NOT NULL DEFAULT false,
+    "minPort" INTEGER NOT NULL DEFAULT 9000,
+    "maxPort" INTEGER NOT NULL DEFAULT 9100,
+    "proxyPassword" TEXT NOT NULL,
+    "proxyUser" TEXT NOT NULL,
+    "proxyHash" TEXT,
+    "proxyDefaultRedirect" TEXT,
+    "isAutoUpdateEnabled" BOOLEAN NOT NULL DEFAULT false,
+    "isDNSCheckEnabled" BOOLEAN NOT NULL DEFAULT true,
+    "DNSServers" TEXT,
+    "isTraefikUsed" BOOLEAN NOT NULL DEFAULT true,
+    "createdAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" DATETIME NOT NULL,
+    "ipv4" TEXT,
+    "ipv6" TEXT,
+    "arch" TEXT,
+    "concurrentBuilds" INTEGER NOT NULL DEFAULT 1,
+    "applicationStoragePathMigrationFinished" BOOLEAN NOT NULL DEFAULT false
+);
+INSERT INTO "new_Setting" ("DNSServers", "arch", "concurrentBuilds", "createdAt", "dualCerts", "fqdn", "id", "ipv4", "ipv6", "isAPIDebuggingEnabled", "isAutoUpdateEnabled", "isDNSCheckEnabled", "isRegistrationEnabled", "isTraefikUsed", "maxPort", "minPort", "proxyDefaultRedirect", "proxyHash", "proxyPassword", "proxyUser", "updatedAt") SELECT "DNSServers", "arch", "concurrentBuilds", "createdAt", "dualCerts", "fqdn", "id", "ipv4", "ipv6", "isAPIDebuggingEnabled", "isAutoUpdateEnabled", "isDNSCheckEnabled", "isRegistrationEnabled", "isTraefikUsed", "maxPort", "minPort", "proxyDefaultRedirect", "proxyHash", "proxyPassword", "proxyUser", "updatedAt" FROM "Setting";
+DROP TABLE "Setting";
+ALTER TABLE "new_Setting" RENAME TO "Setting";
+CREATE UNIQUE INDEX "Setting_fqdn_key" ON "Setting"("fqdn");
+CREATE TABLE "new_ApplicationPersistentStorage" (
+    "id" TEXT NOT NULL PRIMARY KEY,
+    "applicationId" TEXT NOT NULL,
+    "path" TEXT NOT NULL,
+    "oldPath" BOOLEAN NOT NULL DEFAULT false,
+    "createdAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" DATETIME NOT NULL,
+    CONSTRAINT "ApplicationPersistentStorage_applicationId_fkey" FOREIGN KEY ("applicationId") REFERENCES "Application" ("id") ON DELETE RESTRICT ON UPDATE CASCADE
+);
+INSERT INTO "new_ApplicationPersistentStorage" ("applicationId", "createdAt", "id", "path", "updatedAt") SELECT "applicationId", "createdAt", "id", "path", "updatedAt" FROM "ApplicationPersistentStorage";
+DROP TABLE "ApplicationPersistentStorage";
+ALTER TABLE "new_ApplicationPersistentStorage" RENAME TO "ApplicationPersistentStorage";
+CREATE UNIQUE INDEX "ApplicationPersistentStorage_applicationId_path_key" ON "ApplicationPersistentStorage"("applicationId", "path");
+PRAGMA foreign_key_check;
+PRAGMA foreign_keys=ON;

apps/api/prisma/migrations/20221123122143_remote_haproxy_from_db/migration.sql

@@ -0,0 +1,37 @@
+/*
+  Warnings:
+
+  - You are about to drop the column `proxyHash` on the `Setting` table. All the data in the column will be lost.
+  - You are about to drop the column `proxyPassword` on the `Setting` table. All the data in the column will be lost.
+  - You are about to drop the column `proxyUser` on the `Setting` table. All the data in the column will be lost.
+
+*/
+-- RedefineTables
+PRAGMA foreign_keys=OFF;
+CREATE TABLE "new_Setting" (
+    "id" TEXT NOT NULL PRIMARY KEY,
+    "fqdn" TEXT,
+    "dualCerts" BOOLEAN NOT NULL DEFAULT false,
+    "minPort" INTEGER NOT NULL DEFAULT 9000,
+    "maxPort" INTEGER NOT NULL DEFAULT 9100,
+    "DNSServers" TEXT,
+    "ipv4" TEXT,
+    "ipv6" TEXT,
+    "arch" TEXT,
+    "concurrentBuilds" INTEGER NOT NULL DEFAULT 1,
+    "applicationStoragePathMigrationFinished" BOOLEAN NOT NULL DEFAULT false,
+    "proxyDefaultRedirect" TEXT,
+    "isAPIDebuggingEnabled" BOOLEAN DEFAULT false,
+    "isRegistrationEnabled" BOOLEAN NOT NULL DEFAULT false,
+    "isAutoUpdateEnabled" BOOLEAN NOT NULL DEFAULT false,
+    "isDNSCheckEnabled" BOOLEAN NOT NULL DEFAULT true,
+    "isTraefikUsed" BOOLEAN NOT NULL DEFAULT true,
+    "createdAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" DATETIME NOT NULL
+);
+INSERT INTO "new_Setting" ("DNSServers", "applicationStoragePathMigrationFinished", "arch", "concurrentBuilds", "createdAt", "dualCerts", "fqdn", "id", "ipv4", "ipv6", "isAPIDebuggingEnabled", "isAutoUpdateEnabled", "isDNSCheckEnabled", "isRegistrationEnabled", "isTraefikUsed", "maxPort", "minPort", "proxyDefaultRedirect", "updatedAt") SELECT "DNSServers", "applicationStoragePathMigrationFinished", "arch", "concurrentBuilds", "createdAt", "dualCerts", "fqdn", "id", "ipv4", "ipv6", "isAPIDebuggingEnabled", "isAutoUpdateEnabled", "isDNSCheckEnabled", "isRegistrationEnabled", "isTraefikUsed", "maxPort", "minPort", "proxyDefaultRedirect", "updatedAt" FROM "Setting";
+DROP TABLE "Setting";
+ALTER TABLE "new_Setting" RENAME TO "Setting";
+CREATE UNIQUE INDEX "Setting_fqdn_key" ON "Setting"("fqdn");
+PRAGMA foreign_key_check;
+PRAGMA foreign_keys=ON;

apps/api/prisma/migrations/20221123133429_docker_registries/migration.sql

@@ -0,0 +1,59 @@
+-- CreateTable
+CREATE TABLE "DockerRegistry" (
+    "id" TEXT NOT NULL PRIMARY KEY,
+    "name" TEXT NOT NULL,
+    "url" TEXT NOT NULL,
+    "username" TEXT,
+    "password" TEXT,
+    "isSystemWide" BOOLEAN NOT NULL DEFAULT false,
+    "createdAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" DATETIME NOT NULL,
+    "teamId" TEXT,
+    CONSTRAINT "DockerRegistry_teamId_fkey" FOREIGN KEY ("teamId") REFERENCES "Team" ("id") ON DELETE SET NULL ON UPDATE CASCADE
+);
+
+-- RedefineTables
+PRAGMA foreign_keys=OFF;
+CREATE TABLE "new_Application" (
+    "id" TEXT NOT NULL PRIMARY KEY,
+    "name" TEXT NOT NULL,
+    "fqdn" TEXT,
+    "repository" TEXT,
+    "configHash" TEXT,
+    "branch" TEXT,
+    "buildPack" TEXT,
+    "projectId" INTEGER,
+    "port" INTEGER,
+    "exposePort" INTEGER,
+    "installCommand" TEXT,
+    "buildCommand" TEXT,
+    "startCommand" TEXT,
+    "baseDirectory" TEXT,
+    "publishDirectory" TEXT,
+    "deploymentType" TEXT,
+    "phpModules" TEXT,
+    "pythonWSGI" TEXT,
+    "pythonModule" TEXT,
+    "pythonVariable" TEXT,
+    "dockerFileLocation" TEXT,
+    "denoMainFile" TEXT,
+    "denoOptions" TEXT,
+    "dockerComposeFile" TEXT,
+    "dockerComposeFileLocation" TEXT,
+    "dockerComposeConfiguration" TEXT,
+    "createdAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" DATETIME NOT NULL,
+    "destinationDockerId" TEXT,
+    "gitSourceId" TEXT,
+    "baseImage" TEXT,
+    "baseBuildImage" TEXT,
+    "dockerRegistryId" TEXT NOT NULL DEFAULT '0',
+    CONSTRAINT "Application_gitSourceId_fkey" FOREIGN KEY ("gitSourceId") REFERENCES "GitSource" ("id") ON DELETE SET NULL ON UPDATE CASCADE,
+    CONSTRAINT "Application_destinationDockerId_fkey" FOREIGN KEY ("destinationDockerId") REFERENCES "DestinationDocker" ("id") ON DELETE SET NULL ON UPDATE CASCADE,
+    CONSTRAINT "Application_dockerRegistryId_fkey" FOREIGN KEY ("dockerRegistryId") REFERENCES "DockerRegistry" ("id") ON DELETE RESTRICT ON UPDATE CASCADE
+);
+INSERT INTO "new_Application" ("baseBuildImage", "baseDirectory", "baseImage", "branch", "buildCommand", "buildPack", "configHash", "createdAt", "denoMainFile", "denoOptions", "deploymentType", "destinationDockerId", "dockerComposeConfiguration", "dockerComposeFile", "dockerComposeFileLocation", "dockerFileLocation", "exposePort", "fqdn", "gitSourceId", "id", "installCommand", "name", "phpModules", "port", "projectId", "publishDirectory", "pythonModule", "pythonVariable", "pythonWSGI", "repository", "startCommand", "updatedAt") SELECT "baseBuildImage", "baseDirectory", "baseImage", "branch", "buildCommand", "buildPack", "configHash", "createdAt", "denoMainFile", "denoOptions", "deploymentType", "destinationDockerId", "dockerComposeConfiguration", "dockerComposeFile", "dockerComposeFileLocation", "dockerFileLocation", "exposePort", "fqdn", "gitSourceId", "id", "installCommand", "name", "phpModules", "port", "projectId", "publishDirectory", "pythonModule", "pythonVariable", "pythonWSGI", "repository", "startCommand", "updatedAt" FROM "Application";
+DROP TABLE "Application";
+ALTER TABLE "new_Application" RENAME TO "Application";
+PRAGMA foreign_key_check;
+PRAGMA foreign_keys=ON;

apps/api/prisma/migrations/20221128104158_do_not_track/migration.sql

@@ -0,0 +1,30 @@
+-- RedefineTables
+PRAGMA foreign_keys=OFF;
+CREATE TABLE "new_Setting" (
+    "id" TEXT NOT NULL PRIMARY KEY,
+    "fqdn" TEXT,
+    "dualCerts" BOOLEAN NOT NULL DEFAULT false,
+    "minPort" INTEGER NOT NULL DEFAULT 9000,
+    "maxPort" INTEGER NOT NULL DEFAULT 9100,
+    "DNSServers" TEXT,
+    "ipv4" TEXT,
+    "ipv6" TEXT,
+    "arch" TEXT,
+    "concurrentBuilds" INTEGER NOT NULL DEFAULT 1,
+    "applicationStoragePathMigrationFinished" BOOLEAN NOT NULL DEFAULT false,
+    "proxyDefaultRedirect" TEXT,
+    "doNotTrack" BOOLEAN NOT NULL DEFAULT false,
+    "isAPIDebuggingEnabled" BOOLEAN DEFAULT false,
+    "isRegistrationEnabled" BOOLEAN NOT NULL DEFAULT false,
+    "isAutoUpdateEnabled" BOOLEAN NOT NULL DEFAULT false,
+    "isDNSCheckEnabled" BOOLEAN NOT NULL DEFAULT true,
+    "isTraefikUsed" BOOLEAN NOT NULL DEFAULT true,
+    "createdAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" DATETIME NOT NULL
+);
+INSERT INTO "new_Setting" ("DNSServers", "applicationStoragePathMigrationFinished", "arch", "concurrentBuilds", "createdAt", "dualCerts", "fqdn", "id", "ipv4", "ipv6", "isAPIDebuggingEnabled", "isAutoUpdateEnabled", "isDNSCheckEnabled", "isRegistrationEnabled", "isTraefikUsed", "maxPort", "minPort", "proxyDefaultRedirect", "updatedAt") SELECT "DNSServers", "applicationStoragePathMigrationFinished", "arch", "concurrentBuilds", "createdAt", "dualCerts", "fqdn", "id", "ipv4", "ipv6", "isAPIDebuggingEnabled", "isAutoUpdateEnabled", "isDNSCheckEnabled", "isRegistrationEnabled", "isTraefikUsed", "maxPort", "minPort", "proxyDefaultRedirect", "updatedAt" FROM "Setting";
+DROP TABLE "Setting";
+ALTER TABLE "new_Setting" RENAME TO "Setting";
+CREATE UNIQUE INDEX "Setting_fqdn_key" ON "Setting"("fqdn");
+PRAGMA foreign_key_check;
+PRAGMA foreign_keys=ON;

apps/api/prisma/migrations/20221128104718_fix_defaults/migration.sql

@@ -0,0 +1,60 @@
+-- RedefineTables
+PRAGMA foreign_keys=OFF;
+CREATE TABLE "new_Setting" (
+    "id" TEXT NOT NULL PRIMARY KEY,
+    "fqdn" TEXT,
+    "dualCerts" BOOLEAN NOT NULL DEFAULT false,
+    "minPort" INTEGER NOT NULL DEFAULT 9000,
+    "maxPort" INTEGER NOT NULL DEFAULT 9100,
+    "DNSServers" TEXT,
+    "ipv4" TEXT,
+    "ipv6" TEXT,
+    "arch" TEXT,
+    "concurrentBuilds" INTEGER NOT NULL DEFAULT 1,
+    "applicationStoragePathMigrationFinished" BOOLEAN NOT NULL DEFAULT false,
+    "proxyDefaultRedirect" TEXT,
+    "doNotTrack" BOOLEAN NOT NULL DEFAULT false,
+    "isAPIDebuggingEnabled" BOOLEAN NOT NULL DEFAULT false,
+    "isRegistrationEnabled" BOOLEAN NOT NULL DEFAULT false,
+    "isAutoUpdateEnabled" BOOLEAN NOT NULL DEFAULT false,
+    "isDNSCheckEnabled" BOOLEAN NOT NULL DEFAULT true,
+    "isTraefikUsed" BOOLEAN NOT NULL DEFAULT true,
+    "createdAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" DATETIME NOT NULL
+);
+INSERT INTO "new_Setting" ("DNSServers", "applicationStoragePathMigrationFinished", "arch", "concurrentBuilds", "createdAt", "doNotTrack", "dualCerts", "fqdn", "id", "ipv4", "ipv6", "isAPIDebuggingEnabled", "isAutoUpdateEnabled", "isDNSCheckEnabled", "isRegistrationEnabled", "isTraefikUsed", "maxPort", "minPort", "proxyDefaultRedirect", "updatedAt") SELECT "DNSServers", "applicationStoragePathMigrationFinished", "arch", "concurrentBuilds", "createdAt", "doNotTrack", "dualCerts", "fqdn", "id", "ipv4", "ipv6", coalesce("isAPIDebuggingEnabled", false) AS "isAPIDebuggingEnabled", "isAutoUpdateEnabled", "isDNSCheckEnabled", "isRegistrationEnabled", "isTraefikUsed", "maxPort", "minPort", "proxyDefaultRedirect", "updatedAt" FROM "Setting";
+DROP TABLE "Setting";
+ALTER TABLE "new_Setting" RENAME TO "Setting";
+CREATE UNIQUE INDEX "Setting_fqdn_key" ON "Setting"("fqdn");
+CREATE TABLE "new_GlitchTip" (
+    "id" TEXT NOT NULL PRIMARY KEY,
+    "postgresqlUser" TEXT NOT NULL,
+    "postgresqlPassword" TEXT NOT NULL,
+    "postgresqlDatabase" TEXT NOT NULL,
+    "postgresqlPublicPort" INTEGER,
+    "secretKeyBase" TEXT,
+    "defaultEmail" TEXT NOT NULL,
+    "defaultUsername" TEXT NOT NULL,
+    "defaultPassword" TEXT NOT NULL,
+    "defaultEmailFrom" TEXT NOT NULL DEFAULT 'glitchtip@domain.tdl',
+    "emailSmtpHost" TEXT DEFAULT 'domain.tdl',
+    "emailSmtpPort" INTEGER DEFAULT 25,
+    "emailSmtpUser" TEXT,
+    "emailSmtpPassword" TEXT,
+    "emailSmtpUseTls" BOOLEAN NOT NULL DEFAULT false,
+    "emailSmtpUseSsl" BOOLEAN NOT NULL DEFAULT false,
+    "emailBackend" TEXT,
+    "mailgunApiKey" TEXT,
+    "sendgridApiKey" TEXT,
+    "enableOpenUserRegistration" BOOLEAN NOT NULL DEFAULT true,
+    "serviceId" TEXT NOT NULL,
+    "createdAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" DATETIME NOT NULL,
+    CONSTRAINT "GlitchTip_serviceId_fkey" FOREIGN KEY ("serviceId") REFERENCES "Service" ("id") ON DELETE RESTRICT ON UPDATE CASCADE
+);
+INSERT INTO "new_GlitchTip" ("createdAt", "defaultEmail", "defaultEmailFrom", "defaultPassword", "defaultUsername", "emailBackend", "emailSmtpHost", "emailSmtpPassword", "emailSmtpPort", "emailSmtpUseSsl", "emailSmtpUseTls", "emailSmtpUser", "enableOpenUserRegistration", "id", "mailgunApiKey", "postgresqlDatabase", "postgresqlPassword", "postgresqlPublicPort", "postgresqlUser", "secretKeyBase", "sendgridApiKey", "serviceId", "updatedAt") SELECT "createdAt", "defaultEmail", "defaultEmailFrom", "defaultPassword", "defaultUsername", "emailBackend", "emailSmtpHost", "emailSmtpPassword", "emailSmtpPort", coalesce("emailSmtpUseSsl", false) AS "emailSmtpUseSsl", coalesce("emailSmtpUseTls", false) AS "emailSmtpUseTls", "emailSmtpUser", "enableOpenUserRegistration", "id", "mailgunApiKey", "postgresqlDatabase", "postgresqlPassword", "postgresqlPublicPort", "postgresqlUser", "secretKeyBase", "sendgridApiKey", "serviceId", "updatedAt" FROM "GlitchTip";
+DROP TABLE "GlitchTip";
+ALTER TABLE "new_GlitchTip" RENAME TO "GlitchTip";
+CREATE UNIQUE INDEX "GlitchTip_serviceId_key" ON "GlitchTip"("serviceId");
+PRAGMA foreign_key_check;
+PRAGMA foreign_keys=ON;

apps/api/prisma/migrations/20221128105615_custom_sentry/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "Setting" ADD COLUMN "sentryDSN" TEXT;

apps/api/prisma/migrations/20221129081832_fix_defaults/migration.sql

@@ -0,0 +1,31 @@
+-- RedefineTables
+PRAGMA foreign_keys=OFF;
+CREATE TABLE "new_Setting" (
+    "id" TEXT NOT NULL PRIMARY KEY,
+    "fqdn" TEXT,
+    "dualCerts" BOOLEAN NOT NULL DEFAULT false,
+    "minPort" INTEGER NOT NULL DEFAULT 9000,
+    "maxPort" INTEGER NOT NULL DEFAULT 9100,
+    "DNSServers" TEXT NOT NULL DEFAULT '1.1.1.1,8.8.8.8',
+    "ipv4" TEXT,
+    "ipv6" TEXT,
+    "arch" TEXT,
+    "concurrentBuilds" INTEGER NOT NULL DEFAULT 1,
+    "applicationStoragePathMigrationFinished" BOOLEAN NOT NULL DEFAULT false,
+    "proxyDefaultRedirect" TEXT,
+    "doNotTrack" BOOLEAN NOT NULL DEFAULT false,
+    "sentryDSN" TEXT,
+    "isAPIDebuggingEnabled" BOOLEAN NOT NULL DEFAULT false,
+    "isRegistrationEnabled" BOOLEAN NOT NULL DEFAULT true,
+    "isAutoUpdateEnabled" BOOLEAN NOT NULL DEFAULT false,
+    "isDNSCheckEnabled" BOOLEAN NOT NULL DEFAULT true,
+    "isTraefikUsed" BOOLEAN NOT NULL DEFAULT true,
+    "createdAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" DATETIME NOT NULL
+);
+INSERT INTO "new_Setting" ("DNSServers", "applicationStoragePathMigrationFinished", "arch", "concurrentBuilds", "createdAt", "doNotTrack", "dualCerts", "fqdn", "id", "ipv4", "ipv6", "isAPIDebuggingEnabled", "isAutoUpdateEnabled", "isDNSCheckEnabled", "isRegistrationEnabled", "isTraefikUsed", "maxPort", "minPort", "proxyDefaultRedirect", "sentryDSN", "updatedAt") SELECT coalesce("DNSServers", '1.1.1.1,8.8.8.8') AS "DNSServers", "applicationStoragePathMigrationFinished", "arch", "concurrentBuilds", "createdAt", "doNotTrack", "dualCerts", "fqdn", "id", "ipv4", "ipv6", "isAPIDebuggingEnabled", "isAutoUpdateEnabled", "isDNSCheckEnabled", "isRegistrationEnabled", "isTraefikUsed", "maxPort", "minPort", "proxyDefaultRedirect", "sentryDSN", "updatedAt" FROM "Setting";
+DROP TABLE "Setting";
+ALTER TABLE "new_Setting" RENAME TO "Setting";
+CREATE UNIQUE INDEX "Setting_fqdn_key" ON "Setting"("fqdn");
+PRAGMA foreign_key_check;
+PRAGMA foreign_keys=ON;

apps/api/prisma/migrations/20221129121702_preview_separator/migration.sql

@@ -0,0 +1,33 @@
+-- RedefineTables
+PRAGMA foreign_keys=OFF;
+CREATE TABLE "new_Setting" (
+    "id" TEXT NOT NULL PRIMARY KEY,
+    "fqdn" TEXT,
+    "dualCerts" BOOLEAN NOT NULL DEFAULT false,
+    "minPort" INTEGER NOT NULL DEFAULT 9000,
+    "maxPort" INTEGER NOT NULL DEFAULT 9100,
+    "DNSServers" TEXT NOT NULL DEFAULT '1.1.1.1,8.8.8.8',
+    "ipv4" TEXT,
+    "ipv6" TEXT,
+    "arch" TEXT,
+    "concurrentBuilds" INTEGER NOT NULL DEFAULT 1,
+    "applicationStoragePathMigrationFinished" BOOLEAN NOT NULL DEFAULT false,
+    "numberOfDockerImagesKeptLocally" INTEGER NOT NULL DEFAULT 3,
+    "proxyDefaultRedirect" TEXT,
+    "doNotTrack" BOOLEAN NOT NULL DEFAULT false,
+    "sentryDSN" TEXT,
+    "previewSeparator" TEXT NOT NULL DEFAULT '.',
+    "isAPIDebuggingEnabled" BOOLEAN NOT NULL DEFAULT false,
+    "isRegistrationEnabled" BOOLEAN NOT NULL DEFAULT true,
+    "isAutoUpdateEnabled" BOOLEAN NOT NULL DEFAULT false,
+    "isDNSCheckEnabled" BOOLEAN NOT NULL DEFAULT true,
+    "isTraefikUsed" BOOLEAN NOT NULL DEFAULT true,
+    "createdAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" DATETIME NOT NULL
+);
+INSERT INTO "new_Setting" ("DNSServers", "applicationStoragePathMigrationFinished", "arch", "concurrentBuilds", "createdAt", "doNotTrack", "dualCerts", "fqdn", "id", "ipv4", "ipv6", "isAPIDebuggingEnabled", "isAutoUpdateEnabled", "isDNSCheckEnabled", "isRegistrationEnabled", "isTraefikUsed", "maxPort", "minPort", "numberOfDockerImagesKeptLocally", "proxyDefaultRedirect", "sentryDSN", "updatedAt") SELECT "DNSServers", "applicationStoragePathMigrationFinished", "arch", "concurrentBuilds", "createdAt", "doNotTrack", "dualCerts", "fqdn", "id", "ipv4", "ipv6", "isAPIDebuggingEnabled", "isAutoUpdateEnabled", "isDNSCheckEnabled", "isRegistrationEnabled", "isTraefikUsed", "maxPort", "minPort", 3, "proxyDefaultRedirect", "sentryDSN", "updatedAt" FROM "Setting";
+DROP TABLE "Setting";
+ALTER TABLE "new_Setting" RENAME TO "Setting";
+CREATE UNIQUE INDEX "Setting_fqdn_key" ON "Setting"("fqdn");
+PRAGMA foreign_key_check;
+PRAGMA foreign_keys=ON;

apps/api/prisma/migrations/20221129130036_keep_local_docker_images/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "Application" ADD COLUMN "gitCommitHash" TEXT;

apps/api/prisma/migrations/20221130142058_reconfigure_docker_registries/migration.sql

@@ -0,0 +1,66 @@
+/*
+  Warnings:
+
+  - You are about to drop the column `isSystemWide` on the `DockerRegistry` table. All the data in the column will be lost.
+
+*/
+-- RedefineTables
+PRAGMA foreign_keys=OFF;
+CREATE TABLE "new_DockerRegistry" (
+    "id" TEXT NOT NULL PRIMARY KEY,
+    "name" TEXT NOT NULL,
+    "url" TEXT NOT NULL,
+    "username" TEXT,
+    "password" TEXT,
+    "createdAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" DATETIME NOT NULL,
+    "teamId" TEXT,
+    CONSTRAINT "DockerRegistry_teamId_fkey" FOREIGN KEY ("teamId") REFERENCES "Team" ("id") ON DELETE SET NULL ON UPDATE CASCADE
+);
+INSERT INTO "new_DockerRegistry" ("createdAt", "id", "name", "password", "teamId", "updatedAt", "url", "username") SELECT "createdAt", "id", "name", "password", "teamId", "updatedAt", "url", "username" FROM "DockerRegistry";
+DROP TABLE "DockerRegistry";
+ALTER TABLE "new_DockerRegistry" RENAME TO "DockerRegistry";
+CREATE TABLE "new_Application" (
+    "id" TEXT NOT NULL PRIMARY KEY,
+    "name" TEXT NOT NULL,
+    "fqdn" TEXT,
+    "repository" TEXT,
+    "configHash" TEXT,
+    "branch" TEXT,
+    "buildPack" TEXT,
+    "projectId" INTEGER,
+    "port" INTEGER,
+    "exposePort" INTEGER,
+    "installCommand" TEXT,
+    "buildCommand" TEXT,
+    "startCommand" TEXT,
+    "baseDirectory" TEXT,
+    "publishDirectory" TEXT,
+    "deploymentType" TEXT,
+    "phpModules" TEXT,
+    "pythonWSGI" TEXT,
+    "pythonModule" TEXT,
+    "pythonVariable" TEXT,
+    "dockerFileLocation" TEXT,
+    "denoMainFile" TEXT,
+    "denoOptions" TEXT,
+    "dockerComposeFile" TEXT,
+    "dockerComposeFileLocation" TEXT,
+    "dockerComposeConfiguration" TEXT,
+    "createdAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" DATETIME NOT NULL,
+    "destinationDockerId" TEXT,
+    "gitSourceId" TEXT,
+    "gitCommitHash" TEXT,
+    "baseImage" TEXT,
+    "baseBuildImage" TEXT,
+    "dockerRegistryId" TEXT,
+    CONSTRAINT "Application_gitSourceId_fkey" FOREIGN KEY ("gitSourceId") REFERENCES "GitSource" ("id") ON DELETE SET NULL ON UPDATE CASCADE,
+    CONSTRAINT "Application_destinationDockerId_fkey" FOREIGN KEY ("destinationDockerId") REFERENCES "DestinationDocker" ("id") ON DELETE SET NULL ON UPDATE CASCADE,
+    CONSTRAINT "Application_dockerRegistryId_fkey" FOREIGN KEY ("dockerRegistryId") REFERENCES "DockerRegistry" ("id") ON DELETE SET NULL ON UPDATE CASCADE
+);
+INSERT INTO "new_Application" ("baseBuildImage", "baseDirectory", "baseImage", "branch", "buildCommand", "buildPack", "configHash", "createdAt", "denoMainFile", "denoOptions", "deploymentType", "destinationDockerId", "dockerComposeConfiguration", "dockerComposeFile", "dockerComposeFileLocation", "dockerFileLocation", "dockerRegistryId", "exposePort", "fqdn", "gitCommitHash", "gitSourceId", "id", "installCommand", "name", "phpModules", "port", "projectId", "publishDirectory", "pythonModule", "pythonVariable", "pythonWSGI", "repository", "startCommand", "updatedAt") SELECT "baseBuildImage", "baseDirectory", "baseImage", "branch", "buildCommand", "buildPack", "configHash", "createdAt", "denoMainFile", "denoOptions", "deploymentType", "destinationDockerId", "dockerComposeConfiguration", "dockerComposeFile", "dockerComposeFileLocation", "dockerFileLocation", "dockerRegistryId", "exposePort", "fqdn", "gitCommitHash", "gitSourceId", "id", "installCommand", "name", "phpModules", "port", "projectId", "publishDirectory", "pythonModule", "pythonVariable", "pythonWSGI", "repository", "startCommand", "updatedAt" FROM "Application";
+DROP TABLE "Application";
+ALTER TABLE "new_Application" RENAME TO "Application";
+PRAGMA foreign_key_check;
+PRAGMA foreign_keys=ON;

apps/api/prisma/migrations/20221201115801_simple_dockerfile_deployment/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "Application" ADD COLUMN "simpleDockerfile" TEXT;

apps/api/prisma/migrations/20221201133847_push_image_to_docker_registry/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "Application" ADD COLUMN "dockerRegistryImageName" TEXT;

apps/api/prisma/schema.prisma

@@ -8,27 +8,40 @@ datasource db {
   url      = env("COOLIFY_DATABASE_URL")
 }
 
+model Certificate {
+  id        String   @id @default(cuid())
+  key       String
+  cert      String
+  team      Team?    @relation(fields: [teamId], references: [id])
+  createdAt DateTime @default(now())
+  updatedAt DateTime @updatedAt
+  teamId    String?
+}
+
 model Setting {
-  id                    String   @id @default(cuid())
-  fqdn                  String?  @unique
-  isAPIDebuggingEnabled Boolean? @default(false)
-  isRegistrationEnabled Boolean  @default(false)
-  dualCerts             Boolean  @default(false)
-  minPort               Int      @default(9000)
-  maxPort               Int      @default(9100)
-  proxyPassword         String
-  proxyUser             String
-  proxyHash             String?
-  isAutoUpdateEnabled   Boolean  @default(false)
-  isDNSCheckEnabled     Boolean  @default(true)
-  DNSServers            String?
-  isTraefikUsed         Boolean  @default(true)
-  createdAt             DateTime @default(now())
-  updatedAt             DateTime @updatedAt
-  ipv4                  String?
-  ipv6                  String?
-  arch                  String?
-  concurrentBuilds      Int      @default(1)
+  id                                      String   @id @default(cuid())
+  fqdn                                    String?  @unique
+  dualCerts                               Boolean  @default(false)
+  minPort                                 Int      @default(9000)
+  maxPort                                 Int      @default(9100)
+  DNSServers                              String   @default("1.1.1.1,8.8.8.8")
+  ipv4                                    String?
+  ipv6                                    String?
+  arch                                    String?
+  concurrentBuilds                        Int      @default(1)
+  applicationStoragePathMigrationFinished Boolean  @default(false)
+  numberOfDockerImagesKeptLocally         Int      @default(3)
+  proxyDefaultRedirect                    String?
+  doNotTrack                              Boolean  @default(false)
+  sentryDSN                               String?
+  previewSeparator                        String   @default(".")
+  isAPIDebuggingEnabled                   Boolean  @default(false)
+  isRegistrationEnabled                   Boolean  @default(true)
+  isAutoUpdateEnabled                     Boolean  @default(false)
+  isDNSCheckEnabled                       Boolean  @default(true)
+  isTraefikUsed                           Boolean  @default(true)
+  createdAt                               DateTime @default(now())
+  updatedAt                               DateTime @updatedAt
 }
 
 model User {
@@ -70,6 +83,8 @@ model Team {
   gitLabApps        GitlabApp[]
   service           Service[]
   users             User[]
+  certificate       Certificate[]
+  dockerRegistry    DockerRegistry[]
 }
 
 model TeamInvitation {
@@ -83,42 +98,64 @@ model TeamInvitation {
 }
 
 model Application {
-  id                  String                         @id @default(cuid())
-  name                String
-  fqdn                String?
-  repository          String?
-  configHash          String?
-  branch              String?
-  buildPack           String?
-  projectId           Int?
-  port                Int?
-  exposePort          Int?
-  installCommand      String?
-  buildCommand        String?
-  startCommand        String?
-  baseDirectory       String?
-  publishDirectory    String?
-  deploymentType      String?
-  phpModules          String?
-  pythonWSGI          String?
-  pythonModule        String?
-  pythonVariable      String?
-  dockerFileLocation  String?
-  denoMainFile        String?
-  denoOptions         String?
-  createdAt           DateTime                       @default(now())
-  updatedAt           DateTime                       @updatedAt
-  destinationDockerId String?
-  gitSourceId         String?
-  baseImage           String?
-  baseBuildImage      String?
-  gitSource           GitSource?                     @relation(fields: [gitSourceId], references: [id])
-  destinationDocker   DestinationDocker?             @relation(fields: [destinationDockerId], references: [id])
-  persistentStorage   ApplicationPersistentStorage[]
-  settings            ApplicationSettings?
-  secrets             Secret[]
-  teams               Team[]
-  connectedDatabase   ApplicationConnectedDatabase?
+  id                         String               @id @default(cuid())
+  name                       String
+  fqdn                       String?
+  repository                 String?
+  configHash                 String?
+  branch                     String?
+  buildPack                  String?
+  projectId                  Int?
+  port                       Int?
+  exposePort                 Int?
+  installCommand             String?
+  buildCommand               String?
+  startCommand               String?
+  baseDirectory              String?
+  publishDirectory           String?
+  deploymentType             String?
+  phpModules                 String?
+  pythonWSGI                 String?
+  pythonModule               String?
+  pythonVariable             String?
+  dockerFileLocation         String?
+  denoMainFile               String?
+  denoOptions                String?
+  dockerComposeFile          String?
+  dockerComposeFileLocation  String?
+  dockerComposeConfiguration String?
+  createdAt                  DateTime             @default(now())
+  updatedAt                  DateTime             @updatedAt
+  destinationDockerId        String?
+  gitSourceId                String?
+  gitCommitHash              String?
+  baseImage                  String?
+  baseBuildImage             String?
+  settings                   ApplicationSettings?
+  dockerRegistryId           String?
+  dockerRegistryImageName    String?
+  simpleDockerfile           String?
+
+  persistentStorage  ApplicationPersistentStorage[]
+  secrets            Secret[]
+  teams              Team[]
+  connectedDatabase  ApplicationConnectedDatabase?
+  previewApplication PreviewApplication[]
+  gitSource          GitSource?                     @relation(fields: [gitSourceId], references: [id])
+  destinationDocker  DestinationDocker?             @relation(fields: [destinationDockerId], references: [id])
+  dockerRegistry     DockerRegistry?                @relation(fields: [dockerRegistryId], references: [id])
+}
+
+model PreviewApplication {
+  id                 String      @id @default(cuid())
+  pullmergeRequestId String
+  sourceBranch       String
+  isRandomDomain     Boolean     @default(false)
+  customDomain       String?
+  applicationId      String
+  application        Application @relation(fields: [applicationId], references: [id])
+  createdAt          DateTime    @default(now())
+  updatedAt          DateTime    @updatedAt
 }
 
 model ApplicationConnectedDatabase {
@@ -148,6 +185,7 @@ model ApplicationSettings {
   isBot              Boolean     @default(false)
   isPublicRepository Boolean     @default(false)
   isDBBranching      Boolean     @default(false)
+  isCustomSSL        Boolean     @default(false)
   createdAt          DateTime    @default(now())
   updatedAt          DateTime    @updatedAt
   application        Application @relation(fields: [applicationId], references: [id])
@@ -157,6 +195,7 @@ model ApplicationPersistentStorage {
   id            String      @id @default(cuid())
   applicationId String
   path          String
+  oldPath       Boolean     @default(false)
   createdAt     DateTime    @default(now())
   updatedAt     DateTime    @updatedAt
   application   Application @relation(fields: [applicationId], references: [id])
@@ -165,14 +204,17 @@ model ApplicationPersistentStorage {
 }
 
 model ServicePersistentStorage {
-  id        String   @id @default(cuid())
-  serviceId String
-  path      String
-  createdAt DateTime @default(now())
-  updatedAt DateTime @updatedAt
-  service   Service  @relation(fields: [serviceId], references: [id])
+  id          String   @id @default(cuid())
+  serviceId   String
+  path        String
+  volumeName  String?
+  predefined  Boolean  @default(false)
+  containerId String?
+  createdAt   DateTime @default(now())
+  updatedAt   DateTime @updatedAt
+  service     Service  @relation(fields: [serviceId], references: [id])
 
-  @@unique([serviceId, path])
+  @@unique([serviceId, containerId, path])
 }
 
 model Secret {
@@ -210,21 +252,23 @@ model BuildLog {
 }
 
 model Build {
-  id                  String   @id @default(cuid())
-  type                String
-  applicationId       String?
-  destinationDockerId String?
-  gitSourceId         String?
-  githubAppId         String?
-  gitlabAppId         String?
-  commit              String?
-  pullmergeRequestId  String?
-  forceRebuild        Boolean  @default(false)
-  sourceBranch        String?
-  branch              String?
-  status              String?  @default("queued")
-  createdAt           DateTime @default(now())
-  updatedAt           DateTime @updatedAt
+  id                   String   @id @default(cuid())
+  type                 String
+  applicationId        String?
+  destinationDockerId  String?
+  gitSourceId          String?
+  githubAppId          String?
+  gitlabAppId          String?
+  commit               String?
+  pullmergeRequestId   String?
+  previewApplicationId String?
+  forceRebuild         Boolean  @default(false)
+  sourceBranch         String?
+  sourceRepository     String?
+  branch               String?
+  status               String?  @default("queued")
+  createdAt            DateTime @default(now())
+  updatedAt            DateTime @updatedAt
 }
 
 model DestinationDocker {
@@ -260,6 +304,19 @@ model SshKey {
   destinationDocker DestinationDocker[]
 }
 
+model DockerRegistry {
+  id          String        @id @default(cuid())
+  name        String
+  url         String
+  username    String?
+  password    String?
+  createdAt   DateTime      @default(now())
+  updatedAt   DateTime      @updatedAt
+  teamId      String?
+  team        Team?         @relation(fields: [teamId], references: [id])
+  application Application[]
+}
+
 model GitSource {
   id           String        @id @default(cuid())
   name         String
@@ -273,6 +330,7 @@ model GitSource {
   updatedAt    DateTime      @updatedAt
   githubAppId  String?       @unique
   gitlabAppId  String?       @unique
+  isSystemWide Boolean       @default(false)
   gitlabApp    GitlabApp?    @relation(fields: [gitlabAppId], references: [id])
   githubApp    GithubApp?    @relation(fields: [githubAppId], references: [id])
   application  Application[]
@@ -361,12 +419,14 @@ model Service {
   dualCerts           Boolean                    @default(false)
   type                String?
   version             String?
+  templateVersion     String                     @default("0.0.0")
   destinationDockerId String?
   createdAt           DateTime                   @default(now())
   updatedAt           DateTime                   @updatedAt
   destinationDocker   DestinationDocker?         @relation(fields: [destinationDockerId], references: [id])
   persistentStorage   ServicePersistentStorage[]
   serviceSecret       ServiceSecret[]
+  serviceSetting      ServiceSetting[]
   teams               Team[]
 
   fider              Fider?
@@ -386,6 +446,19 @@ model Service {
   taiga              Taiga?
 }
 
+model ServiceSetting {
+  id           String   @id @default(cuid())
+  serviceId    String
+  name         String
+  value        String
+  variableName String
+  createdAt    DateTime @default(now())
+  updatedAt    DateTime @updatedAt
+  service      Service  @relation(fields: [serviceId], references: [id])
+
+  @@unique([serviceId, name])
+}
+
 model PlausibleAnalytics {
   id                   String   @id @default(cuid())
   email                String?
@@ -431,10 +504,10 @@ model Wordpress {
   ownMysql              Boolean  @default(false)
   mysqlHost             String?
   mysqlPort             Int?
-  mysqlUser             String
-  mysqlPassword         String
-  mysqlRootUser         String
-  mysqlRootUserPassword String
+  mysqlUser             String?
+  mysqlPassword         String?
+  mysqlRootUser         String?
+  mysqlRootUserPassword String?
   mysqlDatabase         String?
   mysqlPublicPort       Int?
   ftpEnabled            Boolean  @default(false)
@@ -574,8 +647,8 @@ model GlitchTip {
   emailSmtpPort              Int?     @default(25)
   emailSmtpUser              String?
   emailSmtpPassword          String?
-  emailSmtpUseTls            Boolean? @default(false)
-  emailSmtpUseSsl            Boolean? @default(false)
+  emailSmtpUseTls            Boolean  @default(false)
+  emailSmtpUseSsl            Boolean  @default(false)
   emailBackend               String?
   mailgunApiKey              String?
   sendgridApiKey             String?

apps/api/prisma/seed.js

@@ -1,18 +1,8 @@
 const dotEnvExtended = require('dotenv-extended');
 dotEnvExtended.load();
 const crypto = require('crypto');
-const generator = require('generate-password');
-const cuid = require('cuid');
 const { PrismaClient } = require('@prisma/client');
 const prisma = new PrismaClient();
-
-function generatePassword(length = 24) {
-	return generator.generate({
-		length,
-		numbers: true,
-		strict: true
-	});
-}
 const algorithm = 'aes-256-ctr';
 
 async function main() {
@@ -21,11 +11,8 @@ async function main() {
 	if (!settingsFound) {
 		await prisma.setting.create({
 			data: {
-				isRegistrationEnabled: true,
-				proxyPassword: encrypt(generatePassword()),
-				proxyUser: cuid(),
+				id: '0',
 				arch: process.arch,
-				DNSServers: '1.1.1.1,8.8.8.8'
 			}
 		});
 	} else {
@@ -34,11 +21,11 @@ async function main() {
 				id: settingsFound.id
 			},
 			data: {
-				isTraefikUsed: true,
-				proxyHash: null
+				id: '0'
 			}
 		});
 	}
+	// Create local docker engine
 	const localDocker = await prisma.destinationDocker.findFirst({
 		where: { engine: '/var/run/docker.sock' }
 	});
@@ -55,23 +42,18 @@ async function main() {
 
 	// Set auto-update based on env variable
 	const isAutoUpdateEnabled = process.env['COOLIFY_AUTO_UPDATE'] === 'true';
-	const settings = await prisma.setting.findFirst({});
-	if (settings) {
-		await prisma.setting.update({
-			where: {
-				id: settings.id
-			},
-			data: {
-				isAutoUpdateEnabled
-			}
-		});
-	}
+	await prisma.setting.update({
+		where: {
+			id: '0'
+		},
+		data: {
+			isAutoUpdateEnabled
+		}
+	});
+	// Create public github source
 	const github = await prisma.gitSource.findFirst({
 		where: { htmlUrl: 'https://github.com', forPublic: true }
 	});
-	const gitlab = await prisma.gitSource.findFirst({
-		where: { htmlUrl: 'https://gitlab.com', forPublic: true }
-	});
 	if (!github) {
 		await prisma.gitSource.create({
 			data: {
@@ -83,6 +65,10 @@ async function main() {
 			}
 		});
 	}
+	// Create public gitlab source
+	const gitlab = await prisma.gitSource.findFirst({
+		where: { htmlUrl: 'https://gitlab.com', forPublic: true }
+	});
 	if (!gitlab) {
 		await prisma.gitSource.create({
 			data: {
@@ -94,6 +80,16 @@ async function main() {
 			}
 		});
 	}
+	// Set new preview secrets
+	const secrets = await prisma.secret.findMany({ where: { isPRMRSecret: false } })
+	if (secrets.length > 0) {
+		for (const secret of secrets) {
+			const previewSecrets = await prisma.secret.findMany({ where: { applicationId: secret.applicationId, name: secret.name, isPRMRSecret: true } })
+			if (previewSecrets.length === 0) {
+				await prisma.secret.create({ data: { ...secret, id: undefined, isPRMRSecret: true } })
+			}
+		}
+	}
 }
 main()
 	.catch((e) => {

apps/api/scripts/generateTags.mjs

@@ -0,0 +1,67 @@
+import fs from 'fs/promises';
+import yaml from 'js-yaml';
+import got from 'got';
+
+const repositories = [];
+const templates = await fs.readFile('./apps/api/devTemplates.yaml', 'utf8');
+const devTemplates = yaml.load(templates);
+for (const template of devTemplates) {
+    let image = template.services['$$id'].image.replaceAll(':$$core_version', '');
+    if (!image.includes('/')) {
+        image = `library/${image}`;
+    }
+    repositories.push({ image, name: template.type });
+}
+const services = []
+const numberOfTags = 30;
+// const semverRegex = new RegExp(/^v?(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$/g)
+for (const repository of repositories) {
+    console.log('Querying', repository.name, 'at', repository.image);
+    let semverRegex = new RegExp(/^v?(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)$/g)
+    if (repository.name.startsWith('wordpress')) {
+        semverRegex = new RegExp(/^v?(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)-php(0|[1-9]\d*)$/g)
+    }
+    if (repository.name.startsWith('minio')) {
+        semverRegex = new RegExp(/^RELEASE.*$/g)
+    }
+    if (repository.name.startsWith('fider')) {
+        semverRegex = new RegExp(/^v?(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)-([0-9]+)$/g)
+    }
+    if (repository.name.startsWith('searxng')) {
+        semverRegex = new RegExp(/^\d{4}[\.\-](0?[1-9]|[12][0-9]|3[01])[\.\-](0?[1-9]|1[012]).*$/)
+    }
+    if (repository.name.startsWith('umami')) {
+        semverRegex = new RegExp(/^postgresql-v?(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)-([0-9]+)$/g)
+    }
+    if (repository.image.includes('ghcr.io')) {
+        const { execaCommand } = await import('execa');
+        const { stdout } = await execaCommand(`docker run --rm quay.io/skopeo/stable list-tags docker://${repository.image}`);
+        if (stdout) {
+            const json = JSON.parse(stdout);
+            const semverTags = json.Tags.filter((tag) => semverRegex.test(tag))
+            let tags = semverTags.length > 10 ? semverTags.sort().reverse().slice(0, numberOfTags) : json.Tags.sort().reverse().slice(0, numberOfTags)
+            if (!tags.includes('latest')) {
+                tags.push('latest')
+            }
+            services.push({ name: repository.name, image: repository.image, tags })
+        }
+    } else {
+        const { token } = await got.get(`https://auth.docker.io/token?service=registry.docker.io&scope=repository:${repository.image}:pull`).json()
+        let data = await got.get(`https://registry-1.docker.io/v2/${repository.image}/tags/list`, {
+            headers: {
+                Authorization: `Bearer ${token}`
+            }
+        }).json()
+        const semverTags = data.tags.filter((tag) => semverRegex.test(tag))
+        let tags = semverTags.length > 10 ? semverTags.sort().reverse().slice(0, numberOfTags) : data.tags.sort().reverse().slice(0, numberOfTags)
+        if (!tags.includes('latest')) {
+            tags.push('latest')
+        }
+        services.push({
+            name: repository.name,
+            image: repository.image,
+            tags
+        })
+    }
+}
+await fs.writeFile('./apps/api/devTags.json', JSON.stringify(services));

apps/api/src/index.ts

@@ -3,19 +3,29 @@ import cors from '@fastify/cors';
 import serve from '@fastify/static';
 import env from '@fastify/env';
 import cookie from '@fastify/cookie';
+import multipart from '@fastify/multipart';
 import path, { join } from 'path';
 import autoLoad from '@fastify/autoload';
-import { asyncExecShell, createRemoteEngineConfiguration, getDomain, isDev, listSettings, prisma, version } from './lib/common';
+import socketIO from 'fastify-socket.io'
+import socketIOServer from './realtime'
+
+import { cleanupDockerStorage, createRemoteEngineConfiguration, decrypt, executeCommand, generateDatabaseConfiguration, isDev, listSettings, prisma, sentryDSN, startTraefikProxy, startTraefikTCPProxy, version } from './lib/common';
 import { scheduler } from './lib/scheduler';
 import { compareVersions } from 'compare-versions';
 import Graceful from '@ladjs/graceful'
+import yaml from 'js-yaml'
+import fs from 'fs/promises';
+import { verifyRemoteDockerEngineFn } from './routes/api/v1/destinations/handlers';
+import { checkContainer } from './lib/docker';
+import { migrateApplicationPersistentStorage, migrateServicesToNewTemplate } from './lib';
+import { refreshTags, refreshTemplates } from './routes/api/v1/handlers';
+import * as Sentry from '@sentry/node';
 declare module 'fastify' {
 	interface FastifyInstance {
 		config: {
 			COOLIFY_APP_ID: string,
 			COOLIFY_SECRET_KEY: string,
 			COOLIFY_DATABASE_URL: string,
-			COOLIFY_SENTRY_DSN: string,
 			COOLIFY_IS_ON: string,
 			COOLIFY_WHITE_LABELED: string,
 			COOLIFY_WHITE_LABELED_ICON: string | null,
@@ -26,11 +36,14 @@ declare module 'fastify' {
 
 const port = isDev ? 3001 : 3000;
 const host = '0.0.0.0';
-prisma.setting.findFirst().then(async (settings) => {
+
+(async () => {
+	const settings = await prisma.setting.findFirst()
 	const fastify = Fastify({
 		logger: settings?.isAPIDebuggingEnabled || false,
 		trustProxy: true
 	});
+
 	const schema = {
 		type: 'object',
 		required: ['COOLIFY_SECRET_KEY', 'COOLIFY_DATABASE_URL', 'COOLIFY_IS_ON'],
@@ -45,10 +58,6 @@ prisma.setting.findFirst().then(async (settings) => {
 				type: 'string',
 				default: 'file:../db/dev.db'
 			},
-			COOLIFY_SENTRY_DSN: {
-				type: 'string',
-				default: null
-			},
 			COOLIFY_IS_ON: {
 				type: 'string',
 				default: 'docker'
@@ -68,7 +77,6 @@ prisma.setting.findFirst().then(async (settings) => {
 
 		}
 	};
-
 	const options = {
 		schema,
 		dotenv: true
@@ -88,39 +96,49 @@ prisma.setting.findFirst().then(async (settings) => {
 			return reply.status(200).sendFile('index.html');
 		});
 	}
+	fastify.register(multipart, { limits: { fileSize: 100000 } });
 	fastify.register(autoLoad, {
 		dir: join(__dirname, 'plugins')
 	});
 	fastify.register(autoLoad, {
 		dir: join(__dirname, 'routes')
 	});
-
 	fastify.register(cookie)
 	fastify.register(cors);
-	fastify.addHook('onRequest', async (request, reply) => {
-		let allowedList = ['coolify:3000'];
-		const { ipv4, ipv6, fqdn } = await prisma.setting.findFirst({})
-
-		ipv4 && allowedList.push(`${ipv4}:3000`);
-		ipv6 && allowedList.push(ipv6);
-		fqdn && allowedList.push(getDomain(fqdn));
-		isDev && allowedList.push('localhost:3000') && allowedList.push('localhost:3001') && allowedList.push('host.docker.internal:3001');
-		const remotes = await prisma.destinationDocker.findMany({ where: { remoteEngine: true, remoteVerified: true } })
-		if (remotes.length > 0) {
-			remotes.forEach(remote => {
-				allowedList.push(`${remote.remoteIpAddress}:3000`);
-			})
-		}
-		if (!allowedList.includes(request.headers.host)) {
-			// console.log('not allowed', request.headers.host)
+	fastify.register(socketIO, {
+		cors: {
+			origin: isDev ? "*" : ''
 		}
 	})
-	fastify.listen({ port, host }, async (err: any, address: any) => {
-		if (err) {
-			console.error(err);
-			process.exit(1);
-		}
+	// To detect allowed origins
+	// fastify.addHook('onRequest', async (request, reply) => {
+	// 	console.log(request.headers.host)
+	// 	let allowedList = ['coolify:3000'];
+	// 	const { ipv4, ipv6, fqdn } = await prisma.setting.findFirst({})
+
+	// 	ipv4 && allowedList.push(`${ipv4}:3000`);
+	// 	ipv6 && allowedList.push(ipv6);
+	// 	fqdn && allowedList.push(getDomain(fqdn));
+	// 	isDev && allowedList.push('localhost:3000') && allowedList.push('localhost:3001') && allowedList.push('host.docker.internal:3001');
+	// 	const remotes = await prisma.destinationDocker.findMany({ where: { remoteEngine: true, remoteVerified: true } })
+	// 	if (remotes.length > 0) {
+	// 		remotes.forEach(remote => {
+	// 			allowedList.push(`${remote.remoteIpAddress}:3000`);
+	// 		})
+	// 	}
+	// 	if (!allowedList.includes(request.headers.host)) {
+	// 		// console.log('not allowed', request.headers.host)
+	// 	}
+	// })
+
+
+	try {
+		await fastify.listen({ port, host })
+		await socketIOServer(fastify)
 		console.log(`Coolify's API is listening on ${host}:${port}`);
+
+		migrateServicesToNewTemplate();
+		await migrateApplicationPersistentStorage();
 		await initServer();
 
 		const graceful = new Graceful({ brees: [scheduler] });
@@ -130,38 +148,54 @@ prisma.setting.findFirst().then(async (settings) => {
 			if (!scheduler.workers.has('deployApplication')) {
 				scheduler.run('deployApplication');
 			}
-			if (!scheduler.workers.has('infrastructure')) {
-				scheduler.run('infrastructure');
-			}
 		}, 2000)
 
 		// autoUpdater
 		setInterval(async () => {
-			scheduler.workers.has('infrastructure') && scheduler.workers.get('infrastructure').postMessage("action:autoUpdater")
-		}, isDev ? 5000 : 60000 * 15)
+			await autoUpdater()
+		}, 60000 * 15)
 
 		// cleanupStorage
 		setInterval(async () => {
-			scheduler.workers.has('infrastructure') && scheduler.workers.get('infrastructure').postMessage("action:cleanupStorage")
-		}, isDev ? 6000 : 60000 * 10)
+			await cleanupStorage()
+		}, 60000 * 10)
 
-		// checkProxies
+		// checkProxies, checkFluentBit & refresh templates
 		setInterval(async () => {
-			scheduler.workers.has('infrastructure') && scheduler.workers.get('infrastructure').postMessage("action:checkProxies")
+			await checkProxies();
+			await checkFluentBit();
+		}, 60000)
+
+		// Refresh and check templates
+		setInterval(async () => {
+			await refreshTemplates()
+		}, 60000)
+
+		setInterval(async () => {
+			await refreshTags()
+		}, 60000)
+
+		setInterval(async () => {
+			await migrateServicesToNewTemplate()
+		}, isDev ? 10000 : 60000)
+
+		setInterval(async () => {
+			await copySSLCertificates();
 		}, 10000)
 
-		// cleanupPrismaEngines
-		// setInterval(async () => {
-		// 	scheduler.workers.has('infrastructure') && scheduler.workers.get('infrastructure').postMessage("action:cleanupPrismaEngines")
-		// }, 60000)
-
 		await Promise.all([
+			getTagsTemplates(),
 			getArch(),
 			getIPAddress(),
 			configureRemoteDockers(),
 		])
-	});
-})
+
+	} catch (error) {
+		console.error(error);
+		process.exit(1);
+	}
+})();
+
 
 async function getIPAddress() {
 	const { publicIpv4, publicIpv6 } = await import('public-ip')
@@ -169,31 +203,86 @@ async function getIPAddress() {
 		const settings = await listSettings();
 		if (!settings.ipv4) {
 			const ipv4 = await publicIpv4({ timeout: 2000 })
+			console.log(`Getting public IPv4 address...`);
 			await prisma.setting.update({ where: { id: settings.id }, data: { ipv4 } })
 		}
 
 		if (!settings.ipv6) {
 			const ipv6 = await publicIpv6({ timeout: 2000 })
+			console.log(`Getting public IPv6 address...`);
 			await prisma.setting.update({ where: { id: settings.id }, data: { ipv6 } })
 		}
 
 	} catch (error) { }
 }
-async function initServer() {
+async function getTagsTemplates() {
+	const { default: got } = await import('got')
 	try {
-		await asyncExecShell(`docker network create --attachable coolify`);
+		if (isDev) {
+			const templates = await fs.readFile('./devTemplates.yaml', 'utf8')
+			const tags = await fs.readFile('./devTags.json', 'utf8')
+			await fs.writeFile('./templates.json', JSON.stringify(yaml.load(templates)))
+			await fs.writeFile('./tags.json', tags)
+			console.log('[004] Tags and templates loaded in dev mode...')
+		} else {
+			const tags = await got.get('https://get.coollabs.io/coolify/service-tags.json').text()
+			const response = await got.get('https://get.coollabs.io/coolify/service-templates.yaml').text()
+			await fs.writeFile('/app/templates.json', JSON.stringify(yaml.load(response)))
+			await fs.writeFile('/app/tags.json', tags)
+			console.log('[004] Tags and templates loaded...')
+		}
+
+	} catch (error) {
+		console.log("Couldn't get latest templates.")
+		console.log(error)
+	}
+}
+async function initServer() {
+	const appId = process.env['COOLIFY_APP_ID'];
+	const settings = await prisma.setting.findUnique({ where: { id: '0' } })
+	try {
+		if (settings.doNotTrack === true) {
+			console.log('[000] Telemetry disabled...')
+
+		} else {
+			if (settings.sentryDSN !== sentryDSN) {
+				await prisma.setting.update({ where: { id: '0' }, data: { sentryDSN } })
+			}
+			// Initialize Sentry
+			// Sentry.init({
+			// 	dsn: sentryDSN,
+			// 	environment: isDev ? 'development' : 'production',
+			// 	release: version
+			// });
+			// console.log('[000] Sentry initialized...')
+		}
+	} catch (error) {
+		console.error(error)
+	}
+	try {
+		console.log(`[001] Initializing server...`);
+		await executeCommand({ command: `docker network create --attachable coolify` });
 	} catch (error) { }
 	try {
+		console.log(`[002] Cleanup stucked builds...`);
 		const isOlder = compareVersions('3.8.1', version);
 		if (isOlder === 1) {
 			await prisma.build.updateMany({ where: { status: { in: ['running', 'queued'] } }, data: { status: 'failed' } });
 		}
 	} catch (error) { }
+	try {
+		console.log('[003] Cleaning up old build sources under /tmp/build-sources/...');
+		await fs.rm('/tmp/build-sources', { recursive: true, force: true })
+	} catch (error) {
+		console.log(error)
+	}
 }
+
 async function getArch() {
 	try {
 		const settings = await prisma.setting.findFirst({})
 		if (settings && !settings.arch) {
+			console.log(`Getting architecture...`);
 			await prisma.setting.update({ where: { id: settings.id }, data: { arch: process.arch } })
 		}
 	} catch (error) { }
@@ -205,9 +294,244 @@ async function configureRemoteDockers() {
 			where: { remoteVerified: true, remoteEngine: true }
 		});
 		if (remoteDocker.length > 0) {
+			console.log(`Verifying Remote Docker Engines...`);
 			for (const docker of remoteDocker) {
-				await createRemoteEngineConfiguration(docker.id)
+				console.log('Verifying:', docker.remoteIpAddress)
+				await verifyRemoteDockerEngineFn(docker.id);
 			}
 		}
-	} catch (error) { }
+	} catch (error) {
+		console.log(error)
+	}
 }
+
+async function autoUpdater() {
+	try {
+		const { default: got } = await import('got')
+		const currentVersion = version;
+		const { coolify } = await got.get('https://get.coollabs.io/versions.json', {
+			searchParams: {
+				appId: process.env['COOLIFY_APP_ID'] || undefined,
+				version: currentVersion
+			}
+		}).json()
+		const latestVersion = coolify.main.version;
+		const isUpdateAvailable = compareVersions(latestVersion, currentVersion);
+		if (isUpdateAvailable === 1) {
+			const activeCount = 0
+			if (activeCount === 0) {
+				if (!isDev) {
+					const { isAutoUpdateEnabled } = await prisma.setting.findFirst();
+					if (isAutoUpdateEnabled) {
+						await executeCommand({ command: `docker pull coollabsio/coolify:${latestVersion}` })
+						await executeCommand({ shell: true, command: `env | grep '^COOLIFY' > .env` })
+						await executeCommand({ command: `sed -i '/COOLIFY_AUTO_UPDATE=/cCOOLIFY_AUTO_UPDATE=${isAutoUpdateEnabled}' .env` })
+						await executeCommand({ shell: true, command: `docker run --rm -tid --env-file .env -v /var/run/docker.sock:/var/run/docker.sock -v coolify-db coollabsio/coolify:${latestVersion} /bin/sh -c "env | grep COOLIFY > .env && echo 'TAG=${latestVersion}' >> .env && docker stop -t 0 coolify coolify-fluentbit && docker rm coolify coolify-fluentbit && docker compose pull && docker compose up -d --force-recreate"` })
+					}
+				} else {
+					console.log('Updating (not really in dev mode).');
+				}
+			}
+		}
+	} catch (error) {
+		console.log(error)
+	}
+}
+
+async function checkFluentBit() {
+	try {
+		if (!isDev) {
+			const engine = '/var/run/docker.sock';
+			const { id } = await prisma.destinationDocker.findFirst({
+				where: { engine, network: 'coolify' }
+			});
+			const { found } = await checkContainer({ dockerId: id, container: 'coolify-fluentbit', remove: true });
+			if (!found) {
+				await executeCommand({ shell: true, command: `env | grep '^COOLIFY' > .env` });
+				await executeCommand({ command: `docker compose up -d fluent-bit` });
+			}
+		}
+	} catch (error) {
+		console.log(error)
+	}
+}
+async function checkProxies() {
+	try {
+		const { default: isReachable } = await import('is-port-reachable');
+		let portReachable;
+
+		const { arch, ipv4, ipv6 } = await listSettings();
+
+		// Coolify Proxy local
+		const engine = '/var/run/docker.sock';
+		const localDocker = await prisma.destinationDocker.findFirst({
+			where: { engine, network: 'coolify', isCoolifyProxyUsed: true }
+		});
+		if (localDocker) {
+			portReachable = await isReachable(80, { host: ipv4 || ipv6 })
+			if (!portReachable) {
+				await startTraefikProxy(localDocker.id);
+			}
+		}
+		// Coolify Proxy remote
+		const remoteDocker = await prisma.destinationDocker.findMany({
+			where: { remoteEngine: true, remoteVerified: true }
+		});
+		if (remoteDocker.length > 0) {
+			for (const docker of remoteDocker) {
+				if (docker.isCoolifyProxyUsed) {
+					portReachable = await isReachable(80, { host: docker.remoteIpAddress })
+					if (!portReachable) {
+						await startTraefikProxy(docker.id);
+					}
+				}
+				try {
+					await createRemoteEngineConfiguration(docker.id)
+				} catch (error) { }
+			}
+		}
+		// TCP Proxies
+		const databasesWithPublicPort = await prisma.database.findMany({
+			where: { publicPort: { not: null } },
+			include: { settings: true, destinationDocker: true }
+		});
+		for (const database of databasesWithPublicPort) {
+			const { destinationDockerId, destinationDocker, publicPort, id } = database;
+			if (destinationDockerId && destinationDocker.isCoolifyProxyUsed) {
+				const { privatePort } = generateDatabaseConfiguration(database, arch);
+				await startTraefikTCPProxy(destinationDocker, id, publicPort, privatePort);
+			}
+		}
+		const wordpressWithFtp = await prisma.wordpress.findMany({
+			where: { ftpPublicPort: { not: null } },
+			include: { service: { include: { destinationDocker: true } } }
+		});
+		for (const ftp of wordpressWithFtp) {
+			const { service, ftpPublicPort } = ftp;
+			const { destinationDockerId, destinationDocker, id } = service;
+			if (destinationDockerId && destinationDocker.isCoolifyProxyUsed) {
+				await startTraefikTCPProxy(destinationDocker, id, ftpPublicPort, 22, 'wordpressftp');
+			}
+		}
+
+		// HTTP Proxies
+		// const minioInstances = await prisma.minio.findMany({
+		// 	where: { publicPort: { not: null } },
+		// 	include: { service: { include: { destinationDocker: true } } }
+		// });
+		// for (const minio of minioInstances) {
+		// 	const { service, publicPort } = minio;
+		// 	const { destinationDockerId, destinationDocker, id } = service;
+		// 	if (destinationDockerId && destinationDocker.isCoolifyProxyUsed) {
+		// 		await startTraefikTCPProxy(destinationDocker, id, publicPort, 9000);
+		// 	}
+		// }
+	} catch (error) {
+
+	}
+}
+
+async function copySSLCertificates() {
+	try {
+		const pAll = await import('p-all');
+		const actions = []
+		const certificates = await prisma.certificate.findMany({ include: { team: true } })
+		const teamIds = certificates.map(c => c.teamId)
+		const destinations = await prisma.destinationDocker.findMany({ where: { isCoolifyProxyUsed: true, teams: { some: { id: { in: [...teamIds] } } } } })
+		for (const certificate of certificates) {
+			const { id, key, cert } = certificate
+			const decryptedKey = decrypt(key)
+			await fs.writeFile(`/tmp/${id}-key.pem`, decryptedKey)
+			await fs.writeFile(`/tmp/${id}-cert.pem`, cert)
+			for (const destination of destinations) {
+				if (destination.remoteEngine) {
+					if (destination.remoteVerified) {
+						const { id: dockerId, remoteIpAddress } = destination
+						actions.push(async () => copyRemoteCertificates(id, dockerId, remoteIpAddress))
+					}
+				} else {
+					actions.push(async () => copyLocalCertificates(id))
+				}
+			}
+		}
+		await pAll.default(actions, { concurrency: 1 })
+	} catch (error) {
+		console.log(error)
+	} finally {
+		await executeCommand({ command: `find /tmp/ -maxdepth 1 -type f -name '*-*.pem' -delete` })
+	}
+}
+
+async function copyRemoteCertificates(id: string, dockerId: string, remoteIpAddress: string) {
+	try {
+		await executeCommand({ command: `scp /tmp/${id}-cert.pem /tmp/${id}-key.pem ${remoteIpAddress}:/tmp/` })
+		await executeCommand({ sshCommand: true, shell: true, dockerId, command: `docker exec coolify-proxy sh -c 'test -d /etc/traefik/acme/custom/ || mkdir -p /etc/traefik/acme/custom/'` })
+		await executeCommand({ sshCommand: true, dockerId, command: `docker cp /tmp/${id}-key.pem coolify-proxy:/etc/traefik/acme/custom/` })
+		await executeCommand({ sshCommand: true, dockerId, command: `docker cp /tmp/${id}-cert.pem coolify-proxy:/etc/traefik/acme/custom/` })
+	} catch (error) {
+		console.log({ error })
+	}
+}
+async function copyLocalCertificates(id: string) {
+	try {
+		await executeCommand({ command: `docker exec coolify-proxy sh -c 'test -d /etc/traefik/acme/custom/ || mkdir -p /etc/traefik/acme/custom/'`, shell: true })
+		await executeCommand({ command: `docker cp /tmp/${id}-key.pem coolify-proxy:/etc/traefik/acme/custom/` })
+		await executeCommand({ command: `docker cp /tmp/${id}-cert.pem coolify-proxy:/etc/traefik/acme/custom/` })
+	} catch (error) {
+		console.log({ error })
+	}
+}
+
+async function cleanupStorage() {
+	const destinationDockers = await prisma.destinationDocker.findMany();
+	let enginesDone = new Set()
+	for (const destination of destinationDockers) {
+		if (enginesDone.has(destination.engine) || enginesDone.has(destination.remoteIpAddress)) return
+		if (destination.engine) enginesDone.add(destination.engine)
+		if (destination.remoteIpAddress) enginesDone.add(destination.remoteIpAddress)
+
+		let lowDiskSpace = false;
+		try {
+			let stdout = null
+			if (!isDev) {
+				const output = await executeCommand({ dockerId: destination.id, command: `CONTAINER=$(docker ps -lq | head -1) && docker exec $CONTAINER sh -c 'df -kPT /'`, shell: true })
+				stdout = output.stdout;
+			} else {
+				const output = await executeCommand({
+					command:
+						`df -kPT /`
+				});
+				stdout = output.stdout;
+			}
+			let lines = stdout.trim().split('\n');
+			let header = lines[0];
+			let regex =
+				/^Filesystem\s+|Type\s+|1024-blocks|\s+Used|\s+Available|\s+Capacity|\s+Mounted on\s*$/g;
+			const boundaries = [];
+			let match;
+
+			while ((match = regex.exec(header))) {
+				boundaries.push(match[0].length);
+			}
+
+			boundaries[boundaries.length - 1] = -1;
+			const data = lines.slice(1).map((line) => {
+				const cl = boundaries.map((boundary) => {
+					const column = boundary > 0 ? line.slice(0, boundary) : line;
+					line = line.slice(boundary);
+					return column.trim();
+				});
+				return {
+					capacity: Number.parseInt(cl[5], 10) / 100
+				};
+			});
+			if (data.length > 0) {
+				const { capacity } = data[0];
+				if (capacity > 0.8) {
+					lowDiskSpace = true;
+				}
+			}
+		} catch (error) { }
+		await cleanupDockerStorage(destination.id, lowDiskSpace, false)
+	}
+}

apps/api/src/jobs/deployApplication.ts

@@ -3,8 +3,26 @@ import crypto from 'crypto';
 import fs from 'fs/promises';
 import yaml from 'js-yaml';
 
-import { copyBaseConfigurationFiles, makeLabelForStandaloneApplication, saveBuildLog, setDefaultConfiguration } from '../lib/buildPacks/common';
-import { createDirectories, decrypt, defaultComposeConfiguration, executeDockerCmd, getDomain, prisma, decryptApplication } from '../lib/common';
+import {
+	copyBaseConfigurationFiles,
+	makeLabelForSimpleDockerfile,
+	makeLabelForStandaloneApplication,
+	saveBuildLog,
+	saveDockerRegistryCredentials,
+	setDefaultConfiguration
+} from '../lib/buildPacks/common';
+import {
+	createDirectories,
+	decrypt,
+	defaultComposeConfiguration,
+	getDomain,
+	prisma,
+	decryptApplication,
+	isDev,
+	pushToRegistry,
+	executeCommand,
+	generateSecrets
+} from '../lib/common';
 import * as importers from '../lib/importers';
 import * as buildpacks from '../lib/buildPacks';
 
@@ -14,71 +32,313 @@ import * as buildpacks from '../lib/buildPacks';
 			if (message === 'error') throw new Error('oops');
 			if (message === 'cancel') {
 				parentPort.postMessage('cancelled');
-				await prisma.$disconnect()
+				await prisma.$disconnect();
 				process.exit(0);
 			}
 		});
-		const pThrottle = await import('p-throttle')
+		const pThrottle = await import('p-throttle');
 		const throttle = pThrottle.default({
 			limit: 1,
 			interval: 2000
 		});
 
-
 		const th = throttle(async () => {
 			try {
-				const queuedBuilds = await prisma.build.findMany({ where: { status: { in: ['queued', 'running'] } }, orderBy: { createdAt: 'asc' } });
-				const { concurrentBuilds } = await prisma.setting.findFirst({})
+				const queuedBuilds = await prisma.build.findMany({
+					where: { status: { in: ['queued', 'running'] } },
+					orderBy: { createdAt: 'asc' }
+				});
+				const { concurrentBuilds } = await prisma.setting.findFirst({});
 				if (queuedBuilds.length > 0) {
 					parentPort.postMessage({ deploying: true });
 					const concurrency = concurrentBuilds;
 					const pAll = await import('p-all');
-					const actions = []
+					const actions = [];
 
 					for (const queueBuild of queuedBuilds) {
 						actions.push(async () => {
-							let application = await prisma.application.findUnique({ where: { id: queueBuild.applicationId }, include: { destinationDocker: true, gitSource: { include: { githubApp: true, gitlabApp: true } }, persistentStorage: true, secrets: true, settings: true, teams: true } })
-							let { id: buildId, type, sourceBranch = null, pullmergeRequestId = null, forceRebuild } = queueBuild
-							application = decryptApplication(application)
-							try {
-								if (queueBuild.status === 'running') {
-									await saveBuildLog({ line: 'Building halted, restarting...', buildId, applicationId: application.id });
+							let application = await prisma.application.findUnique({
+								where: { id: queueBuild.applicationId },
+								include: {
+									dockerRegistry: true,
+									destinationDocker: true,
+									gitSource: { include: { githubApp: true, gitlabApp: true } },
+									persistentStorage: true,
+									secrets: true,
+									settings: true,
+									teams: true
 								}
+							});
+
+							let {
+								id: buildId,
+								type,
+								gitSourceId,
+								sourceBranch = null,
+								pullmergeRequestId = null,
+								previewApplicationId = null,
+								forceRebuild,
+								sourceRepository = null
+							} = queueBuild;
+							application = decryptApplication(application);
+
+							if (!gitSourceId && application.simpleDockerfile) {
 								const {
 									id: applicationId,
-									repository,
-									name,
 									destinationDocker,
 									destinationDockerId,
-									gitSource,
-									configHash,
-									fqdn,
-									projectId,
 									secrets,
-									phpModules,
-									settings,
-									persistentStorage,
-									pythonWSGI,
-									pythonModule,
-									pythonVariable,
-									denoOptions,
-									exposePort,
-									baseImage,
-									baseBuildImage,
-									deploymentType,
-								} = application
-								let {
-									branch,
-									buildPack,
 									port,
-									installCommand,
-									buildCommand,
-									startCommand,
-									baseDirectory,
-									publishDirectory,
-									dockerFileLocation,
-									denoMainFile
-								} = application
+									persistentStorage,
+									exposePort,
+									simpleDockerfile,
+									dockerRegistry
+								} = application;
+								const { workdir } = await createDirectories({ repository: applicationId, buildId });
+								try {
+									if (queueBuild.status === 'running') {
+										await saveBuildLog({
+											line: 'Building halted, restarting...',
+											buildId,
+											applicationId: application.id
+										});
+									}
+									const volumes =
+										persistentStorage?.map((storage) => {
+											if (storage.oldPath) {
+												return `${applicationId}${storage.path
+													.replace(/\//gi, '-')
+													.replace('-app', '')}:${storage.path}`;
+											}
+											return `${applicationId}${storage.path.replace(/\//gi, '-')}:${storage.path}`;
+										}) || [];
+
+									if (destinationDockerId) {
+										await prisma.build.update({
+											where: { id: buildId },
+											data: { status: 'running' }
+										});
+										try {
+											const { stdout: containers } = await executeCommand({
+												dockerId: destinationDockerId,
+												command: `docker ps -a --filter 'label=com.docker.compose.service=${applicationId}' --format {{.ID}}`
+											});
+											if (containers) {
+												const containerArray = containers.split('\n');
+												if (containerArray.length > 0) {
+													for (const container of containerArray) {
+														await executeCommand({
+															dockerId: destinationDockerId,
+															command: `docker stop -t 0 ${container}`
+														});
+														await executeCommand({
+															dockerId: destinationDockerId,
+															command: `docker rm --force ${container}`
+														});
+													}
+												}
+											}
+										} catch (error) {
+											//
+										}
+										let envs = ['NODE_ENV=production', `PORT=${port}`];
+										if (secrets.length > 0) {
+											envs = [...envs, ...generateSecrets(secrets, pullmergeRequestId)];
+										}
+										await fs.writeFile(`${workdir}/Dockerfile`, simpleDockerfile);
+										if (dockerRegistry) {
+											const { url, username, password } = dockerRegistry;
+											await saveDockerRegistryCredentials({ url, username, password, workdir });
+										}
+
+										const labels = makeLabelForSimpleDockerfile({
+											applicationId,
+											type,
+											port: exposePort ? `${exposePort}:${port}` : port
+										});
+										try {
+											const composeVolumes = volumes.map((volume) => {
+												return {
+													[`${volume.split(':')[0]}`]: {
+														name: volume.split(':')[0]
+													}
+												};
+											});
+											const composeFile = {
+												version: '3.8',
+												services: {
+													[applicationId]: {
+														build: {
+															context: workdir
+														},
+														image: `${applicationId}:${buildId}`,
+														container_name: applicationId,
+														volumes,
+														labels,
+														environment: envs,
+														depends_on: [],
+														expose: [port],
+														...(exposePort ? { ports: [`${exposePort}:${port}`] } : {}),
+														...defaultComposeConfiguration(destinationDocker.network)
+													}
+												},
+												networks: {
+													[destinationDocker.network]: {
+														external: true
+													}
+												},
+												volumes: Object.assign({}, ...composeVolumes)
+											};
+											await fs.writeFile(`${workdir}/docker-compose.yml`, yaml.dump(composeFile));
+											await executeCommand({
+												debug: true,
+												dockerId: destinationDocker.id,
+												command: `docker compose --project-directory ${workdir} up -d`
+											});
+											await saveBuildLog({ line: 'Deployed 🎉', buildId, applicationId });
+										} catch (error) {
+											await saveBuildLog({ line: error, buildId, applicationId });
+											const foundBuild = await prisma.build.findUnique({ where: { id: buildId } });
+											if (foundBuild) {
+												await prisma.build.update({
+													where: { id: buildId },
+													data: {
+														status: 'failed'
+													}
+												});
+											}
+											throw new Error(error);
+										}
+									}
+								} catch (error) {
+									const foundBuild = await prisma.build.findUnique({ where: { id: buildId } });
+									if (foundBuild) {
+										await prisma.build.update({
+											where: { id: buildId },
+											data: {
+												status: 'failed'
+											}
+										});
+									}
+									if (error !== 1) {
+										await saveBuildLog({ line: error, buildId, applicationId: application.id });
+									}
+									if (error instanceof Error) {
+										await saveBuildLog({
+											line: error.message,
+											buildId,
+											applicationId: application.id
+										});
+									}
+									await fs.rm(workdir, { recursive: true, force: true });
+									return;
+								}
+								try {
+									if (application.dockerRegistryImageName) {
+										const customTag = application.dockerRegistryImageName.split(':')[1] || buildId;
+										const imageName = application.dockerRegistryImageName.split(':')[0];
+										await saveBuildLog({
+											line: `Pushing ${imageName}:${customTag} to Docker Registry... It could take a while...`,
+											buildId,
+											applicationId: application.id
+										});
+										await pushToRegistry(application, workdir, buildId, imageName, customTag);
+										await saveBuildLog({ line: 'Success', buildId, applicationId: application.id });
+									}
+								} catch (error) {
+									if (error.stdout) {
+										await saveBuildLog({ line: error.stdout, buildId, applicationId });
+									}
+									if (error.stderr) {
+										await saveBuildLog({ line: error.stderr, buildId, applicationId });
+									}
+								} finally {
+									await fs.rm(workdir, { recursive: true, force: true });
+									await prisma.build.update({
+										where: { id: buildId },
+										data: { status: 'success' }
+									});
+								}
+								return;
+							}
+
+							const originalApplicationId = application.id;
+							const {
+								id: applicationId,
+								name,
+								destinationDocker,
+								destinationDockerId,
+								gitSource,
+								configHash,
+								fqdn,
+								projectId,
+								secrets,
+								phpModules,
+								settings,
+								persistentStorage,
+								pythonWSGI,
+								pythonModule,
+								pythonVariable,
+								denoOptions,
+								exposePort,
+								baseImage,
+								baseBuildImage,
+								deploymentType,
+								gitCommitHash,
+								dockerRegistry
+							} = application;
+
+							let {
+								branch,
+								repository,
+								buildPack,
+								port,
+								installCommand,
+								buildCommand,
+								startCommand,
+								baseDirectory,
+								publishDirectory,
+								dockerFileLocation,
+								dockerComposeFileLocation,
+								dockerComposeConfiguration,
+								denoMainFile
+							} = application;
+
+							let imageId = applicationId;
+							let domain = getDomain(fqdn);
+
+							let location = null;
+
+							let tag = null;
+							let customTag = null;
+							let imageName = null;
+
+							let imageFoundLocally = false;
+							let imageFoundRemotely = false;
+
+							if (pullmergeRequestId) {
+								const previewApplications = await prisma.previewApplication.findMany({
+									where: { applicationId: originalApplicationId, pullmergeRequestId }
+								});
+								if (previewApplications.length > 0) {
+									previewApplicationId = previewApplications[0].id;
+								}
+								// Previews, we need to get the source branch and set subdomain
+								branch = sourceBranch;
+								domain = `${pullmergeRequestId}.${domain}`;
+								imageId = `${applicationId}-${pullmergeRequestId}`;
+								repository = sourceRepository || repository;
+							}
+							const { workdir, repodir } = await createDirectories({ repository, buildId });
+							try {
+								if (queueBuild.status === 'running') {
+									await saveBuildLog({
+										line: 'Building halted, restarting...',
+										buildId,
+										applicationId: application.id
+									});
+								}
+
 								const currentHash = crypto
 									.createHash('sha256')
 									.update(
@@ -104,31 +364,26 @@ import * as buildpacks from '../lib/buildPacks';
 									)
 									.digest('hex');
 								const { debug } = settings;
-								if (concurrency === 1) {
-									await prisma.build.updateMany({
-										where: {
-											status: { in: ['queued', 'running'] },
-											id: { not: buildId },
-											applicationId,
-											createdAt: { lt: new Date(new Date().getTime() - 10 * 1000) }
-										},
-										data: { status: 'failed' }
+								if (!debug) {
+									await saveBuildLog({
+										line: `Debug logging is disabled. Enable it above if necessary!`,
+										buildId,
+										applicationId
 									});
 								}
-								let imageId = applicationId;
-								let domain = getDomain(fqdn);
 								const volumes =
 									persistentStorage?.map((storage) => {
-										return `${applicationId}${storage.path.replace(/\//gi, '-')}:${buildPack !== 'docker' ? '/app' : ''
-											}${storage.path}`;
+										if (storage.oldPath) {
+											return `${applicationId}${storage.path
+												.replace(/\//gi, '-')
+												.replace('-app', '')}:${storage.path}`;
+										}
+										return `${applicationId}${storage.path.replace(/\//gi, '-')}:${storage.path}`;
 									}) || [];
-								// Previews, we need to get the source branch and set subdomain
-								if (pullmergeRequestId) {
-									branch = sourceBranch;
-									domain = `${pullmergeRequestId}.${domain}`;
-									imageId = `${applicationId}-${pullmergeRequestId}`;
-								}
 
+								try {
+									dockerComposeConfiguration = JSON.parse(dockerComposeConfiguration);
+								} catch (error) {}
 								let deployNeeded = true;
 								let destinationType;
 
@@ -136,8 +391,11 @@ import * as buildpacks from '../lib/buildPacks';
 									destinationType = 'docker';
 								}
 								if (destinationType === 'docker') {
-									await prisma.build.update({ where: { id: buildId }, data: { status: 'running' } });
-									const { workdir, repodir } = await createDirectories({ repository, buildId });
+									await prisma.build.update({
+										where: { id: buildId },
+										data: { status: 'running' }
+									});
+
 									const configuration = await setDefaultConfiguration(application);
 
 									buildPack = configuration.buildPack;
@@ -146,8 +404,9 @@ import * as buildpacks from '../lib/buildPacks';
 									startCommand = configuration.startCommand;
 									buildCommand = configuration.buildCommand;
 									publishDirectory = configuration.publishDirectory;
-									baseDirectory = configuration.baseDirectory;
+									baseDirectory = configuration.baseDirectory || '';
 									dockerFileLocation = configuration.dockerFileLocation;
+									dockerComposeFileLocation = configuration.dockerComposeFileLocation;
 									denoMainFile = configuration.denoMainFile;
 									const commit = await importers[gitSource.type]({
 										applicationId,
@@ -157,6 +416,8 @@ import * as buildpacks from '../lib/buildPacks';
 										githubAppId: gitSource.githubApp?.id,
 										gitlabAppId: gitSource.gitlabApp?.id,
 										customPort: gitSource.customPort,
+										gitCommitHash,
+										configuration,
 										repository,
 										branch,
 										buildId,
@@ -170,20 +431,35 @@ import * as buildpacks from '../lib/buildPacks';
 									if (!commit) {
 										throw new Error('No commit found?');
 									}
-									let tag = commit.slice(0, 7);
+									tag = commit.slice(0, 7);
 									if (pullmergeRequestId) {
 										tag = `${commit.slice(0, 7)}-${pullmergeRequestId}`;
 									}
+									if (application.dockerRegistryImageName) {
+										imageName = application.dockerRegistryImageName.split(':')[0];
+										customTag = application.dockerRegistryImageName.split(':')[1] || tag;
+									} else {
+										customTag = tag;
+										imageName = applicationId;
+									}
+
+									if (pullmergeRequestId) {
+										customTag = `${customTag}-${pullmergeRequestId}`;
+									}
 
 									try {
 										await prisma.build.update({ where: { id: buildId }, data: { commit } });
-									} catch (err) { }
+									} catch (err) {}
 
 									if (!pullmergeRequestId) {
 										if (configHash !== currentHash) {
 											deployNeeded = true;
 											if (configHash) {
-												await saveBuildLog({ line: 'Configuration changed.', buildId, applicationId });
+												await saveBuildLog({
+													line: 'Configuration changed',
+													buildId,
+													applicationId
+												});
 											}
 										} else {
 											deployNeeded = false;
@@ -192,29 +468,78 @@ import * as buildpacks from '../lib/buildPacks';
 										deployNeeded = true;
 									}
 
-									let imageFound = false;
 									try {
-										await executeDockerCmd({
+										await executeCommand({
 											dockerId: destinationDocker.id,
 											command: `docker image inspect ${applicationId}:${tag}`
-										})
-										imageFound = true;
+										});
+										imageFoundLocally = true;
 									} catch (error) {
 										//
 									}
-									await copyBaseConfigurationFiles(buildPack, workdir, buildId, applicationId, baseImage);
+									if (dockerRegistry) {
+										const { url, username, password } = dockerRegistry;
+										location = await saveDockerRegistryCredentials({
+											url,
+											username,
+											password,
+											workdir
+										});
+									}
 
-									if (forceRebuild) deployNeeded = true
-									if (!imageFound || deployNeeded) {
-										// if (true) {
+									try {
+										await executeCommand({
+											dockerId: destinationDocker.id,
+											command: `docker ${
+												location ? `--config ${location}` : ''
+											} pull ${imageName}:${customTag}`
+										});
+										imageFoundRemotely = true;
+									} catch (error) {
+										//
+									}
+									let imageFound = `${applicationId}:${tag}`;
+									if (imageFoundRemotely) {
+										imageFound = `${imageName}:${customTag}`;
+									}
+									await copyBaseConfigurationFiles(
+										buildPack,
+										workdir,
+										buildId,
+										applicationId,
+										baseImage
+									);
+									const labels = makeLabelForStandaloneApplication({
+										applicationId,
+										fqdn,
+										name,
+										type,
+										pullmergeRequestId,
+										buildPack,
+										repository,
+										branch,
+										projectId,
+										port: exposePort ? `${exposePort}:${port}` : port,
+										commit,
+										installCommand,
+										buildCommand,
+										startCommand,
+										baseDirectory,
+										publishDirectory
+									});
+									if (forceRebuild) deployNeeded = true;
+									if ((!imageFoundLocally && !imageFoundRemotely) || deployNeeded) {
 										if (buildpacks[buildPack])
 											await buildpacks[buildPack]({
 												dockerId: destinationDocker.id,
+												network: destinationDocker.network,
 												buildId,
 												applicationId,
 												domain,
 												name,
 												type,
+												volumes,
+												labels,
 												pullmergeRequestId,
 												buildPack,
 												repository,
@@ -236,141 +561,241 @@ import * as buildpacks from '../lib/buildPacks';
 												pythonModule,
 												pythonVariable,
 												dockerFileLocation,
+												dockerComposeConfiguration,
+												dockerComposeFileLocation,
 												denoMainFile,
 												denoOptions,
 												baseImage,
 												baseBuildImage,
-												deploymentType
+												deploymentType,
+												forceRebuild
 											});
 										else {
-											await saveBuildLog({ line: `Build pack ${buildPack} not found`, buildId, applicationId });
+											await saveBuildLog({
+												line: `Build pack ${buildPack} not found`,
+												buildId,
+												applicationId
+											});
 											throw new Error(`Build pack ${buildPack} not found.`);
 										}
 									} else {
-										await saveBuildLog({ line: 'Build image already available - no rebuild required.', buildId, applicationId });
+										if (imageFoundRemotely || deployNeeded) {
+											await saveBuildLog({
+												line: `Container image ${imageFound} found in Docker Registry - reuising it`,
+												buildId,
+												applicationId
+											});
+										} else {
+											if (imageFoundLocally || deployNeeded) {
+												await saveBuildLog({
+													line: `Container image ${imageFound} found locally - reuising it`,
+													buildId,
+													applicationId
+												});
+											}
+										}
 									}
-									try {
-										await executeDockerCmd({ dockerId: destinationDocker.id, command: `docker stop -t 0 ${imageId}` })
-										await executeDockerCmd({ dockerId: destinationDocker.id, command: `docker rm ${imageId}` })
-									} catch (error) {
-										//
-									}
-									const envs = [
-										`PORT=${port}`
-									];
-									if (secrets.length > 0) {
-										secrets.forEach((secret) => {
-											if (pullmergeRequestId) {
-												const isSecretFound = secrets.filter(s => s.name === secret.name && s.isPRMRSecret)
-												if (isSecretFound.length > 0) {
-													envs.push(`${secret.name}=${isSecretFound[0].value}`);
-												} else {
-													envs.push(`${secret.name}=${secret.value}`);
-												}
-											} else {
-												if (!secret.isPRMRSecret) {
-													envs.push(`${secret.name}=${secret.value}`);
+
+									if (buildPack === 'compose') {
+										try {
+											const { stdout: containers } = await executeCommand({
+												dockerId: destinationDockerId,
+												command: `docker ps -a --filter 'label=coolify.applicationId=${applicationId}' --format {{.ID}}`
+											});
+											if (containers) {
+												const containerArray = containers.split('\n');
+												if (containerArray.length > 0) {
+													for (const container of containerArray) {
+														await executeCommand({
+															dockerId: destinationDockerId,
+															command: `docker stop -t 0 ${container}`
+														});
+														await executeCommand({
+															dockerId: destinationDockerId,
+															command: `docker rm --force ${container}`
+														});
+													}
 												}
 											}
-										});
-									}
-									await fs.writeFile(`${workdir}/.env`, envs.join('\n'));
-									const labels = makeLabelForStandaloneApplication({
-										applicationId,
-										fqdn,
-										name,
-										type,
-										pullmergeRequestId,
-										buildPack,
-										repository,
-										branch,
-										projectId,
-										port: exposePort ? `${exposePort}:${port}` : port,
-										commit,
-										installCommand,
-										buildCommand,
-										startCommand,
-										baseDirectory,
-										publishDirectory
-									});
-									let envFound = false;
-									try {
-										envFound = !!(await fs.stat(`${workdir}/.env`));
-									} catch (error) {
-										//
-									}
-									try {
-										await saveBuildLog({ line: 'Deployment started.', buildId, applicationId });
-										const composeVolumes = volumes.map((volume) => {
-											return {
-												[`${volume.split(':')[0]}`]: {
-													name: volume.split(':')[0]
+										} catch (error) {
+											//
+										}
+										try {
+											await executeCommand({
+												debug,
+												buildId,
+												applicationId,
+												dockerId: destinationDocker.id,
+												command: `docker compose --project-directory ${workdir} up -d`
+											});
+											await saveBuildLog({ line: 'Deployed 🎉', buildId, applicationId });
+											await prisma.build.update({
+												where: { id: buildId },
+												data: { status: 'success' }
+											});
+											await prisma.application.update({
+												where: { id: applicationId },
+												data: { configHash: currentHash }
+											});
+										} catch (error) {
+											await saveBuildLog({ line: error, buildId, applicationId });
+											const foundBuild = await prisma.build.findUnique({ where: { id: buildId } });
+											if (foundBuild) {
+												await prisma.build.update({
+													where: { id: buildId },
+													data: {
+														status: 'failed'
+													}
+												});
+											}
+											throw new Error(error);
+										}
+									} else {
+										try {
+											const { stdout: containers } = await executeCommand({
+												dockerId: destinationDockerId,
+												command: `docker ps -a --filter 'label=com.docker.compose.service=${
+													pullmergeRequestId ? imageId : applicationId
+												}' --format {{.ID}}`
+											});
+											if (containers) {
+												const containerArray = containers.split('\n');
+												if (containerArray.length > 0) {
+													for (const container of containerArray) {
+														await executeCommand({
+															dockerId: destinationDockerId,
+															command: `docker stop -t 0 ${container}`
+														});
+														await executeCommand({
+															dockerId: destinationDockerId,
+															command: `docker rm --force ${container}`
+														});
+													}
 												}
+											}
+										} catch (error) {
+											//
+										}
+										let envs = ['NODE_ENV=production', `PORT=${port}`];
+										if (secrets.length > 0) {
+											envs = [...envs, ...generateSecrets(secrets, pullmergeRequestId)];
+										}
+										if (dockerRegistry) {
+											const { url, username, password } = dockerRegistry;
+											await saveDockerRegistryCredentials({ url, username, password, workdir });
+										}
+										try {
+											const composeVolumes = volumes.map((volume) => {
+												return {
+													[`${volume.split(':')[0]}`]: {
+														name: volume.split(':')[0]
+													}
+												};
+											});
+											const composeFile = {
+												version: '3.8',
+												services: {
+													[imageId]: {
+														image: imageFound,
+														container_name: imageId,
+														volumes,
+														environment: envs,
+														labels,
+														depends_on: [],
+														expose: [port],
+														...(exposePort ? { ports: [`${exposePort}:${port}`] } : {}),
+														...defaultComposeConfiguration(destinationDocker.network)
+													}
+												},
+												networks: {
+													[destinationDocker.network]: {
+														external: true
+													}
+												},
+												volumes: Object.assign({}, ...composeVolumes)
 											};
-										});
-										const composeFile = {
-											version: '3.8',
-											services: {
-												[imageId]: {
-													image: `${applicationId}:${tag}`,
-													container_name: imageId,
-													volumes,
-													env_file: envFound ? [`${workdir}/.env`] : [],
-													labels,
-													depends_on: [],
-													expose: [port],
-													...(exposePort ? { ports: [`${exposePort}:${port}`] } : {}),
-													// logging: {
-													// 	driver: 'fluentd',
-													// },
-													...defaultComposeConfiguration(destinationDocker.network),
-												}
-											},
-											networks: {
-												[destinationDocker.network]: {
-													external: true
-												}
-											},
-											volumes: Object.assign({}, ...composeVolumes)
-										};
-										await fs.writeFile(`${workdir}/docker-compose.yml`, yaml.dump(composeFile));
-										await executeDockerCmd({ dockerId: destinationDocker.id, command: `docker compose --project-directory ${workdir} up -d` })
-										await saveBuildLog({ line: 'Deployment successful!', buildId, applicationId });
-									} catch (error) {
-										await saveBuildLog({ line: error, buildId, applicationId });
-										await prisma.build.updateMany({
-											where: { id: buildId, status: { in: ['queued', 'running'] } },
-											data: { status: 'failed' }
-										});
-										throw new Error(error);
+											await fs.writeFile(`${workdir}/docker-compose.yml`, yaml.dump(composeFile));
+											await executeCommand({
+												debug,
+												dockerId: destinationDocker.id,
+												command: `docker compose --project-directory ${workdir} up -d`
+											});
+											await saveBuildLog({ line: 'Deployed 🎉', buildId, applicationId });
+										} catch (error) {
+											await saveBuildLog({ line: error, buildId, applicationId });
+											const foundBuild = await prisma.build.findUnique({ where: { id: buildId } });
+											if (foundBuild) {
+												await prisma.build.update({
+													where: { id: buildId },
+													data: {
+														status: 'failed'
+													}
+												});
+											}
+											throw new Error(error);
+										}
+
+										if (!pullmergeRequestId)
+											await prisma.application.update({
+												where: { id: applicationId },
+												data: { configHash: currentHash }
+											});
 									}
-									await saveBuildLog({ line: 'Proxy will be updated shortly.', buildId, applicationId });
-									await prisma.build.update({ where: { id: buildId }, data: { status: 'success' } });
-									if (!pullmergeRequestId) await prisma.application.update({
-										where: { id: applicationId },
-										data: { configHash: currentHash }
+								}
+							} catch (error) {
+								const foundBuild = await prisma.build.findUnique({ where: { id: buildId } });
+								if (foundBuild) {
+									await prisma.build.update({
+										where: { id: buildId },
+										data: {
+											status: 'failed'
+										}
 									});
 								}
-							}
-							catch (error) {
-								await prisma.build.updateMany({
-									where: { id: buildId, status: { in: ['queued', 'running'] } },
-									data: { status: 'failed' }
-								});
 								if (error !== 1) {
 									await saveBuildLog({ line: error, buildId, applicationId: application.id });
 								}
+								if (error instanceof Error) {
+									await saveBuildLog({
+										line: error.message,
+										buildId,
+										applicationId: application.id
+									});
+								}
+								await fs.rm(workdir, { recursive: true, force: true });
+								return;
+							}
+							try {
+								if (application.dockerRegistryImageName && (!imageFoundRemotely || forceRebuild)) {
+									await saveBuildLog({
+										line: `Pushing ${imageName}:${customTag} to Docker Registry... It could take a while...`,
+										buildId,
+										applicationId: application.id
+									});
+									await pushToRegistry(application, workdir, tag, imageName, customTag);
+									await saveBuildLog({ line: 'Success', buildId, applicationId: application.id });
+								}
+							} catch (error) {
+								if (error.stdout) {
+									await saveBuildLog({ line: error.stdout, buildId, applicationId });
+								}
+								if (error.stderr) {
+									await saveBuildLog({ line: error.stderr, buildId, applicationId });
+								}
+							} finally {
+								await fs.rm(workdir, { recursive: true, force: true });
+								await prisma.build.update({ where: { id: buildId }, data: { status: 'success' } });
 							}
 						});
 					}
-					await pAll.default(actions, { concurrency })
+					await pAll.default(actions, { concurrency });
 				}
 			} catch (error) {
-				console.log(error)
+				console.log(error);
 			}
-		})
+		});
 		while (true) {
-			await th()
+			await th();
 		}
 	} else process.exit(0);
 })();

apps/api/src/jobs/infrastructure.ts

@@ -1,229 +0,0 @@
-import { parentPort } from 'node:worker_threads';
-import axios from 'axios';
-import { compareVersions } from 'compare-versions';
-import { asyncExecShell, cleanupDockerStorage, executeDockerCmd, isDev, prisma, startTraefikTCPProxy, generateDatabaseConfiguration, startTraefikProxy, listSettings, version, createRemoteEngineConfiguration } from '../lib/common';
-
-async function autoUpdater() {
-    try {
-        const currentVersion = version;
-        const { data: versions } = await axios
-            .get(
-                `https://get.coollabs.io/versions.json`
-                , {
-                    params: {
-                        appId: process.env['COOLIFY_APP_ID'] || undefined,
-                        version: currentVersion
-                    }
-                })
-        const latestVersion = versions['coolify'].main.version;
-        const isUpdateAvailable = compareVersions(latestVersion, currentVersion);
-        if (isUpdateAvailable === 1) {
-            const activeCount = 0
-            if (activeCount === 0) {
-                if (!isDev) {
-                    const { isAutoUpdateEnabled } = await prisma.setting.findFirst();
-                    if (isAutoUpdateEnabled) {
-                        await asyncExecShell(`docker pull coollabsio/coolify:${latestVersion}`);
-                        await asyncExecShell(`env | grep COOLIFY > .env`);
-                        await asyncExecShell(
-                            `sed -i '/COOLIFY_AUTO_UPDATE=/cCOOLIFY_AUTO_UPDATE=${isAutoUpdateEnabled}' .env`
-                        );
-                        await asyncExecShell(
-                            `docker run --rm -tid --env-file .env -v /var/run/docker.sock:/var/run/docker.sock -v coolify-db coollabsio/coolify:${latestVersion} /bin/sh -c "env | grep COOLIFY > .env && echo 'TAG=${latestVersion}' >> .env && docker stop -t 0 coolify && docker rm coolify && docker compose up -d --force-recreate"`
-                        );
-                    }
-                } else {
-                    console.log('Updating (not really in dev mode).');
-                }
-            }
-        }
-    } catch (error) { }
-}
-async function checkProxies() {
-    try {
-        const { default: isReachable } = await import('is-port-reachable');
-        let portReachable;
-
-        const { arch, ipv4, ipv6 } = await listSettings();
-
-        // Coolify Proxy local
-        const engine = '/var/run/docker.sock';
-        const localDocker = await prisma.destinationDocker.findFirst({
-            where: { engine, network: 'coolify', isCoolifyProxyUsed: true }
-        });
-        if (localDocker) {
-            portReachable = await isReachable(80, { host: ipv4 || ipv6 })
-            if (!portReachable) {
-                await startTraefikProxy(localDocker.id);
-            }
-        }
-        // Coolify Proxy remote
-        const remoteDocker = await prisma.destinationDocker.findMany({
-            where: { remoteEngine: true, remoteVerified: true }
-        });
-        if (remoteDocker.length > 0) {
-            for (const docker of remoteDocker) {
-                if (docker.isCoolifyProxyUsed) {
-                    portReachable = await isReachable(80, { host: docker.remoteIpAddress })
-                    if (!portReachable) {
-                        await startTraefikProxy(docker.id);
-                    }
-                }
-                try {
-                    await createRemoteEngineConfiguration(docker.id)
-                } catch (error) { }
-            }
-        }
-        // TCP Proxies
-        const databasesWithPublicPort = await prisma.database.findMany({
-            where: { publicPort: { not: null } },
-            include: { settings: true, destinationDocker: true }
-        });
-        for (const database of databasesWithPublicPort) {
-            const { destinationDockerId, destinationDocker, publicPort, id } = database;
-            if (destinationDockerId && destinationDocker.isCoolifyProxyUsed) {
-                const { privatePort } = generateDatabaseConfiguration(database, arch);
-                portReachable = await isReachable(publicPort, { host: destinationDocker.remoteIpAddress || ipv4 || ipv6 })
-                if (!portReachable) {
-                    await startTraefikTCPProxy(destinationDocker, id, publicPort, privatePort);
-                }
-            }
-        }
-        const wordpressWithFtp = await prisma.wordpress.findMany({
-            where: { ftpPublicPort: { not: null } },
-            include: { service: { include: { destinationDocker: true } } }
-        });
-        for (const ftp of wordpressWithFtp) {
-            const { service, ftpPublicPort } = ftp;
-            const { destinationDockerId, destinationDocker, id } = service;
-            if (destinationDockerId && destinationDocker.isCoolifyProxyUsed) {
-                portReachable = await isReachable(ftpPublicPort, { host: destinationDocker.remoteIpAddress || ipv4 || ipv6 })
-                if (!portReachable) {
-                    await startTraefikTCPProxy(destinationDocker, id, ftpPublicPort, 22, 'wordpressftp');
-                }
-            }
-        }
-
-        // HTTP Proxies
-        const minioInstances = await prisma.minio.findMany({
-            where: { publicPort: { not: null } },
-            include: { service: { include: { destinationDocker: true } } }
-        });
-        for (const minio of minioInstances) {
-            const { service, publicPort } = minio;
-            const { destinationDockerId, destinationDocker, id } = service;
-            if (destinationDockerId && destinationDocker.isCoolifyProxyUsed) {
-                portReachable = await isReachable(publicPort, { host: destinationDocker.remoteIpAddress || ipv4 || ipv6 })
-                if (!portReachable) {
-                    await startTraefikTCPProxy(destinationDocker, id, publicPort, 9000);
-                }
-            }
-        }
-    } catch (error) {
-
-    }
-}
-async function cleanupPrismaEngines() {
-    if (!isDev) {
-        try {
-            const { stdout } = await asyncExecShell(`ps -ef | grep /app/prisma-engines/query-engine | grep -v grep | wc -l | xargs`)
-            if (stdout.trim() != null && stdout.trim() != '' && Number(stdout.trim()) > 1) {
-                await asyncExecShell(`killall -q -e /app/prisma-engines/query-engine -o 1m`)
-            }
-        } catch (error) { }
-    }
-}
-async function cleanupStorage() {
-    const destinationDockers = await prisma.destinationDocker.findMany();
-    let enginesDone = new Set()
-    for (const destination of destinationDockers) {
-        if (enginesDone.has(destination.engine) || enginesDone.has(destination.remoteIpAddress)) return
-        if (destination.engine) enginesDone.add(destination.engine)
-        if (destination.remoteIpAddress) enginesDone.add(destination.remoteIpAddress)
-
-        let lowDiskSpace = false;
-        try {
-            let stdout = null
-            if (!isDev) {
-                const output = await executeDockerCmd({ dockerId: destination.id, command: `CONTAINER=$(docker ps -lq | head -1) && docker exec $CONTAINER sh -c 'df -kPT /'` })
-                stdout = output.stdout;
-            } else {
-                const output = await asyncExecShell(
-                    `df -kPT /`
-                );
-                stdout = output.stdout;
-            }
-            let lines = stdout.trim().split('\n');
-            let header = lines[0];
-            let regex =
-                /^Filesystem\s+|Type\s+|1024-blocks|\s+Used|\s+Available|\s+Capacity|\s+Mounted on\s*$/g;
-            const boundaries = [];
-            let match;
-
-            while ((match = regex.exec(header))) {
-                boundaries.push(match[0].length);
-            }
-
-            boundaries[boundaries.length - 1] = -1;
-            const data = lines.slice(1).map((line) => {
-                const cl = boundaries.map((boundary) => {
-                    const column = boundary > 0 ? line.slice(0, boundary) : line;
-                    line = line.slice(boundary);
-                    return column.trim();
-                });
-                return {
-                    capacity: Number.parseInt(cl[5], 10) / 100
-                };
-            });
-            if (data.length > 0) {
-                const { capacity } = data[0];
-                if (capacity > 0.8) {
-                    lowDiskSpace = true;
-                }
-            }
-        } catch (error) { }
-        await cleanupDockerStorage(destination.id, lowDiskSpace, false)
-    }
-}
-
-(async () => {
-    let status = {
-        cleanupStorage: false,
-        autoUpdater: false
-    }
-    if (parentPort) {
-        parentPort.on('message', async (message) => {
-            if (parentPort) {
-                if (message === 'error') throw new Error('oops');
-                if (message === 'cancel') {
-                    parentPort.postMessage('cancelled');
-                    process.exit(1);
-                }
-                if (message === 'action:cleanupStorage') {
-                    if (!status.autoUpdater) {
-                        status.cleanupStorage = true
-                        await cleanupStorage();
-                        status.cleanupStorage = false
-                    }
-                    return;
-                }
-                if (message === 'action:cleanupPrismaEngines') {
-                    await cleanupPrismaEngines();
-                    return;
-                }
-                if (message === 'action:checkProxies') {
-                    await checkProxies();
-                    return;
-                }
-                if (message === 'action:autoUpdater') {
-                    if (!status.cleanupStorage) {
-                        status.autoUpdater = true
-                        await autoUpdater();
-                        status.autoUpdater = false
-                    }
-                    return;
-                }
-            }
-        });
-    } else process.exit(0);
-})();

apps/api/src/lib.ts

@@ -0,0 +1,530 @@
+import cuid from "cuid";
+import { decrypt, encrypt, fixType, generatePassword, generateToken, prisma } from "./lib/common";
+import { getTemplates } from "./lib/services";
+
+export async function migrateApplicationPersistentStorage() {
+    const settings = await prisma.setting.findFirst()
+    if (settings) {
+        const { id: settingsId, applicationStoragePathMigrationFinished } = settings
+        try {
+            if (!applicationStoragePathMigrationFinished) {
+                const applications = await prisma.application.findMany({ include: { persistentStorage: true } });
+                for (const application of applications) {
+                    if (application.persistentStorage && application.persistentStorage.length > 0 && application?.buildPack !== 'docker') {
+                        for (const storage of application.persistentStorage) {
+                            let { id, path } = storage
+                            if (!path.startsWith('/app')) {
+                                path = `/app${path}`
+                                await prisma.applicationPersistentStorage.update({ where: { id }, data: { path, oldPath: true } })
+                            }
+                        }
+                    }
+                }
+            }
+        } catch (error) {
+            console.log(error)
+        } finally {
+            await prisma.setting.update({ where: { id: settingsId }, data: { applicationStoragePathMigrationFinished: true } })
+        }
+    }
+}
+export async function migrateServicesToNewTemplate() {
+    // This function migrates old hardcoded services to the new template based services
+    try {
+        let templates = await getTemplates()
+        const services: any = await prisma.service.findMany({
+            include: {
+                destinationDocker: true,
+                persistentStorage: true,
+                serviceSecret: true,
+                serviceSetting: true,
+                minio: true,
+                plausibleAnalytics: true,
+                vscodeserver: true,
+                wordpress: true,
+                ghost: true,
+                meiliSearch: true,
+                umami: true,
+                hasura: true,
+                fider: true,
+                moodle: true,
+                appwrite: true,
+                glitchTip: true,
+                searxng: true,
+                weblate: true,
+                taiga: true,
+            }
+        })
+        for (const service of services) {
+            try {
+                const { id } = service
+                if (!service.type) {
+                    continue;
+                }
+                let template = templates.find(t => fixType(t.type) === fixType(service.type));
+                if (template) {
+                    template = JSON.parse(JSON.stringify(template).replaceAll('$$id', service.id))
+                    if (service.type === 'plausibleanalytics' && service.plausibleAnalytics) await plausibleAnalytics(service, template)
+                    if (service.type === 'fider' && service.fider) await fider(service, template)
+                    if (service.type === 'minio' && service.minio) await minio(service, template)
+                    if (service.type === 'vscodeserver' && service.vscodeserver) await vscodeserver(service, template)
+                    if (service.type === 'wordpress' && service.wordpress) await wordpress(service, template)
+                    if (service.type === 'ghost' && service.ghost) await ghost(service, template)
+                    if (service.type === 'meilisearch' && service.meiliSearch) await meilisearch(service, template)
+                    if (service.type === 'umami' && service.umami) await umami(service, template)
+                    if (service.type === 'hasura' && service.hasura) await hasura(service, template)
+                    if (service.type === 'glitchTip' && service.glitchTip) await glitchtip(service, template)
+                    if (service.type === 'searxng' && service.searxng) await searxng(service, template)
+                    if (service.type === 'weblate' && service.weblate) await weblate(service, template)
+                    if (service.type === 'appwrite' && service.appwrite) await appwrite(service, template)
+
+                    try {
+                        await createVolumes(service, template);
+                    } catch (error) {
+                        console.log(error)
+                    }
+                    if (template.variables) {
+                        if (template.variables.length > 0) {
+                            for (const variable of template.variables) {
+                                const { defaultValue } = variable;
+                                const regex = /^\$\$.*\((\d+)\)$/g;
+                                const length = Number(regex.exec(defaultValue)?.[1]) || undefined
+                                if (variable.defaultValue.startsWith('$$generate_password')) {
+                                    variable.value = generatePassword({ length });
+                                } else if (variable.defaultValue.startsWith('$$generate_hex')) {
+                                    variable.value = generatePassword({ length, isHex: true });
+                                } else if (variable.defaultValue.startsWith('$$generate_username')) {
+                                    variable.value = cuid();
+                                } else if (variable.defaultValue.startsWith('$$generate_token')) {
+                                    variable.value = generateToken()
+                                } else {
+                                    variable.value = variable.defaultValue || '';
+                                }
+                            }
+                        }
+                        for (const variable of template.variables) {
+                            if (variable.id.startsWith('$$secret_')) {
+                                const found = await prisma.serviceSecret.findFirst({ where: { name: variable.name, serviceId: id } })
+                                if (!found) {
+                                    await prisma.serviceSecret.create({
+                                        data: { name: variable.name, value: encrypt(variable.value) || '', service: { connect: { id } } }
+                                    })
+                                }
+
+                            }
+                            if (variable.id.startsWith('$$config_')) {
+                                const found = await prisma.serviceSetting.findFirst({ where: { name: variable.name, serviceId: id } })
+                                if (!found) {
+                                    await prisma.serviceSetting.create({
+                                        data: { name: variable.name, value: variable.value.toString(), variableName: variable.id, service: { connect: { id } } }
+                                    })
+                                }
+                            }
+                        }
+                    }
+                    for (const s of Object.keys(template.services)) {
+                        if (service.type === 'plausibleanalytics') {
+                            continue;
+                        }
+                        if (template.services[s].volumes) {
+                            for (const volume of template.services[s].volumes) {
+                                const [volumeName, path] = volume.split(':')
+                                if (!volumeName.startsWith('/')) {
+                                    const found = await prisma.servicePersistentStorage.findFirst({ where: { volumeName, serviceId: id } })
+                                    if (!found) {
+                                        await prisma.servicePersistentStorage.create({
+                                            data: { volumeName, path, containerId: s, predefined: true, service: { connect: { id } } }
+                                        });
+                                    }
+                                }
+                            }
+                        }
+                    }
+                    await prisma.service.update({ where: { id }, data: { templateVersion: template.templateVersion } })
+                }
+            } catch (error) {
+                console.log(error)
+            }
+        }
+    } catch (error) {
+        console.log(error)
+
+    }
+}
+async function appwrite(service: any, template: any) {
+    const { opensslKeyV1, executorSecret, mariadbHost, mariadbPort, mariadbUser, mariadbPassword, mariadbRootUserPassword, mariadbDatabase } = service.appwrite
+
+    const secrets = [
+        `_APP_EXECUTOR_SECRET@@@${executorSecret}`,
+        `_APP_OPENSSL_KEY_V1@@@${opensslKeyV1}`,
+        `_APP_DB_PASS@@@${mariadbPassword}`,
+        `_APP_DB_ROOT_PASS@@@${mariadbRootUserPassword}`,
+    ]
+
+    const settings = [
+        `_APP_DB_HOST@@@${mariadbHost}`,
+        `_APP_DB_PORT@@@${mariadbPort}`,
+        `_APP_DB_USER@@@${mariadbUser}`,
+        `_APP_DB_SCHEMA@@@${mariadbDatabase}`,
+    ]
+    await migrateSecrets(secrets, service);
+    await migrateSettings(settings, service, template);
+
+    // Disconnect old service data
+    // await prisma.service.update({ where: { id: service.id }, data: { appwrite: { disconnect: true } } })
+}
+async function weblate(service: any, template: any) {
+    const { adminPassword, postgresqlUser, postgresqlPassword, postgresqlDatabase } = service.weblate
+
+    const secrets = [
+        `WEBLATE_ADMIN_PASSWORD@@@${adminPassword}`,
+        `POSTGRES_PASSWORD@@@${postgresqlPassword}`,
+    ]
+
+    const settings = [
+        `WEBLATE_SITE_DOMAIN@@@$$generate_domain`,
+        `POSTGRES_USER@@@${postgresqlUser}`,
+        `POSTGRES_DATABASE@@@${postgresqlDatabase}`,
+        `POSTGRES_DB@@@${postgresqlDatabase}`,
+        `POSTGRES_HOST@@@$$id-postgres`,
+        `POSTGRES_PORT@@@5432`,
+        `REDIS_HOST@@@$$id-redis`,
+    ]
+    await migrateSettings(settings, service, template);
+    await migrateSecrets(secrets, service);
+
+    // Disconnect old service data
+    // await prisma.service.update({ where: { id: service.id }, data: { weblate: { disconnect: true } } })
+}
+async function searxng(service: any, template: any) {
+    const { secretKey, redisPassword } = service.searxng
+
+    const secrets = [
+        `SECRET_KEY@@@${secretKey}`,
+        `REDIS_PASSWORD@@@${redisPassword}`,
+    ]
+
+    const settings = [
+        `SEARXNG_BASE_URL@@@$$generate_fqdn`
+    ]
+    await migrateSettings(settings, service, template);
+    await migrateSecrets(secrets, service);
+
+    // Disconnect old service data
+    // await prisma.service.update({ where: { id: service.id }, data: { searxng: { disconnect: true } } })
+}
+async function glitchtip(service: any, template: any) {
+    const { postgresqlUser, postgresqlPassword, postgresqlDatabase, secretKeyBase, defaultEmail, defaultUsername, defaultPassword, defaultEmailFrom, emailSmtpHost, emailSmtpPort, emailSmtpUser, emailSmtpPassword, emailSmtpUseTls, emailSmtpUseSsl, emailBackend, mailgunApiKey, sendgridApiKey, enableOpenUserRegistration } = service.glitchTip
+    const { id } = service
+
+    const secrets = [
+        `POSTGRES_PASSWORD@@@${postgresqlPassword}`,
+        `SECRET_KEY@@@${secretKeyBase}`,
+        `MAILGUN_API_KEY@@@${mailgunApiKey}`,
+        `SENDGRID_API_KEY@@@${sendgridApiKey}`,
+        `DJANGO_SUPERUSER_PASSWORD@@@${defaultPassword}`,
+        emailSmtpUser && emailSmtpPassword && emailSmtpHost && emailSmtpPort && `EMAIL_URL@@@${encrypt(`smtp://${emailSmtpUser}:${decrypt(emailSmtpPassword)}@${emailSmtpHost}:${emailSmtpPort}`)} || ''`,
+        `DATABASE_URL@@@${encrypt(`postgres://${postgresqlUser}:${decrypt(postgresqlPassword)}@${id}-postgresql:5432/${postgresqlDatabase}`)}`,
+        `REDIS_URL@@@${encrypt(`redis://${id}-redis:6379`)}`
+    ]
+    const settings = [
+        `POSTGRES_USER@@@${postgresqlUser}`,
+        `POSTGRES_DB@@@${postgresqlDatabase}`,
+        `DEFAULT_FROM_EMAIL@@@${defaultEmailFrom}`,
+        `EMAIL_USE_TLS@@@${emailSmtpUseTls}`,
+        `EMAIL_USE_SSL@@@${emailSmtpUseSsl}`,
+        `EMAIL_BACKEND@@@${emailBackend}`,
+        `ENABLE_OPEN_USER_REGISTRATION@@@${enableOpenUserRegistration}`,
+        `DJANGO_SUPERUSER_EMAIL@@@${defaultEmail}`,
+        `DJANGO_SUPERUSER_USERNAME@@@${defaultUsername}`,
+    ]
+    await migrateSettings(settings, service, template);
+    await migrateSecrets(secrets, service);
+
+    await prisma.service.update({ where: { id: service.id }, data: { type: 'glitchtip' } })
+    // Disconnect old service data
+    // await prisma.service.update({ where: { id: service.id }, data: { glitchTip: { disconnect: true } } })
+}
+async function hasura(service: any, template: any) {
+    const { postgresqlUser, postgresqlPassword, postgresqlDatabase, graphQLAdminPassword } = service.hasura
+    const { id } = service
+
+    const secrets = [
+        `HASURA_GRAPHQL_ADMIN_SECRET@@@${graphQLAdminPassword}`,
+        `HASURA_GRAPHQL_METADATA_DATABASE_URL@@@${encrypt(`postgres://${postgresqlUser}:${decrypt(postgresqlPassword)}@${id}-postgresql:5432/${postgresqlDatabase}`)}`,
+        `POSTGRES_PASSWORD@@@${postgresqlPassword}`,
+    ]
+    const settings = [
+        `POSTGRES_USER@@@${postgresqlUser}`,
+        `POSTGRES_DB@@@${postgresqlDatabase}`,
+    ]
+    await migrateSettings(settings, service, template);
+    await migrateSecrets(secrets, service);
+
+    // Disconnect old service data
+    // await prisma.service.update({ where: { id: service.id }, data: { hasura: { disconnect: true } } })
+}
+async function umami(service: any, template: any) {
+    const { postgresqlUser, postgresqlPassword, postgresqlDatabase, umamiAdminPassword, hashSalt } = service.umami
+    const { id } = service
+    const secrets = [
+        `HASH_SALT@@@${hashSalt}`,
+        `POSTGRES_PASSWORD@@@${postgresqlPassword}`,
+        `ADMIN_PASSWORD@@@${umamiAdminPassword}`,
+        `DATABASE_URL@@@${encrypt(`postgres://${postgresqlUser}:${decrypt(postgresqlPassword)}@${id}-postgresql:5432/${postgresqlDatabase}`)}`,
+    ]
+    const settings = [
+        `DATABASE_TYPE@@@postgresql`,
+        `POSTGRES_USER@@@${postgresqlUser}`,
+        `POSTGRES_DB@@@${postgresqlDatabase}`,
+    ]
+    await migrateSettings(settings, service, template);
+    await migrateSecrets(secrets, service);
+
+    await prisma.service.update({ where: { id: service.id }, data: { type: "umami-postgresql" } })
+
+    // Disconnect old service data
+    // await prisma.service.update({ where: { id: service.id }, data: { umami: { disconnect: true } } })
+}
+async function meilisearch(service: any, template: any) {
+    const { masterKey } = service.meiliSearch
+
+    const secrets = [
+        `MEILI_MASTER_KEY@@@${masterKey}`,
+    ]
+
+    // await migrateSettings(settings, service, template);
+    await migrateSecrets(secrets, service);
+
+    // Disconnect old service data
+    // await prisma.service.update({ where: { id: service.id }, data: { meiliSearch: { disconnect: true } } })
+}
+async function ghost(service: any, template: any) {
+    const { defaultEmail, defaultPassword, mariadbUser, mariadbPassword, mariadbRootUser, mariadbRootUserPassword, mariadbDatabase } = service.ghost
+    const { fqdn } = service
+
+    const isHttps = fqdn.startsWith('https://');
+
+    const secrets = [
+        `GHOST_PASSWORD@@@${defaultPassword}`,
+        `MARIADB_PASSWORD@@@${mariadbPassword}`,
+        `MARIADB_ROOT_PASSWORD@@@${mariadbRootUserPassword}`,
+        `GHOST_DATABASE_PASSWORD@@@${mariadbPassword}`,
+    ]
+    const settings = [
+        `GHOST_EMAIL@@@${defaultEmail}`,
+        `GHOST_DATABASE_HOST@@@${service.id}-mariadb`,
+        `GHOST_DATABASE_USER@@@${mariadbUser}`,
+        `GHOST_DATABASE_NAME@@@${mariadbDatabase}`,
+        `GHOST_DATABASE_PORT_NUMBER@@@3306`,
+        `MARIADB_USER@@@${mariadbUser}`,
+        `MARIADB_DATABASE@@@${mariadbDatabase}`,
+        `MARIADB_ROOT_USER@@@${mariadbRootUser}`,
+        `GHOST_HOST@@@$$generate_domain`,
+        `url@@@$$generate_fqdn`,
+        `GHOST_ENABLE_HTTPS@@@${isHttps ? 'yes' : 'no'}`
+    ]
+    await migrateSettings(settings, service, template);
+    await migrateSecrets(secrets, service);
+
+    await prisma.service.update({ where: { id: service.id }, data: { type: "ghost-mariadb" } })
+
+    // Disconnect old service data
+    // await prisma.service.update({ where: { id: service.id }, data: { ghost: { disconnect: true } } })
+}
+async function wordpress(service: any, template: any) {
+    const { extraConfig, tablePrefix, ownMysql, mysqlHost, mysqlPort, mysqlUser, mysqlPassword, mysqlRootUser, mysqlRootUserPassword, mysqlDatabase, ftpEnabled, ftpUser, ftpPassword, ftpPublicPort, ftpHostKey, ftpHostKeyPrivate } = service.wordpress
+
+    let settings = []
+    let secrets = []
+    if (ownMysql) {
+        secrets = [
+            `WORDPRESS_DB_PASSWORD@@@${mysqlPassword}`,
+            ftpPassword && `COOLIFY_FTP_PASSWORD@@@${ftpPassword}`,
+            ftpHostKeyPrivate && `COOLIFY_FTP_HOST_KEY_PRIVATE@@@${ftpHostKeyPrivate}`,
+            ftpHostKey && `COOLIFY_FTP_HOST_KEY@@@${ftpHostKey}`,
+        ]
+        settings = [
+            `WORDPRESS_CONFIG_EXTRA@@@${extraConfig}`,
+            `WORDPRESS_DB_HOST@@@${mysqlHost}`,
+            `WORDPRESS_DB_PORT@@@${mysqlPort}`,
+            `WORDPRESS_DB_USER@@@${mysqlUser}`,
+            `WORDPRESS_DB_NAME@@@${mysqlDatabase}`,
+        ]
+    } else {
+        secrets = [
+            `MYSQL_ROOT_PASSWORD@@@${mysqlRootUserPassword}`,
+            `MYSQL_PASSWORD@@@${mysqlPassword}`,
+            ftpPassword && `COOLIFY_FTP_PASSWORD@@@${ftpPassword}`,
+            ftpHostKeyPrivate && `COOLIFY_FTP_HOST_KEY_PRIVATE@@@${ftpHostKeyPrivate}`,
+            ftpHostKey && `COOLIFY_FTP_HOST_KEY@@@${ftpHostKey}`,
+        ]
+        settings = [
+            `MYSQL_ROOT_USER@@@${mysqlRootUser}`,
+            `MYSQL_USER@@@${mysqlUser}`,
+            `MYSQL_DATABASE@@@${mysqlDatabase}`,
+            `MYSQL_HOST@@@${service.id}-mysql`,
+            `MYSQL_PORT@@@${mysqlPort}`,
+            `WORDPRESS_CONFIG_EXTRA@@@${extraConfig}`,
+            `WORDPRESS_TABLE_PREFIX@@@${tablePrefix}`,
+            `WORDPRESS_DB_HOST@@@${service.id}-mysql`,
+            `COOLIFY_OWN_DB@@@${ownMysql}`,
+            `COOLIFY_FTP_ENABLED@@@${ftpEnabled}`,
+            `COOLIFY_FTP_USER@@@${ftpUser}`,
+            `COOLIFY_FTP_PUBLIC_PORT@@@${ftpPublicPort}`,
+        ]
+    }
+
+    await migrateSettings(settings, service, template);
+    await migrateSecrets(secrets, service);
+    if (ownMysql) {
+        await prisma.service.update({ where: { id: service.id }, data: { type: "wordpress-only" } })
+    }
+    // Disconnect old service data
+    // await prisma.service.update({ where: { id: service.id }, data: { wordpress: { disconnect: true } } })
+}
+async function vscodeserver(service: any, template: any) {
+    const { password } = service.vscodeserver
+
+    const secrets = [
+        `PASSWORD@@@${password}`,
+    ]
+    await migrateSecrets(secrets, service);
+
+    // Disconnect old service data
+    // await prisma.service.update({ where: { id: service.id }, data: { vscodeserver: { disconnect: true } } })
+}
+async function minio(service: any, template: any) {
+    const { rootUser, rootUserPassword, apiFqdn } = service.minio
+    const secrets = [
+        `MINIO_ROOT_PASSWORD@@@${rootUserPassword}`,
+    ]
+    const settings = [
+        `MINIO_ROOT_USER@@@${rootUser}`,
+        `MINIO_SERVER_URL@@@${apiFqdn}`,
+        `MINIO_BROWSER_REDIRECT_URL@@@$$generate_fqdn`,
+        `MINIO_DOMAIN@@@$$generate_domain`,
+    ]
+    await migrateSettings(settings, service, template);
+    await migrateSecrets(secrets, service);
+
+    // Disconnect old service data
+    // await prisma.service.update({ where: { id: service.id }, data: { minio: { disconnect: true } } })
+}
+async function fider(service: any, template: any) {
+    const { postgresqlUser, postgresqlPassword, postgresqlDatabase, jwtSecret, emailNoreply, emailMailgunApiKey, emailMailgunDomain, emailMailgunRegion, emailSmtpHost, emailSmtpPort, emailSmtpUser, emailSmtpPassword, emailSmtpEnableStartTls } = service.fider
+    const { id } = service
+    const secrets = [
+        `JWT_SECRET@@@${jwtSecret}`,
+        emailMailgunApiKey && `EMAIL_MAILGUN_API@@@${emailMailgunApiKey}`,
+        emailSmtpPassword && `EMAIL_SMTP_PASSWORD@@@${emailSmtpPassword}`,
+        `POSTGRES_PASSWORD@@@${postgresqlPassword}`,
+        `DATABASE_URL@@@${encrypt(`postgresql://${postgresqlUser}:${decrypt(postgresqlPassword)}@${id}-postgresql:5432/${postgresqlDatabase}?sslmode=disable`)}`
+    ]
+    const settings = [
+        `BASE_URL@@@$$generate_fqdn`,
+        `EMAIL_NOREPLY@@@${emailNoreply || 'noreply@example.com'}`,
+        `EMAIL_MAILGUN_DOMAIN@@@${emailMailgunDomain || ''}`,
+        `EMAIL_MAILGUN_REGION@@@${emailMailgunRegion || ''}`,
+        `EMAIL_SMTP_HOST@@@${emailSmtpHost || ''}`,
+        `EMAIL_SMTP_PORT@@@${emailSmtpPort || 587}`,
+        `EMAIL_SMTP_USER@@@${emailSmtpUser || ''}`,
+        `EMAIL_SMTP_PASSWORD@@@${emailSmtpPassword || ''}`,
+        `EMAIL_SMTP_ENABLE_STARTTLS@@@${emailSmtpEnableStartTls || 'false'}`,
+        `POSTGRES_USER@@@${postgresqlUser}`,
+        `POSTGRES_DB@@@${postgresqlDatabase}`,
+    ]
+    await migrateSettings(settings, service, template);
+    await migrateSecrets(secrets, service);
+
+    // Disconnect old service data
+    // await prisma.service.update({ where: { id: service.id }, data: { fider: { disconnect: true } } })
+
+}
+async function plausibleAnalytics(service: any, template: any) {
+    const { email, username, password, postgresqlUser, postgresqlPassword, postgresqlDatabase, secretKeyBase, scriptName } = service.plausibleAnalytics;
+    const { id } = service
+
+    const settings = [
+        `BASE_URL@@@$$generate_fqdn`,
+        `ADMIN_USER_EMAIL@@@${email}`,
+        `ADMIN_USER_NAME@@@${username}`,
+        `DISABLE_AUTH@@@false`,
+        `DISABLE_REGISTRATION@@@true`,
+        `POSTGRESQL_USERNAME@@@${postgresqlUser}`,
+        `POSTGRESQL_DATABASE@@@${postgresqlDatabase}`,
+        `SCRIPT_NAME@@@${scriptName}`,
+    ]
+    const secrets = [
+        `ADMIN_USER_PWD@@@${password}`,
+        `SECRET_KEY_BASE@@@${secretKeyBase}`,
+        `POSTGRESQL_PASSWORD@@@${postgresqlPassword}`,
+        `DATABASE_URL@@@${encrypt(`postgres://${postgresqlUser}:${decrypt(postgresqlPassword)}@${id}-postgresql:5432/${postgresqlDatabase}`)}`,
+    ]
+    await migrateSettings(settings, service, template);
+    await migrateSecrets(secrets, service);
+
+    // Disconnect old service data
+    // await prisma.service.update({ where: { id: service.id }, data: { plausibleAnalytics: { disconnect: true } } })
+}
+async function migrateSettings(settings: any[], service: any, template: any) {
+    for (const setting of settings) {
+        try {
+            if (!setting) continue;
+            let [name, value] = setting.split('@@@')
+            let minio = name
+            if (name === 'MINIO_SERVER_URL') {
+                name = 'coolify_fqdn_minio_console'
+            }
+            if (!value || value === 'null') {
+                continue;
+            }
+            let variableName = template.variables.find((v: any) => v.name === name)?.id
+            if (!variableName) {
+                variableName = `$$config_${name.toLowerCase()}`
+            }
+            // console.log('Migrating setting', name, value, 'for service', service.id, ', service name:', service.name, 'variableName: ', variableName)
+
+            await prisma.serviceSetting.findFirst({ where: { name: minio, serviceId: service.id } }) || await prisma.serviceSetting.create({ data: { name: minio, value, variableName, service: { connect: { id: service.id } } } })
+        } catch (error) {
+            console.log(error)
+        }
+    }
+}
+async function migrateSecrets(secrets: any[], service: any) {
+    for (const secret of secrets) {
+        try {
+            if (!secret) continue;
+            let [name, value] = secret.split('@@@')
+            if (!value || value === 'null') {
+                continue
+            }
+            // console.log('Migrating secret', name, value, 'for service', service.id, ', service name:', service.name)
+            await prisma.serviceSecret.findFirst({ where: { name, serviceId: service.id } }) || await prisma.serviceSecret.create({ data: { name, value, service: { connect: { id: service.id } } } })
+        } catch (error) {
+            console.log(error)
+        }
+    }
+}
+async function createVolumes(service: any, template: any) {
+    const volumes = [];
+    for (const s of Object.keys(template.services)) {
+        if (template.services[s].volumes && template.services[s].volumes.length > 0) {
+            for (const volume of template.services[s].volumes) {
+                let volumeName = volume.split(':')[0]
+                const volumePath = volume.split(':')[1]
+                let volumeService = s
+                if (service.type === 'plausibleanalytics' && service.plausibleAnalytics?.id) {
+                    let volumeId = volumeName.split('-')[0]
+                    volumeName = volumeName.replace(volumeId, service.plausibleAnalytics.id)
+                }
+                volumes.push(`${volumeName}@@@${volumePath}@@@${volumeService}`)
+            }
+        }
+    }
+    for (const volume of volumes) {
+        const [volumeName, path, containerId] = volume.split('@@@')
+        // console.log('Creating volume', volumeName, path, containerId, 'for service', service.id, ', service name:', service.name)
+        await prisma.servicePersistentStorage.findFirst({ where: { volumeName, serviceId: service.id } }) || await prisma.servicePersistentStorage.create({ data: { volumeName, path, containerId, predefined: true, service: { connect: { id: service.id } } } })
+    }
+}

apps/api/src/lib/buildPacks/common.ts

@@ -1,6 +1,18 @@
-import { base64Encode, executeDockerCmd, generateTimestamp, getDomain, isDev, prisma, version } from "../common";
+import {
+	base64Encode,
+	decrypt,
+	encrypt,
+	executeCommand,
+	generateSecrets,
+	generateTimestamp,
+	getDomain,
+	isARM,
+	isDev,
+	prisma,
+	version
+} from '../common';
 import { promises as fs } from 'fs';
-import { day } from "../dayjs";
+import { day } from '../dayjs';
 
 const staticApps = ['static', 'react', 'vuejs', 'svelte', 'gatsby', 'astro', 'eleventy'];
 const nodeBased = [
@@ -17,7 +29,10 @@ const nodeBased = [
 	'nextjs'
 ];
 
-export function setDefaultBaseImage(buildPack: string | null, deploymentType: string | null = null) {
+export function setDefaultBaseImage(
+	buildPack: string | null,
+	deploymentType: string | null = null
+) {
 	const nodeVersions = [
 		{
 			value: 'node:lts',
@@ -52,6 +67,14 @@ export function setDefaultBaseImage(buildPack: string | null, deploymentType: st
 		{
 			value: 'webdevops/apache:alpine',
 			label: 'webdevops/apache:alpine'
+		},
+		{
+			value: 'nginx:alpine',
+			label: 'nginx:alpine'
+		},
+		{
+			value: 'httpd:alpine',
+			label: 'httpd:alpine (Apache)'
 		}
 	];
 	const rustVersions = [
@@ -214,8 +237,20 @@ export function setDefaultBaseImage(buildPack: string | null, deploymentType: st
 			label: 'webdevops/php-apache:7.1-alpine'
 		},
 		{
-			value: 'webdevops/php-nginx:7.1-alpine',
-			label: 'webdevops/php-nginx:7.1-alpine'
+			value: 'php:8.1-fpm',
+			label: 'php:8.1-fpm'
+		},
+		{
+			value: 'php:8.0-fpm',
+			label: 'php:8.0-fpm'
+		},
+		{
+			value: 'php:8.1-fpm-alpine',
+			label: 'php:8.1-fpm-alpine'
+		},
+		{
+			value: 'php:8.0-fpm-alpine',
+			label: 'php:8.0-fpm-alpine'
 		}
 	];
 	const pythonVersions = [
@@ -296,8 +331,8 @@ export function setDefaultBaseImage(buildPack: string | null, deploymentType: st
 		{
 			value: 'heroku/builder-classic:22',
 			label: 'heroku/builder-classic:22'
-		},
-	]
+		}
+	];
 	let payload: any = {
 		baseImage: null,
 		baseBuildImage: null,
@@ -306,8 +341,10 @@ export function setDefaultBaseImage(buildPack: string | null, deploymentType: st
 	};
 	if (nodeBased.includes(buildPack)) {
 		if (deploymentType === 'static') {
-			payload.baseImage = 'webdevops/nginx:alpine';
-			payload.baseImages = staticVersions;
+			payload.baseImage = isARM(process.arch) ? 'nginx:alpine' : 'webdevops/nginx:alpine';
+			payload.baseImages = isARM(process.arch)
+				? staticVersions.filter((version) => !version.value.includes('webdevops'))
+				: staticVersions;
 			payload.baseBuildImage = 'node:lts';
 			payload.baseBuildImages = nodeVersions;
 		} else {
@@ -318,8 +355,10 @@ export function setDefaultBaseImage(buildPack: string | null, deploymentType: st
 		}
 	}
 	if (staticApps.includes(buildPack)) {
-		payload.baseImage = 'webdevops/nginx:alpine';
-		payload.baseImages = staticVersions;
+		payload.baseImage = isARM(process.arch) ? 'nginx:alpine' : 'webdevops/nginx:alpine';
+		payload.baseImages = isARM(process.arch)
+			? staticVersions.filter((version) => !version.value.includes('webdevops'))
+			: staticVersions;
 		payload.baseBuildImage = 'node:lts';
 		payload.baseBuildImages = nodeVersions;
 	}
@@ -337,18 +376,26 @@ export function setDefaultBaseImage(buildPack: string | null, deploymentType: st
 		payload.baseImage = 'denoland/deno:latest';
 	}
 	if (buildPack === 'php') {
-		payload.baseImage = 'webdevops/php-apache:8.2-alpine';
-		payload.baseImages = phpVersions;
+		payload.baseImage = isARM(process.arch)
+			? 'php:8.1-fpm-alpine'
+			: 'webdevops/php-apache:8.2-alpine';
+		payload.baseImages = isARM(process.arch)
+			? phpVersions.filter((version) => !version.value.includes('webdevops'))
+			: phpVersions;
 	}
 	if (buildPack === 'laravel') {
-		payload.baseImage = 'webdevops/php-apache:8.2-alpine';
+		payload.baseImage = isARM(process.arch)
+			? 'php:8.1-fpm-alpine'
+			: 'webdevops/php-apache:8.2-alpine';
+		payload.baseImages = isARM(process.arch)
+			? phpVersions.filter((version) => !version.value.includes('webdevops'))
+			: phpVersions;
 		payload.baseBuildImage = 'node:18';
 		payload.baseBuildImages = nodeVersions;
 	}
 	if (buildPack === 'heroku') {
 		payload.baseImage = 'heroku/buildpacks:20';
 		payload.baseImages = herokuVersions;
-
 	}
 	return payload;
 }
@@ -363,6 +410,7 @@ export const setDefaultConfiguration = async (data: any) => {
 		publishDirectory,
 		baseDirectory,
 		dockerFileLocation,
+		dockerComposeFileLocation,
 		denoMainFile
 	} = data;
 	//@ts-ignore
@@ -384,7 +432,8 @@ export const setDefaultConfiguration = async (data: any) => {
 	if (!publishDirectory) publishDirectory = template?.publishDirectory || null;
 	if (baseDirectory) {
 		if (!baseDirectory.startsWith('/')) baseDirectory = `/${baseDirectory}`;
-		if (!baseDirectory.endsWith('/')) baseDirectory = `${baseDirectory}/`;
+		if (baseDirectory.endsWith('/') && baseDirectory !== '/')
+			baseDirectory = baseDirectory.slice(0, -1);
 	}
 	if (dockerFileLocation) {
 		if (!dockerFileLocation.startsWith('/')) dockerFileLocation = `/${dockerFileLocation}`;
@@ -392,6 +441,14 @@ export const setDefaultConfiguration = async (data: any) => {
 	} else {
 		dockerFileLocation = '/Dockerfile';
 	}
+	if (dockerComposeFileLocation) {
+		if (!dockerComposeFileLocation.startsWith('/'))
+			dockerComposeFileLocation = `/${dockerComposeFileLocation}`;
+		if (dockerComposeFileLocation.endsWith('/'))
+			dockerComposeFileLocation = dockerComposeFileLocation.slice(0, -1);
+	} else {
+		dockerComposeFileLocation = '/Dockerfile';
+	}
 	if (!denoMainFile) {
 		denoMainFile = 'main.ts';
 	}
@@ -405,6 +462,7 @@ export const setDefaultConfiguration = async (data: any) => {
 		publishDirectory,
 		baseDirectory,
 		dockerFileLocation,
+		dockerComposeFileLocation,
 		denoMainFile
 	};
 };
@@ -451,7 +509,6 @@ export const scanningTemplates = {
 	}
 };
 
-
 export const saveBuildLog = async ({
 	line,
 	buildId,
@@ -461,17 +518,46 @@ export const saveBuildLog = async ({
 	buildId: string;
 	applicationId: string;
 }): Promise<any> => {
+	if (buildId === 'undefined' || buildId === 'null' || !buildId) return;
+	if (applicationId === 'undefined' || applicationId === 'null' || !applicationId) return;
+	const { default: got } = await import('got');
+	if (typeof line === 'object' && line) {
+		if (line.shortMessage) {
+			line = line.shortMessage + '\n' + line.stderr;
+		} else {
+			line = JSON.stringify(line);
+		}
+	}
 	if (line && typeof line === 'string' && line.includes('ghs_')) {
 		const regex = /ghs_.*@/g;
 		line = line.replace(regex, '<SENSITIVE_DATA_DELETED>@');
 	}
 	const addTimestamp = `[${generateTimestamp()}] ${line}`;
-	if (isDev) console.debug(`[${applicationId}] ${addTimestamp}`);
-	return await prisma.buildLog.create({
-		data: {
-			line: addTimestamp, buildId, time: Number(day().valueOf()), applicationId
-		}
-	});
+	const fluentBitUrl = isDev
+		? process.env.COOLIFY_CONTAINER_DEV === 'true'
+			? 'http://coolify-fluentbit:24224'
+			: 'http://localhost:24224'
+		: 'http://coolify-fluentbit:24224';
+
+	if (isDev && !process.env.COOLIFY_CONTAINER_DEV) {
+		console.debug(`[${applicationId}] ${addTimestamp}`);
+	}
+	try {
+		return await got.post(`${fluentBitUrl}/${applicationId}_buildlog_${buildId}.csv`, {
+			json: {
+				line: encrypt(line)
+			}
+		});
+	} catch (error) {
+		return await prisma.buildLog.create({
+			data: {
+				line: addTimestamp,
+				buildId,
+				time: Number(day().valueOf()),
+				applicationId
+			}
+		});
+	}
 };
 
 export async function copyBaseConfigurationFiles(
@@ -543,6 +629,7 @@ export async function copyBaseConfigurationFiles(
             `
 			);
 		}
+		// TODO: Add more configuration files for other buildpacks, like apache2, etc.
 	} catch (error) {
 		throw new Error(error);
 	}
@@ -556,6 +643,29 @@ export function checkPnpm(installCommand = null, buildCommand = null, startComma
 	);
 }
 
+export async function saveDockerRegistryCredentials({ url, username, password, workdir }) {
+	if (!username || !password) {
+		return null;
+	}
+
+	let decryptedPassword = decrypt(password);
+	const location = `${workdir}/.docker`;
+
+	try {
+		await fs.mkdir(`${workdir}/.docker`);
+	} catch (error) {
+		// console.log(error);
+	}
+	const payload = JSON.stringify({
+		auths: {
+			[url]: {
+				auth: Buffer.from(`${username}:${decryptedPassword}`).toString('base64')
+			}
+		}
+	});
+	await fs.writeFile(`${location}/config.json`, payload);
+	return location;
+}
 export async function buildImage({
 	applicationId,
 	tag,
@@ -564,58 +674,52 @@ export async function buildImage({
 	dockerId,
 	isCache = false,
 	debug = false,
-	dockerFileLocation = '/Dockerfile'
+	dockerFileLocation = '/Dockerfile',
+	commit,
+	forceRebuild = false
 }) {
 	if (isCache) {
-		await saveBuildLog({ line: `Building cache image started.`, buildId, applicationId });
+		await saveBuildLog({ line: `Building cache image...`, buildId, applicationId });
 	} else {
-		await saveBuildLog({ line: `Building image started.`, buildId, applicationId });
+		await saveBuildLog({ line: `Building production image...`, buildId, applicationId });
 	}
-	if (!debug && isCache) {
-		await saveBuildLog({
-			line: `Debug turned off. To see more details, allow it in the configuration.`,
-			buildId,
-			applicationId
-		});
-	}
-	const dockerFile = isCache ? `${dockerFileLocation}-cache` : `${dockerFileLocation}`
-	const cache = `${applicationId}:${tag}${isCache ? '-cache' : ''}`
-	await executeDockerCmd({ debug, buildId, applicationId, dockerId, command: `docker build --progress plain -f ${workdir}/${dockerFile} -t ${cache} ${workdir}` })
-	const { status } = await prisma.build.findUnique({ where: { id: buildId } })
-	if (status === 'canceled') {
-		throw new Error('Deployment canceled.')
-	}
-	if (isCache) {
-		await saveBuildLog({ line: `Building cache image successful.`, buildId, applicationId });
-	} else {
-		await saveBuildLog({ line: `Building image successful.`, buildId, applicationId });
-	}
-}
+	const dockerFile = isCache ? `${dockerFileLocation}-cache` : `${dockerFileLocation}`;
+	const cache = `${applicationId}:${tag}${isCache ? '-cache' : ''}`;
+	let location = null;
 
-export async function streamEvents({ stream, docker, buildId, applicationId, debug }) {
-	await new Promise((resolve, reject) => {
-		docker.engine.modem.followProgress(stream, onFinished, onProgress);
-		function onFinished(err, res) {
-			if (err) reject(err);
-			resolve(res);
-		}
-		async function onProgress(event) {
-			if (event.error) {
-				reject(event.error);
-			} else if (event.stream) {
-				if (event.stream !== '\n') {
-					if (debug)
-						await saveBuildLog({
-							line: `${event.stream.replace('\n', '')}`,
-							buildId,
-							applicationId
-						});
-				}
-			}
-		}
+	const { dockerRegistry } = await prisma.application.findUnique({
+		where: { id: applicationId },
+		select: { dockerRegistry: true }
 	});
-}
+	if (dockerRegistry) {
+		const { url, username, password } = dockerRegistry;
+		location = await saveDockerRegistryCredentials({ url, username, password, workdir });
+	}
 
+	await executeCommand({
+		stream: true,
+		debug,
+		buildId,
+		applicationId,
+		dockerId,
+		command: `docker ${location ? `--config ${location}` : ''} build ${
+			forceRebuild ? '--no-cache' : ''
+		} --progress plain -f ${workdir}/${dockerFile} -t ${cache} --build-arg SOURCE_COMMIT=${commit} ${workdir}`
+	});
+
+	const { status } = await prisma.build.findUnique({ where: { id: buildId } });
+	if (status === 'canceled') {
+		throw new Error('Canceled.');
+	}
+}
+export function makeLabelForSimpleDockerfile({ applicationId, port, type }) {
+	return [
+		'coolify.managed=true',
+		`coolify.version=${version}`,
+		`coolify.applicationId=${applicationId}`,
+		`coolify.type=standalone-application`
+	];
+}
 export function makeLabelForStandaloneApplication({
 	applicationId,
 	fqdn,
@@ -642,7 +746,9 @@ export function makeLabelForStandaloneApplication({
 	return [
 		'coolify.managed=true',
 		`coolify.version=${version}`,
+		`coolify.applicationId=${applicationId}`,
 		`coolify.type=standalone-application`,
+		`coolify.name=${name}`,
 		`coolify.configuration=${base64Encode(
 			JSON.stringify({
 				applicationId,
@@ -682,21 +788,8 @@ export async function buildCacheImageWithNode(data, imageForBuild) {
 	Dockerfile.push('WORKDIR /app');
 	Dockerfile.push(`LABEL coolify.buildId=${buildId}`);
 	if (secrets.length > 0) {
-		secrets.forEach((secret) => {
-			if (secret.isBuildSecret) {
-				if (pullmergeRequestId) {
-					const isSecretFound = secrets.filter(s => s.name === secret.name && s.isPRMRSecret)
-					if (isSecretFound.length > 0) {
-						Dockerfile.push(`ARG ${secret.name}=${isSecretFound[0].value}`);
-					} else {
-						Dockerfile.push(`ARG ${secret.name}=${secret.value}`);
-					}
-				} else {
-					if (!secret.isPRMRSecret) {
-						Dockerfile.push(`ARG ${secret.name}=${secret.value}`);
-					}
-				}
-			}
+		generateSecrets(secrets, pullmergeRequestId, true).forEach((env) => {
+			Dockerfile.push(env);
 		});
 	}
 	if (isPnpm) {
@@ -707,34 +800,19 @@ export async function buildCacheImageWithNode(data, imageForBuild) {
 		Dockerfile.push(`RUN ${installCommand}`);
 	}
 	Dockerfile.push(`RUN ${buildCommand}`);
-	console.log(Dockerfile.join('\n'))
 	await fs.writeFile(`${workdir}/Dockerfile-cache`, Dockerfile.join('\n'));
 	await buildImage({ ...data, isCache: true });
 }
 
 export async function buildCacheImageForLaravel(data, imageForBuild) {
 	const { workdir, buildId, secrets, pullmergeRequestId } = data;
-
 	const Dockerfile: Array<string> = [];
 	Dockerfile.push(`FROM ${imageForBuild}`);
 	Dockerfile.push('WORKDIR /app');
 	Dockerfile.push(`LABEL coolify.buildId=${buildId}`);
 	if (secrets.length > 0) {
-		secrets.forEach((secret) => {
-			if (secret.isBuildSecret) {
-				if (pullmergeRequestId) {
-					const isSecretFound = secrets.filter(s => s.name === secret.name && s.isPRMRSecret)
-					if (isSecretFound.length > 0) {
-						Dockerfile.push(`ARG ${secret.name}=${isSecretFound[0].value}`);
-					} else {
-						Dockerfile.push(`ARG ${secret.name}=${secret.value}`);
-					}
-				} else {
-					if (!secret.isPRMRSecret) {
-						Dockerfile.push(`ARG ${secret.name}=${secret.value}`);
-					}
-				}
-			}
+		generateSecrets(secrets, pullmergeRequestId, true).forEach((env) => {
+			Dockerfile.push(env);
 		});
 	}
 	Dockerfile.push(`COPY *.json *.mix.js /app/`);
@@ -745,11 +823,7 @@ export async function buildCacheImageForLaravel(data, imageForBuild) {
 }
 
 export async function buildCacheImageWithCargo(data, imageForBuild) {
-	const {
-		applicationId,
-		workdir,
-		buildId,
-	} = data;
+	const { applicationId, workdir, buildId } = data;
 
 	const Dockerfile: Array<string> = [];
 	Dockerfile.push(`FROM ${imageForBuild} as planner-${applicationId}`);
@@ -766,4 +840,4 @@ export async function buildCacheImageWithCargo(data, imageForBuild) {
 	Dockerfile.push('RUN cargo chef cook --release --recipe-path recipe.json');
 	await fs.writeFile(`${workdir}/Dockerfile-cache`, Dockerfile.join('\n'));
 	await buildImage({ ...data, isCache: true });
-}
+}

apps/api/src/lib/buildPacks/compose.ts

@@ -0,0 +1,108 @@
+import { promises as fs } from 'fs';
+import { defaultComposeConfiguration, executeCommand, generateSecrets } from '../common';
+import { saveBuildLog } from './common';
+import yaml from 'js-yaml';
+
+export default async function (data) {
+	let {
+		applicationId,
+		debug,
+		buildId,
+		dockerId,
+		network,
+		volumes,
+		labels,
+		workdir,
+		baseDirectory,
+		secrets,
+		pullmergeRequestId,
+		dockerComposeConfiguration,
+		dockerComposeFileLocation
+	} = data;
+	const fileYaml = `${workdir}${baseDirectory}${dockerComposeFileLocation}`;
+	const dockerComposeRaw = await fs.readFile(fileYaml, 'utf8');
+	const dockerComposeYaml = yaml.load(dockerComposeRaw);
+	if (!dockerComposeYaml.services) {
+		throw 'No Services found in docker-compose file.';
+	}
+	let envs = ['NODE_ENV=production'];
+	if (secrets.length > 0) {
+		envs = [...envs, ...generateSecrets(secrets, pullmergeRequestId)];
+	}
+
+	const composeVolumes = [];
+	if (volumes.length > 0) {
+		for (const volume of volumes) {
+			let [v, path] = volume.split(':');
+			composeVolumes[v] = {
+				name: v
+			};
+		}
+	}
+
+	let networks = {};
+	for (let [key, value] of Object.entries(dockerComposeYaml.services)) {
+		value['container_name'] = `${applicationId}-${key}`;
+		value['environment'] = [...value['environment'], ...envs];
+		value['labels'] = labels;
+		// TODO: If we support separated volume for each service, we need to add it here
+		if (value['volumes']?.length > 0) {
+			value['volumes'] = value['volumes'].map((volume) => {
+				let [v, path, permission] = volume.split(':');
+				if (!path) {
+					path = v;
+					v = `${applicationId}${v.replace(/\//gi, '-').replace(/\./gi, '')}`;
+				} else {
+					v = `${applicationId}${v.replace(/\//gi, '-').replace(/\./gi, '')}`;
+				}
+				composeVolumes[v] = {
+					name: v
+				};
+				return `${v}:${path}${permission ? ':' + permission : ''}`;
+			});
+		}
+		if (volumes.length > 0) {
+			for (const volume of volumes) {
+				value['volumes'].push(volume);
+			}
+		}
+		if (dockerComposeConfiguration[key].port) {
+			value['expose'] = [dockerComposeConfiguration[key].port];
+		}
+		if (value['networks']?.length > 0) {
+			value['networks'].forEach((network) => {
+				networks[network] = {
+					name: network
+				};
+			});
+		}
+		value['networks'] = [...(value['networks'] || ''), network];
+		dockerComposeYaml.services[key] = {
+			...dockerComposeYaml.services[key],
+			restart: defaultComposeConfiguration(network).restart,
+			deploy: defaultComposeConfiguration(network).deploy
+		};
+	}
+	if (Object.keys(composeVolumes).length > 0) {
+		dockerComposeYaml['volumes'] = { ...composeVolumes };
+	}
+	dockerComposeYaml['networks'] = Object.assign({ ...networks }, { [network]: { external: true } });
+
+	await fs.writeFile(fileYaml, yaml.dump(dockerComposeYaml));
+	await executeCommand({
+		debug,
+		buildId,
+		applicationId,
+		dockerId,
+		command: `docker compose --project-directory ${workdir} pull`
+	});
+	await saveBuildLog({ line: 'Pulling images from Compose file...', buildId, applicationId });
+	await executeCommand({
+		debug,
+		buildId,
+		applicationId,
+		dockerId,
+		command: `docker compose --project-directory ${workdir} build --progress plain`
+	});
+	await saveBuildLog({ line: 'Building images from Compose file...', buildId, applicationId });
+}

apps/api/src/lib/buildPacks/deno.ts

@@ -1,4 +1,5 @@
 import { promises as fs } from 'fs';
+import { generateSecrets } from '../common';
 import { buildImage } from './common';
 
 const createDockerfile = async (data, image): Promise<void> => {
@@ -24,21 +25,8 @@ const createDockerfile = async (data, image): Promise<void> => {
 	Dockerfile.push('WORKDIR /app');
 	Dockerfile.push(`LABEL coolify.buildId=${buildId}`);
 	if (secrets.length > 0) {
-		secrets.forEach((secret) => {
-			if (secret.isBuildSecret) {
-				if (pullmergeRequestId) {
-					const isSecretFound = secrets.filter(s => s.name === secret.name && s.isPRMRSecret)
-					if (isSecretFound.length > 0) {
-						Dockerfile.push(`ARG ${secret.name}=${isSecretFound[0].value}`);
-					} else {
-						Dockerfile.push(`ARG ${secret.name}=${secret.value}`);
-					}
-				} else {
-					if (!secret.isPRMRSecret) {
-						Dockerfile.push(`ARG ${secret.name}=${secret.value}`);
-					}
-				}
-			}
+		generateSecrets(secrets, pullmergeRequestId, true).forEach((env) => {
+			Dockerfile.push(env);
 		});
 	}
 	if (depsFound) {
@@ -49,7 +37,7 @@ const createDockerfile = async (data, image): Promise<void> => {
 	Dockerfile.push(`RUN deno cache ${denoMainFile}`);
 	Dockerfile.push(`ENV NO_COLOR true`);
 	Dockerfile.push(`EXPOSE ${port}`);
-	Dockerfile.push(`CMD deno run ${denoOptions ? denoOptions.split(' ') : ''} ${denoMainFile}`);
+	Dockerfile.push(`CMD deno run ${denoOptions || ''} ${denoMainFile}`);
 	await fs.writeFile(`${workdir}/Dockerfile`, Dockerfile.join('\n'));
 };
 

apps/api/src/lib/buildPacks/docker.ts

@@ -1,53 +1,27 @@
 import { promises as fs } from 'fs';
+import { generateSecrets } from '../common';
 import { buildImage } from './common';
 
 export default async function (data) {
-	let {
-		applicationId,
-		debug,
-		tag,
-		workdir,
-		buildId,
-		baseDirectory,
-		secrets,
-		pullmergeRequestId,
-		dockerFileLocation
-	} = data
-	try {
-		const file = `${workdir}${dockerFileLocation}`;
-		let dockerFileOut = `${workdir}`;
-		if (baseDirectory) {
-			dockerFileOut = `${workdir}${baseDirectory}`;
-			workdir = `${workdir}${baseDirectory}`;
+	let { workdir, buildId, baseDirectory, secrets, pullmergeRequestId, dockerFileLocation } = data;
+	const file = `${workdir}${baseDirectory}${dockerFileLocation}`;
+	data.workdir = `${workdir}${baseDirectory}`;
+	const DockerfileRaw = await fs.readFile(`${file}`, 'utf8');
+	const Dockerfile: Array<string> = DockerfileRaw.toString().trim().split('\n');
+	Dockerfile.forEach((line, index) => {
+		if (line.startsWith('FROM')) {
+			Dockerfile.splice(index + 1, 0, `LABEL coolify.buildId=${buildId}`);
 		}
-		const Dockerfile: Array<string> = (await fs.readFile(`${file}`, 'utf8'))
-			.toString()
-			.trim()
-			.split('\n');
-		Dockerfile.push(`LABEL coolify.buildId=${buildId}`);
-		if (secrets.length > 0) {
-			secrets.forEach((secret) => {
-				if (secret.isBuildSecret) {
-					// TODO: fix secrets
-					if (
-						(pullmergeRequestId && secret.isPRMRSecret) ||
-						(!pullmergeRequestId && !secret.isPRMRSecret)
-					) {
-						Dockerfile.unshift(`ARG ${secret.name}=${secret.value}`);
-
-						Dockerfile.forEach((line, index) => {
-							if (line.startsWith('FROM')) {
-								Dockerfile.splice(index + 1, 0, `ARG ${secret.name}`);
-							}
-						});
-					}
+	});
+	if (secrets.length > 0) {
+		generateSecrets(secrets, pullmergeRequestId, true).forEach((env) => {
+			Dockerfile.forEach((line, index) => {
+				if (line.startsWith('FROM')) {
+					Dockerfile.splice(index + 1, 0, env);
 				}
 			});
-		}
-
-		await fs.writeFile(`${dockerFileOut}${dockerFileLocation}`, Dockerfile.join('\n'));
-		await buildImage(data);
-	} catch (error) {
-		throw error;
+		});
 	}
+	await fs.writeFile(`${data.workdir}${dockerFileLocation}`, Dockerfile.join('\n'));
+	await buildImage(data);
 }

apps/api/src/lib/buildPacks/heroku.ts

@@ -1,39 +1,17 @@
-import { executeDockerCmd, prisma } from "../common"
+import { executeCommand } from "../common"
 import { saveBuildLog } from "./common";
 
 export default async function (data: any): Promise<void> {
-    const { buildId, applicationId, tag, dockerId, debug, workdir } = data
+    const { buildId, applicationId, tag, dockerId, debug, workdir, baseDirectory, baseImage } = data
     try {
-
-        await saveBuildLog({ line: `Building image started.`, buildId, applicationId });
-        const { stdout } = await executeDockerCmd({
+        await saveBuildLog({ line: `Building production image...`, buildId, applicationId });
+        await executeCommand({
+            buildId,
+            debug,
             dockerId,
-            command: `pack build -p ${workdir} ${applicationId}:${tag} --builder heroku/buildpacks:20`
+            command: `pack build -p ${workdir}${baseDirectory} ${applicationId}:${tag} --builder ${baseImage}`
         })
-        if (debug) {
-            const array = stdout.split('\n')
-            for (const line of array) {
-                if (line !== '\n') {
-                    await saveBuildLog({
-                        line: `${line.replace('\n', '')}`,
-                        buildId,
-                        applicationId
-                    });
-                }
-            }
-        }
-        await saveBuildLog({ line: `Building image successful.`, buildId, applicationId });
     } catch (error) {
-        const array = error.stdout.split('\n')
-        for (const line of array) {
-            if (line !== '\n') {
-                await saveBuildLog({
-                    line: `${line.replace('\n', '')}`,
-                    buildId,
-                    applicationId
-                });
-            }
-        }
         throw error;
     }
 }

apps/api/src/lib/buildPacks/index.ts

@@ -16,6 +16,7 @@ import python from './python';
 import deno from './deno';
 import laravel from './laravel';
 import heroku from './heroku';
+import compose from './compose'
 
 export {
 	node,
@@ -35,5 +36,6 @@ export {
 	python,
 	deno,
 	laravel,
-    heroku
+    heroku,
+	compose
 };

apps/api/src/lib/buildPacks/nextjs.ts

@@ -1,4 +1,5 @@
 import { promises as fs } from 'fs';
+import { generateSecrets } from '../common';
 import { buildCacheImageWithNode, buildImage, checkPnpm } from './common';
 
 const createDockerfile = async (data, image): Promise<void> => {
@@ -24,21 +25,8 @@ const createDockerfile = async (data, image): Promise<void> => {
 	Dockerfile.push('WORKDIR /app');
 	Dockerfile.push(`LABEL coolify.buildId=${buildId}`);
 	if (secrets.length > 0) {
-		secrets.forEach((secret) => {
-			if (secret.isBuildSecret) {
-				if (pullmergeRequestId) {
-					const isSecretFound = secrets.filter(s => s.name === secret.name && s.isPRMRSecret)
-					if (isSecretFound.length > 0) {
-						Dockerfile.push(`ARG ${secret.name}=${isSecretFound[0].value}`);
-					} else {
-						Dockerfile.push(`ARG ${secret.name}=${secret.value}`);
-					}
-				} else {
-					if (!secret.isPRMRSecret) {
-						Dockerfile.push(`ARG ${secret.name}=${secret.value}`);
-					}
-				}
-			}
+		generateSecrets(secrets, pullmergeRequestId, true).forEach((env) => {
+			Dockerfile.push(env);
 		});
 	}
 	if (isPnpm) {

apps/api/src/lib/buildPacks/node.ts

@@ -1,4 +1,5 @@
 import { promises as fs } from 'fs';
+import { generateSecrets } from '../common';
 import { buildImage, checkPnpm } from './common';
 
 const createDockerfile = async (data, image): Promise<void> => {
@@ -20,21 +21,8 @@ const createDockerfile = async (data, image): Promise<void> => {
 	Dockerfile.push('WORKDIR /app');
 	Dockerfile.push(`LABEL coolify.buildId=${buildId}`);
 	if (secrets.length > 0) {
-		secrets.forEach((secret) => {
-			if (secret.isBuildSecret) {
-				if (pullmergeRequestId) {
-					const isSecretFound = secrets.filter(s => s.name === secret.name && s.isPRMRSecret)
-					if (isSecretFound.length > 0) {
-						Dockerfile.push(`ARG ${secret.name}=${isSecretFound[0].value}`);
-					} else {
-						Dockerfile.push(`ARG ${secret.name}=${secret.value}`);
-					}
-				} else {
-					if (!secret.isPRMRSecret) {
-						Dockerfile.push(`ARG ${secret.name}=${secret.value}`);
-					}
-				}
-			}
+		generateSecrets(secrets, pullmergeRequestId, true).forEach((env) => {
+			Dockerfile.push(env);
 		});
 	}
 	if (isPnpm) {

apps/api/src/lib/buildPacks/nuxtjs.ts

@@ -1,4 +1,5 @@
 import { promises as fs } from 'fs';
+import { generateSecrets } from '../common';
 import { buildCacheImageWithNode, buildImage, checkPnpm } from './common';
 
 const createDockerfile = async (data, image): Promise<void> => {
@@ -24,21 +25,8 @@ const createDockerfile = async (data, image): Promise<void> => {
 	Dockerfile.push('WORKDIR /app');
 	Dockerfile.push(`LABEL coolify.buildId=${buildId}`);
 	if (secrets.length > 0) {
-		secrets.forEach((secret) => {
-			if (secret.isBuildSecret) {
-				if (pullmergeRequestId) {
-					const isSecretFound = secrets.filter(s => s.name === secret.name && s.isPRMRSecret)
-					if (isSecretFound.length > 0) {
-						Dockerfile.push(`ARG ${secret.name}=${isSecretFound[0].value}`);
-					} else {
-						Dockerfile.push(`ARG ${secret.name}=${secret.value}`);
-					}
-				} else {
-					if (!secret.isPRMRSecret) {
-						Dockerfile.push(`ARG ${secret.name}=${secret.value}`);
-					}
-				}
-			}
+		generateSecrets(secrets, pullmergeRequestId, true).forEach((env) => {
+			Dockerfile.push(env);
 		});
 	}
 	if (isPnpm) {

apps/api/src/lib/buildPacks/php.ts

@@ -1,4 +1,5 @@
 import { promises as fs } from 'fs';
+import { generateSecrets } from '../common';
 import { buildImage } from './common';
 
 const createDockerfile = async (data, image, htaccessFound): Promise<void> => {
@@ -13,21 +14,8 @@ const createDockerfile = async (data, image, htaccessFound): Promise<void> => {
 	Dockerfile.push(`FROM ${image}`);
 	Dockerfile.push(`LABEL coolify.buildId=${buildId}`);
 	if (secrets.length > 0) {
-		secrets.forEach((secret) => {
-			if (secret.isBuildSecret) {
-				if (pullmergeRequestId) {
-					const isSecretFound = secrets.filter(s => s.name === secret.name && s.isPRMRSecret)
-					if (isSecretFound.length > 0) {
-						Dockerfile.push(`ARG ${secret.name}=${isSecretFound[0].value}`);
-					} else {
-						Dockerfile.push(`ARG ${secret.name}=${secret.value}`);
-					}
-				} else {
-					if (!secret.isPRMRSecret) {
-						Dockerfile.push(`ARG ${secret.name}=${secret.value}`);
-					}
-				}
-			}
+		generateSecrets(secrets, pullmergeRequestId, true).forEach((env) => {
+			Dockerfile.push(env);
 		});
 	}
 	Dockerfile.push('WORKDIR /app');

apps/api/src/lib/buildPacks/python.ts

@@ -1,4 +1,5 @@
 import { promises as fs } from 'fs';
+import { generateSecrets } from '../common';
 import { buildImage } from './common';
 
 const createDockerfile = async (data, image): Promise<void> => {
@@ -18,21 +19,8 @@ const createDockerfile = async (data, image): Promise<void> => {
 	Dockerfile.push('WORKDIR /app');
 	Dockerfile.push(`LABEL coolify.buildId=${buildId}`);
 	if (secrets.length > 0) {
-		secrets.forEach((secret) => {
-			if (secret.isBuildSecret) {
-				if (pullmergeRequestId) {
-					const isSecretFound = secrets.filter(s => s.name === secret.name && s.isPRMRSecret)
-					if (isSecretFound.length > 0) {
-						Dockerfile.push(`ARG ${secret.name}=${isSecretFound[0].value}`);
-					} else {
-						Dockerfile.push(`ARG ${secret.name}=${secret.value}`);
-					}
-				} else {
-					if (!secret.isPRMRSecret) {
-						Dockerfile.push(`ARG ${secret.name}=${secret.value}`);
-					}
-				}
-			}
+		generateSecrets(secrets, pullmergeRequestId, true).forEach((env) => {
+			Dockerfile.push(env);
 		});
 	}
 	if (pythonWSGI?.toLowerCase() === 'gunicorn') {

apps/api/src/lib/buildPacks/rust.ts

@@ -1,6 +1,6 @@
 import { promises as fs } from 'fs';
 import TOML from '@iarna/toml';
-import { asyncExecShell } from '../common';
+import {  executeCommand } from '../common';
 import { buildCacheImageWithCargo, buildImage } from './common';
 
 const createDockerfile = async (data, image, name): Promise<void> => {
@@ -28,7 +28,7 @@ const createDockerfile = async (data, image, name): Promise<void> => {
 export default async function (data) {
 	try {
 		const { workdir, baseImage, baseBuildImage } = data;
-		const { stdout: cargoToml } = await asyncExecShell(`cat ${workdir}/Cargo.toml`);
+		const { stdout: cargoToml } = await executeCommand({ command: `cat ${workdir}/Cargo.toml` });
 		const parsedToml: any = TOML.parse(cargoToml);
 		const name = parsedToml.package.name;
 		await buildCacheImageWithCargo(data, baseBuildImage);

apps/api/src/lib/buildPacks/static.ts

@@ -1,4 +1,5 @@
 import { promises as fs } from 'fs';
+import { generateSecrets } from '../common';
 import { buildCacheImageWithNode, buildImage } from './common';
 
 const createDockerfile = async (data, image): Promise<void> => {
@@ -18,24 +19,15 @@ const createDockerfile = async (data, image): Promise<void> => {
 	const Dockerfile: Array<string> = [];
 
 	Dockerfile.push(`FROM ${image}`);
-	Dockerfile.push('WORKDIR /app');
+	if (baseImage?.includes('httpd')) {
+		Dockerfile.push('WORKDIR /usr/local/apache2/htdocs/');
+	} else {
+		Dockerfile.push('WORKDIR /app');
+	}
 	Dockerfile.push(`LABEL coolify.buildId=${buildId}`);
 	if (secrets.length > 0) {
-		secrets.forEach((secret) => {
-			if (secret.isBuildSecret) {
-				if (pullmergeRequestId) {
-					const isSecretFound = secrets.filter(s => s.name === secret.name && s.isPRMRSecret)
-					if (isSecretFound.length > 0) {
-						Dockerfile.push(`ARG ${secret.name}=${isSecretFound[0].value}`);
-					} else {
-						Dockerfile.push(`ARG ${secret.name}=${secret.value}`);
-					}
-				} else {
-					if (!secret.isPRMRSecret) {
-						Dockerfile.push(`ARG ${secret.name}=${secret.value}`);
-					}
-				}
-			}
+		generateSecrets(secrets, pullmergeRequestId, true).forEach((env) => {
+			Dockerfile.push(env);
 		});
 	}
 	if (buildCommand) {

apps/api/src/lib/docker.ts

@@ -1,4 +1,4 @@
-import { executeDockerCmd } from './common';
+import { executeCommand } from './common';
 
 export function formatLabelsOnDocker(data) {
 	return data.trim().split('\n').map(a => JSON.parse(a)).map((container) => {
@@ -16,7 +16,7 @@ export function formatLabelsOnDocker(data) {
 export async function checkContainer({ dockerId, container, remove = false }: { dockerId: string, container: string, remove?: boolean }): Promise<{ found: boolean, status?: { isExited: boolean, isRunning: boolean, isRestarting: boolean } }> {
 	let containerFound = false;
 	try {
-		const { stdout } = await executeDockerCmd({
+		const { stdout } = await executeCommand({
 			dockerId,
 			command:
 				`docker inspect --format '{{json .State}}' ${container}`
@@ -28,27 +28,26 @@ export async function checkContainer({ dockerId, container, remove = false }: {
 		const isRestarting = status === 'restarting'
 		const isExited = status === 'exited'
 		if (status === 'created') {
-			await executeDockerCmd({
+			await executeCommand({
 				dockerId,
 				command:
 					`docker rm ${container}`
 			});
 		}
 		if (remove && status === 'exited') {
-			await executeDockerCmd({
+			await executeCommand({
 				dockerId,
 				command:
 					`docker rm ${container}`
 			});
 		}
-		
+
 		return {
 			found: containerFound,
 			status: {
 				isRunning,
 				isRestarting,
 				isExited
-
 			}
 		};
 	} catch (err) {
@@ -63,7 +62,7 @@ export async function checkContainer({ dockerId, container, remove = false }: {
 export async function isContainerExited(dockerId: string, containerName: string): Promise<boolean> {
 	let isExited = false;
 	try {
-		const { stdout } = await executeDockerCmd({ dockerId, command: `docker inspect -f '{{.State.Status}}' ${containerName}` })
+		const { stdout } = await executeCommand({ dockerId, command: `docker inspect -f '{{.State.Status}}' ${containerName}` })
 		if (stdout.trim() === 'exited') {
 			isExited = true;
 		}
@@ -82,10 +81,13 @@ export async function removeContainer({
 	dockerId: string;
 }): Promise<void> {
 	try {
-		const { stdout } = await executeDockerCmd({ dockerId, command: `docker inspect --format '{{json .State}}' ${id}` })
+		const { stdout } = await executeCommand({ dockerId, command: `docker inspect --format '{{json .State}}' ${id}` })
 		if (JSON.parse(stdout).Running) {
-			await executeDockerCmd({ dockerId, command: `docker stop -t 0 ${id}` })
-			await executeDockerCmd({ dockerId, command: `docker rm ${id}` })
+			await executeCommand({ dockerId, command: `docker stop -t 0 ${id}` })
+			await executeCommand({ dockerId, command: `docker rm ${id}` })
+		}
+		if (JSON.parse(stdout).Status === 'exited') {
+			await executeCommand({ dockerId, command: `docker rm ${id}` })
 		}
 	} catch (error) {
 		throw error;

apps/api/src/lib/importers/github.ts

@@ -1,7 +1,7 @@
 
 import jsonwebtoken from 'jsonwebtoken';
 import { saveBuildLog } from '../buildPacks/common';
-import { asyncExecShell, decrypt, prisma } from '../common';
+import { decrypt, executeCommand, prisma } from '../common';
 
 export default async function ({
 	applicationId,
@@ -9,6 +9,7 @@ export default async function ({
 	githubAppId,
 	repository,
 	apiUrl,
+	gitCommitHash,
 	htmlUrl,
 	branch,
 	buildId,
@@ -20,6 +21,7 @@ export default async function ({
 	githubAppId: string;
 	repository: string;
 	apiUrl: string;
+	gitCommitHash?: string;
 	htmlUrl: string;
 	branch: string;
 	buildId: string;
@@ -28,16 +30,24 @@ export default async function ({
 }): Promise<string> {
 	const { default: got } = await import('got')
 	const url = htmlUrl.replace('https://', '').replace('http://', '');
-	await saveBuildLog({ line: 'GitHub importer started.', buildId, applicationId });
 	if (forPublic) {
 		await saveBuildLog({
-			line: `Cloning ${repository}:${branch} branch.`,
+			line: `Cloning ${repository}:${branch}...`,
 			buildId,
 			applicationId
 		});
-		await asyncExecShell(
-			`git clone -q -b ${branch} https://${url}/${repository}.git ${workdir}/ && cd ${workdir} && git submodule update --init --recursive && git lfs pull && cd .. `
-		);
+		if (gitCommitHash) {
+			await saveBuildLog({
+				line: `Checking out ${gitCommitHash} commit...`,
+				buildId,
+				applicationId
+			});
+		}
+		await executeCommand({
+			command:
+				`git clone -q -b ${branch} https://${url}/${repository}.git ${workdir}/ && cd ${workdir} && git checkout ${gitCommitHash || ""} && git submodule update --init --recursive && git lfs pull && cd .. `,
+			shell: true
+		});
 
 	} else {
 		const body = await prisma.githubApp.findUnique({ where: { id: githubAppId } });
@@ -62,17 +72,23 @@ export default async function ({
 			})
 			.json();
 		await saveBuildLog({
-			line: `Cloning ${repository}:${branch} branch.`,
+			line: `Cloning ${repository}:${branch}...`,
 			buildId,
 			applicationId
 		});
-		await asyncExecShell(
-			`git clone -q -b ${branch} https://x-access-token:${token}@${url}/${repository}.git --config core.sshCommand="ssh -p ${customPort}" ${workdir}/ && cd ${workdir} && git submodule update --init --recursive && git lfs pull && cd .. `
-		);
+		if (gitCommitHash) {
+			await saveBuildLog({
+				line: `Checking out ${gitCommitHash} commit...`,
+				buildId,
+				applicationId
+			});
+		}
+		await executeCommand({
+			command:
+				`git clone -q -b ${branch} https://x-access-token:${token}@${url}/${repository}.git --config core.sshCommand="ssh -p ${customPort}" ${workdir}/ && cd ${workdir} && git checkout ${gitCommitHash || ""} && git submodule update --init --recursive && git lfs pull && cd .. `,
+			shell: true
+		});
 	}
-	const { stdout: commit } = await asyncExecShell(`cd ${workdir}/ && git rev-parse HEAD`);
-
+	const { stdout: commit } = await executeCommand({ command: `cd ${workdir}/ && git rev-parse HEAD`, shell: true });
 	return commit.replace('\n', '');
-
-
 }

apps/api/src/lib/importers/gitlab.ts

@@ -1,11 +1,12 @@
 import { saveBuildLog } from "../buildPacks/common";
-import { asyncExecShell } from "../common";
+import { executeCommand } from "../common";
 
 export default async function ({
 	applicationId,
 	workdir,
 	repodir,
 	htmlUrl,
+	gitCommitHash,
 	repository,
 	branch,
 	buildId,
@@ -20,34 +21,43 @@ export default async function ({
 	branch: string;
 	buildId: string;
 	repodir: string;
+	gitCommitHash: string;
 	privateSshKey: string;
 	customPort: number;
 	forPublic: boolean;
 }): Promise<string> {
 	const url = htmlUrl.replace('https://', '').replace('http://', '').replace(/\/$/, '');
-	await saveBuildLog({ line: 'GitLab importer started.', buildId, applicationId });
-
 	if (!forPublic) {
-		await asyncExecShell(`echo '${privateSshKey}' > ${repodir}/id.rsa`);
-		await asyncExecShell(`chmod 600 ${repodir}/id.rsa`);
+		await executeCommand({ command: `echo '${privateSshKey}' > ${repodir}/id.rsa`, shell: true });
+		await executeCommand({ command: `chmod 600 ${repodir}/id.rsa` });
 	}
 
 	await saveBuildLog({
-		line: `Cloning ${repository}:${branch} branch.`,
+		line: `Cloning ${repository}:${branch}...`,
 		buildId,
 		applicationId
 	});
-
+	if (gitCommitHash) {
+		await saveBuildLog({
+			line: `Checking out ${gitCommitHash} commit...`,
+			buildId,
+			applicationId
+		});
+	}
 	if (forPublic) {
-		await asyncExecShell(
-			`git clone -q -b ${branch} https://${url}/${repository}.git ${workdir}/ && cd ${workdir}/ && git submodule update --init --recursive && git lfs pull && cd .. `
+		await executeCommand({
+			command:
+				`git clone -q -b ${branch} https://${url}/${repository}.git ${workdir}/ && cd ${workdir}/ && git checkout ${gitCommitHash || ""} && git submodule update --init --recursive && git lfs pull && cd .. `, shell: true
+		}
 		);
 	} else {
-		await asyncExecShell(
-			`git clone -q -b ${branch} git@${url}:${repository}.git --config core.sshCommand="ssh -p ${customPort} -q -i ${repodir}id.rsa -o StrictHostKeyChecking=no" ${workdir}/ && cd ${workdir}/ && git submodule update --init --recursive && git lfs pull && cd .. `
+		await executeCommand({
+			command:
+				`git clone -q -b ${branch} git@${url}:${repository}.git --config core.sshCommand="ssh -p ${customPort} -q -i ${repodir}id.rsa -o StrictHostKeyChecking=no" ${workdir}/ && cd ${workdir}/ && git checkout ${gitCommitHash || ""} && git submodule update --init --recursive && git lfs pull && cd .. `, shell: true
+		}
 		);
 	}
-	
-	const { stdout: commit } = await asyncExecShell(`cd ${workdir}/ && git rev-parse HEAD`);
+
+	const { stdout: commit } = await executeCommand({ command: `cd ${workdir}/ && git rev-parse HEAD`, shell: true });
 	return commit.replace('\n', '');
 }

apps/api/src/lib/scheduler.ts

@@ -9,17 +9,16 @@ Bree.extend(TSBree);
 
 const options: any = {
 	defaultExtension: 'js',
-	// logger: new Cabin(),
 	logger: false,
-	workerMessageHandler: async ({ name, message }) => {
-		if (name === 'deployApplication' && message?.deploying) {
-			if (scheduler.workers.has('autoUpdater') || scheduler.workers.has('cleanupStorage')) {
-				scheduler.workers.get('deployApplication').postMessage('cancel')
-			}
-		}
-	},
+	// logger: false,
+	// workerMessageHandler: async ({ name, message }) => {
+	// 	if (name === 'deployApplication' && message?.deploying) {
+	// 		if (scheduler.workers.has('autoUpdater') || scheduler.workers.has('cleanupStorage')) {
+	// 			scheduler.workers.get('deployApplication').postMessage('cancel')
+	// 		}
+	// 	}
+	// },
 	jobs: [
-		{ name: 'infrastructure' },
 		{ name: 'deployApplication' },
 	],
 };

apps/api/src/lib/services.ts

@@ -1,20 +1,47 @@
-import { createDirectories, getServiceFromDB, getServiceImage, getServiceMainPort, makeLabelForServices } from "./common";
-
-export async function defaultServiceConfigurations({ id, teamId }) {
-    const service = await getServiceFromDB({ id, teamId });
-    const { destinationDockerId, destinationDocker, type, serviceSecret } = service;
-
-    const network = destinationDockerId && destinationDocker.network;
-    const port = getServiceMainPort(type);
-
-    const { workdir } = await createDirectories({ repository: type, buildId: id });
-
-    const image = getServiceImage(type);
-    let secrets = [];
-    if (serviceSecret.length > 0) {
-        serviceSecret.forEach((secret) => {
-            secrets.push(`${secret.name}=${secret.value}`);
-        });
-    }
-    return { ...service, network, port, workdir, image, secrets }
-}
+import { isARM, isDev } from './common';
+import fs from 'fs/promises';
+export async function getTemplates() {
+	const templatePath = isDev ? './templates.json' : '/app/templates.json';
+	const open = await fs.open(templatePath, 'r');
+	try {
+		let data = await open.readFile({ encoding: 'utf-8' });
+		let jsonData = JSON.parse(data);
+		if (isARM(process.arch)) {
+			jsonData = jsonData.filter((d) => d.arch !== 'amd64');
+		}
+		return jsonData;
+	} catch (error) {
+		return [];
+	} finally {
+		await open?.close();
+	}
+}
+const compareSemanticVersions = (a: string, b: string) => {
+	const a1 = a.split('.');
+	const b1 = b.split('.');
+	const len = Math.min(a1.length, b1.length);
+	for (let i = 0; i < len; i++) {
+		const a2 = +a1[i] || 0;
+		const b2 = +b1[i] || 0;
+		if (a2 !== b2) {
+			return a2 > b2 ? 1 : -1;
+		}
+	}
+	return b1.length - a1.length;
+};
+export async function getTags(type: string) {
+	try {
+		if (type) {
+			const tagsPath = isDev ? './tags.json' : '/app/tags.json';
+			const data = await fs.readFile(tagsPath, 'utf8');
+			let tags = JSON.parse(data);
+			if (tags) {
+				tags = tags.find((tag: any) => tag.name.includes(type));
+				tags.tags = tags.tags.sort(compareSemanticVersions).reverse();
+				return tags;
+			}
+		}
+	} catch (error) {
+		return [];
+	}
+}

apps/api/src/lib/services/common.ts

@@ -1,367 +1,9 @@
 
-import cuid from 'cuid';
-import { encrypt, generatePassword, prisma } from '../common';
-
-export const includeServices: any = {
-	destinationDocker: true,
-	persistentStorage: true,
-	serviceSecret: true,
-	minio: true,
-	plausibleAnalytics: true,
-	vscodeserver: true,
-	wordpress: true,
-	ghost: true,
-	meiliSearch: true,
-	umami: true,
-	hasura: true,
-	fider: true,
-	moodle: true,
-	appwrite: true,
-	glitchTip: true,
-	searxng: true,
-	weblate: true,
-	taiga: true
-};
-export async function configureServiceType({
-	id,
-	type
-}: {
-	id: string;
-	type: string;
-}): Promise<void> {
-	if (type === 'plausibleanalytics') {
-		const password = encrypt(generatePassword({}));
-		const postgresqlUser = cuid();
-		const postgresqlPassword = encrypt(generatePassword({}));
-		const postgresqlDatabase = 'plausibleanalytics';
-		const secretKeyBase = encrypt(generatePassword({ length: 64 }));
-
-		await prisma.service.update({
-			where: { id },
-			data: {
-				type,
-				plausibleAnalytics: {
-					create: {
-						postgresqlDatabase,
-						postgresqlUser,
-						postgresqlPassword,
-						password,
-						secretKeyBase
-					}
-				}
-			}
-		});
-	} else if (type === 'nocodb') {
-		await prisma.service.update({
-			where: { id },
-			data: { type }
-		});
-	} else if (type === 'minio') {
-		const rootUser = cuid();
-		const rootUserPassword = encrypt(generatePassword({}));
-		await prisma.service.update({
-			where: { id },
-			data: { type, minio: { create: { rootUser, rootUserPassword } } }
-		});
-	} else if (type === 'vscodeserver') {
-		const password = encrypt(generatePassword({}));
-		await prisma.service.update({
-			where: { id },
-			data: { type, vscodeserver: { create: { password } } }
-		});
-	} else if (type === 'wordpress') {
-		const mysqlUser = cuid();
-		const mysqlPassword = encrypt(generatePassword({}));
-		const mysqlRootUser = cuid();
-		const mysqlRootUserPassword = encrypt(generatePassword({}));
-		await prisma.service.update({
-			where: { id },
-			data: {
-				type,
-				wordpress: { create: { mysqlPassword, mysqlRootUserPassword, mysqlRootUser, mysqlUser } }
-			}
-		});
-	} else if (type === 'vaultwarden') {
-		await prisma.service.update({
-			where: { id },
-			data: {
-				type
-			}
-		});
-	} else if (type === 'languagetool') {
-		await prisma.service.update({
-			where: { id },
-			data: {
-				type
-			}
-		});
-	} else if (type === 'n8n') {
-		await prisma.service.update({
-			where: { id },
-			data: {
-				type
-			}
-		});
-	} else if (type === 'uptimekuma') {
-		await prisma.service.update({
-			where: { id },
-			data: {
-				type
-			}
-		});
-	} else if (type === 'ghost') {
-		const defaultEmail = `${cuid()}@example.com`;
-		const defaultPassword = encrypt(generatePassword({}));
-		const mariadbUser = cuid();
-		const mariadbPassword = encrypt(generatePassword({}));
-		const mariadbRootUser = cuid();
-		const mariadbRootUserPassword = encrypt(generatePassword({}));
-
-		await prisma.service.update({
-			where: { id },
-			data: {
-				type,
-				ghost: {
-					create: {
-						defaultEmail,
-						defaultPassword,
-						mariadbUser,
-						mariadbPassword,
-						mariadbRootUser,
-						mariadbRootUserPassword
-					}
-				}
-			}
-		});
-	} else if (type === 'meilisearch') {
-		const masterKey = encrypt(generatePassword({ length: 32 }));
-		await prisma.service.update({
-			where: { id },
-			data: {
-				type,
-				meiliSearch: { create: { masterKey } }
-			}
-		});
-	} else if (type === 'umami') {
-		const umamiAdminPassword = encrypt(generatePassword({}));
-		const postgresqlUser = cuid();
-		const postgresqlPassword = encrypt(generatePassword({}));
-		const postgresqlDatabase = 'umami';
-		const hashSalt = encrypt(generatePassword({ length: 64 }));
-		await prisma.service.update({
-			where: { id },
-			data: {
-				type,
-				umami: {
-					create: {
-						umamiAdminPassword,
-						postgresqlDatabase,
-						postgresqlPassword,
-						postgresqlUser,
-						hashSalt
-					}
-				}
-			}
-		});
-	} else if (type === 'hasura') {
-		const postgresqlUser = cuid();
-		const postgresqlPassword = encrypt(generatePassword({}));
-		const postgresqlDatabase = 'hasura';
-		const graphQLAdminPassword = encrypt(generatePassword({}));
-		await prisma.service.update({
-			where: { id },
-			data: {
-				type,
-				hasura: {
-					create: {
-						postgresqlDatabase,
-						postgresqlPassword,
-						postgresqlUser,
-						graphQLAdminPassword
-					}
-				}
-			}
-		});
-	} else if (type === 'fider') {
-		const postgresqlUser = cuid();
-		const postgresqlPassword = encrypt(generatePassword({}));
-		const postgresqlDatabase = 'fider';
-		const jwtSecret = encrypt(generatePassword({ length: 64, symbols: true }));
-		await prisma.service.update({
-			where: { id },
-			data: {
-				type,
-				fider: {
-					create: {
-						postgresqlDatabase,
-						postgresqlPassword,
-						postgresqlUser,
-						jwtSecret
-					}
-				}
-			}
-		});
-	} else if (type === 'moodle') {
-		const defaultUsername = cuid();
-		const defaultPassword = encrypt(generatePassword({}));
-		const defaultEmail = `${cuid()} @example.com`;
-		const mariadbUser = cuid();
-		const mariadbPassword = encrypt(generatePassword({}));
-		const mariadbDatabase = 'moodle_db';
-		const mariadbRootUser = cuid();
-		const mariadbRootUserPassword = encrypt(generatePassword({}));
-		await prisma.service.update({
-			where: { id },
-			data: {
-				type,
-				moodle: {
-					create: {
-						defaultUsername,
-						defaultPassword,
-						defaultEmail,
-						mariadbUser,
-						mariadbPassword,
-						mariadbDatabase,
-						mariadbRootUser,
-						mariadbRootUserPassword
-					}
-				}
-			}
-		});
-	} else if (type === 'appwrite') {
-		const opensslKeyV1 = encrypt(generatePassword({}));
-		const executorSecret = encrypt(generatePassword({}));
-		const redisPassword = encrypt(generatePassword({}));
-		const mariadbHost = `${id}-mariadb`
-		const mariadbUser = cuid();
-		const mariadbPassword = encrypt(generatePassword({}));
-		const mariadbDatabase = 'appwrite';
-		const mariadbRootUser = cuid();
-		const mariadbRootUserPassword = encrypt(generatePassword({}));
-		await prisma.service.update({
-			where: { id },
-			data: {
-				type,
-				appwrite: {
-					create: {
-						opensslKeyV1,
-						executorSecret,
-						redisPassword,
-						mariadbHost,
-						mariadbUser,
-						mariadbPassword,
-						mariadbDatabase,
-						mariadbRootUser,
-						mariadbRootUserPassword
-					}
-				}
-			}
-		});
-	} else if (type === 'glitchTip') {
-		const defaultUsername = cuid();
-		const defaultEmail = `${defaultUsername}@example.com`;
-		const defaultPassword = encrypt(generatePassword({}));
-		const postgresqlUser = cuid();
-		const postgresqlPassword = encrypt(generatePassword({}));
-		const postgresqlDatabase = 'glitchTip';
-		const secretKeyBase = encrypt(generatePassword({ length: 64 }));
-
-		await prisma.service.update({
-			where: { id },
-			data: {
-				type,
-				glitchTip: {
-					create: {
-						postgresqlDatabase,
-						postgresqlUser,
-						postgresqlPassword,
-						secretKeyBase,
-						defaultEmail,
-						defaultUsername,
-						defaultPassword,
-					}
-				}
-			}
-		});
-	} else if (type === 'searxng') {
-		const secretKey = encrypt(generatePassword({ length: 32, isHex: true }))
-		const redisPassword = encrypt(generatePassword({}));
-		await prisma.service.update({
-			where: { id },
-			data: {
-				type,
-				searxng: {
-					create: {
-						secretKey,
-						redisPassword,
-					}
-				}
-			}
-		});
-	} else if (type === 'weblate') {
-		const adminPassword = encrypt(generatePassword({}))
-		const postgresqlUser = cuid();
-		const postgresqlPassword = encrypt(generatePassword({}));
-		const postgresqlDatabase = 'weblate';
-		await prisma.service.update({
-			where: { id },
-			data: {
-				type,
-				weblate: {
-					create: {
-						adminPassword,
-						postgresqlHost: `${id}-postgresql`,
-						postgresqlPort: 5432,
-						postgresqlUser,
-						postgresqlPassword,
-						postgresqlDatabase,
-					}
-				}
-			}
-		});
-	} else if (type === 'taiga') {
-		const secretKey = encrypt(generatePassword({}))
-		const erlangSecret = encrypt(generatePassword({}))
-		const rabbitMQUser = cuid();
-		const djangoAdminUser = cuid();
-		const djangoAdminPassword = encrypt(generatePassword({}))
-		const rabbitMQPassword = encrypt(generatePassword({}))
-		const postgresqlUser = cuid();
-		const postgresqlPassword = encrypt(generatePassword({}));
-		const postgresqlDatabase = 'taiga';
-		await prisma.service.update({
-			where: { id },
-			data: {
-				type,
-				taiga: {
-					create: {
-						secretKey,
-						erlangSecret,
-						djangoAdminUser,
-						djangoAdminPassword,
-						rabbitMQUser,
-						rabbitMQPassword,
-						postgresqlHost: `${id}-postgresql`,
-						postgresqlPort: 5432,
-						postgresqlUser,
-						postgresqlPassword,
-						postgresqlDatabase,
-					}
-				}
-			}
-		});
-	} else {
-		await prisma.service.update({
-			where: { id },
-			data: {
-				type
-			}
-		});
-	}
-}
+import { decrypt, prisma } from '../common';
 
 export async function removeService({ id }: { id: string }): Promise<void> {
 	await prisma.serviceSecret.deleteMany({ where: { serviceId: id } });
+	await prisma.serviceSetting.deleteMany({ where: { serviceId: id } });
 	await prisma.servicePersistentStorage.deleteMany({ where: { serviceId: id } });
 	await prisma.meiliSearch.deleteMany({ where: { serviceId: id } });
 	await prisma.fider.deleteMany({ where: { serviceId: id } });
@@ -378,6 +20,20 @@ export async function removeService({ id }: { id: string }): Promise<void> {
 	await prisma.searxng.deleteMany({ where: { serviceId: id } });
 	await prisma.weblate.deleteMany({ where: { serviceId: id } });
 	await prisma.taiga.deleteMany({ where: { serviceId: id } });
-	
+
 	await prisma.service.delete({ where: { id } });
+}
+export async function verifyAndDecryptServiceSecrets(id: string) {
+	const secrets = await prisma.serviceSecret.findMany({ where: { serviceId: id } })
+	let decryptedSecrets = secrets.map(secret => {
+		const { name, value } = secret
+		if (value) {
+			let rawValue = decrypt(value)
+			rawValue = rawValue.replaceAll(/\$/gi, '$$$')
+			return { name, value: rawValue }
+		}
+		return { name, value }
+
+	})
+	return decryptedSecrets
 }

apps/api/src/lib/services/supportedVersions.ts

@@ -1,215 +0,0 @@
-export const supportedServiceTypesAndVersions = [
-	{
-		name: 'plausibleanalytics',
-		fancyName: 'Plausible Analytics',
-		baseImage: 'plausible/analytics',
-		images: ['bitnami/postgresql:13.2.0', 'yandex/clickhouse-server:21.3.2.5'],
-		versions: ['latest', 'stable'],
-		recommendedVersion: 'stable',
-		ports: {
-			main: 8000
-		}
-	},
-	{
-		name: 'nocodb',
-		fancyName: 'NocoDB',
-		baseImage: 'nocodb/nocodb',
-		versions: ['latest'],
-		recommendedVersion: 'latest',
-		ports: {
-			main: 8080
-		}
-	},
-	{
-		name: 'minio',
-		fancyName: 'MinIO',
-		baseImage: 'minio/minio',
-		versions: ['latest'],
-		recommendedVersion: 'latest',
-		ports: {
-			main: 9001
-		}
-	},
-	{
-		name: 'vscodeserver',
-		fancyName: 'VSCode Server',
-		baseImage: 'codercom/code-server',
-		versions: ['latest'],
-		recommendedVersion: 'latest',
-		ports: {
-			main: 8080
-		}
-	},
-	{
-		name: 'wordpress',
-		fancyName: 'Wordpress',
-		baseImage: 'wordpress',
-		images: ['bitnami/mysql:5.7'],
-		versions: ['latest', 'php8.1', 'php8.0', 'php7.4', 'php7.3'],
-		recommendedVersion: 'latest',
-		ports: {
-			main: 80
-		}
-	},
-	{
-		name: 'vaultwarden',
-		fancyName: 'Vaultwarden',
-		baseImage: 'vaultwarden/server',
-		versions: ['latest'],
-		recommendedVersion: 'latest',
-		ports: {
-			main: 80
-		}
-	},
-	{
-		name: 'languagetool',
-		fancyName: 'LanguageTool',
-		baseImage: 'silviof/docker-languagetool',
-		versions: ['latest'],
-		recommendedVersion: 'latest',
-		ports: {
-			main: 8010
-		}
-	},
-	{
-		name: 'n8n',
-		fancyName: 'n8n',
-		baseImage: 'n8nio/n8n',
-		versions: ['latest'],
-		recommendedVersion: 'latest',
-		ports: {
-			main: 5678
-		}
-	},
-	{
-		name: 'uptimekuma',
-		fancyName: 'Uptime Kuma',
-		baseImage: 'louislam/uptime-kuma',
-		versions: ['latest'],
-		recommendedVersion: 'latest',
-		ports: {
-			main: 3001
-		}
-	},
-	{
-		name: 'ghost',
-		fancyName: 'Ghost',
-		baseImage: 'bitnami/ghost',
-		images: ['bitnami/mariadb'],
-		versions: ['latest'],
-		recommendedVersion: 'latest',
-		ports: {
-			main: 2368
-		}
-	},
-	{
-		name: 'meilisearch',
-		fancyName: 'Meilisearch',
-		baseImage: 'getmeili/meilisearch',
-		images: [],
-		versions: ['latest'],
-		recommendedVersion: 'latest',
-		ports: {
-			main: 7700
-		}
-	},
-	{
-		name: 'umami',
-		fancyName: 'Umami',
-		baseImage: 'ghcr.io/umami-software/umami',
-		images: ['postgres:12-alpine'],
-		versions: ['postgresql-latest'],
-		recommendedVersion: 'postgresql-latest',
-		ports: {
-			main: 3000
-		}
-	},
-	{
-		name: 'hasura',
-		fancyName: 'Hasura',
-		baseImage: 'hasura/graphql-engine',
-		images: ['postgres:12-alpine'],
-		versions: ['latest', 'v2.10.0', 'v2.5.1'],
-		recommendedVersion: 'v2.10.0',
-		ports: {
-			main: 8080
-		}
-	},
-	{
-		name: 'fider',
-		fancyName: 'Fider',
-		baseImage: 'getfider/fider',
-		images: ['postgres:12-alpine'],
-		versions: ['stable'],
-		recommendedVersion: 'stable',
-		ports: {
-			main: 3000
-		}
-	},
-	{
-		name: 'appwrite',
-		fancyName: 'Appwrite',
-		baseImage: 'appwrite/appwrite',
-		images: ['mariadb:10.7', 'redis:6.2-alpine', 'appwrite/telegraf:1.4.0'],
-		versions: ['latest', '0.15.3'],
-		recommendedVersion: '0.15.3',
-		ports: {
-			main: 80
-		}
-	},
-	// {
-	//     name: 'moodle',
-	//     fancyName: 'Moodle',
-	//     baseImage: 'bitnami/moodle',
-	//     images: [],
-	//     versions: ['latest', 'v4.0.2'],
-	//     recommendedVersion: 'latest',
-	//     ports: {
-	//         main: 8080
-	//     }
-	// }
-	{
-		name: 'glitchTip',
-		fancyName: 'GlitchTip',
-		baseImage: 'glitchtip/glitchtip',
-		images: ['postgres:14-alpine', 'redis:7-alpine'],
-		versions: ['latest'],
-		recommendedVersion: 'latest',
-		ports: {
-			main: 8000
-		}
-	},
-	{
-		name: 'searxng',
-		fancyName: 'SearXNG',
-		baseImage: 'searxng/searxng',
-		images: [],
-		versions: ['latest'],
-		recommendedVersion: 'latest',
-		ports: {
-			main: 8080
-		}
-	},
-	{
-		name: 'weblate',
-		fancyName: 'Weblate',
-		baseImage: 'weblate/weblate',
-		images: ['postgres:14-alpine', 'redis:6-alpine'],
-		versions: ['latest'],
-		recommendedVersion: 'latest',
-		ports: {
-			main: 8080
-		}
-	},
-	// {
-	// 	name: 'taiga',
-	// 	fancyName: 'Taiga',
-	// 	baseImage: 'taigaio/taiga-front',
-	// 	images: ['postgres:12.3', 'rabbitmq:3.8-management-alpine', 'taigaio/taiga-back', 'taigaio/taiga-events', 'taigaio/taiga-protected'],
-	// 	versions: ['latest'],
-	// 	recommendedVersion: 'latest',
-	// 	ports: {
-	// 		main: 80
-	// 	}
-	// },
-];

apps/api/src/realtime/index.ts

@@ -0,0 +1,29 @@
+
+export default async (fastify) => {
+    fastify.io.use((socket, next) => {
+        const { token } = socket.handshake.auth;
+        if (token && fastify.jwt.verify(token)) {
+            next();
+        } else {
+            return next(new Error("unauthorized event"));
+        }
+    });
+    fastify.io.on('connection', (socket: any) => {
+        const { token } = socket.handshake.auth;
+        const { teamId } = fastify.jwt.decode(token);
+        socket.join(teamId);
+        // console.info('Socket connected!', socket.id)
+        // console.info('Socket joined team!', teamId)
+        // socket.on('message', (message) => {
+        //     console.log(message)
+        // })
+        // socket.on('error', (err) => {
+        //     console.log(err)
+        // })
+    })
+    // fastify.io.on("error", (err) => {
+    //     if (err && err.message === "unauthorized event") {
+    //         fastify.io.disconnect();
+    //     }
+    // });
+}

apps/api/src/routes/api/v1/applications/index.ts

@@ -1,8 +1,8 @@
 import { FastifyPluginAsync } from 'fastify';
 import { OnlyId } from '../../../../types';
-import { cancelDeployment, checkDNS, checkDomain, checkRepository, deleteApplication, deleteSecret, deleteStorage, deployApplication, getApplication, getApplicationLogs, getApplicationStatus, getBuildIdLogs, getBuildLogs, getBuildPack, getGitHubToken, getGitLabSSHKey, getImages, getPreviews, getSecrets, getStorages, getUsage, listApplications, newApplication, restartApplication, saveApplication, saveApplicationSettings, saveApplicationSource, saveBuildPack, saveConnectedDatabase, saveDeployKey, saveDestination, saveGitLabSSHKey, saveRepository, saveSecret, saveStorage, stopApplication, stopPreviewApplication } from './handlers';
+import { cancelDeployment, checkDNS, checkDomain, checkRepository, cleanupUnconfiguredApplications, deleteApplication, deleteSecret, deleteStorage, deployApplication, getApplication, getApplicationLogs, getApplicationStatus, getBuildIdLogs, getBuildPack, getBuilds, getDockerImages, getGitHubToken, getGitLabSSHKey, getImages, getPreviews, getPreviewStatus, getSecrets, getStorages, getUsage, getUsageByContainer, listApplications, loadPreviews, newApplication, restartApplication, restartPreview, saveApplication, saveApplicationSettings, saveApplicationSource, saveBuildPack, saveConnectedDatabase, saveDeployKey, saveDestination, saveGitLabSSHKey,  saveRegistry,  saveRepository, saveSecret, saveStorage, stopApplication, stopPreviewApplication, updatePreviewSecret, updateSecret } from './handlers';
 
-import type { CancelDeployment, CheckDNS, CheckDomain, CheckRepository, DeleteApplication, DeleteSecret, DeleteStorage, DeployApplication, GetApplicationLogs, GetBuildIdLogs, GetBuildLogs, GetImages, SaveApplication, SaveApplicationSettings, SaveApplicationSource, SaveDeployKey, SaveDestination, SaveSecret, SaveStorage, StopPreviewApplication } from './types';
+import type { CancelDeployment, CheckDNS, CheckDomain, CheckRepository, DeleteApplication, DeleteSecret, DeleteStorage, DeployApplication, GetApplicationLogs, GetBuildIdLogs, GetBuilds, GetImages, RestartApplication, RestartPreviewApplication, SaveApplication, SaveApplicationSettings, SaveApplicationSource, SaveDeployKey, SaveDestination, SaveSecret, SaveStorage, StopPreviewApplication } from './types';
 
 const root: FastifyPluginAsync = async (fastify): Promise<void> => {
     fastify.addHook('onRequest', async (request) => {
@@ -11,6 +11,8 @@ const root: FastifyPluginAsync = async (fastify): Promise<void> => {
     fastify.get('/', async (request) => await listApplications(request));
     fastify.post<GetImages>('/images', async (request) => await getImages(request));
 
+    fastify.post<any>('/cleanup/unconfigured', async (request) => await cleanupUnconfiguredApplications(request));
+
     fastify.post('/new', async (request, reply) => await newApplication(request, reply));
 
     fastify.get<OnlyId>('/:id', async (request) => await getApplication(request));
@@ -19,7 +21,7 @@ const root: FastifyPluginAsync = async (fastify): Promise<void> => {
 
     fastify.get<OnlyId>('/:id/status', async (request) => await getApplicationStatus(request));
 
-    fastify.post<OnlyId>('/:id/restart', async (request, reply) => await restartApplication(request, reply));
+    fastify.post<RestartApplication>('/:id/restart', async (request, reply) => await restartApplication(request, reply));
     fastify.post<OnlyId>('/:id/stop', async (request, reply) => await stopApplication(request, reply));
     fastify.post<StopPreviewApplication>('/:id/stop/preview', async (request, reply) => await stopPreviewApplication(request, reply));
 
@@ -30,6 +32,8 @@ const root: FastifyPluginAsync = async (fastify): Promise<void> => {
 
     fastify.get<OnlyId>('/:id/secrets', async (request) => await getSecrets(request));
     fastify.post<SaveSecret>('/:id/secrets', async (request, reply) => await saveSecret(request, reply));
+    fastify.put<SaveSecret>('/:id/secrets', async (request, reply) => await updateSecret(request, reply));
+    fastify.put<SaveSecret>('/:id/secrets/preview', async (request, reply) => await updatePreviewSecret(request, reply));
     fastify.delete<DeleteSecret>('/:id/secrets', async (request) => await deleteSecret(request));
 
     fastify.get<OnlyId>('/:id/storages', async (request) => await getStorages(request));
@@ -37,12 +41,18 @@ const root: FastifyPluginAsync = async (fastify): Promise<void> => {
     fastify.delete<DeleteStorage>('/:id/storages', async (request) => await deleteStorage(request));
 
     fastify.get<OnlyId>('/:id/previews', async (request) => await getPreviews(request));
+    fastify.post<OnlyId>('/:id/previews/load', async (request) => await loadPreviews(request));
+    fastify.get<RestartPreviewApplication>('/:id/previews/:pullmergeRequestId/status', async (request) => await getPreviewStatus(request));
+    fastify.post<RestartPreviewApplication>('/:id/previews/:pullmergeRequestId/restart', async (request, reply) => await restartPreview(request, reply));
 
-    fastify.get<GetApplicationLogs>('/:id/logs', async (request) => await getApplicationLogs(request));
-    fastify.get<GetBuildLogs>('/:id/logs/build', async (request) => await getBuildLogs(request));
+    fastify.get<GetApplicationLogs>('/:id/logs/:containerId', async (request) => await getApplicationLogs(request));
+    fastify.get<GetBuilds>('/:id/logs/build', async (request) => await getBuilds(request));
     fastify.get<GetBuildIdLogs>('/:id/logs/build/:buildId', async (request) => await getBuildIdLogs(request));
 
     fastify.get('/:id/usage', async (request) => await getUsage(request))
+    fastify.get('/:id/usage/:containerId', async (request) => await getUsageByContainer(request))
+
+    fastify.get('/:id/images', async (request) => await getDockerImages(request))
 
     fastify.post<DeployApplication>('/:id/deploy', async (request) => await deployApplication(request))
     fastify.post<CancelDeployment>('/:id/cancel', async (request, reply) => await cancelDeployment(request, reply));
@@ -55,6 +65,8 @@ const root: FastifyPluginAsync = async (fastify): Promise<void> => {
     fastify.get('/:id/configuration/buildpack', async (request) => await getBuildPack(request));
     fastify.post('/:id/configuration/buildpack', async (request, reply) => await saveBuildPack(request, reply));
 
+    fastify.post('/:id/configuration/registry', async (request, reply) => await saveRegistry(request, reply));
+
     fastify.post('/:id/configuration/database', async (request, reply) => await saveConnectedDatabase(request, reply));
 
     fastify.get<OnlyId>('/:id/configuration/sshkey', async (request) => await getGitLabSSHKey(request));

apps/api/src/routes/api/v1/applications/types.ts

@@ -19,14 +19,20 @@ export interface SaveApplication extends OnlyId {
         denoMainFile: string,
         denoOptions: string,
         baseImage: string,
+        gitCommitHash: string,
         baseBuildImage: string,
         deploymentType: string,
-        baseDatabaseBranch: string
+        baseDatabaseBranch: string,
+        dockerComposeFile: string,
+        dockerComposeFileLocation: string,
+        dockerComposeConfiguration: string,
+        simpleDockerfile: string,
+        dockerRegistryImageName: string
     }
 }
 export interface SaveApplicationSettings extends OnlyId {
     Querystring: { domain: string; };
-    Body: { debug: boolean; previews: boolean; dualCerts: boolean; autodeploy: boolean; branch: string; projectId: number; isBot: boolean; isDBBranching: boolean };
+    Body: { debug: boolean; previews: boolean; dualCerts: boolean; autodeploy: boolean; branch: string; projectId: number; isBot: boolean; isDBBranching: boolean, isCustomSSL: boolean };
 }
 export interface DeleteApplication extends OnlyId {
     Querystring: { domain: string; };
@@ -52,7 +58,7 @@ export interface GetImages {
     Body: { buildPack: string, deploymentType: string }
 }
 export interface SaveApplicationSource extends OnlyId {
-    Body: { gitSourceId?: string | null, forPublic?: boolean, type?: string }
+    Body: { gitSourceId?: string | null, forPublic?: boolean, type?: string, simpleDockerfile?: string }
 }
 export interface CheckRepository extends OnlyId {
     Querystring: { repository: string, branch: string }
@@ -65,7 +71,7 @@ export interface SaveSecret extends OnlyId {
         name: string,
         value: string,
         isBuildSecret: boolean,
-        isPRMRSecret: boolean,
+        previewSecret: boolean,
         isNew: boolean
     }
 }
@@ -84,12 +90,16 @@ export interface DeleteStorage extends OnlyId {
         path: string,
     }
 }
-export interface GetApplicationLogs extends OnlyId {
+export interface GetApplicationLogs {
+    Params: {
+        id: string,
+        containerId: string
+    }
     Querystring: {
         since: number,
     }
 }
-export interface GetBuildLogs extends OnlyId {
+export interface GetBuilds extends OnlyId {
     Querystring: {
         buildId: string
         skip: number,
@@ -97,6 +107,7 @@ export interface GetBuildLogs extends OnlyId {
 }
 export interface GetBuildIdLogs {
     Params: {
+        id: string,
         buildId: string
     },
     Querystring: {
@@ -126,4 +137,18 @@ export interface StopPreviewApplication extends OnlyId {
     Body: {
         pullmergeRequestId: string | null,
     }
+}
+export interface RestartPreviewApplication {
+    Params: {
+        id: string,
+        pullmergeRequestId: string | null,
+    }
+}
+export interface RestartApplication {
+    Params: {
+        id: string,
+    },
+    Body: {
+        imageId: string | null,
+    }
 }

apps/api/src/routes/api/v1/base/index.ts

@@ -1,23 +1,31 @@
 import { FastifyPluginAsync } from 'fastify';
-import { errorHandler, listSettings, version } from '../../../../lib/common';
+import { errorHandler, isARM, listSettings, version } from '../../../../lib/common';
 
 const root: FastifyPluginAsync = async (fastify): Promise<void> => {
-    fastify.get('/', async () => {
-        const settings = await listSettings()
-        try {
-            return {
-                ipv4: settings.ipv4,
-                ipv6: settings.ipv6,
-                version,
-                whiteLabeled: process.env.COOLIFY_WHITE_LABELED === 'true',
-                whiteLabeledIcon: process.env.COOLIFY_WHITE_LABELED_ICON,
-                isRegistrationEnabled: settings.isRegistrationEnabled,
-            }
-        } catch ({ status, message }) {
-            return errorHandler({ status, message })
-        }
-    });
-
+	fastify.addHook('onRequest', async (request) => {
+		try {
+			await request.jwtVerify();
+		} catch (error) {
+			return;
+		}
+	});
+	fastify.get('/', async (request) => {
+		const teamId = request.user?.teamId;
+		const settings = await listSettings();
+		try {
+			return {
+				ipv4: teamId ? settings.ipv4 : null,
+				ipv6: teamId ? settings.ipv6 : null,
+				version,
+				whiteLabeled: process.env.COOLIFY_WHITE_LABELED === 'true',
+				whiteLabeledIcon: process.env.COOLIFY_WHITE_LABELED_ICON,
+				isRegistrationEnabled: settings.isRegistrationEnabled,
+				isARM: isARM(process.arch)
+			};
+		} catch ({ status, message }) {
+			return errorHandler({ status, message });
+		}
+	});
 };
 
 export default root;

apps/api/src/routes/api/v1/databases/handlers.ts

@@ -3,7 +3,7 @@ import type { FastifyRequest } from 'fastify';
 import { FastifyReply } from 'fastify';
 import yaml from 'js-yaml';
 import fs from 'fs/promises';
-import { ComposeFile, createDirectories, decrypt, defaultComposeConfiguration, encrypt, errorHandler, executeDockerCmd, generateDatabaseConfiguration, generatePassword, getContainerUsage, getDatabaseImage, getDatabaseVersions, getFreePublicPort, listSettings, makeLabelForStandaloneDatabase, prisma, startTraefikTCPProxy, stopDatabaseContainer, stopTcpHttpProxy, supportedDatabaseTypesAndVersions, uniqueName, updatePasswordInDb } from '../../../../lib/common';
+import { ComposeFile, createDirectories, decrypt, defaultComposeConfiguration, encrypt, errorHandler, executeCommand, generateDatabaseConfiguration, generatePassword, getContainerUsage, getDatabaseImage, getDatabaseVersions, getFreePublicPort, listSettings, makeLabelForStandaloneDatabase, prisma, startTraefikTCPProxy, stopDatabaseContainer, stopTcpHttpProxy, supportedDatabaseTypesAndVersions, uniqueName, updatePasswordInDb } from '../../../../lib/common';
 import { day } from '../../../../lib/dayjs';
 
 import type { OnlyId } from '../../../../types';
@@ -51,6 +51,30 @@ export async function newDatabase(request: FastifyRequest, reply: FastifyReply)
         return errorHandler({ status, message })
     }
 }
+export async function cleanupUnconfiguredDatabases(request: FastifyRequest) {
+    try {
+        const teamId = request.user.teamId;
+        let databases = await prisma.database.findMany({
+            where: { teams: { some: { id: teamId === "0" ? undefined : teamId } } },
+            include: { settings: true, destinationDocker: true, teams: true },
+        });
+        for (const database of databases) {
+            if (!database?.version) {
+                const { id } = database;
+                if (database.destinationDockerId) {
+                    const everStarted = await stopDatabaseContainer(database);
+                    if (everStarted) await stopTcpHttpProxy(id, database.destinationDocker, database.publicPort);
+                }
+                await prisma.databaseSettings.deleteMany({ where: { databaseId: id } });
+                await prisma.databaseSecret.deleteMany({ where: { databaseId: id } });
+                await prisma.database.delete({ where: { id } });
+            }
+        }
+        return {}
+    } catch ({ status, message }) {
+        return errorHandler({ status, message })
+    }
+}
 export async function getDatabaseStatus(request: FastifyRequest<OnlyId>) {
     try {
         const { id } = request.params;
@@ -65,7 +89,7 @@ export async function getDatabaseStatus(request: FastifyRequest<OnlyId>) {
             const { destinationDockerId, destinationDocker } = database;
             if (destinationDockerId) {
                 try {
-                    const { stdout } = await executeDockerCmd({ dockerId: destinationDocker.id, command: `docker inspect --format '{{json .State}}' ${id}` })
+                    const { stdout } = await executeCommand({ dockerId: destinationDocker.id, command: `docker inspect --format '{{json .State}}' ${id}` })
 
                     if (JSON.parse(stdout).Running) {
                         isRunning = true;
@@ -184,7 +208,7 @@ export async function saveDatabaseDestination(request: FastifyRequest<SaveDataba
         if (destinationDockerId) {
             if (type && version) {
                 const baseImage = getDatabaseImage(type, arch);
-                executeDockerCmd({ dockerId, command: `docker pull ${baseImage}:${version}` })
+                executeCommand({ dockerId, command: `docker pull ${baseImage}:${version}` })
             }
         }
         return reply.code(201).send({})
@@ -274,7 +298,7 @@ export async function startDatabase(request: FastifyRequest<OnlyId>) {
         };
         const composeFileDestination = `${workdir}/docker-compose.yaml`;
         await fs.writeFile(composeFileDestination, yaml.dump(composeFile));
-        await executeDockerCmd({ dockerId: destinationDocker.id, command: `docker compose -f ${composeFileDestination} up -d` })
+        await executeCommand({ dockerId: destinationDocker.id, command: `docker compose -f ${composeFileDestination} up -d` })
         if (isPublic) await startTraefikTCPProxy(destinationDocker, id, publicPort, privatePort);
         return {};
 
@@ -323,7 +347,7 @@ export async function getDatabaseLogs(request: FastifyRequest<GetDatabaseLogs>)
                 // const found = await checkContainer({ dockerId, container: id })
                 // if (found) {
                 const { default: ansi } = await import('strip-ansi')
-                const { stdout, stderr } = await executeDockerCmd({ dockerId, command: `docker logs --since ${since} --tail 5000 --timestamps ${id}` })
+                const { stdout, stderr } = await executeCommand({ dockerId, command: `docker logs --since ${since} --tail 5000 --timestamps ${id}` })
                 const stripLogsStdout = stdout.toString().split('\n').map((l) => ansi(l)).filter((a) => a);
                 const stripLogsStderr = stderr.toString().split('\n').map((l) => ansi(l)).filter((a) => a);
                 const logs = stripLogsStderr.concat(stripLogsStdout)

apps/api/src/routes/api/v1/databases/index.ts

@@ -1,5 +1,5 @@
 import { FastifyPluginAsync } from 'fastify';
-import { deleteDatabase, deleteDatabaseSecret, getDatabase, getDatabaseLogs, getDatabaseSecrets, getDatabaseStatus, getDatabaseTypes, getDatabaseUsage, getVersions, listDatabases, newDatabase, saveDatabase, saveDatabaseDestination, saveDatabaseSecret, saveDatabaseSettings, saveDatabaseType, saveVersion, startDatabase, stopDatabase } from './handlers';
+import { cleanupUnconfiguredDatabases, deleteDatabase, deleteDatabaseSecret, getDatabase, getDatabaseLogs, getDatabaseSecrets, getDatabaseStatus, getDatabaseTypes, getDatabaseUsage, getVersions, listDatabases, newDatabase, saveDatabase, saveDatabaseDestination, saveDatabaseSecret, saveDatabaseSettings, saveDatabaseType, saveVersion, startDatabase, stopDatabase } from './handlers';
 
 import type { OnlyId } from '../../../../types';
 
@@ -12,6 +12,8 @@ const root: FastifyPluginAsync = async (fastify): Promise<void> => {
     fastify.get('/', async (request) => await listDatabases(request));
     fastify.post('/new', async (request, reply) => await newDatabase(request, reply));
 
+    fastify.post<any>('/cleanup/unconfigured', async (request) => await cleanupUnconfiguredDatabases(request));
+
     fastify.get<OnlyId>('/:id', async (request) => await getDatabase(request));
     fastify.post<SaveDatabase>('/:id', async (request, reply) => await saveDatabase(request, reply));
     fastify.delete<DeleteDatabase>('/:id', async (request) => await deleteDatabase(request));

apps/api/src/routes/api/v1/destinations/handlers.ts

@@ -4,7 +4,7 @@ import sshConfig from 'ssh-config'
 import fs from 'fs/promises'
 import os from 'os';
 
-import { asyncExecShell, createRemoteEngineConfiguration, decrypt, errorHandler, executeDockerCmd, listSettings, prisma, startTraefikProxy, stopTraefikProxy } from '../../../../lib/common';
+import { createRemoteEngineConfiguration, decrypt, errorHandler, executeCommand, listSettings, prisma, startTraefikProxy, stopTraefikProxy } from '../../../../lib/common';
 import { checkContainer } from '../../../../lib/docker';
 
 import type { OnlyId } from '../../../../types';
@@ -79,9 +79,9 @@ export async function newDestination(request: FastifyRequest<NewDestination>, re
         let { name, network, engine, isCoolifyProxyUsed, remoteIpAddress, remoteUser, remotePort } = request.body
         if (id === 'new') {
             if (engine) {
-                const { stdout } = await asyncExecShell(`DOCKER_HOST=unix:///var/run/docker.sock docker network ls --filter 'name=^${network}$' --format '{{json .}}'`);
+                const { stdout } = await await executeCommand({ command: `docker network ls --filter 'name=^${network}$' --format '{{json .}}'` });
                 if (stdout === '') {
-                    await asyncExecShell(`DOCKER_HOST=unix:///var/run/docker.sock docker network create --attachable ${network}`);
+                    await await executeCommand({ command: `docker network create --attachable ${network}` });
                 }
                 await prisma.destinationDocker.create({
                     data: { name, teams: { connect: { id: teamId } }, engine, network, isCoolifyProxyUsed }
@@ -103,7 +103,7 @@ export async function newDestination(request: FastifyRequest<NewDestination>, re
                 return reply.code(201).send({ id: destination.id });
             } else {
                 const destination = await prisma.destinationDocker.create({
-                    data: { name, teams: { connect: { id: teamId } }, engine, network, isCoolifyProxyUsed, remoteEngine: true, remoteIpAddress, remoteUser, remotePort }
+                    data: { name, teams: { connect: { id: teamId } }, engine, network, isCoolifyProxyUsed, remoteEngine: true, remoteIpAddress, remoteUser, remotePort: Number(remotePort) }
                 });
                 return reply.code(201).send({ id: destination.id })
             }
@@ -122,13 +122,13 @@ export async function deleteDestination(request: FastifyRequest<OnlyId>) {
         const { network, remoteVerified, engine, isCoolifyProxyUsed } = await prisma.destinationDocker.findUnique({ where: { id } });
         if (isCoolifyProxyUsed) {
             if (engine || remoteVerified) {
-                const { stdout: found } = await executeDockerCmd({
+                const { stdout: found } = await executeCommand({
                     dockerId: id,
                     command: `docker ps -a --filter network=${network} --filter name=coolify-proxy --format '{{.}}'`
                 })
                 if (found) {
-                    await executeDockerCmd({ dockerId: id, command: `docker network disconnect ${network} coolify-proxy` })
-                    await executeDockerCmd({ dockerId: id, command: `docker network rm ${network}` })
+                    await executeCommand({ dockerId: id, command: `docker network disconnect ${network} coolify-proxy` })
+                    await executeCommand({ dockerId: id, command: `docker network rm ${network}` })
                 }
             }
         }
@@ -202,25 +202,65 @@ export async function assignSSHKey(request: FastifyRequest) {
         return errorHandler({ status, message })
     }
 }
-export async function verifyRemoteDockerEngine(request: FastifyRequest<OnlyId>, reply: FastifyReply) {
+export async function verifyRemoteDockerEngineFn(id: string) {
+    const { remoteIpAddress, network, isCoolifyProxyUsed } = await prisma.destinationDocker.findFirst({ where: { id } })
+    const daemonJson = `daemon-${id}.json`
     try {
-        const { id } = request.params;
-        await createRemoteEngineConfiguration(id);
-        const { remoteIpAddress, remoteUser, network, isCoolifyProxyUsed } = await prisma.destinationDocker.findFirst({ where: { id } })
-        const host = `ssh://${remoteUser}@${remoteIpAddress}`
-        const { stdout } = await asyncExecShell(`DOCKER_HOST=${host} docker network ls --filter 'name=${network}' --no-trunc --format "{{json .}}"`);
-        if (!stdout) {
-            await asyncExecShell(`DOCKER_HOST=${host} docker network create --attachable ${network}`);
-        }
-        const { stdout: coolifyNetwork } = await asyncExecShell(`DOCKER_HOST=${host} docker network ls --filter 'name=coolify-infra' --no-trunc --format "{{json .}}"`);
-        if (!coolifyNetwork) {
-            await asyncExecShell(`DOCKER_HOST=${host} docker network create --attachable coolify-infra`);
-        }
-        if (isCoolifyProxyUsed) await startTraefikProxy(id);
-        await prisma.destinationDocker.update({ where: { id }, data: { remoteVerified: true } })
-        return reply.code(201).send()
+        await executeCommand({ sshCommand: true, command: `docker network inspect ${network}`, dockerId: id });
+    } catch (error) {
+        await executeCommand({ command: `docker network create --attachable ${network}`, dockerId: id });
+    }
 
+    try {
+        await executeCommand({ sshCommand: true, command: `docker network inspect coolify-infra`, dockerId: id });
+    } catch (error) {
+        await executeCommand({ command: `docker network create --attachable coolify-infra`, dockerId: id });
+    }
+
+    if (isCoolifyProxyUsed) await startTraefikProxy(id);
+    let isUpdated = false;
+    let daemonJsonParsed = {
+        "live-restore": true,
+        "features": {
+            "buildkit": true
+        }
+    };
+    try {
+        const { stdout: daemonJson } = await executeCommand({ sshCommand: true, dockerId: id, command: `cat /etc/docker/daemon.json` });
+        daemonJsonParsed = JSON.parse(daemonJson);
+        if (!daemonJsonParsed['live-restore'] || daemonJsonParsed['live-restore'] !== true) {
+            isUpdated = true;
+            daemonJsonParsed['live-restore'] = true
+
+        }
+        if (!daemonJsonParsed?.features?.buildkit) {
+            isUpdated = true;
+            daemonJsonParsed.features = {
+                buildkit: true
+            }
+        }
+    } catch (error) {
+        isUpdated = true;
+    }
+    try {
+        if (isUpdated) {
+            await executeCommand({ shell: true, command: `echo '${JSON.stringify(daemonJsonParsed, null, 2)}' > /tmp/${daemonJson}` })
+            await executeCommand({ dockerId: id, command: `scp /tmp/${daemonJson} ${remoteIpAddress}-remote:/etc/docker/daemon.json` });
+            await executeCommand({ command: `rm /tmp/${daemonJson}` })
+            await executeCommand({ sshCommand: true, dockerId: id, command: `systemctl restart docker` });
+        }
+        await prisma.destinationDocker.update({ where: { id }, data: { remoteVerified: true } })
+    } catch (error) {
+        throw new Error('Error while verifying remote docker engine')
+    }
+}
+export async function verifyRemoteDockerEngine(request: FastifyRequest<OnlyId>, reply: FastifyReply) {
+    const { id } = request.params;
+    try {
+        await verifyRemoteDockerEngineFn(id);
+        return reply.code(201).send()
     } catch ({ status, message }) {
+        await prisma.destinationDocker.update({ where: { id }, data: { remoteVerified: false } })
         return errorHandler({ status, message })
     }
 }

apps/api/src/routes/api/v1/handlers.ts

@@ -1,9 +1,9 @@
-import axios from "axios";
 import { compareVersions } from "compare-versions";
 import cuid from "cuid";
 import bcrypt from "bcryptjs";
+import fs from 'fs/promises';
+import yaml from 'js-yaml';
 import {
-	asyncExecShell,
 	asyncSleep,
 	cleanupDockerStorage,
 	errorHandler,
@@ -12,8 +12,9 @@ import {
 	prisma,
 	uniqueName,
 	version,
+	sentryDSN,
+	executeCommand,
 } from "../../../lib/common";
-import { supportedServiceTypesAndVersions } from "../../../lib/services/supportedVersions";
 import { scheduler } from "../../../lib/scheduler";
 import type { FastifyReply, FastifyRequest } from "fastify";
 import type { Login, Update } from ".";
@@ -24,6 +25,35 @@ export async function hashPassword(password: string): Promise<string> {
 	return bcrypt.hash(password, saltRounds);
 }
 
+export async function backup(request: FastifyRequest) {
+	try {
+		const { backupData } = request.params;
+		let std = null;
+		const [id, backupType, type, zipped, storage] = backupData.split(':')
+		console.log(id, backupType, type, zipped, storage)
+		const database = await prisma.database.findUnique({ where: { id } })
+		if (database) {
+			// await executeDockerCmd({
+			// 	dockerId: database.destinationDockerId,
+			// 	command: `docker pull coollabsio/backup:latest`,
+			// })
+			std = await executeCommand({
+				dockerId: database.destinationDockerId,
+				command: `docker run --rm -v /var/run/docker.sock:/var/run/docker.sock -v coolify-local-backup:/app/backups -e CONTAINERS_TO_BACKUP="${backupData}" coollabsio/backup`
+			})
+
+		}
+		if (std.stdout) {
+			return std.stdout;
+		}
+		if (std.stderr) {
+			return std.stderr;
+		}
+		return 'nope';
+	} catch ({ status, message }) {
+		return errorHandler({ status, message });
+	}
+}
 export async function cleanupManually(request: FastifyRequest) {
 	try {
 		const { serverId } = request.body;
@@ -36,16 +66,59 @@ export async function cleanupManually(request: FastifyRequest) {
 		return errorHandler({ status, message });
 	}
 }
+export async function refreshTags() {
+	try {
+		const { default: got } = await import('got')
+		try {
+			if (isDev) {
+				const tags = await fs.readFile('./devTags.json', 'utf8')
+				await fs.writeFile('./tags.json', tags)
+			} else {
+				const tags = await got.get('https://get.coollabs.io/coolify/service-tags.json').text()
+				await fs.writeFile('/app/tags.json', tags)
+			}
+		} catch (error) {
+			console.log(error)
+		}
+
+		return {};
+	} catch ({ status, message }) {
+		return errorHandler({ status, message });
+	}
+}
+export async function refreshTemplates() {
+	try {
+		const { default: got } = await import('got')
+		try {
+			if (isDev) {
+				const response = await fs.readFile('./devTemplates.yaml', 'utf8')
+				await fs.writeFile('./templates.json', JSON.stringify(yaml.load(response)))
+			} else {
+				const response = await got.get('https://get.coollabs.io/coolify/service-templates.yaml').text()
+				await fs.writeFile('/app/templates.json', JSON.stringify(yaml.load(response)))
+			}
+		} catch (error) {
+			console.log(error)
+		}
+		return {};
+	} catch ({ status, message }) {
+		return errorHandler({ status, message });
+	}
+}
 export async function checkUpdate(request: FastifyRequest) {
 	try {
+		const { default: got } = await import('got')
 		const isStaging =
 			request.hostname === "staging.coolify.io" ||
 			request.hostname === "arm.coolify.io";
 		const currentVersion = version;
-		const { data: versions } = await axios.get(
-			`https://get.coollabs.io/versions.json?appId=${process.env["COOLIFY_APP_ID"]}&version=${currentVersion}`
-		);
-		const latestVersion = versions["coolify"].main.version;
+		const { coolify } = await got.get('https://get.coollabs.io/versions.json', {
+			searchParams: {
+				appId: process.env['COOLIFY_APP_ID'] || undefined,
+				version: currentVersion
+			}
+		}).json()
+		const latestVersion = coolify.main.version;
 		const isUpdateAvailable = compareVersions(latestVersion, currentVersion);
 		if (isStaging) {
 			return {
@@ -67,14 +140,10 @@ export async function update(request: FastifyRequest<Update>) {
 	try {
 		if (!isDev) {
 			const { isAutoUpdateEnabled } = await prisma.setting.findFirst();
-			await asyncExecShell(`docker pull coollabsio/coolify:${latestVersion}`);
-			await asyncExecShell(`env | grep COOLIFY > .env`);
-			await asyncExecShell(
-				`sed -i '/COOLIFY_AUTO_UPDATE=/cCOOLIFY_AUTO_UPDATE=${isAutoUpdateEnabled}' .env`
-			);
-			await asyncExecShell(
-				`docker run --rm -tid --env-file .env -v /var/run/docker.sock:/var/run/docker.sock -v coolify-db coollabsio/coolify:${latestVersion} /bin/sh -c "env | grep COOLIFY > .env && echo 'TAG=${latestVersion}' >> .env && docker stop -t 0 coolify && docker rm coolify && docker compose up -d --force-recreate"`
-			);
+			await executeCommand({ command: `docker pull coollabsio/coolify:${latestVersion}` });
+			await executeCommand({ shell: true, command: `env | grep COOLIFY > .env` });
+			await executeCommand({ command: `sed -i '/COOLIFY_AUTO_UPDATE=/cCOOLIFY_AUTO_UPDATE=${isAutoUpdateEnabled}' .env` });
+			await executeCommand({ shell: true, command: `docker run --rm -tid --env-file .env -v /var/run/docker.sock:/var/run/docker.sock -v coolify-db coollabsio/coolify:${latestVersion} /bin/sh -c "env | grep COOLIFY > .env && echo 'TAG=${latestVersion}' >> .env && docker stop -t 0 coolify coolify-fluentbit && docker rm coolify coolify-fluentbit && docker compose pull && docker compose up -d --force-recreate"` });
 			return {};
 		} else {
 			await asyncSleep(2000);
@@ -103,7 +172,7 @@ export async function restartCoolify(request: FastifyRequest<any>) {
 		const teamId = request.user.teamId;
 		if (teamId === "0") {
 			if (!isDev) {
-				asyncExecShell(`docker restart coolify`);
+				await executeCommand({ command: `docker restart coolify` });
 				return {};
 			} else {
 				return {};
@@ -122,7 +191,7 @@ export async function showDashboard(request: FastifyRequest) {
 	try {
 		const userId = request.user.userId;
 		const teamId = request.user.teamId;
-		const applications = await prisma.application.findMany({
+		let applications = await prisma.application.findMany({
 			where: { teams: { some: { id: teamId === "0" ? undefined : teamId } } },
 			include: { settings: true, destinationDocker: true, teams: true },
 		});
@@ -135,7 +204,7 @@ export async function showDashboard(request: FastifyRequest) {
 			include: { destinationDocker: true, teams: true },
 		});
 		const gitSources = await prisma.gitSource.findMany({
-			where: { teams: { some: { id: teamId === "0" ? undefined : teamId } } },
+			where: { OR: [{ teams: { some: { id: teamId === "0" ? undefined : teamId } } }, { isSystemWide: true }] },
 			include: { teams: true },
 		});
 		const destinations = await prisma.destinationDocker.findMany({
@@ -143,7 +212,29 @@ export async function showDashboard(request: FastifyRequest) {
 			include: { teams: true },
 		});
 		const settings = await listSettings();
+
+		let foundUnconfiguredApplication = false;
+		for (const application of applications) {
+			if (((!application.buildPack || !application.branch) && !application.simpleDockerfile) || !application.destinationDockerId || (!application.settings?.isBot && !application?.fqdn) && application.buildPack !== "compose") {
+				foundUnconfiguredApplication = true
+			}
+		}
+		let foundUnconfiguredService = false;
+		for (const service of services) {
+			if (!service.fqdn) {
+				foundUnconfiguredService = true
+			}
+		}
+		let foundUnconfiguredDatabase = false;
+		for (const database of databases) {
+			if (!database.version) {
+				foundUnconfiguredDatabase = true
+			}
+		}
 		return {
+			foundUnconfiguredApplication,
+			foundUnconfiguredDatabase,
+			foundUnconfiguredService,
 			applications,
 			databases,
 			services,
@@ -331,9 +422,11 @@ export async function getCurrentUser(
 			// No new token -> not switching teams
 		}
 	}
+	const pendingInvitations = await prisma.teamInvitation.findMany({ where: { uid: request.user.userId } })
 	return {
-		settings: await prisma.setting.findFirst(),
-		supportedServiceTypesAndVersions,
+		settings: await prisma.setting.findUnique({ where: { id: "0" } }),
+		sentryDSN,
+		pendingInvitations,
 		token,
 		...request.user,
 	};

apps/api/src/routes/api/v1/iam/handlers.ts

@@ -5,9 +5,10 @@ import { decrypt, errorHandler, prisma, uniqueName } from '../../../../lib/commo
 import { day } from '../../../../lib/dayjs';
 
 import type { OnlyId } from '../../../../types';
-import type { BodyId, InviteToTeam, SaveTeam, SetPermission } from './types';
+import type { BodyId, DeleteUserFromTeam, InviteToTeam, SaveTeam, SetPermission } from './types';
 
-export async function listTeams(request: FastifyRequest) {
+
+export async function listAccounts(request: FastifyRequest) {
     try {
         const userId = request.user.userId;
         const teamId = request.user.teamId;
@@ -15,10 +16,24 @@ export async function listTeams(request: FastifyRequest) {
             where: { id: userId },
             select: { id: true, email: true, teams: true }
         });
-        let accounts = [];
-        let allTeams = [];
+        let accounts = await prisma.user.findMany({ where: { teams: { some: { id: teamId } } }, select: { id: true, email: true, teams: true } });
         if (teamId === '0') {
             accounts = await prisma.user.findMany({ select: { id: true, email: true, teams: true } });
+        }
+        return {
+            account,
+            accounts
+        };
+    } catch ({ status, message }) {
+        return errorHandler({ status, message })
+    }
+}
+export async function listTeams(request: FastifyRequest) {
+    try {
+        const userId = request.user.userId;
+        const teamId = request.user.teamId;
+        let allTeams = [];
+        if (teamId === '0') {
             allTeams = await prisma.team.findMany({
                 where: { users: { none: { id: userId } } },
                 include: { permissions: true }
@@ -28,18 +43,30 @@ export async function listTeams(request: FastifyRequest) {
             where: { users: { some: { id: userId } } },
             include: { permissions: true }
         });
-        const invitations = await prisma.teamInvitation.findMany({ where: { uid: userId } });
         return {
             ownTeams,
             allTeams,
-            invitations,
-            account,
-            accounts
         };
     } catch ({ status, message }) {
         return errorHandler({ status, message })
     }
 }
+export async function removeUserFromTeam(request: FastifyRequest<DeleteUserFromTeam>, reply: FastifyReply) {
+    try {
+        const { uid } = request.body;
+        const { id } = request.params;
+        const userId = request.user.userId;
+        const foundUser = await prisma.team.findMany({ where: { id, users: { some: { id: userId } } } });
+        if (foundUser.length === 0) {
+            return errorHandler({ status: 404, message: 'Team not found' });
+        }
+        await prisma.team.update({ where: { id }, data: { users: { disconnect: { id: uid } } } });
+        await prisma.permission.deleteMany({ where: { teamId: id, userId: uid } })
+        return reply.code(201).send()
+    } catch ({ status, message }) {
+        return errorHandler({ status, message })
+    }
+}
 export async function deleteTeam(request: FastifyRequest<OnlyId>, reply: FastifyReply) {
     try {
         const userId = request.user.userId;

apps/api/src/routes/api/v1/iam/index.ts

@@ -1,19 +1,22 @@
 import { FastifyPluginAsync } from 'fastify';
-import { acceptInvitation, changePassword, deleteTeam, getTeam, inviteToTeam, listTeams, newTeam, removeUser, revokeInvitation, saveTeam, setPermission } from './handlers';
+import { acceptInvitation, changePassword, deleteTeam, getTeam, inviteToTeam, listAccounts, listTeams, newTeam, removeUser, removeUserFromTeam, revokeInvitation, saveTeam, setPermission } from './handlers';
 
 import type { OnlyId } from '../../../../types';
-import type { BodyId, InviteToTeam, SaveTeam, SetPermission } from './types';
+import type { BodyId, DeleteUserFromTeam, InviteToTeam, SaveTeam, SetPermission } from './types';
 
 const root: FastifyPluginAsync = async (fastify): Promise<void> => {
     fastify.addHook('onRequest', async (request) => {
         return await request.jwtVerify()
     })
-    fastify.get('/', async (request) => await listTeams(request));
+
+    fastify.get('/', async (request) => await listAccounts(request));
     fastify.post('/new', async (request, reply) => await newTeam(request, reply));
+    fastify.get('/teams', async (request) => await listTeams(request));
 
     fastify.get<OnlyId>('/team/:id', async (request, reply) => await getTeam(request, reply));
     fastify.post<SaveTeam>('/team/:id', async (request, reply) => await saveTeam(request, reply));
     fastify.delete<OnlyId>('/team/:id', async (request, reply) => await deleteTeam(request, reply));
+    fastify.post<DeleteUserFromTeam>('/team/:id/user/remove', async (request, reply) => await removeUserFromTeam(request, reply));
 
     fastify.post<InviteToTeam>('/team/:id/invitation/invite', async (request, reply) => await inviteToTeam(request, reply))
     fastify.post<BodyId>('/team/:id/invitation/accept', async (request) => await acceptInvitation(request));
@@ -23,7 +26,6 @@ const root: FastifyPluginAsync = async (fastify): Promise<void> => {
 
     fastify.delete<BodyId>('/user/remove', async (request, reply) => await removeUser(request, reply));
     fastify.post<BodyId>('/user/password', async (request, reply) => await changePassword(request, reply));
-    // fastify.delete('/user', async (request, reply) => await deleteUser(request, reply));
 
 };
 

apps/api/src/routes/api/v1/iam/types.ts

@@ -5,6 +5,14 @@ export interface SaveTeam extends OnlyId {
         name: string
     }
 }
+export interface DeleteUserFromTeam {
+    Body: {
+        uid: string
+    },
+    Params: {
+        id: string
+    }
+}
 export interface InviteToTeam {
     Body: {
         email: string,

apps/api/src/routes/api/v1/index.ts

@@ -1,5 +1,5 @@
 import { FastifyPluginAsync } from 'fastify';
-import { checkUpdate, login, showDashboard, update, resetQueue, getCurrentUser, cleanupManually, restartCoolify } from './handlers';
+import { checkUpdate, login, showDashboard, update, resetQueue, getCurrentUser, cleanupManually, restartCoolify, backup } from './handlers';
 import { GetCurrentUser } from './types';
 
 export interface Update {
@@ -23,9 +23,7 @@ const root: FastifyPluginAsync = async (fastify): Promise<void> => {
 		onRequest: [fastify.authenticate]
 	}, async (request) => await getCurrentUser(request, fastify));
 
-	fastify.get('/undead', {
-		onRequest: [fastify.authenticate]
-	}, async function () {
+	fastify.get('/undead', async function () {
 		return { message: 'nope' };
 	});
 
@@ -47,13 +45,17 @@ const root: FastifyPluginAsync = async (fastify): Promise<void> => {
 		onRequest: [fastify.authenticate]
 	}, async (request) => await restartCoolify(request));
 
-    fastify.post('/internal/resetQueue', {
+	fastify.post('/internal/resetQueue', {
 		onRequest: [fastify.authenticate]
 	}, async (request) => await resetQueue(request));
 
 	fastify.post('/internal/cleanup', {
 		onRequest: [fastify.authenticate]
 	}, async (request) => await cleanupManually(request));
+
+	// fastify.get('/internal/backup/:backupData', {
+	// 	onRequest: [fastify.authenticate]
+	// }, async (request) => await backup(request));
 };
 
 export default root;

apps/api/src/routes/api/v1/servers/handlers.ts

@@ -1,5 +1,5 @@
 import type { FastifyRequest } from 'fastify';
-import { errorHandler, executeDockerCmd, prisma, createRemoteEngineConfiguration, executeSSHCmd } from '../../../../lib/common';
+import { errorHandler, prisma, executeCommand } from '../../../../lib/common';
 import os from 'node:os';
 import osu from 'node-os-utils';
 
@@ -8,9 +8,16 @@ export async function listServers(request: FastifyRequest) {
     try {
         const userId = request.user.userId;
         const teamId = request.user.teamId;
-        const servers = await prisma.destinationDocker.findMany({ where: { teams: { some: { id: teamId === '0' ? undefined : teamId } }, remoteEngine: false }, distinct: ['engine'] })
-        // const remoteServers = await prisma.destinationDocker.findMany({ where: { teams: { some: { id: teamId === '0' ? undefined : teamId } } }, distinct: ['remoteIpAddress', 'engine'] })
-
+        let servers = await prisma.destinationDocker.findMany({ where: { teams: { some: { id: teamId === '0' ? undefined : teamId } } }, distinct: ['remoteIpAddress', 'engine'] })
+        servers = servers.filter((server) => {
+            if (server.remoteEngine) {
+                if (server.remoteVerified) {
+                    return server
+                }
+            } else {
+                return server
+            }
+        })
         return {
             servers
         }
@@ -64,11 +71,10 @@ export async function showUsage(request: FastifyRequest) {
     let { remoteEngine } = request.query
     remoteEngine = remoteEngine === 'true' ? true : false
     if (remoteEngine) {
-        const { stdout: stats } = await executeSSHCmd({ dockerId: id, command: `vmstat -s` })
-        const { stdout: disks } = await executeSSHCmd({ dockerId: id, command: `df -m / --output=size,used,pcent|grep -v 'Used'| xargs` })
-        const { stdout: cpus } = await executeSSHCmd({ dockerId: id, command: `nproc --all` })
-        // const { stdout: cpuUsage } = await executeSSHCmd({ dockerId: id, command: `echo $[100-$(vmstat 1 2|tail -1|awk '{print $15}')]` })
-        // console.log(cpuUsage)
+        const { stdout: stats } = await executeCommand({ sshCommand: true, dockerId: id, command: `vmstat -s` })
+        const { stdout: disks } = await executeCommand({ sshCommand: true, shell: true, dockerId: id, command: `df -m / --output=size,used,pcent|grep -v 'Used'| xargs` })
+        const { stdout: cpus } = await executeCommand({ sshCommand: true, dockerId: id, command: `nproc --all` })
+        const { stdout: cpuUsage } = await executeCommand({ sshCommand: true, shell: true, dockerId: id, command: `echo $[100-$(vmstat 1 2|tail -1|awk '{print $15}')]` })
         const parsed: any = parseFromText(stats)
         return {
             usage: {
@@ -81,8 +87,8 @@ export async function showUsage(request: FastifyRequest) {
                     freeMemPercentage: (parsed.totalMemoryKB - parsed.usedMemoryKB) / parsed.totalMemoryKB * 100
                 },
                 cpu: {
-                    load: 0,
-                    usage: 0,
+                    load: [0, 0, 0],
+                    usage: cpuUsage,
                     count: cpus
                 },
                 disk: {

apps/api/src/routes/api/v1/services/handlers.ts

@@ -1,15 +1,17 @@
 import type { FastifyReply, FastifyRequest } from 'fastify';
 import fs from 'fs/promises';
 import yaml from 'js-yaml';
-import { prisma, uniqueName, asyncExecShell, getServiceFromDB, getContainerUsage, isDomainConfigured, saveUpdateableFields, fixType, decrypt, encrypt, ComposeFile, getFreePublicPort, getDomain, errorHandler, generatePassword, isDev, stopTcpHttpProxy, executeDockerCmd, checkDomainsIsValidInDNS, checkExposedPort, listSettings } from '../../../../lib/common';
-import { day } from '../../../../lib/dayjs';
-import { checkContainer, isContainerExited } from '../../../../lib/docker';
+import bcrypt from 'bcryptjs';
 import cuid from 'cuid';
 
-import type { OnlyId } from '../../../../types';
+import { prisma, uniqueName, getServiceFromDB, getContainerUsage, isDomainConfigured, fixType, decrypt, encrypt, ComposeFile, getFreePublicPort, getDomain, errorHandler, generatePassword, isDev, stopTcpHttpProxy, checkDomainsIsValidInDNS, checkExposedPort, listSettings, generateToken, executeCommand } from '../../../../lib/common';
+import { day } from '../../../../lib/dayjs';
+import { checkContainer, } from '../../../../lib/docker';
+import { removeService } from '../../../../lib/services/common';
+import { getTags, getTemplates } from '../../../../lib/services';
+
 import type { ActivateWordpressFtp, CheckService, CheckServiceDomain, DeleteServiceSecret, DeleteServiceStorage, GetServiceLogs, SaveService, SaveServiceDestination, SaveServiceSecret, SaveServiceSettings, SaveServiceStorage, SaveServiceType, SaveServiceVersion, ServiceStartStop, SetGlitchTipSettings, SetWordpressSettings } from './types';
-import { supportedServiceTypesAndVersions } from '../../../../lib/services/supportedVersions';
-import { configureServiceType, removeService } from '../../../../lib/services/common';
+import type { OnlyId } from '../../../../types';
 
 export async function listServices(request: FastifyRequest) {
     try {
@@ -36,34 +38,246 @@ export async function newService(request: FastifyRequest, reply: FastifyReply) {
         return errorHandler({ status, message })
     }
 }
+export async function cleanupUnconfiguredServices(request: FastifyRequest) {
+    try {
+        const teamId = request.user.teamId;
+        let services = await prisma.service.findMany({
+            where: { teams: { some: { id: teamId === "0" ? undefined : teamId } } },
+            include: { destinationDocker: true, teams: true },
+        });
+        for (const service of services) {
+            if (!service.fqdn) {
+                if (service.destinationDockerId) {
+                    const { stdout: containers } = await executeCommand({
+                        dockerId: service.destinationDockerId,
+                        command: `docker ps -a --filter 'label=com.docker.compose.project=${service.id}' --format {{.ID}}`
+                    })
+                    if (containers) {
+                        const containerArray = containers.split('\n');
+                        if (containerArray.length > 0) {
+                            for (const container of containerArray) {
+                                await executeCommand({ dockerId: service.destinationDockerId, command: `docker stop -t 0 ${container}` })
+                                await executeCommand({ dockerId: service.destinationDockerId, command: `docker rm --force ${container}` })
+                            }
+                        }
+                    }
+                }
+                await removeService({ id: service.id });
+            }
+        }
+        return {}
+    } catch ({ status, message }) {
+        return errorHandler({ status, message })
+    }
+}
 export async function getServiceStatus(request: FastifyRequest<OnlyId>) {
     try {
         const teamId = request.user.teamId;
         const { id } = request.params;
-
-        let isRunning = false;
-        let isExited = false
-        let isRestarting = false;
         const service = await getServiceFromDB({ id, teamId });
         const { destinationDockerId, settings } = service;
-
+        let payload = {}
         if (destinationDockerId) {
-            const status = await checkContainer({ dockerId: service.destinationDocker.id, container: id });
-            if (status?.found) {
-                isRunning = status.status.isRunning;
-                isExited = status.status.isExited;
-                isRestarting = status.status.isRestarting
+            const { stdout: containers } = await executeCommand({
+                dockerId: service.destinationDocker.id,
+                command:
+                    `docker ps -a --filter "label=com.docker.compose.project=${id}" --format '{{json .}}'`
+            });
+            if (containers) {
+                const containersArray = containers.trim().split('\n');
+                if (containersArray.length > 0 && containersArray[0] !== '') {
+                    const templates = await getTemplates();
+                    let template = templates.find(t => t.type === service.type);
+                    const templateStr = JSON.stringify(template)
+                    if (templateStr) {
+                        template = JSON.parse(templateStr.replaceAll('$$id', service.id));
+                    }
+                    for (const container of containersArray) {
+                        let isRunning = false;
+                        let isExited = false;
+                        let isRestarting = false;
+                        let isExcluded = false;
+                        const containerObj = JSON.parse(container);
+                        const exclude = template?.services[containerObj.Names]?.exclude;
+                        if (exclude) {
+                            payload[containerObj.Names] = {
+                                status: {
+                                    isExcluded: true,
+                                    isRunning: false,
+                                    isExited: false,
+                                    isRestarting: false,
+                                }
+                            }
+                            continue;
+                        }
+
+                        const status = containerObj.State
+                        if (status === 'running') {
+                            isRunning = true;
+                        }
+                        if (status === 'exited') {
+                            isExited = true;
+                        }
+                        if (status === 'restarting') {
+                            isRestarting = true;
+                        }
+                        payload[containerObj.Names] = {
+                            status: {
+                                isExcluded,
+                                isRunning,
+                                isExited,
+                                isRestarting
+                            }
+                        }
+                    }
+                }
             }
+
         }
-        return {
-            isRunning,
-            isExited,
-            settings
-        }
+        return payload
     } catch ({ status, message }) {
         return errorHandler({ status, message })
     }
 }
+export async function parseAndFindServiceTemplates(service: any, workdir?: string, isDeploy: boolean = false) {
+    const templates = await getTemplates()
+    const foundTemplate = templates.find(t => fixType(t.type) === service.type)
+    let parsedTemplate = {}
+    if (foundTemplate) {
+        if (!isDeploy) {
+            for (const [key, value] of Object.entries(foundTemplate.services)) {
+                const realKey = key.replace('$$id', service.id)
+                let name = value.name
+                if (!name) {
+                    if (Object.keys(foundTemplate.services).length === 1) {
+                        name = foundTemplate.name || service.name.toLowerCase()
+                    } else {
+                        if (key === '$$id') {
+                            name = foundTemplate.name || key.replaceAll('$$id-', '') || service.name.toLowerCase()
+                        } else {
+                            name = key.replaceAll('$$id-', '') || service.name.toLowerCase()
+                        }
+                    }
+                }
+                parsedTemplate[realKey] = {
+                    value,
+                    name,
+                    documentation: value.documentation || foundTemplate.documentation || 'https://docs.coollabs.io',
+                    image: value.image,
+                    files: value?.files,
+                    environment: [],
+                    fqdns: [],
+                    hostPorts: [],
+                    proxy: {}
+                }
+                if (value.environment?.length > 0) {
+                    for (const env of value.environment) {
+                        let [envKey, ...envValue] = env.split('=')
+                        envValue = envValue.join("=")
+                        let variable = null
+                        if (foundTemplate?.variables) {
+                            variable = foundTemplate?.variables.find(v => v.name === envKey) || foundTemplate?.variables.find(v => v.id === envValue)
+                        }
+                        if (variable) {
+                            const id = variable.id.replaceAll('$$', '')
+                            const label = variable?.label
+                            const description = variable?.description
+                            const defaultValue = variable?.defaultValue
+                            const main = variable?.main || '$$id'
+                            const type = variable?.type || 'input'
+                            const placeholder = variable?.placeholder || ''
+                            const readOnly = variable?.readOnly || false
+                            const required = variable?.required || false
+                            if (envValue.startsWith('$$config') || variable?.showOnConfiguration) {
+                                if (envValue.startsWith('$$config_coolify')) {
+                                    continue
+                                }
+                                parsedTemplate[realKey].environment.push(
+                                    { id, name: envKey, value: envValue, main, label, description, defaultValue, type, placeholder, required, readOnly }
+                                )
+                            }
+                        }
+
+                    }
+                }
+                if (value?.proxy && value.proxy.length > 0) {
+                    for (const proxyValue of value.proxy) {
+                        if (proxyValue.domain) {
+                            const variable = foundTemplate?.variables.find(v => v.id === proxyValue.domain)
+                            if (variable) {
+                                const { id, name, label, description, defaultValue, required = false } = variable
+                                const found = await prisma.serviceSetting.findFirst({ where: { serviceId: service.id, variableName: proxyValue.domain } })
+                                parsedTemplate[realKey].fqdns.push(
+                                    { id, name, value: found?.value || '', label, description, defaultValue, required }
+                                )
+                            }
+                        }
+                        if (proxyValue.hostPort) {
+                            const variable = foundTemplate?.variables.find(v => v.id === proxyValue.hostPort)
+                            if (variable) {
+                                const { id, name, label, description, defaultValue, required = false } = variable
+                                const found = await prisma.serviceSetting.findFirst({ where: { serviceId: service.id, variableName: proxyValue.hostPort } })
+                                parsedTemplate[realKey].hostPorts.push(
+                                    { id, name, value: found?.value || '', label, description, defaultValue, required }
+                                )
+                            }
+                        }
+                    }
+                }
+            }
+        } else {
+            parsedTemplate = foundTemplate
+        }
+        let strParsedTemplate = JSON.stringify(parsedTemplate)
+
+        // replace $$id and $$workdir
+        strParsedTemplate = strParsedTemplate.replaceAll('$$id', service.id)
+        strParsedTemplate = strParsedTemplate.replaceAll('$$core_version', service.version || foundTemplate.defaultVersion)
+
+        // replace $$workdir
+        if (workdir) {
+            strParsedTemplate = strParsedTemplate.replaceAll('$$workdir', workdir)
+        }
+
+        // replace $$config
+        if (service.serviceSetting.length > 0) {
+            for (const setting of service.serviceSetting) {
+                const { value, variableName } = setting
+                const regex = new RegExp(`\\$\\$config_${variableName.replace('$$config_', '')}\"`, 'gi')
+                if (value === '$$generate_fqdn') {
+                    strParsedTemplate = strParsedTemplate.replaceAll(regex, service.fqdn + '"' || '' + '"')
+                } else if (value === '$$generate_fqdn_slash') {
+                    strParsedTemplate = strParsedTemplate.replaceAll(regex, service.fqdn + '/' + '"')
+                } else if (value === '$$generate_domain') {
+                    strParsedTemplate = strParsedTemplate.replaceAll(regex, getDomain(service.fqdn) + '"')
+                } else if (service.destinationDocker?.network && value === '$$generate_network') {
+                    strParsedTemplate = strParsedTemplate.replaceAll(regex, service.destinationDocker.network + '"')
+                } else {
+                    strParsedTemplate = strParsedTemplate.replaceAll(regex, value + '"')
+                }
+            }
+        }
+
+        // replace $$secret
+        if (service.serviceSecret.length > 0) {
+            for (const secret of service.serviceSecret) {
+                let { name, value } = secret
+                name = name.toLowerCase()
+                const regexHashed = new RegExp(`\\$\\$hashed\\$\\$secret_${name}\"`, 'gi')
+                const regex = new RegExp(`\\$\\$secret_${name}\"`, 'gi')
+                if (value) {
+                    strParsedTemplate = strParsedTemplate.replaceAll(regexHashed, bcrypt.hashSync(value.replaceAll("\"", "\\\""), 10) + '"')
+                    strParsedTemplate = strParsedTemplate.replaceAll(regex, value.replaceAll("\"", "\\\"") + '"')
+                } else {
+                    strParsedTemplate = strParsedTemplate.replaceAll(regexHashed, '' + '"')
+                    strParsedTemplate = strParsedTemplate.replaceAll(regex, '' + '"')
+                }
+            }
+        }
+        parsedTemplate = JSON.parse(strParsedTemplate)
+    }
+    return parsedTemplate
+}
 
 export async function getService(request: FastifyRequest<OnlyId>) {
     try {
@@ -73,9 +287,17 @@ export async function getService(request: FastifyRequest<OnlyId>) {
         if (!service) {
             throw { status: 404, message: 'Service not found.' }
         }
+        let template = {}
+        let tags = []
+        if (service.type) {
+            template = await parseAndFindServiceTemplates(service)
+            tags = await getTags(service.type)
+        }
         return {
             settings: await listSettings(),
-            service
+            service,
+            template,
+            tags
         }
     } catch ({ status, message }) {
         return errorHandler({ status, message })
@@ -84,7 +306,7 @@ export async function getService(request: FastifyRequest<OnlyId>) {
 export async function getServiceType(request: FastifyRequest) {
     try {
         return {
-            types: supportedServiceTypesAndVersions
+            services: await getTemplates()
         }
     } catch ({ status, message }) {
         return errorHandler({ status, message })
@@ -94,25 +316,83 @@ export async function saveServiceType(request: FastifyRequest<SaveServiceType>,
     try {
         const { id } = request.params;
         const { type } = request.body;
-        await configureServiceType({ id, type });
-        return reply.code(201).send()
-    } catch ({ status, message }) {
-        return errorHandler({ status, message })
-    }
-}
-export async function getServiceVersions(request: FastifyRequest<OnlyId>) {
-    try {
-        const teamId = request.user.teamId;
-        const { id } = request.params;
-        const { type } = await getServiceFromDB({ id, teamId });
-        return {
-            type,
-            versions: supportedServiceTypesAndVersions.find((name) => name.name === type).versions
+        const templates = await getTemplates()
+        let foundTemplate = templates.find(t => fixType(t.type) === fixType(type))
+        if (foundTemplate) {
+            foundTemplate = JSON.parse(JSON.stringify(foundTemplate).replaceAll('$$id', id))
+            if (foundTemplate.variables) {
+                if (foundTemplate.variables.length > 0) {
+                    for (const variable of foundTemplate.variables) {
+                        const { defaultValue } = variable;
+                        const regex = /^\$\$.*\((\d+)\)$/g;
+                        const length = Number(regex.exec(defaultValue)?.[1]) || undefined
+                        if (variable.defaultValue.startsWith('$$generate_password')) {
+                            variable.value = generatePassword({ length });
+                        } else if (variable.defaultValue.startsWith('$$generate_hex')) {
+                            variable.value = generatePassword({ length, isHex: true });
+                        } else if (variable.defaultValue.startsWith('$$generate_username')) {
+                            variable.value = cuid();
+                        } else if (variable.defaultValue.startsWith('$$generate_token')) {
+                            variable.value = generateToken()
+                        } else {
+                            variable.value = variable.defaultValue || '';
+                        }
+                        const foundVariableSomewhereElse = foundTemplate.variables.find(v => v.defaultValue.includes(variable.id))
+                        if (foundVariableSomewhereElse) {
+                            foundVariableSomewhereElse.value = foundVariableSomewhereElse.value.replaceAll(variable.id, variable.value)
+                        }
+                    }
+                }
+                for (const variable of foundTemplate.variables) {
+                    if (variable.id.startsWith('$$secret_')) {
+                        const found = await prisma.serviceSecret.findFirst({ where: { name: variable.name, serviceId: id } })
+                        if (!found) {
+                            await prisma.serviceSecret.create({
+                                data: { name: variable.name, value: encrypt(variable.value) || '', service: { connect: { id } } }
+                            })
+                        }
+
+                    }
+                    if (variable.id.startsWith('$$config_')) {
+                        const found = await prisma.serviceSetting.findFirst({ where: { name: variable.name, serviceId: id } })
+                        if (!found) {
+                            await prisma.serviceSetting.create({
+                                data: { name: variable.name, value: variable.value.toString(), variableName: variable.id, service: { connect: { id } } }
+                            })
+                        }
+                    }
+                }
+            }
+            for (const service of Object.keys(foundTemplate.services)) {
+                if (foundTemplate.services[service].volumes) {
+                    for (const volume of foundTemplate.services[service].volumes) {
+                        const [volumeName, path] = volume.split(':')
+                        if (!volumeName.startsWith('/')) {
+                            const found = await prisma.servicePersistentStorage.findFirst({ where: { volumeName, serviceId: id } })
+                            if (!found) {
+                                await prisma.servicePersistentStorage.create({
+                                    data: { volumeName, path, containerId: service, predefined: true, service: { connect: { id } } }
+                                });
+                            }
+                        }
+                    }
+                }
+            }
+            await prisma.service.update({ where: { id }, data: { type, version: foundTemplate.defaultVersion, templateVersion: foundTemplate.templateVersion } })
+
+            if (type.startsWith('wordpress')) {
+                await prisma.service.update({ where: { id }, data: { wordpress: { create: {} } } })
+            }
+            return reply.code(201).send()
+        } else {
+            throw { status: 404, message: 'Service type not found.' }
         }
+
     } catch ({ status, message }) {
         return errorHandler({ status, message })
     }
 }
+
 export async function saveServiceVersion(request: FastifyRequest<SaveServiceVersion>, reply: FastifyReply) {
     try {
         const { id } = request.params;
@@ -159,7 +439,7 @@ export async function getServiceUsage(request: FastifyRequest<OnlyId>) {
 }
 export async function getServiceLogs(request: FastifyRequest<GetServiceLogs>) {
     try {
-        const { id } = request.params;
+        const { id, containerId } = request.params;
         let { since = 0 } = request.query
         if (since !== 0) {
             since = day(since).unix();
@@ -170,10 +450,8 @@ export async function getServiceLogs(request: FastifyRequest<GetServiceLogs>) {
         });
         if (destinationDockerId) {
             try {
-                // const found = await checkContainer({ dockerId, container: id })
-                // if (found) {
                 const { default: ansi } = await import('strip-ansi')
-                const { stdout, stderr } = await executeDockerCmd({ dockerId, command: `docker logs --since ${since} --tail 5000 --timestamps ${id}` })
+                const { stdout, stderr } = await executeCommand({ dockerId, command: `docker logs --since ${since} --tail 5000 --timestamps ${containerId}` })
                 const stripLogsStdout = stdout.toString().split('\n').map((l) => ansi(l)).filter((a) => a);
                 const stripLogsStderr = stderr.toString().split('\n').map((l) => ansi(l)).filter((a) => a);
                 const logs = stripLogsStderr.concat(stripLogsStdout)
@@ -181,7 +459,10 @@ export async function getServiceLogs(request: FastifyRequest<GetServiceLogs>) {
                 return { logs: sortedLogs }
                 // }
             } catch (error) {
-                const { statusCode } = error;
+                const { statusCode, stderr } = error;
+                if (stderr.startsWith('Error: No such container')) {
+                    return { logs: [], noContainer: true }
+                }
                 if (statusCode === 404) {
                     return {
                         logs: []
@@ -231,26 +512,22 @@ export async function checkServiceDomain(request: FastifyRequest<CheckServiceDom
 export async function checkService(request: FastifyRequest<CheckService>) {
     try {
         const { id } = request.params;
-        let { fqdn, exposePort, forceSave, otherFqdns, dualCerts } = request.body;
+        let { fqdn, exposePort, forceSave, dualCerts, otherFqdn = false } = request.body;
+
+        const domainsList = await prisma.serviceSetting.findMany({ where: { variableName: { startsWith: '$$config_coolify_fqdn' } } })
 
         if (fqdn) fqdn = fqdn.toLowerCase();
-        if (otherFqdns && otherFqdns.length > 0) otherFqdns = otherFqdns.map((f) => f.toLowerCase());
         if (exposePort) exposePort = Number(exposePort);
 
         const { destinationDocker: { remoteIpAddress, remoteEngine, engine }, exposePort: configuredPort } = await prisma.service.findUnique({ where: { id }, include: { destinationDocker: true } })
         const { isDNSCheckEnabled } = await prisma.setting.findFirst({});
 
-        let found = await isDomainConfigured({ id, fqdn, remoteIpAddress });
+        let found = await isDomainConfigured({ id, fqdn, remoteIpAddress, checkOwn: otherFqdn });
         if (found) {
             throw { status: 500, message: `Domain ${getDomain(fqdn).replace('www.', '')} is already in use!` }
         }
-        if (otherFqdns && otherFqdns.length > 0) {
-            for (const ofqdn of otherFqdns) {
-                found = await isDomainConfigured({ id, fqdn: ofqdn, remoteIpAddress });
-                if (found) {
-                    throw { status: 500, message: `Domain ${getDomain(ofqdn).replace('www.', '')} is already in use!` }
-                }
-            }
+        if (domainsList.find(d => getDomain(d.value) === getDomain(fqdn))) {
+            throw { status: 500, message: `Domain ${getDomain(fqdn).replace('www.', '')} is already in use!` }
         }
         if (exposePort) await checkExposedPort({ id, configuredPort, exposePort, engine, remoteEngine, remoteIpAddress })
         if (isDNSCheckEnabled && !isDev && !forceSave) {
@@ -266,20 +543,33 @@ export async function checkService(request: FastifyRequest<CheckService>) {
 export async function saveService(request: FastifyRequest<SaveService>, reply: FastifyReply) {
     try {
         const { id } = request.params;
-        let { name, fqdn, exposePort, type } = request.body;
-
+        let { name, fqdn, exposePort, type, serviceSetting, version } = request.body;
         if (fqdn) fqdn = fqdn.toLowerCase();
         if (exposePort) exposePort = Number(exposePort);
-
         type = fixType(type)
-        const update = saveUpdateableFields(type, request.body[type])
+
         const data = {
             fqdn,
             name,
             exposePort,
+            version,
         }
-        if (Object.keys(update).length > 0) {
-            data[type] = { update: update }
+        const templates = await getTemplates()
+        const service = await prisma.service.findUnique({ where: { id } })
+        const foundTemplate = templates.find(t => fixType(t.type) === fixType(service.type))
+        for (const setting of serviceSetting) {
+            let { id: settingId, name, value, changed = false, isNew = false, variableName } = setting
+            if (value) {
+                if (changed) {
+                    await prisma.serviceSetting.update({ where: { id: settingId }, data: { value } })
+                }
+                if (isNew) {
+                    if (!variableName) {
+                        variableName = foundTemplate?.variables.find(v => v.name === name).id
+                    }
+                    await prisma.serviceSetting.create({ data: { name, value, variableName, service: { connect: { id } } } })
+                }
+            }
         }
         await prisma.service.update({
             where: { id }, data
@@ -293,11 +583,19 @@ export async function saveService(request: FastifyRequest<SaveService>, reply: F
 export async function getServiceSecrets(request: FastifyRequest<OnlyId>) {
     try {
         const { id } = request.params
+        const teamId = request.user.teamId;
+        const service = await getServiceFromDB({ id, teamId });
         let secrets = await prisma.serviceSecret.findMany({
             where: { serviceId: id },
             orderBy: { createdAt: 'desc' }
         });
+        const templates = await getTemplates()
+        const foundTemplate = templates.find(t => fixType(t.type) === service.type)
         secrets = secrets.map((secret) => {
+            const foundVariable = foundTemplate?.variables.find(v => v.name === secret.name) || null
+            if (foundVariable) {
+                secret.readOnly = foundVariable.readOnly
+            }
             secret.value = decrypt(secret.value);
             return secret;
         });
@@ -314,7 +612,6 @@ export async function saveServiceSecret(request: FastifyRequest<SaveServiceSecre
     try {
         const { id } = request.params
         let { name, value, isNew } = request.body
-
         if (isNew) {
             const found = await prisma.serviceSecret.findFirst({ where: { name, serviceId: id } });
             if (found) {
@@ -373,16 +670,21 @@ export async function getServiceStorages(request: FastifyRequest<OnlyId>) {
 export async function saveServiceStorage(request: FastifyRequest<SaveServiceStorage>, reply: FastifyReply) {
     try {
         const { id } = request.params
-        const { path, newStorage, storageId } = request.body
+        const { path, isNewStorage, storageId, containerId } = request.body
 
-        if (newStorage) {
+        if (isNewStorage) {
+            const volumeName = `${id}-custom${path.replace(/\//gi, '-')}`
+            const found = await prisma.servicePersistentStorage.findFirst({ where: { path, containerId } });
+            if (found) {
+                throw { status: 500, message: 'Persistent storage already exists for this container and path.' }
+            }
             await prisma.servicePersistentStorage.create({
-                data: { path, service: { connect: { id } } }
+                data: { path, volumeName, containerId, service: { connect: { id } } }
             });
         } else {
             await prisma.servicePersistentStorage.update({
                 where: { id: storageId },
-                data: { path }
+                data: { path, containerId }
             });
         }
         return reply.code(201).send()
@@ -393,9 +695,8 @@ export async function saveServiceStorage(request: FastifyRequest<SaveServiceStor
 
 export async function deleteServiceStorage(request: FastifyRequest<DeleteServiceStorage>) {
     try {
-        const { id } = request.params
-        const { path } = request.body
-        await prisma.servicePersistentStorage.deleteMany({ where: { serviceId: id, path } });
+        const { storageId } = request.body
+        await prisma.servicePersistentStorage.deleteMany({ where: { id: storageId } });
         return {}
     } catch ({ status, message }) {
         return errorHandler({ status, message })
@@ -451,14 +752,17 @@ export async function activatePlausibleUsers(request: FastifyRequest<OnlyId>, re
         const {
             destinationDockerId,
             destinationDocker,
-            plausibleAnalytics: { postgresqlUser, postgresqlPassword, postgresqlDatabase }
+            serviceSecret
         } = await getServiceFromDB({ id, teamId });
         if (destinationDockerId) {
-            await executeDockerCmd({
-                dockerId: destinationDocker.id,
-                command: `docker exec ${id} 'psql -H postgresql://${postgresqlUser}:${postgresqlPassword}@localhost:5432/${postgresqlDatabase} -c "UPDATE users SET email_verified = true;"'`
-            })
-            return await reply.code(201).send()
+            const databaseUrl = serviceSecret.find((secret) => secret.name === 'DATABASE_URL');
+            if (databaseUrl) {
+                await executeCommand({
+                    dockerId: destinationDocker.id,
+                    command: `docker exec ${id}-postgresql psql -H ${databaseUrl.value} -c "UPDATE users SET email_verified = true;"`
+                })
+                return await reply.code(201).send()
+            }
         }
         throw { status: 500, message: 'Could not activate users.' }
     } catch ({ status, message }) {
@@ -474,9 +778,10 @@ export async function cleanupPlausibleLogs(request: FastifyRequest<OnlyId>, repl
             destinationDocker,
         } = await getServiceFromDB({ id, teamId });
         if (destinationDockerId) {
-            await executeDockerCmd({
+            await executeCommand({
                 dockerId: destinationDocker.id,
-                command: `docker exec ${id}-clickhouse sh -c "/usr/bin/clickhouse-client -q \\"SELECT name FROM system.tables WHERE name LIKE '%log%';\\"| xargs -I{} /usr/bin/clickhouse-client -q \"TRUNCATE TABLE system.{};\""`
+                command: `docker exec ${id}-clickhouse /usr/bin/clickhouse-client -q \\"SELECT name FROM system.tables WHERE name LIKE '%log%';\\"| xargs -I{} /usr/bin/clickhouse-client -q \"TRUNCATE TABLE system.{};\"`,
+                shell: true
             })
             return await reply.code(201).send()
         }
@@ -516,36 +821,42 @@ export async function activateWordpressFtp(request: FastifyRequest<ActivateWordp
             if (user) ftpUser = user;
             if (savedPassword) ftpPassword = decrypt(savedPassword);
 
-            const { stdout: password } = await asyncExecShell(
-                `echo ${ftpPassword} | openssl passwd -1 -stdin`
+            // TODO: rewrite these to usable without shell
+            const { stdout: password } = await executeCommand({
+                command:
+                    `echo ${ftpPassword} | openssl passwd -1 -stdin`,
+                shell: true
+            }
             );
             if (destinationDockerId) {
                 try {
                     await fs.stat(hostkeyDir);
                 } catch (error) {
-                    await asyncExecShell(`mkdir -p ${hostkeyDir}`);
+                    await executeCommand({ command: `mkdir -p ${hostkeyDir}` });
                 }
                 if (!ftpHostKey) {
-                    await asyncExecShell(
-                        `ssh-keygen -t ed25519 -f ssh_host_ed25519_key -N "" -q -f ${hostkeyDir}/${id}.ed25519`
+                    await executeCommand({
+                        command:
+                            `ssh-keygen -t ed25519 -f ssh_host_ed25519_key -N "" -q -f ${hostkeyDir}/${id}.ed25519`
+                    }
                     );
-                    const { stdout: ftpHostKey } = await asyncExecShell(`cat ${hostkeyDir}/${id}.ed25519`);
+                    const { stdout: ftpHostKey } = await executeCommand({ command: `cat ${hostkeyDir}/${id}.ed25519` });
                     await prisma.wordpress.update({
                         where: { serviceId: id },
                         data: { ftpHostKey: encrypt(ftpHostKey) }
                     });
                 } else {
-                    await asyncExecShell(`echo "${decrypt(ftpHostKey)}" > ${hostkeyDir}/${id}.ed25519`);
+                    await executeCommand({ command: `echo "${decrypt(ftpHostKey)}" > ${hostkeyDir}/${id}.ed25519`, shell: true });
                 }
                 if (!ftpHostKeyPrivate) {
-                    await asyncExecShell(`ssh-keygen -t rsa -b 4096 -N "" -f ${hostkeyDir}/${id}.rsa`);
-                    const { stdout: ftpHostKeyPrivate } = await asyncExecShell(`cat ${hostkeyDir}/${id}.rsa`);
+                    await executeCommand({ command: `ssh-keygen -t rsa -b 4096 -N "" -f ${hostkeyDir}/${id}.rsa` });
+                    const { stdout: ftpHostKeyPrivate } = await executeCommand({ command: `cat ${hostkeyDir}/${id}.rsa` });
                     await prisma.wordpress.update({
                         where: { serviceId: id },
                         data: { ftpHostKeyPrivate: encrypt(ftpHostKeyPrivate) }
                     });
                 } else {
-                    await asyncExecShell(`echo "${decrypt(ftpHostKeyPrivate)}" > ${hostkeyDir}/${id}.rsa`);
+                    await executeCommand({ command: `echo "${decrypt(ftpHostKeyPrivate)}" > ${hostkeyDir}/${id}.rsa`, shell: true });
                 }
 
                 await prisma.wordpress.update({
@@ -560,9 +871,10 @@ export async function activateWordpressFtp(request: FastifyRequest<ActivateWordp
                 try {
                     const { found: isRunning } = await checkContainer({ dockerId: destinationDocker.id, container: `${id}-ftp` });
                     if (isRunning) {
-                        await executeDockerCmd({
+                        await executeCommand({
                             dockerId: destinationDocker.id,
-                            command: `docker stop -t 0 ${id}-ftp && docker rm ${id}-ftp`
+                            command: `docker stop -t 0 ${id}-ftp && docker rm ${id}-ftp`,
+                            shell: true
                         })
                     }
                 } catch (error) { }
@@ -606,9 +918,9 @@ export async function activateWordpressFtp(request: FastifyRequest<ActivateWordp
                     `${hostkeyDir}/${id}.sh`,
                     `#!/bin/bash\nchmod 600 /etc/ssh/ssh_host_ed25519_key /etc/ssh/ssh_host_rsa_key\nuserdel -f xfs\nchown -R 33:33 /home/${ftpUser}/wordpress/`
                 );
-                await asyncExecShell(`chmod +x ${hostkeyDir}/${id}.sh`);
+                await executeCommand({ command: `chmod +x ${hostkeyDir}/${id}.sh` });
                 await fs.writeFile(`${hostkeyDir}/${id}-docker-compose.yml`, yaml.dump(compose));
-                await executeDockerCmd({
+                await executeCommand({
                     dockerId: destinationDocker.id,
                     command: `docker compose -f ${hostkeyDir}/${id}-docker-compose.yml up -d`
                 })
@@ -625,9 +937,10 @@ export async function activateWordpressFtp(request: FastifyRequest<ActivateWordp
                 data: { ftpPublicPort: null }
             });
             try {
-                await executeDockerCmd({
+                await executeCommand({
                     dockerId: destinationDocker.id,
-                    command: `docker stop -t 0 ${id}-ftp && docker rm ${id}-ftp`
+                    command: `docker stop -t 0 ${id}-ftp && docker rm ${id}-ftp`,
+                    shell: true
                 })
 
             } catch (error) {
@@ -641,8 +954,10 @@ export async function activateWordpressFtp(request: FastifyRequest<ActivateWordp
         return errorHandler({ status, message })
     } finally {
         try {
-            await asyncExecShell(
-                `rm -fr ${hostkeyDir}/${id}-docker-compose.yml ${hostkeyDir}/${id}.ed25519 ${hostkeyDir}/${id}.ed25519.pub ${hostkeyDir}/${id}.rsa ${hostkeyDir}/${id}.rsa.pub ${hostkeyDir}/${id}.sh`
+            await executeCommand({
+                command:
+                    `rm -fr ${hostkeyDir}/${id}-docker-compose.yml ${hostkeyDir}/${id}.ed25519 ${hostkeyDir}/${id}.ed25519.pub ${hostkeyDir}/${id}.rsa ${hostkeyDir}/${id}.rsa.pub ${hostkeyDir}/${id}.sh`
+            }
             );
         } catch (error) { }
 

apps/api/src/routes/api/v1/services/index.ts

@@ -5,6 +5,7 @@ import {
     checkService,
     checkServiceDomain,
     cleanupPlausibleLogs,
+    cleanupUnconfiguredServices,
     deleteService,
     deleteServiceSecret,
     deleteServiceStorage,
@@ -15,7 +16,6 @@ import {
     getServiceStorages,
     getServiceType,
     getServiceUsage,
-    getServiceVersions,
     listServices,
     newService,
     saveService,
@@ -30,7 +30,7 @@ import {
 
 import type { OnlyId } from '../../../../types';
 import type { ActivateWordpressFtp, CheckService, CheckServiceDomain, DeleteServiceSecret, DeleteServiceStorage, GetServiceLogs, SaveService, SaveServiceDestination, SaveServiceSecret, SaveServiceSettings, SaveServiceStorage, SaveServiceType, SaveServiceVersion, ServiceStartStop, SetGlitchTipSettings, SetWordpressSettings } from './types';
-import { startService, stopService } from '../../../../lib/services/handlers';
+import { migrateAppwriteDB, startService, stopService } from '../../../../lib/services/handlers';
 
 const root: FastifyPluginAsync = async (fastify): Promise<void> => {
     fastify.addHook('onRequest', async (request) => {
@@ -39,6 +39,8 @@ const root: FastifyPluginAsync = async (fastify): Promise<void> => {
     fastify.get('/', async (request) => await listServices(request));
     fastify.post('/new', async (request, reply) => await newService(request, reply));
 
+    fastify.post<any>('/cleanup/unconfigured', async (request) => await cleanupUnconfiguredServices(request));
+
     fastify.get<OnlyId>('/:id', async (request) => await getService(request));
     fastify.post<SaveService>('/:id', async (request, reply) => await saveService(request, reply));
     fastify.delete<OnlyId>('/:id', async (request) => await deleteService(request));
@@ -61,21 +63,22 @@ const root: FastifyPluginAsync = async (fastify): Promise<void> => {
     fastify.get('/:id/configuration/type', async (request) => await getServiceType(request));
     fastify.post<SaveServiceType>('/:id/configuration/type', async (request, reply) => await saveServiceType(request, reply));
 
-    fastify.get<OnlyId>('/:id/configuration/version', async (request) => await getServiceVersions(request));
     fastify.post<SaveServiceVersion>('/:id/configuration/version', async (request, reply) => await saveServiceVersion(request, reply));
 
     fastify.post<SaveServiceDestination>('/:id/configuration/destination', async (request, reply) => await saveServiceDestination(request, reply));
 
     fastify.get<OnlyId>('/:id/usage', async (request) => await getServiceUsage(request));
-    fastify.get<GetServiceLogs>('/:id/logs', async (request) => await getServiceLogs(request));
+    fastify.get<GetServiceLogs>('/:id/logs/:containerId', async (request) => await getServiceLogs(request));
 
-    fastify.post<ServiceStartStop>('/:id/:type/start', async (request) => await startService(request));
-    fastify.post<ServiceStartStop>('/:id/:type/stop', async (request) => await stopService(request));
+    fastify.post<ServiceStartStop>('/:id/start', async (request) => await startService(request, fastify));
+    fastify.post<ServiceStartStop>('/:id/stop', async (request) => await stopService(request));
     fastify.post<ServiceStartStop & SetWordpressSettings & SetGlitchTipSettings>('/:id/:type/settings', async (request, reply) => await setSettingsService(request, reply));
 
     fastify.post<OnlyId>('/:id/plausibleanalytics/activate', async (request, reply) => await activatePlausibleUsers(request, reply));
     fastify.post<OnlyId>('/:id/plausibleanalytics/cleanup', async (request, reply) => await cleanupPlausibleLogs(request, reply));
     fastify.post<ActivateWordpressFtp>('/:id/wordpress/ftp', async (request, reply) => await activateWordpressFtp(request, reply));
+
+    fastify.post<OnlyId>('/:id/appwrite/migrate', async (request, reply) => await migrateAppwriteDB(request, reply));
 };
 
 export default root;

apps/api/src/routes/api/v1/services/types.ts

@@ -15,9 +15,13 @@ export interface SaveServiceDestination extends OnlyId {
         destinationId: string
     }
 }
-export interface GetServiceLogs extends OnlyId {
+export interface GetServiceLogs{
+    Params: { 
+        id: string,
+        containerId: string
+     },
     Querystring: {
-        since: number
+        since: number,
     }
 }
 export interface SaveServiceSettings extends OnlyId {
@@ -36,7 +40,7 @@ export interface CheckService extends OnlyId {
         forceSave: boolean,
         dualCerts: boolean,
         exposePort: number,
-        otherFqdns: Array<string>
+        otherFqdn: boolean
     }
 }
 export interface SaveService extends OnlyId {
@@ -44,6 +48,8 @@ export interface SaveService extends OnlyId {
         name: string,
         fqdn: string,
         exposePort: number,
+        version: string,
+        serviceSetting: any
         type: string
     }
 }
@@ -62,14 +68,15 @@ export interface DeleteServiceSecret extends OnlyId {
 export interface SaveServiceStorage extends OnlyId {
     Body: {
         path: string,
-        newStorage: string,
+        containerId: string,
         storageId: string,
+        isNewStorage: boolean,
     }
 }
 
 export interface DeleteServiceStorage extends OnlyId {
     Body: {
-        path: string,
+        storageId: string,
     }
 }
 export interface ServiceStartStop {

apps/api/src/routes/api/v1/settings/handlers.ts

@@ -1,8 +1,9 @@
 import { promises as dns } from 'dns';
-
+import { X509Certificate } from 'node:crypto';
+import * as Sentry from '@sentry/node';
 import type { FastifyReply, FastifyRequest } from 'fastify';
-import { checkDomainsIsValidInDNS, decrypt, encrypt, errorHandler, getDomain, isDNSValid, isDomainConfigured, listSettings, prisma } from '../../../../lib/common';
-import { CheckDNS, CheckDomain, DeleteDomain, DeleteSSHKey, SaveSettings, SaveSSHKey } from './types';
+import { checkDomainsIsValidInDNS, decrypt, encrypt, errorHandler, executeCommand, getDomain, isDev, isDNSValid, isDomainConfigured, listSettings, prisma, sentryDSN, version } from '../../../../lib/common';
+import { AddDefaultRegistry, CheckDNS, CheckDomain, DeleteDomain, OnlyIdInBody, SaveSettings, SaveSSHKey, SetDefaultRegistry } from './types';
 
 
 export async function listAllSettings(request: FastifyRequest) {
@@ -10,15 +11,31 @@ export async function listAllSettings(request: FastifyRequest) {
         const teamId = request.user.teamId;
         const settings = await listSettings();
         const sshKeys = await prisma.sshKey.findMany({ where: { team: { id: teamId } } })
+        let registries = await prisma.dockerRegistry.findMany({ where: { team: { id: teamId } } })
+        registries = registries.map((registry) => {
+            if (registry.password) {
+                registry.password = decrypt(registry.password)
+            }
+            return registry
+        })
         const unencryptedKeys = []
         if (sshKeys.length > 0) {
             for (const key of sshKeys) {
                 unencryptedKeys.push({ id: key.id, name: key.name, privateKey: decrypt(key.privateKey), createdAt: key.createdAt })
             }
         }
+        const certificates = await prisma.certificate.findMany({ where: { team: { id: teamId } } })
+        let cns = [];
+        for (const certificate of certificates) {
+            const x509 = new X509Certificate(certificate.cert);
+            cns.push({ commonName: x509.subject.split('\n').find((s) => s.startsWith('CN=')).replace('CN=', ''), id: certificate.id, createdAt: certificate.createdAt })
+        }
+
         return {
             settings,
-            sshKeys: unencryptedKeys
+            certificates: cns,
+            sshKeys: unencryptedKeys,
+            registries
         }
     } catch ({ status, message }) {
         return errorHandler({ status, message })
@@ -26,7 +43,10 @@ export async function listAllSettings(request: FastifyRequest) {
 }
 export async function saveSettings(request: FastifyRequest<SaveSettings>, reply: FastifyReply) {
     try {
-        const {
+        let {
+            previewSeparator,
+            numberOfDockerImagesKeptLocally,
+            doNotTrack,
             fqdn,
             isAPIDebuggingEnabled,
             isRegistrationEnabled,
@@ -35,19 +55,48 @@ export async function saveSettings(request: FastifyRequest<SaveSettings>, reply:
             maxPort,
             isAutoUpdateEnabled,
             isDNSCheckEnabled,
-            DNSServers
+            DNSServers,
+            proxyDefaultRedirect
         } = request.body
-        const { id } = await listSettings();
+        const { id, previewSeparator: SetPreviewSeparator } = await listSettings();
+        if (numberOfDockerImagesKeptLocally) {
+            numberOfDockerImagesKeptLocally = Number(numberOfDockerImagesKeptLocally)
+        }
+        if (previewSeparator == '') {
+            previewSeparator = '.'
+        }
+        if (SetPreviewSeparator != previewSeparator) {
+            const applications = await prisma.application.findMany({ where: { previewApplication: { some: { id: { not: undefined } } } }, include: { previewApplication: true } })
+            for (const application of applications) {
+                for (const preview of application.previewApplication) {
+                    const { protocol } = new URL(preview.customDomain)
+                    const { pullmergeRequestId } = preview
+                    const { fqdn } = application
+                    const newPreviewDomain = `${protocol}//${pullmergeRequestId}${previewSeparator}${getDomain(fqdn)}`
+                    await prisma.previewApplication.update({ where: { id: preview.id }, data: { customDomain: newPreviewDomain } })
+                }
+            }
+        }
+
         await prisma.setting.update({
             where: { id },
-            data: { isRegistrationEnabled, dualCerts, isAutoUpdateEnabled, isDNSCheckEnabled, DNSServers, isAPIDebuggingEnabled }
+            data: { previewSeparator, numberOfDockerImagesKeptLocally, doNotTrack, isRegistrationEnabled, dualCerts, isAutoUpdateEnabled, isDNSCheckEnabled, DNSServers, isAPIDebuggingEnabled }
         });
         if (fqdn) {
             await prisma.setting.update({ where: { id }, data: { fqdn } });
         }
+        await prisma.setting.update({ where: { id }, data: { proxyDefaultRedirect } });
         if (minPort && maxPort) {
             await prisma.setting.update({ where: { id }, data: { minPort, maxPort } });
         }
+        if (doNotTrack === false) {
+            // Sentry.init({
+            //     dsn: sentryDSN,
+            //     environment: isDev ? 'development' : 'production',
+            //     release: version
+            // });
+            // console.log('Sentry initialized')
+        }
         return reply.code(201).send()
     } catch ({ status, message }) {
         return errorHandler({ status, message })
@@ -80,9 +129,9 @@ export async function checkDomain(request: FastifyRequest<CheckDomain>) {
         if (fqdn) fqdn = fqdn.toLowerCase();
         const found = await isDomainConfigured({ id, fqdn });
         if (found) {
-            throw "Domain already configured";
+            throw { message: "Domain already configured" };
         }
-        if (isDNSCheckEnabled && !forceSave) {
+        if (isDNSCheckEnabled && !forceSave && !isDev) {
             const hostname = request.hostname.split(':')[0]
             return await checkDomainsIsValidInDNS({ hostname, fqdn, dualCerts });
         }
@@ -118,10 +167,67 @@ export async function saveSSHKey(request: FastifyRequest<SaveSSHKey>, reply: Fas
         return errorHandler({ status, message })
     }
 }
-export async function deleteSSHKey(request: FastifyRequest<DeleteSSHKey>, reply: FastifyReply) {
+export async function deleteSSHKey(request: FastifyRequest<OnlyIdInBody>, reply: FastifyReply) {
     try {
+        const teamId = request.user.teamId;
         const { id } = request.body;
-        await prisma.sshKey.delete({ where: { id } })
+        await prisma.sshKey.deleteMany({ where: { id, teamId } })
+        return reply.code(201).send()
+    } catch ({ status, message }) {
+        return errorHandler({ status, message })
+    }
+}
+
+export async function deleteCertificates(request: FastifyRequest<OnlyIdInBody>, reply: FastifyReply) {
+    try {
+        const teamId = request.user.teamId;
+        const { id } = request.body;
+        await executeCommand({ command: `docker exec coolify-proxy sh -c 'rm -f /etc/traefik/acme/custom/${id}-key.pem /etc/traefik/acme/custom/${id}-cert.pem'`, shell: true })
+        await prisma.certificate.deleteMany({ where: { id, teamId } })
+        return reply.code(201).send()
+    } catch ({ status, message }) {
+        return errorHandler({ status, message })
+    }
+}
+
+export async function setDockerRegistry(request: FastifyRequest<SetDefaultRegistry>, reply: FastifyReply) {
+    try {
+        const teamId = request.user.teamId;
+        const { id, username, password } = request.body;
+
+        let encryptedPassword = ''
+        if (password) encryptedPassword = encrypt(password)
+
+        if (teamId === '0') {
+            await prisma.dockerRegistry.update({ where: { id }, data: { username, password: encryptedPassword } })
+        } else {
+            await prisma.dockerRegistry.updateMany({ where: { id, teamId }, data: { username, password: encryptedPassword } })
+        }
+        return reply.code(201).send()
+    } catch ({ status, message }) {
+        return errorHandler({ status, message })
+    }
+}
+export async function addDockerRegistry(request: FastifyRequest<AddDefaultRegistry>, reply: FastifyReply) {
+    try {
+        const teamId = request.user.teamId;
+        const { name, url, username, password } = request.body;
+
+        let encryptedPassword = ''
+        if (password) encryptedPassword = encrypt(password)
+        await prisma.dockerRegistry.create({ data: { name, url, username, password: encryptedPassword, team: { connect: { id: teamId } } } })
+
+        return reply.code(201).send()
+    } catch ({ status, message }) {
+        return errorHandler({ status, message })
+    }
+}
+export async function deleteDockerRegistry(request: FastifyRequest<OnlyIdInBody>, reply: FastifyReply) {
+    try {
+        const teamId = request.user.teamId;
+        const { id } = request.body;
+        await prisma.application.updateMany({ where: { dockerRegistryId: id }, data: { dockerRegistryId: null } })
+        await prisma.dockerRegistry.deleteMany({ where: { id, teamId } })
         return reply.code(201).send()
     } catch ({ status, message }) {
         return errorHandler({ status, message })

apps/api/src/routes/api/v1/settings/index.ts

@@ -1,21 +1,62 @@
 import { FastifyPluginAsync } from 'fastify';
-import { checkDNS, checkDomain, deleteDomain, deleteSSHKey, listAllSettings, saveSettings, saveSSHKey } from './handlers';
-import { CheckDNS, CheckDomain, DeleteDomain, DeleteSSHKey, SaveSettings, SaveSSHKey } from './types';
+import { X509Certificate } from 'node:crypto';
+
+import { encrypt, errorHandler, prisma } from '../../../../lib/common';
+import { addDockerRegistry, checkDNS, checkDomain, deleteCertificates, deleteDockerRegistry, deleteDomain, deleteSSHKey, listAllSettings, saveSettings, saveSSHKey, setDockerRegistry } from './handlers';
+import { AddDefaultRegistry, CheckDNS, CheckDomain, DeleteDomain, OnlyIdInBody, SaveSettings, SaveSSHKey, SetDefaultRegistry } from './types';
 
 
 const root: FastifyPluginAsync = async (fastify): Promise<void> => {
-    fastify.addHook('onRequest', async (request) => {
-        return await request.jwtVerify()
-    })
-    fastify.get('/', async (request) => await listAllSettings(request));
-    fastify.post<SaveSettings>('/', async (request, reply) => await saveSettings(request, reply));
-    fastify.delete<DeleteDomain>('/', async (request, reply) => await deleteDomain(request, reply));
+	fastify.addHook('onRequest', async (request) => {
+		return await request.jwtVerify()
+	})
+	fastify.get('/', async (request) => await listAllSettings(request));
+	fastify.post<SaveSettings>('/', async (request, reply) => await saveSettings(request, reply));
+	fastify.delete<DeleteDomain>('/', async (request, reply) => await deleteDomain(request, reply));
 
-    fastify.get<CheckDNS>('/check', async (request) => await checkDNS(request));
-    fastify.post<CheckDomain>('/check', async (request) => await checkDomain(request));
+	fastify.get<CheckDNS>('/check', async (request) => await checkDNS(request));
+	fastify.post<CheckDomain>('/check', async (request) => await checkDomain(request));
 
-    fastify.post<SaveSSHKey>('/sshKey', async (request, reply) => await saveSSHKey(request, reply));
-    fastify.delete<DeleteSSHKey>('/sshKey', async (request, reply) => await deleteSSHKey(request, reply));
+	fastify.post<SaveSSHKey>('/sshKey', async (request, reply) => await saveSSHKey(request, reply));
+	fastify.delete<OnlyIdInBody>('/sshKey', async (request, reply) => await deleteSSHKey(request, reply));
+
+	fastify.post<SetDefaultRegistry>('/registry', async (request, reply) => await setDockerRegistry(request, reply));
+	fastify.post<AddDefaultRegistry>('/registry/new', async (request, reply) => await addDockerRegistry(request, reply));
+	fastify.delete<OnlyIdInBody>('/registry', async (request, reply) => await deleteDockerRegistry(request, reply));
+
+	fastify.post('/upload', async (request) => {
+		try {
+			const teamId = request.user.teamId;
+			const certificates = await prisma.certificate.findMany({})
+			let cns = [];
+			for (const certificate of certificates) {
+				const x509 = new X509Certificate(certificate.cert);
+				cns.push(x509.subject.split('\n').find((s) => s.startsWith('CN=')).replace('CN=', ''))
+			}
+			const parts = await request.files()
+			let key = null
+			let cert = null
+			for await (const part of parts) {
+				const name = part.fieldname
+				if (name === 'key') key = (await part.toBuffer()).toString()
+				if (name === 'cert') cert = (await part.toBuffer()).toString()
+			}
+			const x509 = new X509Certificate(cert);
+			const cn = x509.subject.split('\n').find((s) => s.startsWith('CN=')).replace('CN=', '')
+			if (cns.includes(cn)) {
+				throw {
+					message: `A certificate with ${cn} common name already exists.`
+				}
+			}
+			await prisma.certificate.create({ data: { cert, key: encrypt(key), team: { connect: { id: teamId } } } })
+			await prisma.applicationSettings.updateMany({ where: { application: { AND: [{ fqdn: { endsWith: cn } }, { fqdn: { startsWith: 'https' } }] } }, data: { isCustomSSL: true } })
+			return { message: 'Certificated uploaded' }
+		} catch ({ status, message }) {
+			return errorHandler({ status, message });
+		}
+
+	});
+	fastify.delete<OnlyIdInBody>('/certificate', async (request, reply) => await deleteCertificates(request, reply))
 };
 
 export default root;

apps/api/src/routes/api/v1/settings/types.ts

@@ -2,6 +2,9 @@ import { OnlyId } from "../../../../types"
 
 export interface SaveSettings {
     Body: {
+        previewSeparator: string,
+        numberOfDockerImagesKeptLocally: number,
+        doNotTrack: boolean,
         fqdn: string,
         isAPIDebuggingEnabled: boolean,
         isRegistrationEnabled: boolean,
@@ -10,7 +13,8 @@ export interface SaveSettings {
         maxPort: number,
         isAutoUpdateEnabled: boolean,
         isDNSCheckEnabled: boolean,
-        DNSServers: string
+        DNSServers: string,
+        proxyDefaultRedirect: string
     }
 }
 export interface DeleteDomain {
@@ -20,25 +24,46 @@ export interface DeleteDomain {
 }
 export interface CheckDomain extends OnlyId {
     Body: {
-        fqdn: string, 
-        forceSave: boolean, 
-        dualCerts: boolean, 
-        isDNSCheckEnabled: boolean, 
+        fqdn: string,
+        forceSave: boolean,
+        dualCerts: boolean,
+        isDNSCheckEnabled: boolean,
     }
 }
 export interface CheckDNS {
     Params: {
-        domain: string, 
+        domain: string,
     }
 }
 export interface SaveSSHKey {
     Body: {
-        privateKey: string, 
+        privateKey: string,
         name: string
     }
 }
 export interface DeleteSSHKey {
     Body: {
-       id: string
+        id: string
+    }
+}
+export interface OnlyIdInBody {
+    Body: {
+        id: string
+    }
+}
+
+export interface SetDefaultRegistry {
+    Body: {
+        id: string
+        username: string
+        password: string
+    }
+}
+export interface AddDefaultRegistry {
+    Body: {
+        url: string
+        name: string
+        username: string
+        password: string
     }
 }

apps/api/src/routes/api/v1/sources/handlers.ts

@@ -9,7 +9,7 @@ export async function listSources(request: FastifyRequest) {
     try {
         const teamId = request.user?.teamId;
         const sources = await prisma.gitSource.findMany({
-            where: { teams: { some: { id: teamId === '0' ? undefined : teamId } } },
+            where: { OR: [{ teams: { some: { id: teamId === "0" ? undefined : teamId } } }, { isSystemWide: true }] },
             include: { teams: true, githubApp: true, gitlabApp: true }
         });
         return {
@@ -22,11 +22,11 @@ export async function listSources(request: FastifyRequest) {
 export async function saveSource(request, reply) {
     try {
         const { id } = request.params
-        let { name, htmlUrl, apiUrl, customPort } = request.body
+        let { name, htmlUrl, apiUrl, customPort, isSystemWide } = request.body
         if (customPort) customPort = Number(customPort)
         await prisma.gitSource.update({
             where: { id },
-            data: { name, htmlUrl, apiUrl, customPort }
+            data: { name, htmlUrl, apiUrl, customPort, isSystemWide }
         });
         return reply.code(201).send()
     } catch ({ status, message }) {
@@ -37,9 +37,7 @@ export async function getSource(request: FastifyRequest<OnlyId>) {
     try {
         const { id } = request.params
         const { teamId } = request.user
-
         const settings = await prisma.setting.findFirst({});
-        if (settings.proxyPassword) settings.proxyPassword = decrypt(settings.proxyPassword);
 
         if (id === 'new') {
             return {
@@ -56,7 +54,7 @@ export async function getSource(request: FastifyRequest<OnlyId>) {
         }
 
         const source = await prisma.gitSource.findFirst({
-            where: { id, teams: { some: { id: teamId === '0' ? undefined : teamId } } },
+            where: { id, OR: [{ teams: { some: { id: teamId === "0" ? undefined : teamId } } }, { isSystemWide: true }] },
             include: { githubApp: true, gitlabApp: true }
         });
         if (!source) {
@@ -104,7 +102,7 @@ export async function saveGitHubSource(request: FastifyRequest<SaveGitHubSource>
         const { teamId } = request.user
 
         const { id } = request.params
-        let { name, htmlUrl, apiUrl, organization, customPort } = request.body
+        let { name, htmlUrl, apiUrl, organization, customPort, isSystemWide } = request.body
 
         if (customPort) customPort = Number(customPort)
         if (id === 'new') {
@@ -117,6 +115,7 @@ export async function saveGitHubSource(request: FastifyRequest<SaveGitHubSource>
                     apiUrl,
                     organization,
                     customPort,
+                    isSystemWide,
                     type: 'github',
                     teams: { connect: { id: teamId } }
                 }

apps/api/src/routes/api/v1/sources/types.ts

@@ -7,6 +7,7 @@ export interface SaveGitHubSource extends OnlyId {
         apiUrl: string,
         organization: string,
         customPort: number,
+        isSystemWide: boolean
     }
 }
 export interface SaveGitLabSource extends OnlyId {

apps/api/src/routes/webhooks/github/handlers.ts

@@ -1,7 +1,6 @@
-import axios from "axios";
 import cuid from "cuid";
 import crypto from "crypto";
-import { encrypt, errorHandler, getUIUrl, isDev, prisma } from "../../../lib/common";
+import { encrypt, errorHandler, getDomain, getUIUrl, isDev, prisma } from "../../../lib/common";
 import { checkContainer, removeContainer } from "../../../lib/docker";
 import { createdBranchDatabase, getApplicationFromDBWebhook, removeBranchDatabase } from "../../api/v1/applications/handlers";
 
@@ -32,13 +31,14 @@ export async function installGithub(request: FastifyRequest<InstallGithub>, repl
 }
 export async function configureGitHubApp(request, reply) {
     try {
+        const { default: got } = await import('got')
         const { code, state } = request.query;
         const { apiUrl } = await prisma.gitSource.findFirst({
             where: { id: state },
             include: { githubApp: true, gitlabApp: true }
         });
 
-        const { data }: any = await axios.post(`${apiUrl}/app-manifests/${code}/conversions`);
+        const data: any = await got.post(`${apiUrl}/app-manifests/${code}/conversions`).json()
         const { id, client_id, slug, client_secret, pem, webhook_secret } = data
 
         const encryptedClientSecret = encrypt(client_secret);
@@ -66,12 +66,18 @@ export async function configureGitHubApp(request, reply) {
 }
 export async function gitHubEvents(request: FastifyRequest<GitHubEvents>): Promise<any> {
     try {
-        const allowedGithubEvents = ['push', 'pull_request'];
+        const allowedGithubEvents = ['push', 'pull_request', 'ping', 'installation'];
         const allowedActions = ['opened', 'reopened', 'synchronize', 'closed'];
         const githubEvent = request.headers['x-github-event']?.toString().toLowerCase();
         const githubSignature = request.headers['x-hub-signature-256']?.toString().toLowerCase();
         if (!allowedGithubEvents.includes(githubEvent)) {
-            throw { status: 500, message: 'Event not allowed.' }
+            throw { status: 500, message: 'Event not allowed.', type: 'webhook' }
+        }
+        if (githubEvent === 'ping') {
+            return { pong: 'cool' }
+        }
+        if (githubEvent === 'installation') {
+            return { status: 'cool' }
         }
         let projectId, branch;
         const body = request.body
@@ -80,12 +86,13 @@ export async function gitHubEvents(request: FastifyRequest<GitHubEvents>): Promi
             branch = body.ref.includes('/') ? body.ref.split('/')[2] : body.ref;
         } else if (githubEvent === 'pull_request') {
             projectId = body.pull_request.base.repo.id;
-            branch = body.pull_request.base.ref.includes('/') ? body.pull_request.base.ref.split('/')[2] : body.pull_request.base.ref;
+            branch = body.pull_request.base.ref
         }
         if (!projectId || !branch) {
-            throw { status: 500, message: 'Cannot parse projectId or branch from the webhook?!' }
+            throw { status: 500, message: 'Cannot parse projectId or branch from the webhook?!', type: 'webhook' }
         }
         const applicationsFound = await getApplicationFromDBWebhook(projectId, branch);
+        const settings = await prisma.setting.findUnique({ where: { id: '0' } });
         if (applicationsFound && applicationsFound.length > 0) {
             for (const application of applicationsFound) {
                 const buildId = cuid();
@@ -100,7 +107,7 @@ export async function gitHubEvents(request: FastifyRequest<GitHubEvents>): Promi
                     const checksum = Buffer.from(githubSignature, 'utf8');
                     //@ts-ignore
                     if (checksum.length !== digest.length || !crypto.timingSafeEqual(digest, checksum)) {
-                        throw { status: 500, message: 'SHA256 checksum failed. Are you doing something fishy?' }
+                        throw { status: 500, message: 'SHA256 checksum failed. Are you doing something fishy?', type: 'webhook' }
                     };
                 }
 
@@ -147,9 +154,10 @@ export async function gitHubEvents(request: FastifyRequest<GitHubEvents>): Promi
                 } else if (githubEvent === 'pull_request') {
                     const pullmergeRequestId = body.number.toString();
                     const pullmergeRequestAction = body.action;
-                    const sourceBranch = body.pull_request.head.ref.includes('/') ? body.pull_request.head.ref.split('/')[2] : body.pull_request.head.ref;
+                    const sourceBranch = body.pull_request.head.ref
+                    const sourceRepository = body.pull_request.head.repo.full_name
                     if (!allowedActions.includes(pullmergeRequestAction)) {
-                        throw { status: 500, message: 'Action not allowed.' }
+                        throw { status: 500, message: 'Action not allowed.', type: 'webhook' }
                     }
 
                     if (application.settings.previews) {
@@ -161,7 +169,7 @@ export async function gitHubEvents(request: FastifyRequest<GitHubEvents>): Promi
                                 }
                             );
                             if (!isRunning) {
-                                throw { status: 500, message: 'Application not running.' }
+                                throw { status: 500, message: 'Application not running.', type: 'webhook' }
                             }
                         }
                         if (
@@ -169,10 +177,29 @@ export async function gitHubEvents(request: FastifyRequest<GitHubEvents>): Promi
                             pullmergeRequestAction === 'reopened' ||
                             pullmergeRequestAction === 'synchronize'
                         ) {
+
                             await prisma.application.update({
                                 where: { id: application.id },
                                 data: { updatedAt: new Date() }
                             });
+                            let previewApplicationId = undefined
+                            if (pullmergeRequestId) {
+                                const foundPreviewApplications = await prisma.previewApplication.findMany({ where: { applicationId: application.id, pullmergeRequestId } })
+                                if (foundPreviewApplications.length > 0) {
+                                    previewApplicationId = foundPreviewApplications[0].id
+                                } else {
+                                    const protocol = application.fqdn.includes('https://') ? 'https://' : 'http://'
+                                    const previewApplication = await prisma.previewApplication.create({
+                                        data: {
+                                            pullmergeRequestId,
+                                            sourceBranch,
+                                            customDomain: `${protocol}${pullmergeRequestId}${settings.previewSeparator}${getDomain(application.fqdn)}`,
+                                            application: { connect: { id: application.id } }
+                                        }
+                                    })
+                                    previewApplicationId = previewApplication.id
+                                }
+                            }
                             // if (application.connectedDatabase && pullmergeRequestAction === 'opened' || pullmergeRequestAction === 'reopened') {
                             //     // Coolify hosted database
                             //     if (application.connectedDatabase.databaseId) {
@@ -186,7 +213,9 @@ export async function gitHubEvents(request: FastifyRequest<GitHubEvents>): Promi
                             await prisma.build.create({
                                 data: {
                                     id: buildId,
+                                    sourceRepository,
                                     pullmergeRequestId,
+                                    previewApplicationId,
                                     sourceBranch,
                                     applicationId: application.id,
                                     destinationDockerId: application.destinationDocker.id,
@@ -198,7 +227,9 @@ export async function gitHubEvents(request: FastifyRequest<GitHubEvents>): Promi
                                 }
                             });
 
-
+                            return {
+                                message: 'Queued. Thank you!'
+                            };
                         } else if (pullmergeRequestAction === 'closed') {
                             if (application.destinationDockerId) {
                                 const id = `${application.id}-${pullmergeRequestId}`;
@@ -206,20 +237,29 @@ export async function gitHubEvents(request: FastifyRequest<GitHubEvents>): Promi
                                     await removeContainer({ id, dockerId: application.destinationDocker.id });
                                 } catch (error) { }
                             }
-                            if (application.connectedDatabase.databaseId) {
-                                const databaseId = application.connectedDatabase.databaseId;
-                                const database = await prisma.database.findUnique({ where: { id: databaseId } });
-                                if (database) {
-                                    await removeBranchDatabase(database, pullmergeRequestId);
+                            const foundPreviewApplications = await prisma.previewApplication.findMany({ where: { applicationId: application.id, pullmergeRequestId } })
+                            if (foundPreviewApplications.length > 0) {
+                                for (const preview of foundPreviewApplications) {
+                                    await prisma.previewApplication.delete({ where: { id: preview.id } })
                                 }
                             }
+                            return {
+                                message: 'PR closed. Thank you!'
+                            };
+                            // if (application?.connectedDatabase?.databaseId) {
+                            //     const databaseId = application.connectedDatabase.databaseId;
+                            //     const database = await prisma.database.findUnique({ where: { id: databaseId } });
+                            //     if (database) {
+                            //         await removeBranchDatabase(database, pullmergeRequestId);
+                            //     }
+                            // }
                         }
                     }
                 }
             }
         }
-    } catch ({ status, message }) {
-        return errorHandler({ status, message })
+    } catch ({ status, message, type }) {
+        return errorHandler({ status, message, type })
     }
 
 }

apps/api/src/routes/webhooks/github/types.ts

@@ -23,6 +23,7 @@ export interface GitHubEvents {
                 ref: string,
                 repo: {
                     id: string,
+                    full_name: string,
                 }
             }
         }