feat(server): improve selfhost deployment

.github/actions/server-test-env/action.yml

@@ -15,6 +15,8 @@ runs:
 
     - name: Run init-db script
       shell: bash
+      env:
+        NODE_ENV: test
       run: |
         yarn workspace @affine/server exec prisma generate
         yarn workspace @affine/server exec prisma db push

.github/deployment/self-host/.env.example

@@ -0,0 +1,23 @@
+# select a revision to deploy, available values: stable, beta, canary
+AFFINE_REVISION=stable
+
+# set the port for the server container it will expose the server on
+PORT=3010
+
+# set the host for the server for outgoing links
+# AFFINE_SERVER_HTTPS=true
+# AFFINE_SERVER_HOST=affine.yourdomain.com
+# or 
+# AFFINE_SERVER_EXTERNAL_URL=https://affine.yourdomain.com
+
+# position of the database data to persist
+DB_DATA_LOCATION=./postgres
+# position of the upload data(images, files, etc.) to persist
+UPLOAD_LOCATION=./storage
+# position of the configuration files to persist
+CONFIG_LOCATION=./config
+
+# database credentials
+DB_USERNAME=affine
+DB_PASSWORD=
+DB_DATABASE=affine

.github/deployment/self-host/compose.yaml

@@ -1,60 +1,66 @@
+name: affine
 services:
   affine:
-    image: ghcr.io/toeverything/affine-graphql:stable
-    container_name: affine_selfhosted
-    command:
-      ['sh', '-c', 'node ./scripts/self-host-predeploy && node ./dist/index.js']
+    image: ghcr.io/toeverything/affine-graphql:${AFFINE_REVISION:-stable}
+    container_name: affine_server
     ports:
-      - '3010:3010'
-      - '5555:5555'
+      - '${PORT:-3010}:3010'
     depends_on:
       redis:
         condition: service_healthy
       postgres:
         condition: service_healthy
+      affine_migration:
+        condition: service_completed_successfully
     volumes:
       # custom configurations
-      - ~/.affine/self-host/config:/root/.affine/config
-      # blob storage
-      - ~/.affine/self-host/storage:/root/.affine/storage
-    logging:
-      driver: 'json-file'
-      options:
-        max-size: '1000m'
+      - ${UPLOAD_LOCATION}:/root/.affine/storage
+      - ${CONFIG_LOCATION}:/root/.affine/config
+    env_file:
+      - .env
+    environment:
+      - DATABASE_URL=postgresql://${DB_USERNAME}:${DB_PASSWORD}@postgres:5432/${DB_DATABASE:-affine}
     restart: unless-stopped
+
+  affine_migration:
+    image: ghcr.io/toeverything/affine-graphql:${AFFINE_VERSION:-stable}
+    container_name: affine_migration_job
+    volumes:
+      # custom configurations
+      - ${UPLOAD_LOCATION}:/root/.affine/storage
+      - ${CONFIG_LOCATION}:/root/.affine/config
+    command: ['sh', '-c', 'node ./scripts/self-host-predeploy.js']
+    env_file:
+      - .env
     environment:
-      - NODE_OPTIONS="--import=./scripts/register.js"
-      - AFFINE_CONFIG_PATH=/root/.affine/config
-      - REDIS_SERVER_HOST=redis
-      - DATABASE_URL=postgres://affine:affine@postgres:5432/affine
-      - NODE_ENV=production
-      # Telemetry allows us to collect data on how you use the affine. This data will helps us improve the app and provide better features.
-      # Uncomment next line if you wish to quit telemetry.
-      # - TELEMETRY_ENABLE=false
+      - DATABASE_URL=postgresql://${DB_USERNAME}:${DB_PASSWORD}@postgres:5432/${DB_DATABASE:-affine}
+    depends_on:
+      postgres:
+        condition: service_healthy
+
   redis:
     image: redis
-    container_name: affine_redis
-    restart: unless-stopped
-    volumes:
-      - ~/.affine/self-host/redis:/data
+    container_name: redis
     healthcheck:
       test: ['CMD', 'redis-cli', '--raw', 'incr', 'ping']
-      interval: 10s
-      timeout: 5s
-      retries: 5
+    restart: unless-stopped
+
   postgres:
     image: postgres:16
-    container_name: affine_postgres
-    restart: unless-stopped
+    container_name: postgres
     volumes:
-      - ~/.affine/self-host/postgres:/var/lib/postgresql/data
-    healthcheck:
-      test: ['CMD-SHELL', 'pg_isready -U affine']
-      interval: 10s
-      timeout: 5s
-      retries: 5
+      - ${DB_DATA_LOCATION}:/var/lib/postgresql/data
     environment:
-      POSTGRES_USER: affine
-      POSTGRES_PASSWORD: affine
-      POSTGRES_DB: affine
-      PGDATA: /var/lib/postgresql/data/pgdata
+      POSTGRES_USER: ${DB_USERNAME}
+      POSTGRES_PASSWORD: ${DB_PASSWORD}
+      POSTGRES_DB: ${DB_DATABASE:-affine}
+      POSTGRES_INITDB_ARGS: '--data-checksums'
+      # you better set a password for you database
+      # or you may add 'POSTGRES_HOST_AUTH_METHOD=trust' to ignore postgres security policy
+      POSTGRES_HOST_AUTH_METHOD: trust
+    healthcheck:
+      test: ['CMD-SHELL', 'pg_isready']
+      interval: 1m
+      start_interval: 10s
+      start_period: 1m
+    restart: unless-stopped

.github/workflows/release-desktop.yml

@@ -415,35 +415,39 @@ jobs:
         uses: actions/download-artifact@v4
         with:
           name: affine-darwin-x64-builds
-          path: ./
+          path: ./release
       - name: Download Artifacts (macos-arm64)
         uses: actions/download-artifact@v4
         with:
           name: affine-darwin-arm64-builds
-          path: ./
+          path: ./release
       - name: Download Artifacts (windows-x64)
         uses: actions/download-artifact@v4
         with:
           name: affine-win32-x64-builds
-          path: ./
+          path: ./release
       - name: Download Artifacts (windows-arm64)
         uses: actions/download-artifact@v4
         with:
           name: affine-win32-arm64-builds
-          path: ./
+          path: ./release
       - name: Download Artifacts (linux-x64)
         uses: actions/download-artifact@v4
         with:
           name: affine-linux-x64-builds
-          path: ./
+          path: ./release
       - uses: actions/setup-node@v4
         with:
           node-version: 20
       - name: Generate Release yml
         run: |
-          node ./packages/frontend/apps/electron/scripts/generate-yml.js
+          node ./scripts/generate-release-yml.js
         env:
           RELEASE_VERSION: ${{ needs.before-make.outputs.RELEASE_VERSION }}
+      - name: Copy Selfhost Release Files
+        run: |
+          cp ./.github/deployment/self-host/compose.yaml ./release/docker-compose.yaml
+          cp ./.github/deployment/self-host/.env.example ./release/.env.example
       - name: Create Release Draft
         if: ${{ github.ref_type == 'tag' }}
         uses: softprops/action-gh-release@v2
@@ -452,16 +456,7 @@ jobs:
           body: ''
           draft: ${{ github.event.inputs.is-draft }}
           prerelease: ${{ github.event.inputs.is-pre-release }}
-          files: |
-            ./VERSION
-            ./*.zip
-            ./*.dmg
-            ./*.exe
-            ./*.appimage
-            ./*.deb
-            ./*.flatpak
-            ./*.apk
-            ./*.yml
+          files: ./release/*
       - name: Create Nightly Release Draft
         if: ${{ github.ref_type == 'branch' }}
         uses: softprops/action-gh-release@v2
@@ -476,13 +471,4 @@ jobs:
           body: ''
           draft: false
           prerelease: true
-          files: |
-            ./VERSION
-            ./*.zip
-            ./*.dmg
-            ./*.exe
-            ./*.appimage
-            ./*.deb
-            ./*.apk
-            ./*.flatpak
-            ./*.yml
+          files: ./release/*

packages/backend/server/.dockerignore

@@ -0,0 +1,2 @@
+# required DATABASE_URL affects app start
+schema.prisma

packages/backend/server/package.json

@@ -165,6 +165,7 @@
       "*.gen.*"
     ],
     "env": {
+      "NODE_ENV": "development",
       "AFFINE_SERVER_EXTERNAL_URL": "http://localhost:8080",
       "TS_NODE_TRANSPILE_ONLY": true,
       "TS_NODE_PROJECT": "./tsconfig.json",

packages/backend/server/scripts/self-host-predeploy.js

@@ -3,61 +3,47 @@ import { generateKeyPairSync } from 'node:crypto';
 import fs from 'node:fs';
 import path from 'node:path';
 
-import { parse } from 'dotenv';
-
 const SELF_HOST_CONFIG_DIR = '/root/.affine/config';
-/**
- * @type {Array<{ from: string; to?: string, modifier?: (content: string): string }>}
- */
-const configFiles = [
-  { from: './.env.example', to: '.env' },
-  { from: './dist/config/affine.js', modifier: configCleaner },
-  { from: './dist/config/affine.env.js', modifier: configCleaner },
-];
 
-function configCleaner(content) {
+function generateConfigFile() {
+  const content = fs.readFileSync('./dist/config/affine.js', 'utf-8');
   return content.replace(
     /(^\/\/#.*$)|(^\/\/\s+TODO.*$)|("use\sstrict";?)|(^.*eslint-disable.*$)/gm,
     ''
   );
 }
 
+function generatePrivateKey() {
+  const key = generateKeyPairSync('ec', {
+    namedCurve: 'prime256v1',
+  }).privateKey.export({
+    type: 'sec1',
+    format: 'pem',
+  });
+
+  if (key instanceof Buffer) {
+    return key.toString('utf-8');
+  }
+
+  return key;
+}
+
+/**
+ * @type {Array<{ to: string; generator: () => string }>}
+ */
+const configFiles = [
+  { to: 'affine.js', generator: generateConfigFile },
+  { to: 'private.key', generator: generatePrivateKey },
+];
+
 function prepare() {
   fs.mkdirSync(SELF_HOST_CONFIG_DIR, { recursive: true });
 
-  for (const { from, to, modifier } of configFiles) {
-    const targetFileName = to ?? path.parse(from).base;
-    const targetFilePath = path.join(SELF_HOST_CONFIG_DIR, targetFileName);
+  for (const { to, generator } of configFiles) {
+    const targetFilePath = path.join(SELF_HOST_CONFIG_DIR, to);
     if (!fs.existsSync(targetFilePath)) {
       console.log(`creating config file [${targetFilePath}].`);
-      if (modifier) {
-        const content = fs.readFileSync(from, 'utf-8');
-        fs.writeFileSync(targetFilePath, modifier(content), 'utf-8');
-      } else {
-        fs.cpSync(from, targetFilePath, {
-          force: false,
-        });
-      }
-    }
-
-    // make the default .env
-    if (to === '.env') {
-      const dotenvFile = fs.readFileSync(targetFilePath, 'utf-8');
-      const envs = parse(dotenvFile);
-      // generate a new private key
-      if (!envs.AFFINE_PRIVATE_KEY) {
-        const privateKey = generateKeyPairSync('ec', {
-          namedCurve: 'prime256v1',
-        }).privateKey.export({
-          type: 'sec1',
-          format: 'pem',
-        });
-
-        fs.writeFileSync(
-          targetFilePath,
-          `AFFINE_PRIVATE_KEY=${privateKey}\n` + dotenvFile
-        );
-      }
+      fs.writeFileSync(targetFilePath, generator(), 'utf-8');
     }
   }
 }

packages/backend/server/src/config/affine.env.ts

@@ -12,7 +12,7 @@ AFFiNE.ENV_MAP = {
   MAILER_PASSWORD: 'mailer.auth.pass',
   MAILER_SENDER: 'mailer.from.address',
   MAILER_SECURE: ['mailer.secure', 'boolean'],
-  DATABASE_URL: 'database.datasourceUrl',
+  DATABASE_URL: 'prisma.datasourceUrl',
   OAUTH_GOOGLE_CLIENT_ID: 'plugins.oauth.providers.google.clientId',
   OAUTH_GOOGLE_CLIENT_SECRET: 'plugins.oauth.providers.google.clientSecret',
   OAUTH_GITHUB_CLIENT_ID: 'plugins.oauth.providers.github.clientId',

packages/backend/server/src/config/affine.self.ts

@@ -17,6 +17,7 @@
 // ====================================================================================
 const env = process.env;
 
+AFFiNE.serverName = 'AFFiNE Cloud';
 AFFiNE.metrics.enabled = !AFFiNE.node.test;
 
 if (env.R2_OBJECT_STORAGE_ACCOUNT_ID) {

packages/backend/server/src/config/affine.ts

@@ -8,22 +8,12 @@
 // Any changes in this file won't take effect before server restarted.
 //
 //
-// > Configurations merge order
-//   1. load environment variables (`.env` if provided, and from system)
-//   2. load `src/fundamentals/config/default.ts` for all default settings
-//   3. apply `./affine.ts` patches (this file)
-//   4. apply `./affine.env.ts` patches
-//
-//
 // ###############################################################
 // ##                       General settings                    ##
 // ###############################################################
 //
-// /* The unique identity of the server */
-// AFFiNE.serverId = 'some-randome-uuid';
-//
 // /* The name of AFFiNE Server, may show on the UI */
-// AFFiNE.serverName = 'Your Cool AFFiNE Selfhosted Cloud';
+AFFiNE.serverName = 'My Selfhosted AFFiNE Cloud';
 //
 // /* Whether the server is deployed behind a HTTPS proxied environment */
 AFFiNE.server.https = false;

packages/backend/server/src/core/config/resolver.ts

@@ -261,7 +261,7 @@ export class ServerServiceConfigResolver {
   }
 
   database(): ServerDatabaseConfig {
-    const url = new URL(this.config.database.datasourceUrl);
+    const url = new URL(this.config.prisma.datasourceUrl);
 
     return {
       host: url.hostname,

packages/backend/server/src/fundamentals/config/default.ts

@@ -14,7 +14,7 @@ import { readEnv } from './env';
 import { defaultStartupConfig } from './register';
 
 function getPredefinedAFFiNEConfig(): PreDefinedAFFiNEConfig {
-  const NODE_ENV = readEnv<NODE_ENV>('NODE_ENV', 'development', [
+  const NODE_ENV = readEnv<NODE_ENV>('NODE_ENV', 'production', [
     'development',
     'test',
     'production',

packages/backend/server/src/fundamentals/index.ts

@@ -29,7 +29,7 @@ export {
   mapSseError,
   OptionalModule,
 } from './nestjs';
-export type { PrismaTransaction } from './prisma';
+export { type PrismaTransaction } from './prisma';
 export * from './storage';
 export { type StorageProvider, StorageProviderFactory } from './storage';
 export { CloudThrottlerGuard, SkipThrottle, Throttle } from './throttler';