From 1f12f0338e44785bd0a416eedd86c581e05025b1 Mon Sep 17 00:00:00 2001 From: vickytechkey Date: Sat, 12 Sep 2026 11:36:56 +0530 Subject: [PATCH] feat: configure docmost deployment for cisystem with s3 wikimedia prefix and forgejo workflows --- .forgejo/workflows/deploy.yml | 62 ++++++ .forgejo/workflows/sync-upstream.yml | 37 ++++ .../integrations/storage/drivers/s3.driver.ts | 31 ++- docker-compose.yml | 41 ++-- patches/s3.driver.js | 193 ++++++++++++++++++ 5 files changed, 333 insertions(+), 31 deletions(-) create mode 100644 .forgejo/workflows/deploy.yml create mode 100644 .forgejo/workflows/sync-upstream.yml create mode 100644 patches/s3.driver.js diff --git a/.forgejo/workflows/deploy.yml b/.forgejo/workflows/deploy.yml new file mode 100644 index 00000000..3e5c856e --- /dev/null +++ b/.forgejo/workflows/deploy.yml @@ -0,0 +1,62 @@ +name: Deploy Docmost Wiki + +on: + push: + branches: + - main + workflow_dispatch: + +jobs: + deploy: + name: Deploy to cisystem + runs-on: ubuntu-latest + steps: + - name: Checkout Code + uses: https://github.com/actions/checkout@v4 + + - name: Deploy via SSH + env: + SSH_KEY: ${{ secrets.CISYSTEM_SSH_KEY }} + HOST: ${{ secrets.CISYSTEM_HOST }} + USER: ${{ secrets.CISYSTEM_USER }} + run: | + if command -v apk >/dev/null 2>&1; then + apk add --no-cache openssh-client curl + elif command -v apt-get >/dev/null 2>&1; then + apt-get update && apt-get install -y openssh-client curl + fi + + mkdir -p ~/.ssh + echo "$SSH_KEY" > ~/.ssh/id_rsa + chmod 600 ~/.ssh/id_rsa + + TARGET_HOST="${HOST:-16.113.106.152}" + TARGET_USER="${USER:-ubuntu}" + + # Create target directories + ssh -o StrictHostKeyChecking=no -i ~/.ssh/id_rsa $TARGET_USER@$TARGET_HOST "mkdir -p /home/ubuntu/docmost/patches" + + # Copy compose configuration and patches + scp -o StrictHostKeyChecking=no -i ~/.ssh/id_rsa docker-compose.yml $TARGET_USER@$TARGET_HOST:/home/ubuntu/docmost/docker-compose.yml + scp -o StrictHostKeyChecking=no -i ~/.ssh/id_rsa patches/s3.driver.js $TARGET_USER@$TARGET_HOST:/home/ubuntu/docmost/patches/s3.driver.js + + # Restart containers cleanly + ssh -o StrictHostKeyChecking=no -i ~/.ssh/id_rsa $TARGET_USER@$TARGET_HOST << 'EOF' + set -e + cd /home/ubuntu/docmost + docker compose pull docmost redis + docker compose up -d --remove-orphans + + echo "Verifying Docmost service health..." + for i in $(seq 1 30); do + STATUS=$(curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1:3001/ || true) + if [ "$STATUS" = "200" ] || [ "$STATUS" = "302" ]; then + echo "Docmost is healthy! HTTP Status: $STATUS" + exit 0 + fi + sleep 2 + done + + echo "Docmost did not respond with 200/302 within 60s. Inspecting logs:" + docker compose logs --tail=50 docmost + EOF diff --git a/.forgejo/workflows/sync-upstream.yml b/.forgejo/workflows/sync-upstream.yml new file mode 100644 index 00000000..ebbf2df0 --- /dev/null +++ b/.forgejo/workflows/sync-upstream.yml @@ -0,0 +1,37 @@ +name: Sync Upstream Updates + +on: + schedule: + - cron: '0 4 * * 0' # Every Sunday at 04:00 UTC + workflow_dispatch: + +jobs: + sync: + name: Sync with upstream Docmost + runs-on: ubuntu-latest + steps: + - name: Checkout Code + uses: https://github.com/actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Fetch and Merge Upstream + run: | + if command -v apk >/dev/null 2>&1; then + apk add --no-cache git + fi + + git config user.name "Forgejo Actions" + git config user.email "actions@pipeline.tradhox.com" + + git remote add upstream https://github.com/docmost/docmost.git || true + git fetch upstream main + + echo "Merging upstream/main into main..." + git merge upstream/main --no-edit -m "chore(sync): sync updates from docmost/docmost" || { + echo "Conflict detected when merging upstream changes. Manual resolution may be required." + exit 1 + } + + echo "Pushing synchronized changes..." + git push origin main diff --git a/apps/server/src/integrations/storage/drivers/s3.driver.ts b/apps/server/src/integrations/storage/drivers/s3.driver.ts index 76d68a75..aa6e3c03 100644 --- a/apps/server/src/integrations/storage/drivers/s3.driver.ts +++ b/apps/server/src/integrations/storage/drivers/s3.driver.ts @@ -16,6 +16,15 @@ import { Upload } from '@aws-sdk/lib-storage'; import { Logger } from '@nestjs/common'; const S3_MAX_SOCKETS = parseInt(process.env.AWS_S3_MAX_SOCKETS) || 200; +const S3_PREFIX = process.env.AWS_S3_PREFIX + ? process.env.AWS_S3_PREFIX.replace(/^\/+/, '').replace(/\/?$/, '/') + : ''; + +function prefixKey(filePath: string): string { + if (!S3_PREFIX) return filePath; + const cleanPath = filePath.replace(/^\/+/, ''); + return cleanPath.startsWith(S3_PREFIX) ? cleanPath : `${S3_PREFIX}${cleanPath}`; +} export class S3Driver implements StorageDriver { private readonly s3Client: S3Client; @@ -40,7 +49,7 @@ export class S3Driver implements StorageDriver { client: this.s3Client, params: { Bucket: this.config.bucket, - Key: filePath, + Key: prefixKey(filePath), Body: file, ContentType: contentType, }, @@ -75,7 +84,7 @@ export class S3Driver implements StorageDriver { client: clientToUse, params: { Bucket: this.config.bucket, - Key: filePath, + Key: prefixKey(filePath), Body: file, ContentType: contentType, }, @@ -98,8 +107,8 @@ export class S3Driver implements StorageDriver { await this.s3Client.send( new CopyObjectCommand({ Bucket: this.config.bucket, - CopySource: `${this.config.bucket}/${fromFilePath}`, - Key: toFilePath, + CopySource: `${this.config.bucket}/${prefixKey(fromFilePath)}`, + Key: prefixKey(toFilePath), }), ); } @@ -112,7 +121,7 @@ export class S3Driver implements StorageDriver { try { const command = new GetObjectCommand({ Bucket: this.config.bucket, - Key: filePath, + Key: prefixKey(filePath), }); const response = await this.s3Client.send(command); @@ -127,7 +136,7 @@ export class S3Driver implements StorageDriver { try { const command = new GetObjectCommand({ Bucket: this.config.bucket, - Key: filePath, + Key: prefixKey(filePath), }); const response = await this.s3Client.send(command); @@ -145,7 +154,7 @@ export class S3Driver implements StorageDriver { try { const command = new GetObjectCommand({ Bucket: this.config.bucket, - Key: filePath, + Key: prefixKey(filePath), Range: `bytes=${range.start}-${range.end}`, }); @@ -161,7 +170,7 @@ export class S3Driver implements StorageDriver { try { const command = new HeadObjectCommand({ Bucket: this.config.bucket, - Key: filePath, + Key: prefixKey(filePath), }); await this.s3Client.send(command); @@ -174,13 +183,13 @@ export class S3Driver implements StorageDriver { } } getUrl(filePath: string): string { - return `${this.config.baseUrl ?? this.config.endpoint}/${this.config.bucket}/${filePath}`; + return `${this.config.baseUrl ?? this.config.endpoint}/${this.config.bucket}/${prefixKey(filePath)}`; } async getSignedUrl(filePath: string, expiresIn: number): Promise { const command = new GetObjectCommand({ Bucket: this.config.bucket, - Key: filePath, + Key: prefixKey(filePath), }); return await getSignedUrl(this.s3Client, command, { expiresIn }); } @@ -189,7 +198,7 @@ export class S3Driver implements StorageDriver { try { const command = new DeleteObjectCommand({ Bucket: this.config.bucket, - Key: filePath, + Key: prefixKey(filePath), }); await this.s3Client.send(command); diff --git a/docker-compose.yml b/docker-compose.yml index 606c229b..0e3327c7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,38 +1,39 @@ services: docmost: image: docmost/docmost:latest + container_name: docmost depends_on: - - db - redis + extra_hosts: + - "host.docker.internal:host-gateway" environment: - APP_URL: 'http://localhost:3000' - APP_SECRET: 'REPLACE_WITH_LONG_SECRET' - DATABASE_URL: 'postgresql://docmost:STRONG_DB_PASSWORD@db:5432/docmost' - REDIS_URL: 'redis://redis:6379' + APP_URL: ${APP_URL:-https://wiki.tradhox.com} + APP_SECRET: ${APP_SECRET} + PORT: 3000 + DATABASE_URL: ${DATABASE_URL:-postgresql://vignesh:vtechnosoft%40123A@host.docker.internal:5432/docmost?schema=public} + REDIS_URL: ${REDIS_URL:-redis://redis:6379} + STORAGE_DRIVER: ${STORAGE_DRIVER:-s3} + AWS_S3_BUCKET: ${AWS_S3_BUCKET:-trypostmedia} + AWS_S3_REGION: ${AWS_S3_REGION:-ap-south-2} + AWS_S3_PREFIX: ${AWS_S3_PREFIX:-wikimedia/} + AWS_S3_URL: ${AWS_S3_URL:-https://d28qhqo5myc2wf.cloudfront.net} + AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID:-AKIA3EDB7OKBXW3M62NR} + AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY:-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} ports: - - "3000:3000" + - "127.0.0.1:3001:3000" restart: unless-stopped volumes: - - docmost:/app/data/storage - - db: - image: postgres:18 - environment: - POSTGRES_DB: docmost - POSTGRES_USER: docmost - POSTGRES_PASSWORD: STRONG_DB_PASSWORD - restart: unless-stopped - volumes: - - db_data:/var/lib/postgresql + - docmost_storage:/app/data/storage + - ./patches/s3.driver.js:/app/apps/server/dist/integrations/storage/drivers/s3.driver.js:ro redis: - image: redis:8 + image: redis:7-alpine + container_name: docmost-redis command: ["redis-server", "--appendonly", "yes", "--maxmemory-policy", "noeviction"] restart: unless-stopped volumes: - redis_data:/data volumes: - docmost: - db_data: + docmost_storage: redis_data: diff --git a/patches/s3.driver.js b/patches/s3.driver.js new file mode 100644 index 00000000..7caa44f9 --- /dev/null +++ b/patches/s3.driver.js @@ -0,0 +1,193 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.S3Driver = void 0; +const interfaces_1 = require("../interfaces"); +const client_s3_1 = require("@aws-sdk/client-s3"); +const storage_utils_1 = require("../storage.utils"); +const s3_request_presigner_1 = require("@aws-sdk/s3-request-presigner"); +const helpers_1 = require("../../../common/helpers"); +const lib_storage_1 = require("@aws-sdk/lib-storage"); +const common_1 = require("@nestjs/common"); +const S3_MAX_SOCKETS = parseInt(process.env.AWS_S3_MAX_SOCKETS) || 200; +const S3_PREFIX = process.env.AWS_S3_PREFIX ? process.env.AWS_S3_PREFIX.replace(/^\/+/, '').replace(/\/?$/, '/') : ''; + +function prefixKey(filePath) { + if (!S3_PREFIX) return filePath; + const cleanPath = filePath.replace(/^\/+/, ''); + return cleanPath.startsWith(S3_PREFIX) ? cleanPath : `${S3_PREFIX}${cleanPath}`; +} + +class S3Driver { + constructor(config) { + this.config = { + ...config, + requestHandler: { + httpAgent: { maxSockets: S3_MAX_SOCKETS }, + httpsAgent: { maxSockets: S3_MAX_SOCKETS }, + }, + }; + this.s3Client = new client_s3_1.S3Client(this.config); + } + async upload(filePath, file) { + try { + const key = prefixKey(filePath); + const contentType = (0, helpers_1.getMimeType)(filePath); + const upload = new lib_storage_1.Upload({ + client: this.s3Client, + params: { + Bucket: this.config.bucket, + Key: key, + Body: file, + ContentType: contentType, + }, + }); + await upload.done(); + } + catch (err) { + common_1.Logger.error(err); + throw new Error(`Failed to upload file: ${err.message}`); + } + } + async uploadStream(filePath, file, options) { + let clientToUse = this.s3Client; + let shouldDestroyClient = false; + if (options?.recreateClient) { + clientToUse = new client_s3_1.S3Client(this.config); + shouldDestroyClient = true; + } + try { + const key = prefixKey(filePath); + const contentType = (0, helpers_1.getMimeType)(filePath); + const upload = new lib_storage_1.Upload({ + client: clientToUse, + params: { + Bucket: this.config.bucket, + Key: key, + Body: file, + ContentType: contentType, + }, + }); + await upload.done(); + } + catch (err) { + common_1.Logger.error(err); + throw new Error(`Failed to upload file: ${err.message}`); + } + finally { + if (shouldDestroyClient && clientToUse) { + clientToUse.destroy(); + } + } + } + async copy(fromFilePath, toFilePath) { + try { + const fromKey = prefixKey(fromFilePath); + const toKey = prefixKey(toFilePath); + if (await this.exists(fromFilePath)) { + await this.s3Client.send(new client_s3_1.CopyObjectCommand({ + Bucket: this.config.bucket, + CopySource: `${this.config.bucket}/${fromKey}`, + Key: toKey, + })); + } + } + catch (err) { + throw new Error(`Failed to copy file: ${err.message}`); + } + } + async read(filePath) { + try { + const key = prefixKey(filePath); + const command = new client_s3_1.GetObjectCommand({ + Bucket: this.config.bucket, + Key: key, + }); + const response = await this.s3Client.send(command); + return (0, storage_utils_1.streamToBuffer)(response.Body); + } + catch (err) { + throw new Error(`Failed to read file from S3: ${err.message}`); + } + } + async readStream(filePath) { + try { + const key = prefixKey(filePath); + const command = new client_s3_1.GetObjectCommand({ + Bucket: this.config.bucket, + Key: key, + }); + const response = await this.s3Client.send(command); + return response.Body; + } + catch (err) { + throw new Error(`Failed to read file from S3: ${err.message}`); + } + } + async readRangeStream(filePath, range) { + try { + const key = prefixKey(filePath); + const command = new client_s3_1.GetObjectCommand({ + Bucket: this.config.bucket, + Key: key, + Range: `bytes=${range.start}-${range.end}`, + }); + const response = await this.s3Client.send(command); + return response.Body; + } + catch (err) { + throw new Error(`Failed to read file from S3: ${err.message}`); + } + } + async exists(filePath) { + try { + const key = prefixKey(filePath); + const command = new client_s3_1.HeadObjectCommand({ + Bucket: this.config.bucket, + Key: key, + }); + await this.s3Client.send(command); + return true; + } + catch (err) { + if (err instanceof client_s3_1.NoSuchKey) { + return false; + } + throw err; + } + } + getUrl(filePath) { + const key = prefixKey(filePath); + return `${this.config.baseUrl ?? this.config.endpoint}/${this.config.bucket}/${key}`; + } + async getSignedUrl(filePath, expiresIn) { + const key = prefixKey(filePath); + const command = new client_s3_1.GetObjectCommand({ + Bucket: this.config.bucket, + Key: key, + }); + return await (0, s3_request_presigner_1.getSignedUrl)(this.s3Client, command, { expiresIn }); + } + async delete(filePath) { + try { + const key = prefixKey(filePath); + const command = new client_s3_1.DeleteObjectCommand({ + Bucket: this.config.bucket, + Key: key, + }); + await this.s3Client.send(command); + } + catch (err) { + throw new Error(`Error deleting file ${filePath} from S3. ${err.message}`); + } + } + getDriver() { + return this.s3Client; + } + getDriverName() { + return interfaces_1.StorageOption.S3; + } + getConfig() { + return this.config; + } +} +exports.S3Driver = S3Driver;