From 0776b8dbab9d139d0b2bfc4f99288206ebd41d9b Mon Sep 17 00:00:00 2001 From: Manoj K Date: Fri, 18 Jul 2025 12:26:48 +0530 Subject: [PATCH] feat: update Docker configurations for trigger service --- docker-compose.aws.yaml | 3 + trigger/.env.example | 1 + trigger/docker-compose.aws.yaml | 280 ++++++++++++++++++++++++++++++++ trigger/docker-compose.yaml | 13 +- 4 files changed, 294 insertions(+), 3 deletions(-) create mode 100644 trigger/docker-compose.aws.yaml diff --git a/docker-compose.aws.yaml b/docker-compose.aws.yaml index bfce1a7..6f00457 100644 --- a/docker-compose.aws.yaml +++ b/docker-compose.aws.yaml @@ -26,6 +26,9 @@ services: - OLLAMA_URL=${OLLAMA_URL} - EMBEDDING_MODEL=${EMBEDDING_MODEL} - MODEL=${MODEL} + - TRIGGER_PROJECT_ID=${TRIGGER_PROJECT_ID} + - TRIGGER_API_URL=${TRIGGER_API_URL} + - TRIGGER_SECRET_KEY=${TRIGGER_SECRET_KEY} ports: - "3033:3000" depends_on: diff --git a/trigger/.env.example b/trigger/.env.example index 67afd24..35ebb4b 100644 --- a/trigger/.env.example +++ b/trigger/.env.example @@ -39,6 +39,7 @@ DB_SCHEMA=sigma # POSTGRES_DB=postgres DATABASE_URL=postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${DB_HOST}:${DB_PORT}/${TRIGGER_DB}?schema=public&sslmode=disable DIRECT_URL=postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${DB_HOST}:${DB_PORT}/${TRIGGER_DB}?schema=public&sslmode=disable +ELECTRIC_DATABASE_URL=postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${DB_HOST}/${TRIGGER_DB} # Trigger image tag # - This is the version of the webapp and worker images to use, they should be locked to a specific version in production diff --git a/trigger/docker-compose.aws.yaml b/trigger/docker-compose.aws.yaml new file mode 100644 index 0000000..37c1478 --- /dev/null +++ b/trigger/docker-compose.aws.yaml @@ -0,0 +1,280 @@ +x-logging: &logging-config + driver: ${LOGGING_DRIVER:-local} + options: + max-size: ${LOGGING_MAX_SIZE:-20m} + max-file: ${LOGGING_MAX_FILES:-5} + compress: ${LOGGING_COMPRESS:-true} + +services: + webapp: + container_name: trigger-webapp + image: ghcr.io/triggerdotdev/trigger.dev:${TRIGGER_IMAGE_TAG:-v4-beta} + restart: ${RESTART_POLICY:-unless-stopped} + logging: *logging-config + ports: + - ${WEBAPP_PUBLISH_IP:-0.0.0.0}:8030:3000 + depends_on: + - clickhouse + networks: + - webapp + - supervisor + volumes: + - type: bind + source: /efs/trigger/shared + target: /home/node/shared + # Only needed for bootstrap + user: root + # Only needed for bootstrap + command: sh -c "chown -R node:node /home/node/shared && exec ./scripts/entrypoint.sh" + healthcheck: + test: + [ + "CMD", + "node", + "-e", + "http.get('http://localhost:3000/healthcheck', res => process.exit(res.statusCode === 200 ? 0 : 1)).on('error', () => process.exit(1))", + ] + interval: 30s + timeout: 10s + retries: 5 + start_period: 10s + environment: + APP_ORIGIN: ${APP_ORIGIN:-http://localhost:8030} + LOGIN_ORIGIN: ${LOGIN_ORIGIN:-http://localhost:8030} + API_ORIGIN: ${API_ORIGIN:-http://localhost:8030} + ELECTRIC_ORIGIN: http://electric:3000 + DATABASE_URL: ${DATABASE_URL:-postgresql://postgres:postgres@postgres:5432/main?schema=public&sslmode=disable} + DIRECT_URL: ${DIRECT_URL:-postgresql://postgres:postgres@postgres:5432/main?schema=public&sslmode=disable} + SESSION_SECRET: ${SESSION_SECRET} + MAGIC_LINK_SECRET: ${MAGIC_LINK_SECRET} + ENCRYPTION_KEY: ${ENCRYPTION_KEY} + MANAGED_WORKER_SECRET: ${MANAGED_WORKER_SECRET} + REDIS_HOST: trigger-redis + REDIS_PORT: 6379 + REDIS_TLS_DISABLED: true + APP_LOG_LEVEL: info + DEV_OTEL_EXPORTER_OTLP_ENDPOINT: ${DEV_OTEL_EXPORTER_OTLP_ENDPOINT:-http://localhost:8030/otel} + DEPLOY_REGISTRY_HOST: ${DOCKER_REGISTRY_URL:-localhost:5000} + OBJECT_STORE_BASE_URL: ${OBJECT_STORE_BASE_URL:-http://minio:9000} + OBJECT_STORE_ACCESS_KEY_ID: ${OBJECT_STORE_ACCESS_KEY_ID} + OBJECT_STORE_SECRET_ACCESS_KEY: ${OBJECT_STORE_SECRET_ACCESS_KEY} + GRACEFUL_SHUTDOWN_TIMEOUT: 1000 + # Bootstrap - this will automatically set up a worker group for you + # This will NOT work for split deployments + TRIGGER_BOOTSTRAP_ENABLED: 1 + TRIGGER_BOOTSTRAP_WORKER_GROUP_NAME: bootstrap + TRIGGER_BOOTSTRAP_WORKER_TOKEN_PATH: /home/node/shared/worker_token + # ClickHouse configuration + CLICKHOUSE_URL: ${CLICKHOUSE_URL:-http://default:password@clickhouse:8123?secure=false} + CLICKHOUSE_LOG_LEVEL: ${CLICKHOUSE_LOG_LEVEL:-info} + # Run replication + RUN_REPLICATION_ENABLED: ${RUN_REPLICATION_ENABLED:-1} + RUN_REPLICATION_CLICKHOUSE_URL: ${RUN_REPLICATION_CLICKHOUSE_URL:-http://default:password@clickhouse:8123} + RUN_REPLICATION_LOG_LEVEL: ${RUN_REPLICATION_LOG_LEVEL:-info} + # Limits + # TASK_PAYLOAD_OFFLOAD_THRESHOLD: 524288 # 512KB + # TASK_PAYLOAD_MAXIMUM_SIZE: 3145728 # 3MB + # BATCH_TASK_PAYLOAD_MAXIMUM_SIZE: 1000000 # 1MB + # TASK_RUN_METADATA_MAXIMUM_SIZE: 262144 # 256KB + # DEFAULT_ENV_EXECUTION_CONCURRENCY_LIMIT: 100 + # DEFAULT_ORG_EXECUTION_CONCURRENCY_LIMIT: 100 + # Internal OTEL configuration + INTERNAL_OTEL_TRACE_LOGGING_ENABLED: ${INTERNAL_OTEL_TRACE_LOGGING_ENABLED:-0} + + electric: + container_name: trigger-electric + image: electricsql/electric:${ELECTRIC_IMAGE_TAG:-1.0.10} + restart: ${RESTART_POLICY:-unless-stopped} + logging: *logging-config + networks: + - webapp + environment: + DATABASE_URL: ${ELECTRIC_DATABASE_URL:-postgresql://postgres:postgres@postgres:5432/main?schema=public&sslmode=disable} + ELECTRIC_INSECURE: true + ELECTRIC_USAGE_REPORTING: false + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:3000/v1/health"] + interval: 10s + timeout: 5s + retries: 5 + start_period: 10s + + clickhouse: + container_name: trigger-clickhouse + image: bitnami/clickhouse:${CLICKHOUSE_IMAGE_TAG:-latest} + restart: ${RESTART_POLICY:-unless-stopped} + logging: *logging-config + ports: + - ${CLICKHOUSE_PUBLISH_IP:-127.0.0.1}:9123:8123 + - ${CLICKHOUSE_PUBLISH_IP:-127.0.0.1}:9090:9000 + environment: + CLICKHOUSE_ADMIN_USER: ${CLICKHOUSE_USER:-default} + CLICKHOUSE_ADMIN_PASSWORD: ${CLICKHOUSE_PASSWORD:-password} + volumes: + - type: bind + source: /efs/trigger/clickhouse + target: /bitnami/clickhouse + - ../clickhouse/override.xml:/bitnami/clickhouse/etc/config.d/override.xml:ro + networks: + - webapp + healthcheck: + test: + [ + "CMD", + "clickhouse-client", + "--host", + "localhost", + "--port", + "9000", + "--user", + "default", + "--password", + "password", + "--query", + "SELECT 1", + ] + interval: 5s + timeout: 5s + retries: 5 + start_period: 10s + + registry: + container_name: trigger-registry + image: registry:${REGISTRY_IMAGE_TAG:-2} + restart: ${RESTART_POLICY:-unless-stopped} + logging: *logging-config + ports: + - ${REGISTRY_PUBLISH_IP:-127.0.0.1}:5000:5000 + networks: + - webapp + volumes: + # registry-user:very-secure-indeed + - ../registry/auth.htpasswd:/auth/htpasswd:ro + environment: + REGISTRY_AUTH: htpasswd + REGISTRY_AUTH_HTPASSWD_REALM: Registry Realm + REGISTRY_AUTH_HTPASSWD_PATH: /auth/htpasswd + healthcheck: + test: ["CMD", "wget", "--spider", "-q", "http://localhost:5000/"] + interval: 10s + timeout: 5s + retries: 5 + start_period: 10s + + minio: + container_name: trigger-minio + image: bitnami/minio:${MINIO_IMAGE_TAG:-latest} + restart: ${RESTART_POLICY:-unless-stopped} + logging: *logging-config + ports: + - ${MINIO_PUBLISH_IP:-127.0.0.1}:9000:9000 + - ${MINIO_PUBLISH_IP:-127.0.0.1}:9001:9001 + networks: + - webapp + volumes: + - type: bind + source: /efs/trigger/minio + target: /bitnami/minio/data + environment: + MINIO_ROOT_USER: ${MINIO_ROOT_USER:-admin} + MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:-very-safe-password} + MINIO_DEFAULT_BUCKETS: packets + MINIO_BROWSER: "on" + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] + interval: 5s + timeout: 10s + retries: 5 + start_period: 10s + + # Worker related + supervisor: + container_name: trigger-supervisor + image: ghcr.io/triggerdotdev/supervisor:${TRIGGER_IMAGE_TAG:-v4-beta} + restart: ${RESTART_POLICY:-unless-stopped} + logging: *logging-config + depends_on: + - docker-proxy + networks: + - supervisor + - docker-proxy + - webapp + volumes: + - type: bind + source: /efs/trigger/shared + target: /home/node/shared + # Only needed for bootstrap + user: root + # Only needed for bootstrap + command: sh -c "chown -R node:node /home/node/shared && exec /usr/bin/dumb-init -- pnpm run --filter supervisor start" + environment: + # This needs to match the token of the worker group you want to connect to + # TRIGGER_WORKER_TOKEN: ${TRIGGER_WORKER_TOKEN} + # Use the bootstrap token created by the webapp + TRIGGER_WORKER_TOKEN: file:///home/node/shared/worker_token + MANAGED_WORKER_SECRET: ${MANAGED_WORKER_SECRET} + TRIGGER_API_URL: ${TRIGGER_API_URL:-http://webapp:3000} + OTEL_EXPORTER_OTLP_ENDPOINT: ${OTEL_EXPORTER_OTLP_ENDPOINT:-http://webapp:3000/otel} + TRIGGER_WORKLOAD_API_DOMAIN: supervisor + TRIGGER_WORKLOAD_API_PORT_EXTERNAL: 8020 + # Optional settings + DEBUG: 1 + ENFORCE_MACHINE_PRESETS: 1 + TRIGGER_DEQUEUE_INTERVAL_MS: 1000 + DOCKER_HOST: tcp://docker-proxy:2375 + DOCKER_RUNNER_NETWORKS: webapp,supervisor + DOCKER_REGISTRY_URL: ${DOCKER_REGISTRY_URL:-localhost:5000} + DOCKER_REGISTRY_USERNAME: ${DOCKER_REGISTRY_USERNAME:-} + DOCKER_REGISTRY_PASSWORD: ${DOCKER_REGISTRY_PASSWORD:-} + DOCKER_AUTOREMOVE_EXITED_CONTAINERS: 0 + healthcheck: + test: + [ + "CMD", + "node", + "-e", + "http.get('http://localhost:8020/health', res => process.exit(res.statusCode === 200 ? 0 : 1)).on('error', () => process.exit(1))", + ] + interval: 30s + timeout: 10s + retries: 5 + start_period: 10s + + docker-proxy: + container_name: trigger-docker-proxy + image: tecnativa/docker-socket-proxy:${DOCKER_PROXY_IMAGE_TAG:-latest} + restart: ${RESTART_POLICY:-unless-stopped} + logging: *logging-config + volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro + networks: + - docker-proxy + environment: + - LOG_LEVEL=info + - POST=1 + - CONTAINERS=1 + - IMAGES=1 + - INFO=1 + - NETWORKS=1 + healthcheck: + test: ["CMD", "nc", "-z", "127.0.0.1", "2375"] + interval: 30s + timeout: 5s + retries: 5 + start_period: 5s + + redis: + container_name: trigger-redis + image: redis:7 + ports: + - "6379:6379" + networks: + - webapp + +networks: + docker-proxy: + name: docker-proxy + supervisor: + name: supervisor + webapp: + name: webapp diff --git a/trigger/docker-compose.yaml b/trigger/docker-compose.yaml index c0aac8c..09699d7 100644 --- a/trigger/docker-compose.yaml +++ b/trigger/docker-compose.yaml @@ -7,6 +7,7 @@ x-logging: &logging-config services: webapp: + container_name: trigger-webapp image: ghcr.io/triggerdotdev/trigger.dev:${TRIGGER_IMAGE_TAG:-v4-beta} restart: ${RESTART_POLICY:-unless-stopped} logging: *logging-config @@ -46,7 +47,7 @@ services: MAGIC_LINK_SECRET: ${MAGIC_LINK_SECRET} ENCRYPTION_KEY: ${ENCRYPTION_KEY} MANAGED_WORKER_SECRET: ${MANAGED_WORKER_SECRET} - REDIS_HOST: host.docker.internal + REDIS_HOST: trigger-redis REDIS_PORT: 6379 REDIS_TLS_DISABLED: true APP_LOG_LEVEL: info @@ -79,13 +80,14 @@ services: INTERNAL_OTEL_TRACE_LOGGING_ENABLED: ${INTERNAL_OTEL_TRACE_LOGGING_ENABLED:-0} electric: - image: electricsql/electric:${ELECTRIC_IMAGE_TAG:-1.0.13} + container_name: trigger-electric + image: electricsql/electric:${ELECTRIC_IMAGE_TAG:-1.0.10} restart: ${RESTART_POLICY:-unless-stopped} logging: *logging-config networks: - webapp environment: - DATABASE_URL: ${DATABASE_URL:-postgresql://postgres:postgres@postgres:5432/main?schema=public&sslmode=disable} + DATABASE_URL: ${ELECTRIC_DATABASE_URL:-postgresql://postgres:postgres@postgres:5432/main?schema=public&sslmode=disable} ELECTRIC_INSECURE: true ELECTRIC_USAGE_REPORTING: false healthcheck: @@ -96,6 +98,7 @@ services: start_period: 10s clickhouse: + container_name: trigger-clickhouse image: bitnami/clickhouse:${CLICKHOUSE_IMAGE_TAG:-latest} restart: ${RESTART_POLICY:-unless-stopped} logging: *logging-config @@ -132,6 +135,7 @@ services: start_period: 10s registry: + container_name: trigger-registry image: registry:${REGISTRY_IMAGE_TAG:-2} restart: ${RESTART_POLICY:-unless-stopped} logging: *logging-config @@ -154,6 +158,7 @@ services: start_period: 10s minio: + container_name: trigger-minio image: bitnami/minio:${MINIO_IMAGE_TAG:-latest} restart: ${RESTART_POLICY:-unless-stopped} logging: *logging-config @@ -178,6 +183,7 @@ services: # Worker related supervisor: + container_name: trigger-supervisor image: ghcr.io/triggerdotdev/supervisor:${TRIGGER_IMAGE_TAG:-v4-beta} restart: ${RESTART_POLICY:-unless-stopped} logging: *logging-config @@ -227,6 +233,7 @@ services: start_period: 10s docker-proxy: + container_name: trigger-docker-proxy image: tecnativa/docker-socket-proxy:${DOCKER_PROXY_IMAGE_TAG:-latest} restart: ${RESTART_POLICY:-unless-stopped} logging: *logging-config