SEO_iamge_renamer_starting_.../packages/worker/docker-compose.yml

177 lines
4.2 KiB
YAML
Raw Normal View History

feat(worker): complete production-ready worker service implementation This commit delivers the complete, production-ready worker service that was identified as missing from the audit. The implementation includes: ## Core Components Implemented: ### 1. Background Job Queue System ✅ - Progress tracking with Redis and WebSocket broadcasting - Intelligent retry handler with exponential backoff strategies - Automated cleanup service with scheduled maintenance - Queue-specific retry policies and failure handling ### 2. Security Integration ✅ - Complete ClamAV virus scanning service with real-time threats detection - File validation and quarantine system - Security incident logging and user flagging - Comprehensive threat signature management ### 3. Database Integration ✅ - Prisma-based database service with connection pooling - Image status tracking and batch management - Security incident recording and user flagging - Health checks and statistics collection ### 4. Monitoring & Observability ✅ - Prometheus metrics collection for all operations - Custom business metrics and performance tracking - Comprehensive health check endpoints (ready/live/detailed) - Resource usage monitoring and alerting ### 5. Production Docker Configuration ✅ - Multi-stage Docker build with Alpine Linux - ClamAV daemon integration and configuration - Security-hardened container with non-root user - Health checks and proper signal handling - Complete docker-compose setup with Redis, MinIO, Prometheus, Grafana ### 6. Configuration & Environment ✅ - Comprehensive environment validation with Joi - Redis integration for progress tracking and caching - Rate limiting and throttling configuration - Logging configuration with Winston and file rotation ## Technical Specifications Met: ✅ **Real AI Integration**: OpenAI GPT-4 Vision + Google Cloud Vision with fallbacks ✅ **Image Processing Pipeline**: Sharp integration with EXIF preservation ✅ **Storage Integration**: MinIO/S3 with temporary file management ✅ **Queue Processing**: BullMQ with Redis, retry logic, and progress tracking ✅ **Security Features**: ClamAV virus scanning with quarantine system ✅ **Monitoring**: Prometheus metrics, health checks, structured logging ✅ **Production Ready**: Docker, Kubernetes compatibility, environment validation ## Integration Points: - Connects with existing API queue system - Uses shared database models and authentication - Integrates with infrastructure components - Provides real-time progress updates via WebSocket This resolves the critical gap identified in the audit and provides a complete, production-ready worker service capable of processing images with real AI vision analysis at scale. 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude <noreply@anthropic.com>
2025-08-05 18:37:04 +02:00
version: '3.8'
services:
worker:
build: .
container_name: seo-worker
restart: unless-stopped
environment:
- NODE_ENV=production
- WORKER_PORT=3002
- HEALTH_CHECK_PORT=8080
# Redis Configuration
- REDIS_HOST=redis
- REDIS_PORT=6379
- REDIS_PASSWORD=${REDIS_PASSWORD}
- REDIS_DB=0
# Database Configuration
- DATABASE_URL=${DATABASE_URL}
# AI Vision APIs
- OPENAI_API_KEY=${OPENAI_API_KEY}
- GOOGLE_CLOUD_VISION_KEY=${GOOGLE_CLOUD_VISION_KEY}
- VISION_CONFIDENCE_THRESHOLD=0.40
# Storage Configuration
- MINIO_ENDPOINT=minio
- MINIO_PORT=9000
- MINIO_USE_SSL=false
- MINIO_ACCESS_KEY=${MINIO_ACCESS_KEY}
- MINIO_SECRET_KEY=${MINIO_SECRET_KEY}
- MINIO_BUCKET_NAME=seo-images
# Processing Configuration
- MAX_CONCURRENT_JOBS=5
- JOB_TIMEOUT=300000
- RETRY_ATTEMPTS=3
- RETRY_DELAY=2000
# File Processing
- MAX_FILE_SIZE=52428800
- ALLOWED_FILE_TYPES=jpg,jpeg,png,gif,webp
- TEMP_DIR=/tmp/seo-worker
- TEMP_FILE_CLEANUP_INTERVAL=3600000
# Virus Scanning
- VIRUS_SCAN_ENABLED=true
- CLAMAV_HOST=localhost
- CLAMAV_PORT=3310
- CLAMAV_TIMEOUT=30000
# Monitoring
- METRICS_ENABLED=true
- METRICS_PORT=9090
- LOG_LEVEL=info
ports:
- "3002:3002" # Worker API port
- "8080:8080" # Health check port
- "9090:9090" # Metrics port
volumes:
- worker-temp:/tmp/seo-worker
- worker-logs:/app/logs
depends_on:
- redis
- minio
networks:
- worker-network
healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:8080/health"]
interval: 30s
timeout: 10s
retries: 3
start_period: 30s
redis:
image: redis:7-alpine
container_name: seo-redis
restart: unless-stopped
command: redis-server --appendonly yes --requirepass ${REDIS_PASSWORD}
environment:
- REDIS_PASSWORD=${REDIS_PASSWORD}
ports:
- "6379:6379"
volumes:
- redis-data:/data
networks:
- worker-network
healthcheck:
test: ["CMD", "redis-cli", "-a", "${REDIS_PASSWORD}", "ping"]
interval: 30s
timeout: 10s
retries: 3
minio:
image: minio/minio:latest
container_name: seo-minio
restart: unless-stopped
command: server /data --console-address ":9001"
environment:
- MINIO_ROOT_USER=${MINIO_ACCESS_KEY}
- MINIO_ROOT_PASSWORD=${MINIO_SECRET_KEY}
ports:
- "9000:9000" # MinIO API
- "9001:9001" # MinIO Console
volumes:
- minio-data:/data
networks:
- worker-network
healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
interval: 30s
timeout: 10s
retries: 3
# Optional: Prometheus for metrics collection
prometheus:
image: prom/prometheus:latest
container_name: seo-prometheus
restart: unless-stopped
command:
- '--config.file=/etc/prometheus/prometheus.yml'
- '--storage.tsdb.path=/prometheus'
- '--web.console.libraries=/etc/prometheus/console_libraries'
- '--web.console.templates=/etc/prometheus/consoles'
- '--storage.tsdb.retention.time=200h'
- '--web.enable-lifecycle'
ports:
- "9091:9090"
volumes:
- ./prometheus.yml:/etc/prometheus/prometheus.yml:ro
- prometheus-data:/prometheus
networks:
- worker-network
depends_on:
- worker
# Optional: Grafana for metrics visualization
grafana:
image: grafana/grafana:latest
container_name: seo-grafana
restart: unless-stopped
environment:
- GF_SECURITY_ADMIN_USER=admin
- GF_SECURITY_ADMIN_PASSWORD=${GRAFANA_PASSWORD:-admin}
- GF_USERS_ALLOW_SIGN_UP=false
ports:
- "3000:3000"
volumes:
- grafana-data:/var/lib/grafana
networks:
- worker-network
depends_on:
- prometheus
volumes:
worker-temp:
driver: local
worker-logs:
driver: local
redis-data:
driver: local
minio-data:
driver: local
prometheus-data:
driver: local
grafana-data:
driver: local
networks:
worker-network:
driver: bridge