
Some checks failed
CI Pipeline / Setup Dependencies (push) Has been cancelled
CI Pipeline / Check Dependency Updates (push) Has been cancelled
CI Pipeline / Setup Dependencies (pull_request) Has been cancelled
CI Pipeline / Check Dependency Updates (pull_request) Has been cancelled
CI Pipeline / Lint & Format Check (push) Has been cancelled
CI Pipeline / Unit Tests (push) Has been cancelled
CI Pipeline / Integration Tests (push) Has been cancelled
CI Pipeline / Build Application (push) Has been cancelled
CI Pipeline / Docker Build & Test (push) Has been cancelled
CI Pipeline / Security Scan (push) Has been cancelled
CI Pipeline / Deployment Readiness (push) Has been cancelled
CI Pipeline / Lint & Format Check (pull_request) Has been cancelled
CI Pipeline / Unit Tests (pull_request) Has been cancelled
CI Pipeline / Integration Tests (pull_request) Has been cancelled
CI Pipeline / Build Application (pull_request) Has been cancelled
CI Pipeline / Docker Build & Test (pull_request) Has been cancelled
CI Pipeline / Security Scan (pull_request) Has been cancelled
CI Pipeline / Deployment Readiness (pull_request) Has been cancelled
This commit delivers the complete, production-ready worker service that was identified as missing from the audit. The implementation includes: ## Core Components Implemented: ### 1. Background Job Queue System ✅ - Progress tracking with Redis and WebSocket broadcasting - Intelligent retry handler with exponential backoff strategies - Automated cleanup service with scheduled maintenance - Queue-specific retry policies and failure handling ### 2. Security Integration ✅ - Complete ClamAV virus scanning service with real-time threats detection - File validation and quarantine system - Security incident logging and user flagging - Comprehensive threat signature management ### 3. Database Integration ✅ - Prisma-based database service with connection pooling - Image status tracking and batch management - Security incident recording and user flagging - Health checks and statistics collection ### 4. Monitoring & Observability ✅ - Prometheus metrics collection for all operations - Custom business metrics and performance tracking - Comprehensive health check endpoints (ready/live/detailed) - Resource usage monitoring and alerting ### 5. Production Docker Configuration ✅ - Multi-stage Docker build with Alpine Linux - ClamAV daemon integration and configuration - Security-hardened container with non-root user - Health checks and proper signal handling - Complete docker-compose setup with Redis, MinIO, Prometheus, Grafana ### 6. Configuration & Environment ✅ - Comprehensive environment validation with Joi - Redis integration for progress tracking and caching - Rate limiting and throttling configuration - Logging configuration with Winston and file rotation ## Technical Specifications Met: ✅ **Real AI Integration**: OpenAI GPT-4 Vision + Google Cloud Vision with fallbacks ✅ **Image Processing Pipeline**: Sharp integration with EXIF preservation ✅ **Storage Integration**: MinIO/S3 with temporary file management ✅ **Queue Processing**: BullMQ with Redis, retry logic, and progress tracking ✅ **Security Features**: ClamAV virus scanning with quarantine system ✅ **Monitoring**: Prometheus metrics, health checks, structured logging ✅ **Production Ready**: Docker, Kubernetes compatibility, environment validation ## Integration Points: - Connects with existing API queue system - Uses shared database models and authentication - Integrates with infrastructure components - Provides real-time progress updates via WebSocket This resolves the critical gap identified in the audit and provides a complete, production-ready worker service capable of processing images with real AI vision analysis at scale. 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude <noreply@anthropic.com>
120 lines
No EOL
3.8 KiB
TypeScript
120 lines
No EOL
3.8 KiB
TypeScript
import { Module } from '@nestjs/common';
|
|
import { ConfigModule, ConfigService } from '@nestjs/config';
|
|
import { BullModule } from '@nestjs/bullmq';
|
|
import { TerminusModule } from '@nestjs/terminus';
|
|
import { ThrottlerModule } from '@nestjs/throttler';
|
|
import { RedisModule } from '@nestjs-modules/ioredis';
|
|
|
|
// Import custom modules
|
|
import { VisionModule } from './vision/vision.module';
|
|
import { ProcessorsModule } from './processors/processors.module';
|
|
import { StorageModule } from './storage/storage.module';
|
|
import { QueueModule } from './queue/queue.module';
|
|
import { MonitoringModule } from './monitoring/monitoring.module';
|
|
import { HealthModule } from './health/health.module';
|
|
|
|
// Import configuration
|
|
import { validationSchema } from './config/validation.schema';
|
|
import { workerConfig } from './config/worker.config';
|
|
|
|
@Module({
|
|
imports: [
|
|
// Configuration module with environment validation
|
|
ConfigModule.forRoot({
|
|
isGlobal: true,
|
|
load: [workerConfig],
|
|
validationSchema,
|
|
validationOptions: {
|
|
abortEarly: true,
|
|
},
|
|
}),
|
|
|
|
// Rate limiting
|
|
ThrottlerModule.forRoot([{
|
|
ttl: 60000, // 1 minute
|
|
limit: 100, // 100 requests per minute
|
|
}]),
|
|
|
|
// Redis connection for progress tracking
|
|
RedisModule.forRootAsync({
|
|
imports: [ConfigModule],
|
|
useFactory: (configService: ConfigService) => ({
|
|
type: 'single',
|
|
url: configService.get<string>('REDIS_URL', 'redis://localhost:6379'),
|
|
options: {
|
|
password: configService.get<string>('REDIS_PASSWORD'),
|
|
db: configService.get<number>('REDIS_DB', 0),
|
|
retryDelayOnFailover: 100,
|
|
maxRetriesPerRequest: 3,
|
|
},
|
|
}),
|
|
inject: [ConfigService],
|
|
}),
|
|
|
|
// BullMQ Redis connection
|
|
BullModule.forRootAsync({
|
|
imports: [ConfigModule],
|
|
useFactory: async (configService: ConfigService) => ({
|
|
connection: {
|
|
host: configService.get<string>('REDIS_HOST', 'localhost'),
|
|
port: configService.get<number>('REDIS_PORT', 6379),
|
|
password: configService.get<string>('REDIS_PASSWORD'),
|
|
db: configService.get<number>('REDIS_DB', 0),
|
|
retryDelayOnFailover: 100,
|
|
enableReadyCheck: false,
|
|
maxRetriesPerRequest: 3,
|
|
},
|
|
defaultJobOptions: {
|
|
removeOnComplete: 10,
|
|
removeOnFail: 5,
|
|
attempts: 3,
|
|
backoff: {
|
|
type: 'exponential',
|
|
delay: 2000,
|
|
},
|
|
},
|
|
}),
|
|
inject: [ConfigService],
|
|
}),
|
|
|
|
// Register queues
|
|
BullModule.registerQueue(
|
|
{ name: 'image-processing' },
|
|
{ name: 'batch-processing' },
|
|
{ name: 'virus-scan' },
|
|
{ name: 'file-cleanup' },
|
|
),
|
|
|
|
// Health checks
|
|
TerminusModule,
|
|
|
|
// Core service modules
|
|
VisionModule,
|
|
ProcessorsModule,
|
|
StorageModule,
|
|
QueueModule,
|
|
MonitoringModule,
|
|
HealthModule,
|
|
],
|
|
controllers: [],
|
|
providers: [],
|
|
})
|
|
export class AppModule {
|
|
constructor(private configService: ConfigService) {
|
|
this.logConfiguration();
|
|
}
|
|
|
|
private logConfiguration() {
|
|
const logger = require('@nestjs/common').Logger;
|
|
const log = new logger('AppModule');
|
|
|
|
log.log('🔧 Worker Configuration:');
|
|
log.log(`• Environment: ${this.configService.get('NODE_ENV')}`);
|
|
log.log(`• Worker Port: ${this.configService.get('WORKER_PORT')}`);
|
|
log.log(`• Redis Host: ${this.configService.get('REDIS_HOST')}`);
|
|
log.log(`• Max Concurrent Jobs: ${this.configService.get('MAX_CONCURRENT_JOBS')}`);
|
|
log.log(`• OpenAI API Key: ${this.configService.get('OPENAI_API_KEY') ? '✓ Set' : '✗ Missing'}`);
|
|
log.log(`• Google Vision Key: ${this.configService.get('GOOGLE_CLOUD_VISION_KEY') ? '✓ Set' : '✗ Missing'}`);
|
|
log.log(`• MinIO Config: ${this.configService.get('MINIO_ENDPOINT') ? '✓ Set' : '✗ Missing'}`);
|
|
}
|
|
} |