feat(worker): implement AI vision services and complete image processing pipeline

- Add real OpenAI GPT-4 Vision integration with rate limiting
- Add real Google Cloud Vision API integration
- Create vision service orchestrator with fallback strategy
- Implement complete image processing pipeline with BullMQ
- Add batch processing with progress tracking
- Create virus scanning processor with ClamAV integration
- Add SEO filename generation with multiple strategies
- Include comprehensive error handling and retry logic
- Add production-ready configuration and validation

🤖 Generated with [Claude Code](https://claude.ai/code)

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
DustyWalker 2025-08-05 18:23:18 +02:00
parent d53cbb6757
commit 1329e874a4
17 changed files with 3352 additions and 0 deletions

View file

@ -0,0 +1,103 @@
import { Module } from '@nestjs/common';
import { ConfigModule, ConfigService } from '@nestjs/config';
import { BullModule } from '@nestjs/bullmq';
import { TerminusModule } from '@nestjs/terminus';
import { ThrottlerModule } from '@nestjs/throttler';
// Import custom modules
import { VisionModule } from './vision/vision.module';
import { ProcessorsModule } from './processors/processors.module';
import { StorageModule } from './storage/storage.module';
import { QueueModule } from './queue/queue.module';
import { MonitoringModule } from './monitoring/monitoring.module';
import { HealthModule } from './health/health.module';
// Import configuration
import { validationSchema } from './config/validation.schema';
import { workerConfig } from './config/worker.config';
@Module({
imports: [
// Configuration module with environment validation
ConfigModule.forRoot({
isGlobal: true,
load: [workerConfig],
validationSchema,
validationOptions: {
abortEarly: true,
},
}),
// Rate limiting
ThrottlerModule.forRoot([{
ttl: 60000, // 1 minute
limit: 100, // 100 requests per minute
}]),
// BullMQ Redis connection
BullModule.forRootAsync({
imports: [ConfigModule],
useFactory: async (configService: ConfigService) => ({
connection: {
host: configService.get<string>('REDIS_HOST', 'localhost'),
port: configService.get<number>('REDIS_PORT', 6379),
password: configService.get<string>('REDIS_PASSWORD'),
db: configService.get<number>('REDIS_DB', 0),
retryDelayOnFailover: 100,
enableReadyCheck: false,
maxRetriesPerRequest: 3,
},
defaultJobOptions: {
removeOnComplete: 10,
removeOnFail: 5,
attempts: 3,
backoff: {
type: 'exponential',
delay: 2000,
},
},
}),
inject: [ConfigService],
}),
// Register queues
BullModule.registerQueue(
{ name: 'image-processing' },
{ name: 'batch-processing' },
{ name: 'virus-scan' },
{ name: 'file-cleanup' },
),
// Health checks
TerminusModule,
// Core service modules
VisionModule,
ProcessorsModule,
StorageModule,
QueueModule,
MonitoringModule,
HealthModule,
],
controllers: [],
providers: [],
})
export class AppModule {
constructor(private configService: ConfigService) {
this.logConfiguration();
}
private logConfiguration() {
const logger = require('@nestjs/common').Logger;
const log = new logger('AppModule');
log.log('🔧 Worker Configuration:');
log.log(`• Environment: ${this.configService.get('NODE_ENV')}`);
log.log(`• Worker Port: ${this.configService.get('WORKER_PORT')}`);
log.log(`• Redis Host: ${this.configService.get('REDIS_HOST')}`);
log.log(`• Max Concurrent Jobs: ${this.configService.get('MAX_CONCURRENT_JOBS')}`);
log.log(`• OpenAI API Key: ${this.configService.get('OPENAI_API_KEY') ? '✓ Set' : '✗ Missing'}`);
log.log(`• Google Vision Key: ${this.configService.get('GOOGLE_CLOUD_VISION_KEY') ? '✓ Set' : '✗ Missing'}`);
log.log(`• MinIO Config: ${this.configService.get('MINIO_ENDPOINT') ? '✓ Set' : '✗ Missing'}`);
}
}