feat(worker): implement AI vision services and complete image processing pipeline
- Add real OpenAI GPT-4 Vision integration with rate limiting - Add real Google Cloud Vision API integration - Create vision service orchestrator with fallback strategy - Implement complete image processing pipeline with BullMQ - Add batch processing with progress tracking - Create virus scanning processor with ClamAV integration - Add SEO filename generation with multiple strategies - Include comprehensive error handling and retry logic - Add production-ready configuration and validation 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
parent
d53cbb6757
commit
1329e874a4
17 changed files with 3352 additions and 0 deletions
103
packages/worker/src/app.module.ts
Normal file
103
packages/worker/src/app.module.ts
Normal file
|
@ -0,0 +1,103 @@
|
|||
import { Module } from '@nestjs/common';
|
||||
import { ConfigModule, ConfigService } from '@nestjs/config';
|
||||
import { BullModule } from '@nestjs/bullmq';
|
||||
import { TerminusModule } from '@nestjs/terminus';
|
||||
import { ThrottlerModule } from '@nestjs/throttler';
|
||||
|
||||
// Import custom modules
|
||||
import { VisionModule } from './vision/vision.module';
|
||||
import { ProcessorsModule } from './processors/processors.module';
|
||||
import { StorageModule } from './storage/storage.module';
|
||||
import { QueueModule } from './queue/queue.module';
|
||||
import { MonitoringModule } from './monitoring/monitoring.module';
|
||||
import { HealthModule } from './health/health.module';
|
||||
|
||||
// Import configuration
|
||||
import { validationSchema } from './config/validation.schema';
|
||||
import { workerConfig } from './config/worker.config';
|
||||
|
||||
@Module({
|
||||
imports: [
|
||||
// Configuration module with environment validation
|
||||
ConfigModule.forRoot({
|
||||
isGlobal: true,
|
||||
load: [workerConfig],
|
||||
validationSchema,
|
||||
validationOptions: {
|
||||
abortEarly: true,
|
||||
},
|
||||
}),
|
||||
|
||||
// Rate limiting
|
||||
ThrottlerModule.forRoot([{
|
||||
ttl: 60000, // 1 minute
|
||||
limit: 100, // 100 requests per minute
|
||||
}]),
|
||||
|
||||
// BullMQ Redis connection
|
||||
BullModule.forRootAsync({
|
||||
imports: [ConfigModule],
|
||||
useFactory: async (configService: ConfigService) => ({
|
||||
connection: {
|
||||
host: configService.get<string>('REDIS_HOST', 'localhost'),
|
||||
port: configService.get<number>('REDIS_PORT', 6379),
|
||||
password: configService.get<string>('REDIS_PASSWORD'),
|
||||
db: configService.get<number>('REDIS_DB', 0),
|
||||
retryDelayOnFailover: 100,
|
||||
enableReadyCheck: false,
|
||||
maxRetriesPerRequest: 3,
|
||||
},
|
||||
defaultJobOptions: {
|
||||
removeOnComplete: 10,
|
||||
removeOnFail: 5,
|
||||
attempts: 3,
|
||||
backoff: {
|
||||
type: 'exponential',
|
||||
delay: 2000,
|
||||
},
|
||||
},
|
||||
}),
|
||||
inject: [ConfigService],
|
||||
}),
|
||||
|
||||
// Register queues
|
||||
BullModule.registerQueue(
|
||||
{ name: 'image-processing' },
|
||||
{ name: 'batch-processing' },
|
||||
{ name: 'virus-scan' },
|
||||
{ name: 'file-cleanup' },
|
||||
),
|
||||
|
||||
// Health checks
|
||||
TerminusModule,
|
||||
|
||||
// Core service modules
|
||||
VisionModule,
|
||||
ProcessorsModule,
|
||||
StorageModule,
|
||||
QueueModule,
|
||||
MonitoringModule,
|
||||
HealthModule,
|
||||
],
|
||||
controllers: [],
|
||||
providers: [],
|
||||
})
|
||||
export class AppModule {
|
||||
constructor(private configService: ConfigService) {
|
||||
this.logConfiguration();
|
||||
}
|
||||
|
||||
private logConfiguration() {
|
||||
const logger = require('@nestjs/common').Logger;
|
||||
const log = new logger('AppModule');
|
||||
|
||||
log.log('🔧 Worker Configuration:');
|
||||
log.log(`• Environment: ${this.configService.get('NODE_ENV')}`);
|
||||
log.log(`• Worker Port: ${this.configService.get('WORKER_PORT')}`);
|
||||
log.log(`• Redis Host: ${this.configService.get('REDIS_HOST')}`);
|
||||
log.log(`• Max Concurrent Jobs: ${this.configService.get('MAX_CONCURRENT_JOBS')}`);
|
||||
log.log(`• OpenAI API Key: ${this.configService.get('OPENAI_API_KEY') ? '✓ Set' : '✗ Missing'}`);
|
||||
log.log(`• Google Vision Key: ${this.configService.get('GOOGLE_CLOUD_VISION_KEY') ? '✓ Set' : '✗ Missing'}`);
|
||||
log.log(`• MinIO Config: ${this.configService.get('MINIO_ENDPOINT') ? '✓ Set' : '✗ Missing'}`);
|
||||
}
|
||||
}
|
Loading…
Add table
Add a link
Reference in a new issue