diff --git a/packages/worker/.dockerignore b/packages/worker/.dockerignore deleted file mode 100644 index 3bbf7c6..0000000 --- a/packages/worker/.dockerignore +++ /dev/null @@ -1,23 +0,0 @@ -node_modules -npm-debug.log -.git -.gitignore -README.md -.env -.env.local -.env.development -.env.test -.env.production -Dockerfile -.dockerignore -coverage -.nyc_output -dist -logs -*.log -.DS_Store -.vscode -.idea -*.swp -*.swo -*~ \ No newline at end of file diff --git a/packages/worker/.env.example b/packages/worker/.env.example deleted file mode 100644 index 0a57adf..0000000 --- a/packages/worker/.env.example +++ /dev/null @@ -1,79 +0,0 @@ -# SEO Image Renamer Worker Service - Environment Configuration - -# Application Settings -NODE_ENV=development -WORKER_PORT=3002 -HEALTH_CHECK_PORT=8080 - -# Redis Configuration -REDIS_HOST=localhost -REDIS_PORT=6379 -REDIS_PASSWORD=your_redis_password -REDIS_DB=0 -REDIS_URL=redis://localhost:6379 - -# Database Configuration -DATABASE_URL=postgresql://user:password@localhost:5432/seo_renamer - -# AI Vision APIs (at least one is required) -OPENAI_API_KEY=your_openai_api_key -OPENAI_MODEL=gpt-4-vision-preview -OPENAI_MAX_TOKENS=500 -OPENAI_TEMPERATURE=0.1 -OPENAI_REQUESTS_PER_MINUTE=50 -OPENAI_TOKENS_PER_MINUTE=10000 - -GOOGLE_CLOUD_VISION_KEY=path/to/google-service-account.json -GOOGLE_CLOUD_PROJECT_ID=your_project_id -GOOGLE_CLOUD_LOCATION=global -GOOGLE_REQUESTS_PER_MINUTE=100 - -VISION_CONFIDENCE_THRESHOLD=0.40 - -# Storage Configuration (MinIO or AWS S3) -# MinIO Configuration -MINIO_ENDPOINT=localhost -MINIO_PORT=9000 -MINIO_USE_SSL=false -MINIO_ACCESS_KEY=minioadmin -MINIO_SECRET_KEY=minioadmin -MINIO_BUCKET_NAME=seo-images - -# AWS S3 Configuration (alternative to MinIO) -# AWS_REGION=us-east-1 -# AWS_ACCESS_KEY_ID=your_aws_access_key -# AWS_SECRET_ACCESS_KEY=your_aws_secret_key -# AWS_BUCKET_NAME=your_bucket_name - -# Processing Configuration -MAX_CONCURRENT_JOBS=5 -JOB_TIMEOUT=300000 -RETRY_ATTEMPTS=3 -RETRY_DELAY=2000 - -# File Processing -MAX_FILE_SIZE=52428800 -ALLOWED_FILE_TYPES=jpg,jpeg,png,gif,webp -TEMP_DIR=/tmp/seo-worker -TEMP_FILE_CLEANUP_INTERVAL=3600000 - -# Virus Scanning (optional) -VIRUS_SCAN_ENABLED=false -CLAMAV_HOST=localhost -CLAMAV_PORT=3310 -CLAMAV_TIMEOUT=30000 - -# Monitoring -METRICS_ENABLED=true -METRICS_PORT=9090 -LOG_LEVEL=info -FILE_LOGGING_ENABLED=false -LOG_DIR=./logs - -# Rate Limiting for AI APIs -OPENAI_REQUESTS_PER_MINUTE=50 -OPENAI_TOKENS_PER_MINUTE=10000 -GOOGLE_REQUESTS_PER_MINUTE=100 - -# Optional: Grafana -GRAFANA_PASSWORD=admin \ No newline at end of file diff --git a/packages/worker/Dockerfile b/packages/worker/Dockerfile deleted file mode 100644 index 85f0690..0000000 --- a/packages/worker/Dockerfile +++ /dev/null @@ -1,228 +0,0 @@ -# SEO Image Renamer Worker Service Dockerfile -FROM node:18-alpine AS base - -# Install system dependencies for image processing and virus scanning -RUN apk add --no-cache \ - python3 \ - make \ - g++ \ - cairo-dev \ - jpeg-dev \ - pango-dev \ - musl-dev \ - giflib-dev \ - pixman-dev \ - pangomm-dev \ - libjpeg-turbo-dev \ - freetype-dev \ - clamav \ - clamav-daemon \ - freshclam \ - && rm -rf /var/cache/apk/* - -# Set working directory -WORKDIR /app - -# Copy package files -COPY package*.json ./ -COPY tsconfig.json ./ -COPY nest-cli.json ./ - -# Install dependencies -FROM base AS dependencies -RUN npm ci --only=production && npm cache clean --force - -# Install dev dependencies for building -FROM base AS build-dependencies -RUN npm ci - -# Build the application -FROM build-dependencies AS build -COPY src/ ./src/ -RUN npm run build - -# Production image -FROM base AS production - -# Create non-root user for security -RUN addgroup -g 1001 -S worker && \ - adduser -S worker -u 1001 -G worker - -# Copy production dependencies -COPY --from=dependencies /app/node_modules ./node_modules - -# Copy built application -COPY --from=build /app/dist ./dist -COPY --from=build /app/package*.json ./ - -# Create required directories -RUN mkdir -p /tmp/seo-worker /app/logs && \ - chown -R worker:worker /tmp/seo-worker /app/logs /app - -# Configure ClamAV -RUN mkdir -p /var/lib/clamav /var/log/clamav && \ - chown -R clamav:clamav /var/lib/clamav /var/log/clamav && \ - chmod 755 /var/lib/clamav /var/log/clamav - -# Copy ClamAV configuration -COPY < /dev/null 2>&1; then - echo "ClamAV is ready" - break - fi - sleep 1 - done -fi - -# Start the worker service -echo "Starting worker service..." -exec node dist/main.js -EOF - -RUN chmod +x /app/start.sh - -# Switch to non-root user -USER worker - -# Expose health check port -EXPOSE 3002 -EXPOSE 8080 - -# Health check -HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \ - CMD curl -f http://localhost:8080/health || exit 1 - -# Set environment variables -ENV NODE_ENV=production -ENV WORKER_PORT=3002 -ENV HEALTH_CHECK_PORT=8080 -ENV TEMP_DIR=/tmp/seo-worker - -# Start the application -CMD ["/app/start.sh"] - -# Labels for metadata -LABEL maintainer="SEO Image Renamer Team" \ - description="AI-powered image processing worker service" \ - version="1.0.0" \ - service="worker" \ No newline at end of file diff --git a/packages/worker/README.md b/packages/worker/README.md deleted file mode 100644 index f98d8b1..0000000 --- a/packages/worker/README.md +++ /dev/null @@ -1,280 +0,0 @@ -# SEO Image Renamer Worker Service - -A production-ready NestJS worker service that processes images using AI vision analysis to generate SEO-optimized filenames. - -## Features - -### πŸ€– AI Vision Analysis -- **OpenAI GPT-4 Vision**: Advanced image understanding with custom prompts -- **Google Cloud Vision**: Label detection with confidence scoring -- **Fallback Strategy**: Automatic failover between providers -- **Rate Limiting**: Respects API quotas with intelligent throttling - -### πŸ–ΌοΈ Image Processing Pipeline -- **File Validation**: Format validation and virus scanning -- **Metadata Extraction**: EXIF, IPTC, and XMP data preservation -- **Image Optimization**: Sharp-powered processing with quality control -- **Format Support**: JPG, PNG, GIF, WebP with conversion capabilities - -### πŸ“¦ Storage Integration -- **MinIO Support**: S3-compatible object storage -- **AWS S3 Support**: Native AWS integration -- **Temporary Files**: Automatic cleanup and management -- **ZIP Creation**: Batch downloads with EXIF preservation - -### πŸ”’ Security Features -- **Virus Scanning**: ClamAV integration for file safety -- **File Validation**: Comprehensive format and size checking -- **Quarantine System**: Automatic threat isolation -- **Security Logging**: Incident tracking and alerting - -### ⚑ Queue Processing -- **BullMQ Integration**: Reliable job processing with Redis -- **Retry Logic**: Exponential backoff with intelligent failure handling -- **Progress Tracking**: Real-time WebSocket updates -- **Batch Processing**: Efficient multi-image workflows - -### πŸ“Š Monitoring & Observability -- **Prometheus Metrics**: Comprehensive performance monitoring -- **Health Checks**: Kubernetes-ready health endpoints -- **Structured Logging**: Winston-powered logging with rotation -- **Error Tracking**: Detailed error reporting and analysis - -## Quick Start - -### Development Setup - -1. **Clone and Install** - ```bash - cd packages/worker - npm install - ``` - -2. **Environment Configuration** - ```bash - cp .env.example .env - # Edit .env with your configuration - ``` - -3. **Start Dependencies** - ```bash - docker-compose up redis minio -d - ``` - -4. **Run Development Server** - ```bash - npm run start:dev - ``` - -### Production Deployment - -1. **Docker Compose** - ```bash - docker-compose up -d - ``` - -2. **Kubernetes** - ```bash - kubectl apply -f ../k8s/worker-deployment.yaml - ``` - -## Configuration - -### Required Environment Variables - -```env -# Database -DATABASE_URL=postgresql://user:pass@host:5432/db - -# Redis -REDIS_URL=redis://localhost:6379 - -# AI Vision (at least one required) -OPENAI_API_KEY=your_key -# OR -GOOGLE_CLOUD_VISION_KEY=path/to/service-account.json - -# Storage (choose one) -MINIO_ENDPOINT=localhost -MINIO_ACCESS_KEY=access_key -MINIO_SECRET_KEY=secret_key -# OR -AWS_ACCESS_KEY_ID=your_key -AWS_SECRET_ACCESS_KEY=your_secret -AWS_BUCKET_NAME=your_bucket -``` - -### Optional Configuration - -```env -# Processing -MAX_CONCURRENT_JOBS=5 -VISION_CONFIDENCE_THRESHOLD=0.40 -MAX_FILE_SIZE=52428800 - -# Security -VIRUS_SCAN_ENABLED=true -CLAMAV_HOST=localhost - -# Monitoring -METRICS_ENABLED=true -LOG_LEVEL=info -``` - -## API Endpoints - -### Health Checks -- `GET /health` - Basic health check -- `GET /health/detailed` - Comprehensive system status -- `GET /health/ready` - Kubernetes readiness probe -- `GET /health/live` - Kubernetes liveness probe - -### Metrics -- `GET /metrics` - Prometheus metrics endpoint - -## Architecture - -### Processing Pipeline - -``` -Image Upload β†’ Virus Scan β†’ Metadata Extraction β†’ AI Analysis β†’ Filename Generation β†’ Database Update - ↓ ↓ ↓ ↓ ↓ ↓ - Security Validation EXIF/IPTC Vision APIs SEO Optimization Progress Update -``` - -### Queue Structure - -``` -β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” -β”‚ image-processingβ”‚ β”‚ batch-processing β”‚ β”‚ virus-scan β”‚ -β”‚ - Individual β”‚ β”‚ - Batch coord. β”‚ β”‚ - Security β”‚ -β”‚ - AI analysis β”‚ β”‚ - ZIP creation β”‚ β”‚ - Quarantine β”‚ -β”‚ - Filename gen. β”‚ β”‚ - Progress agg. β”‚ β”‚ - Cleanup β”‚ -β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ -``` - -## Performance - -### Throughput -- **Images/minute**: 50-100 (depending on AI provider limits) -- **Concurrent jobs**: Configurable (default: 5) -- **File size limit**: 50MB (configurable) - -### Resource Usage -- **Memory**: ~200MB base + ~50MB per concurrent job -- **CPU**: ~100% per active image processing job -- **Storage**: Temporary files cleaned automatically - -## Monitoring - -### Key Metrics -- `seo_worker_jobs_total` - Total jobs processed -- `seo_worker_job_duration_seconds` - Processing time distribution -- `seo_worker_vision_api_calls_total` - AI API usage -- `seo_worker_processing_errors_total` - Error rates - -### Alerts -- High error rates (>5%) -- API rate limit approaching -- Queue backlog growing -- Storage space low -- Memory usage high - -## Troubleshooting - -### Common Issues - -1. **AI Vision API Failures** - ```bash - # Check API keys and quotas - curl -H "Authorization: Bearer $OPENAI_API_KEY" https://api.openai.com/v1/models - ``` - -2. **Storage Connection Issues** - ```bash - # Test MinIO connection - mc alias set local http://localhost:9000 access_key secret_key - mc ls local - ``` - -3. **Queue Processing Stopped** - ```bash - # Check Redis connection - redis-cli ping - - # Check queue status - curl http://localhost:3002/health/detailed - ``` - -4. **High Memory Usage** - ```bash - # Check temp file cleanup - ls -la /tmp/seo-worker/ - - # Force cleanup - curl -X POST http://localhost:3002/admin/cleanup - ``` - -### Debugging - -Enable debug logging: -```env -LOG_LEVEL=debug -NODE_ENV=development -``` - -Monitor processing in real-time: -```bash -# Follow logs -docker logs -f seo-worker - -# Monitor metrics -curl http://localhost:9090/metrics | grep seo_worker -``` - -## Development - -### Project Structure -``` -src/ -β”œβ”€β”€ config/ # Configuration and validation -β”œβ”€β”€ vision/ # AI vision services -β”œβ”€β”€ processors/ # BullMQ job processors -β”œβ”€β”€ storage/ # File and cloud storage -β”œβ”€β”€ queue/ # Queue management and tracking -β”œβ”€β”€ security/ # Virus scanning and validation -β”œβ”€β”€ database/ # Database integration -β”œβ”€β”€ monitoring/ # Metrics and logging -└── health/ # Health check endpoints -``` - -### Testing -```bash -# Unit tests -npm test - -# Integration tests -npm run test:e2e - -# Coverage report -npm run test:cov -``` - -### Contributing - -1. Fork the repository -2. Create a feature branch -3. Add comprehensive tests -4. Update documentation -5. Submit a pull request - -## License - -Proprietary - SEO Image Renamer Platform - -## Support - -For technical support and questions: -- Documentation: [Internal Wiki] -- Issues: [Project Board] -- Contact: engineering@seo-image-renamer.com \ No newline at end of file diff --git a/packages/worker/docker-compose.yml b/packages/worker/docker-compose.yml deleted file mode 100644 index 895ea71..0000000 --- a/packages/worker/docker-compose.yml +++ /dev/null @@ -1,177 +0,0 @@ -version: '3.8' - -services: - worker: - build: . - container_name: seo-worker - restart: unless-stopped - environment: - - NODE_ENV=production - - WORKER_PORT=3002 - - HEALTH_CHECK_PORT=8080 - - # Redis Configuration - - REDIS_HOST=redis - - REDIS_PORT=6379 - - REDIS_PASSWORD=${REDIS_PASSWORD} - - REDIS_DB=0 - - # Database Configuration - - DATABASE_URL=${DATABASE_URL} - - # AI Vision APIs - - OPENAI_API_KEY=${OPENAI_API_KEY} - - GOOGLE_CLOUD_VISION_KEY=${GOOGLE_CLOUD_VISION_KEY} - - VISION_CONFIDENCE_THRESHOLD=0.40 - - # Storage Configuration - - MINIO_ENDPOINT=minio - - MINIO_PORT=9000 - - MINIO_USE_SSL=false - - MINIO_ACCESS_KEY=${MINIO_ACCESS_KEY} - - MINIO_SECRET_KEY=${MINIO_SECRET_KEY} - - MINIO_BUCKET_NAME=seo-images - - # Processing Configuration - - MAX_CONCURRENT_JOBS=5 - - JOB_TIMEOUT=300000 - - RETRY_ATTEMPTS=3 - - RETRY_DELAY=2000 - - # File Processing - - MAX_FILE_SIZE=52428800 - - ALLOWED_FILE_TYPES=jpg,jpeg,png,gif,webp - - TEMP_DIR=/tmp/seo-worker - - TEMP_FILE_CLEANUP_INTERVAL=3600000 - - # Virus Scanning - - VIRUS_SCAN_ENABLED=true - - CLAMAV_HOST=localhost - - CLAMAV_PORT=3310 - - CLAMAV_TIMEOUT=30000 - - # Monitoring - - METRICS_ENABLED=true - - METRICS_PORT=9090 - - LOG_LEVEL=info - - ports: - - "3002:3002" # Worker API port - - "8080:8080" # Health check port - - "9090:9090" # Metrics port - - volumes: - - worker-temp:/tmp/seo-worker - - worker-logs:/app/logs - - depends_on: - - redis - - minio - - networks: - - worker-network - - healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:8080/health"] - interval: 30s - timeout: 10s - retries: 3 - start_period: 30s - - redis: - image: redis:7-alpine - container_name: seo-redis - restart: unless-stopped - command: redis-server --appendonly yes --requirepass ${REDIS_PASSWORD} - environment: - - REDIS_PASSWORD=${REDIS_PASSWORD} - ports: - - "6379:6379" - volumes: - - redis-data:/data - networks: - - worker-network - healthcheck: - test: ["CMD", "redis-cli", "-a", "${REDIS_PASSWORD}", "ping"] - interval: 30s - timeout: 10s - retries: 3 - - minio: - image: minio/minio:latest - container_name: seo-minio - restart: unless-stopped - command: server /data --console-address ":9001" - environment: - - MINIO_ROOT_USER=${MINIO_ACCESS_KEY} - - MINIO_ROOT_PASSWORD=${MINIO_SECRET_KEY} - ports: - - "9000:9000" # MinIO API - - "9001:9001" # MinIO Console - volumes: - - minio-data:/data - networks: - - worker-network - healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] - interval: 30s - timeout: 10s - retries: 3 - - # Optional: Prometheus for metrics collection - prometheus: - image: prom/prometheus:latest - container_name: seo-prometheus - restart: unless-stopped - command: - - '--config.file=/etc/prometheus/prometheus.yml' - - '--storage.tsdb.path=/prometheus' - - '--web.console.libraries=/etc/prometheus/console_libraries' - - '--web.console.templates=/etc/prometheus/consoles' - - '--storage.tsdb.retention.time=200h' - - '--web.enable-lifecycle' - ports: - - "9091:9090" - volumes: - - ./prometheus.yml:/etc/prometheus/prometheus.yml:ro - - prometheus-data:/prometheus - networks: - - worker-network - depends_on: - - worker - - # Optional: Grafana for metrics visualization - grafana: - image: grafana/grafana:latest - container_name: seo-grafana - restart: unless-stopped - environment: - - GF_SECURITY_ADMIN_USER=admin - - GF_SECURITY_ADMIN_PASSWORD=${GRAFANA_PASSWORD:-admin} - - GF_USERS_ALLOW_SIGN_UP=false - ports: - - "3000:3000" - volumes: - - grafana-data:/var/lib/grafana - networks: - - worker-network - depends_on: - - prometheus - -volumes: - worker-temp: - driver: local - worker-logs: - driver: local - redis-data: - driver: local - minio-data: - driver: local - prometheus-data: - driver: local - grafana-data: - driver: local - -networks: - worker-network: - driver: bridge \ No newline at end of file diff --git a/packages/worker/nest-cli.json b/packages/worker/nest-cli.json deleted file mode 100644 index 6260dd9..0000000 --- a/packages/worker/nest-cli.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "$schema": "https://json.schemastore.org/nest-cli", - "collection": "@nestjs/schematics", - "sourceRoot": "src", - "compilerOptions": { - "deleteOutDir": true, - "tsConfigPath": "tsconfig.json" - } -} \ No newline at end of file diff --git a/packages/worker/package.json b/packages/worker/package.json deleted file mode 100644 index 93dae61..0000000 --- a/packages/worker/package.json +++ /dev/null @@ -1,105 +0,0 @@ -{ - "name": "@seo-image-renamer/worker", - "version": "1.0.0", - "description": "Worker service for AI-powered image processing and SEO filename generation", - "main": "dist/main.js", - "scripts": { - "build": "nest build", - "format": "prettier --write \"src/**/*.ts\" \"test/**/*.ts\"", - "start": "nest start", - "start:dev": "nest start --watch", - "start:debug": "nest start --debug --watch", - "start:prod": "node dist/main", - "lint": "eslint \"{src,apps,libs,test}/**/*.ts\" --fix", - "test": "jest", - "test:watch": "jest --watch", - "test:cov": "jest --coverage", - "test:debug": "node --inspect-brk -r tsconfig-paths/register -r ts-node/register node_modules/.bin/jest --runInBand", - "test:e2e": "jest --config ./test/jest-e2e.json" - }, - "dependencies": { - "@nestjs/common": "^10.0.0", - "@nestjs/core": "^10.0.0", - "@nestjs/platform-express": "^10.0.0", - "@nestjs/config": "^3.1.1", - "@nestjs/bullmq": "^10.0.1", - "@nestjs/schedule": "^4.0.0", - "@nestjs-modules/ioredis": "^2.0.2", - "@nestjs/terminus": "^10.2.0", - "@nestjs/throttler": "^5.0.1", - "@prisma/client": "^5.6.0", - "bullmq": "^4.15.0", - "redis": "^4.6.10", - "ioredis": "^5.3.2", - "sharp": "^0.32.6", - "exifr": "^7.1.3", - "piexifjs": "^1.0.6", - "archiver": "^6.0.1", - "minio": "^7.1.3", - "aws-sdk": "^2.1489.0", - "openai": "^4.20.1", - "@google-cloud/vision": "^4.0.2", - "node-clamav": "^0.8.5", - "axios": "^1.6.0", - "class-validator": "^0.14.0", - "class-transformer": "^0.5.1", - "reflect-metadata": "^0.1.13", - "rxjs": "^7.8.1", - "uuid": "^9.0.1", - "lodash": "^4.17.21", - "mime-types": "^2.1.35", - "file-type": "^18.7.0", - "sanitize-filename": "^1.6.3", - "winston": "^3.11.0", - "winston-daily-rotate-file": "^4.7.1", - "@nestjs/websockets": "^10.2.7", - "@nestjs/platform-socket.io": "^10.2.7", - "socket.io": "^4.7.4", - "prom-client": "^15.0.0", - "joi": "^17.11.0", - "curl": "^0.1.4" - }, - "devDependencies": { - "@nestjs/cli": "^10.0.0", - "@nestjs/schematics": "^10.0.0", - "@nestjs/testing": "^10.0.0", - "@types/express": "^4.17.17", - "@types/jest": "^29.5.2", - "@types/node": "^20.3.1", - "@types/uuid": "^9.0.7", - "@types/lodash": "^4.14.202", - "@types/mime-types": "^2.1.4", - "@types/archiver": "^6.0.2", - "@typescript-eslint/eslint-plugin": "^6.0.0", - "@typescript-eslint/parser": "^6.0.0", - "eslint": "^8.42.0", - "eslint-config-prettier": "^9.0.0", - "eslint-plugin-prettier": "^5.0.0", - "jest": "^29.5.0", - "prettier": "^3.0.0", - "source-map-support": "^0.5.21", - "supertest": "^6.3.3", - "ts-jest": "^29.1.0", - "ts-loader": "^9.4.3", - "ts-node": "^10.9.1", - "tsconfig-paths": "^4.2.1", - "typescript": "^5.1.3" - }, - "jest": { - "moduleFileExtensions": [ - "js", - "json", - "ts" - ], - "rootDir": "src", - "testRegex": ".*\\.spec\\.ts$", - "transform": { - "^.+\\.(t|j)s$": "ts-jest" - }, - "collectCoverageFrom": [ - "**/*.(t|j)s" - ], - "coverageDirectory": "../coverage", - "testEnvironment": "node" - } -} \ No newline at end of file diff --git a/packages/worker/prometheus.yml b/packages/worker/prometheus.yml deleted file mode 100644 index b52eeca..0000000 --- a/packages/worker/prometheus.yml +++ /dev/null @@ -1,31 +0,0 @@ -global: - scrape_interval: 15s - evaluation_interval: 15s - -rule_files: - # - "first_rules.yml" - # - "second_rules.yml" - -scrape_configs: - - job_name: 'prometheus' - static_configs: - - targets: ['localhost:9090'] - - - job_name: 'seo-worker' - static_configs: - - targets: ['worker:9090'] - metrics_path: '/metrics' - scrape_interval: 30s - scrape_timeout: 10s - - - job_name: 'redis' - static_configs: - - targets: ['redis:6379'] - metrics_path: '/metrics' - scrape_interval: 30s - - - job_name: 'minio' - static_configs: - - targets: ['minio:9000'] - metrics_path: '/minio/v2/metrics/cluster' - scrape_interval: 30s \ No newline at end of file diff --git a/packages/worker/src/app.module.ts b/packages/worker/src/app.module.ts deleted file mode 100644 index 447917c..0000000 --- a/packages/worker/src/app.module.ts +++ /dev/null @@ -1,120 +0,0 @@ -import { Module } from '@nestjs/common'; -import { ConfigModule, ConfigService } from '@nestjs/config'; -import { BullModule } from '@nestjs/bullmq'; -import { TerminusModule } from '@nestjs/terminus'; -import { ThrottlerModule } from '@nestjs/throttler'; -import { RedisModule } from '@nestjs-modules/ioredis'; - -// Import custom modules -import { VisionModule } from './vision/vision.module'; -import { ProcessorsModule } from './processors/processors.module'; -import { StorageModule } from './storage/storage.module'; -import { QueueModule } from './queue/queue.module'; -import { MonitoringModule } from './monitoring/monitoring.module'; -import { HealthModule } from './health/health.module'; - -// Import configuration -import { validationSchema } from './config/validation.schema'; -import { workerConfig } from './config/worker.config'; - -@Module({ - imports: [ - // Configuration module with environment validation - ConfigModule.forRoot({ - isGlobal: true, - load: [workerConfig], - validationSchema, - validationOptions: { - abortEarly: true, - }, - }), - - // Rate limiting - ThrottlerModule.forRoot([{ - ttl: 60000, // 1 minute - limit: 100, // 100 requests per minute - }]), - - // Redis connection for progress tracking - RedisModule.forRootAsync({ - imports: [ConfigModule], - useFactory: (configService: ConfigService) => ({ - type: 'single', - url: configService.get('REDIS_URL', 'redis://localhost:6379'), - options: { - password: configService.get('REDIS_PASSWORD'), - db: configService.get('REDIS_DB', 0), - retryDelayOnFailover: 100, - maxRetriesPerRequest: 3, - }, - }), - inject: [ConfigService], - }), - - // BullMQ Redis connection - BullModule.forRootAsync({ - imports: [ConfigModule], - useFactory: async (configService: ConfigService) => ({ - connection: { - host: configService.get('REDIS_HOST', 'localhost'), - port: configService.get('REDIS_PORT', 6379), - password: configService.get('REDIS_PASSWORD'), - db: configService.get('REDIS_DB', 0), - retryDelayOnFailover: 100, - enableReadyCheck: false, - maxRetriesPerRequest: 3, - }, - defaultJobOptions: { - removeOnComplete: 10, - removeOnFail: 5, - attempts: 3, - backoff: { - type: 'exponential', - delay: 2000, - }, - }, - }), - inject: [ConfigService], - }), - - // Register queues - BullModule.registerQueue( - { name: 'image-processing' }, - { name: 'batch-processing' }, - { name: 'virus-scan' }, - { name: 'file-cleanup' }, - ), - - // Health checks - TerminusModule, - - // Core service modules - VisionModule, - ProcessorsModule, - StorageModule, - QueueModule, - MonitoringModule, - HealthModule, - ], - controllers: [], - providers: [], -}) -export class AppModule { - constructor(private configService: ConfigService) { - this.logConfiguration(); - } - - private logConfiguration() { - const logger = require('@nestjs/common').Logger; - const log = new logger('AppModule'); - - log.log('πŸ”§ Worker Configuration:'); - log.log(`β€’ Environment: ${this.configService.get('NODE_ENV')}`); - log.log(`β€’ Worker Port: ${this.configService.get('WORKER_PORT')}`); - log.log(`β€’ Redis Host: ${this.configService.get('REDIS_HOST')}`); - log.log(`β€’ Max Concurrent Jobs: ${this.configService.get('MAX_CONCURRENT_JOBS')}`); - log.log(`β€’ OpenAI API Key: ${this.configService.get('OPENAI_API_KEY') ? 'βœ“ Set' : 'βœ— Missing'}`); - log.log(`β€’ Google Vision Key: ${this.configService.get('GOOGLE_CLOUD_VISION_KEY') ? 'βœ“ Set' : 'βœ— Missing'}`); - log.log(`β€’ MinIO Config: ${this.configService.get('MINIO_ENDPOINT') ? 'βœ“ Set' : 'βœ— Missing'}`); - } -} \ No newline at end of file diff --git a/packages/worker/src/config/validation.schema.ts b/packages/worker/src/config/validation.schema.ts deleted file mode 100644 index f82ca50..0000000 --- a/packages/worker/src/config/validation.schema.ts +++ /dev/null @@ -1,102 +0,0 @@ -const Joi = require('joi'); - -export const validationSchema = Joi.object({ - // Application settings - NODE_ENV: Joi.string().valid('development', 'production', 'test').default('development'), - WORKER_PORT: Joi.number().port().default(3002), - - // Redis configuration - REDIS_HOST: Joi.string().default('localhost'), - REDIS_PORT: Joi.number().port().default(6379), - REDIS_PASSWORD: Joi.string().optional(), - REDIS_DB: Joi.number().integer().min(0).max(15).default(0), - REDIS_URL: Joi.string().uri().default('redis://localhost:6379'), - - // Processing configuration - MAX_CONCURRENT_JOBS: Joi.number().integer().min(1).max(50).default(5), - JOB_TIMEOUT: Joi.number().integer().min(30000).max(3600000).default(300000), - RETRY_ATTEMPTS: Joi.number().integer().min(1).max(10).default(3), - RETRY_DELAY: Joi.number().integer().min(1000).max(60000).default(2000), - - // AI Vision APIs (at least one is required) - OPENAI_API_KEY: Joi.string().when('GOOGLE_CLOUD_VISION_KEY', { - is: Joi.exist(), - then: Joi.optional(), - otherwise: Joi.required(), - }), - OPENAI_MODEL: Joi.string().default('gpt-4-vision-preview'), - OPENAI_MAX_TOKENS: Joi.number().integer().min(100).max(4000).default(500), - OPENAI_TEMPERATURE: Joi.number().min(0).max(2).default(0.1), - OPENAI_REQUESTS_PER_MINUTE: Joi.number().integer().min(1).max(1000).default(50), - OPENAI_TOKENS_PER_MINUTE: Joi.number().integer().min(1000).max(100000).default(10000), - - GOOGLE_CLOUD_VISION_KEY: Joi.string().when('OPENAI_API_KEY', { - is: Joi.exist(), - then: Joi.optional(), - otherwise: Joi.required(), - }), - GOOGLE_CLOUD_PROJECT_ID: Joi.string().optional(), - GOOGLE_CLOUD_LOCATION: Joi.string().default('global'), - GOOGLE_REQUESTS_PER_MINUTE: Joi.number().integer().min(1).max(1000).default(100), - - VISION_CONFIDENCE_THRESHOLD: Joi.number().min(0).max(1).default(0.40), - - // Storage configuration (MinIO or AWS S3) - MINIO_ENDPOINT: Joi.string().when('AWS_BUCKET_NAME', { - is: Joi.exist(), - then: Joi.optional(), - otherwise: Joi.required(), - }), - MINIO_PORT: Joi.number().port().default(9000), - MINIO_USE_SSL: Joi.boolean().default(false), - MINIO_ACCESS_KEY: Joi.string().when('MINIO_ENDPOINT', { - is: Joi.exist(), - then: Joi.required(), - otherwise: Joi.optional(), - }), - MINIO_SECRET_KEY: Joi.string().when('MINIO_ENDPOINT', { - is: Joi.exist(), - then: Joi.required(), - otherwise: Joi.optional(), - }), - MINIO_BUCKET_NAME: Joi.string().default('seo-images'), - - AWS_REGION: Joi.string().default('us-east-1'), - AWS_ACCESS_KEY_ID: Joi.string().when('AWS_BUCKET_NAME', { - is: Joi.exist(), - then: Joi.required(), - otherwise: Joi.optional(), - }), - AWS_SECRET_ACCESS_KEY: Joi.string().when('AWS_BUCKET_NAME', { - is: Joi.exist(), - then: Joi.required(), - otherwise: Joi.optional(), - }), - AWS_BUCKET_NAME: Joi.string().optional(), - - // Database - DATABASE_URL: Joi.string().uri().required(), - DB_MAX_CONNECTIONS: Joi.number().integer().min(1).max(100).default(10), - - // File processing - MAX_FILE_SIZE: Joi.number().integer().min(1024).max(100 * 1024 * 1024).default(50 * 1024 * 1024), // Max 100MB - ALLOWED_FILE_TYPES: Joi.string().default('jpg,jpeg,png,gif,webp'), - TEMP_DIR: Joi.string().default('/tmp/seo-worker'), - TEMP_FILE_CLEANUP_INTERVAL: Joi.number().integer().min(60000).max(86400000).default(3600000), // 1 minute to 24 hours - - // Virus scanning (optional) - VIRUS_SCAN_ENABLED: Joi.boolean().default(false), - CLAMAV_HOST: Joi.string().default('localhost'), - CLAMAV_PORT: Joi.number().port().default(3310), - CLAMAV_TIMEOUT: Joi.number().integer().min(5000).max(120000).default(30000), - - // Monitoring - METRICS_ENABLED: Joi.boolean().default(true), - METRICS_PORT: Joi.number().port().default(9090), - HEALTH_CHECK_PORT: Joi.number().port().default(8080), - - // Logging - LOG_LEVEL: Joi.string().valid('error', 'warn', 'info', 'debug', 'verbose').default('info'), - FILE_LOGGING_ENABLED: Joi.boolean().default(false), - LOG_DIR: Joi.string().default('./logs'), -}); \ No newline at end of file diff --git a/packages/worker/src/config/worker.config.ts b/packages/worker/src/config/worker.config.ts deleted file mode 100644 index 635b36d..0000000 --- a/packages/worker/src/config/worker.config.ts +++ /dev/null @@ -1,105 +0,0 @@ -import { registerAs } from '@nestjs/config'; - -export const workerConfig = registerAs('worker', () => ({ - // Application settings - port: parseInt(process.env.WORKER_PORT, 10) || 3002, - environment: process.env.NODE_ENV || 'development', - - // Redis/Queue configuration - redis: { - host: process.env.REDIS_HOST || 'localhost', - port: parseInt(process.env.REDIS_PORT, 10) || 6379, - password: process.env.REDIS_PASSWORD, - db: parseInt(process.env.REDIS_DB, 10) || 0, - url: process.env.REDIS_URL || 'redis://localhost:6379', - }, - - // Processing limits - processing: { - maxConcurrentJobs: parseInt(process.env.MAX_CONCURRENT_JOBS, 10) || 5, - jobTimeout: parseInt(process.env.JOB_TIMEOUT, 10) || 300000, // 5 minutes - retryAttempts: parseInt(process.env.RETRY_ATTEMPTS, 10) || 3, - retryDelay: parseInt(process.env.RETRY_DELAY, 10) || 2000, // 2 seconds - }, - - // AI Vision APIs - ai: { - openai: { - apiKey: process.env.OPENAI_API_KEY, - model: process.env.OPENAI_MODEL || 'gpt-4-vision-preview', - maxTokens: parseInt(process.env.OPENAI_MAX_TOKENS, 10) || 500, - temperature: parseFloat(process.env.OPENAI_TEMPERATURE) || 0.1, - }, - google: { - apiKey: process.env.GOOGLE_CLOUD_VISION_KEY, - projectId: process.env.GOOGLE_CLOUD_PROJECT_ID, - location: process.env.GOOGLE_CLOUD_LOCATION || 'global', - }, - confidenceThreshold: parseFloat(process.env.VISION_CONFIDENCE_THRESHOLD) || 0.40, - }, - - // Storage configuration - storage: { - minio: { - endpoint: process.env.MINIO_ENDPOINT || 'localhost', - port: parseInt(process.env.MINIO_PORT, 10) || 9000, - useSSL: process.env.MINIO_USE_SSL === 'true', - accessKey: process.env.MINIO_ACCESS_KEY || 'minioadmin', - secretKey: process.env.MINIO_SECRET_KEY || 'minioadmin', - bucketName: process.env.MINIO_BUCKET_NAME || 'seo-images', - }, - aws: { - region: process.env.AWS_REGION || 'us-east-1', - accessKeyId: process.env.AWS_ACCESS_KEY_ID, - secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY, - bucketName: process.env.AWS_BUCKET_NAME, - }, - }, - - // Database (shared with API) - database: { - url: process.env.DATABASE_URL, - maxConnections: parseInt(process.env.DB_MAX_CONNECTIONS, 10) || 10, - }, - - // File processing - files: { - maxFileSize: parseInt(process.env.MAX_FILE_SIZE, 10) || 50 * 1024 * 1024, // 50MB - allowedTypes: (process.env.ALLOWED_FILE_TYPES || 'jpg,jpeg,png,gif,webp').split(','), - tempDir: process.env.TEMP_DIR || '/tmp/seo-worker', - cleanupInterval: parseInt(process.env.TEMP_FILE_CLEANUP_INTERVAL, 10) || 3600000, // 1 hour - }, - - // Virus scanning - virusScan: { - enabled: process.env.VIRUS_SCAN_ENABLED === 'true', - clamavHost: process.env.CLAMAV_HOST || 'localhost', - clamavPort: parseInt(process.env.CLAMAV_PORT, 10) || 3310, - timeout: parseInt(process.env.CLAMAV_TIMEOUT, 10) || 30000, // 30 seconds - }, - - // Monitoring - monitoring: { - metricsEnabled: process.env.METRICS_ENABLED !== 'false', - metricsPort: parseInt(process.env.METRICS_PORT, 10) || 9090, - healthCheckPort: parseInt(process.env.HEALTH_CHECK_PORT, 10) || 8080, - }, - - // Logging - logging: { - level: process.env.LOG_LEVEL || 'info', - fileLogging: process.env.FILE_LOGGING_ENABLED === 'true', - logDir: process.env.LOG_DIR || './logs', - }, - - // Rate limiting for AI APIs - rateLimiting: { - openai: { - requestsPerMinute: parseInt(process.env.OPENAI_REQUESTS_PER_MINUTE, 10) || 50, - tokensPerMinute: parseInt(process.env.OPENAI_TOKENS_PER_MINUTE, 10) || 10000, - }, - google: { - requestsPerMinute: parseInt(process.env.GOOGLE_REQUESTS_PER_MINUTE, 10) || 100, - }, - }, -})); \ No newline at end of file diff --git a/packages/worker/src/database/database.module.ts b/packages/worker/src/database/database.module.ts deleted file mode 100644 index 3041bbb..0000000 --- a/packages/worker/src/database/database.module.ts +++ /dev/null @@ -1,10 +0,0 @@ -import { Module } from '@nestjs/common'; -import { ConfigModule } from '@nestjs/config'; -import { DatabaseService } from './database.service'; - -@Module({ - imports: [ConfigModule], - providers: [DatabaseService], - exports: [DatabaseService], -}) -export class DatabaseModule {} \ No newline at end of file diff --git a/packages/worker/src/database/database.service.ts b/packages/worker/src/database/database.service.ts deleted file mode 100644 index 447ca5f..0000000 --- a/packages/worker/src/database/database.service.ts +++ /dev/null @@ -1,338 +0,0 @@ -import { Injectable, Logger } from '@nestjs/common'; -import { ConfigService } from '@nestjs/config'; -import { PrismaClient } from '@prisma/client'; - -@Injectable() -export class DatabaseService extends PrismaClient { - private readonly logger = new Logger(DatabaseService.name); - - constructor(private configService: ConfigService) { - const databaseUrl = configService.get('DATABASE_URL'); - - super({ - datasources: { - db: { - url: databaseUrl, - }, - }, - log: [ - { level: 'warn', emit: 'event' }, - { level: 'error', emit: 'event' }, - ], - }); - - // Set up logging - this.$on('warn' as never, (e: any) => { - this.logger.warn('Database warning:', e); - }); - - this.$on('error' as never, (e: any) => { - this.logger.error('Database error:', e); - }); - - this.logger.log('Database service initialized'); - } - - async onModuleInit() { - try { - await this.$connect(); - this.logger.log('βœ… Database connected successfully'); - } catch (error) { - this.logger.error('❌ Failed to connect to database:', error.message); - throw error; - } - } - - async onModuleDestroy() { - await this.$disconnect(); - this.logger.log('Database disconnected'); - } - - /** - * Update image processing status - */ - async updateImageStatus( - imageId: string, - status: string, - additionalData: any = {} - ): Promise { - try { - await this.image.update({ - where: { id: imageId }, - data: { - status, - ...additionalData, - updatedAt: new Date(), - }, - }); - } catch (error) { - this.logger.error(`Failed to update image status ${imageId}:`, error.message); - throw error; - } - } - - /** - * Update image processing result - */ - async updateImageProcessingResult( - imageId: string, - result: any - ): Promise { - try { - await this.image.update({ - where: { id: imageId }, - data: { - ...result, - updatedAt: new Date(), - }, - }); - } catch (error) { - this.logger.error(`Failed to update image processing result ${imageId}:`, error.message); - throw error; - } - } - - /** - * Update batch processing status - */ - async updateBatchStatus( - batchId: string, - status: string, - additionalData: any = {} - ): Promise { - try { - await this.batch.update({ - where: { id: batchId }, - data: { - status, - ...additionalData, - updatedAt: new Date(), - }, - }); - } catch (error) { - this.logger.error(`Failed to update batch status ${batchId}:`, error.message); - throw error; - } - } - - /** - * Get images by IDs - */ - async getImagesByIds(imageIds: string[]): Promise { - try { - return await this.image.findMany({ - where: { - id: { in: imageIds }, - }, - select: { - id: true, - originalName: true, - proposedName: true, - s3Key: true, - status: true, - visionAnalysis: true, - metadata: true, - }, - }); - } catch (error) { - this.logger.error('Failed to get images by IDs:', error.message); - throw error; - } - } - - /** - * Get image statuses for multiple images - */ - async getImageStatuses(imageIds: string[]): Promise { - try { - return await this.image.findMany({ - where: { - id: { in: imageIds }, - }, - select: { - id: true, - status: true, - proposedName: true, - visionAnalysis: true, - error: true, - }, - }); - } catch (error) { - this.logger.error('Failed to get image statuses:', error.message); - throw error; - } - } - - /** - * Update image filename - */ - async updateImageFilename( - imageId: string, - filenameData: any - ): Promise { - try { - await this.image.update({ - where: { id: imageId }, - data: { - ...filenameData, - updatedAt: new Date(), - }, - }); - } catch (error) { - this.logger.error(`Failed to update image filename ${imageId}:`, error.message); - throw error; - } - } - - /** - * Update file scan status - */ - async updateFileScanStatus( - fileId: string, - status: string, - scanData: any = {} - ): Promise { - try { - // This would update a file_scans table or similar - // For now, we'll update the image record - await this.image.update({ - where: { id: fileId }, - data: { - scanStatus: status, - scanData, - updatedAt: new Date(), - }, - }); - } catch (error) { - this.logger.error(`Failed to update file scan status ${fileId}:`, error.message); - throw error; - } - } - - /** - * Create security incident record - */ - async createSecurityIncident(incidentData: any): Promise { - try { - // This would create a record in a security_incidents table - // For now, we'll log it and store minimal data - this.logger.warn('Security incident created:', incidentData); - - // In production, you'd have a proper security_incidents table - // await this.securityIncident.create({ data: incidentData }); - - } catch (error) { - this.logger.error('Failed to create security incident:', error.message); - throw error; - } - } - - /** - * Get user's recent threats - */ - async getUserRecentThreats(userId: string, days: number): Promise { - try { - const since = new Date(); - since.setDate(since.getDate() - days); - - // This would query a security_incidents or file_scans table - // For now, return empty array - return []; - - // In production: - // return await this.securityIncident.findMany({ - // where: { - // userId, - // createdAt: { gte: since }, - // type: 'virus-detected', - // }, - // }); - - } catch (error) { - this.logger.error(`Failed to get user recent threats ${userId}:`, error.message); - return []; - } - } - - /** - * Flag user for review - */ - async flagUserForReview(userId: string, flagData: any): Promise { - try { - // This would update a user_flags table or user record - this.logger.warn(`User ${userId} flagged for review:`, flagData); - - // In production: - // await this.user.update({ - // where: { id: userId }, - // data: { - // flagged: true, - // flagReason: flagData.reason, - // flaggedAt: flagData.flaggedAt, - // }, - // }); - - } catch (error) { - this.logger.error(`Failed to flag user ${userId}:`, error.message); - throw error; - } - } - - /** - * Health check for database - */ - async isHealthy(): Promise { - try { - // Simple query to test database connectivity - await this.$queryRaw`SELECT 1`; - return true; - } catch (error) { - this.logger.error('Database health check failed:', error.message); - return false; - } - } - - /** - * Get database statistics - */ - async getStats(): Promise<{ - totalImages: number; - processingImages: number; - completedImages: number; - failedImages: number; - totalBatches: number; - }> { - try { - const [ - totalImages, - processingImages, - completedImages, - failedImages, - totalBatches, - ] = await Promise.all([ - this.image.count(), - this.image.count({ where: { status: 'processing' } }), - this.image.count({ where: { status: 'completed' } }), - this.image.count({ where: { status: 'failed' } }), - this.batch.count(), - ]); - - return { - totalImages, - processingImages, - completedImages, - failedImages, - totalBatches, - }; - } catch (error) { - this.logger.error('Failed to get database stats:', error.message); - return { - totalImages: 0, - processingImages: 0, - completedImages: 0, - failedImages: 0, - totalBatches: 0, - }; - } - } -} \ No newline at end of file diff --git a/packages/worker/src/health/health.controller.ts b/packages/worker/src/health/health.controller.ts deleted file mode 100644 index bb165ce..0000000 --- a/packages/worker/src/health/health.controller.ts +++ /dev/null @@ -1,394 +0,0 @@ -import { Controller, Get, Logger } from '@nestjs/common'; -import { ConfigService } from '@nestjs/config'; -import { - HealthCheckService, - HealthCheck, - HealthCheckResult, - MemoryHealthIndicator, - DiskHealthIndicator, -} from '@nestjs/terminus'; -import { DatabaseService } from '../database/database.service'; -import { StorageService } from '../storage/storage.service'; -import { VirusScanService } from '../security/virus-scan.service'; -import { VisionService } from '../vision/vision.service'; -import { CleanupService } from '../queue/cleanup.service'; -import { MetricsService } from '../monitoring/services/metrics.service'; - -@Controller('health') -export class HealthController { - private readonly logger = new Logger(HealthController.name); - - constructor( - private health: HealthCheckService, - private memory: MemoryHealthIndicator, - private disk: DiskHealthIndicator, - private configService: ConfigService, - private databaseService: DatabaseService, - private storageService: StorageService, - private virusScanService: VirusScanService, - private visionService: VisionService, - private cleanupService: CleanupService, - private metricsService: MetricsService, - ) {} - - @Get() - @HealthCheck() - check(): Promise { - return this.health.check([ - // Basic system health - () => this.memory.checkHeap('memory_heap', 150 * 1024 * 1024), // 150MB - () => this.memory.checkRSS('memory_rss', 300 * 1024 * 1024), // 300MB - () => this.disk.checkStorage('storage', { - path: '/', - thresholdPercent: 0.9 // 90% threshold - }), - - // Core services health - () => this.checkDatabase(), - () => this.checkStorage(), - () => this.checkVisionServices(), - () => this.checkSecurity(), - () => this.checkQueues(), - () => this.checkMetrics(), - ]); - } - - @Get('detailed') - async getDetailedHealth(): Promise<{ - status: string; - timestamp: string; - uptime: number; - services: any; - system: any; - configuration: any; - }> { - const startTime = Date.now(); - - try { - // Gather detailed health information - const [ - databaseHealth, - storageHealth, - visionHealth, - securityHealth, - queueHealth, - metricsHealth, - systemHealth, - ] = await Promise.allSettled([ - this.getDatabaseHealth(), - this.getStorageHealth(), - this.getVisionHealth(), - this.getSecurityHealth(), - this.getQueueHealth(), - this.getMetricsHealth(), - this.getSystemHealth(), - ]); - - const services = { - database: this.getResultValue(databaseHealth), - storage: this.getResultValue(storageHealth), - vision: this.getResultValue(visionHealth), - security: this.getResultValue(securityHealth), - queues: this.getResultValue(queueHealth), - metrics: this.getResultValue(metricsHealth), - }; - - // Determine overall status - const allHealthy = Object.values(services).every(service => - service && service.healthy !== false - ); - - const healthCheckDuration = Date.now() - startTime; - - return { - status: allHealthy ? 'healthy' : 'degraded', - timestamp: new Date().toISOString(), - uptime: process.uptime(), - services, - system: this.getResultValue(systemHealth), - configuration: { - environment: this.configService.get('NODE_ENV'), - workerPort: this.configService.get('WORKER_PORT'), - healthCheckDuration, - }, - }; - - } catch (error) { - this.logger.error('Detailed health check failed:', error.message); - return { - status: 'error', - timestamp: new Date().toISOString(), - uptime: process.uptime(), - services: {}, - system: {}, - configuration: { - error: error.message, - }, - }; - } - } - - @Get('ready') - async readinessCheck(): Promise<{ ready: boolean; checks: any }> { - try { - // Critical services that must be available for the worker to be ready - const checks = await Promise.allSettled([ - this.databaseService.isHealthy(), - this.storageService.testConnection(), - this.visionService.getHealthStatus(), - ]); - - const ready = checks.every(check => - check.status === 'fulfilled' && check.value === true - ); - - return { - ready, - checks: { - database: this.getResultValue(checks[0]), - storage: this.getResultValue(checks[1]), - vision: this.getResultValue(checks[2]), - }, - }; - - } catch (error) { - this.logger.error('Readiness check failed:', error.message); - return { - ready: false, - checks: { error: error.message }, - }; - } - } - - @Get('live') - async livenessCheck(): Promise<{ alive: boolean }> { - // Simple liveness check - just verify the process is responding - return { alive: true }; - } - - // Individual health check methods - private async checkDatabase() { - const isHealthy = await this.databaseService.isHealthy(); - - if (isHealthy) { - return { database: { status: 'up' } }; - } else { - throw new Error('Database connection failed'); - } - } - - private async checkStorage() { - const isHealthy = await this.storageService.testConnection(); - - if (isHealthy) { - return { storage: { status: 'up' } }; - } else { - throw new Error('Storage connection failed'); - } - } - - private async checkVisionServices() { - const healthStatus = await this.visionService.getHealthStatus(); - - if (healthStatus.healthy) { - return { vision: { status: 'up', providers: healthStatus.providers } }; - } else { - throw new Error('Vision services unavailable'); - } - } - - private async checkSecurity() { - const isHealthy = await this.virusScanService.isHealthy(); - const enabled = this.virusScanService.isEnabled(); - - if (!enabled || isHealthy) { - return { security: { status: 'up', virusScanEnabled: enabled } }; - } else { - throw new Error('Security services degraded'); - } - } - - private async checkQueues() { - const isHealthy = await this.cleanupService.isHealthy(); - - if (isHealthy) { - return { queues: { status: 'up' } }; - } else { - throw new Error('Queue services unavailable'); - } - } - - private async checkMetrics() { - const isHealthy = this.metricsService.isHealthy(); - - if (isHealthy) { - return { metrics: { status: 'up' } }; - } else { - throw new Error('Metrics collection failed'); - } - } - - // Detailed health methods - private async getDatabaseHealth() { - try { - const [isHealthy, stats] = await Promise.all([ - this.databaseService.isHealthy(), - this.databaseService.getStats(), - ]); - - return { - healthy: isHealthy, - stats, - lastCheck: new Date().toISOString(), - }; - } catch (error) { - return { - healthy: false, - error: error.message, - lastCheck: new Date().toISOString(), - }; - } - } - - private async getStorageHealth() { - try { - const [isHealthy, stats] = await Promise.all([ - this.storageService.testConnection(), - this.storageService.getStorageStats(), - ]); - - return { - healthy: isHealthy, - stats, - lastCheck: new Date().toISOString(), - }; - } catch (error) { - return { - healthy: false, - error: error.message, - lastCheck: new Date().toISOString(), - }; - } - } - - private async getVisionHealth() { - try { - const healthStatus = await this.visionService.getHealthStatus(); - const serviceInfo = this.visionService.getServiceInfo(); - - return { - healthy: healthStatus.healthy, - providers: healthStatus.providers, - configuration: serviceInfo, - lastCheck: new Date().toISOString(), - }; - } catch (error) { - return { - healthy: false, - error: error.message, - lastCheck: new Date().toISOString(), - }; - } - } - - private async getSecurityHealth() { - try { - const [isHealthy, stats, config] = await Promise.all([ - this.virusScanService.isHealthy(), - this.virusScanService.getScanStats(), - Promise.resolve(this.virusScanService.getConfiguration()), - ]); - - return { - healthy: !config.enabled || isHealthy, // Healthy if disabled or working - configuration: config, - stats, - lastCheck: new Date().toISOString(), - }; - } catch (error) { - return { - healthy: false, - error: error.message, - lastCheck: new Date().toISOString(), - }; - } - } - - private async getQueueHealth() { - try { - const [isHealthy, stats] = await Promise.all([ - this.cleanupService.isHealthy(), - this.cleanupService.getCleanupStats(), - ]); - - return { - healthy: isHealthy, - stats, - lastCheck: new Date().toISOString(), - }; - } catch (error) { - return { - healthy: false, - error: error.message, - lastCheck: new Date().toISOString(), - }; - } - } - - private async getMetricsHealth() { - try { - const isHealthy = this.metricsService.isHealthy(); - const config = this.metricsService.getConfiguration(); - - return { - healthy: isHealthy, - configuration: config, - lastCheck: new Date().toISOString(), - }; - } catch (error) { - return { - healthy: false, - error: error.message, - lastCheck: new Date().toISOString(), - }; - } - } - - private async getSystemHealth() { - try { - const memoryUsage = process.memoryUsage(); - const cpuUsage = process.cpuUsage(); - - return { - healthy: true, - uptime: process.uptime(), - memory: { - rss: memoryUsage.rss, - heapTotal: memoryUsage.heapTotal, - heapUsed: memoryUsage.heapUsed, - external: memoryUsage.external, - }, - cpu: cpuUsage, - platform: process.platform, - nodeVersion: process.version, - pid: process.pid, - }; - } catch (error) { - return { - healthy: false, - error: error.message, - }; - } - } - - private getResultValue(result: PromiseSettledResult): any { - if (result.status === 'fulfilled') { - return result.value; - } else { - return { - error: result.reason?.message || 'Unknown error', - healthy: false, - }; - } - } -} \ No newline at end of file diff --git a/packages/worker/src/health/health.module.ts b/packages/worker/src/health/health.module.ts deleted file mode 100644 index 19ddfea..0000000 --- a/packages/worker/src/health/health.module.ts +++ /dev/null @@ -1,25 +0,0 @@ -import { Module } from '@nestjs/common'; -import { TerminusModule } from '@nestjs/terminus'; -import { ConfigModule } from '@nestjs/config'; -import { HealthController } from './health.controller'; -import { DatabaseModule } from '../database/database.module'; -import { StorageModule } from '../storage/storage.module'; -import { SecurityModule } from '../security/security.module'; -import { VisionModule } from '../vision/vision.module'; -import { QueueModule } from '../queue/queue.module'; -import { MonitoringModule } from '../monitoring/monitoring.module'; - -@Module({ - imports: [ - TerminusModule, - ConfigModule, - DatabaseModule, - StorageModule, - SecurityModule, - VisionModule, - QueueModule, - MonitoringModule, - ], - controllers: [HealthController], -}) -export class HealthModule {} \ No newline at end of file diff --git a/packages/worker/src/main.ts b/packages/worker/src/main.ts deleted file mode 100644 index 797e244..0000000 --- a/packages/worker/src/main.ts +++ /dev/null @@ -1,78 +0,0 @@ -import { NestFactory } from '@nestjs/core'; -import { Logger, ValidationPipe } from '@nestjs/common'; -import { ConfigService } from '@nestjs/config'; -import { AppModule } from './app.module'; - -async function bootstrap() { - const logger = new Logger('WorkerMain'); - - try { - // Create NestJS application - const app = await NestFactory.create(AppModule, { - logger: ['error', 'warn', 'log', 'debug', 'verbose'], - }); - - // Get configuration service - const configService = app.get(ConfigService); - - // Setup global validation pipe - app.useGlobalPipes(new ValidationPipe({ - whitelist: true, - forbidNonWhitelisted: true, - transform: true, - disableErrorMessages: false, - })); - - // Enable shutdown hooks for graceful shutdown - app.enableShutdownHooks(); - - // Get port from environment - const port = configService.get('WORKER_PORT', 3002); - const redisUrl = configService.get('REDIS_URL', 'redis://localhost:6379'); - const environment = configService.get('NODE_ENV', 'development'); - - logger.log(`Starting SEO Image Renamer Worker Service...`); - logger.log(`Environment: ${environment}`); - logger.log(`Port: ${port}`); - logger.log(`Redis URL: ${redisUrl}`); - - // Start the application - await app.listen(port); - - logger.log(`πŸš€ Worker service is running on port ${port}`); - logger.log(`πŸ”„ Queue processors are active and ready`); - logger.log(`πŸ€– AI vision services initialized`); - logger.log(`πŸ“¦ Storage services connected`); - - } catch (error) { - logger.error('Failed to start worker service', error.stack); - process.exit(1); - } -} - -// Handle uncaught exceptions -process.on('uncaughtException', (error) => { - const logger = new Logger('UncaughtException'); - logger.error('Uncaught Exception:', error); - process.exit(1); -}); - -// Handle unhandled promise rejections -process.on('unhandledRejection', (reason, promise) => { - const logger = new Logger('UnhandledRejection'); - logger.error('Unhandled Rejection at:', promise, 'reason:', reason); - process.exit(1); -}); - -// Graceful shutdown -process.on('SIGTERM', () => { - const logger = new Logger('SIGTERM'); - logger.log('Received SIGTERM signal. Starting graceful shutdown...'); -}); - -process.on('SIGINT', () => { - const logger = new Logger('SIGINT'); - logger.log('Received SIGINT signal. Starting graceful shutdown...'); -}); - -bootstrap(); \ No newline at end of file diff --git a/packages/worker/src/monitoring/monitoring.module.ts b/packages/worker/src/monitoring/monitoring.module.ts deleted file mode 100644 index 410cc44..0000000 --- a/packages/worker/src/monitoring/monitoring.module.ts +++ /dev/null @@ -1,10 +0,0 @@ -import { Module } from '@nestjs/common'; -import { ConfigModule } from '@nestjs/config'; -import { MetricsService } from './services/metrics.service'; - -@Module({ - imports: [ConfigModule], - providers: [MetricsService], - exports: [MetricsService], -}) -export class MonitoringModule {} \ No newline at end of file diff --git a/packages/worker/src/monitoring/services/metrics.service.ts b/packages/worker/src/monitoring/services/metrics.service.ts deleted file mode 100644 index 628e134..0000000 --- a/packages/worker/src/monitoring/services/metrics.service.ts +++ /dev/null @@ -1,296 +0,0 @@ -import { Injectable, Logger } from '@nestjs/common'; -import { ConfigService } from '@nestjs/config'; -import { register, collectDefaultMetrics, Counter, Histogram, Gauge } from 'prom-client'; - -@Injectable() -export class MetricsService { - private readonly logger = new Logger(MetricsService.name); - private readonly enabled: boolean; - - // Metrics collectors - private readonly jobsTotal: Counter; - private readonly jobDuration: Histogram; - private readonly jobsActive: Gauge; - private readonly processingErrors: Counter; - private readonly visionApiCalls: Counter; - private readonly visionApiDuration: Histogram; - private readonly storageOperations: Counter; - private readonly virusScansTotal: Counter; - private readonly tempFilesCount: Gauge; - - constructor(private configService: ConfigService) { - this.enabled = this.configService.get('METRICS_ENABLED', true); - - if (this.enabled) { - this.initializeMetrics(); - this.logger.log('Metrics service initialized'); - } else { - this.logger.warn('Metrics collection is disabled'); - } - } - - private initializeMetrics(): void { - // Enable default metrics collection - collectDefaultMetrics({ prefix: 'seo_worker_' }); - - // Job processing metrics - this.jobsTotal = new Counter({ - name: 'seo_worker_jobs_total', - help: 'Total number of jobs processed', - labelNames: ['queue', 'status'], - }); - - this.jobDuration = new Histogram({ - name: 'seo_worker_job_duration_seconds', - help: 'Duration of job processing', - labelNames: ['queue', 'type'], - buckets: [0.1, 0.5, 1, 2, 5, 10, 30, 60, 300, 600], // 0.1s to 10m - }); - - this.jobsActive = new Gauge({ - name: 'seo_worker_jobs_active', - help: 'Number of currently active jobs', - labelNames: ['queue'], - }); - - // Error metrics - this.processingErrors = new Counter({ - name: 'seo_worker_processing_errors_total', - help: 'Total number of processing errors', - labelNames: ['queue', 'error_type'], - }); - - // Vision API metrics - this.visionApiCalls = new Counter({ - name: 'seo_worker_vision_api_calls_total', - help: 'Total number of vision API calls', - labelNames: ['provider', 'status'], - }); - - this.visionApiDuration = new Histogram({ - name: 'seo_worker_vision_api_duration_seconds', - help: 'Duration of vision API calls', - labelNames: ['provider'], - buckets: [0.5, 1, 2, 5, 10, 15, 30, 60], // 0.5s to 1m - }); - - // Storage metrics - this.storageOperations = new Counter({ - name: 'seo_worker_storage_operations_total', - help: 'Total number of storage operations', - labelNames: ['operation', 'status'], - }); - - // Security metrics - this.virusScansTotal = new Counter({ - name: 'seo_worker_virus_scans_total', - help: 'Total number of virus scans performed', - labelNames: ['result'], - }); - - // Resource metrics - this.tempFilesCount = new Gauge({ - name: 'seo_worker_temp_files_count', - help: 'Number of temporary files currently stored', - }); - } - - /** - * Record job start - */ - recordJobStart(queue: string): void { - if (!this.enabled) return; - - this.jobsActive.inc({ queue }); - this.logger.debug(`Job started in queue: ${queue}`); - } - - /** - * Record job completion - */ - recordJobComplete(queue: string, duration: number, status: 'success' | 'failed'): void { - if (!this.enabled) return; - - this.jobsTotal.inc({ queue, status }); - this.jobDuration.observe({ queue, type: 'total' }, duration / 1000); // Convert to seconds - this.jobsActive.dec({ queue }); - - this.logger.debug(`Job completed in queue: ${queue}, status: ${status}, duration: ${duration}ms`); - } - - /** - * Record processing error - */ - recordProcessingError(queue: string, errorType: string): void { - if (!this.enabled) return; - - this.processingErrors.inc({ queue, error_type: errorType }); - this.logger.debug(`Processing error recorded: ${queue} - ${errorType}`); - } - - /** - * Record vision API call - */ - recordVisionApiCall(provider: string, duration: number, status: 'success' | 'failed'): void { - if (!this.enabled) return; - - this.visionApiCalls.inc({ provider, status }); - this.visionApiDuration.observe({ provider }, duration / 1000); - - this.logger.debug(`Vision API call: ${provider}, status: ${status}, duration: ${duration}ms`); - } - - /** - * Record storage operation - */ - recordStorageOperation(operation: string, status: 'success' | 'failed'): void { - if (!this.enabled) return; - - this.storageOperations.inc({ operation, status }); - this.logger.debug(`Storage operation: ${operation}, status: ${status}`); - } - - /** - * Record virus scan - */ - recordVirusScan(result: 'clean' | 'infected' | 'error'): void { - if (!this.enabled) return; - - this.virusScansTotal.inc({ result }); - this.logger.debug(`Virus scan recorded: ${result}`); - } - - /** - * Update temp files count - */ - updateTempFilesCount(count: number): void { - if (!this.enabled) return; - - this.tempFilesCount.set(count); - } - - /** - * Get metrics for Prometheus scraping - */ - async getMetrics(): Promise { - if (!this.enabled) { - return '# Metrics collection is disabled\n'; - } - - try { - return await register.metrics(); - } catch (error) { - this.logger.error('Failed to collect metrics:', error.message); - return '# Error collecting metrics\n'; - } - } - - /** - * Get metrics in JSON format - */ - async getMetricsJson(): Promise { - if (!this.enabled) { - return { enabled: false }; - } - - try { - const metrics = await register.getMetricsAsJSON(); - return { - enabled: true, - timestamp: new Date().toISOString(), - metrics, - }; - } catch (error) { - this.logger.error('Failed to get metrics as JSON:', error.message); - return { enabled: true, error: error.message }; - } - } - - /** - * Reset all metrics (useful for testing) - */ - reset(): void { - if (!this.enabled) return; - - register.clear(); - this.initializeMetrics(); - this.logger.log('Metrics reset'); - } - - /** - * Custom counter increment - */ - incrementCounter(name: string, labels: Record = {}): void { - if (!this.enabled) return; - - try { - const counter = register.getSingleMetric(name) as Counter; - if (counter) { - counter.inc(labels); - } - } catch (error) { - this.logger.warn(`Failed to increment counter ${name}:`, error.message); - } - } - - /** - * Custom histogram observation - */ - observeHistogram(name: string, value: number, labels: Record = {}): void { - if (!this.enabled) return; - - try { - const histogram = register.getSingleMetric(name) as Histogram; - if (histogram) { - histogram.observe(labels, value); - } - } catch (error) { - this.logger.warn(`Failed to observe histogram ${name}:`, error.message); - } - } - - /** - * Custom gauge set - */ - setGauge(name: string, value: number, labels: Record = {}): void { - if (!this.enabled) return; - - try { - const gauge = register.getSingleMetric(name) as Gauge; - if (gauge) { - gauge.set(labels, value); - } - } catch (error) { - this.logger.warn(`Failed to set gauge ${name}:`, error.message); - } - } - - /** - * Health check for metrics service - */ - isHealthy(): boolean { - if (!this.enabled) return true; - - try { - // Test if we can collect metrics - register.metrics(); - return true; - } catch (error) { - this.logger.error('Metrics service health check failed:', error.message); - return false; - } - } - - /** - * Get service configuration - */ - getConfiguration(): { - enabled: boolean; - registeredMetrics: number; - } { - return { - enabled: this.enabled, - registeredMetrics: this.enabled ? register.getMetricsAsArray().length : 0, - }; - } -} \ No newline at end of file diff --git a/packages/worker/src/processors/batch.processor.ts b/packages/worker/src/processors/batch.processor.ts deleted file mode 100644 index 0dfc616..0000000 --- a/packages/worker/src/processors/batch.processor.ts +++ /dev/null @@ -1,470 +0,0 @@ -import { Processor, WorkerHost, OnWorkerEvent } from '@nestjs/bullmq'; -import { Logger } from '@nestjs/common'; -import { ConfigService } from '@nestjs/config'; -import { Job } from 'bullmq'; -import { DatabaseService } from '../database/database.service'; -import { ProgressTrackerService } from '../queue/progress-tracker.service'; -import { ZipCreatorService } from '../storage/zip-creator.service'; -import { StorageService } from '../storage/storage.service'; - -export interface BatchProcessingJobData { - batchId: string; - userId: string; - imageIds: string[]; - keywords?: string[]; - processingOptions?: { - createZip?: boolean; - zipName?: string; - notifyUser?: boolean; - }; -} - -export interface BatchProgress { - percentage: number; - completedImages: number; - totalImages: number; - failedImages: number; - status: string; - currentStep?: string; - estimatedTimeRemaining?: number; -} - -@Processor('batch-processing') -export class BatchProcessor extends WorkerHost { - private readonly logger = new Logger(BatchProcessor.name); - - constructor( - private configService: ConfigService, - private databaseService: DatabaseService, - private progressTracker: ProgressTrackerService, - private zipCreatorService: ZipCreatorService, - private storageService: StorageService, - ) { - super(); - } - - async process(job: Job): Promise { - const startTime = Date.now(); - const { batchId, userId, imageIds, keywords, processingOptions } = job.data; - - this.logger.log(`πŸš€ Starting batch processing: ${batchId} (${imageIds.length} images)`); - - try { - // Step 1: Initialize batch processing (5%) - await this.updateBatchProgress(job, { - percentage: 5, - completedImages: 0, - totalImages: imageIds.length, - failedImages: 0, - status: 'initializing', - currentStep: 'Initializing batch processing', - }); - - // Update batch status in database - await this.databaseService.updateBatchStatus(batchId, 'processing', { - startedAt: new Date(), - totalImages: imageIds.length, - processingJobId: job.id, - }); - - // Step 2: Wait for all image processing jobs to complete (80%) - await this.updateBatchProgress(job, { - percentage: 10, - completedImages: 0, - totalImages: imageIds.length, - failedImages: 0, - status: 'processing-images', - currentStep: 'Processing individual images', - }); - - const completionResults = await this.waitForImageCompletion(job, batchId, imageIds); - - const { completed, failed } = completionResults; - const successfulImageIds = completed.map(result => result.imageId); - const failedImageIds = failed.map(result => result.imageId); - - this.logger.log(`Batch ${batchId}: ${completed.length} successful, ${failed.length} failed`); - - // Step 3: Generate batch summary (85%) - await this.updateBatchProgress(job, { - percentage: 85, - completedImages: completed.length, - totalImages: imageIds.length, - failedImages: failed.length, - status: 'generating-summary', - currentStep: 'Generating batch summary', - }); - - const batchSummary = await this.generateBatchSummary(batchId, completed, failed, keywords); - - // Step 4: Create ZIP file if requested (90%) - let zipDownloadUrl: string | null = null; - if (processingOptions?.createZip && successfulImageIds.length > 0) { - await this.updateBatchProgress(job, { - percentage: 90, - completedImages: completed.length, - totalImages: imageIds.length, - failedImages: failed.length, - status: 'creating-zip', - currentStep: 'Creating downloadable ZIP file', - }); - - zipDownloadUrl = await this.createBatchZip( - batchId, - successfulImageIds, - processingOptions.zipName || `batch-${batchId}-renamed` - ); - } - - // Step 5: Finalize batch (95%) - await this.updateBatchProgress(job, { - percentage: 95, - completedImages: completed.length, - totalImages: imageIds.length, - failedImages: failed.length, - status: 'finalizing', - currentStep: 'Finalizing batch processing', - }); - - // Update batch in database with final results - const finalStatus = failed.length === 0 ? 'completed' : 'completed_with_errors'; - await this.databaseService.updateBatchStatus(batchId, finalStatus, { - completedAt: new Date(), - completedImages: completed.length, - failedImages: failed.length, - summary: batchSummary, - zipDownloadUrl, - processingTime: Date.now() - startTime, - }); - - // Step 6: Complete (100%) - await this.updateBatchProgress(job, { - percentage: 100, - completedImages: completed.length, - totalImages: imageIds.length, - failedImages: failed.length, - status: 'completed', - currentStep: 'Batch processing completed', - }); - - // Send notification if requested - if (processingOptions?.notifyUser) { - await this.sendBatchCompletionNotification(userId, batchId, batchSummary, zipDownloadUrl); - } - - const totalProcessingTime = Date.now() - startTime; - this.logger.log(`βœ… Batch processing completed: ${batchId} in ${totalProcessingTime}ms`); - - return { - batchId, - success: true, - summary: batchSummary, - zipDownloadUrl, - processingTime: totalProcessingTime, - completedImages: completed.length, - failedImages: failed.length, - }; - - } catch (error) { - const processingTime = Date.now() - startTime; - this.logger.error(`❌ Batch processing failed: ${batchId} - ${error.message}`, error.stack); - - // Update batch with error status - await this.databaseService.updateBatchStatus(batchId, 'failed', { - error: error.message, - failedAt: new Date(), - processingTime, - }); - - // Update progress - Failed - await this.updateBatchProgress(job, { - percentage: 0, - completedImages: 0, - totalImages: imageIds.length, - failedImages: imageIds.length, - status: 'failed', - currentStep: `Batch processing failed: ${error.message}`, - }); - - throw error; - } - } - - /** - * Wait for all image processing jobs to complete - */ - private async waitForImageCompletion( - job: Job, - batchId: string, - imageIds: string[] - ): Promise<{ completed: any[]; failed: any[] }> { - const completed: any[] = []; - const failed: any[] = []; - const pollingInterval = 2000; // 2 seconds - const maxWaitTime = 30 * 60 * 1000; // 30 minutes - const startTime = Date.now(); - - while (completed.length + failed.length < imageIds.length) { - // Check if we've exceeded max wait time - if (Date.now() - startTime > maxWaitTime) { - const remaining = imageIds.length - completed.length - failed.length; - this.logger.warn(`Batch ${batchId} timeout: ${remaining} images still processing`); - - // Mark remaining images as failed due to timeout - for (let i = completed.length + failed.length; i < imageIds.length; i++) { - failed.push({ - imageId: imageIds[i], - error: 'Processing timeout', - }); - } - break; - } - - // Get current status from database - const imageStatuses = await this.databaseService.getImageStatuses(imageIds); - - // Count completed and failed images - const newCompleted = imageStatuses.filter(img => - img.status === 'completed' && !completed.some(c => c.imageId === img.id) - ); - - const newFailed = imageStatuses.filter(img => - img.status === 'failed' && !failed.some(f => f.imageId === img.id) - ); - - // Add new completions - completed.push(...newCompleted.map(img => ({ - imageId: img.id, - proposedName: img.proposedName, - visionAnalysis: img.visionAnalysis, - }))); - - // Add new failures - failed.push(...newFailed.map(img => ({ - imageId: img.id, - error: img.error || 'Unknown processing error', - }))); - - // Update progress - const progressPercentage = Math.min( - 85, // Max 85% for image processing phase - 10 + (completed.length + failed.length) / imageIds.length * 75 - ); - - await this.updateBatchProgress(job, { - percentage: progressPercentage, - completedImages: completed.length, - totalImages: imageIds.length, - failedImages: failed.length, - status: 'processing-images', - currentStep: `Processing images: ${completed.length + failed.length}/${imageIds.length}`, - estimatedTimeRemaining: this.estimateRemainingTime( - startTime, - completed.length + failed.length, - imageIds.length - ), - }); - - // Wait before next polling - if (completed.length + failed.length < imageIds.length) { - await this.sleep(pollingInterval); - } - } - - return { completed, failed }; - } - - /** - * Generate comprehensive batch summary - */ - private async generateBatchSummary( - batchId: string, - completed: any[], - failed: any[], - keywords?: string[] - ): Promise { - const totalImages = completed.length + failed.length; - const successRate = (completed.length / totalImages) * 100; - - // Analyze vision results - const visionStats = this.analyzeVisionResults(completed); - - // Generate keyword analysis - const keywordAnalysis = this.analyzeKeywords(completed, keywords); - - return { - batchId, - totalImages, - completedImages: completed.length, - failedImages: failed.length, - successRate: Math.round(successRate * 100) / 100, - visionStats, - keywordAnalysis, - completedAt: new Date(), - failureReasons: failed.map(f => f.error), - }; - } - - private analyzeVisionResults(completed: any[]): any { - if (completed.length === 0) return null; - - const confidences = completed - .map(img => img.visionAnalysis?.confidence) - .filter(conf => conf !== undefined); - - const avgConfidence = confidences.length > 0 - ? confidences.reduce((sum, conf) => sum + conf, 0) / confidences.length - : 0; - - const providersUsed = completed - .flatMap(img => img.visionAnalysis?.providersUsed || []) - .reduce((acc, provider) => { - acc[provider] = (acc[provider] || 0) + 1; - return acc; - }, {} as Record); - - const commonObjects = this.findCommonElements( - completed.flatMap(img => img.visionAnalysis?.objects || []) - ); - - const commonColors = this.findCommonElements( - completed.flatMap(img => img.visionAnalysis?.colors || []) - ); - - return { - averageConfidence: Math.round(avgConfidence * 100) / 100, - providersUsed, - commonObjects: commonObjects.slice(0, 10), - commonColors: commonColors.slice(0, 5), - }; - } - - private analyzeKeywords(completed: any[], userKeywords?: string[]): any { - const generatedKeywords = completed.flatMap(img => img.visionAnalysis?.tags || []); - const keywordFrequency = this.findCommonElements(generatedKeywords); - - return { - userKeywords: userKeywords || [], - generatedKeywords: keywordFrequency.slice(0, 20), - totalUniqueKeywords: new Set(generatedKeywords).size, - }; - } - - private findCommonElements(array: string[]): Array<{ element: string; count: number }> { - const frequency = array.reduce((acc, element) => { - acc[element] = (acc[element] || 0) + 1; - return acc; - }, {} as Record); - - return Object.entries(frequency) - .map(([element, count]) => ({ element, count })) - .sort((a, b) => b.count - a.count); - } - - /** - * Create ZIP file with renamed images - */ - private async createBatchZip( - batchId: string, - imageIds: string[], - zipName: string - ): Promise { - try { - const zipPath = await this.zipCreatorService.createBatchZip( - batchId, - imageIds, - zipName - ); - - // Upload ZIP to storage and get download URL - const zipKey = `downloads/${batchId}/${zipName}.zip`; - await this.storageService.uploadFile(zipPath, zipKey); - - const downloadUrl = await this.storageService.generateSignedUrl(zipKey, 24 * 60 * 60); // 24 hours - - // Cleanup local ZIP file - await this.zipCreatorService.cleanupZipFile(zipPath); - - return downloadUrl; - - } catch (error) { - this.logger.error(`Failed to create ZIP for batch ${batchId}:`, error.message); - throw new Error(`ZIP creation failed: ${error.message}`); - } - } - - /** - * Send batch completion notification - */ - private async sendBatchCompletionNotification( - userId: string, - batchId: string, - summary: any, - zipDownloadUrl?: string | null - ): Promise { - try { - // Broadcast via WebSocket - await this.progressTracker.broadcastBatchComplete(batchId, { - summary, - zipDownloadUrl, - completedAt: new Date(), - }); - - // TODO: Send email notification if configured - this.logger.log(`Batch completion notification sent for batch ${batchId}`); - - } catch (error) { - this.logger.warn(`Failed to send notification for batch ${batchId}:`, error.message); - } - } - - private estimateRemainingTime( - startTime: number, - completed: number, - total: number - ): number | undefined { - if (completed === 0) return undefined; - - const elapsed = Date.now() - startTime; - const avgTimePerImage = elapsed / completed; - const remaining = total - completed; - - return Math.round(avgTimePerImage * remaining); - } - - private sleep(ms: number): Promise { - return new Promise(resolve => setTimeout(resolve, ms)); - } - - private async updateBatchProgress(job: Job, progress: BatchProgress): Promise { - try { - await job.updateProgress(progress); - - // Broadcast progress to WebSocket clients - await this.progressTracker.broadcastBatchProgress(job.data.batchId, progress); - - } catch (error) { - this.logger.warn(`Failed to update batch progress for job ${job.id}:`, error.message); - } - } - - @OnWorkerEvent('completed') - onCompleted(job: Job) { - this.logger.log(`βœ… Batch processing job completed: ${job.id}`); - } - - @OnWorkerEvent('failed') - onFailed(job: Job, err: Error) { - this.logger.error(`❌ Batch processing job failed: ${job.id}`, err.stack); - } - - @OnWorkerEvent('progress') - onProgress(job: Job, progress: BatchProgress) { - this.logger.debug(`πŸ“Š Batch processing progress: ${job.id} - ${progress.percentage}% (${progress.currentStep})`); - } - - @OnWorkerEvent('stalled') - onStalled(jobId: string) { - this.logger.warn(`⚠️ Batch processing job stalled: ${jobId}`); - } -} \ No newline at end of file diff --git a/packages/worker/src/processors/filename-generator.processor.ts b/packages/worker/src/processors/filename-generator.processor.ts deleted file mode 100644 index 11c4d72..0000000 --- a/packages/worker/src/processors/filename-generator.processor.ts +++ /dev/null @@ -1,553 +0,0 @@ -import { Processor, WorkerHost, OnWorkerEvent } from '@nestjs/bullmq'; -import { Logger } from '@nestjs/common'; -import { ConfigService } from '@nestjs/config'; -import { Job } from 'bullmq'; -import { VisionService } from '../vision/vision.service'; -import { DatabaseService } from '../database/database.service'; -import sanitize from 'sanitize-filename'; -import * as _ from 'lodash'; - -export interface FilenameGenerationJobData { - imageId: string; - batchId?: string; - userId: string; - visionAnalysis?: any; - userKeywords?: string[]; - originalFilename: string; - options?: { - maxLength?: number; - includeColors?: boolean; - includeDimensions?: boolean; - customPattern?: string; - preserveExtension?: boolean; - }; -} - -export interface FilenameProgress { - percentage: number; - status: string; - currentStep?: string; - generatedNames?: string[]; - selectedName?: string; -} - -@Processor('filename-generation') -export class FilenameGeneratorProcessor extends WorkerHost { - private readonly logger = new Logger(FilenameGeneratorProcessor.name); - - // Common words to filter out from filenames - private readonly STOP_WORDS = [ - 'the', 'and', 'or', 'but', 'in', 'on', 'at', 'to', 'for', 'of', 'with', - 'by', 'from', 'up', 'about', 'into', 'through', 'during', 'before', - 'after', 'above', 'below', 'is', 'are', 'was', 'were', 'be', 'been', - 'being', 'have', 'has', 'had', 'do', 'does', 'did', 'will', 'would', - 'could', 'should', 'may', 'might', 'must', 'can', 'image', 'photo', - 'picture', 'file', 'jpeg', 'jpg', 'png', 'gif', 'webp' - ]; - - constructor( - private configService: ConfigService, - private visionService: VisionService, - private databaseService: DatabaseService, - ) { - super(); - } - - async process(job: Job): Promise { - const startTime = Date.now(); - const { - imageId, - batchId, - userId, - visionAnalysis, - userKeywords, - originalFilename, - options - } = job.data; - - this.logger.log(`πŸ“ Starting filename generation: ${imageId}`); - - try { - // Step 1: Initialize (10%) - await this.updateProgress(job, { - percentage: 10, - status: 'initializing', - currentStep: 'Preparing filename generation', - }); - - // Step 2: Extract and process keywords (30%) - await this.updateProgress(job, { - percentage: 30, - status: 'extracting-keywords', - currentStep: 'Extracting keywords from vision analysis', - }); - - const processedKeywords = await this.extractAndProcessKeywords( - visionAnalysis, - userKeywords, - options - ); - - // Step 3: Generate multiple filename variations (60%) - await this.updateProgress(job, { - percentage: 60, - status: 'generating-variations', - currentStep: 'Generating filename variations', - }); - - const filenameVariations = await this.generateFilenameVariations( - processedKeywords, - originalFilename, - visionAnalysis, - options - ); - - // Step 4: Select best filename (80%) - await this.updateProgress(job, { - percentage: 80, - status: 'selecting-best', - currentStep: 'Selecting optimal filename', - }); - - const selectedFilename = await this.selectBestFilename( - filenameVariations, - visionAnalysis, - options - ); - - // Step 5: Validate and finalize (95%) - await this.updateProgress(job, { - percentage: 95, - status: 'finalizing', - currentStep: 'Validating and finalizing filename', - }); - - const finalFilename = await this.validateAndSanitizeFilename( - selectedFilename, - originalFilename, - options - ); - - // Step 6: Update database (100%) - await this.updateProgress(job, { - percentage: 100, - status: 'completed', - currentStep: 'Saving generated filename', - selectedName: finalFilename, - }); - - // Save the generated filename to database - await this.databaseService.updateImageFilename(imageId, { - proposedName: finalFilename, - variations: filenameVariations, - keywords: processedKeywords, - generatedAt: new Date(), - generationStats: { - processingTime: Date.now() - startTime, - variationsGenerated: filenameVariations.length, - keywordsUsed: processedKeywords.length, - }, - }); - - const totalProcessingTime = Date.now() - startTime; - this.logger.log(`βœ… Filename generation completed: ${imageId} -> "${finalFilename}" in ${totalProcessingTime}ms`); - - return { - imageId, - success: true, - finalFilename, - variations: filenameVariations, - keywords: processedKeywords, - processingTime: totalProcessingTime, - }; - - } catch (error) { - const processingTime = Date.now() - startTime; - this.logger.error(`❌ Filename generation failed: ${imageId} - ${error.message}`, error.stack); - - // Update progress - Failed - await this.updateProgress(job, { - percentage: 0, - status: 'failed', - currentStep: `Generation failed: ${error.message}`, - }); - - // Fallback to sanitized original filename - const fallbackName = this.sanitizeFilename(originalFilename); - await this.databaseService.updateImageFilename(imageId, { - proposedName: fallbackName, - error: error.message, - fallback: true, - generatedAt: new Date(), - }); - - throw error; - } - } - - /** - * Extract and process keywords from various sources - */ - private async extractAndProcessKeywords( - visionAnalysis: any, - userKeywords?: string[], - options?: any - ): Promise { - const keywords: string[] = []; - - // 1. Add user keywords with highest priority - if (userKeywords && userKeywords.length > 0) { - keywords.push(...userKeywords.slice(0, 5)); // Limit to 5 user keywords - } - - // 2. Add vision analysis objects - if (visionAnalysis?.objects) { - keywords.push(...visionAnalysis.objects.slice(0, 6)); - } - - // 3. Add high-confidence vision tags - if (visionAnalysis?.tags) { - keywords.push(...visionAnalysis.tags.slice(0, 4)); - } - - // 4. Add colors if enabled - if (options?.includeColors && visionAnalysis?.colors) { - keywords.push(...visionAnalysis.colors.slice(0, 2)); - } - - // 5. Extract keywords from scene description - if (visionAnalysis?.scene) { - const sceneKeywords = this.extractKeywordsFromText(visionAnalysis.scene); - keywords.push(...sceneKeywords.slice(0, 3)); - } - - // Process and clean keywords - return this.processKeywords(keywords); - } - - /** - * Process and clean keywords - */ - private processKeywords(keywords: string[]): string[] { - return keywords - .map(keyword => keyword.toLowerCase().trim()) - .filter(keyword => keyword.length > 2) // Remove very short words - .filter(keyword => !this.STOP_WORDS.includes(keyword)) // Remove stop words - .filter(keyword => /^[a-z0-9\s-]+$/i.test(keyword)) // Only alphanumeric and basic chars - .map(keyword => keyword.replace(/\s+/g, '-')) // Replace spaces with hyphens - .filter((keyword, index, arr) => arr.indexOf(keyword) === index) // Remove duplicates - .slice(0, 10); // Limit total keywords - } - - /** - * Extract keywords from text description - */ - private extractKeywordsFromText(text: string): string[] { - return text - .toLowerCase() - .split(/[^a-z0-9]+/) - .filter(word => word.length > 3) - .filter(word => !this.STOP_WORDS.includes(word)) - .slice(0, 5); - } - - /** - * Generate multiple filename variations - */ - private async generateFilenameVariations( - keywords: string[], - originalFilename: string, - visionAnalysis: any, - options?: any - ): Promise { - const variations: string[] = []; - const extension = this.getFileExtension(originalFilename); - - if (keywords.length === 0) { - return [this.sanitizeFilename(originalFilename)]; - } - - // Strategy 1: Main objects + descriptive words - if (keywords.length >= 3) { - const mainKeywords = keywords.slice(0, 4); - variations.push(this.buildFilename(mainKeywords, extension, options)); - } - - // Strategy 2: Scene-based naming - if (visionAnalysis?.scene && keywords.length >= 2) { - const sceneKeywords = [ - ...this.extractKeywordsFromText(visionAnalysis.scene).slice(0, 2), - ...keywords.slice(0, 3) - ]; - variations.push(this.buildFilename(sceneKeywords, extension, options)); - } - - // Strategy 3: Object + color combination - if (options?.includeColors && visionAnalysis?.colors?.length > 0) { - const colorKeywords = [ - ...keywords.slice(0, 3), - ...visionAnalysis.colors.slice(0, 1) - ]; - variations.push(this.buildFilename(colorKeywords, extension, options)); - } - - // Strategy 4: Descriptive approach - if (visionAnalysis?.description) { - const descriptiveKeywords = [ - ...this.extractKeywordsFromText(visionAnalysis.description).slice(0, 2), - ...keywords.slice(0, 3) - ]; - variations.push(this.buildFilename(descriptiveKeywords, extension, options)); - } - - // Strategy 5: Short and concise - const shortKeywords = keywords.slice(0, 3); - variations.push(this.buildFilename(shortKeywords, extension, options)); - - // Strategy 6: Long descriptive (if many keywords available) - if (keywords.length >= 5) { - const longKeywords = keywords.slice(0, 6); - variations.push(this.buildFilename(longKeywords, extension, options)); - } - - // Strategy 7: Custom pattern if provided - if (options?.customPattern) { - const customFilename = this.applyCustomPattern( - options.customPattern, - keywords, - visionAnalysis, - extension - ); - if (customFilename) { - variations.push(customFilename); - } - } - - // Remove duplicates and empty strings - return [...new Set(variations)].filter(name => name && name.length > 0); - } - - /** - * Build filename from keywords - */ - private buildFilename( - keywords: string[], - extension: string, - options?: any - ): string { - if (keywords.length === 0) return ''; - - let filename = keywords - .filter(keyword => keyword && keyword.length > 0) - .join('-') - .toLowerCase() - .replace(/[^a-z0-9-]/g, '') // Remove special characters - .replace(/-+/g, '-') // Replace multiple hyphens with single - .replace(/^-|-$/g, ''); // Remove leading/trailing hyphens - - // Apply length limit - const maxLength = options?.maxLength || 60; - if (filename.length > maxLength) { - filename = filename.substring(0, maxLength).replace(/-[^-]*$/, ''); // Cut at word boundary - } - - return filename ? `${filename}.${extension}` : ''; - } - - /** - * Apply custom filename pattern - */ - private applyCustomPattern( - pattern: string, - keywords: string[], - visionAnalysis: any, - extension: string - ): string { - try { - let filename = pattern; - - // Replace placeholders - filename = filename.replace(/{keywords}/g, keywords.slice(0, 5).join('-')); - filename = filename.replace(/{objects}/g, (visionAnalysis?.objects || []).slice(0, 3).join('-')); - filename = filename.replace(/{colors}/g, (visionAnalysis?.colors || []).slice(0, 2).join('-')); - filename = filename.replace(/{scene}/g, this.extractKeywordsFromText(visionAnalysis?.scene || '').slice(0, 2).join('-')); - filename = filename.replace(/{timestamp}/g, new Date().toISOString().slice(0, 10)); - - // Clean and sanitize - filename = filename - .toLowerCase() - .replace(/[^a-z0-9-]/g, '') - .replace(/-+/g, '-') - .replace(/^-|-$/g, ''); - - return filename ? `${filename}.${extension}` : ''; - - } catch (error) { - this.logger.warn(`Failed to apply custom pattern: ${error.message}`); - return ''; - } - } - - /** - * Select the best filename from variations - */ - private async selectBestFilename( - variations: string[], - visionAnalysis: any, - options?: any - ): Promise { - if (variations.length === 0) { - throw new Error('No filename variations generated'); - } - - if (variations.length === 1) { - return variations[0]; - } - - // Score each variation based on different criteria - const scoredVariations = variations.map(filename => ({ - filename, - score: this.scoreFilename(filename, visionAnalysis, options), - })); - - // Sort by score (highest first) - scoredVariations.sort((a, b) => b.score - a.score); - - this.logger.debug(`Filename scoring results:`, scoredVariations); - - return scoredVariations[0].filename; - } - - /** - * Score filename based on SEO and usability criteria - */ - private scoreFilename(filename: string, visionAnalysis: any, options?: any): number { - let score = 0; - const nameWithoutExtension = filename.replace(/\.[^.]+$/, ''); - const keywords = nameWithoutExtension.split('-'); - - // Length scoring (optimal 30-50 characters) - const nameLength = nameWithoutExtension.length; - if (nameLength >= 20 && nameLength <= 50) { - score += 20; - } else if (nameLength >= 15 && nameLength <= 60) { - score += 10; - } else if (nameLength < 15) { - score += 5; - } - - // Keyword count scoring (optimal 3-5 keywords) - const keywordCount = keywords.length; - if (keywordCount >= 3 && keywordCount <= 5) { - score += 15; - } else if (keywordCount >= 2 && keywordCount <= 6) { - score += 10; - } - - // Keyword quality scoring - if (visionAnalysis?.confidence) { - score += Math.round(visionAnalysis.confidence * 10); - } - - // Readability scoring (avoid too many hyphens in a row) - if (!/--/.test(nameWithoutExtension)) { - score += 10; - } - - // Avoid starting or ending with numbers - if (!/^[0-9]/.test(nameWithoutExtension) && !/[0-9]$/.test(nameWithoutExtension)) { - score += 5; - } - - // Bonus for including high-confidence objects - if (visionAnalysis?.objects) { - const objectsIncluded = visionAnalysis.objects.filter((obj: string) => - nameWithoutExtension.includes(obj.toLowerCase()) - ).length; - score += objectsIncluded * 3; - } - - return score; - } - - /** - * Validate and sanitize final filename - */ - private async validateAndSanitizeFilename( - filename: string, - originalFilename: string, - options?: any - ): Promise { - if (!filename || filename.trim().length === 0) { - return this.sanitizeFilename(originalFilename); - } - - // Sanitize using sanitize-filename library - let sanitized = sanitize(filename, { replacement: '-' }); - - // Additional cleanup - sanitized = sanitized - .toLowerCase() - .replace(/[^a-z0-9.-]/g, '-') - .replace(/-+/g, '-') - .replace(/^-|-$/g, ''); - - // Ensure it has an extension - if (!sanitized.includes('.')) { - const extension = this.getFileExtension(originalFilename); - sanitized = `${sanitized}.${extension}`; - } - - // Ensure minimum length - const nameWithoutExtension = sanitized.replace(/\.[^.]+$/, ''); - if (nameWithoutExtension.length < 3) { - const fallback = this.sanitizeFilename(originalFilename); - this.logger.warn(`Generated filename too short: "${sanitized}", using fallback: "${fallback}"`); - return fallback; - } - - return sanitized; - } - - private sanitizeFilename(filename: string): string { - return sanitize(filename, { replacement: '-' }) - .toLowerCase() - .replace(/[^a-z0-9.-]/g, '-') - .replace(/-+/g, '-') - .replace(/^-|-$/g, ''); - } - - private getFileExtension(filename: string): string { - const parts = filename.split('.'); - return parts.length > 1 ? parts.pop()!.toLowerCase() : 'jpg'; - } - - private async updateProgress(job: Job, progress: FilenameProgress): Promise { - try { - await job.updateProgress(progress); - } catch (error) { - this.logger.warn(`Failed to update filename generation progress for job ${job.id}:`, error.message); - } - } - - @OnWorkerEvent('completed') - onCompleted(job: Job) { - const result = job.returnvalue; - this.logger.log(`βœ… Filename generation completed: ${job.id} -> "${result?.finalFilename}"`); - } - - @OnWorkerEvent('failed') - onFailed(job: Job, err: Error) { - this.logger.error(`❌ Filename generation job failed: ${job.id}`, err.stack); - } - - @OnWorkerEvent('progress') - onProgress(job: Job, progress: FilenameProgress) { - this.logger.debug(`πŸ“ Filename generation progress: ${job.id} - ${progress.percentage}% (${progress.currentStep})`); - } - - @OnWorkerEvent('stalled') - onStalled(jobId: string) { - this.logger.warn(`⚠️ Filename generation job stalled: ${jobId}`); - } -} \ No newline at end of file diff --git a/packages/worker/src/processors/image.processor.ts b/packages/worker/src/processors/image.processor.ts deleted file mode 100644 index f1bbef2..0000000 --- a/packages/worker/src/processors/image.processor.ts +++ /dev/null @@ -1,348 +0,0 @@ -import { Processor, WorkerHost, OnWorkerEvent } from '@nestjs/bullmq'; -import { Logger, Inject } from '@nestjs/common'; -import { ConfigService } from '@nestjs/config'; -import { Job } from 'bullmq'; -import { VisionService } from '../vision/vision.service'; -import { StorageService } from '../storage/storage.service'; -import { VirusScanService } from '../security/virus-scan.service'; -import { FileProcessorService } from '../storage/file-processor.service'; -import { DatabaseService } from '../database/database.service'; -import { ProgressTrackerService } from '../queue/progress-tracker.service'; - -export interface ImageProcessingJobData { - imageId: string; - batchId: string; - s3Key: string; - originalName: string; - userId: string; - keywords?: string[]; - processingOptions?: { - skipVirusScan?: boolean; - preferredVisionProvider?: string; - maxRetries?: number; - }; -} - -export interface JobProgress { - percentage: number; - currentImage?: string; - processedCount: number; - totalCount: number; - status: string; - currentStep?: string; - error?: string; -} - -@Processor('image-processing') -export class ImageProcessor extends WorkerHost { - private readonly logger = new Logger(ImageProcessor.name); - - constructor( - private configService: ConfigService, - private visionService: VisionService, - private storageService: StorageService, - private virusScanService: VirusScanService, - private fileProcessorService: FileProcessorService, - private databaseService: DatabaseService, - private progressTracker: ProgressTrackerService, - ) { - super(); - } - - async process(job: Job): Promise { - const startTime = Date.now(); - const { imageId, batchId, s3Key, originalName, userId, keywords, processingOptions } = job.data; - - this.logger.log(`πŸš€ Starting image processing: ${imageId} (${originalName})`); - - let tempFilePath: string | null = null; - let processedFilePath: string | null = null; - - try { - // Step 1: Initialize progress tracking (5%) - await this.updateProgress(job, { - percentage: 5, - currentImage: originalName, - processedCount: 0, - totalCount: 1, - status: 'initializing', - currentStep: 'Setting up processing pipeline', - }); - - // Update database with processing status - await this.databaseService.updateImageStatus(imageId, 'processing', { - startedAt: new Date(), - processingJobId: job.id, - }); - - // Step 2: Download image from storage (15%) - await this.updateProgress(job, { - percentage: 15, - currentImage: originalName, - processedCount: 0, - totalCount: 1, - status: 'downloading', - currentStep: 'Downloading image from cloud storage', - }); - - tempFilePath = await this.storageService.downloadToTemp(s3Key); - this.logger.debug(`Image downloaded to temp: ${tempFilePath}`); - - // Step 3: Validate file and extract metadata (25%) - await this.updateProgress(job, { - percentage: 25, - currentImage: originalName, - processedCount: 0, - totalCount: 1, - status: 'validating', - currentStep: 'Validating file and extracting metadata', - }); - - const metadata = await this.fileProcessorService.extractMetadata(tempFilePath); - this.logger.debug(`Extracted metadata:`, metadata); - - // Step 4: Virus scan (35% - optional) - if (!processingOptions?.skipVirusScan && this.virusScanService.isEnabled()) { - await this.updateProgress(job, { - percentage: 35, - currentImage: originalName, - processedCount: 0, - totalCount: 1, - status: 'scanning', - currentStep: 'Performing virus scan', - }); - - const scanResult = await this.virusScanService.scanFile(tempFilePath); - if (!scanResult.clean) { - throw new Error(`Virus detected: ${scanResult.threat || 'Unknown threat'}`); - } - this.logger.debug('Virus scan passed'); - } - - // Step 5: Process and optimize image (45%) - await this.updateProgress(job, { - percentage: 45, - currentImage: originalName, - processedCount: 0, - totalCount: 1, - status: 'processing', - currentStep: 'Optimizing image quality and format', - }); - - processedFilePath = await this.fileProcessorService.optimizeImage(tempFilePath, { - quality: 85, - maxWidth: 2048, - maxHeight: 2048, - preserveExif: true, - }); - - // Step 6: Upload to storage for AI analysis (55%) - await this.updateProgress(job, { - percentage: 55, - currentImage: originalName, - processedCount: 0, - totalCount: 1, - status: 'uploading', - currentStep: 'Preparing image for AI analysis', - }); - - const analysisUrl = await this.storageService.getPublicUrl(s3Key); - - // Step 7: AI Vision analysis (75%) - await this.updateProgress(job, { - percentage: 75, - currentImage: originalName, - processedCount: 0, - totalCount: 1, - status: 'analyzing', - currentStep: 'Performing AI vision analysis', - }); - - const visionResult = await this.visionService.analyzeImage( - analysisUrl, - keywords, - undefined, - processingOptions?.preferredVisionProvider - ); - - if (!visionResult.success) { - throw new Error(`Vision analysis failed: ${visionResult.error}`); - } - - this.logger.debug(`Vision analysis completed with confidence: ${visionResult.finalConfidence}`); - - // Step 8: Generate SEO filename (85%) - await this.updateProgress(job, { - percentage: 85, - currentImage: originalName, - processedCount: 0, - totalCount: 1, - status: 'generating-filename', - currentStep: 'Generating SEO-optimized filename', - }); - - const proposedName = await this.visionService.generateSeoFilename( - visionResult, - originalName, - 80 - ); - - // Step 9: Update database with results (95%) - await this.updateProgress(job, { - percentage: 95, - currentImage: originalName, - processedCount: 0, - totalCount: 1, - status: 'updating-database', - currentStep: 'Saving analysis results', - }); - - const processingResult = { - visionAnalysis: { - objects: visionResult.finalObjects, - colors: visionResult.finalColors, - scene: visionResult.finalScene, - description: visionResult.finalDescription, - tags: visionResult.finalTags, - confidence: visionResult.finalConfidence, - providersUsed: visionResult.providersUsed, - processingTime: visionResult.totalProcessingTime, - }, - proposedName, - metadata: { - ...metadata, - fileSize: metadata.size, - dimensions: `${metadata.width}x${metadata.height}`, - format: metadata.format, - }, - processingStats: { - totalTime: Date.now() - startTime, - completedAt: new Date(), - }, - }; - - await this.databaseService.updateImageProcessingResult(imageId, { - status: 'completed', - proposedName, - visionAnalysis: processingResult.visionAnalysis, - metadata: processingResult.metadata, - processingStats: processingResult.processingStats, - }); - - // Step 10: Finalize (100%) - await this.updateProgress(job, { - percentage: 100, - currentImage: originalName, - processedCount: 1, - totalCount: 1, - status: 'completed', - currentStep: 'Processing completed successfully', - }); - - // Notify batch processor if this was the last image - await this.progressTracker.notifyImageCompleted(batchId, imageId); - - const totalProcessingTime = Date.now() - startTime; - this.logger.log(`βœ… Image processing completed: ${imageId} in ${totalProcessingTime}ms`); - - return { - imageId, - success: true, - proposedName, - visionAnalysis: processingResult.visionAnalysis, - metadata: processingResult.metadata, - processingTime: totalProcessingTime, - }; - - } catch (error) { - const processingTime = Date.now() - startTime; - this.logger.error(`❌ Image processing failed: ${imageId} - ${error.message}`, error.stack); - - // Update progress - Failed - await this.updateProgress(job, { - percentage: 0, - currentImage: originalName, - processedCount: 0, - totalCount: 1, - status: 'failed', - error: error.message, - }); - - // Update database with error - await this.databaseService.updateImageStatus(imageId, 'failed', { - error: error.message, - failedAt: new Date(), - processingTime, - }); - - // Notify batch processor of failure - await this.progressTracker.notifyImageFailed(batchId, imageId, error.message); - - throw error; - - } finally { - // Cleanup temporary files - if (tempFilePath) { - await this.fileProcessorService.cleanupTempFile(tempFilePath); - } - if (processedFilePath && processedFilePath !== tempFilePath) { - await this.fileProcessorService.cleanupTempFile(processedFilePath); - } - } - } - - @OnWorkerEvent('completed') - onCompleted(job: Job) { - this.logger.log(`βœ… Image processing job completed: ${job.id}`); - } - - @OnWorkerEvent('failed') - onFailed(job: Job, err: Error) { - this.logger.error(`❌ Image processing job failed: ${job.id}`, err.stack); - } - - @OnWorkerEvent('progress') - onProgress(job: Job, progress: JobProgress) { - this.logger.debug(`πŸ“Š Image processing progress: ${job.id} - ${progress.percentage}% (${progress.currentStep})`); - } - - @OnWorkerEvent('stalled') - onStalled(jobId: string) { - this.logger.warn(`⚠️ Image processing job stalled: ${jobId}`); - } - - /** - * Update job progress and broadcast via WebSocket - */ - private async updateProgress(job: Job, progress: JobProgress): Promise { - try { - await job.updateProgress(progress); - - // Broadcast progress to WebSocket clients - await this.progressTracker.broadcastProgress(job.data.batchId, { - jobId: job.id as string, - imageId: job.data.imageId, - progress, - }); - - } catch (error) { - this.logger.warn(`Failed to update progress for job ${job.id}:`, error.message); - } - } - - /** - * Validate processing options - */ - private validateProcessingOptions(options?: ImageProcessingJobData['processingOptions']): void { - if (!options) return; - - if (options.maxRetries && (options.maxRetries < 0 || options.maxRetries > 10)) { - throw new Error('maxRetries must be between 0 and 10'); - } - - if (options.preferredVisionProvider && - !['openai', 'google'].includes(options.preferredVisionProvider)) { - throw new Error('preferredVisionProvider must be either "openai" or "google"'); - } - } -} \ No newline at end of file diff --git a/packages/worker/src/processors/processors.module.ts b/packages/worker/src/processors/processors.module.ts deleted file mode 100644 index 662f1f5..0000000 --- a/packages/worker/src/processors/processors.module.ts +++ /dev/null @@ -1,46 +0,0 @@ -import { Module } from '@nestjs/common'; -import { BullModule } from '@nestjs/bullmq'; -import { ConfigModule } from '@nestjs/config'; - -// Import processors -import { ImageProcessor } from './image.processor'; -import { BatchProcessor } from './batch.processor'; -import { VirusScanProcessor } from './virus-scan.processor'; -import { FilenameGeneratorProcessor } from './filename-generator.processor'; - -// Import required services -import { VisionModule } from '../vision/vision.module'; -import { StorageModule } from '../storage/storage.module'; -import { QueueModule } from '../queue/queue.module'; -import { SecurityModule } from '../security/security.module'; -import { DatabaseModule } from '../database/database.module'; - -@Module({ - imports: [ - ConfigModule, - BullModule.registerQueue( - { name: 'image-processing' }, - { name: 'batch-processing' }, - { name: 'virus-scan' }, - { name: 'filename-generation' }, - ), - VisionModule, - StorageModule, - QueueModule, - SecurityModule, - DatabaseModule, - ], - providers: [ - ImageProcessor, - BatchProcessor, - VirusScanProcessor, - FilenameGeneratorProcessor, - ], - exports: [ - ImageProcessor, - BatchProcessor, - VirusScanProcessor, - FilenameGeneratorProcessor, - ], -}) -export class ProcessorsModule {} \ No newline at end of file diff --git a/packages/worker/src/processors/virus-scan.processor.ts b/packages/worker/src/processors/virus-scan.processor.ts deleted file mode 100644 index 1bbd73a..0000000 --- a/packages/worker/src/processors/virus-scan.processor.ts +++ /dev/null @@ -1,360 +0,0 @@ -import { Processor, WorkerHost, OnWorkerEvent } from '@nestjs/bullmq'; -import { Logger } from '@nestjs/common'; -import { ConfigService } from '@nestjs/config'; -import { Job } from 'bullmq'; -import { VirusScanService } from '../security/virus-scan.service'; -import { StorageService } from '../storage/storage.service'; -import { DatabaseService } from '../database/database.service'; - -export interface VirusScanJobData { - fileId: string; - filePath: string; - s3Key?: string; - userId: string; - scanType: 'upload' | 'periodic' | 'suspicious'; - metadata?: { - originalName: string; - fileSize: number; - mimeType: string; - }; -} - -export interface ScanProgress { - percentage: number; - status: string; - currentStep?: string; - scanResult?: { - clean: boolean; - threat?: string; - scanTime: number; - }; -} - -@Processor('virus-scan') -export class VirusScanProcessor extends WorkerHost { - private readonly logger = new Logger(VirusScanProcessor.name); - - constructor( - private configService: ConfigService, - private virusScanService: VirusScanService, - private storageService: StorageService, - private databaseService: DatabaseService, - ) { - super(); - } - - async process(job: Job): Promise { - const startTime = Date.now(); - const { fileId, filePath, s3Key, userId, scanType, metadata } = job.data; - - this.logger.log(`πŸ” Starting virus scan: ${fileId} (${scanType})`); - - let tempFilePath: string | null = null; - - try { - // Step 1: Initialize scan (10%) - await this.updateScanProgress(job, { - percentage: 10, - status: 'initializing', - currentStep: 'Preparing file for virus scan', - }); - - // Update database with scan status - await this.databaseService.updateFileScanStatus(fileId, 'scanning', { - startedAt: new Date(), - scanType, - scanJobId: job.id, - }); - - // Step 2: Download file if needed (20%) - let scanFilePath = filePath; - if (s3Key && !filePath) { - await this.updateScanProgress(job, { - percentage: 20, - status: 'downloading', - currentStep: 'Downloading file from storage', - }); - - tempFilePath = await this.storageService.downloadToTemp(s3Key); - scanFilePath = tempFilePath; - } - - // Step 3: Validate file exists and is readable (30%) - await this.updateScanProgress(job, { - percentage: 30, - status: 'validating', - currentStep: 'Validating file accessibility', - }); - - const fileExists = await this.virusScanService.validateFile(scanFilePath); - if (!fileExists) { - throw new Error(`File not accessible: ${scanFilePath}`); - } - - // Step 4: Perform virus scan (80%) - await this.updateScanProgress(job, { - percentage: 40, - status: 'scanning', - currentStep: 'Performing virus scan with ClamAV', - }); - - const scanResult = await this.virusScanService.scanFile(scanFilePath); - - this.logger.log(`Scan result for ${fileId}: ${scanResult.clean ? 'Clean' : `Threat: ${scanResult.threat}`}`); - - // Step 5: Process scan results (90%) - await this.updateScanProgress(job, { - percentage: 90, - status: 'processing-results', - currentStep: 'Processing scan results', - scanResult, - }); - - // Handle scan results - if (!scanResult.clean) { - await this.handleThreatDetected(fileId, s3Key, scanResult, userId, metadata); - } else { - await this.handleCleanFile(fileId, scanResult); - } - - // Step 6: Complete (100%) - await this.updateScanProgress(job, { - percentage: 100, - status: scanResult.clean ? 'clean' : 'threat-detected', - currentStep: 'Virus scan completed', - scanResult, - }); - - const totalScanTime = Date.now() - startTime; - this.logger.log(`βœ… Virus scan completed: ${fileId} in ${totalScanTime}ms - ${scanResult.clean ? 'Clean' : 'Threat detected'}`); - - return { - fileId, - success: true, - scanResult: { - ...scanResult, - scanTime: totalScanTime, - scanType, - }, - }; - - } catch (error) { - const scanTime = Date.now() - startTime; - this.logger.error(`❌ Virus scan failed: ${fileId} - ${error.message}`, error.stack); - - // Update database with error - await this.databaseService.updateFileScanStatus(fileId, 'failed', { - error: error.message, - failedAt: new Date(), - scanTime, - }); - - // Update progress - Failed - await this.updateScanProgress(job, { - percentage: 0, - status: 'failed', - currentStep: `Scan failed: ${error.message}`, - }); - - throw error; - - } finally { - // Cleanup temporary file - if (tempFilePath) { - try { - await this.storageService.deleteTempFile(tempFilePath); - } catch (cleanupError) { - this.logger.warn(`Failed to cleanup temp file ${tempFilePath}:`, cleanupError.message); - } - } - } - } - - /** - * Handle threat detected scenario - */ - private async handleThreatDetected( - fileId: string, - s3Key: string | undefined, - scanResult: any, - userId: string, - metadata?: any - ): Promise { - this.logger.warn(`🚨 THREAT DETECTED in file ${fileId}: ${scanResult.threat}`); - - try { - // 1. Update database with threat information - await this.databaseService.updateFileScanStatus(fileId, 'threat-detected', { - threat: scanResult.threat, - threatDetails: scanResult.details, - detectedAt: new Date(), - quarantined: true, - }); - - // 2. Quarantine file if in storage - if (s3Key) { - await this.quarantineFile(s3Key, fileId, scanResult.threat); - } - - // 3. Log security incident - await this.logSecurityIncident({ - fileId, - userId, - threat: scanResult.threat, - s3Key, - metadata, - timestamp: new Date(), - }); - - // 4. Notify security team if configured - await this.notifySecurityTeam({ - fileId, - userId, - threat: scanResult.threat, - metadata, - }); - - // 5. Block user if multiple threats detected - await this.checkUserThreatHistory(userId); - - } catch (error) { - this.logger.error(`Failed to handle threat for file ${fileId}:`, error.message); - throw error; - } - } - - /** - * Handle clean file scenario - */ - private async handleCleanFile(fileId: string, scanResult: any): Promise { - // Update database with clean status - await this.databaseService.updateFileScanStatus(fileId, 'clean', { - scannedAt: new Date(), - scanEngine: scanResult.engine || 'ClamAV', - scanVersion: scanResult.version, - }); - - this.logger.debug(`βœ… File ${fileId} is clean`); - } - - /** - * Quarantine infected file - */ - private async quarantineFile(s3Key: string, fileId: string, threat: string): Promise { - try { - const quarantineKey = `quarantine/${fileId}_${Date.now()}`; - - // Move file to quarantine bucket/folder - await this.storageService.moveFile(s3Key, quarantineKey); - - this.logger.warn(`πŸ”’ File quarantined: ${s3Key} -> ${quarantineKey} (Threat: ${threat})`); - - } catch (error) { - this.logger.error(`Failed to quarantine file ${s3Key}:`, error.message); - - // If quarantine fails, delete the file as a safety measure - try { - await this.storageService.deleteFile(s3Key); - this.logger.warn(`πŸ—‘οΈ Infected file deleted as quarantine failed: ${s3Key}`); - } catch (deleteError) { - this.logger.error(`CRITICAL: Failed to delete infected file ${s3Key}:`, deleteError.message); - } - } - } - - /** - * Log security incident - */ - private async logSecurityIncident(incident: any): Promise { - try { - await this.databaseService.createSecurityIncident({ - type: 'virus-detected', - severity: 'high', - details: incident, - status: 'active', - createdAt: new Date(), - }); - - this.logger.warn(`🚨 Security incident logged: ${incident.fileId}`); - - } catch (error) { - this.logger.error(`Failed to log security incident:`, error.message); - } - } - - /** - * Notify security team - */ - private async notifySecurityTeam(threat: any): Promise { - try { - // TODO: Implement actual notification system (email, Slack, etc.) - this.logger.warn(`🚨 SECURITY ALERT: Virus detected in file ${threat.fileId} - ${threat.threat}`); - - // For now, just log the alert. In production, this would: - // - Send email to security team - // - Post to Slack security channel - // - Create ticket in security system - // - Trigger incident response workflow - - } catch (error) { - this.logger.error(`Failed to notify security team:`, error.message); - } - } - - /** - * Check user threat history and take action if needed - */ - private async checkUserThreatHistory(userId: string): Promise { - try { - const recentThreats = await this.databaseService.getUserRecentThreats(userId, 7); // Last 7 days - - if (recentThreats.length >= 3) { - this.logger.warn(`🚨 User ${userId} has ${recentThreats.length} recent threats - considering account restriction`); - - // TODO: Implement user restriction logic - // - Temporarily suspend account - // - Require manual review - // - Notify administrators - - await this.databaseService.flagUserForReview(userId, { - reason: 'multiple-virus-detections', - threatCount: recentThreats.length, - flaggedAt: new Date(), - }); - } - - } catch (error) { - this.logger.error(`Failed to check user threat history for ${userId}:`, error.message); - } - } - - private async updateScanProgress(job: Job, progress: ScanProgress): Promise { - try { - await job.updateProgress(progress); - } catch (error) { - this.logger.warn(`Failed to update scan progress for job ${job.id}:`, error.message); - } - } - - @OnWorkerEvent('completed') - onCompleted(job: Job) { - const result = job.returnvalue; - const status = result?.scanResult?.clean ? 'βœ… Clean' : '🚨 Threat detected'; - this.logger.log(`Virus scan completed: ${job.id} - ${status}`); - } - - @OnWorkerEvent('failed') - onFailed(job: Job, err: Error) { - this.logger.error(`❌ Virus scan job failed: ${job.id}`, err.stack); - } - - @OnWorkerEvent('progress') - onProgress(job: Job, progress: ScanProgress) { - this.logger.debug(`πŸ” Virus scan progress: ${job.id} - ${progress.percentage}% (${progress.currentStep})`); - } - - @OnWorkerEvent('stalled') - onStalled(jobId: string) { - this.logger.warn(`⚠️ Virus scan job stalled: ${jobId}`); - } -} \ No newline at end of file diff --git a/packages/worker/src/queue/cleanup.service.ts b/packages/worker/src/queue/cleanup.service.ts deleted file mode 100644 index 8ca7abf..0000000 --- a/packages/worker/src/queue/cleanup.service.ts +++ /dev/null @@ -1,487 +0,0 @@ -import { Injectable, Logger } from '@nestjs/common'; -import { ConfigService } from '@nestjs/config'; -import { InjectQueue } from '@nestjs/bullmq'; -import { Queue } from 'bullmq'; -import { Cron, CronExpression } from '@nestjs/schedule'; -import { InjectRedis } from '@nestjs-modules/ioredis'; -import { Redis } from 'ioredis'; -import { StorageService } from '../storage/storage.service'; -import { FileProcessorService } from '../storage/file-processor.service'; - -@Injectable() -export class CleanupService { - private readonly logger = new Logger(CleanupService.name); - private readonly cleanupInterval: number; - private readonly maxJobAge: number; - private readonly maxTempFileAge: number; - - constructor( - private configService: ConfigService, - @InjectQueue('image-processing') private imageQueue: Queue, - @InjectQueue('batch-processing') private batchQueue: Queue, - @InjectQueue('virus-scan') private virusScanQueue: Queue, - @InjectQueue('filename-generation') private filenameQueue: Queue, - @InjectRedis() private redis: Redis, - private storageService: StorageService, - private fileProcessorService: FileProcessorService, - ) { - this.cleanupInterval = this.configService.get('TEMP_FILE_CLEANUP_INTERVAL', 3600000); // 1 hour - this.maxJobAge = this.configService.get('MAX_JOB_AGE', 24 * 60 * 60 * 1000); // 24 hours - this.maxTempFileAge = this.configService.get('MAX_TEMP_FILE_AGE', 2 * 60 * 60 * 1000); // 2 hours - - this.logger.log(`Cleanup service initialized with interval: ${this.cleanupInterval}ms`); - } - - /** - * Main cleanup routine - runs every hour - */ - @Cron(CronExpression.EVERY_HOUR) - async performScheduledCleanup(): Promise { - const startTime = Date.now(); - this.logger.log('🧹 Starting scheduled cleanup routine'); - - try { - const results = await Promise.allSettled([ - this.cleanupCompletedJobs(), - this.cleanupFailedJobs(), - this.cleanupTempFiles(), - this.cleanupRedisData(), - this.cleanupStorageTemp(), - ]); - - // Log results - const cleanupStats = this.processCleanupResults(results); - const duration = Date.now() - startTime; - - this.logger.log( - `βœ… Cleanup completed in ${duration}ms: ${JSON.stringify(cleanupStats)}` - ); - - } catch (error) { - this.logger.error('❌ Cleanup routine failed:', error.message); - } - } - - /** - * Clean up completed jobs from all queues - */ - async cleanupCompletedJobs(): Promise<{ - imageProcessing: number; - batchProcessing: number; - virusScan: number; - filenameGeneration: number; - }> { - const results = { - imageProcessing: 0, - batchProcessing: 0, - virusScan: 0, - filenameGeneration: 0, - }; - - try { - this.logger.debug('Cleaning up completed jobs...'); - - // Clean completed jobs from each queue - const cleanupPromises = [ - this.cleanQueueJobs(this.imageQueue, 'completed').then(count => results.imageProcessing = count), - this.cleanQueueJobs(this.batchQueue, 'completed').then(count => results.batchProcessing = count), - this.cleanQueueJobs(this.virusScanQueue, 'completed').then(count => results.virusScan = count), - this.cleanQueueJobs(this.filenameQueue, 'completed').then(count => results.filenameGeneration = count), - ]; - - await Promise.all(cleanupPromises); - - const totalCleaned = Object.values(results).reduce((sum, count) => sum + count, 0); - this.logger.debug(`Cleaned ${totalCleaned} completed jobs`); - - } catch (error) { - this.logger.error('Failed to cleanup completed jobs:', error.message); - } - - return results; - } - - /** - * Clean up failed jobs from all queues - */ - async cleanupFailedJobs(): Promise<{ - imageProcessing: number; - batchProcessing: number; - virusScan: number; - filenameGeneration: number; - }> { - const results = { - imageProcessing: 0, - batchProcessing: 0, - virusScan: 0, - filenameGeneration: 0, - }; - - try { - this.logger.debug('Cleaning up old failed jobs...'); - - // Clean failed jobs older than maxJobAge - const cleanupPromises = [ - this.cleanQueueJobs(this.imageQueue, 'failed').then(count => results.imageProcessing = count), - this.cleanQueueJobs(this.batchQueue, 'failed').then(count => results.batchProcessing = count), - this.cleanQueueJobs(this.virusScanQueue, 'failed').then(count => results.virusScan = count), - this.cleanQueueJobs(this.filenameQueue, 'failed').then(count => results.filenameGeneration = count), - ]; - - await Promise.all(cleanupPromises); - - const totalCleaned = Object.values(results).reduce((sum, count) => sum + count, 0); - this.logger.debug(`Cleaned ${totalCleaned} failed jobs`); - - } catch (error) { - this.logger.error('Failed to cleanup failed jobs:', error.message); - } - - return results; - } - - /** - * Clean up temporary files - */ - async cleanupTempFiles(): Promise<{ - fileProcessor: number; - storage: number; - }> { - const results = { - fileProcessor: 0, - storage: 0, - }; - - try { - this.logger.debug('Cleaning up temporary files...'); - - // Clean temporary files from file processor - results.fileProcessor = await this.fileProcessorService.cleanupOldTempFiles(this.maxTempFileAge); - - // Clean temporary files from storage service - await this.storageService.cleanupTempFiles(this.maxTempFileAge); - - // Get storage stats for logging - const storageStats = await this.storageService.getStorageStats(); - results.storage = storageStats.tempFilesCount; // Remaining files after cleanup - - this.logger.debug(`Cleaned temporary files: processor=${results.fileProcessor}, storage temp files remaining=${results.storage}`); - - } catch (error) { - this.logger.error('Failed to cleanup temporary files:', error.message); - } - - return results; - } - - /** - * Clean up Redis data - */ - async cleanupRedisData(): Promise<{ - progressData: number; - retryData: number; - sessionData: number; - }> { - const results = { - progressData: 0, - retryData: 0, - sessionData: 0, - }; - - try { - this.logger.debug('Cleaning up Redis data...'); - - // Clean up different types of Redis data - results.progressData = await this.cleanupRedisPattern('job:progress:*', 3600); // 1 hour - results.retryData = await this.cleanupRedisPattern('job:retry:*', 7200); // 2 hours - results.sessionData = await this.cleanupRedisPattern('session:*', 86400); // 24 hours - - // Clean up expired keys - await this.cleanupExpiredKeys(); - - const totalCleaned = Object.values(results).reduce((sum, count) => sum + count, 0); - this.logger.debug(`Cleaned ${totalCleaned} Redis entries`); - - } catch (error) { - this.logger.error('Failed to cleanup Redis data:', error.message); - } - - return results; - } - - /** - * Clean up storage temporary files - */ - async cleanupStorageTemp(): Promise<{ deletedFiles: number }> { - try { - this.logger.debug('Cleaning up storage temporary files...'); - - // This is handled in cleanupTempFiles, but we can add additional logic here - // for cloud storage cleanup if needed - - return { deletedFiles: 0 }; - - } catch (error) { - this.logger.error('Failed to cleanup storage temp files:', error.message); - return { deletedFiles: 0 }; - } - } - - /** - * Clean jobs from a specific queue - */ - private async cleanQueueJobs(queue: Queue, jobType: 'completed' | 'failed'): Promise { - try { - const maxAge = jobType === 'completed' ? this.maxJobAge : this.maxJobAge * 2; // Keep failed jobs longer - const gracePeriod = 5 * 60 * 1000; // 5 minutes grace period - - // Get jobs of specified type - const jobs = jobType === 'completed' - ? await queue.getCompleted() - : await queue.getFailed(); - - let cleanedCount = 0; - const now = Date.now(); - - for (const job of jobs) { - try { - // Calculate job age - const jobAge = now - (job.finishedOn || job.processedOn || job.timestamp || now); - - if (jobAge > maxAge + gracePeriod) { - await job.remove(); - cleanedCount++; - } - } catch (jobError) { - this.logger.warn(`Failed to remove job ${job.id}:`, jobError.message); - } - } - - return cleanedCount; - - } catch (error) { - this.logger.error(`Failed to clean ${jobType} jobs from queue ${queue.name}:`, error.message); - return 0; - } - } - - /** - * Clean up Redis keys matching a pattern - */ - private async cleanupRedisPattern(pattern: string, maxAge: number): Promise { - try { - const keys = await this.redis.keys(pattern); - let cleanedCount = 0; - - for (const key of keys) { - const ttl = await this.redis.ttl(key); - - // If TTL is very low or key has no expiration but is old - if (ttl < 300 || ttl === -1) { - // For keys without TTL, try to determine age from the key content - if (ttl === -1) { - try { - const data = await this.redis.get(key); - if (data) { - const parsed = JSON.parse(data); - if (parsed.timestamp) { - const age = Date.now() - new Date(parsed.timestamp).getTime(); - if (age < maxAge * 1000) { - continue; // Skip if not old enough - } - } - } - } catch (parseError) { - // If we can't parse, assume it's old and delete it - } - } - - await this.redis.del(key); - cleanedCount++; - } - } - - return cleanedCount; - - } catch (error) { - this.logger.error(`Failed to cleanup Redis pattern ${pattern}:`, error.message); - return 0; - } - } - - /** - * Clean up expired keys - */ - private async cleanupExpiredKeys(): Promise { - try { - // Get Redis info to check expired keys - const info = await this.redis.info('keyspace'); - - // Force Redis to clean up expired keys - await this.redis.eval(` - local keys = redis.call('RANDOMKEY') - if keys then - redis.call('TTL', keys) - end - return 'OK' - `, 0); - - } catch (error) { - this.logger.warn('Failed to trigger expired keys cleanup:', error.message); - } - } - - /** - * Process cleanup results and return statistics - */ - private processCleanupResults(results: PromiseSettledResult[]): any { - const stats: any = { - successful: 0, - failed: 0, - details: {}, - }; - - results.forEach((result, index) => { - const taskNames = [ - 'completedJobs', - 'failedJobs', - 'tempFiles', - 'redisData', - 'storageTemp' - ]; - - if (result.status === 'fulfilled') { - stats.successful++; - stats.details[taskNames[index]] = result.value; - } else { - stats.failed++; - stats.details[taskNames[index]] = { error: result.reason?.message || 'Unknown error' }; - } - }); - - return stats; - } - - /** - * Manual cleanup trigger (for testing or emergency cleanup) - */ - async performManualCleanup(options: { - includeJobs?: boolean; - includeTempFiles?: boolean; - includeRedis?: boolean; - force?: boolean; - } = {}): Promise { - const startTime = Date.now(); - this.logger.log('🧹 Starting manual cleanup routine'); - - const { - includeJobs = true, - includeTempFiles = true, - includeRedis = true, - force = false - } = options; - - try { - const tasks: Promise[] = []; - - if (includeJobs) { - tasks.push(this.cleanupCompletedJobs()); - tasks.push(this.cleanupFailedJobs()); - } - - if (includeTempFiles) { - tasks.push(this.cleanupTempFiles()); - tasks.push(this.cleanupStorageTemp()); - } - - if (includeRedis) { - tasks.push(this.cleanupRedisData()); - } - - const results = await Promise.allSettled(tasks); - const cleanupStats = this.processCleanupResults(results); - const duration = Date.now() - startTime; - - this.logger.log( - `βœ… Manual cleanup completed in ${duration}ms: ${JSON.stringify(cleanupStats)}` - ); - - return { - success: true, - duration, - stats: cleanupStats, - }; - - } catch (error) { - this.logger.error('❌ Manual cleanup failed:', error.message); - return { - success: false, - error: error.message, - }; - } - } - - /** - * Get cleanup statistics - */ - async getCleanupStats(): Promise<{ - lastCleanup: Date | null; - tempFilesCount: number; - queueSizes: { [queueName: string]: number }; - redisMemoryUsage: number; - }> { - try { - // Get storage stats - const storageStats = await this.storageService.getStorageStats(); - - // Get queue sizes - const queueSizes = { - 'image-processing': (await this.imageQueue.getWaiting()).length + (await this.imageQueue.getActive()).length, - 'batch-processing': (await this.batchQueue.getWaiting()).length + (await this.batchQueue.getActive()).length, - 'virus-scan': (await this.virusScanQueue.getWaiting()).length + (await this.virusScanQueue.getActive()).length, - 'filename-generation': (await this.filenameQueue.getWaiting()).length + (await this.filenameQueue.getActive()).length, - }; - - // Get Redis memory usage - const redisInfo = await this.redis.info('memory'); - const memoryMatch = redisInfo.match(/used_memory:(\d+)/); - const redisMemoryUsage = memoryMatch ? parseInt(memoryMatch[1]) : 0; - - return { - lastCleanup: null, // Could track this in Redis if needed - tempFilesCount: storageStats.tempFilesCount, - queueSizes, - redisMemoryUsage, - }; - - } catch (error) { - this.logger.error('Failed to get cleanup stats:', error.message); - return { - lastCleanup: null, - tempFilesCount: 0, - queueSizes: {}, - redisMemoryUsage: 0, - }; - } - } - - /** - * Health check for cleanup service - */ - async isHealthy(): Promise { - try { - // Check if we can access all required services - await Promise.all([ - this.redis.ping(), - this.storageService.getStorageStats(), - this.imageQueue.getWaiting(), - ]); - - return true; - - } catch (error) { - this.logger.error('Cleanup service health check failed:', error.message); - return false; - } - } -} \ No newline at end of file diff --git a/packages/worker/src/queue/progress-tracker.service.ts b/packages/worker/src/queue/progress-tracker.service.ts deleted file mode 100644 index c5a3a89..0000000 --- a/packages/worker/src/queue/progress-tracker.service.ts +++ /dev/null @@ -1,436 +0,0 @@ -import { Injectable, Logger } from '@nestjs/common'; -import { ConfigService } from '@nestjs/config'; -import { Server } from 'socket.io'; -import { InjectRedis } from '@nestjs-modules/ioredis'; -import { Redis } from 'ioredis'; - -export interface ProgressUpdate { - jobId: string; - imageId?: string; - batchId?: string; - progress: any; - timestamp?: Date; -} - -export interface BatchProgressUpdate { - batchId: string; - progress: any; - timestamp?: Date; -} - -@Injectable() -export class ProgressTrackerService { - private readonly logger = new Logger(ProgressTrackerService.name); - private webSocketServer: Server | null = null; - private readonly progressCacheTime = 3600; // 1 hour in seconds - - constructor( - private configService: ConfigService, - @InjectRedis() private redis: Redis, - ) {} - - /** - * Set WebSocket server instance for broadcasting - */ - setWebSocketServer(server: Server): void { - this.webSocketServer = server; - this.logger.log('WebSocket server configured for progress broadcasting'); - } - - /** - * Broadcast progress update via WebSocket - */ - async broadcastProgress(batchId: string, update: ProgressUpdate): Promise { - try { - const progressData = { - ...update, - timestamp: new Date(), - }; - - // Store progress in Redis for persistence - await this.storeProgress(batchId, update.jobId, progressData); - - // Broadcast via WebSocket if available - if (this.webSocketServer) { - this.webSocketServer.to(`batch-${batchId}`).emit('imageProgress', progressData); - this.logger.debug(`Progress broadcasted for batch ${batchId}, job ${update.jobId}`); - } - - } catch (error) { - this.logger.error(`Failed to broadcast progress for batch ${batchId}:`, error.message); - } - } - - /** - * Broadcast batch-level progress update - */ - async broadcastBatchProgress(batchId: string, progress: any): Promise { - try { - const progressData = { - batchId, - progress, - timestamp: new Date(), - }; - - // Store batch progress in Redis - await this.storeBatchProgress(batchId, progressData); - - // Broadcast via WebSocket if available - if (this.webSocketServer) { - this.webSocketServer.to(`batch-${batchId}`).emit('batchProgress', progressData); - this.logger.debug(`Batch progress broadcasted for batch ${batchId}`); - } - - } catch (error) { - this.logger.error(`Failed to broadcast batch progress for batch ${batchId}:`, error.message); - } - } - - /** - * Broadcast batch completion - */ - async broadcastBatchComplete(batchId: string, completionData: any): Promise { - try { - const completeData = { - batchId, - ...completionData, - timestamp: new Date(), - }; - - // Store completion data - await this.redis.setex( - `batch:complete:${batchId}`, - this.progressCacheTime, - JSON.stringify(completeData) - ); - - // Broadcast via WebSocket if available - if (this.webSocketServer) { - this.webSocketServer.to(`batch-${batchId}`).emit('batchComplete', completeData); - this.logger.log(`Batch completion broadcasted for batch ${batchId}`); - } - - } catch (error) { - this.logger.error(`Failed to broadcast batch completion for batch ${batchId}:`, error.message); - } - } - - /** - * Notify when an image processing is completed - */ - async notifyImageCompleted(batchId: string, imageId: string): Promise { - try { - const key = `batch:images:${batchId}`; - - // Add completed image to set - await this.redis.sadd(`${key}:completed`, imageId); - - // Get progress statistics - const stats = await this.getBatchImageStats(batchId); - - // Check if batch is complete - if (stats.completed >= stats.total && stats.total > 0) { - await this.broadcastBatchComplete(batchId, { - message: 'All images processed successfully', - stats, - }); - } - - } catch (error) { - this.logger.error(`Failed to notify image completion for batch ${batchId}:`, error.message); - } - } - - /** - * Notify when an image processing fails - */ - async notifyImageFailed(batchId: string, imageId: string, error: string): Promise { - try { - const key = `batch:images:${batchId}`; - - // Add failed image to set with error info - await this.redis.sadd(`${key}:failed`, imageId); - await this.redis.hset(`${key}:errors`, imageId, error); - - // Get progress statistics - const stats = await this.getBatchImageStats(batchId); - - // Broadcast failure update - if (this.webSocketServer) { - this.webSocketServer.to(`batch-${batchId}`).emit('imageFailed', { - batchId, - imageId, - error, - stats, - timestamp: new Date(), - }); - } - - } catch (error) { - this.logger.error(`Failed to notify image failure for batch ${batchId}:`, error.message); - } - } - - /** - * Get batch image processing statistics - */ - async getBatchImageStats(batchId: string): Promise<{ - total: number; - completed: number; - failed: number; - pending: number; - }> { - try { - const key = `batch:images:${batchId}`; - - const [totalImages, completedImages, failedImages] = await Promise.all([ - this.redis.scard(`${key}:total`), - this.redis.scard(`${key}:completed`), - this.redis.scard(`${key}:failed`), - ]); - - return { - total: totalImages, - completed: completedImages, - failed: failedImages, - pending: totalImages - completedImages - failedImages, - }; - - } catch (error) { - this.logger.error(`Failed to get batch stats for ${batchId}:`, error.message); - return { total: 0, completed: 0, failed: 0, pending: 0 }; - } - } - - /** - * Initialize batch tracking - */ - async initializeBatchTracking(batchId: string, imageIds: string[]): Promise { - try { - const key = `batch:images:${batchId}`; - - // Store total images in the batch - if (imageIds.length > 0) { - await this.redis.sadd(`${key}:total`, ...imageIds); - } - - // Initialize empty completed and failed sets - await this.redis.del(`${key}:completed`, `${key}:failed`, `${key}:errors`); - - // Set expiration for cleanup - await this.redis.expire(`${key}:total`, this.progressCacheTime); - await this.redis.expire(`${key}:completed`, this.progressCacheTime); - await this.redis.expire(`${key}:failed`, this.progressCacheTime); - await this.redis.expire(`${key}:errors`, this.progressCacheTime); - - this.logger.debug(`Batch tracking initialized for ${batchId} with ${imageIds.length} images`); - - } catch (error) { - this.logger.error(`Failed to initialize batch tracking for ${batchId}:`, error.message); - } - } - - /** - * Get current progress for a batch - */ - async getBatchProgress(batchId: string): Promise { - try { - const progressKey = `batch:progress:${batchId}`; - const progressData = await this.redis.get(progressKey); - - if (progressData) { - return JSON.parse(progressData); - } - - // If no stored progress, calculate from image stats - const stats = await this.getBatchImageStats(batchId); - return { - batchId, - progress: { - percentage: stats.total > 0 ? Math.round((stats.completed / stats.total) * 100) : 0, - completedImages: stats.completed, - totalImages: stats.total, - failedImages: stats.failed, - status: stats.completed === stats.total ? 'completed' : 'processing', - }, - timestamp: new Date(), - }; - - } catch (error) { - this.logger.error(`Failed to get batch progress for ${batchId}:`, error.message); - return null; - } - } - - /** - * Get current progress for a specific job - */ - async getJobProgress(batchId: string, jobId: string): Promise { - try { - const progressKey = `job:progress:${batchId}:${jobId}`; - const progressData = await this.redis.get(progressKey); - - return progressData ? JSON.parse(progressData) : null; - - } catch (error) { - this.logger.error(`Failed to get job progress for ${jobId}:`, error.message); - return null; - } - } - - /** - * Store progress data in Redis - */ - private async storeProgress(batchId: string, jobId: string, progressData: any): Promise { - try { - const progressKey = `job:progress:${batchId}:${jobId}`; - await this.redis.setex(progressKey, this.progressCacheTime, JSON.stringify(progressData)); - - } catch (error) { - this.logger.warn(`Failed to store progress for job ${jobId}:`, error.message); - } - } - - /** - * Store batch progress data in Redis - */ - private async storeBatchProgress(batchId: string, progressData: any): Promise { - try { - const progressKey = `batch:progress:${batchId}`; - await this.redis.setex(progressKey, this.progressCacheTime, JSON.stringify(progressData)); - - } catch (error) { - this.logger.warn(`Failed to store batch progress for ${batchId}:`, error.message); - } - } - - /** - * Clean up old progress data - */ - async cleanupOldProgress(maxAge: number = 86400): Promise { - try { - const pattern = 'job:progress:*'; - const keys = await this.redis.keys(pattern); - let cleanedCount = 0; - - for (const key of keys) { - const ttl = await this.redis.ttl(key); - - // If TTL is very low or expired, delete immediately - if (ttl < 300) { // Less than 5 minutes - await this.redis.del(key); - cleanedCount++; - } - } - - // Also clean batch progress - const batchPattern = 'batch:progress:*'; - const batchKeys = await this.redis.keys(batchPattern); - - for (const key of batchKeys) { - const ttl = await this.redis.ttl(key); - - if (ttl < 300) { - await this.redis.del(key); - cleanedCount++; - } - } - - if (cleanedCount > 0) { - this.logger.log(`Cleaned up ${cleanedCount} old progress entries`); - } - - return cleanedCount; - - } catch (error) { - this.logger.error('Failed to cleanup old progress data:', error.message); - return 0; - } - } - - /** - * Get all active batches - */ - async getActiveBatches(): Promise { - try { - const pattern = 'batch:progress:*'; - const keys = await this.redis.keys(pattern); - - return keys.map(key => key.replace('batch:progress:', '')); - - } catch (error) { - this.logger.error('Failed to get active batches:', error.message); - return []; - } - } - - /** - * Subscribe to Redis pub/sub for distributed progress updates - */ - async subscribeToProgressUpdates(): Promise { - try { - const subscriber = this.redis.duplicate(); - - await subscriber.subscribe('progress:updates'); - - subscriber.on('message', async (channel, message) => { - try { - if (channel === 'progress:updates') { - const update = JSON.parse(message); - - // Re-broadcast via WebSocket - if (this.webSocketServer && update.batchId) { - this.webSocketServer.to(`batch-${update.batchId}`).emit('progressUpdate', update); - } - } - } catch (error) { - this.logger.warn('Failed to process progress update message:', error.message); - } - }); - - this.logger.log('Subscribed to progress updates channel'); - - } catch (error) { - this.logger.error('Failed to subscribe to progress updates:', error.message); - } - } - - /** - * Publish progress update to Redis pub/sub - */ - async publishProgressUpdate(update: any): Promise { - try { - await this.redis.publish('progress:updates', JSON.stringify(update)); - } catch (error) { - this.logger.warn('Failed to publish progress update:', error.message); - } - } - - /** - * Get service statistics - */ - async getProgressStats(): Promise<{ - activeBatches: number; - totalProgressEntries: number; - webSocketConnected: boolean; - }> { - try { - const activeBatches = await this.getActiveBatches(); - const progressKeys = await this.redis.keys('job:progress:*'); - - return { - activeBatches: activeBatches.length, - totalProgressEntries: progressKeys.length, - webSocketConnected: !!this.webSocketServer, - }; - - } catch (error) { - this.logger.error('Failed to get progress stats:', error.message); - return { - activeBatches: 0, - totalProgressEntries: 0, - webSocketConnected: !!this.webSocketServer, - }; - } - } -} \ No newline at end of file diff --git a/packages/worker/src/queue/queue.module.ts b/packages/worker/src/queue/queue.module.ts deleted file mode 100644 index e45ecfd..0000000 --- a/packages/worker/src/queue/queue.module.ts +++ /dev/null @@ -1,33 +0,0 @@ -import { Module } from '@nestjs/common'; -import { BullModule } from '@nestjs/bullmq'; -import { ConfigModule } from '@nestjs/config'; -import { ScheduleModule } from '@nestjs/schedule'; -import { ProgressTrackerService } from './progress-tracker.service'; -import { RetryHandlerService } from './retry-handler.service'; -import { CleanupService } from './cleanup.service'; -import { StorageModule } from '../storage/storage.module'; - -@Module({ - imports: [ - ConfigModule, - ScheduleModule.forRoot(), - BullModule.registerQueue( - { name: 'image-processing' }, - { name: 'batch-processing' }, - { name: 'virus-scan' }, - { name: 'filename-generation' }, - ), - StorageModule, - ], - providers: [ - ProgressTrackerService, - RetryHandlerService, - CleanupService, - ], - exports: [ - ProgressTrackerService, - RetryHandlerService, - CleanupService, - ], -}) -export class QueueModule {} \ No newline at end of file diff --git a/packages/worker/src/queue/retry-handler.service.ts b/packages/worker/src/queue/retry-handler.service.ts deleted file mode 100644 index 942e5bd..0000000 --- a/packages/worker/src/queue/retry-handler.service.ts +++ /dev/null @@ -1,496 +0,0 @@ -import { Injectable, Logger } from '@nestjs/common'; -import { ConfigService } from '@nestjs/config'; -import { InjectQueue } from '@nestjs/bullmq'; -import { Queue, Job } from 'bullmq'; -import { InjectRedis } from '@nestjs-modules/ioredis'; -import { Redis } from 'ioredis'; - -export interface RetryPolicy { - maxAttempts: number; - backoffStrategy: 'exponential' | 'fixed' | 'linear'; - baseDelay: number; - maxDelay: number; - jitter: boolean; -} - -export interface RetryContext { - jobId: string; - attemptNumber: number; - previousError: string; - retryAfter: Date; - retryPolicy: RetryPolicy; -} - -@Injectable() -export class RetryHandlerService { - private readonly logger = new Logger(RetryHandlerService.name); - private readonly defaultRetryPolicy: RetryPolicy; - - // Queue-specific retry policies - private readonly retryPolicies: Map = new Map(); - - constructor( - private configService: ConfigService, - @InjectQueue('image-processing') private imageQueue: Queue, - @InjectQueue('batch-processing') private batchQueue: Queue, - @InjectQueue('virus-scan') private virusScanQueue: Queue, - @InjectQueue('filename-generation') private filenameQueue: Queue, - @InjectRedis() private redis: Redis, - ) { - // Default retry policy - this.defaultRetryPolicy = { - maxAttempts: this.configService.get('RETRY_ATTEMPTS', 3), - backoffStrategy: 'exponential', - baseDelay: this.configService.get('RETRY_DELAY', 2000), - maxDelay: 60000, // 1 minute max delay - jitter: true, - }; - - this.initializeRetryPolicies(); - } - - /** - * Initialize queue-specific retry policies - */ - private initializeRetryPolicies(): void { - // Image processing - critical, more retries - this.retryPolicies.set('image-processing', { - maxAttempts: 5, - backoffStrategy: 'exponential', - baseDelay: 3000, - maxDelay: 120000, // 2 minutes - jitter: true, - }); - - // Batch processing - important, moderate retries - this.retryPolicies.set('batch-processing', { - maxAttempts: 3, - backoffStrategy: 'exponential', - baseDelay: 5000, - maxDelay: 180000, // 3 minutes - jitter: true, - }); - - // Virus scan - security critical, many retries - this.retryPolicies.set('virus-scan', { - maxAttempts: 7, - backoffStrategy: 'exponential', - baseDelay: 1000, - maxDelay: 60000, // 1 minute - jitter: true, - }); - - // Filename generation - less critical, fewer retries - this.retryPolicies.set('filename-generation', { - maxAttempts: 2, - backoffStrategy: 'linear', - baseDelay: 2000, - maxDelay: 30000, // 30 seconds - jitter: false, - }); - - this.logger.log('Retry policies initialized for all queues'); - } - - /** - * Handle job failure and determine retry strategy - */ - async handleJobFailure( - job: Job, - error: Error, - queueName: string - ): Promise<{ - shouldRetry: boolean; - retryDelay?: number; - finalFailure?: boolean; - }> { - try { - const retryPolicy = this.retryPolicies.get(queueName) || this.defaultRetryPolicy; - const attemptsMade = job.attemptsMade || 0; - - this.logger.warn(`Job ${job.id} failed (attempt ${attemptsMade}/${retryPolicy.maxAttempts}): ${error.message}`); - - // Check if we've exceeded max attempts - if (attemptsMade >= retryPolicy.maxAttempts) { - await this.handleFinalFailure(job, error, queueName); - return { shouldRetry: false, finalFailure: true }; - } - - // Determine if error is retryable - const isRetryable = this.isErrorRetryable(error, queueName); - if (!isRetryable) { - await this.handleNonRetryableFailure(job, error, queueName); - return { shouldRetry: false, finalFailure: true }; - } - - // Calculate retry delay - const retryDelay = this.calculateRetryDelay( - attemptsMade + 1, - retryPolicy - ); - - // Log retry context - await this.logRetryAttempt(job, error, attemptsMade + 1, retryDelay, queueName); - - return { - shouldRetry: true, - retryDelay, - }; - - } catch (retryError) { - this.logger.error(`Error in retry handler for job ${job.id}:`, retryError.message); - return { shouldRetry: false, finalFailure: true }; - } - } - - /** - * Calculate retry delay based on policy - */ - private calculateRetryDelay(attemptNumber: number, policy: RetryPolicy): number { - let delay: number; - - switch (policy.backoffStrategy) { - case 'exponential': - delay = policy.baseDelay * Math.pow(2, attemptNumber - 1); - break; - - case 'linear': - delay = policy.baseDelay * attemptNumber; - break; - - case 'fixed': - default: - delay = policy.baseDelay; - break; - } - - // Apply max delay limit - delay = Math.min(delay, policy.maxDelay); - - // Apply jitter to prevent thundering herd - if (policy.jitter) { - const jitterAmount = delay * 0.1; // 10% jitter - delay += (Math.random() - 0.5) * 2 * jitterAmount; - } - - return Math.max(delay, 1000); // Minimum 1 second delay - } - - /** - * Determine if an error is retryable - */ - private isErrorRetryable(error: Error, queueName: string): boolean { - const errorMessage = error.message.toLowerCase(); - - // Non-retryable errors (permanent failures) - const nonRetryableErrors = [ - 'file not found', - 'invalid file format', - 'file too large', - 'virus detected', - 'authentication failed', - 'permission denied', - 'quota exceeded permanently', - 'invalid api key', - 'account suspended', - ]; - - // Check for non-retryable error patterns - for (const nonRetryable of nonRetryableErrors) { - if (errorMessage.includes(nonRetryable)) { - this.logger.warn(`Non-retryable error detected: ${error.message}`); - return false; - } - } - - // Queue-specific retryable checks - switch (queueName) { - case 'virus-scan': - // Virus scan errors are usually retryable unless it's a virus detection - return !errorMessage.includes('threat detected'); - - case 'image-processing': - // Image processing errors are usually retryable unless it's file corruption - return !errorMessage.includes('corrupted') && !errorMessage.includes('invalid image'); - - default: - return true; // Default to retryable - } - } - - /** - * Handle final failure after all retries exhausted - */ - private async handleFinalFailure(job: Job, error: Error, queueName: string): Promise { - try { - this.logger.error(`Job ${job.id} finally failed after all retry attempts: ${error.message}`); - - // Store failure information - const failureData = { - jobId: job.id, - queueName, - finalError: error.message, - totalAttempts: job.attemptsMade || 0, - failedAt: new Date().toISOString(), - jobData: job.data, - }; - - await this.redis.setex( - `job:final-failure:${job.id}`, - 86400, // 24 hours retention - JSON.stringify(failureData) - ); - - // Update failure metrics - await this.updateFailureMetrics(queueName, 'final_failure'); - - // Trigger alerts for critical queues - if (['image-processing', 'virus-scan'].includes(queueName)) { - await this.triggerFailureAlert(job, error, queueName); - } - - } catch (logError) { - this.logger.error(`Failed to log final failure for job ${job.id}:`, logError.message); - } - } - - /** - * Handle non-retryable failure - */ - private async handleNonRetryableFailure(job: Job, error: Error, queueName: string): Promise { - try { - this.logger.error(`Job ${job.id} failed with non-retryable error: ${error.message}`); - - // Store non-retryable failure information - const failureData = { - jobId: job.id, - queueName, - error: error.message, - reason: 'non_retryable', - failedAt: new Date().toISOString(), - jobData: job.data, - }; - - await this.redis.setex( - `job:non-retryable-failure:${job.id}`, - 86400, // 24 hours retention - JSON.stringify(failureData) - ); - - // Update failure metrics - await this.updateFailureMetrics(queueName, 'non_retryable'); - - } catch (logError) { - this.logger.error(`Failed to log non-retryable failure for job ${job.id}:`, logError.message); - } - } - - /** - * Log retry attempt - */ - private async logRetryAttempt( - job: Job, - error: Error, - attemptNumber: number, - retryDelay: number, - queueName: string - ): Promise { - try { - const retryContext: RetryContext = { - jobId: job.id as string, - attemptNumber, - previousError: error.message, - retryAfter: new Date(Date.now() + retryDelay), - retryPolicy: this.retryPolicies.get(queueName) || this.defaultRetryPolicy, - }; - - // Store retry context - await this.redis.setex( - `job:retry:${job.id}`, - 3600, // 1 hour retention - JSON.stringify(retryContext) - ); - - // Update retry metrics - await this.updateRetryMetrics(queueName, attemptNumber); - - this.logger.log( - `Job ${job.id} will retry in ${Math.round(retryDelay / 1000)}s (attempt ${attemptNumber})` - ); - - } catch (logError) { - this.logger.warn(`Failed to log retry attempt for job ${job.id}:`, logError.message); - } - } - - /** - * Update failure metrics - */ - private async updateFailureMetrics(queueName: string, failureType: string): Promise { - try { - const today = new Date().toISOString().split('T')[0]; // YYYY-MM-DD - const key = `metrics:failures:${queueName}:${failureType}:${today}`; - - await this.redis.incr(key); - await this.redis.expire(key, 7 * 24 * 3600); // 7 days retention - - } catch (error) { - this.logger.warn(`Failed to update failure metrics:`, error.message); - } - } - - /** - * Update retry metrics - */ - private async updateRetryMetrics(queueName: string, attemptNumber: number): Promise { - try { - const today = new Date().toISOString().split('T')[0]; - const key = `metrics:retries:${queueName}:${today}`; - - await this.redis.hincrby(key, `attempt_${attemptNumber}`, 1); - await this.redis.expire(key, 7 * 24 * 3600); // 7 days retention - - } catch (error) { - this.logger.warn(`Failed to update retry metrics:`, error.message); - } - } - - /** - * Trigger failure alert for critical jobs - */ - private async triggerFailureAlert(job: Job, error: Error, queueName: string): Promise { - try { - const alertData = { - jobId: job.id, - queueName, - error: error.message, - jobData: job.data, - timestamp: new Date().toISOString(), - severity: 'high', - }; - - // Publish alert to monitoring system - await this.redis.publish('alerts:job-failures', JSON.stringify(alertData)); - - // Log critical failure - this.logger.error(`🚨 CRITICAL FAILURE ALERT: Job ${job.id} in queue ${queueName} - ${error.message}`); - - } catch (alertError) { - this.logger.error(`Failed to trigger failure alert:`, alertError.message); - } - } - - /** - * Get retry statistics for a queue - */ - async getRetryStats(queueName: string, days: number = 7): Promise<{ - totalRetries: number; - finalFailures: number; - nonRetryableFailures: number; - retryDistribution: { [attempt: string]: number }; - }> { - try { - const stats = { - totalRetries: 0, - finalFailures: 0, - nonRetryableFailures: 0, - retryDistribution: {} as { [attempt: string]: number }, - }; - - // Get stats for each day - for (let i = 0; i < days; i++) { - const date = new Date(); - date.setDate(date.getDate() - i); - const dateStr = date.toISOString().split('T')[0]; - - // Get retry distribution - const retryKey = `metrics:retries:${queueName}:${dateStr}`; - const retryData = await this.redis.hgetall(retryKey); - - for (const [attempt, count] of Object.entries(retryData)) { - stats.retryDistribution[attempt] = (stats.retryDistribution[attempt] || 0) + parseInt(count); - stats.totalRetries += parseInt(count); - } - - // Get failure counts - const finalFailureKey = `metrics:failures:${queueName}:final_failure:${dateStr}`; - const nonRetryableKey = `metrics:failures:${queueName}:non_retryable:${dateStr}`; - - const [finalFailures, nonRetryableFailures] = await Promise.all([ - this.redis.get(finalFailureKey).then(val => parseInt(val || '0')), - this.redis.get(nonRetryableKey).then(val => parseInt(val || '0')), - ]); - - stats.finalFailures += finalFailures; - stats.nonRetryableFailures += nonRetryableFailures; - } - - return stats; - - } catch (error) { - this.logger.error(`Failed to get retry stats for ${queueName}:`, error.message); - return { - totalRetries: 0, - finalFailures: 0, - nonRetryableFailures: 0, - retryDistribution: {}, - }; - } - } - - /** - * Get current retry policy for a queue - */ - getRetryPolicy(queueName: string): RetryPolicy { - return this.retryPolicies.get(queueName) || this.defaultRetryPolicy; - } - - /** - * Update retry policy for a queue - */ - updateRetryPolicy(queueName: string, policy: Partial): void { - const currentPolicy = this.getRetryPolicy(queueName); - const newPolicy = { ...currentPolicy, ...policy }; - - this.retryPolicies.set(queueName, newPolicy); - this.logger.log(`Updated retry policy for queue ${queueName}:`, newPolicy); - } - - /** - * Clean up old retry and failure data - */ - async cleanupOldData(maxAge: number = 7 * 24 * 3600): Promise { - try { - const patterns = [ - 'job:retry:*', - 'job:final-failure:*', - 'job:non-retryable-failure:*', - ]; - - let cleanedCount = 0; - - for (const pattern of patterns) { - const keys = await this.redis.keys(pattern); - - for (const key of keys) { - const ttl = await this.redis.ttl(key); - - if (ttl < 300) { // Less than 5 minutes remaining - await this.redis.del(key); - cleanedCount++; - } - } - } - - if (cleanedCount > 0) { - this.logger.log(`Cleaned up ${cleanedCount} old retry/failure records`); - } - - return cleanedCount; - - } catch (error) { - this.logger.error('Failed to cleanup old retry data:', error.message); - return 0; - } - } -} \ No newline at end of file diff --git a/packages/worker/src/security/security.module.ts b/packages/worker/src/security/security.module.ts deleted file mode 100644 index 3561896..0000000 --- a/packages/worker/src/security/security.module.ts +++ /dev/null @@ -1,10 +0,0 @@ -import { Module } from '@nestjs/common'; -import { ConfigModule } from '@nestjs/config'; -import { VirusScanService } from './virus-scan.service'; - -@Module({ - imports: [ConfigModule], - providers: [VirusScanService], - exports: [VirusScanService], -}) -export class SecurityModule {} \ No newline at end of file diff --git a/packages/worker/src/security/virus-scan.service.ts b/packages/worker/src/security/virus-scan.service.ts deleted file mode 100644 index 0d53027..0000000 --- a/packages/worker/src/security/virus-scan.service.ts +++ /dev/null @@ -1,504 +0,0 @@ -import { Injectable, Logger } from '@nestjs/common'; -import { ConfigService } from '@nestjs/config'; -import * as NodeClam from 'node-clamav'; -import * as fs from 'fs/promises'; -import * as net from 'net'; - -export interface ScanResult { - clean: boolean; - threat?: string; - engine?: string; - version?: string; - scanTime: number; - details?: any; -} - -export interface ScanOptions { - timeout?: number; - maxFileSize?: number; - skipArchives?: boolean; - scanMetadata?: boolean; -} - -@Injectable() -export class VirusScanService { - private readonly logger = new Logger(VirusScanService.name); - private readonly enabled: boolean; - private readonly clamavHost: string; - private readonly clamavPort: number; - private readonly timeout: number; - private readonly maxFileSize: number; - private clamAV: any; - - constructor(private configService: ConfigService) { - this.enabled = this.configService.get('VIRUS_SCAN_ENABLED', false); - this.clamavHost = this.configService.get('CLAMAV_HOST', 'localhost'); - this.clamavPort = this.configService.get('CLAMAV_PORT', 3310); - this.timeout = this.configService.get('CLAMAV_TIMEOUT', 30000); - this.maxFileSize = this.configService.get('MAX_FILE_SIZE', 50 * 1024 * 1024); - - if (this.enabled) { - this.initializeClamAV(); - } else { - this.logger.warn('Virus scanning is disabled. Set VIRUS_SCAN_ENABLED=true to enable.'); - } - } - - /** - * Initialize ClamAV connection - */ - private async initializeClamAV(): Promise { - try { - this.clamAV = NodeClam.init({ - remove_infected: false, // Don't auto-remove infected files - quarantine_infected: false, // Don't auto-quarantine - scan_log: null, // Disable file logging - debug_mode: this.configService.get('NODE_ENV') === 'development', - file_list: null, - scan_timeout: this.timeout, - clamdscan: { - host: this.clamavHost, - port: this.clamavPort, - timeout: this.timeout, - local_fallback: false, // Don't fallback to local scanning - }, - }); - - // Test connection - await this.testConnection(); - - this.logger.log(`ClamAV initialized: ${this.clamavHost}:${this.clamavPort}`); - - } catch (error) { - this.logger.error('Failed to initialize ClamAV:', error.message); - throw new Error(`ClamAV initialization failed: ${error.message}`); - } - } - - /** - * Test ClamAV connection and functionality - */ - async testConnection(): Promise { - if (!this.enabled) { - this.logger.warn('Virus scanning is disabled'); - return false; - } - - try { - // Test socket connection - const isConnected = await this.testSocketConnection(); - if (!isConnected) { - throw new Error('Cannot connect to ClamAV daemon'); - } - - // Test with EICAR test file - const testResult = await this.scanEicarTestString(); - if (!testResult) { - throw new Error('EICAR test failed - ClamAV may not be working properly'); - } - - this.logger.log('βœ… ClamAV connection test passed'); - return true; - - } catch (error) { - this.logger.error('❌ ClamAV connection test failed:', error.message); - return false; - } - } - - /** - * Test socket connection to ClamAV daemon - */ - private async testSocketConnection(): Promise { - return new Promise((resolve) => { - const socket = new net.Socket(); - - const cleanup = () => { - socket.removeAllListeners(); - socket.destroy(); - }; - - socket.setTimeout(5000); - - socket.on('connect', () => { - cleanup(); - resolve(true); - }); - - socket.on('error', () => { - cleanup(); - resolve(false); - }); - - socket.on('timeout', () => { - cleanup(); - resolve(false); - }); - - socket.connect(this.clamavPort, this.clamavHost); - }); - } - - /** - * Test ClamAV with EICAR test string - */ - private async scanEicarTestString(): Promise { - try { - // EICAR Anti-Virus Test File - const eicarString = 'X5O!P%@AP[4\\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-FILE!$H+H*'; - - // Create temporary test file - const testFilePath = '/tmp/eicar-test.txt'; - await fs.writeFile(testFilePath, eicarString); - - // Scan the test file - const result = await this.scanFile(testFilePath); - - // Clean up test file - try { - await fs.unlink(testFilePath); - } catch (cleanupError) { - this.logger.warn('Failed to cleanup EICAR test file:', cleanupError.message); - } - - // EICAR should be detected as a threat - return !result.clean && result.threat?.includes('EICAR'); - - } catch (error) { - this.logger.error('EICAR test failed:', error.message); - return false; - } - } - - /** - * Scan a file for viruses - */ - async scanFile(filePath: string, options: ScanOptions = {}): Promise { - const startTime = Date.now(); - - if (!this.enabled) { - return { - clean: true, - engine: 'disabled', - scanTime: Date.now() - startTime, - }; - } - - try { - this.logger.debug(`Scanning file: ${filePath}`); - - // Validate file exists and is readable - const isValid = await this.validateFile(filePath); - if (!isValid) { - throw new Error(`File not accessible: ${filePath}`); - } - - // Check file size - const stats = await fs.stat(filePath); - if (stats.size > (options.maxFileSize || this.maxFileSize)) { - throw new Error(`File too large: ${stats.size} bytes (max: ${this.maxFileSize})`); - } - - // Perform the scan - const scanResult = await this.performScan(filePath, options); - const scanTime = Date.now() - startTime; - - const result: ScanResult = { - clean: scanResult.isInfected === false, - threat: scanResult.viruses && scanResult.viruses.length > 0 ? scanResult.viruses[0] : undefined, - engine: 'ClamAV', - version: scanResult.version, - scanTime, - details: { - file: scanResult.file, - goodFiles: scanResult.goodFiles, - badFiles: scanResult.badFiles, - totalFiles: scanResult.totalFiles, - }, - }; - - if (!result.clean) { - this.logger.warn(`🚨 VIRUS DETECTED in ${filePath}: ${result.threat}`); - } else { - this.logger.debug(`βœ… File clean: ${filePath} (${scanTime}ms)`); - } - - return result; - - } catch (error) { - const scanTime = Date.now() - startTime; - this.logger.error(`Scan failed for ${filePath}:`, error.message); - - throw new Error(`Virus scan failed: ${error.message}`); - } - } - - /** - * Scan buffer/stream content - */ - async scanBuffer(buffer: Buffer, fileName: string = 'buffer'): Promise { - const startTime = Date.now(); - - if (!this.enabled) { - return { - clean: true, - engine: 'disabled', - scanTime: Date.now() - startTime, - }; - } - - try { - this.logger.debug(`Scanning buffer: ${fileName} (${buffer.length} bytes)`); - - // Check buffer size - if (buffer.length > this.maxFileSize) { - throw new Error(`Buffer too large: ${buffer.length} bytes (max: ${this.maxFileSize})`); - } - - // Write buffer to temporary file for scanning - const tempFilePath = `/tmp/scan_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`; - await fs.writeFile(tempFilePath, buffer); - - try { - // Scan the temporary file - const result = await this.scanFile(tempFilePath); - return result; - - } finally { - // Clean up temporary file - try { - await fs.unlink(tempFilePath); - } catch (cleanupError) { - this.logger.warn(`Failed to cleanup temp scan file: ${cleanupError.message}`); - } - } - - } catch (error) { - const scanTime = Date.now() - startTime; - this.logger.error(`Buffer scan failed for ${fileName}:`, error.message); - - throw new Error(`Buffer virus scan failed: ${error.message}`); - } - } - - /** - * Perform the actual ClamAV scan - */ - private async performScan(filePath: string, options: ScanOptions): Promise { - return new Promise((resolve, reject) => { - const scanOptions = { - timeout: options.timeout || this.timeout, - // Additional ClamAV options can be configured here - }; - - this.clamAV.scan_file(filePath, (error: any, object: any, malicious: string) => { - if (error) { - if (error.toString().includes('TIMEOUT')) { - reject(new Error('Scan timeout - file may be too large or ClamAV is overloaded')); - } else { - reject(error); - } - return; - } - - // Parse ClamAV response - const result = { - isInfected: object && object.is_infected, - file: object ? object.file : filePath, - viruses: malicious ? [malicious] : [], - goodFiles: object ? object.good_files : 1, - badFiles: object ? object.bad_files : 0, - totalFiles: 1, - version: object ? object.version : 'unknown', - }; - - resolve(result); - }); - }); - } - - /** - * Validate file exists and is readable - */ - async validateFile(filePath: string): Promise { - try { - const stats = await fs.stat(filePath); - return stats.isFile(); - } catch (error) { - return false; - } - } - - /** - * Get ClamAV version information - */ - async getVersion(): Promise { - if (!this.enabled) { - return 'disabled'; - } - - try { - return new Promise((resolve, reject) => { - this.clamAV.get_version((error: any, version: string) => { - if (error) { - reject(error); - } else { - resolve(version); - } - }); - }); - - } catch (error) { - this.logger.error('Failed to get ClamAV version:', error.message); - return 'unknown'; - } - } - - /** - * Update virus definitions - */ - async updateDefinitions(): Promise { - if (!this.enabled) { - this.logger.warn('Cannot update definitions - virus scanning is disabled'); - return false; - } - - try { - this.logger.log('Updating ClamAV virus definitions...'); - - // Note: This requires freshclam to be properly configured - // In production, definitions should be updated via freshclam daemon - - return new Promise((resolve) => { - this.clamAV.update_db((error: any) => { - if (error) { - this.logger.error('Failed to update virus definitions:', error.message); - resolve(false); - } else { - this.logger.log('βœ… Virus definitions updated successfully'); - resolve(true); - } - }); - }); - - } catch (error) { - this.logger.error('Failed to update virus definitions:', error.message); - return false; - } - } - - /** - * Get scan statistics - */ - async getScanStats(): Promise<{ - enabled: boolean; - version: string; - lastUpdate?: Date; - totalScans: number; - threatsDetected: number; - }> { - try { - const version = await this.getVersion(); - - // In a production system, you'd track these stats in Redis or database - return { - enabled: this.enabled, - version, - totalScans: 0, // Would be tracked - threatsDetected: 0, // Would be tracked - }; - - } catch (error) { - return { - enabled: this.enabled, - version: 'error', - totalScans: 0, - threatsDetected: 0, - }; - } - } - - /** - * Health check for virus scanning service - */ - async isHealthy(): Promise { - if (!this.enabled) { - return true; // If disabled, consider it "healthy" - } - - try { - return await this.testConnection(); - } catch (error) { - return false; - } - } - - /** - * Check if virus scanning is enabled - */ - isEnabled(): boolean { - return this.enabled; - } - - /** - * Get service configuration - */ - getConfiguration(): { - enabled: boolean; - host: string; - port: number; - timeout: number; - maxFileSize: number; - } { - return { - enabled: this.enabled, - host: this.clamavHost, - port: this.clamavPort, - timeout: this.timeout, - maxFileSize: this.maxFileSize, - }; - } - - /** - * Scan multiple files in batch - */ - async scanFiles(filePaths: string[], options: ScanOptions = {}): Promise { - const results: ScanResult[] = []; - - for (const filePath of filePaths) { - try { - const result = await this.scanFile(filePath, options); - results.push(result); - } catch (error) { - results.push({ - clean: false, - threat: `Scan error: ${error.message}`, - engine: 'ClamAV', - scanTime: 0, - }); - } - } - - return results; - } - - /** - * Check if a specific threat signature exists - */ - async checkThreatSignature(signature: string): Promise { - if (!this.enabled) { - return false; - } - - try { - // This would typically query ClamAV database for specific signatures - // Implementation depends on ClamAV setup and requirements - this.logger.debug(`Checking for threat signature: ${signature}`); - return false; // Placeholder implementation - - } catch (error) { - this.logger.error(`Failed to check threat signature ${signature}:`, error.message); - return false; - } - } -} \ No newline at end of file diff --git a/packages/worker/src/storage/exif-preserver.service.ts b/packages/worker/src/storage/exif-preserver.service.ts deleted file mode 100644 index 473a9cf..0000000 --- a/packages/worker/src/storage/exif-preserver.service.ts +++ /dev/null @@ -1,455 +0,0 @@ -import { Injectable, Logger } from '@nestjs/common'; -import { ConfigService } from '@nestjs/config'; -import * as exifr from 'exifr'; -import * as piexif from 'piexifjs'; -import * as fs from 'fs/promises'; - -export interface ExifData { - exif?: any; - iptc?: any; - xmp?: any; - icc?: any; - tiff?: any; - gps?: any; -} - -export interface GpsCoordinates { - latitude: number; - longitude: number; - altitude?: number; -} - -@Injectable() -export class ExifPreserverService { - private readonly logger = new Logger(ExifPreserverService.name); - - constructor(private configService: ConfigService) { - this.logger.log('EXIF Preserver Service initialized'); - } - - /** - * Extract all EXIF data from image file - */ - async extractExif(filePath: string): Promise { - try { - this.logger.debug(`Extracting EXIF data from: ${filePath}`); - - // Use exifr to extract comprehensive metadata - const exifData = await exifr.parse(filePath, { - exif: true, - iptc: true, - xmp: true, - icc: true, - tiff: true, - gps: true, - sanitize: false, // Keep all data - reviveValues: true, - translateKeys: false, - translateValues: false, - mergeOutput: false, - }); - - if (!exifData) { - this.logger.debug(`No EXIF data found in: ${filePath}`); - return {}; - } - - // Separate different metadata types - const result: ExifData = { - exif: exifData.exif || exifData.EXIF, - iptc: exifData.iptc || exifData.IPTC, - xmp: exifData.xmp || exifData.XMP, - icc: exifData.icc || exifData.ICC, - tiff: exifData.tiff || exifData.TIFF, - gps: exifData.gps || exifData.GPS, - }; - - // Log extracted data summary - const hasExif = !!result.exif; - const hasGps = !!result.gps && (result.gps.latitude || result.gps.GPSLatitude); - const hasIptc = !!result.iptc; - const hasXmp = !!result.xmp; - - this.logger.debug(`EXIF extraction summary: EXIF=${hasExif}, GPS=${hasGps}, IPTC=${hasIptc}, XMP=${hasXmp}`); - - return result; - - } catch (error) { - this.logger.warn(`Failed to extract EXIF data from ${filePath}:`, error.message); - return {}; - } - } - - /** - * Preserve EXIF data by writing it to processed image - */ - async preserveExif(filePath: string, exifData: ExifData): Promise { - try { - if (!exifData || Object.keys(exifData).length === 0) { - this.logger.debug(`No EXIF data to preserve for: ${filePath}`); - return; - } - - this.logger.debug(`Preserving EXIF data for: ${filePath}`); - - // Read the processed image file - const imageBuffer = await fs.readFile(filePath); - - // Convert image to base64 for piexif processing - const imageBase64 = imageBuffer.toString('binary'); - - // Prepare EXIF data for piexif - const exifDict = this.prepareExifDict(exifData); - - if (Object.keys(exifDict).length === 0) { - this.logger.debug('No valid EXIF data to embed'); - return; - } - - // Convert EXIF dict to bytes - const exifBytes = piexif.dump(exifDict); - - // Insert EXIF data into image - const newImageBase64 = piexif.insert(exifBytes, imageBase64); - - // Convert back to buffer and save - const newImageBuffer = Buffer.from(newImageBase64, 'binary'); - await fs.writeFile(filePath, newImageBuffer); - - this.logger.debug(`EXIF data preserved successfully for: ${filePath}`); - - } catch (error) { - this.logger.warn(`Failed to preserve EXIF data for ${filePath}:`, error.message); - // Don't throw error as EXIF preservation is not critical for image processing - } - } - - /** - * Remove sensitive EXIF data while preserving useful metadata - */ - async sanitizeExif(filePath: string, options: { - removeGps?: boolean; - removeCamera?: boolean; - removePersonalInfo?: boolean; - preserveOrientation?: boolean; - preserveDateTime?: boolean; - } = {}): Promise { - try { - const exifData = await this.extractExif(filePath); - - if (!exifData.exif) { - this.logger.debug(`No EXIF data to sanitize in: ${filePath}`); - return; - } - - // Create sanitized EXIF data - const sanitizedExif = { ...exifData }; - - // Remove GPS data if requested - if (options.removeGps !== false) { - delete sanitizedExif.gps; - if (sanitizedExif.exif) { - delete sanitizedExif.exif.GPSLatitude; - delete sanitizedExif.exif.GPSLongitude; - delete sanitizedExif.exif.GPSAltitude; - delete sanitizedExif.exif.GPSLatitudeRef; - delete sanitizedExif.exif.GPSLongitudeRef; - delete sanitizedExif.exif.GPSAltitudeRef; - } - } - - // Remove camera/device specific info if requested - if (options.removeCamera) { - if (sanitizedExif.exif) { - delete sanitizedExif.exif.Make; - delete sanitizedExif.exif.Model; - delete sanitizedExif.exif.Software; - delete sanitizedExif.exif.SerialNumber; - delete sanitizedExif.exif.LensModel; - delete sanitizedExif.exif.LensSerialNumber; - } - } - - // Remove personal information if requested - if (options.removePersonalInfo) { - if (sanitizedExif.exif) { - delete sanitizedExif.exif.Artist; - delete sanitizedExif.exif.Copyright; - delete sanitizedExif.exif.UserComment; - } - if (sanitizedExif.iptc) { - delete sanitizedExif.iptc.By_line; - delete sanitizedExif.iptc.Copyright_Notice; - delete sanitizedExif.iptc.Contact; - } - } - - // Preserve orientation if requested (default: preserve) - if (options.preserveOrientation !== false && exifData.exif?.Orientation) { - if (!sanitizedExif.exif) sanitizedExif.exif = {}; - sanitizedExif.exif.Orientation = exifData.exif.Orientation; - } - - // Preserve date/time if requested (default: preserve) - if (options.preserveDateTime !== false && exifData.exif) { - if (!sanitizedExif.exif) sanitizedExif.exif = {}; - if (exifData.exif.DateTime) sanitizedExif.exif.DateTime = exifData.exif.DateTime; - if (exifData.exif.DateTimeOriginal) sanitizedExif.exif.DateTimeOriginal = exifData.exif.DateTimeOriginal; - if (exifData.exif.DateTimeDigitized) sanitizedExif.exif.DateTimeDigitized = exifData.exif.DateTimeDigitized; - } - - // Apply sanitized EXIF data - await this.preserveExif(filePath, sanitizedExif); - - this.logger.debug(`EXIF data sanitized for: ${filePath}`); - - } catch (error) { - this.logger.warn(`Failed to sanitize EXIF data for ${filePath}:`, error.message); - } - } - - /** - * Extract GPS coordinates from EXIF data - */ - extractGpsCoordinates(exifData: ExifData): GpsCoordinates | null { - try { - const gps = exifData.gps || exifData.exif; - if (!gps) return null; - - // Handle different GPS coordinate formats - let latitude: number | undefined; - let longitude: number | undefined; - let altitude: number | undefined; - - // Modern format (decimal degrees) - if (typeof gps.latitude === 'number' && typeof gps.longitude === 'number') { - latitude = gps.latitude; - longitude = gps.longitude; - altitude = gps.altitude; - } - // Legacy EXIF format (degrees, minutes, seconds) - else if (gps.GPSLatitude && gps.GPSLongitude) { - latitude = this.dmsToDecimal(gps.GPSLatitude, gps.GPSLatitudeRef); - longitude = this.dmsToDecimal(gps.GPSLongitude, gps.GPSLongitudeRef); - - if (gps.GPSAltitude) { - altitude = gps.GPSAltitude; - if (gps.GPSAltitudeRef === 1) { - altitude = -altitude; // Below sea level - } - } - } - - if (latitude !== undefined && longitude !== undefined) { - const coordinates: GpsCoordinates = { latitude, longitude }; - if (altitude !== undefined) { - coordinates.altitude = altitude; - } - return coordinates; - } - - return null; - - } catch (error) { - this.logger.warn('Failed to extract GPS coordinates:', error.message); - return null; - } - } - - /** - * Get camera information from EXIF data - */ - getCameraInfo(exifData: ExifData): { - make?: string; - model?: string; - software?: string; - lens?: string; - settings?: { - fNumber?: number; - exposureTime?: string; - iso?: number; - focalLength?: number; - }; - } { - const exif = exifData.exif || {}; - - return { - make: exif.Make, - model: exif.Model, - software: exif.Software, - lens: exif.LensModel, - settings: { - fNumber: exif.FNumber, - exposureTime: exif.ExposureTime, - iso: exif.ISO || exif.ISOSpeedRatings, - focalLength: exif.FocalLength, - }, - }; - } - - /** - * Get image capture date from EXIF data - */ - getCaptureDate(exifData: ExifData): Date | null { - try { - const exif = exifData.exif || {}; - - // Try different date fields in order of preference - const dateFields = [ - 'DateTimeOriginal', - 'DateTimeDigitized', - 'DateTime', - 'CreateDate', - ]; - - for (const field of dateFields) { - if (exif[field]) { - const dateStr = exif[field]; - - // Parse EXIF date format: "YYYY:MM:DD HH:MM:SS" - if (typeof dateStr === 'string') { - const normalizedDate = dateStr.replace(/:/g, '-', 2); - const date = new Date(normalizedDate); - - if (!isNaN(date.getTime())) { - return date; - } - } - } - } - - return null; - - } catch (error) { - this.logger.warn('Failed to extract capture date:', error.message); - return null; - } - } - - /** - * Prepare EXIF dictionary for piexif - */ - private prepareExifDict(exifData: ExifData): any { - const exifDict: any = {}; - - try { - // Map EXIF data to piexif format - if (exifData.exif) { - exifDict['Exif'] = this.convertExifTags(exifData.exif); - } - - if (exifData.tiff) { - exifDict['0th'] = this.convertExifTags(exifData.tiff); - } - - if (exifData.gps) { - exifDict['GPS'] = this.convertGpsTags(exifData.gps); - } - - // Handle thumbnail data if present - if (exifData.exif && exifData.exif.thumbnail) { - exifDict['1st'] = {}; - } - - } catch (error) { - this.logger.warn('Error preparing EXIF dictionary:', error.message); - } - - return exifDict; - } - - /** - * Convert EXIF tags to piexif format - */ - private convertExifTags(tags: any): any { - const converted: any = {}; - - for (const [key, value] of Object.entries(tags)) { - if (value !== null && value !== undefined) { - // Convert specific tag formats - if (key === 'Orientation' && typeof value === 'number') { - converted[piexif.ExifIFD.Orientation] = value; - } else if (key === 'DateTime' && typeof value === 'string') { - converted[piexif.ImageIFD.DateTime] = value; - } else if (key === 'DateTimeOriginal' && typeof value === 'string') { - converted[piexif.ExifIFD.DateTimeOriginal] = value; - } - // Add more tag conversions as needed - } - } - - return converted; - } - - /** - * Convert GPS tags to piexif format - */ - private convertGpsTags(gps: any): any { - const converted: any = {}; - - if (gps.latitude && gps.longitude) { - const latDMS = this.decimalToDMS(Math.abs(gps.latitude)); - const lonDMS = this.decimalToDMS(Math.abs(gps.longitude)); - - converted[piexif.GPSIFD.GPSLatitude] = latDMS; - converted[piexif.GPSIFD.GPSLatitudeRef] = gps.latitude >= 0 ? 'N' : 'S'; - converted[piexif.GPSIFD.GPSLongitude] = lonDMS; - converted[piexif.GPSIFD.GPSLongitudeRef] = gps.longitude >= 0 ? 'E' : 'W'; - - if (gps.altitude) { - converted[piexif.GPSIFD.GPSAltitude] = [Math.abs(gps.altitude) * 1000, 1000]; - converted[piexif.GPSIFD.GPSAltitudeRef] = gps.altitude >= 0 ? 0 : 1; - } - } - - return converted; - } - - /** - * Convert DMS (Degrees, Minutes, Seconds) to decimal degrees - */ - private dmsToDecimal(dms: number[], ref: string): number { - if (!Array.isArray(dms) || dms.length < 3) return 0; - - const degrees = dms[0] || 0; - const minutes = dms[1] || 0; - const seconds = dms[2] || 0; - - let decimal = degrees + minutes / 60 + seconds / 3600; - - // Apply hemisphere reference - if (ref === 'S' || ref === 'W') { - decimal = -decimal; - } - - return decimal; - } - - /** - * Convert decimal degrees to DMS format - */ - private decimalToDMS(decimal: number): [number[], number[], number[]] { - const degrees = Math.floor(decimal); - const minutesFloat = (decimal - degrees) * 60; - const minutes = Math.floor(minutesFloat); - const seconds = (minutesFloat - minutes) * 60; - - return [ - [degrees, 1], - [minutes, 1], - [Math.round(seconds * 1000), 1000], // Preserve precision - ]; - } - - /** - * Check if file has EXIF data - */ - async hasExifData(filePath: string): Promise { - try { - const exifData = await this.extractExif(filePath); - return !!(exifData.exif || exifData.tiff || exifData.gps); - } catch (error) { - return false; - } - } -} \ No newline at end of file diff --git a/packages/worker/src/storage/file-processor.service.ts b/packages/worker/src/storage/file-processor.service.ts deleted file mode 100644 index 3a41578..0000000 --- a/packages/worker/src/storage/file-processor.service.ts +++ /dev/null @@ -1,480 +0,0 @@ -import { Injectable, Logger } from '@nestjs/common'; -import { ConfigService } from '@nestjs/config'; -import * as Sharp from 'sharp'; -import * as fs from 'fs/promises'; -import * as path from 'path'; -import { v4 as uuidv4 } from 'uuid'; -import { ExifPreserverService } from './exif-preserver.service'; -import { fileTypeFromFile } from 'file-type'; - -export interface ImageMetadata { - width: number; - height: number; - format: string; - size: number; - density?: number; - hasAlpha: boolean; - channels: number; - space: string; - exif?: any; - iptc?: any; - xmp?: any; -} - -export interface OptimizationOptions { - quality?: number; - maxWidth?: number; - maxHeight?: number; - format?: 'jpeg' | 'png' | 'webp' | 'auto'; - preserveExif?: boolean; - progressive?: boolean; - lossless?: boolean; -} - -@Injectable() -export class FileProcessorService { - private readonly logger = new Logger(FileProcessorService.name); - private readonly tempDir: string; - private readonly maxFileSize: number; - private readonly allowedTypes: string[]; - - constructor( - private configService: ConfigService, - private exifPreserverService: ExifPreserverService, - ) { - this.tempDir = this.configService.get('TEMP_DIR', '/tmp/seo-worker'); - this.maxFileSize = this.configService.get('MAX_FILE_SIZE', 50 * 1024 * 1024); // 50MB - this.allowedTypes = this.configService.get('ALLOWED_FILE_TYPES', 'jpg,jpeg,png,gif,webp').split(','); - } - - /** - * Extract comprehensive metadata from image file - */ - async extractMetadata(filePath: string): Promise { - try { - this.logger.debug(`Extracting metadata from: ${filePath}`); - - // Validate file exists and is readable - const fileStats = await fs.stat(filePath); - if (fileStats.size > this.maxFileSize) { - throw new Error(`File size ${fileStats.size} exceeds maximum allowed size ${this.maxFileSize}`); - } - - // Detect file type - const fileType = await fileTypeFromFile(filePath); - if (!fileType) { - throw new Error('Unable to determine file type'); - } - - // Validate file type is allowed - const extension = fileType.ext.toLowerCase(); - if (!this.allowedTypes.includes(extension)) { - throw new Error(`File type ${extension} is not allowed. Allowed types: ${this.allowedTypes.join(', ')}`); - } - - // Extract image metadata using Sharp - const sharpInstance = Sharp(filePath); - const sharpMetadata = await sharpInstance.metadata(); - - // Extract EXIF data - const exifData = await this.exifPreserverService.extractExif(filePath); - - const metadata: ImageMetadata = { - width: sharpMetadata.width || 0, - height: sharpMetadata.height || 0, - format: sharpMetadata.format || extension, - size: fileStats.size, - density: sharpMetadata.density, - hasAlpha: sharpMetadata.hasAlpha || false, - channels: sharpMetadata.channels || 3, - space: sharpMetadata.space || 'srgb', - exif: exifData.exif, - iptc: exifData.iptc, - xmp: exifData.xmp, - }; - - this.logger.debug(`Metadata extracted: ${metadata.width}x${metadata.height} ${metadata.format} (${metadata.size} bytes)`); - return metadata; - - } catch (error) { - this.logger.error(`Failed to extract metadata from ${filePath}:`, error.message); - throw error; - } - } - - /** - * Optimize image with various options - */ - async optimizeImage( - filePath: string, - options: OptimizationOptions = {} - ): Promise { - try { - this.logger.debug(`Optimizing image: ${filePath}`); - - // Extract original metadata if EXIF preservation is enabled - let originalExif: any = null; - if (options.preserveExif) { - originalExif = await this.exifPreserverService.extractExif(filePath); - } - - // Generate unique output filename - const outputFileName = `optimized_${uuidv4()}.${options.format || 'jpg'}`; - const outputPath = path.join(this.tempDir, outputFileName); - - // Initialize Sharp processing pipeline - let pipeline = Sharp(filePath); - - // Apply resizing if specified - if (options.maxWidth || options.maxHeight) { - pipeline = pipeline.resize(options.maxWidth, options.maxHeight, { - fit: 'inside', - withoutEnlargement: true, - }); - } - - // Apply format-specific optimizations - const quality = options.quality || 85; - const progressive = options.progressive !== false; - - switch (options.format) { - case 'jpeg': - pipeline = pipeline.jpeg({ - quality, - progressive, - mozjpeg: true, // Use mozjpeg for better compression - }); - break; - - case 'png': - pipeline = pipeline.png({ - quality, - progressive, - compressionLevel: 9, - adaptiveFiltering: true, - }); - break; - - case 'webp': - pipeline = pipeline.webp({ - quality, - lossless: options.lossless || false, - effort: 6, // High effort for better compression - }); - break; - - default: - // Auto-detect best format based on content - const metadata = await pipeline.metadata(); - if (metadata.hasAlpha) { - pipeline = pipeline.png({ quality, progressive }); - } else { - pipeline = pipeline.jpeg({ quality, progressive, mozjpeg: true }); - } - } - - // Process and save the image - await pipeline.toFile(outputPath); - - // Restore EXIF data if preservation was requested - if (options.preserveExif && originalExif) { - await this.exifPreserverService.preserveExif(outputPath, originalExif); - } - - // Log optimization results - const originalStats = await fs.stat(filePath); - const optimizedStats = await fs.stat(outputPath); - const compressionRatio = ((originalStats.size - optimizedStats.size) / originalStats.size * 100).toFixed(1); - - this.logger.debug( - `Image optimized: ${originalStats.size} -> ${optimizedStats.size} bytes (${compressionRatio}% reduction)` - ); - - return outputPath; - - } catch (error) { - this.logger.error(`Failed to optimize image ${filePath}:`, error.message); - throw error; - } - } - - /** - * Create thumbnail image - */ - async createThumbnail( - filePath: string, - width: number = 300, - height: number = 300, - quality: number = 80 - ): Promise { - try { - const thumbnailFileName = `thumb_${uuidv4()}.jpg`; - const thumbnailPath = path.join(this.tempDir, thumbnailFileName); - - await Sharp(filePath) - .resize(width, height, { - fit: 'cover', - position: 'center', - }) - .jpeg({ quality, progressive: true }) - .toFile(thumbnailPath); - - this.logger.debug(`Thumbnail created: ${thumbnailPath} (${width}x${height})`); - return thumbnailPath; - - } catch (error) { - this.logger.error(`Failed to create thumbnail for ${filePath}:`, error.message); - throw error; - } - } - - /** - * Convert image to different format - */ - async convertFormat( - filePath: string, - targetFormat: 'jpeg' | 'png' | 'webp', - quality: number = 85 - ): Promise { - try { - const convertedFileName = `converted_${uuidv4()}.${targetFormat}`; - const convertedPath = path.join(this.tempDir, convertedFileName); - - let pipeline = Sharp(filePath); - - switch (targetFormat) { - case 'jpeg': - pipeline = pipeline.jpeg({ quality, progressive: true, mozjpeg: true }); - break; - case 'png': - pipeline = pipeline.png({ quality, progressive: true }); - break; - case 'webp': - pipeline = pipeline.webp({ quality, effort: 6 }); - break; - } - - await pipeline.toFile(convertedPath); - - this.logger.debug(`Image converted to ${targetFormat}: ${convertedPath}`); - return convertedPath; - - } catch (error) { - this.logger.error(`Failed to convert image ${filePath} to ${targetFormat}:`, error.message); - throw error; - } - } - - /** - * Rotate image based on EXIF orientation - */ - async autoRotate(filePath: string): Promise { - try { - const rotatedFileName = `rotated_${uuidv4()}.jpg`; - const rotatedPath = path.join(this.tempDir, rotatedFileName); - - await Sharp(filePath) - .rotate() // Auto-rotate based on EXIF orientation - .jpeg({ quality: 95, progressive: true }) - .toFile(rotatedPath); - - this.logger.debug(`Image auto-rotated: ${rotatedPath}`); - return rotatedPath; - - } catch (error) { - this.logger.error(`Failed to auto-rotate image ${filePath}:`, error.message); - throw error; - } - } - - /** - * Generate multiple sizes of an image - */ - async generateMultipleSizes( - filePath: string, - sizes: Array<{ width: number; height: number; suffix: string }> - ): Promise { - try { - const generatedFiles: string[] = []; - - for (const size of sizes) { - const sizedFileName = `${size.suffix}_${uuidv4()}.jpg`; - const sizedPath = path.join(this.tempDir, sizedFileName); - - await Sharp(filePath) - .resize(size.width, size.height, { - fit: 'inside', - withoutEnlargement: true, - }) - .jpeg({ quality: 85, progressive: true }) - .toFile(sizedPath); - - generatedFiles.push(sizedPath); - } - - this.logger.debug(`Generated ${generatedFiles.length} different sizes`); - return generatedFiles; - - } catch (error) { - this.logger.error(`Failed to generate multiple sizes for ${filePath}:`, error.message); - throw error; - } - } - - /** - * Apply watermark to image - */ - async applyWatermark( - filePath: string, - watermarkPath: string, - position: 'top-left' | 'top-right' | 'bottom-left' | 'bottom-right' | 'center' = 'bottom-right', - opacity: number = 0.5 - ): Promise { - try { - const watermarkedFileName = `watermarked_${uuidv4()}.jpg`; - const watermarkedPath = path.join(this.tempDir, watermarkedFileName); - - // Prepare watermark - const watermark = await Sharp(watermarkPath) - .png() - .composite([{ - input: Buffer.from([255, 255, 255, Math.round(255 * opacity)]), - raw: { width: 1, height: 1, channels: 4 }, - tile: true, - blend: 'dest-in' - }]) - .toBuffer(); - - // Determine position - const gravity = this.getGravityFromPosition(position); - - await Sharp(filePath) - .composite([{ input: watermark, gravity }]) - .jpeg({ quality: 90, progressive: true }) - .toFile(watermarkedPath); - - this.logger.debug(`Watermark applied: ${watermarkedPath}`); - return watermarkedPath; - - } catch (error) { - this.logger.error(`Failed to apply watermark to ${filePath}:`, error.message); - throw error; - } - } - - /** - * Validate image file integrity - */ - async validateImage(filePath: string): Promise<{ - valid: boolean; - error?: string; - metadata?: ImageMetadata; - }> { - try { - // Try to extract metadata - this will fail if image is corrupted - const metadata = await this.extractMetadata(filePath); - - // Try to create a test thumbnail - this will catch most corruption issues - const testThumb = await this.createThumbnail(filePath, 100, 100); - await this.cleanupTempFile(testThumb); - - return { - valid: true, - metadata, - }; - - } catch (error) { - return { - valid: false, - error: error.message, - }; - } - } - - /** - * Clean up temporary file - */ - async cleanupTempFile(filePath: string): Promise { - try { - // Safety check: only delete files in our temp directory - if (!filePath.startsWith(this.tempDir)) { - this.logger.warn(`Skipping cleanup of file outside temp directory: ${filePath}`); - return; - } - - await fs.unlink(filePath); - this.logger.debug(`Temporary file cleaned up: ${filePath}`); - - } catch (error) { - if (error.code !== 'ENOENT') { - this.logger.warn(`Failed to cleanup temporary file ${filePath}:`, error.message); - } - } - } - - /** - * Batch cleanup of old temporary files - */ - async cleanupOldTempFiles(maxAge: number = 3600000): Promise { - try { - const files = await fs.readdir(this.tempDir); - const now = Date.now(); - let cleanedCount = 0; - - for (const file of files) { - try { - const filePath = path.join(this.tempDir, file); - const stats = await fs.stat(filePath); - const age = now - stats.mtime.getTime(); - - if (age > maxAge) { - await fs.unlink(filePath); - cleanedCount++; - } - } catch (error) { - // Skip files that can't be processed - continue; - } - } - - if (cleanedCount > 0) { - this.logger.log(`Cleaned up ${cleanedCount} old temporary files`); - } - - return cleanedCount; - - } catch (error) { - this.logger.error('Failed to cleanup old temporary files:', error.message); - return 0; - } - } - - private getGravityFromPosition( - position: 'top-left' | 'top-right' | 'bottom-left' | 'bottom-right' | 'center' - ): string { - const gravityMap = { - 'top-left': 'northwest', - 'top-right': 'northeast', - 'bottom-left': 'southwest', - 'bottom-right': 'southeast', - 'center': 'center', - }; - - return gravityMap[position] || 'southeast'; - } - - /** - * Get processing statistics - */ - getProcessingStats(): { - tempDir: string; - maxFileSize: number; - allowedTypes: string[]; - } { - return { - tempDir: this.tempDir, - maxFileSize: this.maxFileSize, - allowedTypes: this.allowedTypes, - }; - } -} \ No newline at end of file diff --git a/packages/worker/src/storage/minio.service.ts b/packages/worker/src/storage/minio.service.ts deleted file mode 100644 index be1f348..0000000 --- a/packages/worker/src/storage/minio.service.ts +++ /dev/null @@ -1,367 +0,0 @@ -import { Injectable, Logger } from '@nestjs/common'; -import { ConfigService } from '@nestjs/config'; -import { Client as MinioClient, BucketItem } from 'minio'; -import { StorageProvider } from './storage.service'; -import * as fs from 'fs'; -import * as path from 'path'; - -@Injectable() -export class MinioService implements StorageProvider { - private readonly logger = new Logger(MinioService.name); - private readonly client: MinioClient; - private readonly bucketName: string; - - constructor(private configService: ConfigService) { - const endpoint = this.configService.get('MINIO_ENDPOINT'); - const port = this.configService.get('MINIO_PORT', 9000); - const useSSL = this.configService.get('MINIO_USE_SSL', false); - const accessKey = this.configService.get('MINIO_ACCESS_KEY'); - const secretKey = this.configService.get('MINIO_SECRET_KEY'); - - if (!endpoint || !accessKey || !secretKey) { - throw new Error('MinIO configuration incomplete. Required: MINIO_ENDPOINT, MINIO_ACCESS_KEY, MINIO_SECRET_KEY'); - } - - this.bucketName = this.configService.get('MINIO_BUCKET_NAME', 'seo-images'); - - this.client = new MinioClient({ - endPoint: endpoint, - port, - useSSL, - accessKey, - secretKey, - }); - - this.logger.log(`MinIO client initialized: ${endpoint}:${port} (SSL: ${useSSL})`); - this.initializeBucket(); - } - - private async initializeBucket(): Promise { - try { - const bucketExists = await this.client.bucketExists(this.bucketName); - - if (!bucketExists) { - await this.client.makeBucket(this.bucketName, 'us-east-1'); - this.logger.log(`Created MinIO bucket: ${this.bucketName}`); - } else { - this.logger.log(`MinIO bucket exists: ${this.bucketName}`); - } - } catch (error) { - this.logger.error(`Failed to initialize MinIO bucket ${this.bucketName}:`, error.message); - throw error; - } - } - - async uploadFile(filePath: string, key: string, metadata?: any): Promise { - try { - // Prepare metadata - const fileStats = fs.statSync(filePath); - const metadataObj = { - 'Content-Type': this.getContentType(filePath), - 'X-Amz-Meta-Upload-Time': new Date().toISOString(), - 'X-Amz-Meta-Original-Name': path.basename(filePath), - ...metadata, - }; - - // Upload file - await this.client.fPutObject( - this.bucketName, - key, - filePath, - metadataObj - ); - - this.logger.debug(`File uploaded to MinIO: ${key} (${fileStats.size} bytes)`); - - // Return the object URL - return `${this.getEndpointUrl()}/${this.bucketName}/${key}`; - - } catch (error) { - this.logger.error(`Failed to upload file to MinIO: ${key}`, error.message); - throw error; - } - } - - async downloadFile(key: string, destPath: string): Promise { - try { - // Ensure destination directory exists - const destDir = path.dirname(destPath); - fs.mkdirSync(destDir, { recursive: true }); - - // Download file - await this.client.fGetObject(this.bucketName, key, destPath); - - this.logger.debug(`File downloaded from MinIO: ${key} -> ${destPath}`); - - } catch (error) { - this.logger.error(`Failed to download file from MinIO: ${key}`, error.message); - throw error; - } - } - - async deleteFile(key: string): Promise { - try { - await this.client.removeObject(this.bucketName, key); - this.logger.debug(`File deleted from MinIO: ${key}`); - - } catch (error) { - this.logger.error(`Failed to delete file from MinIO: ${key}`, error.message); - throw error; - } - } - - async moveFile(sourceKey: string, destKey: string): Promise { - try { - // Copy file to new location - await this.client.copyObject( - this.bucketName, - destKey, - `/${this.bucketName}/${sourceKey}` - ); - - // Delete original file - await this.client.removeObject(this.bucketName, sourceKey); - - this.logger.debug(`File moved in MinIO: ${sourceKey} -> ${destKey}`); - - } catch (error) { - this.logger.error(`Failed to move file in MinIO: ${sourceKey} -> ${destKey}`, error.message); - throw error; - } - } - - async getPublicUrl(key: string): Promise { - // MinIO doesn't have built-in public URLs, so we return the direct URL - // This assumes the bucket is configured for public read access - return `${this.getEndpointUrl()}/${this.bucketName}/${key}`; - } - - async generateSignedUrl(key: string, expiresIn: number): Promise { - try { - // Generate presigned URL for GET request - const signedUrl = await this.client.presignedGetObject( - this.bucketName, - key, - expiresIn - ); - - this.logger.debug(`Generated signed URL for MinIO object: ${key} (expires in ${expiresIn}s)`); - return signedUrl; - - } catch (error) { - this.logger.error(`Failed to generate signed URL for MinIO object: ${key}`, error.message); - throw error; - } - } - - async fileExists(key: string): Promise { - try { - await this.client.statObject(this.bucketName, key); - return true; - } catch (error) { - if (error.code === 'NotFound') { - return false; - } - this.logger.error(`Error checking if file exists in MinIO: ${key}`, error.message); - throw error; - } - } - - async getFileMetadata(key: string): Promise { - try { - const stat = await this.client.statObject(this.bucketName, key); - - return { - size: stat.size, - lastModified: stat.lastModified, - etag: stat.etag, - contentType: stat.metaData['content-type'], - metadata: stat.metaData, - }; - - } catch (error) { - this.logger.error(`Failed to get metadata for MinIO object: ${key}`, error.message); - throw error; - } - } - - async listFiles(prefix?: string, maxKeys: number = 1000): Promise { - try { - const objects: BucketItem[] = []; - const stream = this.client.listObjects(this.bucketName, prefix, true); - - return new Promise((resolve, reject) => { - stream.on('data', (obj) => { - objects.push(obj); - if (objects.length >= maxKeys) { - stream.destroy(); - } - }); - - stream.on('end', () => { - const keys = objects.map(obj => obj.name).filter(name => name !== undefined) as string[]; - resolve(keys); - }); - - stream.on('error', (error) => { - this.logger.error('Error listing MinIO objects:', error.message); - reject(error); - }); - }); - - } catch (error) { - this.logger.error('Failed to list MinIO objects:', error.message); - throw error; - } - } - - /** - * Upload file from buffer/stream - */ - async uploadBuffer( - buffer: Buffer, - key: string, - contentType?: string, - metadata?: any - ): Promise { - try { - const metadataObj = { - 'Content-Type': contentType || 'application/octet-stream', - 'X-Amz-Meta-Upload-Time': new Date().toISOString(), - ...metadata, - }; - - await this.client.putObject( - this.bucketName, - key, - buffer, - buffer.length, - metadataObj - ); - - this.logger.debug(`Buffer uploaded to MinIO: ${key} (${buffer.length} bytes)`); - return `${this.getEndpointUrl()}/${this.bucketName}/${key}`; - - } catch (error) { - this.logger.error(`Failed to upload buffer to MinIO: ${key}`, error.message); - throw error; - } - } - - /** - * Get file as buffer - */ - async getFileBuffer(key: string): Promise { - try { - const stream = await this.client.getObject(this.bucketName, key); - const chunks: Buffer[] = []; - - return new Promise((resolve, reject) => { - stream.on('data', (chunk) => chunks.push(chunk)); - stream.on('end', () => resolve(Buffer.concat(chunks))); - stream.on('error', reject); - }); - - } catch (error) { - this.logger.error(`Failed to get buffer from MinIO: ${key}`, error.message); - throw error; - } - } - - /** - * Generate upload URL for direct client uploads - */ - async generateUploadUrl( - key: string, - expiresIn: number = 3600, - conditions?: any - ): Promise<{ url: string; fields: any }> { - try { - const policy = this.client.newPostPolicy(); - policy.setBucket(this.bucketName); - policy.setKey(key); - policy.setExpires(new Date(Date.now() + expiresIn * 1000)); - - if (conditions) { - // Add custom conditions to policy - for (const [field, value] of Object.entries(conditions)) { - policy.setContentLengthRange(0, value as number); - } - } - - const result = await this.client.presignedPostPolicy(policy); - - this.logger.debug(`Generated upload URL for MinIO: ${key}`); - return { - url: result.postURL, - fields: result.formData, - }; - - } catch (error) { - this.logger.error(`Failed to generate upload URL for MinIO: ${key}`, error.message); - throw error; - } - } - - /** - * Get bucket statistics - */ - async getBucketStats(): Promise<{ - name: string; - objectCount: number; - totalSize: number; - }> { - try { - const objects: BucketItem[] = []; - const stream = this.client.listObjects(this.bucketName, '', true); - - return new Promise((resolve, reject) => { - stream.on('data', (obj) => objects.push(obj)); - - stream.on('end', () => { - const totalSize = objects.reduce((sum, obj) => sum + (obj.size || 0), 0); - resolve({ - name: this.bucketName, - objectCount: objects.length, - totalSize, - }); - }); - - stream.on('error', reject); - }); - - } catch (error) { - this.logger.error('Failed to get MinIO bucket stats:', error.message); - throw error; - } - } - - private getContentType(filePath: string): string { - const ext = path.extname(filePath).toLowerCase(); - const mimeTypes: { [key: string]: string } = { - '.jpg': 'image/jpeg', - '.jpeg': 'image/jpeg', - '.png': 'image/png', - '.gif': 'image/gif', - '.webp': 'image/webp', - '.svg': 'image/svg+xml', - '.pdf': 'application/pdf', - '.zip': 'application/zip', - '.txt': 'text/plain', - '.json': 'application/json', - }; - - return mimeTypes[ext] || 'application/octet-stream'; - } - - private getEndpointUrl(): string { - const endpoint = this.configService.get('MINIO_ENDPOINT'); - const port = this.configService.get('MINIO_PORT', 9000); - const useSSL = this.configService.get('MINIO_USE_SSL', false); - - const protocol = useSSL ? 'https' : 'http'; - const portSuffix = (useSSL && port === 443) || (!useSSL && port === 80) ? '' : `:${port}`; - - return `${protocol}://${endpoint}${portSuffix}`; - } -} \ No newline at end of file diff --git a/packages/worker/src/storage/s3.service.ts b/packages/worker/src/storage/s3.service.ts deleted file mode 100644 index aec62fb..0000000 --- a/packages/worker/src/storage/s3.service.ts +++ /dev/null @@ -1,401 +0,0 @@ -import { Injectable, Logger } from '@nestjs/common'; -import { ConfigService } from '@nestjs/config'; -import { S3 } from 'aws-sdk'; -import { StorageProvider } from './storage.service'; -import * as fs from 'fs'; -import * as path from 'path'; - -@Injectable() -export class S3Service implements StorageProvider { - private readonly logger = new Logger(S3Service.name); - private readonly s3: S3; - private readonly bucketName: string; - - constructor(private configService: ConfigService) { - const region = this.configService.get('AWS_REGION', 'us-east-1'); - const accessKeyId = this.configService.get('AWS_ACCESS_KEY_ID'); - const secretAccessKey = this.configService.get('AWS_SECRET_ACCESS_KEY'); - - if (!accessKeyId || !secretAccessKey) { - throw new Error('AWS S3 configuration incomplete. Required: AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY'); - } - - this.bucketName = this.configService.get('AWS_BUCKET_NAME'); - if (!this.bucketName) { - throw new Error('AWS_BUCKET_NAME is required for S3 storage'); - } - - this.s3 = new S3({ - region, - accessKeyId, - secretAccessKey, - signatureVersion: 'v4', - }); - - this.logger.log(`AWS S3 client initialized: ${this.bucketName} (${region})`); - } - - async uploadFile(filePath: string, key: string, metadata?: any): Promise { - try { - const fileStream = fs.createReadStream(filePath); - const fileStats = fs.statSync(filePath); - - const uploadParams: S3.PutObjectRequest = { - Bucket: this.bucketName, - Key: key, - Body: fileStream, - ContentType: this.getContentType(filePath), - Metadata: { - 'upload-time': new Date().toISOString(), - 'original-name': path.basename(filePath), - ...metadata, - }, - }; - - const result = await this.s3.upload(uploadParams).promise(); - - this.logger.debug(`File uploaded to S3: ${key} (${fileStats.size} bytes)`); - return result.Location; - - } catch (error) { - this.logger.error(`Failed to upload file to S3: ${key}`, error.message); - throw error; - } - } - - async downloadFile(key: string, destPath: string): Promise { - try { - // Ensure destination directory exists - const destDir = path.dirname(destPath); - fs.mkdirSync(destDir, { recursive: true }); - - const downloadParams: S3.GetObjectRequest = { - Bucket: this.bucketName, - Key: key, - }; - - const result = await this.s3.getObject(downloadParams).promise(); - - if (!result.Body) { - throw new Error('No data received from S3'); - } - - // Write file to destination - fs.writeFileSync(destPath, result.Body as Buffer); - - this.logger.debug(`File downloaded from S3: ${key} -> ${destPath}`); - - } catch (error) { - this.logger.error(`Failed to download file from S3: ${key}`, error.message); - throw error; - } - } - - async deleteFile(key: string): Promise { - try { - const deleteParams: S3.DeleteObjectRequest = { - Bucket: this.bucketName, - Key: key, - }; - - await this.s3.deleteObject(deleteParams).promise(); - this.logger.debug(`File deleted from S3: ${key}`); - - } catch (error) { - this.logger.error(`Failed to delete file from S3: ${key}`, error.message); - throw error; - } - } - - async moveFile(sourceKey: string, destKey: string): Promise { - try { - // Copy object to new location - const copyParams: S3.CopyObjectRequest = { - Bucket: this.bucketName, - CopySource: `${this.bucketName}/${sourceKey}`, - Key: destKey, - }; - - await this.s3.copyObject(copyParams).promise(); - - // Delete original object - await this.deleteFile(sourceKey); - - this.logger.debug(`File moved in S3: ${sourceKey} -> ${destKey}`); - - } catch (error) { - this.logger.error(`Failed to move file in S3: ${sourceKey} -> ${destKey}`, error.message); - throw error; - } - } - - async getPublicUrl(key: string): Promise { - // Return the public S3 URL (assumes bucket is public) - const region = this.configService.get('AWS_REGION', 'us-east-1'); - return `https://${this.bucketName}.s3.${region}.amazonaws.com/${key}`; - } - - async generateSignedUrl(key: string, expiresIn: number): Promise { - try { - const params: S3.GetObjectRequest = { - Bucket: this.bucketName, - Key: key, - }; - - const signedUrl = this.s3.getSignedUrl('getObject', { - ...params, - Expires: expiresIn, - }); - - this.logger.debug(`Generated signed URL for S3 object: ${key} (expires in ${expiresIn}s)`); - return signedUrl; - - } catch (error) { - this.logger.error(`Failed to generate signed URL for S3 object: ${key}`, error.message); - throw error; - } - } - - async fileExists(key: string): Promise { - try { - const params: S3.HeadObjectRequest = { - Bucket: this.bucketName, - Key: key, - }; - - await this.s3.headObject(params).promise(); - return true; - - } catch (error) { - if (error.code === 'NotFound' || error.statusCode === 404) { - return false; - } - this.logger.error(`Error checking if file exists in S3: ${key}`, error.message); - throw error; - } - } - - async getFileMetadata(key: string): Promise { - try { - const params: S3.HeadObjectRequest = { - Bucket: this.bucketName, - Key: key, - }; - - const result = await this.s3.headObject(params).promise(); - - return { - size: result.ContentLength, - lastModified: result.LastModified, - etag: result.ETag, - contentType: result.ContentType, - metadata: result.Metadata, - storageClass: result.StorageClass, - }; - - } catch (error) { - this.logger.error(`Failed to get metadata for S3 object: ${key}`, error.message); - throw error; - } - } - - async listFiles(prefix?: string, maxKeys: number = 1000): Promise { - try { - const params: S3.ListObjectsV2Request = { - Bucket: this.bucketName, - Prefix: prefix, - MaxKeys: maxKeys, - }; - - const result = await this.s3.listObjectsV2(params).promise(); - - return (result.Contents || []) - .map(obj => obj.Key) - .filter(key => key !== undefined) as string[]; - - } catch (error) { - this.logger.error('Failed to list S3 objects:', error.message); - throw error; - } - } - - /** - * Upload file from buffer - */ - async uploadBuffer( - buffer: Buffer, - key: string, - contentType?: string, - metadata?: any - ): Promise { - try { - const uploadParams: S3.PutObjectRequest = { - Bucket: this.bucketName, - Key: key, - Body: buffer, - ContentType: contentType || 'application/octet-stream', - Metadata: { - 'upload-time': new Date().toISOString(), - ...metadata, - }, - }; - - const result = await this.s3.upload(uploadParams).promise(); - - this.logger.debug(`Buffer uploaded to S3: ${key} (${buffer.length} bytes)`); - return result.Location; - - } catch (error) { - this.logger.error(`Failed to upload buffer to S3: ${key}`, error.message); - throw error; - } - } - - /** - * Get file as buffer - */ - async getFileBuffer(key: string): Promise { - try { - const params: S3.GetObjectRequest = { - Bucket: this.bucketName, - Key: key, - }; - - const result = await this.s3.getObject(params).promise(); - - if (!result.Body) { - throw new Error('No data received from S3'); - } - - return result.Body as Buffer; - - } catch (error) { - this.logger.error(`Failed to get buffer from S3: ${key}`, error.message); - throw error; - } - } - - /** - * Generate upload URL for direct client uploads - */ - async generateUploadUrl( - key: string, - expiresIn: number = 3600, - conditions?: any - ): Promise<{ url: string; fields: any }> { - try { - const params: any = { - Bucket: this.bucketName, - Fields: { - key, - }, - Expires: expiresIn, - }; - - if (conditions) { - params.Conditions = conditions; - } - - return new Promise((resolve, reject) => { - this.s3.createPresignedPost(params, (error, data) => { - if (error) { - reject(error); - } else { - this.logger.debug(`Generated upload URL for S3: ${key}`); - resolve({ - url: data.url, - fields: data.fields, - }); - } - }); - }); - - } catch (error) { - this.logger.error(`Failed to generate upload URL for S3: ${key}`, error.message); - throw error; - } - } - - /** - * Get bucket statistics - */ - async getBucketStats(): Promise<{ - name: string; - objectCount: number; - totalSize: number; - }> { - try { - const params: S3.ListObjectsV2Request = { - Bucket: this.bucketName, - }; - - let objectCount = 0; - let totalSize = 0; - let continuationToken: string | undefined; - - do { - if (continuationToken) { - params.ContinuationToken = continuationToken; - } - - const result = await this.s3.listObjectsV2(params).promise(); - - if (result.Contents) { - objectCount += result.Contents.length; - totalSize += result.Contents.reduce((sum, obj) => sum + (obj.Size || 0), 0); - } - - continuationToken = result.NextContinuationToken; - } while (continuationToken); - - return { - name: this.bucketName, - objectCount, - totalSize, - }; - - } catch (error) { - this.logger.error('Failed to get S3 bucket stats:', error.message); - throw error; - } - } - - /** - * Enable versioning on bucket - */ - async enableVersioning(): Promise { - try { - const params: S3.PutBucketVersioningRequest = { - Bucket: this.bucketName, - VersioningConfiguration: { - Status: 'Enabled', - }, - }; - - await this.s3.putBucketVersioning(params).promise(); - this.logger.log(`Versioning enabled for S3 bucket: ${this.bucketName}`); - - } catch (error) { - this.logger.error(`Failed to enable versioning for S3 bucket: ${this.bucketName}`, error.message); - throw error; - } - } - - private getContentType(filePath: string): string { - const ext = path.extname(filePath).toLowerCase(); - const mimeTypes: { [key: string]: string } = { - '.jpg': 'image/jpeg', - '.jpeg': 'image/jpeg', - '.png': 'image/png', - '.gif': 'image/gif', - '.webp': 'image/webp', - '.svg': 'image/svg+xml', - '.pdf': 'application/pdf', - '.zip': 'application/zip', - '.txt': 'text/plain', - '.json': 'application/json', - }; - - return mimeTypes[ext] || 'application/octet-stream'; - } -} \ No newline at end of file diff --git a/packages/worker/src/storage/storage.module.ts b/packages/worker/src/storage/storage.module.ts deleted file mode 100644 index 8061690..0000000 --- a/packages/worker/src/storage/storage.module.ts +++ /dev/null @@ -1,29 +0,0 @@ -import { Module } from '@nestjs/common'; -import { ConfigModule } from '@nestjs/config'; -import { StorageService } from './storage.service'; -import { MinioService } from './minio.service'; -import { S3Service } from './s3.service'; -import { FileProcessorService } from './file-processor.service'; -import { ExifPreserverService } from './exif-preserver.service'; -import { ZipCreatorService } from './zip-creator.service'; - -@Module({ - imports: [ConfigModule], - providers: [ - StorageService, - MinioService, - S3Service, - FileProcessorService, - ExifPreserverService, - ZipCreatorService, - ], - exports: [ - StorageService, - MinioService, - S3Service, - FileProcessorService, - ExifPreserverService, - ZipCreatorService, - ], -}) -export class StorageModule {} \ No newline at end of file diff --git a/packages/worker/src/storage/storage.service.ts b/packages/worker/src/storage/storage.service.ts deleted file mode 100644 index bff178b..0000000 --- a/packages/worker/src/storage/storage.service.ts +++ /dev/null @@ -1,343 +0,0 @@ -import { Injectable, Logger } from '@nestjs/common'; -import { ConfigService } from '@nestjs/config'; -import { MinioService } from './minio.service'; -import { S3Service } from './s3.service'; -import * as fs from 'fs/promises'; -import * as path from 'path'; -import { v4 as uuidv4 } from 'uuid'; - -export interface StorageProvider { - uploadFile(filePath: string, key: string, metadata?: any): Promise; - downloadFile(key: string, destPath: string): Promise; - deleteFile(key: string): Promise; - moveFile(sourceKey: string, destKey: string): Promise; - getPublicUrl(key: string): Promise; - generateSignedUrl(key: string, expiresIn: number): Promise; - fileExists(key: string): Promise; - getFileMetadata(key: string): Promise; - listFiles(prefix?: string, maxKeys?: number): Promise; -} - -@Injectable() -export class StorageService { - private readonly logger = new Logger(StorageService.name); - private readonly provider: StorageProvider; - private readonly tempDir: string; - - constructor( - private configService: ConfigService, - private minioService: MinioService, - private s3Service: S3Service, - ) { - // Determine which storage provider to use - const useMinIO = !!this.configService.get('MINIO_ENDPOINT'); - const useS3 = !!this.configService.get('AWS_BUCKET_NAME'); - - if (useMinIO) { - this.provider = this.minioService; - this.logger.log('Using MinIO storage provider'); - } else if (useS3) { - this.provider = this.s3Service; - this.logger.log('Using AWS S3 storage provider'); - } else { - throw new Error('No storage provider configured. Please configure either MinIO or AWS S3.'); - } - - this.tempDir = this.configService.get('TEMP_DIR', '/tmp/seo-worker'); - this.initializeTempDirectory(); - } - - private async initializeTempDirectory(): Promise { - try { - await fs.mkdir(this.tempDir, { recursive: true }); - this.logger.log(`Temporary directory initialized: ${this.tempDir}`); - } catch (error) { - this.logger.error(`Failed to create temp directory ${this.tempDir}:`, error.message); - throw error; - } - } - - /** - * Upload file to storage - */ - async uploadFile( - filePath: string, - key: string, - metadata?: { [key: string]: string } - ): Promise { - try { - this.logger.debug(`Uploading file: ${filePath} -> ${key}`); - - const uploadedUrl = await this.provider.uploadFile(filePath, key, metadata); - - this.logger.debug(`File uploaded successfully: ${key}`); - return uploadedUrl; - - } catch (error) { - this.logger.error(`Failed to upload file ${filePath} to ${key}:`, error.message); - throw error; - } - } - - /** - * Download file from storage to local temporary directory - */ - async downloadToTemp(key: string): Promise { - try { - const tempFileName = `${uuidv4()}_${path.basename(key)}`; - const tempFilePath = path.join(this.tempDir, tempFileName); - - this.logger.debug(`Downloading file: ${key} -> ${tempFilePath}`); - - await this.provider.downloadFile(key, tempFilePath); - - this.logger.debug(`File downloaded successfully: ${tempFilePath}`); - return tempFilePath; - - } catch (error) { - this.logger.error(`Failed to download file ${key}:`, error.message); - throw error; - } - } - - /** - * Download file from storage to specific path - */ - async downloadFile(key: string, destPath: string): Promise { - try { - // Ensure destination directory exists - const destDir = path.dirname(destPath); - await fs.mkdir(destDir, { recursive: true }); - - await this.provider.downloadFile(key, destPath); - this.logger.debug(`File downloaded: ${key} -> ${destPath}`); - - } catch (error) { - this.logger.error(`Failed to download file ${key} to ${destPath}:`, error.message); - throw error; - } - } - - /** - * Delete file from storage - */ - async deleteFile(key: string): Promise { - try { - await this.provider.deleteFile(key); - this.logger.debug(`File deleted: ${key}`); - - } catch (error) { - this.logger.error(`Failed to delete file ${key}:`, error.message); - throw error; - } - } - - /** - * Move/rename file in storage - */ - async moveFile(sourceKey: string, destKey: string): Promise { - try { - await this.provider.moveFile(sourceKey, destKey); - this.logger.debug(`File moved: ${sourceKey} -> ${destKey}`); - - } catch (error) { - this.logger.error(`Failed to move file ${sourceKey} to ${destKey}:`, error.message); - throw error; - } - } - - /** - * Get public URL for file (if supported) - */ - async getPublicUrl(key: string): Promise { - try { - return await this.provider.getPublicUrl(key); - } catch (error) { - this.logger.error(`Failed to get public URL for ${key}:`, error.message); - throw error; - } - } - - /** - * Generate signed URL for temporary access - */ - async generateSignedUrl(key: string, expiresIn: number = 3600): Promise { - try { - return await this.provider.generateSignedUrl(key, expiresIn); - } catch (error) { - this.logger.error(`Failed to generate signed URL for ${key}:`, error.message); - throw error; - } - } - - /** - * Check if file exists in storage - */ - async fileExists(key: string): Promise { - try { - return await this.provider.fileExists(key); - } catch (error) { - this.logger.error(`Failed to check if file exists ${key}:`, error.message); - return false; - } - } - - /** - * Get file metadata - */ - async getFileMetadata(key: string): Promise { - try { - return await this.provider.getFileMetadata(key); - } catch (error) { - this.logger.error(`Failed to get metadata for ${key}:`, error.message); - throw error; - } - } - - /** - * List files with optional prefix - */ - async listFiles(prefix?: string, maxKeys: number = 1000): Promise { - try { - return await this.provider.listFiles(prefix, maxKeys); - } catch (error) { - this.logger.error(`Failed to list files with prefix ${prefix}:`, error.message); - throw error; - } - } - - /** - * Delete temporary file - */ - async deleteTempFile(filePath: string): Promise { - try { - // Only delete files in our temp directory for safety - if (!filePath.startsWith(this.tempDir)) { - this.logger.warn(`Skipping deletion of file outside temp directory: ${filePath}`); - return; - } - - await fs.unlink(filePath); - this.logger.debug(`Temporary file deleted: ${filePath}`); - - } catch (error) { - if (error.code !== 'ENOENT') { // Ignore file not found errors - this.logger.warn(`Failed to delete temporary file ${filePath}:`, error.message); - } - } - } - - /** - * Clean up old temporary files - */ - async cleanupTempFiles(maxAge: number = 3600000): Promise { - try { - const files = await fs.readdir(this.tempDir); - const now = Date.now(); - let cleanedCount = 0; - - for (const file of files) { - const filePath = path.join(this.tempDir, file); - - try { - const stats = await fs.stat(filePath); - const age = now - stats.mtime.getTime(); - - if (age > maxAge) { - await fs.unlink(filePath); - cleanedCount++; - } - } catch (error) { - // Skip files that can't be processed - continue; - } - } - - if (cleanedCount > 0) { - this.logger.log(`Cleaned up ${cleanedCount} old temporary files`); - } - - } catch (error) { - this.logger.error('Failed to cleanup temporary files:', error.message); - } - } - - /** - * Get storage statistics - */ - async getStorageStats(): Promise<{ - provider: string; - tempDir: string; - tempFilesCount: number; - tempDirSize: number; - }> { - try { - const files = await fs.readdir(this.tempDir); - let totalSize = 0; - - for (const file of files) { - try { - const filePath = path.join(this.tempDir, file); - const stats = await fs.stat(filePath); - totalSize += stats.size; - } catch (error) { - // Skip files that can't be processed - } - } - - return { - provider: this.provider.constructor.name, - tempDir: this.tempDir, - tempFilesCount: files.length, - tempDirSize: totalSize, - }; - - } catch (error) { - this.logger.error('Failed to get storage stats:', error.message); - return { - provider: this.provider.constructor.name, - tempDir: this.tempDir, - tempFilesCount: 0, - tempDirSize: 0, - }; - } - } - - /** - * Test storage connectivity - */ - async testConnection(): Promise { - try { - // Create a small test file - const testKey = `test/${uuidv4()}.txt`; - const testContent = 'Storage connection test'; - const testFilePath = path.join(this.tempDir, 'connection-test.txt'); - - // Write test file - await fs.writeFile(testFilePath, testContent); - - // Upload test file - await this.uploadFile(testFilePath, testKey); - - // Download test file - const downloadPath = path.join(this.tempDir, 'connection-test-download.txt'); - await this.downloadFile(testKey, downloadPath); - - // Verify content - const downloadedContent = await fs.readFile(downloadPath, 'utf8'); - const isValid = downloadedContent === testContent; - - // Cleanup - await this.deleteFile(testKey); - await this.deleteTempFile(testFilePath); - await this.deleteTempFile(downloadPath); - - this.logger.log(`Storage connection test: ${isValid ? 'PASSED' : 'FAILED'}`); - return isValid; - - } catch (error) { - this.logger.error('Storage connection test failed:', error.message); - return false; - } - } -} \ No newline at end of file diff --git a/packages/worker/src/storage/zip-creator.service.ts b/packages/worker/src/storage/zip-creator.service.ts deleted file mode 100644 index d06f8f8..0000000 --- a/packages/worker/src/storage/zip-creator.service.ts +++ /dev/null @@ -1,465 +0,0 @@ -import { Injectable, Logger } from '@nestjs/common'; -import { ConfigService } from '@nestjs/config'; -import * as archiver from 'archiver'; -import * as fs from 'fs'; -import * as path from 'path'; -import { v4 as uuidv4 } from 'uuid'; -import { StorageService } from './storage.service'; -import { DatabaseService } from '../database/database.service'; - -export interface ZipEntry { - fileName: string; - originalName: string; - proposedName: string; - filePath?: string; - s3Key?: string; -} - -export interface ZipCreationOptions { - includeOriginals?: boolean; - compressionLevel?: number; - password?: string; - excludeMetadata?: boolean; - customStructure?: boolean; -} - -@Injectable() -export class ZipCreatorService { - private readonly logger = new Logger(ZipCreatorService.name); - private readonly tempDir: string; - - constructor( - private configService: ConfigService, - private storageService: StorageService, - private databaseService: DatabaseService, - ) { - this.tempDir = this.configService.get('TEMP_DIR', '/tmp/seo-worker'); - } - - /** - * Create ZIP file for a batch of processed images - */ - async createBatchZip( - batchId: string, - imageIds: string[], - zipName: string, - options: ZipCreationOptions = {} - ): Promise { - const startTime = Date.now(); - this.logger.log(`πŸ—‚οΈ Creating ZIP for batch ${batchId} with ${imageIds.length} images`); - - const zipFileName = `${zipName}_${uuidv4()}.zip`; - const zipPath = path.join(this.tempDir, zipFileName); - - try { - // Get image details from database - const images = await this.databaseService.getImagesByIds(imageIds); - - if (images.length === 0) { - throw new Error('No images found for ZIP creation'); - } - - // Create ZIP entries - const zipEntries = await this.prepareZipEntries(images, options); - - // Create the ZIP file - await this.createZipFromEntries(zipPath, zipEntries, options); - - const stats = fs.statSync(zipPath); - const processingTime = Date.now() - startTime; - - this.logger.log( - `βœ… ZIP created successfully: ${zipPath} (${stats.size} bytes) in ${processingTime}ms` - ); - - return zipPath; - - } catch (error) { - this.logger.error(`❌ Failed to create ZIP for batch ${batchId}:`, error.message); - - // Cleanup failed ZIP file - try { - if (fs.existsSync(zipPath)) { - fs.unlinkSync(zipPath); - } - } catch (cleanupError) { - this.logger.warn(`Failed to cleanup failed ZIP file: ${cleanupError.message}`); - } - - throw error; - } - } - - /** - * Create ZIP file from individual files - */ - async createZipFromFiles( - files: Array<{ filePath: string; zipPath: string }>, - outputPath: string, - options: ZipCreationOptions = {} - ): Promise { - return new Promise((resolve, reject) => { - const output = fs.createWriteStream(outputPath); - const archive = archiver('zip', { - zlib: { level: options.compressionLevel || 6 }, - }); - - // Handle stream events - output.on('close', () => { - this.logger.debug(`ZIP file created: ${outputPath} (${archive.pointer()} bytes)`); - resolve(); - }); - - archive.on('error', (error) => { - this.logger.error('ZIP creation error:', error.message); - reject(error); - }); - - archive.on('warning', (warning) => { - this.logger.warn('ZIP creation warning:', warning.message); - }); - - // Pipe archive data to output file - archive.pipe(output); - - // Add files to archive - for (const file of files) { - if (fs.existsSync(file.filePath)) { - archive.file(file.filePath, { name: file.zipPath }); - } else { - this.logger.warn(`File not found, skipping: ${file.filePath}`); - } - } - - // Add password protection if specified - if (options.password) { - // Note: Basic archiver doesn't support password protection - // For production, consider using node-7z or yazl with encryption - this.logger.warn('Password protection requested but not implemented in basic archiver'); - } - - // Finalize the archive - archive.finalize(); - }); - } - - /** - * Create ZIP with custom folder structure - */ - async createStructuredZip( - batchId: string, - structure: { - [folderName: string]: string[]; // folder name -> array of image IDs - }, - zipName: string, - options: ZipCreationOptions = {} - ): Promise { - const zipFileName = `${zipName}_structured_${uuidv4()}.zip`; - const zipPath = path.join(this.tempDir, zipFileName); - - return new Promise(async (resolve, reject) => { - try { - const output = fs.createWriteStream(zipPath); - const archive = archiver('zip', { - zlib: { level: options.compressionLevel || 6 }, - }); - - // Handle stream events - output.on('close', () => { - this.logger.log(`Structured ZIP created: ${zipPath} (${archive.pointer()} bytes)`); - resolve(zipPath); - }); - - archive.on('error', reject); - archive.pipe(output); - - // Process each folder - for (const [folderName, imageIds] of Object.entries(structure)) { - if (imageIds.length === 0) continue; - - const images = await this.databaseService.getImagesByIds(imageIds); - - for (const image of images) { - try { - // Download image to temp location - const tempFilePath = await this.storageService.downloadToTemp(image.s3Key); - - // Determine filename to use in ZIP - const fileName = image.proposedName || image.originalName; - const zipEntryPath = `${folderName}/${fileName}`; - - // Add file to archive - archive.file(tempFilePath, { name: zipEntryPath }); - - // Schedule cleanup of temp file after archive is complete - output.on('close', () => { - this.storageService.deleteTempFile(tempFilePath).catch(() => {}); - }); - - } catch (error) { - this.logger.warn(`Failed to add image ${image.id} to ZIP:`, error.message); - } - } - } - - // Add README file if requested - if (options.includeOriginals !== false) { - const readmeContent = this.generateReadmeContent(batchId, structure); - archive.append(readmeContent, { name: 'README.txt' }); - } - - archive.finalize(); - - } catch (error) { - reject(error); - } - }); - } - - /** - * Prepare ZIP entries from image data - */ - private async prepareZipEntries( - images: any[], - options: ZipCreationOptions - ): Promise { - const entries: ZipEntry[] = []; - const usedNames = new Set(); - - for (const image of images) { - try { - // Determine the filename to use - let fileName = image.proposedName || image.originalName; - - // Ensure unique filenames - fileName = this.ensureUniqueFilename(fileName, usedNames); - usedNames.add(fileName.toLowerCase()); - - const entry: ZipEntry = { - fileName, - originalName: image.originalName, - proposedName: image.proposedName || image.originalName, - s3Key: image.s3Key, - }; - - entries.push(entry); - - } catch (error) { - this.logger.warn(`Failed to prepare ZIP entry for image ${image.id}:`, error.message); - } - } - - this.logger.debug(`Prepared ${entries.length} ZIP entries`); - return entries; - } - - /** - * Create ZIP file from prepared entries - */ - private async createZipFromEntries( - zipPath: string, - entries: ZipEntry[], - options: ZipCreationOptions - ): Promise { - return new Promise(async (resolve, reject) => { - const output = fs.createWriteStream(zipPath); - const archive = archiver('zip', { - zlib: { level: options.compressionLevel || 6 }, - }); - - const tempFiles: string[] = []; - - // Handle stream events - output.on('close', () => { - // Cleanup temp files - this.cleanupTempFiles(tempFiles); - resolve(); - }); - - archive.on('error', (error) => { - this.cleanupTempFiles(tempFiles); - reject(error); - }); - - archive.pipe(output); - - try { - // Process each entry - for (const entry of entries) { - if (entry.s3Key) { - // Download file from storage - const tempFilePath = await this.storageService.downloadToTemp(entry.s3Key); - tempFiles.push(tempFilePath); - - // Add to archive - archive.file(tempFilePath, { name: entry.fileName }); - } else if (entry.filePath) { - // Use local file - archive.file(entry.filePath, { name: entry.fileName }); - } - } - - // Add metadata file if not excluded - if (!options.excludeMetadata) { - const metadataContent = this.generateMetadataContent(entries); - archive.append(metadataContent, { name: 'metadata.json' }); - } - - // Add processing summary - const summaryContent = this.generateSummaryContent(entries); - archive.append(summaryContent, { name: 'processing_summary.txt' }); - - archive.finalize(); - - } catch (error) { - this.cleanupTempFiles(tempFiles); - reject(error); - } - }); - } - - /** - * Ensure filename is unique within the ZIP - */ - private ensureUniqueFilename(fileName: string, usedNames: Set): string { - const originalName = fileName; - const baseName = path.parse(fileName).name; - const extension = path.parse(fileName).ext; - - let counter = 1; - let uniqueName = fileName; - - while (usedNames.has(uniqueName.toLowerCase())) { - uniqueName = `${baseName}_${counter}${extension}`; - counter++; - } - - if (uniqueName !== originalName) { - this.logger.debug(`Renamed duplicate file: ${originalName} -> ${uniqueName}`); - } - - return uniqueName; - } - - /** - * Generate metadata JSON content - */ - private generateMetadataContent(entries: ZipEntry[]): string { - const metadata = { - createdAt: new Date().toISOString(), - totalFiles: entries.length, - processingInfo: { - service: 'SEO Image Renamer Worker', - version: '1.0.0', - }, - files: entries.map(entry => ({ - fileName: entry.fileName, - originalName: entry.originalName, - proposedName: entry.proposedName, - })), - }; - - return JSON.stringify(metadata, null, 2); - } - - /** - * Generate summary text content - */ - private generateSummaryContent(entries: ZipEntry[]): string { - const renamedCount = entries.filter(e => e.fileName !== e.originalName).length; - const unchangedCount = entries.length - renamedCount; - - return `SEO Image Renamer - Processing Summary -========================================== - -Total Files: ${entries.length} -Renamed Files: ${renamedCount} -Unchanged Files: ${unchangedCount} - -Processing Date: ${new Date().toISOString()} - -File List: -${entries.map(entry => { - const status = entry.fileName !== entry.originalName ? 'βœ“ RENAMED' : '- unchanged'; - return `${status}: ${entry.originalName} -> ${entry.fileName}`; -}).join('\n')} - -Generated by SEO Image Renamer Worker Service -For support, visit: https://seo-image-renamer.com -`; - } - - /** - * Generate README content for structured ZIPs - */ - private generateReadmeContent(batchId: string, structure: { [key: string]: string[] }): string { - const folderList = Object.entries(structure) - .map(([folder, imageIds]) => ` ${folder}/ (${imageIds.length} images)`) - .join('\n'); - - return `SEO Image Renamer - Batch Processing Results -============================================= - -Batch ID: ${batchId} -Created: ${new Date().toISOString()} - -Folder Structure: -${folderList} - -Instructions: -- Each folder contains images organized by your specified criteria -- Filenames have been optimized for SEO based on AI vision analysis -- Original filenames are preserved in the metadata.json file - -For more information about our AI-powered image renaming service, -visit: https://seo-image-renamer.com -`; - } - - /** - * Cleanup temporary files - */ - private async cleanupTempFiles(filePaths: string[]): Promise { - for (const filePath of filePaths) { - try { - await this.storageService.deleteTempFile(filePath); - } catch (error) { - this.logger.warn(`Failed to cleanup temp file ${filePath}:`, error.message); - } - } - } - - /** - * Cleanup ZIP file - */ - async cleanupZipFile(zipPath: string): Promise { - try { - // Only delete files in our temp directory for safety - if (zipPath.startsWith(this.tempDir)) { - fs.unlinkSync(zipPath); - this.logger.debug(`ZIP file cleaned up: ${zipPath}`); - } else { - this.logger.warn(`Skipping cleanup of ZIP file outside temp directory: ${zipPath}`); - } - } catch (error) { - if (error.code !== 'ENOENT') { - this.logger.warn(`Failed to cleanup ZIP file ${zipPath}:`, error.message); - } - } - } - - /** - * Get ZIP creation statistics - */ - getZipStats(): { - tempDir: string; - supportedFormats: string[]; - defaultCompression: number; - } { - return { - tempDir: this.tempDir, - supportedFormats: ['zip'], - defaultCompression: 6, - }; - } -} \ No newline at end of file diff --git a/packages/worker/src/vision/google-vision.service.ts b/packages/worker/src/vision/google-vision.service.ts deleted file mode 100644 index 586a261..0000000 --- a/packages/worker/src/vision/google-vision.service.ts +++ /dev/null @@ -1,324 +0,0 @@ -import { Injectable, Logger } from '@nestjs/common'; -import { ConfigService } from '@nestjs/config'; -import { ImageAnnotatorClient } from '@google-cloud/vision'; -import { VisionAnalysisResult, VisionProvider } from './types/vision.types'; - -@Injectable() -export class GoogleVisionService implements VisionProvider { - private readonly logger = new Logger(GoogleVisionService.name); - private readonly client: ImageAnnotatorClient; - private readonly confidenceThreshold: number; - - // Rate limiting - private requestCount = 0; - private lastResetTime = Date.now(); - private readonly requestsPerMinute: number; - - constructor(private configService: ConfigService) { - const apiKey = this.configService.get('GOOGLE_CLOUD_VISION_KEY'); - if (!apiKey) { - throw new Error('Google Cloud Vision API key is required'); - } - - // Initialize the client with API key - this.client = new ImageAnnotatorClient({ - keyFilename: apiKey, // If it's a file path - // Or use the key directly if it's a JSON string - ...(apiKey.startsWith('{') ? { credentials: JSON.parse(apiKey) } : {}), - }); - - this.confidenceThreshold = this.configService.get('VISION_CONFIDENCE_THRESHOLD', 0.40); - this.requestsPerMinute = this.configService.get('GOOGLE_REQUESTS_PER_MINUTE', 100); - - this.logger.log('Google Cloud Vision Service initialized'); - } - - async analyzeImage( - imageUrl: string, - keywords?: string[], - customPrompt?: string - ): Promise { - await this.checkRateLimit(); - - const startTime = Date.now(); - - try { - this.logger.debug(`Analyzing image with Google Cloud Vision: ${imageUrl}`); - - // Perform multiple types of detection - const [labelResult] = await this.client.labelDetection({ - image: { source: { imageUri: imageUrl } }, - maxResults: 20, - }); - - const [objectResult] = await this.client.objectLocalization({ - image: { source: { imageUri: imageUrl } }, - maxResults: 10, - }); - - const [propertiesResult] = await this.client.imageProperties({ - image: { source: { imageUri: imageUrl } } - }); - - const [textResult] = await this.client.textDetection({ - image: { source: { imageUri: imageUrl } } - }); - - // Update rate limiting counter - this.requestCount += 4; // We made 4 API calls - - const processingTime = Date.now() - startTime; - - // Process the results - const result = this.processGoogleVisionResults( - labelResult, - objectResult, - propertiesResult, - textResult, - processingTime, - keywords - ); - - this.logger.debug(`Google Vision analysis completed in ${processingTime}ms`); - return result; - - } catch (error) { - const processingTime = Date.now() - startTime; - this.logger.error(`Google Vision analysis failed: ${error.message}`, error.stack); - - // Return error result with fallback data - return { - provider: 'google', - success: false, - error: error.message, - objects: [], - colors: [], - scene: '', - description: '', - confidence: 0, - processingTime, - keywords: keywords || [], - tags: [], - labels: [], - }; - } - } - - private processGoogleVisionResults( - labelResult: any, - objectResult: any, - propertiesResult: any, - textResult: any, - processingTime: number, - keywords?: string[] - ): VisionAnalysisResult { - - // Process labels with confidence filtering - const labels = (labelResult.labelAnnotations || []) - .filter((label: any) => label.score >= this.confidenceThreshold) - .map((label: any) => ({ - name: label.description.toLowerCase(), - confidence: label.score, - })) - .sort((a: any, b: any) => b.confidence - a.confidence); - - // Process detected objects - const objects = (objectResult.localizedObjectAnnotations || []) - .filter((obj: any) => obj.score >= this.confidenceThreshold) - .map((obj: any) => obj.name.toLowerCase()) - .slice(0, 10); - - // Process dominant colors - const colors = this.extractDominantColors(propertiesResult); - - // Process detected text - const detectedText = textResult.textAnnotations && textResult.textAnnotations.length > 0 - ? textResult.textAnnotations[0].description - : ''; - - // Combine all tags - const allTags = [ - ...labels.map((l: any) => l.name), - ...objects, - ...(keywords || []), - ]; - - // Remove duplicates and filter - const uniqueTags = [...new Set(allTags)] - .filter(tag => tag.length > 2) - .filter(tag => !['image', 'photo', 'picture', 'file'].includes(tag)) - .slice(0, 15); - - // Generate scene description - const topLabels = labels.slice(0, 3).map((l: any) => l.name); - const scene = this.generateSceneDescription(topLabels, objects.slice(0, 3)); - - // Generate overall description - const description = this.generateDescription(labels, objects, colors, detectedText); - - // Calculate overall confidence (average of top 5 labels) - const topConfidences = labels.slice(0, 5).map((l: any) => l.confidence); - const averageConfidence = topConfidences.length > 0 - ? topConfidences.reduce((sum, conf) => sum + conf, 0) / topConfidences.length - : 0; - - return { - provider: 'google', - success: true, - objects: objects.slice(0, 8), - colors: colors.slice(0, 3), - scene, - description, - confidence: averageConfidence, - processingTime, - keywords: keywords || [], - tags: uniqueTags, - labels, - detectedText: detectedText ? detectedText.substring(0, 200) : undefined, - rawResponse: { - labels: labelResult.labelAnnotations, - objects: objectResult.localizedObjectAnnotations, - properties: propertiesResult.imagePropertiesAnnotation, - text: textResult.textAnnotations, - }, - }; - } - - private extractDominantColors(propertiesResult: any): string[] { - if (!propertiesResult.imagePropertiesAnnotation?.dominantColors?.colors) { - return []; - } - - return propertiesResult.imagePropertiesAnnotation.dominantColors.colors - .slice(0, 5) // Take top 5 colors - .map((colorInfo: any) => { - const { red = 0, green = 0, blue = 0 } = colorInfo.color; - return this.rgbToColorName(red, green, blue); - }) - .filter((color: string) => color !== 'unknown') - .slice(0, 3); // Keep top 3 recognizable colors - } - - private rgbToColorName(r: number, g: number, b: number): string { - // Simple color name mapping based on RGB values - const colors = [ - { name: 'red', r: 255, g: 0, b: 0 }, - { name: 'green', r: 0, g: 255, b: 0 }, - { name: 'blue', r: 0, g: 0, b: 255 }, - { name: 'yellow', r: 255, g: 255, b: 0 }, - { name: 'orange', r: 255, g: 165, b: 0 }, - { name: 'purple', r: 128, g: 0, b: 128 }, - { name: 'pink', r: 255, g: 192, b: 203 }, - { name: 'brown', r: 165, g: 42, b: 42 }, - { name: 'gray', r: 128, g: 128, b: 128 }, - { name: 'black', r: 0, g: 0, b: 0 }, - { name: 'white', r: 255, g: 255, b: 255 }, - ]; - - let closestColor = 'unknown'; - let minDistance = Infinity; - - for (const color of colors) { - const distance = Math.sqrt( - Math.pow(r - color.r, 2) + - Math.pow(g - color.g, 2) + - Math.pow(b - color.b, 2) - ); - - if (distance < minDistance) { - minDistance = distance; - closestColor = color.name; - } - } - - return closestColor; - } - - private generateSceneDescription(labels: string[], objects: string[]): string { - const combined = [...new Set([...labels, ...objects])].slice(0, 4); - - if (combined.length === 0) return ''; - if (combined.length === 1) return combined[0]; - if (combined.length === 2) return combined.join(' and '); - - const last = combined.pop(); - return combined.join(', ') + ', and ' + last; - } - - private generateDescription(labels: any[], objects: string[], colors: string[], text: string): string { - const parts = []; - - if (objects.length > 0) { - parts.push(`Image containing ${objects.slice(0, 3).join(', ')}`); - } else if (labels.length > 0) { - parts.push(`Image featuring ${labels.slice(0, 3).map(l => l.name).join(', ')}`); - } - - if (colors.length > 0) { - parts.push(`with ${colors.join(' and ')} colors`); - } - - if (text && text.trim()) { - parts.push(`including text elements`); - } - - return parts.join(' ') || 'Image analysis'; - } - - private async checkRateLimit(): Promise { - const now = Date.now(); - const timeSinceReset = now - this.lastResetTime; - - // Reset counters every minute - if (timeSinceReset >= 60000) { - this.requestCount = 0; - this.lastResetTime = now; - return; - } - - // Check if we're hitting rate limits - if (this.requestCount >= this.requestsPerMinute) { - const waitTime = 60000 - timeSinceReset; - this.logger.warn(`Google Vision request rate limit reached, waiting ${waitTime}ms`); - await this.sleep(waitTime); - this.requestCount = 0; - this.lastResetTime = Date.now(); - } - } - - private sleep(ms: number): Promise { - return new Promise(resolve => setTimeout(resolve, ms)); - } - - async isHealthy(): Promise { - try { - // Simple health check - try to detect labels on a small test image - // Using Google's test image URL - const testImageUrl = 'https://cloud.google.com/vision/docs/images/bicycle_example.png'; - - const [result] = await this.client.labelDetection({ - image: { source: { imageUri: testImageUrl } }, - maxResults: 1, - }); - - return !!(result.labelAnnotations && result.labelAnnotations.length > 0); - } catch (error) { - this.logger.error('Google Vision health check failed:', error.message); - return false; - } - } - - getProviderName(): string { - return 'google'; - } - - getConfiguration() { - return { - provider: 'google', - confidenceThreshold: this.confidenceThreshold, - rateLimits: { - requestsPerMinute: this.requestsPerMinute, - }, - }; - } -} \ No newline at end of file diff --git a/packages/worker/src/vision/openai-vision.service.ts b/packages/worker/src/vision/openai-vision.service.ts deleted file mode 100644 index 8a4dda2..0000000 --- a/packages/worker/src/vision/openai-vision.service.ts +++ /dev/null @@ -1,267 +0,0 @@ -import { Injectable, Logger } from '@nestjs/common'; -import { ConfigService } from '@nestjs/config'; -import OpenAI from 'openai'; -import { VisionAnalysisResult, VisionProvider } from './types/vision.types'; - -@Injectable() -export class OpenAIVisionService implements VisionProvider { - private readonly logger = new Logger(OpenAIVisionService.name); - private readonly openai: OpenAI; - private readonly model: string; - private readonly maxTokens: number; - private readonly temperature: number; - - // Rate limiting - private requestCount = 0; - private tokenCount = 0; - private lastResetTime = Date.now(); - private readonly requestsPerMinute: number; - private readonly tokensPerMinute: number; - - constructor(private configService: ConfigService) { - const apiKey = this.configService.get('OPENAI_API_KEY'); - if (!apiKey) { - throw new Error('OpenAI API key is required'); - } - - this.openai = new OpenAI({ - apiKey, - timeout: 30000, // 30 seconds timeout - }); - - this.model = this.configService.get('OPENAI_MODEL', 'gpt-4-vision-preview'); - this.maxTokens = this.configService.get('OPENAI_MAX_TOKENS', 500); - this.temperature = this.configService.get('OPENAI_TEMPERATURE', 0.1); - this.requestsPerMinute = this.configService.get('OPENAI_REQUESTS_PER_MINUTE', 50); - this.tokensPerMinute = this.configService.get('OPENAI_TOKENS_PER_MINUTE', 10000); - - this.logger.log(`OpenAI Vision Service initialized with model: ${this.model}`); - } - - async analyzeImage( - imageUrl: string, - keywords?: string[], - customPrompt?: string - ): Promise { - await this.checkRateLimit(); - - const startTime = Date.now(); - - try { - this.logger.debug(`Analyzing image with OpenAI: ${imageUrl}`); - - const prompt = customPrompt || this.buildAnalysisPrompt(keywords); - - const response = await this.openai.chat.completions.create({ - model: this.model, - max_tokens: this.maxTokens, - temperature: this.temperature, - messages: [ - { - role: 'user', - content: [ - { - type: 'text', - text: prompt, - }, - { - type: 'image_url', - image_url: { - url: imageUrl, - detail: 'high', // Use high detail for better analysis - }, - }, - ], - }, - ], - }); - - // Update rate limiting counters - this.requestCount++; - this.tokenCount += response.usage?.total_tokens || 0; - - const processingTime = Date.now() - startTime; - const content = response.choices[0]?.message?.content; - - if (!content) { - throw new Error('No content received from OpenAI API'); - } - - // Parse the structured response - const result = this.parseOpenAIResponse(content, processingTime); - - this.logger.debug(`OpenAI analysis completed in ${processingTime}ms`); - return result; - - } catch (error) { - const processingTime = Date.now() - startTime; - this.logger.error(`OpenAI vision analysis failed: ${error.message}`, error.stack); - - // Return error result with fallback data - return { - provider: 'openai', - success: false, - error: error.message, - objects: [], - colors: [], - scene: '', - description: '', - confidence: 0, - processingTime, - keywords: keywords || [], - tags: [], - labels: [], - }; - } - } - - private buildAnalysisPrompt(keywords?: string[]): string { - const keywordContext = keywords && keywords.length > 0 - ? `\n\nUser context keywords: ${keywords.join(', ')}` - : ''; - - return `Analyze this image and provide a detailed description suitable for SEO filename generation. -Please provide your response as a JSON object with the following structure: - -{ - "objects": ["object1", "object2", "object3"], - "colors": ["color1", "color2"], - "scene": "brief scene description", - "description": "detailed description of the image", - "confidence": 0.95, - "tags": ["tag1", "tag2", "tag3"], - "labels": [ - {"name": "label1", "confidence": 0.9}, - {"name": "label2", "confidence": 0.8} - ] -} - -Focus on: -1. Main objects and subjects in the image -2. Dominant colors (max 3) -3. Scene type (indoor/outdoor, setting) -4. Style, mood, or theme -5. Any text or branding visible -6. Technical aspects if relevant (photography style, lighting) - -Provide specific, descriptive terms that would be valuable for SEO and image search optimization.${keywordContext}`; - } - - private parseOpenAIResponse(content: string, processingTime: number): VisionAnalysisResult { - try { - // Try to extract JSON from the response - const jsonMatch = content.match(/\{[\s\S]*\}/); - const jsonContent = jsonMatch ? jsonMatch[0] : content; - - const parsed = JSON.parse(jsonContent); - - return { - provider: 'openai', - success: true, - objects: Array.isArray(parsed.objects) ? parsed.objects : [], - colors: Array.isArray(parsed.colors) ? parsed.colors : [], - scene: parsed.scene || '', - description: parsed.description || '', - confidence: typeof parsed.confidence === 'number' ? parsed.confidence : 0.85, - processingTime, - keywords: [], - tags: Array.isArray(parsed.tags) ? parsed.tags : [], - labels: Array.isArray(parsed.labels) ? parsed.labels : [], - rawResponse: content, - }; - } catch (parseError) { - this.logger.warn('Failed to parse OpenAI JSON response, using fallback parsing'); - - // Fallback parsing - extract keywords from plain text - const words = content.toLowerCase() - .split(/[^a-z0-9]+/) - .filter(word => word.length > 2) - .filter(word => !['the', 'and', 'with', 'for', 'are', 'was', 'this', 'that'].includes(word)) - .slice(0, 10); - - return { - provider: 'openai', - success: true, - objects: words.slice(0, 5), - colors: [], - scene: content.substring(0, 100), - description: content, - confidence: 0.7, // Lower confidence for fallback parsing - processingTime, - keywords: [], - tags: words, - labels: words.map(word => ({ name: word, confidence: 0.7 })), - rawResponse: content, - }; - } - } - - private async checkRateLimit(): Promise { - const now = Date.now(); - const timeSinceReset = now - this.lastResetTime; - - // Reset counters every minute - if (timeSinceReset >= 60000) { - this.requestCount = 0; - this.tokenCount = 0; - this.lastResetTime = now; - return; - } - - // Check if we're hitting rate limits - if (this.requestCount >= this.requestsPerMinute) { - const waitTime = 60000 - timeSinceReset; - this.logger.warn(`OpenAI request rate limit reached, waiting ${waitTime}ms`); - await this.sleep(waitTime); - this.requestCount = 0; - this.tokenCount = 0; - this.lastResetTime = Date.now(); - } - - if (this.tokenCount >= this.tokensPerMinute) { - const waitTime = 60000 - timeSinceReset; - this.logger.warn(`OpenAI token rate limit reached, waiting ${waitTime}ms`); - await this.sleep(waitTime); - this.requestCount = 0; - this.tokenCount = 0; - this.lastResetTime = Date.now(); - } - } - - private sleep(ms: number): Promise { - return new Promise(resolve => setTimeout(resolve, ms)); - } - - async isHealthy(): Promise { - try { - // Simple health check - try to create a completion with minimal tokens - const response = await this.openai.chat.completions.create({ - model: 'gpt-3.5-turbo', // Use cheaper model for health check - max_tokens: 5, - messages: [{ role: 'user', content: 'Hello' }], - }); - - return !!response.choices[0]?.message?.content; - } catch (error) { - this.logger.error('OpenAI health check failed:', error.message); - return false; - } - } - - getProviderName(): string { - return 'openai'; - } - - getConfiguration() { - return { - provider: 'openai', - model: this.model, - maxTokens: this.maxTokens, - temperature: this.temperature, - rateLimits: { - requestsPerMinute: this.requestsPerMinute, - tokensPerMinute: this.tokensPerMinute, - }, - }; - } -} \ No newline at end of file diff --git a/packages/worker/src/vision/types/vision.types.ts b/packages/worker/src/vision/types/vision.types.ts deleted file mode 100644 index 2c55c73..0000000 --- a/packages/worker/src/vision/types/vision.types.ts +++ /dev/null @@ -1,62 +0,0 @@ -export interface VisionLabel { - name: string; - confidence: number; -} - -export interface VisionAnalysisResult { - provider: string; - success: boolean; - error?: string; - - // Core analysis results - objects: string[]; - colors: string[]; - scene: string; - description: string; - confidence: number; - processingTime: number; - - // Additional data - keywords: string[]; - tags: string[]; - labels: VisionLabel[]; - - // Optional fields - detectedText?: string; - emotions?: string[]; - faces?: number; - - // Raw provider response (for debugging) - rawResponse?: any; -} - -export interface VisionProvider { - analyzeImage( - imageUrl: string, - keywords?: string[], - customPrompt?: string - ): Promise; - - isHealthy(): Promise; - getProviderName(): string; - getConfiguration(): any; -} - -export interface CombinedVisionResult { - primary: VisionAnalysisResult; - secondary?: VisionAnalysisResult; - - // Merged results - finalObjects: string[]; - finalColors: string[]; - finalScene: string; - finalDescription: string; - finalTags: string[]; - finalConfidence: number; - - // Metadata - providersUsed: string[]; - totalProcessingTime: number; - success: boolean; - error?: string; -} \ No newline at end of file diff --git a/packages/worker/src/vision/vision.module.ts b/packages/worker/src/vision/vision.module.ts deleted file mode 100644 index 9b25b54..0000000 --- a/packages/worker/src/vision/vision.module.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { Module } from '@nestjs/common'; -import { ConfigModule } from '@nestjs/config'; -import { VisionService } from './vision.service'; -import { OpenAIVisionService } from './openai-vision.service'; -import { GoogleVisionService } from './google-vision.service'; - -@Module({ - imports: [ConfigModule], - providers: [ - VisionService, - OpenAIVisionService, - GoogleVisionService, - ], - exports: [ - VisionService, - OpenAIVisionService, - GoogleVisionService, - ], -}) -export class VisionModule {} \ No newline at end of file diff --git a/packages/worker/src/vision/vision.service.ts b/packages/worker/src/vision/vision.service.ts deleted file mode 100644 index 8b2d10d..0000000 --- a/packages/worker/src/vision/vision.service.ts +++ /dev/null @@ -1,370 +0,0 @@ -import { Injectable, Logger } from '@nestjs/common'; -import { ConfigService } from '@nestjs/config'; -import { OpenAIVisionService } from './openai-vision.service'; -import { GoogleVisionService } from './google-vision.service'; -import { VisionAnalysisResult, CombinedVisionResult, VisionProvider } from './types/vision.types'; - -@Injectable() -export class VisionService { - private readonly logger = new Logger(VisionService.name); - private readonly providers: VisionProvider[] = []; - private readonly confidenceThreshold: number; - - constructor( - private configService: ConfigService, - private openaiVisionService: OpenAIVisionService, - private googleVisionService: GoogleVisionService, - ) { - this.confidenceThreshold = this.configService.get('VISION_CONFIDENCE_THRESHOLD', 0.40); - - // Initialize available providers - this.initializeProviders(); - } - - private initializeProviders() { - const openaiKey = this.configService.get('OPENAI_API_KEY'); - const googleKey = this.configService.get('GOOGLE_CLOUD_VISION_KEY'); - - if (openaiKey) { - this.providers.push(this.openaiVisionService); - this.logger.log('OpenAI Vision provider initialized'); - } - - if (googleKey) { - this.providers.push(this.googleVisionService); - this.logger.log('Google Vision provider initialized'); - } - - if (this.providers.length === 0) { - throw new Error('No vision providers available. Please configure at least one AI vision service.'); - } - - this.logger.log(`Vision service initialized with ${this.providers.length} provider(s)`); - } - - /** - * Analyze image using all available providers with fallback strategy - */ - async analyzeImage( - imageUrl: string, - keywords?: string[], - customPrompt?: string, - preferredProvider?: string - ): Promise { - const startTime = Date.now(); - - this.logger.debug(`Starting vision analysis for image: ${imageUrl}`); - - // Determine provider order based on preference and availability - const orderedProviders = this.getOrderedProviders(preferredProvider); - - let primaryResult: VisionAnalysisResult | null = null; - let secondaryResult: VisionAnalysisResult | null = null; - const providersUsed: string[] = []; - - // Try primary provider - for (const provider of orderedProviders) { - try { - this.logger.debug(`Attempting analysis with ${provider.getProviderName()}`); - - const result = await provider.analyzeImage(imageUrl, keywords, customPrompt); - - if (result.success && result.confidence >= this.confidenceThreshold) { - primaryResult = result; - providersUsed.push(result.provider); - this.logger.debug(`Primary analysis successful with ${result.provider} (confidence: ${result.confidence})`); - break; - } else if (result.success) { - this.logger.warn(`Provider ${result.provider} returned low confidence: ${result.confidence}`); - } - - } catch (error) { - this.logger.warn(`Provider ${provider.getProviderName()} failed: ${error.message}`); - } - } - - // If primary result has low confidence, try secondary provider for validation - if (primaryResult && primaryResult.confidence < 0.8 && orderedProviders.length > 1) { - const secondaryProvider = orderedProviders.find(p => p.getProviderName() !== primaryResult!.provider); - - if (secondaryProvider) { - try { - this.logger.debug(`Getting secondary validation from ${secondaryProvider.getProviderName()}`); - - secondaryResult = await secondaryProvider.analyzeImage(imageUrl, keywords, customPrompt); - - if (secondaryResult.success) { - providersUsed.push(secondaryResult.provider); - this.logger.debug(`Secondary analysis completed with ${secondaryResult.provider}`); - } - - } catch (error) { - this.logger.warn(`Secondary provider ${secondaryProvider.getProviderName()} failed: ${error.message}`); - } - } - } - - const totalProcessingTime = Date.now() - startTime; - - // If no successful analysis, return error result - if (!primaryResult) { - this.logger.error('All vision providers failed'); - return { - primary: { - provider: 'none', - success: false, - error: 'All vision providers failed', - objects: [], - colors: [], - scene: '', - description: '', - confidence: 0, - processingTime: totalProcessingTime, - keywords: keywords || [], - tags: [], - labels: [], - }, - finalObjects: [], - finalColors: [], - finalScene: '', - finalDescription: '', - finalTags: [], - finalConfidence: 0, - providersUsed, - totalProcessingTime, - success: false, - error: 'All vision providers failed', - }; - } - - // Combine results from both providers - const combinedResult = this.combineResults(primaryResult, secondaryResult, keywords); - combinedResult.providersUsed = providersUsed; - combinedResult.totalProcessingTime = totalProcessingTime; - - this.logger.log(`Vision analysis completed in ${totalProcessingTime}ms using ${providersUsed.join(', ')}`); - - return combinedResult; - } - - /** - * Combine results from multiple providers using weighted scoring - */ - private combineResults( - primary: VisionAnalysisResult, - secondary?: VisionAnalysisResult, - keywords?: string[] - ): CombinedVisionResult { - - if (!secondary) { - // Single provider result - return { - primary, - finalObjects: primary.objects, - finalColors: primary.colors, - finalScene: primary.scene, - finalDescription: primary.description, - finalTags: this.mergeWithKeywords(primary.tags, keywords), - finalConfidence: primary.confidence, - providersUsed: [primary.provider], - totalProcessingTime: primary.processingTime, - success: primary.success, - }; - } - - // Combine results from both providers - const weightedObjects = this.combineWeightedArrays( - primary.objects, - secondary.objects, - primary.confidence, - secondary.confidence - ); - - const weightedColors = this.combineWeightedArrays( - primary.colors, - secondary.colors, - primary.confidence, - secondary.confidence - ); - - const weightedTags = this.combineWeightedArrays( - primary.tags, - secondary.tags, - primary.confidence, - secondary.confidence - ); - - // Choose the better scene description - const finalScene = primary.confidence >= secondary.confidence - ? primary.scene - : secondary.scene; - - // Combine descriptions - const finalDescription = this.combineDescriptions(primary, secondary); - - // Calculate combined confidence - const finalConfidence = (primary.confidence + secondary.confidence) / 2; - - return { - primary, - secondary, - finalObjects: weightedObjects.slice(0, 8), - finalColors: weightedColors.slice(0, 3), - finalScene, - finalDescription, - finalTags: this.mergeWithKeywords(weightedTags, keywords).slice(0, 12), - finalConfidence, - providersUsed: [primary.provider, secondary.provider], - totalProcessingTime: primary.processingTime + secondary.processingTime, - success: true, - }; - } - - private combineWeightedArrays( - arr1: string[], - arr2: string[], - weight1: number, - weight2: number - ): string[] { - const scoreMap = new Map(); - - // Score items from first array - arr1.forEach((item, index) => { - const positionScore = (arr1.length - index) / arr1.length; // Higher position = higher score - const weightedScore = positionScore * weight1; - scoreMap.set(item.toLowerCase(), (scoreMap.get(item.toLowerCase()) || 0) + weightedScore); - }); - - // Score items from second array - arr2.forEach((item, index) => { - const positionScore = (arr2.length - index) / arr2.length; - const weightedScore = positionScore * weight2; - scoreMap.set(item.toLowerCase(), (scoreMap.get(item.toLowerCase()) || 0) + weightedScore); - }); - - // Sort by combined score and return - return Array.from(scoreMap.entries()) - .sort(([, scoreA], [, scoreB]) => scoreB - scoreA) - .map(([item]) => item); - } - - private combineDescriptions(primary: VisionAnalysisResult, secondary: VisionAnalysisResult): string { - if (primary.confidence >= secondary.confidence) { - return primary.description; - } else { - return secondary.description; - } - } - - private mergeWithKeywords(tags: string[], keywords?: string[]): string[] { - if (!keywords || keywords.length === 0) { - return tags; - } - - // Combine and prioritize user keywords (70% vision tags, 30% user keywords) - const visionTags = tags.slice(0, Math.ceil(tags.length * 0.7)); - const userKeywords = keywords.slice(0, Math.ceil(keywords.length * 0.3)); - - const combined = [...userKeywords, ...visionTags]; - - // Remove duplicates while preserving order - return [...new Set(combined.map(tag => tag.toLowerCase()))]; - } - - private getOrderedProviders(preferredProvider?: string): VisionProvider[] { - if (!preferredProvider) { - return [...this.providers]; // Default order - } - - const preferred = this.providers.find(p => p.getProviderName() === preferredProvider); - const others = this.providers.filter(p => p.getProviderName() !== preferredProvider); - - return preferred ? [preferred, ...others] : [...this.providers]; - } - - /** - * Generate SEO-optimized filename from vision analysis - */ - async generateSeoFilename( - visionResult: CombinedVisionResult, - originalFilename: string, - maxLength: number = 80 - ): Promise { - try { - // Use the final combined tags - const tags = visionResult.finalTags.slice(0, 6); // Limit to 6 tags - - if (tags.length === 0) { - return this.sanitizeFilename(originalFilename); - } - - // Create SEO-friendly filename - let filename = tags - .join('-') - .toLowerCase() - .replace(/[^a-z0-9\s-]/g, '') // Remove special characters - .replace(/\s+/g, '-') // Replace spaces with hyphens - .replace(/-+/g, '-') // Replace multiple hyphens with single - .substring(0, maxLength); - - // Get file extension from original name - const extension = originalFilename.split('.').pop()?.toLowerCase() || 'jpg'; - - // Ensure filename is not empty - if (!filename || filename === '-') { - filename = 'image'; - } - - // Remove trailing hyphens - filename = filename.replace(/-+$/, ''); - - return `${filename}.${extension}`; - - } catch (error) { - this.logger.error('Failed to generate SEO filename', error.stack); - return this.sanitizeFilename(originalFilename); - } - } - - private sanitizeFilename(filename: string): string { - return filename - .toLowerCase() - .replace(/[^a-z0-9.-]/g, '-') - .replace(/-+/g, '-') - .replace(/^-|-$/g, ''); - } - - /** - * Health check for all providers - */ - async getHealthStatus(): Promise<{ - healthy: boolean; - providers: Array<{ name: string; healthy: boolean; config: any }>; - }> { - const providerStatus = await Promise.all( - this.providers.map(async (provider) => ({ - name: provider.getProviderName(), - healthy: await provider.isHealthy(), - config: provider.getConfiguration(), - })) - ); - - const healthy = providerStatus.some(p => p.healthy); - - return { - healthy, - providers: providerStatus, - }; - } - - /** - * Get service configuration and statistics - */ - getServiceInfo() { - return { - availableProviders: this.providers.map(p => p.getProviderName()), - confidenceThreshold: this.confidenceThreshold, - providerConfigs: this.providers.map(p => p.getConfiguration()), - }; - } -} \ No newline at end of file diff --git a/packages/worker/tsconfig.json b/packages/worker/tsconfig.json deleted file mode 100644 index 14ddffd..0000000 --- a/packages/worker/tsconfig.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - "compilerOptions": { - "module": "commonjs", - "declaration": true, - "removeComments": true, - "emitDecoratorMetadata": true, - "experimentalDecorators": true, - "allowSyntheticDefaultImports": true, - "target": "ES2021", - "sourceMap": true, - "outDir": "./dist", - "baseUrl": "./", - "incremental": true, - "skipLibCheck": true, - "strictNullChecks": false, - "noImplicitAny": false, - "strictBindCallApply": false, - "forceConsistentCasingInFileNames": false, - "noFallthroughCasesInSwitch": false, - "resolveJsonModule": true, - "esModuleInterop": true, - "paths": { - "@/*": ["src/*"], - "@/vision/*": ["src/vision/*"], - "@/processors/*": ["src/processors/*"], - "@/storage/*": ["src/storage/*"], - "@/queue/*": ["src/queue/*"], - "@/config/*": ["src/config/*"], - "@/utils/*": ["src/utils/*"] - } - }, - "include": ["src/**/*"], - "exclude": ["node_modules", "dist", "test", "**/*spec.ts"] -} \ No newline at end of file