feat(api): add storage module for MinIO/S3 integration
- Implement StorageService with MinIO client integration - Add file upload, download, and metadata operations - Support SHA-256 checksum calculation for deduplication - Include presigned URL generation for secure downloads - Add batch file management and cleanup operations - Validate image MIME types for security Resolves requirements §28-§30 for file storage architecture. 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
parent
0197a2f7ca
commit
d2c988303f
2 changed files with 273 additions and 0 deletions
10
packages/api/src/storage/storage.module.ts
Normal file
10
packages/api/src/storage/storage.module.ts
Normal file
|
@ -0,0 +1,10 @@
|
||||||
|
import { Module } from '@nestjs/common';
|
||||||
|
import { ConfigModule } from '@nestjs/config';
|
||||||
|
import { StorageService } from './storage.service';
|
||||||
|
|
||||||
|
@Module({
|
||||||
|
imports: [ConfigModule],
|
||||||
|
providers: [StorageService],
|
||||||
|
exports: [StorageService],
|
||||||
|
})
|
||||||
|
export class StorageModule {}
|
263
packages/api/src/storage/storage.service.ts
Normal file
263
packages/api/src/storage/storage.service.ts
Normal file
|
@ -0,0 +1,263 @@
|
||||||
|
import { Injectable, Logger } from '@nestjs/common';
|
||||||
|
import { ConfigService } from '@nestjs/config';
|
||||||
|
import * as Minio from 'minio';
|
||||||
|
import { v4 as uuidv4 } from 'uuid';
|
||||||
|
import * as crypto from 'crypto';
|
||||||
|
|
||||||
|
export interface StorageFile {
|
||||||
|
buffer: Buffer;
|
||||||
|
originalName: string;
|
||||||
|
mimeType: string;
|
||||||
|
size: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface UploadResult {
|
||||||
|
key: string;
|
||||||
|
etag: string;
|
||||||
|
size: number;
|
||||||
|
checksum: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Injectable()
|
||||||
|
export class StorageService {
|
||||||
|
private readonly logger = new Logger(StorageService.name);
|
||||||
|
private readonly minioClient: Minio.Client;
|
||||||
|
private readonly bucketName: string;
|
||||||
|
|
||||||
|
constructor(private configService: ConfigService) {
|
||||||
|
// Initialize MinIO client
|
||||||
|
this.minioClient = new Minio.Client({
|
||||||
|
endPoint: this.configService.get<string>('MINIO_ENDPOINT', 'localhost'),
|
||||||
|
port: this.configService.get<number>('MINIO_PORT', 9000),
|
||||||
|
useSSL: this.configService.get<boolean>('MINIO_USE_SSL', false),
|
||||||
|
accessKey: this.configService.get<string>('MINIO_ACCESS_KEY', 'minioadmin'),
|
||||||
|
secretKey: this.configService.get<string>('MINIO_SECRET_KEY', 'minioadmin'),
|
||||||
|
});
|
||||||
|
|
||||||
|
this.bucketName = this.configService.get<string>('MINIO_BUCKET_NAME', 'seo-image-renamer');
|
||||||
|
this.initializeBucket();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Initialize the bucket if it doesn't exist
|
||||||
|
*/
|
||||||
|
private async initializeBucket(): Promise<void> {
|
||||||
|
try {
|
||||||
|
const bucketExists = await this.minioClient.bucketExists(this.bucketName);
|
||||||
|
if (!bucketExists) {
|
||||||
|
await this.minioClient.makeBucket(this.bucketName, 'us-east-1');
|
||||||
|
this.logger.log(`Created bucket: ${this.bucketName}`);
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
this.logger.error(`Failed to initialize bucket: ${error.message}`, error.stack);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Upload a file to MinIO storage
|
||||||
|
* @param file File data to upload
|
||||||
|
* @param batchId Batch UUID for organizing files
|
||||||
|
* @returns Upload result with key and metadata
|
||||||
|
*/
|
||||||
|
async uploadFile(file: StorageFile, batchId: string): Promise<UploadResult> {
|
||||||
|
try {
|
||||||
|
// Generate file checksum
|
||||||
|
const checksum = crypto.createHash('sha256').update(file.buffer).digest('hex');
|
||||||
|
|
||||||
|
// Generate unique filename with batch organization
|
||||||
|
const fileExtension = this.getFileExtension(file.originalName);
|
||||||
|
const fileName = `${uuidv4()}${fileExtension}`;
|
||||||
|
const objectKey = `batches/${batchId}/${fileName}`;
|
||||||
|
|
||||||
|
// Upload metadata
|
||||||
|
const metaData = {
|
||||||
|
'Content-Type': file.mimeType,
|
||||||
|
'Original-Name': file.originalName,
|
||||||
|
'Upload-Date': new Date().toISOString(),
|
||||||
|
'Checksum-SHA256': checksum,
|
||||||
|
};
|
||||||
|
|
||||||
|
// Upload file to MinIO
|
||||||
|
const uploadInfo = await this.minioClient.putObject(
|
||||||
|
this.bucketName,
|
||||||
|
objectKey,
|
||||||
|
file.buffer,
|
||||||
|
file.size,
|
||||||
|
metaData
|
||||||
|
);
|
||||||
|
|
||||||
|
this.logger.log(`File uploaded successfully: ${objectKey}`);
|
||||||
|
|
||||||
|
return {
|
||||||
|
key: objectKey,
|
||||||
|
etag: uploadInfo.etag,
|
||||||
|
size: file.size,
|
||||||
|
checksum,
|
||||||
|
};
|
||||||
|
} catch (error) {
|
||||||
|
this.logger.error(`Failed to upload file: ${error.message}`, error.stack);
|
||||||
|
throw new Error(`File upload failed: ${error.message}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get a file from MinIO storage
|
||||||
|
* @param objectKey Object key to retrieve
|
||||||
|
* @returns File stream
|
||||||
|
*/
|
||||||
|
async getFile(objectKey: string): Promise<NodeJS.ReadableStream> {
|
||||||
|
try {
|
||||||
|
return await this.minioClient.getObject(this.bucketName, objectKey);
|
||||||
|
} catch (error) {
|
||||||
|
this.logger.error(`Failed to retrieve file: ${objectKey}`, error.stack);
|
||||||
|
throw new Error(`File retrieval failed: ${error.message}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get file metadata
|
||||||
|
* @param objectKey Object key to get metadata for
|
||||||
|
* @returns File metadata
|
||||||
|
*/
|
||||||
|
async getFileMetadata(objectKey: string): Promise<any> {
|
||||||
|
try {
|
||||||
|
return await this.minioClient.statObject(this.bucketName, objectKey);
|
||||||
|
} catch (error) {
|
||||||
|
this.logger.error(`Failed to get file metadata: ${objectKey}`, error.stack);
|
||||||
|
throw new Error(`File metadata retrieval failed: ${error.message}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Delete a file from MinIO storage
|
||||||
|
* @param objectKey Object key to delete
|
||||||
|
*/
|
||||||
|
async deleteFile(objectKey: string): Promise<void> {
|
||||||
|
try {
|
||||||
|
await this.minioClient.removeObject(this.bucketName, objectKey);
|
||||||
|
this.logger.log(`File deleted successfully: ${objectKey}`);
|
||||||
|
} catch (error) {
|
||||||
|
this.logger.error(`Failed to delete file: ${objectKey}`, error.stack);
|
||||||
|
throw new Error(`File deletion failed: ${error.message}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* List files in a batch folder
|
||||||
|
* @param batchId Batch UUID
|
||||||
|
* @returns Array of object keys
|
||||||
|
*/
|
||||||
|
async listBatchFiles(batchId: string): Promise<string[]> {
|
||||||
|
try {
|
||||||
|
const objects: string[] = [];
|
||||||
|
const objectStream = this.minioClient.listObjects(
|
||||||
|
this.bucketName,
|
||||||
|
`batches/${batchId}/`,
|
||||||
|
true
|
||||||
|
);
|
||||||
|
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
objectStream.on('data', (obj) => {
|
||||||
|
objects.push(obj.name);
|
||||||
|
});
|
||||||
|
|
||||||
|
objectStream.on('error', (err) => {
|
||||||
|
this.logger.error(`Failed to list batch files: ${batchId}`, err);
|
||||||
|
reject(new Error(`Failed to list batch files: ${err.message}`));
|
||||||
|
});
|
||||||
|
|
||||||
|
objectStream.on('end', () => {
|
||||||
|
resolve(objects);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
} catch (error) {
|
||||||
|
this.logger.error(`Failed to list batch files: ${batchId}`, error.stack);
|
||||||
|
throw new Error(`Batch file listing failed: ${error.message}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Delete all files in a batch folder
|
||||||
|
* @param batchId Batch UUID
|
||||||
|
*/
|
||||||
|
async deleteBatchFiles(batchId: string): Promise<void> {
|
||||||
|
try {
|
||||||
|
const objectKeys = await this.listBatchFiles(batchId);
|
||||||
|
|
||||||
|
if (objectKeys.length > 0) {
|
||||||
|
await this.minioClient.removeObjects(this.bucketName, objectKeys);
|
||||||
|
this.logger.log(`Deleted ${objectKeys.length} files for batch: ${batchId}`);
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
this.logger.error(`Failed to delete batch files: ${batchId}`, error.stack);
|
||||||
|
throw new Error(`Batch file deletion failed: ${error.message}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Generate a presigned URL for file download
|
||||||
|
* @param objectKey Object key
|
||||||
|
* @param expiry Expiry time in seconds (default: 1 hour)
|
||||||
|
* @returns Presigned URL
|
||||||
|
*/
|
||||||
|
async getPresignedUrl(objectKey: string, expiry: number = 3600): Promise<string> {
|
||||||
|
try {
|
||||||
|
return await this.minioClient.presignedGetObject(this.bucketName, objectKey, expiry);
|
||||||
|
} catch (error) {
|
||||||
|
this.logger.error(`Failed to generate presigned URL: ${objectKey}`, error.stack);
|
||||||
|
throw new Error(`Presigned URL generation failed: ${error.message}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Check if file exists in storage
|
||||||
|
* @param objectKey Object key to check
|
||||||
|
* @returns Whether file exists
|
||||||
|
*/
|
||||||
|
async fileExists(objectKey: string): Promise<boolean> {
|
||||||
|
try {
|
||||||
|
await this.minioClient.statObject(this.bucketName, objectKey);
|
||||||
|
return true;
|
||||||
|
} catch (error) {
|
||||||
|
if (error.code === 'NotFound') {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Calculate SHA-256 checksum for duplicate detection
|
||||||
|
* @param buffer File buffer
|
||||||
|
* @returns SHA-256 checksum
|
||||||
|
*/
|
||||||
|
calculateChecksum(buffer: Buffer): string {
|
||||||
|
return crypto.createHash('sha256').update(buffer).digest('hex');
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get file extension from filename
|
||||||
|
* @param filename Original filename
|
||||||
|
* @returns File extension with dot
|
||||||
|
*/
|
||||||
|
private getFileExtension(filename: string): string {
|
||||||
|
const lastDotIndex = filename.lastIndexOf('.');
|
||||||
|
return lastDotIndex !== -1 ? filename.substring(lastDotIndex) : '';
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Validate file MIME type for image uploads
|
||||||
|
* @param mimeType MIME type to validate
|
||||||
|
* @returns Whether MIME type is valid
|
||||||
|
*/
|
||||||
|
isValidImageMimeType(mimeType: string): boolean {
|
||||||
|
const validMimeTypes = [
|
||||||
|
'image/jpeg',
|
||||||
|
'image/jpg',
|
||||||
|
'image/png',
|
||||||
|
'image/gif',
|
||||||
|
'image/webp',
|
||||||
|
];
|
||||||
|
return validMimeTypes.includes(mimeType.toLowerCase());
|
||||||
|
}
|
||||||
|
}
|
Loading…
Add table
Add a link
Reference in a new issue