feat(api): add storage module for MinIO/S3 integration
- Implement StorageService with MinIO client integration - Add file upload, download, and metadata operations - Support SHA-256 checksum calculation for deduplication - Include presigned URL generation for secure downloads - Add batch file management and cleanup operations - Validate image MIME types for security Resolves requirements §28-§30 for file storage architecture. 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
DustyWalker
parent
0197a2f7ca
commit
d2c988303f
2 changed files with 273 additions and 0 deletions
10
packages/api/src/storage/storage.module.ts
Normal file
10
packages/api/src/storage/storage.module.ts
Normal file
|
|
@ -0,0 +1,10 @@
|
|||
import { Module } from '@nestjs/common';
|
||||
import { ConfigModule } from '@nestjs/config';
|
||||
import { StorageService } from './storage.service';
|
||||
|
||||
@Module({
|
||||
imports: [ConfigModule],
|
||||
providers: [StorageService],
|
||||
exports: [StorageService],
|
||||
})
|
||||
export class StorageModule {}
|
||||
263
packages/api/src/storage/storage.service.ts
Normal file
263
packages/api/src/storage/storage.service.ts
Normal file
|
|
@ -0,0 +1,263 @@
|
|||
import { Injectable, Logger } from '@nestjs/common';
|
||||
import { ConfigService } from '@nestjs/config';
|
||||
import * as Minio from 'minio';
|
||||
import { v4 as uuidv4 } from 'uuid';
|
||||
import * as crypto from 'crypto';
|
||||
|
||||
export interface StorageFile {
|
||||
buffer: Buffer;
|
||||
originalName: string;
|
||||
mimeType: string;
|
||||
size: number;
|
||||
}
|
||||
|
||||
export interface UploadResult {
|
||||
key: string;
|
||||
etag: string;
|
||||
size: number;
|
||||
checksum: string;
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class StorageService {
|
||||
private readonly logger = new Logger(StorageService.name);
|
||||
private readonly minioClient: Minio.Client;
|
||||
private readonly bucketName: string;
|
||||
|
||||
constructor(private configService: ConfigService) {
|
||||
// Initialize MinIO client
|
||||
this.minioClient = new Minio.Client({
|
||||
endPoint: this.configService.get<string>('MINIO_ENDPOINT', 'localhost'),
|
||||
port: this.configService.get<number>('MINIO_PORT', 9000),
|
||||
useSSL: this.configService.get<boolean>('MINIO_USE_SSL', false),
|
||||
accessKey: this.configService.get<string>('MINIO_ACCESS_KEY', 'minioadmin'),
|
||||
secretKey: this.configService.get<string>('MINIO_SECRET_KEY', 'minioadmin'),
|
||||
});
|
||||
|
||||
this.bucketName = this.configService.get<string>('MINIO_BUCKET_NAME', 'seo-image-renamer');
|
||||
this.initializeBucket();
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize the bucket if it doesn't exist
|
||||
*/
|
||||
private async initializeBucket(): Promise<void> {
|
||||
try {
|
||||
const bucketExists = await this.minioClient.bucketExists(this.bucketName);
|
||||
if (!bucketExists) {
|
||||
await this.minioClient.makeBucket(this.bucketName, 'us-east-1');
|
||||
this.logger.log(`Created bucket: ${this.bucketName}`);
|
||||
}
|
||||
} catch (error) {
|
||||
this.logger.error(`Failed to initialize bucket: ${error.message}`, error.stack);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Upload a file to MinIO storage
|
||||
* @param file File data to upload
|
||||
* @param batchId Batch UUID for organizing files
|
||||
* @returns Upload result with key and metadata
|
||||
*/
|
||||
async uploadFile(file: StorageFile, batchId: string): Promise<UploadResult> {
|
||||
try {
|
||||
// Generate file checksum
|
||||
const checksum = crypto.createHash('sha256').update(file.buffer).digest('hex');
|
||||
|
||||
// Generate unique filename with batch organization
|
||||
const fileExtension = this.getFileExtension(file.originalName);
|
||||
const fileName = `${uuidv4()}${fileExtension}`;
|
||||
const objectKey = `batches/${batchId}/${fileName}`;
|
||||
|
||||
// Upload metadata
|
||||
const metaData = {
|
||||
'Content-Type': file.mimeType,
|
||||
'Original-Name': file.originalName,
|
||||
'Upload-Date': new Date().toISOString(),
|
||||
'Checksum-SHA256': checksum,
|
||||
};
|
||||
|
||||
// Upload file to MinIO
|
||||
const uploadInfo = await this.minioClient.putObject(
|
||||
this.bucketName,
|
||||
objectKey,
|
||||
file.buffer,
|
||||
file.size,
|
||||
metaData
|
||||
);
|
||||
|
||||
this.logger.log(`File uploaded successfully: ${objectKey}`);
|
||||
|
||||
return {
|
||||
key: objectKey,
|
||||
etag: uploadInfo.etag,
|
||||
size: file.size,
|
||||
checksum,
|
||||
};
|
||||
} catch (error) {
|
||||
this.logger.error(`Failed to upload file: ${error.message}`, error.stack);
|
||||
throw new Error(`File upload failed: ${error.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a file from MinIO storage
|
||||
* @param objectKey Object key to retrieve
|
||||
* @returns File stream
|
||||
*/
|
||||
async getFile(objectKey: string): Promise<NodeJS.ReadableStream> {
|
||||
try {
|
||||
return await this.minioClient.getObject(this.bucketName, objectKey);
|
||||
} catch (error) {
|
||||
this.logger.error(`Failed to retrieve file: ${objectKey}`, error.stack);
|
||||
throw new Error(`File retrieval failed: ${error.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get file metadata
|
||||
* @param objectKey Object key to get metadata for
|
||||
* @returns File metadata
|
||||
*/
|
||||
async getFileMetadata(objectKey: string): Promise<any> {
|
||||
try {
|
||||
return await this.minioClient.statObject(this.bucketName, objectKey);
|
||||
} catch (error) {
|
||||
this.logger.error(`Failed to get file metadata: ${objectKey}`, error.stack);
|
||||
throw new Error(`File metadata retrieval failed: ${error.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete a file from MinIO storage
|
||||
* @param objectKey Object key to delete
|
||||
*/
|
||||
async deleteFile(objectKey: string): Promise<void> {
|
||||
try {
|
||||
await this.minioClient.removeObject(this.bucketName, objectKey);
|
||||
this.logger.log(`File deleted successfully: ${objectKey}`);
|
||||
} catch (error) {
|
||||
this.logger.error(`Failed to delete file: ${objectKey}`, error.stack);
|
||||
throw new Error(`File deletion failed: ${error.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* List files in a batch folder
|
||||
* @param batchId Batch UUID
|
||||
* @returns Array of object keys
|
||||
*/
|
||||
async listBatchFiles(batchId: string): Promise<string[]> {
|
||||
try {
|
||||
const objects: string[] = [];
|
||||
const objectStream = this.minioClient.listObjects(
|
||||
this.bucketName,
|
||||
`batches/${batchId}/`,
|
||||
true
|
||||
);
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
objectStream.on('data', (obj) => {
|
||||
objects.push(obj.name);
|
||||
});
|
||||
|
||||
objectStream.on('error', (err) => {
|
||||
this.logger.error(`Failed to list batch files: ${batchId}`, err);
|
||||
reject(new Error(`Failed to list batch files: ${err.message}`));
|
||||
});
|
||||
|
||||
objectStream.on('end', () => {
|
||||
resolve(objects);
|
||||
});
|
||||
});
|
||||
} catch (error) {
|
||||
this.logger.error(`Failed to list batch files: ${batchId}`, error.stack);
|
||||
throw new Error(`Batch file listing failed: ${error.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete all files in a batch folder
|
||||
* @param batchId Batch UUID
|
||||
*/
|
||||
async deleteBatchFiles(batchId: string): Promise<void> {
|
||||
try {
|
||||
const objectKeys = await this.listBatchFiles(batchId);
|
||||
|
||||
if (objectKeys.length > 0) {
|
||||
await this.minioClient.removeObjects(this.bucketName, objectKeys);
|
||||
this.logger.log(`Deleted ${objectKeys.length} files for batch: ${batchId}`);
|
||||
}
|
||||
} catch (error) {
|
||||
this.logger.error(`Failed to delete batch files: ${batchId}`, error.stack);
|
||||
throw new Error(`Batch file deletion failed: ${error.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate a presigned URL for file download
|
||||
* @param objectKey Object key
|
||||
* @param expiry Expiry time in seconds (default: 1 hour)
|
||||
* @returns Presigned URL
|
||||
*/
|
||||
async getPresignedUrl(objectKey: string, expiry: number = 3600): Promise<string> {
|
||||
try {
|
||||
return await this.minioClient.presignedGetObject(this.bucketName, objectKey, expiry);
|
||||
} catch (error) {
|
||||
this.logger.error(`Failed to generate presigned URL: ${objectKey}`, error.stack);
|
||||
throw new Error(`Presigned URL generation failed: ${error.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if file exists in storage
|
||||
* @param objectKey Object key to check
|
||||
* @returns Whether file exists
|
||||
*/
|
||||
async fileExists(objectKey: string): Promise<boolean> {
|
||||
try {
|
||||
await this.minioClient.statObject(this.bucketName, objectKey);
|
||||
return true;
|
||||
} catch (error) {
|
||||
if (error.code === 'NotFound') {
|
||||
return false;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate SHA-256 checksum for duplicate detection
|
||||
* @param buffer File buffer
|
||||
* @returns SHA-256 checksum
|
||||
*/
|
||||
calculateChecksum(buffer: Buffer): string {
|
||||
return crypto.createHash('sha256').update(buffer).digest('hex');
|
||||
}
|
||||
|
||||
/**
|
||||
* Get file extension from filename
|
||||
* @param filename Original filename
|
||||
* @returns File extension with dot
|
||||
*/
|
||||
private getFileExtension(filename: string): string {
|
||||
const lastDotIndex = filename.lastIndexOf('.');
|
||||
return lastDotIndex !== -1 ? filename.substring(lastDotIndex) : '';
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate file MIME type for image uploads
|
||||
* @param mimeType MIME type to validate
|
||||
* @returns Whether MIME type is valid
|
||||
*/
|
||||
isValidImageMimeType(mimeType: string): boolean {
|
||||
const validMimeTypes = [
|
||||
'image/jpeg',
|
||||
'image/jpg',
|
||||
'image/png',
|
||||
'image/gif',
|
||||
'image/webp',
|
||||
];
|
||||
return validMimeTypes.includes(mimeType.toLowerCase());
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue