diff --git a/.claude/settings.local.json b/.claude/settings.local.json index b660554..7411d53 100644 --- a/.claude/settings.local.json +++ b/.claude/settings.local.json @@ -6,7 +6,26 @@ "Bash(npm run lint:*)", "Bash(npm run backend:lint)", "Bash(npm run backend:build:*)", - "Bash(npm run frontend:build:*)" + "Bash(npm run frontend:build:*)", + "Bash(rm:*)", + "Bash(git rm:*)", + "Bash(git add:*)", + "Bash(git commit:*)", + "Bash(git push:*)", + "Bash(npx tsc:*)", + "Bash(npx nest:*)", + "Read(//Users/david/Documents/xpeditis/**)", + "Bash(find:*)", + "Bash(npm test)", + "Bash(git checkout:*)", + "Bash(git reset:*)", + "Bash(curl:*)", + "Read(//private/tmp/**)", + "Bash(lsof:*)", + "Bash(awk:*)", + "Bash(xargs kill:*)", + "Read(//dev/**)", + "Bash(psql:*)" ], "deny": [], "ask": [] diff --git a/apps/backend/src/application/controllers/admin/csv-rates.controller.ts b/apps/backend/src/application/controllers/admin/csv-rates.controller.ts index b602343..082e77f 100644 --- a/apps/backend/src/application/controllers/admin/csv-rates.controller.ts +++ b/apps/backend/src/application/controllers/admin/csv-rates.controller.ts @@ -38,6 +38,8 @@ import { CsvFileValidationDto, } from '../../dto/csv-rate-upload.dto'; import { CsvRateMapper } from '../../mappers/csv-rate.mapper'; +import { S3StorageAdapter } from '@infrastructure/storage/s3-storage.adapter'; +import { ConfigService } from '@nestjs/config'; /** * CSV Rates Admin Controller @@ -57,7 +59,9 @@ export class CsvRatesAdminController { private readonly csvLoader: CsvRateLoaderAdapter, private readonly csvConverter: CsvConverterService, private readonly csvConfigRepository: TypeOrmCsvRateConfigRepository, - private readonly csvRateMapper: CsvRateMapper + private readonly csvRateMapper: CsvRateMapper, + private readonly s3Storage: S3StorageAdapter, + private readonly configService: ConfigService ) {} /** @@ -197,6 +201,34 @@ export class CsvRatesAdminController { fs.renameSync(filePathToValidate, finalPath); this.logger.log(`Renamed ${file.filename} to ${finalFilename}`); + // Upload CSV file to MinIO/S3 + let minioObjectKey: string | null = null; + try { + const csvBuffer = fs.readFileSync(finalPath); + const bucket = this.configService.get('AWS_S3_BUCKET', 'xpeditis-csv-rates'); + const objectKey = `csv-rates/${finalFilename}`; + + await this.s3Storage.upload({ + bucket, + key: objectKey, + body: csvBuffer, + contentType: 'text/csv', + metadata: { + companyName: dto.companyName, + companyEmail: dto.companyEmail, + uploadedBy: user.email, + uploadedAt: new Date().toISOString(), + }, + }); + + minioObjectKey = objectKey; + this.logger.log(`✅ CSV file uploaded to MinIO: ${bucket}/${objectKey}`); + } catch (error: any) { + this.logger.error(`⚠️ Failed to upload CSV to MinIO (will continue with local storage): ${error.message}`); + // Don't fail the entire operation if MinIO upload fails + // The file is still available locally + } + // Check if config exists for this company const existingConfig = await this.csvConfigRepository.findByCompanyName(dto.companyName); @@ -211,6 +243,7 @@ export class CsvRatesAdminController { metadata: { ...existingConfig.metadata, companyEmail: dto.companyEmail, // Store email in metadata + minioObjectKey, // Store MinIO object key lastUpload: { timestamp: new Date().toISOString(), by: user.email, @@ -237,6 +270,7 @@ export class CsvRatesAdminController { uploadedBy: user.email, description: `${dto.companyName} shipping rates`, companyEmail: dto.companyEmail, // Store email in metadata + minioObjectKey, // Store MinIO object key }, }); @@ -492,6 +526,19 @@ export class CsvRatesAdminController { this.logger.error(`Failed to delete file ${filePath}: ${error.message}`); } + // Delete from MinIO/S3 if it exists there + const minioObjectKey = config.metadata?.minioObjectKey as string | undefined; + if (minioObjectKey) { + try { + const bucket = this.configService.get('AWS_S3_BUCKET', 'xpeditis-csv-rates'); + await this.s3Storage.delete({ bucket, key: minioObjectKey }); + this.logger.log(`✅ Deleted file from MinIO: ${bucket}/${minioObjectKey}`); + } catch (error: any) { + this.logger.error(`⚠️ Failed to delete file from MinIO: ${error.message}`); + // Don't fail the operation if MinIO deletion fails + } + } + // Delete the configuration await this.csvConfigRepository.delete(config.companyName); diff --git a/apps/backend/src/infrastructure/carriers/csv-loader/csv-rate-loader.adapter.ts b/apps/backend/src/infrastructure/carriers/csv-loader/csv-rate-loader.adapter.ts index 8888fd7..5ca7c3b 100644 --- a/apps/backend/src/infrastructure/carriers/csv-loader/csv-rate-loader.adapter.ts +++ b/apps/backend/src/infrastructure/carriers/csv-loader/csv-rate-loader.adapter.ts @@ -1,4 +1,5 @@ -import { Injectable, Logger } from '@nestjs/common'; +import { Injectable, Logger, Optional } from '@nestjs/common'; +import { ConfigService } from '@nestjs/config'; import { parse } from 'csv-parse/sync'; import * as fs from 'fs/promises'; import * as path from 'path'; @@ -9,6 +10,8 @@ import { ContainerType } from '@domain/value-objects/container-type.vo'; import { Money } from '@domain/value-objects/money.vo'; import { Surcharge, SurchargeType, SurchargeCollection } from '@domain/value-objects/surcharge.vo'; import { DateRange } from '@domain/value-objects/date-range.vo'; +import { S3StorageAdapter } from '@infrastructure/storage/s3-storage.adapter'; +import { TypeOrmCsvRateConfigRepository } from '@infrastructure/persistence/typeorm/repositories/typeorm-csv-rate-config.repository'; /** * CSV Row Interface @@ -63,7 +66,11 @@ export class CsvRateLoaderAdapter implements CsvRateLoaderPort { ['NVO Consolidation', 'nvo-consolidation.csv'], ]); - constructor() { + constructor( + @Optional() private readonly s3Storage?: S3StorageAdapter, + @Optional() private readonly configService?: ConfigService, + @Optional() private readonly csvConfigRepository?: TypeOrmCsvRateConfigRepository + ) { // CSV files are stored in infrastructure/storage/csv-storage/rates/ // Use absolute path based on project root (works in both dev and production) // In production, process.cwd() points to the backend app directory @@ -77,18 +84,50 @@ export class CsvRateLoaderAdapter implements CsvRateLoaderPort { 'rates' ); this.logger.log(`CSV directory initialized: ${this.csvDirectory}`); + + if (this.s3Storage && this.configService) { + this.logger.log('✅ MinIO/S3 storage support enabled for CSV files'); + } } async loadRatesFromCsv(filePath: string, companyEmail: string, companyNameOverride?: string): Promise { this.logger.log(`Loading rates from CSV: ${filePath} (email: ${companyEmail}, company: ${companyNameOverride || 'from CSV'})`); try { - // Read CSV file - const fullPath = path.isAbsolute(filePath) - ? filePath - : path.join(this.csvDirectory, filePath); + let fileContent: string; - const fileContent = await fs.readFile(fullPath, 'utf-8'); + // Try to load from MinIO first if configured + if (this.s3Storage && this.configService && this.csvConfigRepository && companyNameOverride) { + try { + const config = await this.csvConfigRepository.findByCompanyName(companyNameOverride); + const minioObjectKey = config?.metadata?.minioObjectKey as string | undefined; + + if (minioObjectKey) { + const bucket = this.configService.get('AWS_S3_BUCKET', 'xpeditis-csv-rates'); + this.logger.log(`📥 Loading CSV from MinIO: ${bucket}/${minioObjectKey}`); + + const buffer = await this.s3Storage.download({ bucket, key: minioObjectKey }); + fileContent = buffer.toString('utf-8'); + this.logger.log(`✅ Successfully loaded CSV from MinIO`); + } else { + // Fallback to local file + throw new Error('No MinIO object key found, using local file'); + } + } catch (minioError: any) { + this.logger.warn(`⚠️ Failed to load from MinIO: ${minioError.message}. Falling back to local file.`); + // Fallback to local file system + const fullPath = path.isAbsolute(filePath) + ? filePath + : path.join(this.csvDirectory, filePath); + fileContent = await fs.readFile(fullPath, 'utf-8'); + } + } else { + // Read from local file system + const fullPath = path.isAbsolute(filePath) + ? filePath + : path.join(this.csvDirectory, filePath); + fileContent = await fs.readFile(fullPath, 'utf-8'); + } // Parse CSV const records: CsvRow[] = parse(fileContent, { diff --git a/apps/backend/src/scripts/migrate-csv-to-minio.ts b/apps/backend/src/scripts/migrate-csv-to-minio.ts new file mode 100644 index 0000000..48feee4 --- /dev/null +++ b/apps/backend/src/scripts/migrate-csv-to-minio.ts @@ -0,0 +1,118 @@ +import { NestFactory } from '@nestjs/core'; +import { AppModule } from '../app.module'; +import { S3StorageAdapter } from '@infrastructure/storage/s3-storage.adapter'; +import { TypeOrmCsvRateConfigRepository } from '@infrastructure/persistence/typeorm/repositories/typeorm-csv-rate-config.repository'; +import { ConfigService } from '@nestjs/config'; +import * as fs from 'fs'; +import * as path from 'path'; + +/** + * Script to migrate existing CSV files to MinIO + * Usage: npm run ts-node src/scripts/migrate-csv-to-minio.ts + */ +async function migrateCsvFilesToMinio() { + const app = await NestFactory.createApplicationContext(AppModule); + const s3Storage = app.get(S3StorageAdapter); + const csvConfigRepository = app.get(TypeOrmCsvRateConfigRepository); + const configService = app.get(ConfigService); + + try { + console.log('🚀 Starting CSV migration to MinIO...\n'); + + const bucket = configService.get('AWS_S3_BUCKET', 'xpeditis-csv-rates'); + const csvDirectory = path.join( + process.cwd(), + 'src', + 'infrastructure', + 'storage', + 'csv-storage', + 'rates' + ); + + // Get all CSV configurations + const configs = await csvConfigRepository.findAll(); + console.log(`📋 Found ${configs.length} CSV configurations\n`); + + let migratedCount = 0; + let skippedCount = 0; + let errorCount = 0; + + for (const config of configs) { + const filename = config.csvFilePath; + const filePath = path.join(csvDirectory, filename); + + console.log(`📄 Processing: ${config.companyName} - ${filename}`); + + // Check if already in MinIO + const existingMinioKey = config.metadata?.minioObjectKey as string | undefined; + if (existingMinioKey) { + console.log(` ⏭️ Already in MinIO: ${existingMinioKey}`); + skippedCount++; + continue; + } + + // Check if file exists locally + if (!fs.existsSync(filePath)) { + console.log(` ⚠️ Local file not found: ${filePath}`); + errorCount++; + continue; + } + + try { + // Read local file + const fileBuffer = fs.readFileSync(filePath); + const objectKey = `csv-rates/${filename}`; + + // Upload to MinIO + await s3Storage.upload({ + bucket, + key: objectKey, + body: fileBuffer, + contentType: 'text/csv', + metadata: { + companyName: config.companyName, + uploadedBy: 'migration-script', + migratedAt: new Date().toISOString(), + }, + }); + + // Update configuration with MinIO object key + await csvConfigRepository.update(config.id, { + metadata: { + ...config.metadata, + minioObjectKey: objectKey, + migratedToMinioAt: new Date().toISOString(), + }, + }); + + console.log(` ✅ Uploaded to MinIO: ${bucket}/${objectKey}`); + migratedCount++; + } catch (error: any) { + console.log(` ❌ Error uploading ${filename}: ${error.message}`); + errorCount++; + } + } + + console.log('\n' + '='.repeat(60)); + console.log('📊 Migration Summary:'); + console.log(` ✅ Migrated: ${migratedCount}`); + console.log(` ⏭️ Skipped (already in MinIO): ${skippedCount}`); + console.log(` ❌ Errors: ${errorCount}`); + console.log('='.repeat(60) + '\n'); + + if (migratedCount > 0) { + console.log('🎉 Migration completed successfully!'); + } else if (skippedCount === configs.length) { + console.log('✅ All files are already in MinIO'); + } else { + console.log('⚠️ Migration completed with errors'); + } + } catch (error: any) { + console.error('❌ Migration failed:', error.message); + process.exit(1); + } + + await app.close(); +} + +migrateCsvFilesToMinio();