import { Injectable, Logger } from '@nestjs/common'; import * as fs from 'fs/promises'; import * as path from 'path'; /** * CSV Converter Service * * Detects and converts CSV files to the standard 33-column Xpeditis format. * * Standard format columns (33): * companyName, companyEmail, originCFS, originCode, portOfLoading, routing, * destinationCFS, destinationCode, destinationCountry, containerType, * freightCurrency, freightRatePerCBM, freightMinimum, * fobCurrency, fobDocumentation, fobISPS, fobHandling, fobHandlingUnit, * fobHandlingMinimum, fobSolas, fobCustoms, fobAMS_ACI, fobISF5, fobDGAdmin, * dgSurchargeCurrency, dgSurchargeRate, dgSurchargeUnit, dgSurchargeMin, * remarks, frequency, transitDays, validFrom, validUntil */ @Injectable() export class CsvConverterService { private readonly logger = new Logger(CsvConverterService.name); private readonly STANDARD_HEADERS = [ 'companyName', 'companyEmail', 'originCFS', 'originCode', 'portOfLoading', 'routing', 'destinationCFS', 'destinationCode', 'destinationCountry', 'containerType', 'freightCurrency', 'freightRatePerCBM', 'freightMinimum', 'fobCurrency', 'fobDocumentation', 'fobISPS', 'fobHandling', 'fobHandlingUnit', 'fobHandlingMinimum', 'fobSolas', 'fobCustoms', 'fobAMS_ACI', 'fobISF5', 'fobDGAdmin', 'dgSurchargeCurrency', 'dgSurchargeRate', 'dgSurchargeUnit', 'dgSurchargeMin', 'remarks', 'frequency', 'transitDays', 'validFrom', 'validUntil', ]; // Legacy "Frais FOB FRET" format indicators (older Excel exports) private readonly FOB_FRET_HEADERS = [ 'Origine UN code', 'Destination UN code', 'Devise FRET', 'Taux de FRET (UP)', 'Minimum FRET (LS)', 'Transit time', ]; async detectFormat(filePath: string): Promise<'STANDARD' | 'FOB_FRET' | 'UNKNOWN'> { try { const content = await fs.readFile(filePath, 'utf-8'); const lines = content.split('\n').filter(l => l.trim()); if (lines.length === 0) return 'UNKNOWN'; for (let i = 0; i < Math.min(2, lines.length); i++) { const headers = this.parseCSVLine(lines[i]); if (this.STANDARD_HEADERS.some(h => headers.includes(h))) return 'STANDARD'; if (this.FOB_FRET_HEADERS.some(h => headers.includes(h))) return 'FOB_FRET'; } return 'UNKNOWN'; } catch { return 'UNKNOWN'; } } async autoConvert( inputPath: string, companyName: string ): Promise<{ convertedPath: string; wasConverted: boolean; rowsConverted?: number }> { const format = await this.detectFormat(inputPath); this.logger.log(`Detected CSV format: ${format} for ${inputPath}`); if (format === 'STANDARD') { return { convertedPath: inputPath, wasConverted: false }; } if (format === 'FOB_FRET') { const result = await this.convertFobFretToStandard(inputPath, companyName); return { convertedPath: result.outputPath, wasConverted: true, rowsConverted: result.rowsConverted, }; } throw new Error( 'Unknown CSV format. Please provide a file matching the standard 33-column schema.' ); } async convertFobFretToStandard( inputPath: string, companyName: string ): Promise<{ outputPath: string; rowsConverted: number }> { this.logger.log(`Converting legacy FOB FRET CSV: ${inputPath}`); const content = await fs.readFile(inputPath, 'utf-8'); const lines = content.split('\n').filter(l => l.trim()); if (lines.length < 2) throw new Error('CSV file is empty or has no data rows'); // Find the header line let headerLineIndex = 0; for (let i = 0; i < Math.min(2, lines.length); i++) { const headers = this.parseCSVLine(lines[i]); if (this.FOB_FRET_HEADERS.some(h => headers.includes(h))) { headerLineIndex = i; break; } } const headers = this.parseCSVLine(lines[headerLineIndex]); const dataRows: Record[] = []; for (let i = headerLineIndex + 1; i < lines.length; i++) { const values = this.parseCSVLine(lines[i]); const row: Record = {}; headers.forEach((header, idx) => (row[header] = values[idx] || '')); if (row['Origine UN code'] && row['Destination UN code']) { dataRows.push(row); } } const convertedRows = dataRows.map(row => this.convertFobFretRow(row, companyName)); const outputLines: string[] = [this.STANDARD_HEADERS.join(',')]; convertedRows.forEach(row => { const values = this.STANDARD_HEADERS.map(header => { const value = row[header] ?? ''; if (typeof value === 'string' && (value.includes(',') || value.includes('"'))) { return `"${value.replace(/"/g, '""')}"`; } return value; }); outputLines.push(values.join(',')); }); const outputPath = path.isAbsolute(inputPath) ? inputPath.replace('.csv', '-converted.csv') : path.resolve(process.cwd(), inputPath.replace('.csv', '-converted.csv')); await fs.writeFile(outputPath, outputLines.join('\n'), 'utf-8'); this.logger.log(`Conversion complete: ${outputPath} (${convertedRows.length} rows)`); return { outputPath, rowsConverted: convertedRows.length }; } private convertFobFretRow(row: Record, companyName: string): Record { const freightCurrency = row['Devise FRET'] || 'USD'; const freightRatePerCBM = parseFloat(row['Taux de FRET (UP)']) || 0; const freightMinimum = parseFloat(row['Minimum FRET (LS)']) || 0; const transitDays = parseInt(row['Transit time'], 10) || 0; const fobCurrency = row['Devise FOB'] || 'EUR'; const validFrom = new Date().toISOString().split('T')[0]; const validUntil = new Date(Date.now() + 90 * 24 * 60 * 60 * 1000).toISOString().split('T')[0]; const originCode = row['Origine UN code'] || ''; const destinationCode = row['Destination UN code'] || ''; return { companyName, companyEmail: row['Email'] || '', originCFS: row['Origine CFS'] || originCode, originCode, portOfLoading: row['Port of Loading'] || originCode, routing: row['Routing'] || 'Direct', destinationCFS: row['Destination CFS'] || destinationCode, destinationCode, destinationCountry: row['Destination Country'] || '', containerType: 'LCL', freightCurrency, freightRatePerCBM, freightMinimum, fobCurrency, fobDocumentation: parseInt(row['Documentation (LS et Minimum)'], 10) || 0, fobISPS: parseInt(row['ISPS (LS et Minimum)'], 10) || 0, fobHandling: parseInt(row['Manutention'], 10) || 0, fobHandlingUnit: row['Unité de manutention (UP;Tonne)'] || 'W', fobHandlingMinimum: parseInt(row['Minimum manutention'], 10) || 0, fobSolas: parseInt(row['Solas (LS et Minimum)'], 10) || 0, fobCustoms: parseInt(row['Douane (LS et Minimum)'], 10) || 0, fobAMS_ACI: parseFloat(row['AMS/ACI (LS et Minimum)']) || 0, fobISF5: parseFloat(row['ISF5 (LS et Minimum)']) || 0, fobDGAdmin: parseInt(row['Frais admin de dangereux (LS et Minimum)'], 10) || 0, dgSurchargeCurrency: row['Devise surcharge DG'] || fobCurrency, dgSurchargeRate: row['Taux surcharge DG'] || '0', dgSurchargeUnit: row['Unité surcharge DG'] || 'LS', dgSurchargeMin: row['Minimum surcharge DG'] || '0', remarks: row['Remarques'] || '', frequency: row['Frequence'] || 'Weekly', transitDays, validFrom, validUntil, }; } private parseCSVLine(line: string): string[] { const result: string[] = []; let current = ''; let inQuotes = false; for (const char of line) { if (char === '"') { inQuotes = !inQuotes; } else if (char === ',' && !inQuotes) { result.push(current.trim()); current = ''; } else { current += char; } } result.push(current.trim()); return result; } }