From 634b9adc4a20b3ffca9753060eb0d3b62a1ea072 Mon Sep 17 00:00:00 2001 From: David Date: Wed, 29 Oct 2025 21:18:38 +0100 Subject: [PATCH] feature csv rates --- .../csv-loader/csv-converter.service.ts | 314 ++++++++++++++++++ scripts/convert-rates-csv.js | 267 +++++++++++++++ 2 files changed, 581 insertions(+) create mode 100644 apps/backend/src/infrastructure/carriers/csv-loader/csv-converter.service.ts create mode 100644 scripts/convert-rates-csv.js diff --git a/apps/backend/src/infrastructure/carriers/csv-loader/csv-converter.service.ts b/apps/backend/src/infrastructure/carriers/csv-loader/csv-converter.service.ts new file mode 100644 index 0000000..5d71698 --- /dev/null +++ b/apps/backend/src/infrastructure/carriers/csv-loader/csv-converter.service.ts @@ -0,0 +1,314 @@ +import { Injectable, Logger } from '@nestjs/common'; +import * as fs from 'fs/promises'; +import * as path from 'path'; + +/** + * CSV Converter Service + * + * Détecte automatiquement le format du CSV et convertit au format attendu + * Supporte: + * - Format standard Xpeditis + * - Format "Frais FOB FRET" + */ +@Injectable() +export class CsvConverterService { + private readonly logger = new Logger(CsvConverterService.name); + + // Headers du format standard attendu + private readonly STANDARD_HEADERS = [ + 'companyName', + 'origin', + 'destination', + 'containerType', + 'minVolumeCBM', + 'maxVolumeCBM', + 'minWeightKG', + 'maxWeightKG', + 'palletCount', + 'pricePerCBM', + 'pricePerKG', + 'basePriceUSD', + 'basePriceEUR', + 'currency', + 'hasSurcharges', + 'surchargeBAF', + 'surchargeCAF', + 'surchargeDetails', + 'transitDays', + 'validFrom', + 'validUntil', + ]; + + // Headers du format "Frais FOB FRET" + private readonly FOB_FRET_HEADERS = [ + 'Origine UN code', + 'Destination UN code', + 'Devise FRET', + 'Taux de FRET (UP)', + 'Minimum FRET (LS)', + 'Transit time', + ]; + + /** + * Parse une ligne CSV en gérant les champs entre guillemets + */ + private parseCSVLine(line: string): string[] { + const result: string[] = []; + let current = ''; + let inQuotes = false; + + for (let i = 0; i < line.length; i++) { + const char = line[i]; + + if (char === '"') { + inQuotes = !inQuotes; + } else if (char === ',' && !inQuotes) { + result.push(current.trim()); + current = ''; + } else { + current += char; + } + } + result.push(current.trim()); + + return result; + } + + /** + * Détecte le format du CSV + */ + async detectFormat(filePath: string): Promise<'STANDARD' | 'FOB_FRET' | 'UNKNOWN'> { + try { + const content = await fs.readFile(filePath, 'utf-8'); + const lines = content.split('\n').filter(line => line.trim()); + + if (lines.length === 0) { + return 'UNKNOWN'; + } + + // Vérifier les 2 premières lignes (parfois la vraie ligne d'en-tête est la ligne 2) + for (let i = 0; i < Math.min(2, lines.length); i++) { + const headers = this.parseCSVLine(lines[i]); + + // Vérifier format standard + const hasStandardHeaders = this.STANDARD_HEADERS.some(h => headers.includes(h)); + if (hasStandardHeaders) { + return 'STANDARD'; + } + + // Vérifier format FOB FRET + const hasFobFretHeaders = this.FOB_FRET_HEADERS.some(h => headers.includes(h)); + if (hasFobFretHeaders) { + return 'FOB_FRET'; + } + } + + return 'UNKNOWN'; + } catch (error) { + const errorMessage = error instanceof Error ? error.message : 'Unknown error'; + this.logger.error(`Error detecting CSV format: ${errorMessage}`); + return 'UNKNOWN'; + } + } + + /** + * Calcule les surcharges à partir des colonnes FOB + */ + private calculateSurcharges(row: Record): string { + const surcharges: string[] = []; + + const surchargeFields = [ + { key: 'Documentation (LS et Minimum)', prefix: 'DOC' }, + { key: 'ISPS (LS et Minimum)', prefix: 'ISPS' }, + { key: 'Manutention', prefix: 'HANDLING' }, + { key: 'Solas (LS et Minimum)', prefix: 'SOLAS' }, + { key: 'Douane (LS et Minimum)', prefix: 'CUSTOMS' }, + { key: 'AMS/ACI (LS et Minimum)', prefix: 'AMS_ACI' }, + { key: 'ISF5 (LS et Minimum)', prefix: 'ISF5' }, + { key: 'Frais admin de dangereux (LS et Minimum)', prefix: 'DG_FEE' }, + ]; + + surchargeFields.forEach(({ key, prefix }) => { + if (row[key]) { + const unit = key === 'Manutention' ? row['Unité de manutention (UP;Tonne)'] || 'UP' : ''; + surcharges.push(`${prefix}:${row[key]}${unit ? ' ' + unit : ''}`); + } + }); + + return surcharges.join(' | '); + } + + /** + * Convertit une ligne FOB FRET vers le format standard + */ + private convertFobFretRow(row: Record, companyName: string): Record { + const currency = row['Devise FRET'] || 'USD'; + const freightRate = parseFloat(row['Taux de FRET (UP)']) || 0; + const minFreight = parseFloat(row['Minimum FRET (LS)']) || 0; + const transitDays = parseInt(row['Transit time']) || 0; + + // Calcul des surcharges + const surchargeDetails = this.calculateSurcharges(row); + const hasSurcharges = surchargeDetails.length > 0; + + // Dates de validité (90 jours par défaut) + const validFrom = new Date().toISOString().split('T')[0]; + const validUntil = new Date(Date.now() + 90 * 24 * 60 * 60 * 1000).toISOString().split('T')[0]; + + // Volumes et poids standards pour LCL + const minVolumeCBM = 1; + const maxVolumeCBM = 20; + const minWeightKG = 100; + const maxWeightKG = 20000; + + // Prix par CBM + const pricePerCBM = freightRate > 0 ? freightRate : minFreight; + + // Prix par KG (estimation: prix CBM / 200 kg/m³) + const pricePerKG = pricePerCBM > 0 ? (pricePerCBM / 200).toFixed(2) : '0'; + + return { + companyName, + origin: row['Origine UN code'] || '', + destination: row['Destination UN code'] || '', + containerType: 'LCL', + minVolumeCBM, + maxVolumeCBM, + minWeightKG, + maxWeightKG, + palletCount: 0, + pricePerCBM, + pricePerKG, + basePriceUSD: currency === 'USD' ? pricePerCBM : 0, + basePriceEUR: currency === 'EUR' ? pricePerCBM : 0, + currency, + hasSurcharges, + surchargeBAF: '', + surchargeCAF: '', + surchargeDetails, + transitDays, + validFrom, + validUntil, + }; + } + + /** + * Convertit un CSV FOB FRET vers le format standard + */ + async convertFobFretToStandard( + inputPath: string, + companyName: string + ): Promise<{ outputPath: string; rowsConverted: number }> { + this.logger.log(`Converting FOB FRET CSV: ${inputPath}`); + + try { + // Lire le fichier + const content = await fs.readFile(inputPath, 'utf-8'); + const lines = content.split('\n').filter(line => line.trim()); + + if (lines.length < 2) { + throw new Error('CSV file is empty or has no data rows'); + } + + // Trouver la ligne d'en-tête réelle (chercher celle avec "Devise FRET") + let headerLineIndex = 0; + for (let i = 0; i < Math.min(2, lines.length); i++) { + const headers = this.parseCSVLine(lines[i]); + if (this.FOB_FRET_HEADERS.some(h => headers.includes(h))) { + headerLineIndex = i; + break; + } + } + + // Parse headers + const headers = this.parseCSVLine(lines[headerLineIndex]); + this.logger.log(`Found FOB FRET headers at line ${headerLineIndex + 1}`); + + // Parse data rows (commencer après la ligne d'en-tête) + const dataRows: Record[] = []; + for (let i = headerLineIndex + 1; i < lines.length; i++) { + const values = this.parseCSVLine(lines[i]); + const row: Record = {}; + + headers.forEach((header, index) => { + row[header] = values[index] || ''; + }); + + // Vérifier que la ligne a des données valides + if (row['Origine UN code'] && row['Destination UN code']) { + dataRows.push(row); + } + } + + this.logger.log(`Found ${dataRows.length} valid data rows`); + + // Convertir les lignes + const convertedRows = dataRows.map(row => this.convertFobFretRow(row, companyName)); + + // Générer le CSV de sortie + const outputLines: string[] = [this.STANDARD_HEADERS.join(',')]; + + convertedRows.forEach(row => { + const values = this.STANDARD_HEADERS.map(header => { + const value = row[header]; + + // Échapper les virgules et quotes + if ( + typeof value === 'string' && + (value.includes(',') || value.includes('"') || value.includes('\n')) + ) { + return `"${value.replace(/"/g, '""')}"`; + } + return value; + }); + outputLines.push(values.join(',')); + }); + + // Écrire le fichier converti + const outputPath = inputPath.replace('.csv', '-converted.csv'); + await fs.writeFile(outputPath, outputLines.join('\n'), 'utf-8'); + + this.logger.log(`Conversion completed: ${outputPath} (${convertedRows.length} rows)`); + + return { + outputPath, + rowsConverted: convertedRows.length, + }; + } catch (error) { + const errorMessage = error instanceof Error ? error.message : 'Unknown error'; + const errorStack = error instanceof Error ? error.stack : undefined; + this.logger.error(`Error converting CSV: ${errorMessage}`, errorStack); + throw new Error(`CSV conversion failed: ${errorMessage}`); + } + } + + /** + * Convertit automatiquement un CSV si nécessaire + */ + async autoConvert( + inputPath: string, + companyName: string + ): Promise<{ convertedPath: string; wasConverted: boolean; rowsConverted?: number }> { + const format = await this.detectFormat(inputPath); + + this.logger.log(`Detected CSV format: ${format}`); + + if (format === 'STANDARD') { + return { + convertedPath: inputPath, + wasConverted: false, + }; + } + + if (format === 'FOB_FRET') { + const result = await this.convertFobFretToStandard(inputPath, companyName); + return { + convertedPath: result.outputPath, + wasConverted: true, + rowsConverted: result.rowsConverted, + }; + } + + throw new Error(`Unknown CSV format. Please provide a valid CSV file.`); + } +} diff --git a/scripts/convert-rates-csv.js b/scripts/convert-rates-csv.js new file mode 100644 index 0000000..9fad8d3 --- /dev/null +++ b/scripts/convert-rates-csv.js @@ -0,0 +1,267 @@ +#!/usr/bin/env node + +/** + * Script de conversion CSV pour les tarifs maritimes + * Convertit le format "Frais FOB FRET" vers le format d'import Xpeditis + */ + +const fs = require('fs'); +const path = require('path'); + +// Configuration +const INPUT_FILE = process.argv[2] || path.join(__dirname, '../Frais FOB FRET.csv'); +const OUTPUT_FILE = process.argv[3] || path.join(__dirname, '../rates-import-ready.csv'); +const COMPANY_NAME = process.argv[4] || 'Consolidation Maritime'; + +// Headers attendus par l'API +const OUTPUT_HEADERS = [ + 'companyName', + 'origin', + 'destination', + 'containerType', + 'minVolumeCBM', + 'maxVolumeCBM', + 'minWeightKG', + 'maxWeightKG', + 'palletCount', + 'pricePerCBM', + 'pricePerKG', + 'basePriceUSD', + 'basePriceEUR', + 'currency', + 'hasSurcharges', + 'surchargeBAF', + 'surchargeCAF', + 'surchargeDetails', + 'transitDays', + 'validFrom', + 'validUntil' +]; + +/** + * Parse CSV line handling quoted fields + */ +function parseCSVLine(line) { + const result = []; + let current = ''; + let inQuotes = false; + + for (let i = 0; i < line.length; i++) { + const char = line[i]; + + if (char === '"') { + inQuotes = !inQuotes; + } else if (char === ',' && !inQuotes) { + result.push(current.trim()); + current = ''; + } else { + current += char; + } + } + result.push(current.trim()); + + return result; +} + +/** + * Calcule les frais supplémentaires à partir des colonnes FOB + */ +function calculateSurcharges(row) { + const surcharges = []; + + // Documentation (LS et Minimum) + if (row['Documentation (LS et Minimum)']) { + surcharges.push(`DOC:${row['Documentation (LS et Minimum)']}`); + } + + // ISPS (LS et Minimum) + if (row['ISPS (LS et Minimum)']) { + surcharges.push(`ISPS:${row['ISPS (LS et Minimum)']}`); + } + + // Manutention + if (row['Manutention']) { + surcharges.push(`HANDLING:${row['Manutention']} ${row['Unité de manutention (UP;Tonne)'] || 'UP'}`); + } + + // SOLAS + if (row['Solas (LS et Minimum)']) { + surcharges.push(`SOLAS:${row['Solas (LS et Minimum)']}`); + } + + // Douane + if (row['Douane (LS et Minimum)']) { + surcharges.push(`CUSTOMS:${row['Douane (LS et Minimum)']}`); + } + + // AMS/ACI + if (row['AMS/ACI (LS et Minimum)']) { + surcharges.push(`AMS_ACI:${row['AMS/ACI (LS et Minimum)']}`); + } + + // ISF5 + if (row['ISF5 (LS et Minimum)']) { + surcharges.push(`ISF5:${row['ISF5 (LS et Minimum)']}`); + } + + // Frais admin dangereux + if (row['Frais admin de dangereux (LS et Minimum)']) { + surcharges.push(`DG_FEE:${row['Frais admin de dangereux (LS et Minimum)']}`); + } + + return surcharges.join(' | '); +} + +/** + * Conversion d'une ligne du CSV source vers le format de destination + */ +function convertRow(sourceRow) { + const currency = sourceRow['Devise FRET'] || 'USD'; + const freightRate = parseFloat(sourceRow['Taux de FRET (UP)']) || 0; + const minFreight = parseFloat(sourceRow['Minimum FRET (LS)']) || 0; + const transitDays = parseInt(sourceRow['Transit time']) || 0; + + // Calcul des surcharges + const surchargeDetails = calculateSurcharges(sourceRow); + const hasSurcharges = surchargeDetails.length > 0; + + // Dates de validité (par défaut: 90 jours à partir d'aujourd'hui) + const validFrom = new Date().toISOString().split('T')[0]; + const validUntil = new Date(Date.now() + 90 * 24 * 60 * 60 * 1000).toISOString().split('T')[0]; + + // Estimation volume min/max (1-20 CBM pour LCL standard) + const minVolumeCBM = 1; + const maxVolumeCBM = 20; + + // Estimation poids min/max (100-20000 KG) + const minWeightKG = 100; + const maxWeightKG = 20000; + + // Prix par CBM (basé sur le taux de fret) + const pricePerCBM = freightRate > 0 ? freightRate : minFreight; + + // Prix par KG (estimation: prix CBM / 200 kg/m³) + const pricePerKG = pricePerCBM > 0 ? (pricePerCBM / 200).toFixed(2) : 0; + + return { + companyName: COMPANY_NAME, + origin: sourceRow['Origine UN code'] || '', + destination: sourceRow['Destination UN code'] || '', + containerType: 'LCL', + minVolumeCBM: minVolumeCBM, + maxVolumeCBM: maxVolumeCBM, + minWeightKG: minWeightKG, + maxWeightKG: maxWeightKG, + palletCount: 0, + pricePerCBM: pricePerCBM, + pricePerKG: pricePerKG, + basePriceUSD: currency === 'USD' ? pricePerCBM : 0, + basePriceEUR: currency === 'EUR' ? pricePerCBM : 0, + currency: currency, + hasSurcharges: hasSurcharges, + surchargeBAF: '', // BAF non spécifié dans le CSV source + surchargeCAF: '', // CAF non spécifié dans le CSV source + surchargeDetails: surchargeDetails, + transitDays: transitDays, + validFrom: validFrom, + validUntil: validUntil + }; +} + +/** + * Main conversion function + */ +async function convertCSV() { + console.log('🔄 Conversion CSV - Tarifs Maritimes\n'); + console.log(`📥 Fichier source: ${INPUT_FILE}`); + console.log(`📤 Fichier destination: ${OUTPUT_FILE}`); + console.log(`🏢 Compagnie: ${COMPANY_NAME}\n`); + + try { + // Lecture du fichier source + if (!fs.existsSync(INPUT_FILE)) { + throw new Error(`Fichier source introuvable: ${INPUT_FILE}`); + } + + const fileContent = fs.readFileSync(INPUT_FILE, 'utf-8'); + const lines = fileContent.split('\n').filter(line => line.trim()); + + console.log(`📊 ${lines.length} lignes trouvées\n`); + + // Parse headers + const headers = parseCSVLine(lines[0]); + console.log(`📋 Colonnes source détectées: ${headers.length}`); + + // Parse data rows + const dataRows = []; + for (let i = 1; i < lines.length; i++) { + const values = parseCSVLine(lines[i]); + + // Créer un objet avec les headers comme clés + const row = {}; + headers.forEach((header, index) => { + row[header] = values[index] || ''; + }); + + // Vérifier que la ligne a des données valides + if (row['Origine UN code'] && row['Destination UN code']) { + dataRows.push(row); + } + } + + console.log(`✅ ${dataRows.length} lignes valides à convertir\n`); + + // Conversion des lignes + const convertedRows = dataRows.map(convertRow); + + // Génération du CSV de sortie + const outputLines = [OUTPUT_HEADERS.join(',')]; + + convertedRows.forEach(row => { + const values = OUTPUT_HEADERS.map(header => { + const value = row[header]; + // Échapper les virgules et quotes + if (typeof value === 'string' && (value.includes(',') || value.includes('"') || value.includes('\n'))) { + return `"${value.replace(/"/g, '""')}"`; + } + return value; + }); + outputLines.push(values.join(',')); + }); + + // Écriture du fichier + fs.writeFileSync(OUTPUT_FILE, outputLines.join('\n'), 'utf-8'); + + console.log('✅ Conversion terminée avec succès!\n'); + console.log('📊 Statistiques:'); + console.log(` - Lignes converties: ${convertedRows.length}`); + console.log(` - Origines uniques: ${new Set(convertedRows.map(r => r.origin)).size}`); + console.log(` - Destinations uniques: ${new Set(convertedRows.map(r => r.destination)).size}`); + console.log(` - Devises: ${new Set(convertedRows.map(r => r.currency)).size} (${[...new Set(convertedRows.map(r => r.currency))].join(', ')})`); + + // Exemples de tarifs + console.log('\n📦 Exemples de tarifs convertis:'); + convertedRows.slice(0, 3).forEach((row, idx) => { + console.log(` ${idx + 1}. ${row.origin} → ${row.destination}`); + console.log(` Prix: ${row.pricePerCBM} ${row.currency}/CBM (${row.pricePerKG} ${row.currency}/KG)`); + console.log(` Transit: ${row.transitDays} jours`); + }); + + console.log(`\n🎯 Fichier prêt à importer: ${OUTPUT_FILE}`); + console.log('\n📝 Commande d\'import:'); + console.log(` curl -X POST http://localhost:4000/api/v1/admin/rates/csv/upload \\`); + console.log(` -H "Authorization: Bearer $TOKEN" \\`); + console.log(` -F "file=@${OUTPUT_FILE}"`); + + } catch (error) { + console.error('❌ Erreur lors de la conversion:', error.message); + process.exit(1); + } +} + +// Exécution +if (require.main === module) { + convertCSV(); +} + +module.exports = { convertCSV };