CSVファイルの完全ガイド｜文字化け・エスケープ・エラー解決

import csv

# BOM付きUTF-8でCSV書き出し
def write_csv_with_bom(data, filename):
    with open(filename, 'w', encoding='utf-8-sig', newline='') as csvfile:
        writer = csv.writer(csvfile)
        for row in data:
            writer.writerow(row)

// BOM付きCSVをダウンロード
function downloadCsvWithBom(data, filename) {
    const bom = new Uint8Array([0xEF, 0xBB, 0xBF]);
    const csvContent = data.map(row => row.join(',')).join('\n');
    const blob = new Blob([bom, csvContent], { type: 'text/csv' });
    
    const url = URL.createObjectURL(blob);
    const a = document.createElement('a');
    a.href = url;
    a.download = filename;
    a.click();
}

import csv
import chardet
import logging
from typing import List, Dict, Optional

class CsvProcessor:
    def __init__(self):
        self.logger = logging.getLogger(__name__)
    
    def process_uploaded_csv(self, file_path: str) -> List[Dict]:
        try:
            # ファイルエンコーディング検出
            with open(file_path, 'rb') as f:
                raw_data = f.read()
                encoding_result = chardet.detect(raw_data)
                encoding = encoding_result.get('encoding', 'utf-8')
            
            self.logger.info(f"検出されたエンコーディング: {encoding}")
            
            # CSV読み込み
            data = []
            with open(file_path, 'r', encoding=encoding) as csvfile:
                # 方言を自動検出
                sample = csvfile.read(1024)
                csvfile.seek(0)
                sniffer = csv.Sniffer()
                delimiter = sniffer.sniff(sample).delimiter
                
                reader = csv.DictReader(csvfile, delimiter=delimiter)
                
                for line_num, row in enumerate(reader, 1):
                    try:
                        validated_row = self.validate_and_sanitize_row(row, line_num)
                        if validated_row:
                            data.append(validated_row)
                    except ValueError as e:
                        self.logger.warning(f"行{line_num}でエラー: {e}")
                        continue
            
            return data
            
        except Exception as e:
            self.logger.error(f"CSV処理エラー: {e}")
            raise
    
    def validate_and_sanitize_row(self, row: Dict, line_num: int) -> Optional[Dict]:
        # 空行チェック
        if all(not value.strip() for value in row.values()):
            return None
        
        # データサニタイズ
        sanitized = {}
        for key, value in row.items():
            if value is not None:
                # 改行文字を削除、前後の空白を削除
                sanitized[key] = value.replace('\r\n', ' ').replace('\r', ' ').replace('\n', ' ').strip()
            else:
                sanitized[key] = ''
        
        return sanitized

const fs = require('fs');
const csv = require('csv-parser');
const iconv = require('iconv-lite');
const jschardet = require('jschardet');
const { Transform } = require('stream');

class CsvProcessor {
    constructor() {
        this.logger = console; // 実際にはWinstonなどを使用
    }
    
    async processUploadedCsv(filePath) {
        return new Promise((resolve, reject) => {
            try {
                // ファイルエンコーディング検出
                const buffer = fs.readFileSync(filePath);
                const detected = jschardet.detect(buffer);
                const encoding = detected.encoding || 'utf8';
                
                this.logger.info(`検出されたエンコーディング: ${encoding}`);
                
                const results = [];
                const errors = [];
                
                // エンコーディング変換ストリーム
                const decodedStream = encoding.toLowerCase() === 'utf8' || encoding.toLowerCase() === 'utf-8'
                    ? fs.createReadStream(filePath)
                    : fs.createReadStream(filePath).pipe(iconv.decodeStream(encoding));
                
                // データ変換・検証ストリーム
                const validateTransform = new Transform({
                    objectMode: true,
                    transform(chunk, encoding, callback) {
                        try {
                            const validated = this.validateAndSanitizeRow(chunk);
                            if (validated) {
                                this.push(validated);
                            }
                        } catch (error) {
                            errors.push({ line: results.length + 1, error: error.message });
                        }
                        callback();
                    }.bind(this)
                });
                
                decodedStream
                    .pipe(csv({ skipEmptyLines: true }))
                    .pipe(validateTransform)
                    .on('data', (data) => results.push(data))
                    .on('end', () => {
                        if (errors.length > 0) {
                            this.logger.warn(`${errors.length}件のエラーがありました:`, errors);
                        }
                        resolve(results);
                    })
                    .on('error', reject);
                    
            } catch (error) {
                this.logger.error('CSV処理エラー:', error);
                reject(error);
            }
        });
    }
    
    validateAndSanitizeRow(row) {
        // 空行チェック
        const hasData = Object.values(row).some(value => 
            value && typeof value === 'string' && value.trim().length > 0
        );
        
        if (!hasData) {
            return null;
        }
        
        // データサニタイズ
        const sanitized = {};
        for (const [key, value] of Object.entries(row)) {
            if (typeof value === 'string') {
                sanitized[key] = value
                    .replace(/[\r\n]+/g, ' ')
                    .trim();
            } else {
                sanitized[key] = value || '';
            }
        }
        
        return sanitized;
    }
}

module.exports = CsvProcessor;