feat: 优化执行效率

scripts/optimize-images.js

@@ -4,130 +4,332 @@ import path from 'path';
 import { fileURLToPath } from 'url';
 import { glob } from 'glob';
 import os from 'os';
+import { Worker, isMainThread, parentPort, workerData } from 'worker_threads';
 
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = path.dirname(__filename);
 
 const sourceDir = 'public/images';
 const outputDir = 'dist/images';
-
-// 统计数据
-let totalFiles = 0;
-let totalOriginalSize = 0;
-let totalOptimizedSize = 0;
-let successCount = 0;
-let failCount = 0;
-let startTime = Date.now();
-
-// 文件大小阈值（字节）
 const SMALL_FILE_THRESHOLD = 5 * 1024; // 5KB
 
-async function optimizeImage(inputPath) {
+// 动态计算最佳参数
-    const relativePath = path.relative(sourceDir, inputPath);
+async function calculateOptimalParams() {
-    const outputPath = path.join(outputDir, relativePath);
+    const totalMemory = os.totalmem();
-    const outputDirPath = path.dirname(outputPath);
+    const freeMemory = os.freemem();
+    const cpuCount = os.cpus().length;
+    const cpuUsage = os.loadavg()[0] / cpuCount; // 1分钟平均负载
 
-    await fs.mkdir(outputDirPath, { recursive: true });
+    // 根据系统内存使用情况动态调整内存使用比例
-
+    const memoryUsageRatio = 1 - (freeMemory / totalMemory);
-    try {
+    let memoryAllocationRatio;
-        const inputStats = await fs.stat(inputPath);
+    if (memoryUsageRatio < 0.7) { // 内存使用率低于70%
-        const isSmallFile = inputStats.size < SMALL_FILE_THRESHOLD;
+        memoryAllocationRatio = 0.4; // 可以使用40%的总内存
-
+    } else if (memoryUsageRatio < 0.85) { // 内存使用率在70%-85%之间
-        // 根据文件大小选择不同的压缩策略
+        memoryAllocationRatio = 0.3; // 使用30%的总内存
-        let sharpInstance = sharp(inputPath);
+    } else { // 内存使用率高于85%
-        
+        memoryAllocationRatio = 0.2; // 只使用20%的总内存
-        if (isSmallFile) {
-            // 小文件使用无损压缩策略
-            await sharpInstance
-                .png({
-                    compressionLevel: 9,
-                    effort: 10,
-                    palette: false // 不使用调色板模式
-                })
-                .toFile(outputPath);
-        } else {
-            // 大文件使用有损压缩策略
-            await sharpInstance
-                .png({
-                    quality: 80,
-                    compressionLevel: 9,
-                    palette: true,
-                    colors: 256,
-                    dither: 0.5,
-                    effort: 10
-                })
-                .toFile(outputPath);
-        }
-
-        const outputStats = await fs.stat(outputPath);
-        const savings = ((inputStats.size - outputStats.size) / inputStats.size * 100).toFixed(2);
-        
-        // 如果优化后文件更大，使用原始文件
-        if (outputStats.size > inputStats.size) {
-            await fs.copyFile(inputPath, outputPath);
-            console.log(`⚠️ ${relativePath} (使用原始文件)`);
-        } else {
-            console.log(`✓ ${relativePath}`);
-        }
-        
-        // 更新统计数据
-        totalOriginalSize += inputStats.size;
-        totalOptimizedSize += Math.min(inputStats.size, outputStats.size);
-        successCount++;
-        
-        console.log(`  原始大小: ${(inputStats.size / 1024).toFixed(2)}KB`);
-        console.log(`  优化大小: ${(Math.min(inputStats.size, outputStats.size) / 1024).toFixed(2)}KB`);
-        console.log(`  节省: ${((inputStats.size - Math.min(inputStats.size, outputStats.size)) / inputStats.size * 100).toFixed(2)}%\n`);
-    } catch (error) {
-        console.error(`✗ 处理 ${relativePath} 时出错:`, error);
-        failCount++;
     }
-}
 
-function printReport() {
+    // 估算单个文件处理的平均内存占用（根据文件大小动态调整）
-    const endTime = Date.now();
+    const estimatedMemoryPerFile = Math.max(
-    const duration = ((endTime - startTime) / 1000).toFixed(2);
+        512 * 1024, // 最小512KB
-    const totalSavings = ((totalOriginalSize - totalOptimizedSize) / totalOriginalSize * 100).toFixed(2);
+        Math.min(
+            2 * 1024 * 1024, // 最大2MB
+            Math.floor(totalMemory / (1024 * cpuCount)) // 根据系统配置动态计算
+        )
+    );
     
-    console.log('\n========== 优化结果报告 ==========');
+    // 计算批处理大小
-    console.log(`处理总文件数: ${totalFiles}`);
+    const memoryForProcessing = totalMemory * memoryAllocationRatio;
-    console.log(`成功处理: ${successCount} 个文件`);
+    let batchSize = Math.floor(memoryForProcessing / (estimatedMemoryPerFile * cpuCount));
-    console.log(`处理失败: ${failCount} 个文件`);
+    
-    console.log(`原始总大小: ${(totalOriginalSize / 1024 / 1024).toFixed(2)}MB`);
+    // 动态调整批处理大小的上下限
-    console.log(`优化后总大小: ${(totalOptimizedSize / 1024 / 1024).toFixed(2)}MB`);
+    const minBatchSize = Math.max(50, Math.floor(200 / cpuCount)); // 确保每个CPU至少处理50个文件
-    console.log(`总体积减少: ${totalSavings}%`);
+    const maxBatchSize = Math.min(
-    console.log(`处理耗时: ${duration}秒`);
+        2000, // 硬上限
-    console.log(`CPU核心数: ${os.cpus().length}`);
+        Math.floor(memoryForProcessing / (512 * 1024)) // 根据分配内存动态计算上限
-    console.log('================================\n');
+    );
-}
+    batchSize = Math.max(minBatchSize, Math.min(maxBatchSize, batchSize));
+    
+    // 优化工作线程数量计算
+    let workerCount;
+    const maxThreads = cpuCount * 2; // 最大允许CPU核心数的2倍线程
+    const systemLoad = os.loadavg()[0] / cpuCount;
+    const memoryConstraint = memoryUsageRatio > 0.85; // 改为85%
 
-async function processImages() {
+    if (memoryConstraint) {
-    try {
+        // 内存压力大时，限制线程数
-        // 清空输出目录
+        workerCount = Math.max(2, Math.min(cpuCount - 1, 4));
-        await fs.rm(outputDir, { recursive: true, force: true });
+    } else if (systemLoad < 0.5) {
-        await fs.mkdir(outputDir, { recursive: true });
+        // 系统负载很低，可以用最大线程数
-        
+        workerCount = Math.max(2, Math.min(maxThreads, 6));
-        // 获取所有PNG文件
+    } else if (systemLoad < 1.0) {
-        const files = await glob(path.join(sourceDir, '**/*.png'));
+        // 系统负载适中，使用较多线程
-        totalFiles = files.length;
+        workerCount = Math.max(2, Math.min(cpuCount + 2, 6));
-        
+    } else {
-        console.log(`找到 ${files.length} 个PNG文件需要优化\n`);
+        // 系统负载较高，但仍有余力
-        
+        workerCount = Math.max(2, Math.min(cpuCount, 4));
-        // 使用工作池并行处理图片，限制并发数为CPU核心数
-        const concurrency = os.cpus().length;
-        const chunks = [];
-        for (let i = 0; i < files.length; i += concurrency) {
-            chunks.push(files.slice(i, i + concurrency));
-        }
-        
-        for (const chunk of chunks) {
-            await Promise.all(chunk.map(file => optimizeImage(file)));
-        }
-        
-        printReport();
-    } catch (error) {
-        console.error('处理过程中发生错误:', error);
     }
+
+    // 根据批处理大小调整线程数
+    // 如果批次太小，减少线程数以避免线程切换开销
+    if (batchSize < 100) {
+        workerCount = Math.min(workerCount, 2);
+    } else if (batchSize < 200) {
+        workerCount = Math.min(workerCount, 3);
+    }
+    
+    // 检测是否在RAM disk上
+    const isRamDisk = await checkIfRamDisk(outputDir);
+    
+    return {
+        batchSize,
+        workerCount,
+        isRamDisk,
+        systemInfo: {
+            totalMemory: formatBytes(totalMemory),
+            freeMemory: formatBytes(freeMemory),
+            cpuCount,
+            cpuUsage: (cpuUsage * 100).toFixed(1) + '%',
+            memoryUsage: (memoryUsageRatio * 100).toFixed(1) + '%',
+            memoryAllocationRatio: (memoryAllocationRatio * 100).toFixed(1) + '%',
+            estimatedMemoryPerFile: formatBytes(estimatedMemoryPerFile)
+        }
+    };
 }
 
-processImages(); 
+// 检查目录是否在RAM disk上
+async function checkIfRamDisk(dir) {
+    try {
+        if (process.platform === 'darwin') {
+            // macOS: 检查是否在 /Volumes/RAMDisk
+            return dir.startsWith('/Volumes/RAMDisk');
+        } else if (process.platform === 'linux') {
+            // Linux: 检查是否在 tmpfs
+            const { stdout } = await import('child_process').then(cp => 
+                new Promise((resolve) => {
+                    cp.exec(`df -T "${dir}" | grep tmpfs`, (error, stdout) => resolve({ stdout }));
+                })
+            );
+            return stdout.includes('tmpfs');
+        }
+    } catch (error) {
+        return false;
+    }
+    return false;
+}
+
+// 格式化字节数
+function formatBytes(bytes) {
+    const units = ['B', 'KB', 'MB', 'GB'];
+    let size = bytes;
+    let unitIndex = 0;
+    while (size >= 1024 && unitIndex < units.length - 1) {
+        size /= 1024;
+        unitIndex++;
+    }
+    return `${size.toFixed(2)}${units[unitIndex]}`;
+}
+
+// 工作线程逻辑
+if (!isMainThread) {
+    const { files, sourceDir, outputDir, isRamDisk } = workerData;
+    
+    async function optimizeImage(inputPath) {
+        const relativePath = path.relative(sourceDir, inputPath);
+        const outputPath = path.join(outputDir, relativePath);
+        const outputDirPath = path.dirname(outputPath);
+        
+        try {
+            await fs.mkdir(outputDirPath, { recursive: true });
+            const inputStats = await fs.stat(inputPath);
+            const isSmallFile = inputStats.size < SMALL_FILE_THRESHOLD;
+            
+            // 根据是否在RAM disk上调整缓冲策略
+            const sharpOptions = {
+                failOnError: false,
+                limitInputPixels: false,
+                sequentialRead: !isRamDisk, // 如果不是RAM disk，使用顺序读取
+            };
+            
+            let sharpInstance = sharp(inputPath, sharpOptions);
+            
+            if (isSmallFile) {
+                await sharpInstance
+                    .png({
+                        compressionLevel: 9,
+                        effort: 10,
+                        palette: true,
+                        colors: 128,
+                        dither: 0.4
+                    })
+                    .toFile(outputPath);
+            } else {
+                await sharpInstance
+                    .png({
+                        quality: 70,
+                        compressionLevel: 9,
+                        palette: true,
+                        colors: 128,
+                        dither: 0.4,
+                        effort: 10
+                    })
+                    .toFile(outputPath);
+            }
+            
+            const outputStats = await fs.stat(outputPath);
+            if (outputStats.size > inputStats.size) {
+                await fs.copyFile(inputPath, outputPath);
+            }
+            
+            return {
+                success: true,
+                inputSize: inputStats.size,
+                outputSize: Math.min(inputStats.size, outputStats.size),
+                path: relativePath
+            };
+        } catch (error) {
+            return {
+                success: false,
+                path: relativePath,
+                error: error.message
+            };
+        }
+    }
+    
+    Promise.all(files.map(file => optimizeImage(file)))
+        .then(results => parentPort.postMessage(results));
+}
+
+// 主线程逻辑
+else {
+    async function processImages() {
+        try {
+            // 计算最佳参数
+            const optimalParams = await calculateOptimalParams();
+            console.log('\n========== 系统资源信息 ==========');
+            console.log(`总内存: ${optimalParams.systemInfo.totalMemory}`);
+            console.log(`可用内存: ${optimalParams.systemInfo.freeMemory}`);
+            console.log(`CPU核心数: ${optimalParams.systemInfo.cpuCount}`);
+            console.log(`CPU使用率: ${optimalParams.systemInfo.cpuUsage}`);
+            console.log(`内存使用率: ${optimalParams.systemInfo.memoryUsage}`);
+            console.log(`内存分配比例: ${optimalParams.systemInfo.memoryAllocationRatio}`);
+            console.log(`估计每个文件内存占用: ${optimalParams.systemInfo.estimatedMemoryPerFile}`);
+            console.log(`优化批次大小: ${optimalParams.batchSize}`);
+            console.log(`工作线程数: ${optimalParams.workerCount}`);
+            console.log(`RAM Disk: ${optimalParams.isRamDisk ? '是' : '否'}`);
+            console.log('==================================\n');
+            
+            // 清空输出目录
+            await fs.rm(outputDir, { recursive: true, force: true });
+            await fs.mkdir(outputDir, { recursive: true });
+            
+            // 获取所有PNG文件
+            const files = await glob(path.join(sourceDir, '**/*.png'));
+            const totalFiles = files.length;
+            
+            console.log(`找到 ${files.length} 个PNG文件需要优化\n`);
+            
+            // 初始化统计数据
+            let totalOriginalSize = 0;
+            let totalOptimizedSize = 0;
+            let successCount = 0;
+            let failCount = 0;
+            const startTime = Date.now();
+            
+            const results = [];
+            
+            // 将文件分成多个批次
+            const batches = [];
+            for (let i = 0; i < files.length; i += optimalParams.batchSize) {
+                batches.push(files.slice(i, Math.min(i + optimalParams.batchSize, files.length)));
+            }
+            
+            let batchIndex = 0;
+            
+            // 处理每个批次
+            const processBatch = async () => {
+                if (batchIndex >= batches.length) return null;
+                
+                const currentBatch = batches[batchIndex++];
+                const worker = new Worker(new URL(import.meta.url), {
+                    workerData: {
+                        files: currentBatch,
+                        sourceDir,
+                        outputDir,
+                        isRamDisk: optimalParams.isRamDisk
+                    }
+                });
+                
+                return new Promise((resolve, reject) => {
+                    worker.on('message', (batchResults) => {
+                        results.push(...batchResults);
+                        
+                        // 更新进度
+                        const progress = Math.round((results.length / totalFiles) * 100);
+                        const elapsedTime = ((Date.now() - startTime) / 1000).toFixed(1);
+                        const estimatedTotal = (elapsedTime / progress * 100).toFixed(1);
+                        process.stdout.write(`\r处理进度: ${progress}% (${results.length}/${totalFiles}) - 已用时间: ${elapsedTime}秒 - 预计总时间: ${estimatedTotal}秒`);
+                        
+                        worker.terminate();
+                        resolve();
+                    });
+                    
+                    worker.on('error', (err) => {
+                        worker.terminate();
+                        reject(err);
+                    });
+                    
+                    worker.on('exit', (code) => {
+                        if (code !== 0 && !worker.exitCode) {
+                            reject(new Error(`Worker stopped with exit code ${code}`));
+                        }
+                    });
+                });
+            };
+            
+            // 并行处理所有批次
+            while (batchIndex < batches.length) {
+                const workerPromises = [];
+                for (let i = 0; i < optimalParams.workerCount && batchIndex < batches.length; i++) {
+                    workerPromises.push(processBatch());
+                }
+                await Promise.all(workerPromises);
+            }
+            
+            // 统计结果
+            for (const result of results) {
+                if (result.success) {
+                    successCount++;
+                    totalOriginalSize += result.inputSize;
+                    totalOptimizedSize += result.outputSize;
+                } else {
+                    failCount++;
+                    console.error(`\n✗ 处理 ${result.path} 时出错:`, result.error);
+                }
+            }
+            
+            // 打印报告
+            const endTime = Date.now();
+            const duration = ((endTime - startTime) / 1000).toFixed(2);
+            const totalSavings = ((totalOriginalSize - totalOptimizedSize) / totalOriginalSize * 100).toFixed(2);
+            
+            console.log('\n\n========== 优化结果报告 ==========');
+            console.log(`处理总文件数: ${totalFiles}`);
+            console.log(`成功处理: ${successCount} 个文件`);
+            console.log(`处理失败: ${failCount} 个文件`);
+            console.log(`原始总大小: ${(totalOriginalSize / 1024 / 1024).toFixed(2)}MB`);
+            console.log(`优化后总大小: ${(totalOptimizedSize / 1024 / 1024).toFixed(2)}MB`);
+            console.log(`总体积减少: ${totalSavings}%`);
+            console.log(`处理耗时: ${duration}秒`);
+            console.log(`平均处理速度: ${(totalFiles / duration).toFixed(2)}个/秒`);
+            console.log('================================\n');
+            
+        } catch (error) {
+            console.error('处理过程中发生错误:', error);
+        }
+    }
+    
+    processImages();
+}