From 27b95ca5042cb6beb5af922d22a6d745892cfc85 Mon Sep 17 00:00:00 2001 From: lirtual Date: Sat, 21 Dec 2024 20:31:17 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E6=B7=BB=E5=8A=A0JSON=E6=96=87?= =?UTF-8?q?=E4=BB=B6=E4=BC=98=E5=8C=96=E8=84=9A=E6=9C=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/optimize-json.js | 301 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 301 insertions(+) create mode 100644 scripts/optimize-json.js diff --git a/scripts/optimize-json.js b/scripts/optimize-json.js new file mode 100644 index 00000000000..c6032685678 --- /dev/null +++ b/scripts/optimize-json.js @@ -0,0 +1,301 @@ +import { promises as fs } from 'fs'; +import path from 'path'; +import { fileURLToPath } from 'url'; +import { glob } from 'glob'; +import os from 'os'; +import { Worker, isMainThread, parentPort, workerData } from 'worker_threads'; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); + +const sourceDir = 'public'; +const SMALL_FILE_THRESHOLD = 5 * 1024; // 5KB + +// 动态计算最佳参数 +async function calculateOptimalParams() { + const totalMemory = os.totalmem(); + const freeMemory = os.freemem(); + const cpuCount = os.cpus().length; + const cpuUsage = os.loadavg()[0] / cpuCount; + + // 根据系统内存使用情况动态调整内存使用比例 + const memoryUsageRatio = 1 - (freeMemory / totalMemory); + let memoryAllocationRatio; + if (memoryUsageRatio < 0.7) { + memoryAllocationRatio = 0.4; + } else if (memoryUsageRatio < 0.85) { + memoryAllocationRatio = 0.3; + } else { + memoryAllocationRatio = 0.2; + } + + // 估算单个文件处理的平均内存占用 + const estimatedMemoryPerFile = Math.max( + 256 * 1024, // 最小256KB + Math.min( + 1 * 1024 * 1024, // 最大1MB + Math.floor(totalMemory / (1024 * cpuCount)) + ) + ); + + // 计算批处理大小 + const memoryForProcessing = totalMemory * memoryAllocationRatio; + let batchSize = Math.floor(memoryForProcessing / (estimatedMemoryPerFile * cpuCount)); + + // 动态调整批处理大小的上下限 + const minBatchSize = Math.max(50, Math.floor(200 / cpuCount)); + const maxBatchSize = Math.min( + 2000, + Math.floor(memoryForProcessing / (256 * 1024)) + ); + batchSize = Math.max(minBatchSize, Math.min(maxBatchSize, batchSize)); + + // 优化工作线程数量 + let workerCount; + const maxThreads = cpuCount * 2; + const systemLoad = os.loadavg()[0] / cpuCount; + const memoryConstraint = memoryUsageRatio > 0.85; + + if (memoryConstraint) { + workerCount = Math.max(2, Math.min(cpuCount - 1, 4)); + } else if (systemLoad < 0.5) { + workerCount = Math.max(2, Math.min(maxThreads, 6)); + } else if (systemLoad < 1.0) { + workerCount = Math.max(2, Math.min(cpuCount + 2, 6)); + } else { + workerCount = Math.max(2, Math.min(cpuCount, 4)); + } + + if (batchSize < 100) { + workerCount = Math.min(workerCount, 2); + } else if (batchSize < 200) { + workerCount = Math.min(workerCount, 3); + } + + return { + batchSize, + workerCount, + systemInfo: { + totalMemory: formatBytes(totalMemory), + freeMemory: formatBytes(freeMemory), + cpuCount, + cpuUsage: (cpuUsage * 100).toFixed(1) + '%', + memoryUsage: (memoryUsageRatio * 100).toFixed(1) + '%', + memoryAllocationRatio: (memoryAllocationRatio * 100).toFixed(1) + '%', + estimatedMemoryPerFile: formatBytes(estimatedMemoryPerFile) + } + }; +} + +// 格式化字节数 +function formatBytes(bytes) { + const units = ['B', 'KB', 'MB', 'GB']; + let size = bytes; + let unitIndex = 0; + while (size >= 1024 && unitIndex < units.length - 1) { + size /= 1024; + unitIndex++; + } + return `${size.toFixed(2)}${units[unitIndex]}`; +} + +// 工作线程逻辑 +if (!isMainThread) { + const { files, sourceDir } = workerData; + + async function optimizeJson(inputPath) { + const relativePath = path.relative(sourceDir, inputPath); + const tempPath = `${inputPath}.temp`; + + try { + const inputStats = await fs.stat(inputPath); + const isSmallFile = inputStats.size < SMALL_FILE_THRESHOLD; + + // 读取JSON文件 + const jsonContent = await fs.readFile(inputPath, 'utf8'); + let jsonData; + + try { + jsonData = JSON.parse(jsonContent); + } catch (error) { + return { + success: false, + path: relativePath, + error: '无效的JSON文件' + }; + } + + // 压缩JSON + const optimizedJson = JSON.stringify(jsonData); + + // 写入临时文件 + await fs.writeFile(tempPath, optimizedJson, 'utf8'); + + const outputStats = await fs.stat(tempPath); + if (outputStats.size < inputStats.size) { + // 如果优化后的文件更小，则替换原文件 + await fs.rename(tempPath, inputPath); + return { + success: true, + inputSize: inputStats.size, + outputSize: outputStats.size, + path: relativePath + }; + } else { + // 如果优化后的文件更大，则删除临时文件 + await fs.unlink(tempPath); + return { + success: true, + inputSize: inputStats.size, + outputSize: inputStats.size, + path: relativePath, + skipped: true + }; + } + } catch (error) { + // 清理临时文件 + try { + await fs.unlink(tempPath); + } catch {} + + return { + success: false, + path: relativePath, + error: error.message + }; + } + } + + Promise.all(files.map(file => optimizeJson(file))) + .then(results => parentPort.postMessage(results)); +} + +// 主线程逻辑 +else { + async function processJsonFiles() { + try { + // 计算最佳参数 + const optimalParams = await calculateOptimalParams(); + console.log('\n========== 系统资源信息 =========='); + console.log(`总内存: ${optimalParams.systemInfo.totalMemory}`); + console.log(`可用内存: ${optimalParams.systemInfo.freeMemory}`); + console.log(`CPU核心数: ${optimalParams.systemInfo.cpuCount}`); + console.log(`CPU使用率: ${optimalParams.systemInfo.cpuUsage}`); + console.log(`内存使用率: ${optimalParams.systemInfo.memoryUsage}`); + console.log(`内存分配比例: ${optimalParams.systemInfo.memoryAllocationRatio}`); + console.log(`估计每个文件内存占用: ${optimalParams.systemInfo.estimatedMemoryPerFile}`); + console.log(`优化批次大小: ${optimalParams.batchSize}`); + console.log(`工作线程数: ${optimalParams.workerCount}`); + console.log('==================================\n'); + + // 获取所有JSON文件 + const files = await glob(path.join(sourceDir, '**/*.json')); + const totalFiles = files.length; + + console.log(`找到 ${files.length} 个JSON文件需要优化\n`); + + // 初始化统计数据 + let totalOriginalSize = 0; + let totalOptimizedSize = 0; + let successCount = 0; + let failCount = 0; + let skippedCount = 0; + const startTime = Date.now(); + + const results = []; + + // 将文件分成多个批次 + const batches = []; + for (let i = 0; i < files.length; i += optimalParams.batchSize) { + batches.push(files.slice(i, Math.min(i + optimalParams.batchSize, files.length))); + } + + let batchIndex = 0; + + // 处理每个批次 + const processBatch = async () => { + if (batchIndex >= batches.length) return null; + + const currentBatch = batches[batchIndex++]; + const worker = new Worker(new URL(import.meta.url), { + workerData: { + files: currentBatch, + sourceDir + } + }); + + return new Promise((resolve, reject) => { + worker.on('message', (batchResults) => { + results.push(...batchResults); + + // 更新进度 + const progress = Math.round((results.length / totalFiles) * 100); + const elapsedTime = ((Date.now() - startTime) / 1000).toFixed(1); + const estimatedTotal = (elapsedTime / progress * 100).toFixed(1); + process.stdout.write(`\r处理进度: ${progress}% (${results.length}/${totalFiles}) - 已用时间: ${elapsedTime}秒 - 预计总时间: ${estimatedTotal}秒`); + + worker.terminate(); + resolve(); + }); + + worker.on('error', (err) => { + worker.terminate(); + reject(err); + }); + + worker.on('exit', (code) => { + if (code !== 0 && !worker.exitCode) { + reject(new Error(`Worker stopped with exit code ${code}`)); + } + }); + }); + }; + + // 并行处理所有批次 + while (batchIndex < batches.length) { + const workerPromises = []; + for (let i = 0; i < optimalParams.workerCount && batchIndex < batches.length; i++) { + workerPromises.push(processBatch()); + } + await Promise.all(workerPromises); + } + + // 统计结果 + for (const result of results) { + if (result.success) { + successCount++; + totalOriginalSize += result.inputSize; + totalOptimizedSize += result.outputSize; + if (result.skipped) { + skippedCount++; + } + } else { + failCount++; + console.error(`\n✗ 处理 ${result.path} 时出错:`, result.error); + } + } + + // 打印报告 + const endTime = Date.now(); + const duration = ((endTime - startTime) / 1000).toFixed(2); + const totalSavings = ((totalOriginalSize - totalOptimizedSize) / totalOriginalSize * 100).toFixed(2); + + console.log('\n\n========== 优化结果报告 =========='); + console.log(`处理总文件数: ${totalFiles}`); + console.log(`成功处理: ${successCount} 个文件`); + console.log(`跳过处理: ${skippedCount} 个文件（优化后体积更大）`); + console.log(`处理失败: ${failCount} 个文件`); + console.log(`原始总大小: ${(totalOriginalSize / 1024 / 1024).toFixed(2)}MB`); + console.log(`优化后总大小: ${(totalOptimizedSize / 1024 / 1024).toFixed(2)}MB`); + console.log(`总体积减少: ${totalSavings}%`); + console.log(`处理耗时: ${duration}秒`); + console.log(`平均处理速度: ${(totalFiles / duration).toFixed(2)}个/秒`); + console.log('================================\n'); + + } catch (error) { + console.error('处理过程中发生错误:', error); + } + } + + processJsonFiles(); +} \ No newline at end of file