feat(cook): 支持菜谱数据分片加载与增量更新
- CLI 转换命令新增 --chunk-size 参数,支持将菜谱数据拆分为多个分片文件 - 前端 DB 初始化重构,支持单文件和分片两种模式 - 分片模式下通过 hash 比对实现增量更新,减少重复数据加载 - .gitignore 新增 recipe-meta.json 和 recipe-chunk-*.json 忽略规则 - 升级 vite、vitest 等依赖
This commit is contained in:
@@ -14,7 +14,7 @@
|
||||
"dependencies": {
|
||||
"@clack/prompts": "^1.1.0",
|
||||
"@cook/types": "workspace:*",
|
||||
"@larksuiteoapi/node-sdk": "^1.59.0",
|
||||
"@larksuiteoapi/node-sdk": "^1.60.0",
|
||||
"cac": "^7.0.0",
|
||||
"consola": "^3.4.2",
|
||||
"papaparse": "^5.5.3"
|
||||
|
||||
@@ -1,11 +1,15 @@
|
||||
import crypto from 'node:crypto'
|
||||
import fs from 'node:fs'
|
||||
import path from 'node:path'
|
||||
import consola from 'consola'
|
||||
import {
|
||||
CHUNK_SIZE,
|
||||
CHUNK_THRESHOLD,
|
||||
incompatibleFoodsCsvFile,
|
||||
incompatibleFoodsJsonFile,
|
||||
recipeCsvFile,
|
||||
recipeJsonFile,
|
||||
recipeMetaJsonFile,
|
||||
rootDir,
|
||||
} from '../utils/config.js'
|
||||
import { parseIncompatibleFoodsCsv, parseRecipeCsv } from '../utils/csv.js'
|
||||
@@ -13,9 +17,30 @@ import { parseIncompatibleFoodsCsv, parseRecipeCsv } from '../utils/csv.js'
|
||||
// 正则表达式预编译,避免每次调用时重新编译
|
||||
const DATE_SLASH_REGEX = /\//g
|
||||
const LAST_DB_UPDATED_REGEX = /export const lastDbUpdated = '.+'/
|
||||
const CHUNK_FILE_REGEX = /^recipe-chunk-\d+\.json$/
|
||||
|
||||
/**
|
||||
* 转换 recipe CSV → JSON
|
||||
* 清理旧的分片文件
|
||||
*/
|
||||
function cleanChunkFiles() {
|
||||
const dataDir = path.dirname(recipeJsonFile)
|
||||
const files = fs.readdirSync(dataDir)
|
||||
for (const file of files) {
|
||||
if (CHUNK_FILE_REGEX.test(file)) {
|
||||
fs.unlinkSync(path.join(dataDir, file))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 计算内容的短 hash(MD5 前 8 位)
|
||||
*/
|
||||
function shortHash(content: string): string {
|
||||
return crypto.createHash('md5').update(content).digest('hex').slice(0, 8)
|
||||
}
|
||||
|
||||
/**
|
||||
* 转换 recipe CSV → JSON(支持分片输出)
|
||||
*/
|
||||
function convertRecipes() {
|
||||
consola.info('Converting recipe data...')
|
||||
@@ -28,8 +53,50 @@ function convertRecipes() {
|
||||
return
|
||||
}
|
||||
|
||||
fs.writeFileSync(recipeJsonFile, JSON.stringify(recipes))
|
||||
consola.success(`Generated: ${recipeJsonFile} (${recipes.length} recipes)`)
|
||||
const dataDir = path.dirname(recipeJsonFile)
|
||||
|
||||
if (recipes.length > CHUNK_THRESHOLD) {
|
||||
// 分片模式:删除旧文件,按 CHUNK_SIZE 分片写入
|
||||
cleanChunkFiles()
|
||||
if (fs.existsSync(recipeJsonFile)) {
|
||||
fs.unlinkSync(recipeJsonFile)
|
||||
}
|
||||
|
||||
const chunks: { index: number, hash: string, count: number }[] = []
|
||||
for (let i = 0; i < recipes.length; i += CHUNK_SIZE) {
|
||||
const chunkIndex = Math.floor(i / CHUNK_SIZE)
|
||||
const chunk = recipes.slice(i, i + CHUNK_SIZE)
|
||||
const content = JSON.stringify(chunk)
|
||||
const chunkFile = path.join(dataDir, `recipe-chunk-${chunkIndex}.json`)
|
||||
fs.writeFileSync(chunkFile, content)
|
||||
chunks.push({ index: chunkIndex, hash: shortHash(content), count: chunk.length })
|
||||
consola.success(`Generated: ${chunkFile} (${chunk.length} recipes)`)
|
||||
}
|
||||
|
||||
const meta = {
|
||||
chunked: true,
|
||||
total: recipes.length,
|
||||
chunkSize: CHUNK_SIZE,
|
||||
chunks,
|
||||
version: Date.now(),
|
||||
}
|
||||
fs.writeFileSync(recipeMetaJsonFile, JSON.stringify(meta, null, 2))
|
||||
consola.success(`Generated: ${recipeMetaJsonFile} (${chunks.length} chunks, ${recipes.length} total)`)
|
||||
}
|
||||
else {
|
||||
// 单文件模式:保持原有行为,同时生成 meta
|
||||
cleanChunkFiles()
|
||||
fs.writeFileSync(recipeJsonFile, JSON.stringify(recipes))
|
||||
consola.success(`Generated: ${recipeJsonFile} (${recipes.length} recipes)`)
|
||||
|
||||
const meta = {
|
||||
chunked: false,
|
||||
total: recipes.length,
|
||||
version: Date.now(),
|
||||
}
|
||||
fs.writeFileSync(recipeMetaJsonFile, JSON.stringify(meta, null, 2))
|
||||
consola.success(`Generated: ${recipeMetaJsonFile}`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -15,9 +15,14 @@ export const envFile = path.resolve(root, '.env')
|
||||
// Data 文件路径
|
||||
export const recipeCsvFile = path.resolve(root, 'app/data/recipe.csv')
|
||||
export const recipeJsonFile = path.resolve(root, 'app/data/recipe.json')
|
||||
export const recipeMetaJsonFile = path.resolve(root, 'app/data/recipe-meta.json')
|
||||
export const incompatibleFoodsCsvFile = path.resolve(root, 'app/data/incompatible-foods.csv')
|
||||
export const incompatibleFoodsJsonFile = path.resolve(root, 'app/data/incompatible-foods.json')
|
||||
|
||||
// 分片配置
|
||||
export const CHUNK_SIZE = 200 // 每片菜谱数
|
||||
export const CHUNK_THRESHOLD = 1000 // 超过此数量启用分片
|
||||
|
||||
// CSV Headers
|
||||
export const RECIPE_CSV_HEADERS = 'name,stuff,bv,difficulty,tags,methods,tools,'
|
||||
export const INCOMPATIBLE_FOODS_CSV_HEADERS = 'foodA,foodB,reason'
|
||||
|
||||
Reference in New Issue
Block a user