1
0
mirror of synced 2026-05-21 01:36:28 +08:00

feat(cook): 支持菜谱数据分片加载与增量更新

- CLI 转换命令新增 --chunk-size 参数,支持将菜谱数据拆分为多个分片文件
- 前端 DB 初始化重构,支持单文件和分片两种模式
- 分片模式下通过 hash 比对实现增量更新,减少重复数据加载
- .gitignore 新增 recipe-meta.json 和 recipe-chunk-*.json 忽略规则
- 升级 vite、vitest 等依赖
This commit is contained in:
YunYouJun
2026-03-28 23:12:23 +08:00
parent 0ee8941115
commit 7b4c1a6029
9 changed files with 484 additions and 309 deletions

View File

@@ -14,7 +14,7 @@
"dependencies": {
"@clack/prompts": "^1.1.0",
"@cook/types": "workspace:*",
"@larksuiteoapi/node-sdk": "^1.59.0",
"@larksuiteoapi/node-sdk": "^1.60.0",
"cac": "^7.0.0",
"consola": "^3.4.2",
"papaparse": "^5.5.3"

View File

@@ -1,11 +1,15 @@
import crypto from 'node:crypto'
import fs from 'node:fs'
import path from 'node:path'
import consola from 'consola'
import {
CHUNK_SIZE,
CHUNK_THRESHOLD,
incompatibleFoodsCsvFile,
incompatibleFoodsJsonFile,
recipeCsvFile,
recipeJsonFile,
recipeMetaJsonFile,
rootDir,
} from '../utils/config.js'
import { parseIncompatibleFoodsCsv, parseRecipeCsv } from '../utils/csv.js'
@@ -13,9 +17,30 @@ import { parseIncompatibleFoodsCsv, parseRecipeCsv } from '../utils/csv.js'
// 正则表达式预编译,避免每次调用时重新编译
const DATE_SLASH_REGEX = /\//g
const LAST_DB_UPDATED_REGEX = /export const lastDbUpdated = '.+'/
const CHUNK_FILE_REGEX = /^recipe-chunk-\d+\.json$/
/**
* 转换 recipe CSV → JSON
* 清理旧的分片文件
*/
function cleanChunkFiles() {
const dataDir = path.dirname(recipeJsonFile)
const files = fs.readdirSync(dataDir)
for (const file of files) {
if (CHUNK_FILE_REGEX.test(file)) {
fs.unlinkSync(path.join(dataDir, file))
}
}
}
/**
* 计算内容的短 hashMD5 前 8 位)
*/
function shortHash(content: string): string {
return crypto.createHash('md5').update(content).digest('hex').slice(0, 8)
}
/**
* 转换 recipe CSV → JSON支持分片输出
*/
function convertRecipes() {
consola.info('Converting recipe data...')
@@ -28,8 +53,50 @@ function convertRecipes() {
return
}
fs.writeFileSync(recipeJsonFile, JSON.stringify(recipes))
consola.success(`Generated: ${recipeJsonFile} (${recipes.length} recipes)`)
const dataDir = path.dirname(recipeJsonFile)
if (recipes.length > CHUNK_THRESHOLD) {
// 分片模式:删除旧文件,按 CHUNK_SIZE 分片写入
cleanChunkFiles()
if (fs.existsSync(recipeJsonFile)) {
fs.unlinkSync(recipeJsonFile)
}
const chunks: { index: number, hash: string, count: number }[] = []
for (let i = 0; i < recipes.length; i += CHUNK_SIZE) {
const chunkIndex = Math.floor(i / CHUNK_SIZE)
const chunk = recipes.slice(i, i + CHUNK_SIZE)
const content = JSON.stringify(chunk)
const chunkFile = path.join(dataDir, `recipe-chunk-${chunkIndex}.json`)
fs.writeFileSync(chunkFile, content)
chunks.push({ index: chunkIndex, hash: shortHash(content), count: chunk.length })
consola.success(`Generated: ${chunkFile} (${chunk.length} recipes)`)
}
const meta = {
chunked: true,
total: recipes.length,
chunkSize: CHUNK_SIZE,
chunks,
version: Date.now(),
}
fs.writeFileSync(recipeMetaJsonFile, JSON.stringify(meta, null, 2))
consola.success(`Generated: ${recipeMetaJsonFile} (${chunks.length} chunks, ${recipes.length} total)`)
}
else {
// 单文件模式:保持原有行为,同时生成 meta
cleanChunkFiles()
fs.writeFileSync(recipeJsonFile, JSON.stringify(recipes))
consola.success(`Generated: ${recipeJsonFile} (${recipes.length} recipes)`)
const meta = {
chunked: false,
total: recipes.length,
version: Date.now(),
}
fs.writeFileSync(recipeMetaJsonFile, JSON.stringify(meta, null, 2))
consola.success(`Generated: ${recipeMetaJsonFile}`)
}
}
/**

View File

@@ -15,9 +15,14 @@ export const envFile = path.resolve(root, '.env')
// Data 文件路径
export const recipeCsvFile = path.resolve(root, 'app/data/recipe.csv')
export const recipeJsonFile = path.resolve(root, 'app/data/recipe.json')
export const recipeMetaJsonFile = path.resolve(root, 'app/data/recipe-meta.json')
export const incompatibleFoodsCsvFile = path.resolve(root, 'app/data/incompatible-foods.csv')
export const incompatibleFoodsJsonFile = path.resolve(root, 'app/data/incompatible-foods.json')
// 分片配置
export const CHUNK_SIZE = 200 // 每片菜谱数
export const CHUNK_THRESHOLD = 1000 // 超过此数量启用分片
// CSV Headers
export const RECIPE_CSV_HEADERS = 'name,stuff,bv,difficulty,tags,methods,tools,'
export const INCOMPATIBLE_FOODS_CSV_HEADERS = 'foodA,foodB,reason'