1
0
mirror of synced 2026-05-20 17:28:31 +08:00

feat(cook): 支持菜谱数据分片加载与增量更新

- CLI 转换命令新增 --chunk-size 参数,支持将菜谱数据拆分为多个分片文件
- 前端 DB 初始化重构,支持单文件和分片两种模式
- 分片模式下通过 hash 比对实现增量更新,减少重复数据加载
- .gitignore 新增 recipe-meta.json 和 recipe-chunk-*.json 忽略规则
- 升级 vite、vitest 等依赖
This commit is contained in:
YunYouJun
2026-03-28 23:12:23 +08:00
parent 0ee8941115
commit 7b4c1a6029
9 changed files with 484 additions and 309 deletions

2
.gitignore vendored
View File

@@ -1,5 +1,7 @@
# auto generate
app/data/recipe.json
app/data/recipe-meta.json
app/data/recipe-chunk-*.json
app/data/incompatible-foods.json
.DS_Store

View File

@@ -2,7 +2,7 @@ export const appName = '食用手册'
export const appDescription = '好的,今天我们来做菜!'
export const namespace = 'cook'
export const lastDbUpdated = '2026-3-25 02:52:00'
export const lastDbUpdated = '2026-3-26 16:45:17'
export const icp = '苏ICP备17038157号'

View File

@@ -7,6 +7,28 @@ export interface DbRecipeItem extends RecipeItem {
id?: number
}
interface RecipeMetaBase {
total: number
version: number
}
interface RecipeMetaSingle extends RecipeMetaBase {
chunked: false
}
interface RecipeMetaChunked extends RecipeMetaBase {
chunked: true
chunkSize: number
chunks: { index: number, hash: string, count: number }[]
}
type RecipeMeta = RecipeMetaSingle | RecipeMetaChunked
const CHUNK_HASH_STORAGE_KEY = 'cook:chunk-hashes'
// 预注册分片文件Vite 会在构建时处理这些 glob 导入
const chunkModules = import.meta.glob<{ default: RecipeItem[] }>('../data/recipe-chunk-*.json')
export class MySubClassedDexie extends Dexie {
recipes!: Table<DbRecipeItem>
@@ -20,7 +42,30 @@ export class MySubClassedDexie extends Dexie {
export const db = new MySubClassedDexie()
export async function initDb() {
/**
* 加载已记录的分片 hash用于增量更新判断
*/
function loadChunkHashes(): Record<string, string> {
try {
const raw = localStorage.getItem(CHUNK_HASH_STORAGE_KEY)
return raw ? JSON.parse(raw) : {}
}
catch {
return {}
}
}
/**
* 保存分片 hash 记录
*/
function saveChunkHashes(hashes: Record<string, string>) {
localStorage.setItem(CHUNK_HASH_STORAGE_KEY, JSON.stringify(hashes))
}
/**
* 单文件模式:全量加载 recipe.json
*/
async function loadSingleFile() {
const { default: recipeData } = await import('../data/recipe.json')
return db.recipes.bulkPut(
@@ -30,3 +75,57 @@ export async function initDb() {
})),
)
}
/**
* 分片模式:增量加载变化的分片
*/
async function loadChunks(meta: RecipeMetaChunked) {
const savedHashes = loadChunkHashes()
const newHashes = { ...savedHashes }
// 找出需要加载的分片hash 变化或新增的)
const chunksToLoad = meta.chunks.filter(
chunk => savedHashes[String(chunk.index)] !== chunk.hash,
)
if (chunksToLoad.length === 0) {
return
}
// 并行加载所有需要更新的分片
await Promise.all(
chunksToLoad.map(async (chunk) => {
const modulePath = `../data/recipe-chunk-${chunk.index}.json`
const loader = chunkModules[modulePath]
if (!loader) {
console.warn(`Chunk module not found: ${modulePath}`)
return
}
const { default: chunkData } = await loader()
const offset = chunk.index * meta.chunkSize
await db.recipes.bulkPut(
(chunkData as RecipeItem[]).map((item, i) => ({
id: offset + i,
...item,
})),
)
newHashes[String(chunk.index)] = chunk.hash
}),
)
saveChunkHashes(newHashes)
}
export async function initDb() {
const { default: meta } = await import('../data/recipe-meta.json') as { default: RecipeMeta }
if (meta.chunked) {
return loadChunks(meta)
}
else {
return loadSingleFile()
}
}

View File

@@ -22,6 +22,6 @@
"unplugin-vue-components": "^32.0.0",
"vite-plugin-vue-devtools": "^8.1.1",
"vitepress": "^2.0.0-alpha.17",
"vitepress-plugin-group-icons": "^1.7.1"
"vitepress-plugin-group-icons": "^1.7.3"
}
}

View File

@@ -54,7 +54,7 @@
"@capacitor/cli": "7.4.3",
"@capacitor/dialog": "^8.0.1",
"@headlessui/vue": "^1.7.23",
"@iconify-json/carbon": "^1.2.19",
"@iconify-json/carbon": "^1.2.20",
"@iconify-json/fe": "^1.2.4",
"@iconify-json/gg": "^1.2.2",
"@iconify-json/ic": "^1.2.4",
@@ -75,12 +75,13 @@
"@unocss/nuxt": "^66.6.7",
"@vite-pwa/nuxt": "^1.1.1",
"@vue/test-utils": "^2.4.6",
"@vueuse/core": "^14.2.1",
"@vueuse/nuxt": "^14.2.1",
"@yunlefun/vue": "^0.1.1",
"baseline-browser-mapping": "^2.10.10",
"baseline-browser-mapping": "^2.10.12",
"bumpp": "^11.0.1",
"consola": "^3.4.2",
"dexie": "^4.3.0",
"dexie": "^4.4.1",
"eslint": "^10.1.0",
"eslint-plugin-format": "^2.0.1",
"fake-indexeddb": "^6.2.5",
@@ -96,12 +97,12 @@
"tsx": "^4.21.0",
"typescript": "^5.9.3",
"unocss": "^66.6.7",
"vitest": "^4.1.1",
"vitest": "^4.1.2",
"vue-tsc": "^3.2.6"
},
"resolutions": {
"unplugin": "^3.0.0",
"vite": "^8.0.2"
"vite": "^8.0.3"
},
"simple-git-hooks": {
"pre-commit": "pnpm lint-staged"

View File

@@ -14,7 +14,7 @@
"dependencies": {
"@clack/prompts": "^1.1.0",
"@cook/types": "workspace:*",
"@larksuiteoapi/node-sdk": "^1.59.0",
"@larksuiteoapi/node-sdk": "^1.60.0",
"cac": "^7.0.0",
"consola": "^3.4.2",
"papaparse": "^5.5.3"

View File

@@ -1,11 +1,15 @@
import crypto from 'node:crypto'
import fs from 'node:fs'
import path from 'node:path'
import consola from 'consola'
import {
CHUNK_SIZE,
CHUNK_THRESHOLD,
incompatibleFoodsCsvFile,
incompatibleFoodsJsonFile,
recipeCsvFile,
recipeJsonFile,
recipeMetaJsonFile,
rootDir,
} from '../utils/config.js'
import { parseIncompatibleFoodsCsv, parseRecipeCsv } from '../utils/csv.js'
@@ -13,9 +17,30 @@ import { parseIncompatibleFoodsCsv, parseRecipeCsv } from '../utils/csv.js'
// 正则表达式预编译,避免每次调用时重新编译
const DATE_SLASH_REGEX = /\//g
const LAST_DB_UPDATED_REGEX = /export const lastDbUpdated = '.+'/
const CHUNK_FILE_REGEX = /^recipe-chunk-\d+\.json$/
/**
* 转换 recipe CSV → JSON
* 清理旧的分片文件
*/
function cleanChunkFiles() {
const dataDir = path.dirname(recipeJsonFile)
const files = fs.readdirSync(dataDir)
for (const file of files) {
if (CHUNK_FILE_REGEX.test(file)) {
fs.unlinkSync(path.join(dataDir, file))
}
}
}
/**
* 计算内容的短 hashMD5 前 8 位)
*/
function shortHash(content: string): string {
return crypto.createHash('md5').update(content).digest('hex').slice(0, 8)
}
/**
* 转换 recipe CSV → JSON支持分片输出
*/
function convertRecipes() {
consola.info('Converting recipe data...')
@@ -28,8 +53,50 @@ function convertRecipes() {
return
}
const dataDir = path.dirname(recipeJsonFile)
if (recipes.length > CHUNK_THRESHOLD) {
// 分片模式:删除旧文件,按 CHUNK_SIZE 分片写入
cleanChunkFiles()
if (fs.existsSync(recipeJsonFile)) {
fs.unlinkSync(recipeJsonFile)
}
const chunks: { index: number, hash: string, count: number }[] = []
for (let i = 0; i < recipes.length; i += CHUNK_SIZE) {
const chunkIndex = Math.floor(i / CHUNK_SIZE)
const chunk = recipes.slice(i, i + CHUNK_SIZE)
const content = JSON.stringify(chunk)
const chunkFile = path.join(dataDir, `recipe-chunk-${chunkIndex}.json`)
fs.writeFileSync(chunkFile, content)
chunks.push({ index: chunkIndex, hash: shortHash(content), count: chunk.length })
consola.success(`Generated: ${chunkFile} (${chunk.length} recipes)`)
}
const meta = {
chunked: true,
total: recipes.length,
chunkSize: CHUNK_SIZE,
chunks,
version: Date.now(),
}
fs.writeFileSync(recipeMetaJsonFile, JSON.stringify(meta, null, 2))
consola.success(`Generated: ${recipeMetaJsonFile} (${chunks.length} chunks, ${recipes.length} total)`)
}
else {
// 单文件模式:保持原有行为,同时生成 meta
cleanChunkFiles()
fs.writeFileSync(recipeJsonFile, JSON.stringify(recipes))
consola.success(`Generated: ${recipeJsonFile} (${recipes.length} recipes)`)
const meta = {
chunked: false,
total: recipes.length,
version: Date.now(),
}
fs.writeFileSync(recipeMetaJsonFile, JSON.stringify(meta, null, 2))
consola.success(`Generated: ${recipeMetaJsonFile}`)
}
}
/**

View File

@@ -15,9 +15,14 @@ export const envFile = path.resolve(root, '.env')
// Data 文件路径
export const recipeCsvFile = path.resolve(root, 'app/data/recipe.csv')
export const recipeJsonFile = path.resolve(root, 'app/data/recipe.json')
export const recipeMetaJsonFile = path.resolve(root, 'app/data/recipe-meta.json')
export const incompatibleFoodsCsvFile = path.resolve(root, 'app/data/incompatible-foods.csv')
export const incompatibleFoodsJsonFile = path.resolve(root, 'app/data/incompatible-foods.json')
// 分片配置
export const CHUNK_SIZE = 200 // 每片菜谱数
export const CHUNK_THRESHOLD = 1000 // 超过此数量启用分片
// CSV Headers
export const RECIPE_CSV_HEADERS = 'name,stuff,bv,difficulty,tags,methods,tools,'
export const INCOMPATIBLE_FOODS_CSV_HEADERS = 'foodA,foodB,reason'

595
pnpm-lock.yaml generated

File diff suppressed because it is too large Load Diff