data: scaffold word deck registry

This commit is contained in:
gitea-actions[bot]
2026-05-10 11:42:46 +09:00
commit 3b9e3a2d36
14 changed files with 700 additions and 0 deletions
+4
View File
@@ -0,0 +1,4 @@
__pycache__/
.pytest_cache/
.DS_Store
*.tmp
+63
View File
@@ -0,0 +1,63 @@
# Japanese Word Trainer Deck Registry
This repository is the public word-deck data registry for the 0.3 package model.
It keeps JSON as the source of truth and publishes zip package artifacts that the
app can download and verify.
## Layout
- `registry.json` is the root registry file read by the app.
- `sources/official/<slug>/` contains editable official deck package JSON.
- `sources/custom/<slug>/` contains editable published custom deck package JSON.
- `packages/*.zip` contains generated app-installable package artifacts.
- `scripts/build_packages.py` builds packages and refreshes registry checksums.
- `scripts/validate_registry.py` validates source JSON, packages, and checksums.
Each package zip contains exactly:
```text
manifest.json
words.json
refs.json
```
## Build
Run from the repository root:
```bash
python3 scripts/build_packages.py
python3 scripts/validate_registry.py
```
`build_packages.py` fills each word `contentHash`, creates deterministic zip
files in `packages/`, and updates `registry.json` with `wordCount`, `packageUrl`,
and `sha256`.
By default, `generatedAt` is left unchanged to avoid noisy rebuild diffs. Use
`--update-generated-at` when intentionally publishing a freshly generated
registry:
```bash
python3 scripts/build_packages.py --update-generated-at
```
## Package Rules
- Stable deck keys use `{registryId}:deck:{slug}`.
- Stable word keys use `{registryId}:word:{slug-or-id}`.
- `versionCode` changes when package content changes.
- `contentHash` is computed from a word record excluding `contentHash`.
- `refs.json` owns deck membership and display order.
- The same package shape is used for official, registry custom, imported, and
local-exported decks.
## Current Samples
- `official:deck:jlpt-n5` is a tiny official sample based lightly on the app
seed vocabulary. It is not the full JLPT N5 set.
- `official:deck:travel-basic` is a sample published custom deck under the
official registry.
The sample data is intentionally small so app migration, import, and checksum
handling can be tested before the full public deck corpus is populated.
Binary file not shown.
Binary file not shown.
+30
View File
@@ -0,0 +1,30 @@
{
"schemaVersion": 1,
"registryId": "official",
"registryName": "Japanese Word Trainer Official Registry",
"trustLevel": "OFFICIAL",
"generatedAt": 1778400000000,
"decks": [
{
"stableDeckKey": "official:deck:jlpt-n5",
"deckKind": "OFFICIAL",
"title": "JLPT N5 Sample",
"description": "Small JLPT N5 sample deck for validating the 0.3 package format.",
"versionCode": 1,
"wordCount": 5,
"packageUrl": "packages/jlpt-n5-v1.zip",
"sha256": "653f65d387965a0a57b70bfd52c6fb1dd75a4268d1c392f35f1463147fa395aa"
},
{
"stableDeckKey": "official:deck:travel-basic",
"deckKind": "CUSTOM",
"title": "Travel Basic",
"description": "Sample custom travel vocabulary deck published through the official registry.",
"versionCode": 1,
"wordCount": 3,
"packageUrl": "packages/travel-basic-v1.zip",
"sha256": "97450f805f3a4feb7c3f29cae44f60e69612087e751cdb7fe2f9db16c18b7f81"
}
],
"additionalRegistries": []
}
+95
View File
@@ -0,0 +1,95 @@
#!/usr/bin/env python3
"""Build deck package zips and update registry package metadata."""
from __future__ import annotations
import argparse
import time
from pathlib import Path
from deck_registry import (
ROOT,
dump_json,
load_json,
load_source,
materialize_words,
package_url_for,
sha256_file,
source_dirs,
validate_source,
write_package,
)
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
"--root",
type=Path,
default=ROOT,
help="Repository root. Defaults to the parent of scripts/.",
)
parser.add_argument(
"--update-generated-at",
action="store_true",
help="Set registry.generatedAt to the current Unix epoch milliseconds.",
)
return parser.parse_args()
def main() -> int:
args = parse_args()
root = args.root.resolve()
registry_path = root / "registry.json"
registry = load_json(registry_path)
registry_entries = {
deck["stableDeckKey"]: deck
for deck in registry.setdefault("decks", [])
if isinstance(deck, dict) and "stableDeckKey" in deck
}
for source_dir in source_dirs(root):
manifest, source_words, refs = load_source(source_dir)
errors = validate_source(source_dir, manifest, source_words, refs)
if errors:
for error in errors:
print(f"ERROR: {error}")
return 1
words = materialize_words(source_words)
if words != source_words:
dump_json(source_dir / "words.json", words)
package_url = package_url_for(manifest)
package_path = root / package_url
write_package(package_path, manifest, words, refs)
sha256 = sha256_file(package_path)
stable_deck_key = manifest["stableDeckKey"]
entry = registry_entries.get(stable_deck_key)
if entry is None:
entry = {"stableDeckKey": stable_deck_key}
registry["decks"].append(entry)
registry_entries[stable_deck_key] = entry
entry.update(
{
"deckKind": manifest["deckKind"],
"title": manifest["title"],
"description": manifest["description"],
"versionCode": manifest["versionCode"],
"wordCount": len(refs),
"packageUrl": package_url,
"sha256": sha256,
}
)
print(f"built {package_url} ({len(refs)} refs, sha256 {sha256})")
registry["decks"].sort(key=lambda deck: deck["stableDeckKey"])
if args.update_generated_at:
registry["generatedAt"] = int(time.time() * 1000)
dump_json(registry_path, registry)
return 0
if __name__ == "__main__":
raise SystemExit(main())
+293
View File
@@ -0,0 +1,293 @@
#!/usr/bin/env python3
"""Shared helpers for the Japanese Word Trainer 0.3 deck registry."""
from __future__ import annotations
import hashlib
import json
import re
import zipfile
from pathlib import Path
from typing import Any
ROOT = Path(__file__).resolve().parents[1]
PACKAGE_FILES = ("manifest.json", "words.json", "refs.json")
STABLE_DECK_RE = re.compile(r"^[a-z0-9][a-z0-9_-]*:deck:[a-z0-9][a-z0-9_-]*$")
STABLE_WORD_RE = re.compile(r"^[a-z0-9][a-z0-9_-]*:word:[a-z0-9][a-z0-9_-]*$")
MANIFEST_REQUIRED = {
"schemaVersion",
"stableDeckKey",
"deckKind",
"title",
"description",
"versionCode",
"createdAt",
"updatedAt",
}
WORD_REQUIRED = {
"stableWordKey",
"readingJa",
"readingKo",
"partOfSpeech",
"grammar",
"kanji",
"meaningJa",
"meaningKo",
"exampleJa",
"exampleKo",
"tag",
"note",
"contentHash",
}
REF_REQUIRED = {"stableWordKey", "displayOrder"}
REGISTRY_DECK_REQUIRED = {
"stableDeckKey",
"deckKind",
"title",
"description",
"versionCode",
"wordCount",
"packageUrl",
"sha256",
}
class ValidationError(Exception):
"""Raised when registry or package data does not match the 0.3 contract."""
def load_json(path: Path) -> Any:
with path.open("r", encoding="utf-8") as handle:
return json.load(handle)
def dump_json(path: Path, data: Any) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
text = json.dumps(data, ensure_ascii=False, indent=2) + "\n"
path.write_text(text, encoding="utf-8")
def json_bytes(data: Any) -> bytes:
return (json.dumps(data, ensure_ascii=False, indent=2) + "\n").encode("utf-8")
def canonical_bytes(data: Any) -> bytes:
return json.dumps(
data,
ensure_ascii=False,
sort_keys=True,
separators=(",", ":"),
).encode("utf-8")
def sha256_file(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as handle:
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def stable_slug(stable_deck_key: str) -> str:
return stable_deck_key.split(":deck:", 1)[1]
def package_url_for(manifest: dict[str, Any]) -> str:
return f"packages/{stable_slug(manifest['stableDeckKey'])}-v{manifest['versionCode']}.zip"
def word_content_hash(word: dict[str, Any]) -> str:
payload = {key: value for key, value in word.items() if key != "contentHash"}
digest = hashlib.sha256(b"word-v1:" + canonical_bytes(payload)).hexdigest()
return f"sha256:{digest}"
def source_dirs(root: Path = ROOT) -> list[Path]:
manifests = sorted((root / "sources").glob("*/*/manifest.json"))
return [manifest.parent for manifest in manifests]
def load_source(source_dir: Path) -> tuple[dict[str, Any], list[dict[str, Any]], list[dict[str, Any]]]:
return (
load_json(source_dir / "manifest.json"),
load_json(source_dir / "words.json"),
load_json(source_dir / "refs.json"),
)
def materialize_words(words: list[dict[str, Any]]) -> list[dict[str, Any]]:
materialized: list[dict[str, Any]] = []
for word in words:
next_word = dict(word)
next_word["contentHash"] = word_content_hash(next_word)
materialized.append(next_word)
return materialized
def write_package(
package_path: Path,
manifest: dict[str, Any],
words: list[dict[str, Any]],
refs: list[dict[str, Any]],
) -> None:
package_path.parent.mkdir(parents=True, exist_ok=True)
payloads = {
"manifest.json": json_bytes(manifest),
"words.json": json_bytes(words),
"refs.json": json_bytes(refs),
}
with zipfile.ZipFile(package_path, "w", compression=zipfile.ZIP_DEFLATED) as package:
for name in PACKAGE_FILES:
info = zipfile.ZipInfo(name)
info.date_time = (1980, 1, 1, 0, 0, 0)
info.compress_type = zipfile.ZIP_DEFLATED
info.external_attr = 0o644 << 16
package.writestr(info, payloads[name])
def validate_source(
source_dir: Path,
manifest: dict[str, Any],
words: list[dict[str, Any]],
refs: list[dict[str, Any]],
) -> list[str]:
errors: list[str] = []
context = str(source_dir.relative_to(ROOT)) if source_dir.is_relative_to(ROOT) else str(source_dir)
missing_manifest = MANIFEST_REQUIRED - set(manifest)
if missing_manifest:
errors.append(f"{context}/manifest.json missing fields: {sorted(missing_manifest)}")
if manifest.get("schemaVersion") != 1:
errors.append(f"{context}/manifest.json schemaVersion must be 1")
if manifest.get("deckKind") not in {"OFFICIAL", "CUSTOM"}:
errors.append(f"{context}/manifest.json deckKind must be OFFICIAL or CUSTOM")
if not STABLE_DECK_RE.match(str(manifest.get("stableDeckKey", ""))):
errors.append(f"{context}/manifest.json stableDeckKey has invalid format")
if not isinstance(manifest.get("versionCode"), int) or manifest.get("versionCode", 0) < 1:
errors.append(f"{context}/manifest.json versionCode must be a positive integer")
if not isinstance(words, list):
errors.append(f"{context}/words.json must be an array")
words = []
if not isinstance(refs, list):
errors.append(f"{context}/refs.json must be an array")
refs = []
word_keys: set[str] = set()
for index, word in enumerate(words):
missing_word = WORD_REQUIRED - set(word)
if missing_word:
errors.append(f"{context}/words.json[{index}] missing fields: {sorted(missing_word)}")
stable_word_key = str(word.get("stableWordKey", ""))
if not STABLE_WORD_RE.match(stable_word_key):
errors.append(f"{context}/words.json[{index}] stableWordKey has invalid format")
if stable_word_key in word_keys:
errors.append(f"{context}/words.json duplicates {stable_word_key}")
word_keys.add(stable_word_key)
expected_hash = word_content_hash(word)
actual_hash = word.get("contentHash")
if actual_hash not in {"", expected_hash}:
errors.append(
f"{context}/words.json[{index}] contentHash is {actual_hash!r}; expected {expected_hash!r}"
)
ref_orders: list[int] = []
ref_keys: set[str] = set()
for index, ref in enumerate(refs):
missing_ref = REF_REQUIRED - set(ref)
if missing_ref:
errors.append(f"{context}/refs.json[{index}] missing fields: {sorted(missing_ref)}")
stable_word_key = str(ref.get("stableWordKey", ""))
if not STABLE_WORD_RE.match(stable_word_key):
errors.append(f"{context}/refs.json[{index}] stableWordKey has invalid format")
if stable_word_key in ref_keys:
errors.append(f"{context}/refs.json duplicates {stable_word_key}")
ref_keys.add(stable_word_key)
if stable_word_key not in word_keys:
errors.append(f"{context}/refs.json[{index}] references missing package word {stable_word_key}")
display_order = ref.get("displayOrder")
if not isinstance(display_order, int):
errors.append(f"{context}/refs.json[{index}] displayOrder must be an integer")
else:
ref_orders.append(display_order)
if sorted(ref_orders) != list(range(len(ref_orders))):
errors.append(f"{context}/refs.json displayOrder must be contiguous from 0")
return errors
def validate_registry(root: Path = ROOT) -> list[str]:
errors: list[str] = []
registry_path = root / "registry.json"
registry = load_json(registry_path)
for field in ("schemaVersion", "registryId", "registryName", "trustLevel", "generatedAt", "decks"):
if field not in registry:
errors.append(f"registry.json missing field: {field}")
if registry.get("schemaVersion") != 1:
errors.append("registry.json schemaVersion must be 1")
if registry.get("trustLevel") not in {"OFFICIAL", "COMMUNITY"}:
errors.append("registry.json trustLevel must be OFFICIAL or COMMUNITY")
if not isinstance(registry.get("decks"), list):
errors.append("registry.json decks must be an array")
return errors
source_by_deck: dict[str, tuple[dict[str, Any], list[dict[str, Any]], list[dict[str, Any]]]] = {}
for source_dir in source_dirs(root):
manifest, words, refs = load_source(source_dir)
errors.extend(validate_source(source_dir, manifest, words, refs))
source_by_deck[manifest.get("stableDeckKey", "")] = (manifest, words, refs)
seen_decks: set[str] = set()
for index, deck in enumerate(registry["decks"]):
missing_deck = REGISTRY_DECK_REQUIRED - set(deck)
if missing_deck:
errors.append(f"registry.json decks[{index}] missing fields: {sorted(missing_deck)}")
stable_deck_key = str(deck.get("stableDeckKey", ""))
if stable_deck_key in seen_decks:
errors.append(f"registry.json duplicates {stable_deck_key}")
seen_decks.add(stable_deck_key)
if stable_deck_key not in source_by_deck:
errors.append(f"registry.json decks[{index}] has no source directory for {stable_deck_key}")
continue
manifest, words, refs = source_by_deck[stable_deck_key]
materialized_words = materialize_words(words)
expected_url = package_url_for(manifest)
package_path = root / expected_url
if deck.get("packageUrl") != expected_url:
errors.append(f"registry.json decks[{index}] packageUrl should be {expected_url}")
if deck.get("deckKind") != manifest.get("deckKind"):
errors.append(f"registry.json decks[{index}] deckKind does not match manifest")
if deck.get("versionCode") != manifest.get("versionCode"):
errors.append(f"registry.json decks[{index}] versionCode does not match manifest")
if deck.get("wordCount") != len(refs):
errors.append(f"registry.json decks[{index}] wordCount should be {len(refs)}")
if not package_path.exists():
errors.append(f"{expected_url} does not exist")
continue
sha256 = sha256_file(package_path)
if deck.get("sha256") != sha256:
errors.append(f"registry.json decks[{index}] sha256 should be {sha256}")
with zipfile.ZipFile(package_path, "r") as package:
names = package.namelist()
if names != list(PACKAGE_FILES):
errors.append(f"{expected_url} must contain only {list(PACKAGE_FILES)} in that order")
package_manifest = json.loads(package.read("manifest.json"))
package_words = json.loads(package.read("words.json"))
package_refs = json.loads(package.read("refs.json"))
if package_manifest != manifest:
errors.append(f"{expected_url}/manifest.json does not match source manifest")
if package_words != materialized_words:
errors.append(f"{expected_url}/words.json does not match hashed source words")
if package_refs != refs:
errors.append(f"{expected_url}/refs.json does not match source refs")
missing_registry_entries = set(source_by_deck) - seen_decks
for stable_deck_key in sorted(missing_registry_entries):
errors.append(f"source deck {stable_deck_key} is missing from registry.json")
return errors
+35
View File
@@ -0,0 +1,35 @@
#!/usr/bin/env python3
"""Validate registry.json, source deck JSON, and built package artifacts."""
from __future__ import annotations
import argparse
from pathlib import Path
from deck_registry import ROOT, validate_registry
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
"--root",
type=Path,
default=ROOT,
help="Repository root. Defaults to the parent of scripts/.",
)
return parser.parse_args()
def main() -> int:
args = parse_args()
errors = validate_registry(args.root.resolve())
if errors:
for error in errors:
print(f"ERROR: {error}")
return 1
print("registry, sources, and packages are valid")
return 0
if __name__ == "__main__":
raise SystemExit(main())
+10
View File
@@ -0,0 +1,10 @@
{
"schemaVersion": 1,
"stableDeckKey": "official:deck:travel-basic",
"deckKind": "CUSTOM",
"title": "Travel Basic",
"description": "Sample custom travel vocabulary deck published through the official registry.",
"versionCode": 1,
"createdAt": 1778400000000,
"updatedAt": 1778400000000
}
+14
View File
@@ -0,0 +1,14 @@
[
{
"stableWordKey": "official:word:airport",
"displayOrder": 0
},
{
"stableWordKey": "official:word:reservation",
"displayOrder": 1
},
{
"stableWordKey": "official:word:ticket",
"displayOrder": 2
}
]
+47
View File
@@ -0,0 +1,47 @@
[
{
"stableWordKey": "official:word:airport",
"readingJa": "くうこう",
"readingKo": "",
"partOfSpeech": "noun",
"grammar": "",
"kanji": "空港",
"meaningJa": "",
"meaningKo": "공항",
"exampleJa": "空港まで行きたいです。",
"exampleKo": "공항까지 가고 싶습니다.",
"tag": "travel",
"note": "Sample custom package word.",
"contentHash": "sha256:01d44ce6ba568f17dca98d970f742df22edc93bef49817b4d468ab6c7bfb63f6"
},
{
"stableWordKey": "official:word:reservation",
"readingJa": "よやく",
"readingKo": "",
"partOfSpeech": "noun",
"grammar": "",
"kanji": "予約",
"meaningJa": "",
"meaningKo": "예약",
"exampleJa": "予約があります。",
"exampleKo": "예약이 있습니다.",
"tag": "travel",
"note": "Sample custom package word.",
"contentHash": "sha256:679b7b32ce7c576e1f8b8e75243deca7af60022e214feb9875ca72e3f1a38478"
},
{
"stableWordKey": "official:word:ticket",
"readingJa": "きっぷ",
"readingKo": "",
"partOfSpeech": "noun",
"grammar": "",
"kanji": "切符",
"meaningJa": "",
"meaningKo": "표",
"exampleJa": "切符を一枚ください。",
"exampleKo": "표 한 장 주세요.",
"tag": "travel",
"note": "Sample custom package word.",
"contentHash": "sha256:a27fd3edcaa3871b7588d67302e59b71f4d4aee19a452475e415122c7ee4e1d1"
}
]
+10
View File
@@ -0,0 +1,10 @@
{
"schemaVersion": 1,
"stableDeckKey": "official:deck:jlpt-n5",
"deckKind": "OFFICIAL",
"title": "JLPT N5 Sample",
"description": "Small JLPT N5 sample deck for validating the 0.3 package format.",
"versionCode": 1,
"createdAt": 1778400000000,
"updatedAt": 1778400000000
}
+22
View File
@@ -0,0 +1,22 @@
[
{
"stableWordKey": "official:word:au",
"displayOrder": 0
},
{
"stableWordKey": "official:word:aoi",
"displayOrder": 1
},
{
"stableWordKey": "official:word:akai",
"displayOrder": 2
},
{
"stableWordKey": "official:word:agaru",
"displayOrder": 3
},
{
"stableWordKey": "official:word:akarui",
"displayOrder": 4
}
]
+77
View File
@@ -0,0 +1,77 @@
[
{
"stableWordKey": "official:word:au",
"readingJa": "あう",
"readingKo": "",
"partOfSpeech": "동사",
"grammar": "5단동사, 자동사",
"kanji": "会う",
"meaningJa": "",
"meaningKo": "만나다",
"exampleJa": "前に彼にあったのを覚えている。",
"exampleKo": "",
"tag": "JLPT N5",
"note": "Light sample adapted from the app seed data.",
"contentHash": "sha256:6f9bc1f6327f865957a4bd96178ce14ada1a97350990e0702bd204013fc9a152"
},
{
"stableWordKey": "official:word:aoi",
"readingJa": "あおい",
"readingKo": "",
"partOfSpeech": "형용사",
"grammar": "い형용사",
"kanji": "青い",
"meaningJa": "",
"meaningKo": "파랗다. 푸르다",
"exampleJa": "なぜ空が青いか知っているか。",
"exampleKo": "",
"tag": "JLPT N5",
"note": "Light sample adapted from the app seed data.",
"contentHash": "sha256:0d198f6f0b5adc4fe2f8d60fad7d80d58a41f730a64717f33a7129353854f466"
},
{
"stableWordKey": "official:word:akai",
"readingJa": "あかい",
"readingKo": "",
"partOfSpeech": "형용사",
"grammar": "い형용사",
"kanji": "赤い",
"meaningJa": "",
"meaningKo": "붉다. 빨갛다",
"exampleJa": "彼女は赤いスカートをはいていた。",
"exampleKo": "",
"tag": "JLPT N5",
"note": "Light sample adapted from the app seed data.",
"contentHash": "sha256:27a23262a70221a3c30c53ce595bf10fda2408a03da87002a79339f41752dadd"
},
{
"stableWordKey": "official:word:agaru",
"readingJa": "あがる",
"readingKo": "",
"partOfSpeech": "동사",
"grammar": "5단동사, 자동사, 타동사",
"kanji": "上がる",
"meaningJa": "",
"meaningKo": "오르다",
"exampleJa": "3月1日から鉄道運賃が1割あがると発表された。",
"exampleKo": "",
"tag": "JLPT N5",
"note": "Light sample adapted from the app seed data.",
"contentHash": "sha256:e9fd51e06175c0793e1c128c3c9b3dd7e19142e8552beff800818f6ed020401e"
},
{
"stableWordKey": "official:word:akarui",
"readingJa": "あかるい",
"readingKo": "",
"partOfSpeech": "형용사",
"grammar": "い형용사",
"kanji": "明るい",
"meaningJa": "",
"meaningKo": "밝다. 환하다. 명랑하다",
"exampleJa": "ダイヤは明るく光った。",
"exampleKo": "",
"tag": "JLPT N5",
"note": "Light sample adapted from the app seed data.",
"contentHash": "sha256:1f07f834e0f1ca89d7b70e59f22d981d3f62682f5aeac22adc7028e5a54d3980"
}
]