Merge pull request 'feat: import fruit DB from XML (story #03)' (#3) from feature/03-import-fruit-db into feature/02-manage-fruits
Reviewed-on: #3
This commit was merged in pull request #3.
This commit is contained in:
@@ -12,6 +12,10 @@ frontend/.vite/
|
||||
.env
|
||||
backend/.env
|
||||
|
||||
# Import data — large binary files, not committed
|
||||
03-data/
|
||||
obstsorten.zip
|
||||
|
||||
# macOS
|
||||
.DS_Store
|
||||
|
||||
|
||||
@@ -30,10 +30,11 @@ run-backend:
|
||||
run-frontend:
|
||||
cd frontend && npm run dev
|
||||
|
||||
## test: run all backend and frontend tests
|
||||
## test: run all backend, frontend, and script tests
|
||||
test:
|
||||
cd backend && go test ./...
|
||||
cd frontend && npm run test -- --run
|
||||
cd scripts && python3 -m unittest import_fruits_test -v
|
||||
|
||||
## fmt: format all code
|
||||
fmt:
|
||||
|
||||
@@ -6,6 +6,7 @@ A full-stack fruit-variety database (Go + Vue 3).
|
||||
|
||||
- Users can view a "Hello, OSDB!" landing page that confirms the backend is reachable via the Vite proxy.
|
||||
- Users can create, view, edit, and delete fruit varieties, including managing synonyms and uploading images.
|
||||
- Administrators can bulk-import the legacy fruit database from XML using `scripts/import_fruits.py`.
|
||||
|
||||
## Quick Start
|
||||
|
||||
|
||||
@@ -0,0 +1,249 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Import fruits from 03-data/obstsorten.xml into the OSDB database.
|
||||
|
||||
Usage:
|
||||
DATABASE_URL=postgres://... python3 scripts/import_fruits.py
|
||||
|
||||
Idempotent: deletes synonyms and images for each fruit before re-inserting.
|
||||
Upserts the fruit row by osdb_number.
|
||||
"""
|
||||
|
||||
import glob
|
||||
import os
|
||||
import sys
|
||||
import xml.etree.ElementTree as ET
|
||||
from datetime import date, datetime
|
||||
|
||||
import psycopg2
|
||||
|
||||
DATA_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "03-data")
|
||||
|
||||
# typ values that are aggregate categories — coerced with a logged warning
|
||||
AGGREGATE_TYPS = {"x", "bq", "aprpfi", "mirren", "pflzwe", "bo", "k"}
|
||||
|
||||
TYP_MAP = {
|
||||
"a": "Apfelsorten",
|
||||
"x": "Apfelsorten",
|
||||
"b": "Birnensorten",
|
||||
"bq": "Birnensorten",
|
||||
"q": "Quittensorten",
|
||||
"apr": "Aprikosen",
|
||||
"aprpfi": "Aprikosen",
|
||||
"pfi": "Pfirsiche",
|
||||
"p": "Pfirsiche",
|
||||
"mir": "Mirabellen",
|
||||
"mirren": "Mirabellen",
|
||||
"ren": "Renekloden",
|
||||
"pfl": "Pflaumen",
|
||||
"pflzwe": "Pflaumen",
|
||||
"zwe": "Zwetschen",
|
||||
"k": "Sauerkirschen",
|
||||
"bo": "Brombeeren",
|
||||
"bob": "Brombeeren",
|
||||
"boe": "Erdbeeren",
|
||||
"boh": "Himbeeren",
|
||||
"boj": "Johannisbeeren",
|
||||
"bos": "Stachelbeeren",
|
||||
"wei": "Wein",
|
||||
"w": "Wein",
|
||||
}
|
||||
|
||||
IMAGE_DIRS = [
|
||||
("einzelfruechte", "fruit"),
|
||||
("blueten", "flower"),
|
||||
("baume", "tree"),
|
||||
]
|
||||
|
||||
|
||||
def map_typ(typ: str):
|
||||
"""Map XML typ code to fruit_type enum value. Returns None for unknowns."""
|
||||
return TYP_MAP.get(typ)
|
||||
|
||||
|
||||
def parse_synonyms(text) -> list:
|
||||
"""Split comma-separated synonym text into a clean list."""
|
||||
if not text:
|
||||
return []
|
||||
return [s.strip() for s in text.split(",") if s.strip()]
|
||||
|
||||
|
||||
def parse_date(text) -> date | None:
|
||||
"""Parse YYYYMMDD string to date. Returns None on failure or empty input."""
|
||||
if not text:
|
||||
return None
|
||||
try:
|
||||
return datetime.strptime(text.strip(), "%Y%m%d").date()
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def find_images(fruit_id: str, data_dir: str) -> list:
|
||||
"""
|
||||
Find all _s0 images for a fruit across the three image directories.
|
||||
Returns list of dicts with keys: filename, data, image_type.
|
||||
"""
|
||||
results = []
|
||||
for subdir, image_type in IMAGE_DIRS:
|
||||
pattern = os.path.join(data_dir, subdir, f"{glob.escape(fruit_id)}_*_s0.*")
|
||||
for path in sorted(glob.glob(pattern)):
|
||||
basename = os.path.basename(path)
|
||||
# Verify the fruit_id is the exact prefix, not just a string prefix of
|
||||
# another fruit's id (e.g. "mitschurins" must not steal
|
||||
# "mitschurins_fruchtbare_nda_s0.jpg").
|
||||
# Convention: basename = {id}_{type_code}_s0.{ext}
|
||||
# So stripping the last _-segment before _s0 gives the exact id.
|
||||
stem = basename.split("_s0.")[0]
|
||||
actual_id = "_".join(stem.split("_")[:-1])
|
||||
if actual_id != fruit_id:
|
||||
continue
|
||||
with open(path, "rb") as f:
|
||||
data = f.read()
|
||||
results.append(
|
||||
{
|
||||
"filename": basename,
|
||||
"data": data,
|
||||
"image_type": image_type,
|
||||
}
|
||||
)
|
||||
return results
|
||||
|
||||
|
||||
def import_fruits(conn, data_dir: str = None) -> dict:
|
||||
"""
|
||||
Import all fruits from obstsorten.xml.
|
||||
Returns counts dict: fruits, synonyms, images, warnings, skipped.
|
||||
"""
|
||||
if data_dir is None:
|
||||
data_dir = DATA_DIR
|
||||
xml_path = os.path.join(data_dir, "obstsorten.xml")
|
||||
|
||||
tree = ET.parse(xml_path)
|
||||
root = tree.getroot()
|
||||
|
||||
counts = {"fruits": 0, "synonyms": 0, "images": {img_type: 0 for _, img_type in IMAGE_DIRS}, "warnings": 0, "skipped": 0}
|
||||
warnings = []
|
||||
|
||||
with conn.cursor() as cur:
|
||||
for sorte in root.findall("sorte"):
|
||||
osdb_number = (sorte.findtext("id") or "").strip()
|
||||
name = (sorte.findtext("name") or "").strip()
|
||||
typ = (sorte.findtext("typ") or "").strip()
|
||||
synonym_text = sorte.findtext("synonym")
|
||||
added_text = (sorte.findtext("added") or "").strip()
|
||||
|
||||
if not osdb_number or not name:
|
||||
counts["skipped"] += 1
|
||||
counts["warnings"] += 1
|
||||
warnings.append(f"SKIP: missing id or name (id={osdb_number!r})")
|
||||
continue
|
||||
|
||||
fruit_type = map_typ(typ)
|
||||
if fruit_type is None:
|
||||
counts["skipped"] += 1
|
||||
warnings.append(f"SKIP: unknown typ={typ!r} for id={osdb_number!r}")
|
||||
counts["warnings"] += 1
|
||||
continue
|
||||
|
||||
if typ in AGGREGATE_TYPS:
|
||||
warnings.append(f"COERCE: typ={typ!r} → {fruit_type!r} for id={osdb_number!r}")
|
||||
counts["warnings"] += 1
|
||||
|
||||
created_at = parse_date(added_text)
|
||||
synonyms = parse_synonyms(synonym_text)
|
||||
|
||||
# Upsert fruit — get id back for child inserts
|
||||
if created_at:
|
||||
cur.execute(
|
||||
"""
|
||||
INSERT INTO fruits (name, osdb_number, fruit_type, created_at, updated_at)
|
||||
VALUES (%s, %s, %s, %s, NOW())
|
||||
ON CONFLICT (osdb_number) DO UPDATE
|
||||
SET name = EXCLUDED.name,
|
||||
fruit_type = EXCLUDED.fruit_type,
|
||||
comment = fruits.comment,
|
||||
updated_at = NOW()
|
||||
RETURNING id
|
||||
""",
|
||||
(name, osdb_number, fruit_type, created_at),
|
||||
)
|
||||
else:
|
||||
cur.execute(
|
||||
"""
|
||||
INSERT INTO fruits (name, osdb_number, fruit_type, updated_at)
|
||||
VALUES (%s, %s, %s, NOW())
|
||||
ON CONFLICT (osdb_number) DO UPDATE
|
||||
SET name = EXCLUDED.name,
|
||||
fruit_type = EXCLUDED.fruit_type,
|
||||
comment = fruits.comment,
|
||||
updated_at = NOW()
|
||||
RETURNING id
|
||||
""",
|
||||
(name, osdb_number, fruit_type),
|
||||
)
|
||||
fruit_id = cur.fetchone()[0]
|
||||
counts["fruits"] += 1
|
||||
|
||||
# Idempotent: delete existing synonyms and images
|
||||
cur.execute("DELETE FROM fruit_synonyms WHERE fruit_id = %s", (fruit_id,))
|
||||
cur.execute("DELETE FROM fruit_images WHERE fruit_id = %s", (fruit_id,))
|
||||
|
||||
# Insert synonyms
|
||||
for syn in synonyms:
|
||||
cur.execute(
|
||||
"INSERT INTO fruit_synonyms (fruit_id, synonym) VALUES (%s, %s)",
|
||||
(fruit_id, syn),
|
||||
)
|
||||
counts["synonyms"] += 1
|
||||
|
||||
# Insert images
|
||||
for img in find_images(osdb_number, data_dir):
|
||||
cur.execute(
|
||||
"""
|
||||
INSERT INTO fruit_images (fruit_id, filename, data, image_type, created_at)
|
||||
VALUES (%s, %s, %s, %s, NOW())
|
||||
""",
|
||||
(fruit_id, img["filename"], psycopg2.Binary(img["data"]), img["image_type"]),
|
||||
)
|
||||
counts["images"][img["image_type"]] += 1
|
||||
|
||||
conn.commit()
|
||||
|
||||
for w in warnings[:20]:
|
||||
print(f" WARNING: {w}", file=sys.stderr)
|
||||
if len(warnings) > 20:
|
||||
print(f" ... and {len(warnings) - 20} more warnings", file=sys.stderr)
|
||||
|
||||
return counts
|
||||
|
||||
|
||||
def main():
|
||||
database_url = os.environ.get("DATABASE_URL")
|
||||
if not database_url:
|
||||
print("Error: DATABASE_URL environment variable not set", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
print("Connecting to database...")
|
||||
conn = psycopg2.connect(database_url)
|
||||
try:
|
||||
print("Importing fruits...")
|
||||
counts = import_fruits(conn)
|
||||
total_images = sum(counts["images"].values())
|
||||
print(f"\nDone.")
|
||||
print(f" Fruits upserted: {counts['fruits']}")
|
||||
print(f" Synonyms inserted: {counts['synonyms']}")
|
||||
print(f" Images inserted: {total_images}")
|
||||
for img_type, n in counts["images"].items():
|
||||
print(f" {img_type}: {n}")
|
||||
print(f" Warnings: {counts['warnings']}")
|
||||
print(f" Skipped: {counts['skipped']}")
|
||||
except Exception as e:
|
||||
print(f"Error: {e}", file=sys.stderr)
|
||||
conn.rollback()
|
||||
sys.exit(1)
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,193 @@
|
||||
import os
|
||||
import shutil
|
||||
import tempfile
|
||||
import unittest
|
||||
from datetime import date
|
||||
|
||||
from import_fruits import map_typ, parse_synonyms, parse_date, find_images
|
||||
|
||||
|
||||
class TestMapTyp(unittest.TestCase):
|
||||
def test_maps_a_to_apfelsorten(self):
|
||||
self.assertEqual(map_typ("a"), "Apfelsorten")
|
||||
|
||||
def test_maps_b_to_birnensorten(self):
|
||||
self.assertEqual(map_typ("b"), "Birnensorten")
|
||||
|
||||
def test_maps_bq_aggregate_to_birnensorten(self):
|
||||
self.assertEqual(map_typ("bq"), "Birnensorten")
|
||||
|
||||
def test_maps_q_to_quittensorten(self):
|
||||
self.assertEqual(map_typ("q"), "Quittensorten")
|
||||
|
||||
def test_maps_apr_to_aprikosen(self):
|
||||
self.assertEqual(map_typ("apr"), "Aprikosen")
|
||||
|
||||
def test_maps_aprpfi_aggregate_to_aprikosen(self):
|
||||
self.assertEqual(map_typ("aprpfi"), "Aprikosen")
|
||||
|
||||
def test_maps_pfi_to_pfirsiche(self):
|
||||
self.assertEqual(map_typ("pfi"), "Pfirsiche")
|
||||
|
||||
def test_maps_p_to_pfirsiche(self):
|
||||
self.assertEqual(map_typ("p"), "Pfirsiche")
|
||||
|
||||
def test_maps_mir_to_mirabellen(self):
|
||||
self.assertEqual(map_typ("mir"), "Mirabellen")
|
||||
|
||||
def test_maps_mirren_aggregate_to_mirabellen(self):
|
||||
self.assertEqual(map_typ("mirren"), "Mirabellen")
|
||||
|
||||
def test_maps_ren_to_renekloden(self):
|
||||
self.assertEqual(map_typ("ren"), "Renekloden")
|
||||
|
||||
def test_maps_pfl_to_pflaumen(self):
|
||||
self.assertEqual(map_typ("pfl"), "Pflaumen")
|
||||
|
||||
def test_maps_pflzwe_aggregate_to_pflaumen(self):
|
||||
self.assertEqual(map_typ("pflzwe"), "Pflaumen")
|
||||
|
||||
def test_maps_zwe_to_zwetschen(self):
|
||||
self.assertEqual(map_typ("zwe"), "Zwetschen")
|
||||
|
||||
def test_maps_k_aggregate_to_sauerkirschen(self):
|
||||
self.assertEqual(map_typ("k"), "Sauerkirschen")
|
||||
|
||||
def test_maps_bo_aggregate_to_brombeeren(self):
|
||||
self.assertEqual(map_typ("bo"), "Brombeeren")
|
||||
|
||||
def test_maps_bob_to_brombeeren(self):
|
||||
self.assertEqual(map_typ("bob"), "Brombeeren")
|
||||
|
||||
def test_maps_boe_to_erdbeeren(self):
|
||||
self.assertEqual(map_typ("boe"), "Erdbeeren")
|
||||
|
||||
def test_maps_boh_to_himbeeren(self):
|
||||
self.assertEqual(map_typ("boh"), "Himbeeren")
|
||||
|
||||
def test_maps_boj_to_johannisbeeren(self):
|
||||
self.assertEqual(map_typ("boj"), "Johannisbeeren")
|
||||
|
||||
def test_maps_bos_to_stachelbeeren(self):
|
||||
self.assertEqual(map_typ("bos"), "Stachelbeeren")
|
||||
|
||||
def test_maps_wei_to_wein(self):
|
||||
self.assertEqual(map_typ("wei"), "Wein")
|
||||
|
||||
def test_maps_w_to_wein(self):
|
||||
self.assertEqual(map_typ("w"), "Wein")
|
||||
|
||||
def test_maps_x_aggregate_to_apfelsorten(self):
|
||||
self.assertEqual(map_typ("x"), "Apfelsorten")
|
||||
|
||||
def test_returns_none_for_unknown_typ(self):
|
||||
self.assertIsNone(map_typ("zzz"))
|
||||
|
||||
|
||||
class TestParseSynonyms(unittest.TestCase):
|
||||
def test_splits_by_comma(self):
|
||||
self.assertEqual(parse_synonyms("Boskop, Boskoop"), ["Boskop", "Boskoop"])
|
||||
|
||||
def test_strips_whitespace_and_newlines(self):
|
||||
result = parse_synonyms("\nBoskop Renette [IH.1],\nBoskoop\n")
|
||||
self.assertEqual(result, ["Boskop Renette [IH.1]", "Boskoop"])
|
||||
|
||||
def test_filters_empty_parts(self):
|
||||
self.assertEqual(parse_synonyms(" , , Boskop ,"), ["Boskop"])
|
||||
|
||||
def test_empty_string_returns_empty_list(self):
|
||||
self.assertEqual(parse_synonyms(""), [])
|
||||
|
||||
def test_single_synonym_no_comma(self):
|
||||
self.assertEqual(parse_synonyms("White Paradise"), ["White Paradise"])
|
||||
|
||||
def test_none_returns_empty_list(self):
|
||||
self.assertEqual(parse_synonyms(None), [])
|
||||
|
||||
|
||||
class TestParseDate(unittest.TestCase):
|
||||
def test_parses_yyyymmdd(self):
|
||||
self.assertEqual(parse_date("20070806"), date(2007, 8, 6))
|
||||
|
||||
def test_empty_string_returns_none(self):
|
||||
self.assertIsNone(parse_date(""))
|
||||
|
||||
def test_none_returns_none(self):
|
||||
self.assertIsNone(parse_date(None))
|
||||
|
||||
def test_invalid_format_returns_none(self):
|
||||
self.assertIsNone(parse_date("not-a-date"))
|
||||
|
||||
|
||||
class TestFindImages(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmpdir = tempfile.mkdtemp()
|
||||
self.ef_dir = os.path.join(self.tmpdir, "einzelfruechte")
|
||||
self.bl_dir = os.path.join(self.tmpdir, "blueten")
|
||||
self.bau_dir = os.path.join(self.tmpdir, "baume")
|
||||
for d in (self.ef_dir, self.bl_dir, self.bau_dir):
|
||||
os.makedirs(d)
|
||||
|
||||
def tearDown(self):
|
||||
shutil.rmtree(self.tmpdir, ignore_errors=True)
|
||||
|
||||
def _touch(self, path, content=b"img"):
|
||||
with open(path, "wb") as f:
|
||||
f.write(content)
|
||||
|
||||
def test_finds_fruit_image(self):
|
||||
self._touch(os.path.join(self.ef_dir, "adams_apfel_ef_s0.jpg"))
|
||||
imgs = find_images("adams_apfel", self.tmpdir)
|
||||
types = [i["image_type"] for i in imgs]
|
||||
self.assertIn("fruit", types)
|
||||
|
||||
def test_finds_flower_image(self):
|
||||
self._touch(os.path.join(self.bl_dir, "adams_apfel_frucht_s0.jpg"))
|
||||
imgs = find_images("adams_apfel", self.tmpdir)
|
||||
types = [i["image_type"] for i in imgs]
|
||||
self.assertIn("flower", types)
|
||||
|
||||
def test_finds_tree_image(self):
|
||||
self._touch(os.path.join(self.bau_dir, "adams_apfel_bau_s0.jpg"))
|
||||
imgs = find_images("adams_apfel", self.tmpdir)
|
||||
types = [i["image_type"] for i in imgs]
|
||||
self.assertIn("tree", types)
|
||||
|
||||
def test_skips_tn_files_but_finds_s0(self):
|
||||
# _tn file must be excluded; _s0 file must be included (positive control)
|
||||
self._touch(os.path.join(self.ef_dir, "adams_apfel_ef_tn.jpg"))
|
||||
self._touch(os.path.join(self.ef_dir, "adams_apfel_ef_s0.jpg"))
|
||||
imgs = find_images("adams_apfel", self.tmpdir)
|
||||
self.assertEqual(len(imgs), 1)
|
||||
self.assertEqual(imgs[0]["filename"], "adams_apfel_ef_s0.jpg")
|
||||
|
||||
def test_returns_empty_when_no_match(self):
|
||||
imgs = find_images("unknown_fruit", self.tmpdir)
|
||||
self.assertEqual(imgs, [])
|
||||
|
||||
def test_image_contains_bytes_and_filename(self):
|
||||
content = b"\x89PNG\r\n"
|
||||
self._touch(os.path.join(self.ef_dir, "adams_apfel_ef_s0.jpg"), content)
|
||||
imgs = find_images("adams_apfel", self.tmpdir)
|
||||
self.assertEqual(len(imgs), 1)
|
||||
self.assertEqual(imgs[0]["data"], content)
|
||||
self.assertEqual(imgs[0]["filename"], "adams_apfel_ef_s0.jpg")
|
||||
|
||||
def test_finds_all_three_types(self):
|
||||
self._touch(os.path.join(self.ef_dir, "abbe_fetel_ef_s0.jpg"))
|
||||
self._touch(os.path.join(self.bl_dir, "abbe_fetel_frucht_s0.jpg"))
|
||||
self._touch(os.path.join(self.bau_dir, "abbe_fetel_bau_s0.jpg"))
|
||||
imgs = find_images("abbe_fetel", self.tmpdir)
|
||||
self.assertEqual(len(imgs), 3)
|
||||
|
||||
def test_does_not_steal_images_from_fruit_with_longer_id(self):
|
||||
# "mitschurins" must not match "mitschurins_fruchtbare_nda_s0.jpg"
|
||||
self._touch(os.path.join(self.ef_dir, "mitschurins_ef_s0.jpg"))
|
||||
self._touch(os.path.join(self.ef_dir, "mitschurins_fruchtbare_ef_s0.jpg"))
|
||||
imgs = find_images("mitschurins", self.tmpdir)
|
||||
self.assertEqual(len(imgs), 1)
|
||||
self.assertEqual(imgs[0]["filename"], "mitschurins_ef_s0.jpg")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user