Merge pull request 'feat: import fruit DB from XML (story #03)' (#3) from feature/03-import-fruit-db into feature/02-manage-fruits

Reviewed-on: #3

.gitignore

@@ -12,6 +12,10 @@ frontend/.vite/
 .env
 backend/.env
 
+# Import data — large binary files, not committed
+03-data/
+obstsorten.zip
+
 # macOS
 .DS_Store
 

Makefile

@@ -30,10 +30,11 @@ run-backend:
 run-frontend:
 	cd frontend && npm run dev
 
-## test: run all backend and frontend tests
+## test: run all backend, frontend, and script tests
 test:
 	cd backend && go test ./...
 	cd frontend && npm run test -- --run
+	cd scripts && python3 -m unittest import_fruits_test -v
 
 ## fmt: format all code
 fmt:

README.md

@@ -6,6 +6,7 @@ A full-stack fruit-variety database (Go + Vue 3).
 
 - Users can view a "Hello, OSDB!" landing page that confirms the backend is reachable via the Vite proxy.
 - Users can create, view, edit, and delete fruit varieties, including managing synonyms and uploading images.
+- Administrators can bulk-import the legacy fruit database from XML using `scripts/import_fruits.py`.
 
 ## Quick Start
 

scripts/import_fruits.py

@@ -0,0 +1,249 @@
+#!/usr/bin/env python3
+"""
+Import fruits from 03-data/obstsorten.xml into the OSDB database.
+
+Usage:
+    DATABASE_URL=postgres://... python3 scripts/import_fruits.py
+
+Idempotent: deletes synonyms and images for each fruit before re-inserting.
+Upserts the fruit row by osdb_number.
+"""
+
+import glob
+import os
+import sys
+import xml.etree.ElementTree as ET
+from datetime import date, datetime
+
+import psycopg2
+
+DATA_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "03-data")
+
+# typ values that are aggregate categories — coerced with a logged warning
+AGGREGATE_TYPS = {"x", "bq", "aprpfi", "mirren", "pflzwe", "bo", "k"}
+
+TYP_MAP = {
+    "a": "Apfelsorten",
+    "x": "Apfelsorten",
+    "b": "Birnensorten",
+    "bq": "Birnensorten",
+    "q": "Quittensorten",
+    "apr": "Aprikosen",
+    "aprpfi": "Aprikosen",
+    "pfi": "Pfirsiche",
+    "p": "Pfirsiche",
+    "mir": "Mirabellen",
+    "mirren": "Mirabellen",
+    "ren": "Renekloden",
+    "pfl": "Pflaumen",
+    "pflzwe": "Pflaumen",
+    "zwe": "Zwetschen",
+    "k": "Sauerkirschen",
+    "bo": "Brombeeren",
+    "bob": "Brombeeren",
+    "boe": "Erdbeeren",
+    "boh": "Himbeeren",
+    "boj": "Johannisbeeren",
+    "bos": "Stachelbeeren",
+    "wei": "Wein",
+    "w": "Wein",
+}
+
+IMAGE_DIRS = [
+    ("einzelfruechte", "fruit"),
+    ("blueten", "flower"),
+    ("baume", "tree"),
+]
+
+
+def map_typ(typ: str):
+    """Map XML typ code to fruit_type enum value. Returns None for unknowns."""
+    return TYP_MAP.get(typ)
+
+
+def parse_synonyms(text) -> list:
+    """Split comma-separated synonym text into a clean list."""
+    if not text:
+        return []
+    return [s.strip() for s in text.split(",") if s.strip()]
+
+
+def parse_date(text) -> date | None:
+    """Parse YYYYMMDD string to date. Returns None on failure or empty input."""
+    if not text:
+        return None
+    try:
+        return datetime.strptime(text.strip(), "%Y%m%d").date()
+    except ValueError:
+        return None
+
+
+def find_images(fruit_id: str, data_dir: str) -> list:
+    """
+    Find all _s0 images for a fruit across the three image directories.
+    Returns list of dicts with keys: filename, data, image_type.
+    """
+    results = []
+    for subdir, image_type in IMAGE_DIRS:
+        pattern = os.path.join(data_dir, subdir, f"{glob.escape(fruit_id)}_*_s0.*")
+        for path in sorted(glob.glob(pattern)):
+            basename = os.path.basename(path)
+            # Verify the fruit_id is the exact prefix, not just a string prefix of
+            # another fruit's id (e.g. "mitschurins" must not steal
+            # "mitschurins_fruchtbare_nda_s0.jpg").
+            # Convention: basename = {id}_{type_code}_s0.{ext}
+            # So stripping the last _-segment before _s0 gives the exact id.
+            stem = basename.split("_s0.")[0]
+            actual_id = "_".join(stem.split("_")[:-1])
+            if actual_id != fruit_id:
+                continue
+            with open(path, "rb") as f:
+                data = f.read()
+            results.append(
+                {
+                    "filename": basename,
+                    "data": data,
+                    "image_type": image_type,
+                }
+            )
+    return results
+
+
+def import_fruits(conn, data_dir: str = None) -> dict:
+    """
+    Import all fruits from obstsorten.xml.
+    Returns counts dict: fruits, synonyms, images, warnings, skipped.
+    """
+    if data_dir is None:
+        data_dir = DATA_DIR
+    xml_path = os.path.join(data_dir, "obstsorten.xml")
+
+    tree = ET.parse(xml_path)
+    root = tree.getroot()
+
+    counts = {"fruits": 0, "synonyms": 0, "images": {img_type: 0 for _, img_type in IMAGE_DIRS}, "warnings": 0, "skipped": 0}
+    warnings = []
+
+    with conn.cursor() as cur:
+        for sorte in root.findall("sorte"):
+            osdb_number = (sorte.findtext("id") or "").strip()
+            name = (sorte.findtext("name") or "").strip()
+            typ = (sorte.findtext("typ") or "").strip()
+            synonym_text = sorte.findtext("synonym")
+            added_text = (sorte.findtext("added") or "").strip()
+
+            if not osdb_number or not name:
+                counts["skipped"] += 1
+                counts["warnings"] += 1
+                warnings.append(f"SKIP: missing id or name (id={osdb_number!r})")
+                continue
+
+            fruit_type = map_typ(typ)
+            if fruit_type is None:
+                counts["skipped"] += 1
+                warnings.append(f"SKIP: unknown typ={typ!r} for id={osdb_number!r}")
+                counts["warnings"] += 1
+                continue
+
+            if typ in AGGREGATE_TYPS:
+                warnings.append(f"COERCE: typ={typ!r} → {fruit_type!r} for id={osdb_number!r}")
+                counts["warnings"] += 1
+
+            created_at = parse_date(added_text)
+            synonyms = parse_synonyms(synonym_text)
+
+            # Upsert fruit — get id back for child inserts
+            if created_at:
+                cur.execute(
+                    """
+                    INSERT INTO fruits (name, osdb_number, fruit_type, created_at, updated_at)
+                    VALUES (%s, %s, %s, %s, NOW())
+                    ON CONFLICT (osdb_number) DO UPDATE
+                      SET name = EXCLUDED.name,
+                          fruit_type = EXCLUDED.fruit_type,
+                          comment = fruits.comment,
+                          updated_at = NOW()
+                    RETURNING id
+                    """,
+                    (name, osdb_number, fruit_type, created_at),
+                )
+            else:
+                cur.execute(
+                    """
+                    INSERT INTO fruits (name, osdb_number, fruit_type, updated_at)
+                    VALUES (%s, %s, %s, NOW())
+                    ON CONFLICT (osdb_number) DO UPDATE
+                      SET name = EXCLUDED.name,
+                          fruit_type = EXCLUDED.fruit_type,
+                          comment = fruits.comment,
+                          updated_at = NOW()
+                    RETURNING id
+                    """,
+                    (name, osdb_number, fruit_type),
+                )
+            fruit_id = cur.fetchone()[0]
+            counts["fruits"] += 1
+
+            # Idempotent: delete existing synonyms and images
+            cur.execute("DELETE FROM fruit_synonyms WHERE fruit_id = %s", (fruit_id,))
+            cur.execute("DELETE FROM fruit_images WHERE fruit_id = %s", (fruit_id,))
+
+            # Insert synonyms
+            for syn in synonyms:
+                cur.execute(
+                    "INSERT INTO fruit_synonyms (fruit_id, synonym) VALUES (%s, %s)",
+                    (fruit_id, syn),
+                )
+                counts["synonyms"] += 1
+
+            # Insert images
+            for img in find_images(osdb_number, data_dir):
+                cur.execute(
+                    """
+                    INSERT INTO fruit_images (fruit_id, filename, data, image_type, created_at)
+                    VALUES (%s, %s, %s, %s, NOW())
+                    """,
+                    (fruit_id, img["filename"], psycopg2.Binary(img["data"]), img["image_type"]),
+                )
+                counts["images"][img["image_type"]] += 1
+
+    conn.commit()
+
+    for w in warnings[:20]:
+        print(f"  WARNING: {w}", file=sys.stderr)
+    if len(warnings) > 20:
+        print(f"  ... and {len(warnings) - 20} more warnings", file=sys.stderr)
+
+    return counts
+
+
+def main():
+    database_url = os.environ.get("DATABASE_URL")
+    if not database_url:
+        print("Error: DATABASE_URL environment variable not set", file=sys.stderr)
+        sys.exit(1)
+
+    print("Connecting to database...")
+    conn = psycopg2.connect(database_url)
+    try:
+        print("Importing fruits...")
+        counts = import_fruits(conn)
+        total_images = sum(counts["images"].values())
+        print(f"\nDone.")
+        print(f"  Fruits upserted:  {counts['fruits']}")
+        print(f"  Synonyms inserted: {counts['synonyms']}")
+        print(f"  Images inserted:  {total_images}")
+        for img_type, n in counts["images"].items():
+            print(f"    {img_type}: {n}")
+        print(f"  Warnings:         {counts['warnings']}")
+        print(f"  Skipped:          {counts['skipped']}")
+    except Exception as e:
+        print(f"Error: {e}", file=sys.stderr)
+        conn.rollback()
+        sys.exit(1)
+    finally:
+        conn.close()
+
+
+if __name__ == "__main__":
+    main()

scripts/import_fruits_test.py

@@ -0,0 +1,193 @@
+import os
+import shutil
+import tempfile
+import unittest
+from datetime import date
+
+from import_fruits import map_typ, parse_synonyms, parse_date, find_images
+
+
+class TestMapTyp(unittest.TestCase):
+    def test_maps_a_to_apfelsorten(self):
+        self.assertEqual(map_typ("a"), "Apfelsorten")
+
+    def test_maps_b_to_birnensorten(self):
+        self.assertEqual(map_typ("b"), "Birnensorten")
+
+    def test_maps_bq_aggregate_to_birnensorten(self):
+        self.assertEqual(map_typ("bq"), "Birnensorten")
+
+    def test_maps_q_to_quittensorten(self):
+        self.assertEqual(map_typ("q"), "Quittensorten")
+
+    def test_maps_apr_to_aprikosen(self):
+        self.assertEqual(map_typ("apr"), "Aprikosen")
+
+    def test_maps_aprpfi_aggregate_to_aprikosen(self):
+        self.assertEqual(map_typ("aprpfi"), "Aprikosen")
+
+    def test_maps_pfi_to_pfirsiche(self):
+        self.assertEqual(map_typ("pfi"), "Pfirsiche")
+
+    def test_maps_p_to_pfirsiche(self):
+        self.assertEqual(map_typ("p"), "Pfirsiche")
+
+    def test_maps_mir_to_mirabellen(self):
+        self.assertEqual(map_typ("mir"), "Mirabellen")
+
+    def test_maps_mirren_aggregate_to_mirabellen(self):
+        self.assertEqual(map_typ("mirren"), "Mirabellen")
+
+    def test_maps_ren_to_renekloden(self):
+        self.assertEqual(map_typ("ren"), "Renekloden")
+
+    def test_maps_pfl_to_pflaumen(self):
+        self.assertEqual(map_typ("pfl"), "Pflaumen")
+
+    def test_maps_pflzwe_aggregate_to_pflaumen(self):
+        self.assertEqual(map_typ("pflzwe"), "Pflaumen")
+
+    def test_maps_zwe_to_zwetschen(self):
+        self.assertEqual(map_typ("zwe"), "Zwetschen")
+
+    def test_maps_k_aggregate_to_sauerkirschen(self):
+        self.assertEqual(map_typ("k"), "Sauerkirschen")
+
+    def test_maps_bo_aggregate_to_brombeeren(self):
+        self.assertEqual(map_typ("bo"), "Brombeeren")
+
+    def test_maps_bob_to_brombeeren(self):
+        self.assertEqual(map_typ("bob"), "Brombeeren")
+
+    def test_maps_boe_to_erdbeeren(self):
+        self.assertEqual(map_typ("boe"), "Erdbeeren")
+
+    def test_maps_boh_to_himbeeren(self):
+        self.assertEqual(map_typ("boh"), "Himbeeren")
+
+    def test_maps_boj_to_johannisbeeren(self):
+        self.assertEqual(map_typ("boj"), "Johannisbeeren")
+
+    def test_maps_bos_to_stachelbeeren(self):
+        self.assertEqual(map_typ("bos"), "Stachelbeeren")
+
+    def test_maps_wei_to_wein(self):
+        self.assertEqual(map_typ("wei"), "Wein")
+
+    def test_maps_w_to_wein(self):
+        self.assertEqual(map_typ("w"), "Wein")
+
+    def test_maps_x_aggregate_to_apfelsorten(self):
+        self.assertEqual(map_typ("x"), "Apfelsorten")
+
+    def test_returns_none_for_unknown_typ(self):
+        self.assertIsNone(map_typ("zzz"))
+
+
+class TestParseSynonyms(unittest.TestCase):
+    def test_splits_by_comma(self):
+        self.assertEqual(parse_synonyms("Boskop, Boskoop"), ["Boskop", "Boskoop"])
+
+    def test_strips_whitespace_and_newlines(self):
+        result = parse_synonyms("\nBoskop Renette [IH.1],\nBoskoop\n")
+        self.assertEqual(result, ["Boskop Renette [IH.1]", "Boskoop"])
+
+    def test_filters_empty_parts(self):
+        self.assertEqual(parse_synonyms("  ,  , Boskop ,"), ["Boskop"])
+
+    def test_empty_string_returns_empty_list(self):
+        self.assertEqual(parse_synonyms(""), [])
+
+    def test_single_synonym_no_comma(self):
+        self.assertEqual(parse_synonyms("White Paradise"), ["White Paradise"])
+
+    def test_none_returns_empty_list(self):
+        self.assertEqual(parse_synonyms(None), [])
+
+
+class TestParseDate(unittest.TestCase):
+    def test_parses_yyyymmdd(self):
+        self.assertEqual(parse_date("20070806"), date(2007, 8, 6))
+
+    def test_empty_string_returns_none(self):
+        self.assertIsNone(parse_date(""))
+
+    def test_none_returns_none(self):
+        self.assertIsNone(parse_date(None))
+
+    def test_invalid_format_returns_none(self):
+        self.assertIsNone(parse_date("not-a-date"))
+
+
+class TestFindImages(unittest.TestCase):
+    def setUp(self):
+        self.tmpdir = tempfile.mkdtemp()
+        self.ef_dir = os.path.join(self.tmpdir, "einzelfruechte")
+        self.bl_dir = os.path.join(self.tmpdir, "blueten")
+        self.bau_dir = os.path.join(self.tmpdir, "baume")
+        for d in (self.ef_dir, self.bl_dir, self.bau_dir):
+            os.makedirs(d)
+
+    def tearDown(self):
+        shutil.rmtree(self.tmpdir, ignore_errors=True)
+
+    def _touch(self, path, content=b"img"):
+        with open(path, "wb") as f:
+            f.write(content)
+
+    def test_finds_fruit_image(self):
+        self._touch(os.path.join(self.ef_dir, "adams_apfel_ef_s0.jpg"))
+        imgs = find_images("adams_apfel", self.tmpdir)
+        types = [i["image_type"] for i in imgs]
+        self.assertIn("fruit", types)
+
+    def test_finds_flower_image(self):
+        self._touch(os.path.join(self.bl_dir, "adams_apfel_frucht_s0.jpg"))
+        imgs = find_images("adams_apfel", self.tmpdir)
+        types = [i["image_type"] for i in imgs]
+        self.assertIn("flower", types)
+
+    def test_finds_tree_image(self):
+        self._touch(os.path.join(self.bau_dir, "adams_apfel_bau_s0.jpg"))
+        imgs = find_images("adams_apfel", self.tmpdir)
+        types = [i["image_type"] for i in imgs]
+        self.assertIn("tree", types)
+
+    def test_skips_tn_files_but_finds_s0(self):
+        # _tn file must be excluded; _s0 file must be included (positive control)
+        self._touch(os.path.join(self.ef_dir, "adams_apfel_ef_tn.jpg"))
+        self._touch(os.path.join(self.ef_dir, "adams_apfel_ef_s0.jpg"))
+        imgs = find_images("adams_apfel", self.tmpdir)
+        self.assertEqual(len(imgs), 1)
+        self.assertEqual(imgs[0]["filename"], "adams_apfel_ef_s0.jpg")
+
+    def test_returns_empty_when_no_match(self):
+        imgs = find_images("unknown_fruit", self.tmpdir)
+        self.assertEqual(imgs, [])
+
+    def test_image_contains_bytes_and_filename(self):
+        content = b"\x89PNG\r\n"
+        self._touch(os.path.join(self.ef_dir, "adams_apfel_ef_s0.jpg"), content)
+        imgs = find_images("adams_apfel", self.tmpdir)
+        self.assertEqual(len(imgs), 1)
+        self.assertEqual(imgs[0]["data"], content)
+        self.assertEqual(imgs[0]["filename"], "adams_apfel_ef_s0.jpg")
+
+    def test_finds_all_three_types(self):
+        self._touch(os.path.join(self.ef_dir, "abbe_fetel_ef_s0.jpg"))
+        self._touch(os.path.join(self.bl_dir, "abbe_fetel_frucht_s0.jpg"))
+        self._touch(os.path.join(self.bau_dir, "abbe_fetel_bau_s0.jpg"))
+        imgs = find_images("abbe_fetel", self.tmpdir)
+        self.assertEqual(len(imgs), 3)
+
+    def test_does_not_steal_images_from_fruit_with_longer_id(self):
+        # "mitschurins" must not match "mitschurins_fruchtbare_nda_s0.jpg"
+        self._touch(os.path.join(self.ef_dir, "mitschurins_ef_s0.jpg"))
+        self._touch(os.path.join(self.ef_dir, "mitschurins_fruchtbare_ef_s0.jpg"))
+        imgs = find_images("mitschurins", self.tmpdir)
+        self.assertEqual(len(imgs), 1)
+        self.assertEqual(imgs[0]["filename"], "mitschurins_ef_s0.jpg")
+
+
+if __name__ == "__main__":
+    unittest.main()