diff --git a/.gitignore b/.gitignore index 434abb6..c3bee36 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,10 @@ frontend/.vite/ .env backend/.env +# Import data — large binary files, not committed +03-data/ +obstsorten.zip + # macOS .DS_Store diff --git a/Makefile b/Makefile index 16d5a6f..1d1a440 100644 --- a/Makefile +++ b/Makefile @@ -30,10 +30,11 @@ run-backend: run-frontend: cd frontend && npm run dev -## test: run all backend and frontend tests +## test: run all backend, frontend, and script tests test: cd backend && go test ./... cd frontend && npm run test -- --run + cd scripts && python3 -m unittest import_fruits_test -v ## fmt: format all code fmt: diff --git a/scripts/import_fruits.py b/scripts/import_fruits.py new file mode 100644 index 0000000..13cf469 --- /dev/null +++ b/scripts/import_fruits.py @@ -0,0 +1,237 @@ +#!/usr/bin/env python3 +""" +Import fruits from 03-data/obstsorten.xml into the OSDB database. + +Usage: + DATABASE_URL=postgres://... python3 scripts/import_fruits.py + +Idempotent: deletes synonyms and images for each fruit before re-inserting. +Upserts the fruit row by osdb_number. +""" + +import glob +import os +import sys +import xml.etree.ElementTree as ET +from datetime import date, datetime + +import psycopg2 + +DATA_DIR = os.path.join(os.path.dirname(__file__), "..", "03-data") +XML_PATH = os.path.join(DATA_DIR, "obstsorten.xml") + +# typ values that are aggregate categories — coerced with a logged warning +AGGREGATE_TYPS = {"x", "bq", "aprpfi", "mirren", "pflzwe", "bo", "k"} + +TYP_MAP = { + "a": "Apfelsorten", + "x": "Apfelsorten", + "b": "Birnensorten", + "bq": "Birnensorten", + "q": "Quittensorten", + "apr": "Aprikosen", + "aprpfi": "Aprikosen", + "pfi": "Pfirsiche", + "p": "Pfirsiche", + "mir": "Mirabellen", + "mirren": "Mirabellen", + "ren": "Renekloden", + "pfl": "Pflaumen", + "pflzwe": "Pflaumen", + "zwe": "Zwetschen", + "k": "Sauerkirschen", + "bo": "Brombeeren", + "bob": "Brombeeren", + "boe": "Erdbeeren", + "boh": "Himbeeren", + "boj": "Johannisbeeren", + "bos": "Stachelbeeren", + "wei": "Wein", + "w": "Wein", +} + +IMAGE_DIRS = [ + ("einzelfruechte", "fruit"), + ("blueten", "flower"), + ("baume", "tree"), +] + + +def map_typ(typ: str): + """Map XML typ code to fruit_type enum value. Returns None for unknowns.""" + return TYP_MAP.get(typ) + + +def parse_synonyms(text) -> list: + """Split comma-separated synonym text into a clean list.""" + if not text: + return [] + return [s.strip() for s in text.split(",") if s.strip()] + + +def parse_date(text) -> date | None: + """Parse YYYYMMDD string to date. Returns None on failure or empty input.""" + if not text: + return None + try: + return datetime.strptime(text.strip(), "%Y%m%d").date() + except ValueError: + return None + + +def find_images(fruit_id: str, data_dir: str) -> list: + """ + Find all _s0 images for a fruit across the three image directories. + Returns list of dicts with keys: filename, data, image_type. + """ + results = [] + for subdir, image_type in IMAGE_DIRS: + pattern = os.path.join(data_dir, subdir, f"{fruit_id}_*_s0.*") + for path in sorted(glob.glob(pattern)): + with open(path, "rb") as f: + data = f.read() + results.append( + { + "filename": os.path.basename(path), + "data": data, + "image_type": image_type, + } + ) + return results + + +def import_fruits(conn, data_dir: str = None) -> dict: + """ + Import all fruits from obstsorten.xml. + Returns counts dict: fruits, synonyms, images, warnings, skipped. + """ + if data_dir is None: + data_dir = DATA_DIR + xml_path = os.path.join(data_dir, "obstsorten.xml") + + tree = ET.parse(xml_path) + root = tree.getroot() + + counts = {"fruits": 0, "synonyms": 0, "images": {"fruit": 0, "flower": 0, "tree": 0}, "warnings": 0, "skipped": 0} + warnings = [] + + with conn.cursor() as cur: + for sorte in root.findall("sorte"): + osdb_number = (sorte.findtext("id") or "").strip() + name = (sorte.findtext("name") or "").strip() + typ = (sorte.findtext("typ") or "").strip() + synonym_text = sorte.findtext("synonym") + added_text = (sorte.findtext("added") or "").strip() + + if not osdb_number or not name: + counts["skipped"] += 1 + warnings.append(f"SKIP: missing id or name (id={osdb_number!r})") + continue + + fruit_type = map_typ(typ) + if fruit_type is None: + counts["skipped"] += 1 + warnings.append(f"SKIP: unknown typ={typ!r} for id={osdb_number!r}") + counts["warnings"] += 1 + continue + + if typ in AGGREGATE_TYPS: + warnings.append(f"COERCE: typ={typ!r} → {fruit_type!r} for id={osdb_number!r}") + counts["warnings"] += 1 + + created_at = parse_date(added_text) + synonyms = parse_synonyms(synonym_text) + + # Upsert fruit — get id back for child inserts + if created_at: + cur.execute( + """ + INSERT INTO fruits (name, osdb_number, fruit_type, created_at, updated_at) + VALUES (%s, %s, %s, %s, NOW()) + ON CONFLICT (osdb_number) DO UPDATE + SET name = EXCLUDED.name, + fruit_type = EXCLUDED.fruit_type, + comment = NULL, + updated_at = NOW() + RETURNING id + """, + (name, osdb_number, fruit_type, created_at), + ) + else: + cur.execute( + """ + INSERT INTO fruits (name, osdb_number, fruit_type, updated_at) + VALUES (%s, %s, %s, NOW()) + ON CONFLICT (osdb_number) DO UPDATE + SET name = EXCLUDED.name, + fruit_type = EXCLUDED.fruit_type, + comment = NULL, + updated_at = NOW() + RETURNING id + """, + (name, osdb_number, fruit_type), + ) + fruit_id = cur.fetchone()[0] + counts["fruits"] += 1 + + # Idempotent: delete existing synonyms and images + cur.execute("DELETE FROM fruit_synonyms WHERE fruit_id = %s", (fruit_id,)) + cur.execute("DELETE FROM fruit_images WHERE fruit_id = %s", (fruit_id,)) + + # Insert synonyms + for syn in synonyms: + cur.execute( + "INSERT INTO fruit_synonyms (fruit_id, synonym) VALUES (%s, %s)", + (fruit_id, syn), + ) + counts["synonyms"] += 1 + + # Insert images + for img in find_images(osdb_number, data_dir): + cur.execute( + """ + INSERT INTO fruit_images (fruit_id, filename, data, image_type, created_at) + VALUES (%s, %s, %s, %s, NOW()) + """, + (fruit_id, img["filename"], psycopg2.Binary(img["data"]), img["image_type"]), + ) + counts["images"][img["image_type"]] += 1 + + conn.commit() + + for w in warnings[:20]: + print(f" WARNING: {w}", file=sys.stderr) + if len(warnings) > 20: + print(f" ... and {len(warnings) - 20} more warnings", file=sys.stderr) + + return counts + + +def main(): + database_url = os.environ.get("DATABASE_URL") + if not database_url: + print("Error: DATABASE_URL environment variable not set", file=sys.stderr) + sys.exit(1) + + print("Connecting to database...") + conn = psycopg2.connect(database_url) + try: + print("Importing fruits...") + counts = import_fruits(conn) + finally: + conn.close() + + total_images = sum(counts["images"].values()) + print(f"\nDone.") + print(f" Fruits upserted: {counts['fruits']}") + print(f" Synonyms inserted: {counts['synonyms']}") + print(f" Images inserted: {total_images}") + print(f" fruit: {counts['images']['fruit']}") + print(f" flower: {counts['images']['flower']}") + print(f" tree: {counts['images']['tree']}") + print(f" Warnings: {counts['warnings']}") + print(f" Skipped: {counts['skipped']}") + + +if __name__ == "__main__": + main() diff --git a/scripts/import_fruits_test.py b/scripts/import_fruits_test.py new file mode 100644 index 0000000..d9b0d11 --- /dev/null +++ b/scripts/import_fruits_test.py @@ -0,0 +1,178 @@ +import os +import tempfile +import unittest +from datetime import date + +from import_fruits import map_typ, parse_synonyms, parse_date, find_images + + +class TestMapTyp(unittest.TestCase): + def test_maps_a_to_apfelsorten(self): + self.assertEqual(map_typ("a"), "Apfelsorten") + + def test_maps_b_to_birnensorten(self): + self.assertEqual(map_typ("b"), "Birnensorten") + + def test_maps_bq_aggregate_to_birnensorten(self): + self.assertEqual(map_typ("bq"), "Birnensorten") + + def test_maps_q_to_quittensorten(self): + self.assertEqual(map_typ("q"), "Quittensorten") + + def test_maps_apr_to_aprikosen(self): + self.assertEqual(map_typ("apr"), "Aprikosen") + + def test_maps_aprpfi_aggregate_to_aprikosen(self): + self.assertEqual(map_typ("aprpfi"), "Aprikosen") + + def test_maps_pfi_to_pfirsiche(self): + self.assertEqual(map_typ("pfi"), "Pfirsiche") + + def test_maps_p_to_pfirsiche(self): + self.assertEqual(map_typ("p"), "Pfirsiche") + + def test_maps_mir_to_mirabellen(self): + self.assertEqual(map_typ("mir"), "Mirabellen") + + def test_maps_mirren_aggregate_to_mirabellen(self): + self.assertEqual(map_typ("mirren"), "Mirabellen") + + def test_maps_ren_to_renekloden(self): + self.assertEqual(map_typ("ren"), "Renekloden") + + def test_maps_pfl_to_pflaumen(self): + self.assertEqual(map_typ("pfl"), "Pflaumen") + + def test_maps_pflzwe_aggregate_to_pflaumen(self): + self.assertEqual(map_typ("pflzwe"), "Pflaumen") + + def test_maps_zwe_to_zwetschen(self): + self.assertEqual(map_typ("zwe"), "Zwetschen") + + def test_maps_k_aggregate_to_sauerkirschen(self): + self.assertEqual(map_typ("k"), "Sauerkirschen") + + def test_maps_bo_aggregate_to_brombeeren(self): + self.assertEqual(map_typ("bo"), "Brombeeren") + + def test_maps_bob_to_brombeeren(self): + self.assertEqual(map_typ("bob"), "Brombeeren") + + def test_maps_boe_to_erdbeeren(self): + self.assertEqual(map_typ("boe"), "Erdbeeren") + + def test_maps_boh_to_himbeeren(self): + self.assertEqual(map_typ("boh"), "Himbeeren") + + def test_maps_boj_to_johannisbeeren(self): + self.assertEqual(map_typ("boj"), "Johannisbeeren") + + def test_maps_bos_to_stachelbeeren(self): + self.assertEqual(map_typ("bos"), "Stachelbeeren") + + def test_maps_wei_to_wein(self): + self.assertEqual(map_typ("wei"), "Wein") + + def test_maps_w_to_wein(self): + self.assertEqual(map_typ("w"), "Wein") + + def test_maps_x_aggregate_to_apfelsorten(self): + self.assertEqual(map_typ("x"), "Apfelsorten") + + def test_returns_none_for_unknown_typ(self): + self.assertIsNone(map_typ("zzz")) + + +class TestParseSynonyms(unittest.TestCase): + def test_splits_by_comma(self): + self.assertEqual(parse_synonyms("Boskop, Boskoop"), ["Boskop", "Boskoop"]) + + def test_strips_whitespace_and_newlines(self): + result = parse_synonyms("\nBoskop Renette [IH.1],\nBoskoop\n") + self.assertEqual(result, ["Boskop Renette [IH.1]", "Boskoop"]) + + def test_filters_empty_parts(self): + self.assertEqual(parse_synonyms(" , , Boskop ,"), ["Boskop"]) + + def test_empty_string_returns_empty_list(self): + self.assertEqual(parse_synonyms(""), []) + + def test_single_synonym_no_comma(self): + self.assertEqual(parse_synonyms("White Paradise"), ["White Paradise"]) + + def test_none_returns_empty_list(self): + self.assertEqual(parse_synonyms(None), []) + + +class TestParseDate(unittest.TestCase): + def test_parses_yyyymmdd(self): + self.assertEqual(parse_date("20070806"), date(2007, 8, 6)) + + def test_empty_string_returns_none(self): + self.assertIsNone(parse_date("")) + + def test_none_returns_none(self): + self.assertIsNone(parse_date(None)) + + def test_invalid_format_returns_none(self): + self.assertIsNone(parse_date("not-a-date")) + + +class TestFindImages(unittest.TestCase): + def setUp(self): + self.tmpdir = tempfile.mkdtemp() + self.ef_dir = os.path.join(self.tmpdir, "einzelfruechte") + self.bl_dir = os.path.join(self.tmpdir, "blueten") + self.bau_dir = os.path.join(self.tmpdir, "baume") + for d in (self.ef_dir, self.bl_dir, self.bau_dir): + os.makedirs(d) + + def _touch(self, path, content=b"img"): + with open(path, "wb") as f: + f.write(content) + + def test_finds_fruit_image(self): + self._touch(os.path.join(self.ef_dir, "adams_apfel_ef_s0.jpg")) + imgs = find_images("adams_apfel", self.tmpdir) + types = [i["image_type"] for i in imgs] + self.assertIn("fruit", types) + + def test_finds_flower_image(self): + self._touch(os.path.join(self.bl_dir, "adams_apfel_frucht_s0.jpg")) + imgs = find_images("adams_apfel", self.tmpdir) + types = [i["image_type"] for i in imgs] + self.assertIn("flower", types) + + def test_finds_tree_image(self): + self._touch(os.path.join(self.bau_dir, "adams_apfel_bau_s0.jpg")) + imgs = find_images("adams_apfel", self.tmpdir) + types = [i["image_type"] for i in imgs] + self.assertIn("tree", types) + + def test_skips_tn_files(self): + self._touch(os.path.join(self.ef_dir, "adams_apfel_ef_tn.jpg")) + imgs = find_images("adams_apfel", self.tmpdir) + self.assertEqual(imgs, []) + + def test_returns_empty_when_no_match(self): + imgs = find_images("unknown_fruit", self.tmpdir) + self.assertEqual(imgs, []) + + def test_image_contains_bytes_and_filename(self): + content = b"\x89PNG\r\n" + self._touch(os.path.join(self.ef_dir, "adams_apfel_ef_s0.jpg"), content) + imgs = find_images("adams_apfel", self.tmpdir) + self.assertEqual(len(imgs), 1) + self.assertEqual(imgs[0]["data"], content) + self.assertEqual(imgs[0]["filename"], "adams_apfel_ef_s0.jpg") + + def test_finds_all_three_types(self): + self._touch(os.path.join(self.ef_dir, "abbe_fetel_ef_s0.jpg")) + self._touch(os.path.join(self.bl_dir, "abbe_fetel_frucht_s0.jpg")) + self._touch(os.path.join(self.bau_dir, "abbe_fetel_bau_s0.jpg")) + imgs = find_images("abbe_fetel", self.tmpdir) + self.assertEqual(len(imgs), 3) + + +if __name__ == "__main__": + unittest.main()