fix: apply code-review findings to import_fruits script and tests

- find_images: verify exact fruit_id prefix (stem-parse before _s0.) to
  prevent cross-fruit image contamination (e.g. mitschurins steals
  mitschurins_fruchtbare images)
- DATA_DIR uses os.path.abspath to handle symlinks / relative invocation
- counts["images"] derived from IMAGE_DIRS constant, not hardcoded keys
- counts["warnings"] incremented on missing-id/name skip (was missing)
- UPSERT preserves existing comment (comment = fruits.comment, not NULL)
- Print summary and rollback moved inside try/except to avoid NameError
- Remove dead XML_PATH constant
- Tests: add tearDown to clean up tmpdir; fix vacuous _tn exclusion test
  to include positive-control _s0 file; add prefix-collision regression test

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
2026-06-17 13:38:05 +02:00
co-authored by Claude Sonnet 4.6
parent 4d4c5f399f
commit 7fc8204924
3 changed files with 48 additions and 20 deletions
+30 -18
View File
@@ -17,8 +17,7 @@ from datetime import date, datetime
import psycopg2
DATA_DIR = os.path.join(os.path.dirname(__file__), "..", "03-data")
XML_PATH = os.path.join(DATA_DIR, "obstsorten.xml")
DATA_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "03-data")
# typ values that are aggregate categories — coerced with a logged warning
AGGREGATE_TYPS = {"x", "bq", "aprpfi", "mirren", "pflzwe", "bo", "k"}
@@ -86,13 +85,23 @@ def find_images(fruit_id: str, data_dir: str) -> list:
"""
results = []
for subdir, image_type in IMAGE_DIRS:
pattern = os.path.join(data_dir, subdir, f"{fruit_id}_*_s0.*")
pattern = os.path.join(data_dir, subdir, f"{glob.escape(fruit_id)}_*_s0.*")
for path in sorted(glob.glob(pattern)):
basename = os.path.basename(path)
# Verify the fruit_id is the exact prefix, not just a string prefix of
# another fruit's id (e.g. "mitschurins" must not steal
# "mitschurins_fruchtbare_nda_s0.jpg").
# Convention: basename = {id}_{type_code}_s0.{ext}
# So stripping the last _-segment before _s0 gives the exact id.
stem = basename.split("_s0.")[0]
actual_id = "_".join(stem.split("_")[:-1])
if actual_id != fruit_id:
continue
with open(path, "rb") as f:
data = f.read()
results.append(
{
"filename": os.path.basename(path),
"filename": basename,
"data": data,
"image_type": image_type,
}
@@ -112,7 +121,7 @@ def import_fruits(conn, data_dir: str = None) -> dict:
tree = ET.parse(xml_path)
root = tree.getroot()
counts = {"fruits": 0, "synonyms": 0, "images": {"fruit": 0, "flower": 0, "tree": 0}, "warnings": 0, "skipped": 0}
counts = {"fruits": 0, "synonyms": 0, "images": {img_type: 0 for _, img_type in IMAGE_DIRS}, "warnings": 0, "skipped": 0}
warnings = []
with conn.cursor() as cur:
@@ -125,6 +134,7 @@ def import_fruits(conn, data_dir: str = None) -> dict:
if not osdb_number or not name:
counts["skipped"] += 1
counts["warnings"] += 1
warnings.append(f"SKIP: missing id or name (id={osdb_number!r})")
continue
@@ -151,7 +161,7 @@ def import_fruits(conn, data_dir: str = None) -> dict:
ON CONFLICT (osdb_number) DO UPDATE
SET name = EXCLUDED.name,
fruit_type = EXCLUDED.fruit_type,
comment = NULL,
comment = fruits.comment,
updated_at = NOW()
RETURNING id
""",
@@ -165,7 +175,7 @@ def import_fruits(conn, data_dir: str = None) -> dict:
ON CONFLICT (osdb_number) DO UPDATE
SET name = EXCLUDED.name,
fruit_type = EXCLUDED.fruit_type,
comment = NULL,
comment = fruits.comment,
updated_at = NOW()
RETURNING id
""",
@@ -218,20 +228,22 @@ def main():
try:
print("Importing fruits...")
counts = import_fruits(conn)
total_images = sum(counts["images"].values())
print(f"\nDone.")
print(f" Fruits upserted: {counts['fruits']}")
print(f" Synonyms inserted: {counts['synonyms']}")
print(f" Images inserted: {total_images}")
for img_type, n in counts["images"].items():
print(f" {img_type}: {n}")
print(f" Warnings: {counts['warnings']}")
print(f" Skipped: {counts['skipped']}")
except Exception as e:
print(f"Error: {e}", file=sys.stderr)
conn.rollback()
sys.exit(1)
finally:
conn.close()
total_images = sum(counts["images"].values())
print(f"\nDone.")
print(f" Fruits upserted: {counts['fruits']}")
print(f" Synonyms inserted: {counts['synonyms']}")
print(f" Images inserted: {total_images}")
print(f" fruit: {counts['images']['fruit']}")
print(f" flower: {counts['images']['flower']}")
print(f" tree: {counts['images']['tree']}")
print(f" Warnings: {counts['warnings']}")
print(f" Skipped: {counts['skipped']}")
if __name__ == "__main__":
main()