471 lines
17 KiB
Python
471 lines
17 KiB
Python
"""
|
||
========================================================================
|
||
NormVokabular Mapper – Übersicht
|
||
========================================================================
|
||
|
||
Dieses Skript dient dazu, Begriffe aus Datenbeständen mit einem
|
||
vordefinierten Normvokabular abzugleichen. Es identifiziert Treffer,
|
||
gibt bei fehlenden Treffern Vorschläge aus und ermöglicht optional
|
||
einen Abgleich mit externen APIs (GND, Wikidata).
|
||
|
||
Hauptfunktionen:
|
||
|
||
1. **Input verarbeiten**
|
||
- Liest CSV-, Excel- und ODS-Dateien aus dem Ordner "Input CSV".
|
||
- Extrahiert relevante Begriffe aus Spalten wie "Objektbeschreibung",
|
||
filtert Stopwords und Zahlen.
|
||
|
||
2. **Normvokabular laden**
|
||
- Liest die Masterdatei NV_MASTER.ods ein.
|
||
- Berücksichtigt Hierarchie-IDs, um übergeordnete Begriffe zuordnen zu können.
|
||
- Erstellt ein Index für gestemmte Begriffe, um auch ähnliche Schreibweisen zu erkennen.
|
||
|
||
3. **Mapping auf Normvokabular**
|
||
- Prüft, ob ein Begriff exakt oder gestemmt im Normvokabular vorkommt.
|
||
- Wenn kein Treffer vorliegt, werden alternative Vorschläge generiert.
|
||
|
||
4. **API-Abgleich (optional)**
|
||
- Fragt GND und Wikidata ab, um den Top-1 Treffer für jeden Begriff zu ermitteln.
|
||
- Nutzt einen Cache, um wiederholte Requests zu vermeiden.
|
||
- Bietet einen Dry-Run-Modus für Tests ohne Internetzugang.
|
||
|
||
5. **Ergebnis speichern**
|
||
- Speichert die Auswertung in einem eigenen Ordner "Auswertung Ergebnisse".
|
||
- Markiert Treffer visuell: grün = Treffer, rot = kein Treffer (bei Excel),
|
||
bzw. fügt Statusspalte bei ODS-Dateien hinzu.
|
||
- Enthält alle relevanten Informationen pro Begriff: Originalbegriff, Normbegriff,
|
||
Norm-ID, Vorschläge, GND/Wikidata Top1 Treffer.
|
||
|
||
6. **Logging**
|
||
- Informiert über Fortschritt, Anzahl der Begriffe, Treffer und mögliche Fehler.
|
||
|
||
"""
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
import os
|
||
import sys
|
||
import re
|
||
import time
|
||
import json
|
||
import pandas as pd
|
||
import requests
|
||
from pathlib import Path
|
||
from collections import defaultdict
|
||
from difflib import SequenceMatcher
|
||
|
||
# RapidFuzz für Token-basierte Fuzzy-Suche
|
||
try:
|
||
from rapidfuzz import fuzz
|
||
RAPIDFUZZ_AVAILABLE = True
|
||
print("RapidFuzz verfügbar")
|
||
except ImportError:
|
||
RAPIDFUZZ_AVAILABLE = False
|
||
print("RapidFuzz nicht verfügbar – nutze SequenceMatcher")
|
||
|
||
# Spacy Lemmatizer
|
||
try:
|
||
import spacy
|
||
nlp = spacy.load("de_core_news_sm")
|
||
SPACY_AVAILABLE = True
|
||
print("Spacy Lemmatizer aktiviert")
|
||
except:
|
||
SPACY_AVAILABLE = False
|
||
nlp = None
|
||
print("Spacy nicht verfügbar – nutze naive Stemmer")
|
||
|
||
# =========================
|
||
# Pfade & Config
|
||
# =========================
|
||
INPUT_DIR = Path("Input CSV")
|
||
OUTPUT_DIR = Path("Auswertung Ergebnisse")
|
||
OUTPUT_DIR.mkdir(exist_ok=True)
|
||
NORMVOC_FILE = Path("Input CSV/Normvokabular_INTERN/NV_MASTER.ods")
|
||
CACHE_FILE = "api_cache.json"
|
||
STOPWORDS = {"mit","ohne","der","die","das","ein","eine","und","zu","von","im","in","auf","an","als","bei","für","aus","dem","den","des","eines","einer"}
|
||
CONF_THRESHOLD = 0.75
|
||
TIMEOUT = 5
|
||
MAX_RETRIES = 3
|
||
BACKOFF_FACTOR = 2
|
||
HEADERS = {"User-Agent": "VLG_API_multi/1.0 (projekt@example.com)"}
|
||
API_ACTIVE = {"gnd": True, "wikidata": True}
|
||
FAIL_COUNTER = {"gnd": 0, "wikidata": 0}
|
||
|
||
# Cache
|
||
if os.path.exists(CACHE_FILE):
|
||
with open(CACHE_FILE,"r",encoding="utf-8") as f:
|
||
CACHE = json.load(f)
|
||
else:
|
||
CACHE = {}
|
||
|
||
def save_cache():
|
||
with open(CACHE_FILE,"w",encoding="utf-8") as f:
|
||
json.dump(CACHE, f, indent=2, ensure_ascii=False)
|
||
|
||
# =========================
|
||
# Normalisierung / Lemma
|
||
# =========================
|
||
def normalize_text(s):
|
||
if not s:
|
||
return ""
|
||
s = str(s).lower().strip()
|
||
s = re.sub(r"[\(\)\[\]\"'\\.,;:\?!]", "", s)
|
||
s = re.sub(r"\s+"," ",s)
|
||
return s
|
||
|
||
lemma_cache = {}
|
||
|
||
def lemmatize_term(term):
|
||
term_norm = normalize_text(term)
|
||
if term_norm in lemma_cache:
|
||
return lemma_cache[term_norm]
|
||
if SPACY_AVAILABLE and nlp:
|
||
doc = nlp(term_norm)
|
||
lemma = " ".join([token.lemma_ for token in doc])
|
||
else:
|
||
lemma = term_norm
|
||
lemma_cache[term_norm] = lemma
|
||
return lemma
|
||
|
||
# =========================
|
||
# Kompositum-Zerlegung (erweitert)
|
||
# =========================
|
||
def compound_split(term, norm_dict):
|
||
"""
|
||
Zerlegt Komposita durch Prüfen auf Substrings, die im Normvokabular vorkommen.
|
||
"""
|
||
term_norm = normalize_text(term)
|
||
matches = []
|
||
for i in range(len(term_norm)):
|
||
for j in range(i+3, len(term_norm)+1):
|
||
sub = term_norm[i:j]
|
||
if sub in norm_dict and sub not in matches:
|
||
matches.append(sub)
|
||
if not matches:
|
||
matches = [term_norm]
|
||
return matches
|
||
|
||
# =========================
|
||
# Normvokabular laden & Lemma vorbereiten
|
||
# =========================
|
||
def load_normvokabular(file_path):
|
||
sheets = pd.read_excel(file_path, sheet_name=None, engine="odf" if file_path.suffix.lower()==".ods" else None)
|
||
norm_dict = {}
|
||
stem_index = defaultdict(list)
|
||
lemma_norm_map = {}
|
||
|
||
for sheet_name, df in sheets.items():
|
||
if sheet_name.lower() in ["master", "übersicht"]:
|
||
continue
|
||
df = df.dropna(how="all", axis=1)
|
||
df.columns = [str(c).strip() for c in df.columns]
|
||
id_col = next((c for c in df.columns if "ID" in c), None)
|
||
word_col = next((c for c in df.columns if "Wort" in c or "Vokabel" in c), None)
|
||
if not id_col or not word_col:
|
||
continue
|
||
|
||
current_parent_id = None
|
||
for _, row in df.iterrows():
|
||
row_id = str(row[id_col]).strip() if pd.notna(row[id_col]) else None
|
||
row_word = str(row[word_col]).strip() if pd.notna(row[word_col]) else None
|
||
if row_id:
|
||
current_parent_id = row_id
|
||
if not row_word:
|
||
continue
|
||
assigned_parent_id = current_parent_id
|
||
entry = {
|
||
"Name": row_word,
|
||
"ID": assigned_parent_id,
|
||
"Sheet": sheet_name,
|
||
"Own_ID": row_id or ""
|
||
}
|
||
key = normalize_text(row_word)
|
||
norm_dict[key] = entry
|
||
lemma = lemmatize_term(key)
|
||
stem_index[lemma].append(entry)
|
||
if lemma not in lemma_norm_map:
|
||
lemma_norm_map[lemma] = entry
|
||
return norm_dict, stem_index, lemma_norm_map
|
||
|
||
# =========================
|
||
# Vorschläge & Fuzzy Matching
|
||
# =========================
|
||
def get_suggestions(term_lemma, lemma_norm_map, top_n=3, threshold=CONF_THRESHOLD):
|
||
candidates = []
|
||
for key_lemma, entry in lemma_norm_map.items():
|
||
if RAPIDFUZZ_AVAILABLE:
|
||
score_token = fuzz.token_set_ratio(term_lemma, key_lemma)/100
|
||
score_partial = fuzz.partial_ratio(term_lemma, key_lemma)/100
|
||
score = max(score_token, score_partial)
|
||
else:
|
||
score_seq = SequenceMatcher(None, term_lemma.lower(), key_lemma.lower()).ratio()
|
||
score = score_seq
|
||
|
||
# Substring-Boost
|
||
if term_lemma in key_lemma or key_lemma in term_lemma:
|
||
score = max(score, 0.9)
|
||
|
||
if score >= threshold:
|
||
candidates.append((score, entry["Name"], entry["ID"]))
|
||
|
||
candidates.sort(reverse=True)
|
||
return [f"{name} ({id_})" for _, name, id_ in candidates[:top_n]]
|
||
|
||
# =========================
|
||
# Mapping auf Normvokabular
|
||
# =========================
|
||
def map_to_norm(term, norm_dict, stem_index, lemma_norm_map, top_n=3):
|
||
term_norm = normalize_text(term)
|
||
term_lemma = lemmatize_term(term)
|
||
|
||
# Exakter Treffer
|
||
if term_norm in norm_dict:
|
||
e = norm_dict[term_norm]
|
||
return e["Name"], e["ID"], []
|
||
|
||
# Lemma-Treffer
|
||
if term_lemma in stem_index:
|
||
e = stem_index[term_lemma][0]
|
||
return e["Name"], e["ID"], []
|
||
|
||
# KEIN TREFFER → Kompositum-Split & Teilbegriffe prüfen
|
||
tokens = compound_split(term, norm_dict)
|
||
token_matches = []
|
||
all_suggestions = []
|
||
for t in tokens:
|
||
t_lemma = lemmatize_term(t)
|
||
if t_lemma in stem_index:
|
||
e = stem_index[t_lemma][0]
|
||
token_matches.append((t, e["Name"], e["ID"]))
|
||
else:
|
||
sugg = get_suggestions(t_lemma, lemma_norm_map, top_n)
|
||
all_suggestions.extend(sugg)
|
||
token_matches.append((t, "KEIN TREFFER", "", sugg))
|
||
|
||
combined_matches = [m[1] for m in token_matches if m[1] != "KEIN TREFFER"]
|
||
|
||
if combined_matches:
|
||
return "KEIN TREFFER", "", combined_matches
|
||
elif all_suggestions:
|
||
return "KEIN TREFFER", "", all_suggestions
|
||
else:
|
||
return "KEIN TREFFER", "", []
|
||
|
||
# =========================
|
||
# API-Abfragen
|
||
# =========================
|
||
def request_with_retries(api_name,url,params=None):
|
||
cache_key = url + str(params)
|
||
if cache_key in CACHE:
|
||
return CACHE[cache_key]
|
||
retries = 0
|
||
while retries < MAX_RETRIES:
|
||
try:
|
||
r = requests.get(url, params=params, timeout=TIMEOUT, headers=HEADERS)
|
||
if r.status_code == 200:
|
||
try: data = r.json()
|
||
except: data = r.text
|
||
CACHE[cache_key] = data
|
||
FAIL_COUNTER[api_name] = 0
|
||
return data
|
||
except:
|
||
pass
|
||
retries += 1
|
||
time.sleep(min(BACKOFF_FACTOR**retries,30))
|
||
FAIL_COUNTER[api_name] += 1
|
||
if FAIL_COUNTER[api_name] >= 10:
|
||
API_ACTIVE[api_name] = False
|
||
return None
|
||
|
||
def batch_query_gnd(terms):
|
||
results={}
|
||
if not API_ACTIVE.get("gnd", False):
|
||
for t in terms: results[t] = ""
|
||
return results
|
||
for t in terms:
|
||
url="https://lobid.org/gnd/search"
|
||
params={"q":t,"format":"json"}
|
||
data = request_with_retries("gnd", url, params)
|
||
top = ""
|
||
if data and "member" in data:
|
||
cands = [(doc.get("preferredName","") or doc.get("name",""), SequenceMatcher(None,t.lower(),(doc.get("preferredName","") or doc.get("name","")).lower()).ratio()) for doc in data["member"] if doc.get("preferredName","") or doc.get("name","")]
|
||
cands = [c for c in cands if c[1]>=0.75]
|
||
if cands:
|
||
top = sorted(cands,key=lambda x:x[1],reverse=True)[0][0]
|
||
results[t] = top
|
||
return results
|
||
|
||
def batch_query_wikidata(terms):
|
||
results={}
|
||
if not API_ACTIVE.get("wikidata", False):
|
||
for t in terms: results[t] = ""
|
||
return results
|
||
for t in terms:
|
||
url="https://www.wikidata.org/w/api.php"
|
||
params={"action":"wbsearchentities","search":t,"language":"de","format":"json"}
|
||
data = request_with_retries("wikidata", url, params)
|
||
top = ""
|
||
if data and "search" in data:
|
||
cands = [(e.get("label",""), SequenceMatcher(None,t.lower(),e.get("label","").lower()).ratio()) for e in data["search"] if e.get("label","")]
|
||
cands = [c for c in cands if c[1]>=0.70]
|
||
if cands:
|
||
top = sorted(cands,key=lambda x:x[1],reverse=True)[0][0]
|
||
results[t] = top
|
||
return results
|
||
|
||
# =========================
|
||
# Markierung / Export
|
||
# =========================
|
||
def mark_norm_hits(file_path):
|
||
ext = file_path.suffix.lower()
|
||
if ext in [".xlsx", ".xls"]:
|
||
from openpyxl import load_workbook
|
||
from openpyxl.styles import PatternFill
|
||
wb = load_workbook(file_path)
|
||
ws = wb.active
|
||
green_fill = PatternFill(start_color="C6EFCE", end_color="C6EFCE", fill_type="solid")
|
||
red_fill = PatternFill(start_color="FFC7CE", end_color="FFC7CE", fill_type="solid")
|
||
col_map = {cell.value: idx+1 for idx, cell in enumerate(ws[1])}
|
||
norm_col = col_map.get("Norm_Treffer", None)
|
||
if not norm_col:
|
||
print("Spalte 'Norm_Treffer' nicht gefunden")
|
||
return
|
||
for row in ws.iter_rows(min_row=2, min_col=norm_col, max_col=norm_col):
|
||
cell = row[0]
|
||
if cell.value and cell.value != "KEIN TREFFER":
|
||
cell.fill = green_fill
|
||
else:
|
||
cell.fill = red_fill
|
||
wb.save(file_path)
|
||
elif ext==".ods":
|
||
df = pd.read_excel(file_path, engine="odf")
|
||
df["Norm_Status"] = df["Norm_Treffer"].apply(lambda x: "Treffer" if pd.notna(x) and str(x).strip() and x!="KEIN TREFFER" else "Kein Treffer")
|
||
df.to_excel(file_path, index=False, engine="odf")
|
||
|
||
# =========================
|
||
# Export mit zweitem Sheet für Begriffe ohne Treffer und Vorschlag
|
||
# =========================
|
||
def export_results_with_no_hits(out_df, output_file):
|
||
"""
|
||
Exportiert das Mapping-Ergebnis und zusätzlich ein zweites Sheet
|
||
mit allen Begriffen, deren Norm_Treffer == 'KEIN TREFFER' und Norm_Vorschlag leer ist.
|
||
"""
|
||
# Begriffe ohne Treffer und ohne Vorschlag
|
||
no_match_df = out_df[(out_df["Norm_Treffer"]=="KEIN TREFFER") & (out_df["Norm_Vorschlag"].isna() | (out_df["Norm_Vorschlag"].str.strip()==""))].copy()
|
||
|
||
ext = output_file.suffix.lower()
|
||
|
||
if ext in [".xlsx", ".xls"]:
|
||
with pd.ExcelWriter(output_file, engine="openpyxl") as writer:
|
||
out_df.to_excel(writer, index=False, sheet_name="Mapping")
|
||
no_match_df.to_excel(writer, index=False, sheet_name="Keine Treffer")
|
||
elif ext == ".ods":
|
||
# ODS-Export via odf-Engine
|
||
with pd.ExcelWriter(output_file, engine="odf") as writer:
|
||
out_df.to_excel(writer, index=False, sheet_name="Mapping")
|
||
no_match_df.to_excel(writer, index=False, sheet_name="Keine Treffer")
|
||
|
||
|
||
# =========================
|
||
# Verarbeitung Input-Dateien
|
||
# =========================
|
||
def process_files():
|
||
norm_dict, stem_index, lemma_norm_map = load_normvokabular(NORMVOC_FILE)
|
||
total_terms = 0
|
||
total_hits = 0
|
||
|
||
if not INPUT_DIR.exists():
|
||
print(f"Eingabeordner {INPUT_DIR} fehlt")
|
||
sys.exit(1)
|
||
files = list(INPUT_DIR.glob("*"))
|
||
if not files:
|
||
print("Keine Dateien gefunden")
|
||
return
|
||
|
||
for file_path in files:
|
||
if not file_path.suffix.lower() in [".csv",".ods",".xls",".xlsx"]:
|
||
continue
|
||
print(f"Verarbeite Datei: {file_path.name}")
|
||
try:
|
||
if file_path.suffix.lower() == ".csv":
|
||
df = pd.read_csv(file_path)
|
||
else:
|
||
df = pd.read_excel(file_path, engine="odf" if file_path.suffix.lower()==".ods" else None)
|
||
except Exception as e:
|
||
print(f"Fehler beim Lesen von {file_path.name}: {e}")
|
||
continue
|
||
|
||
df = df.dropna(how="all")
|
||
df.columns = [str(c).strip() for c in df.columns]
|
||
|
||
besch_col = next((c for c in df.columns if "Objektbeschreibung" in c), None)
|
||
box_col = next((c for c in df.columns if "Objekt/Ebene" in c), None)
|
||
urh_col = next((c for c in df.columns if "Urheber" in c), None)
|
||
if not besch_col: continue
|
||
|
||
row_terms_map = []
|
||
for _, row in df.iterrows():
|
||
besch = str(row[besch_col]).strip() if pd.notna(row[besch_col]) else ""
|
||
if not besch: continue
|
||
obj_box = row[box_col] if box_col else ""
|
||
urheber = row[urh_col] if urh_col else ""
|
||
clauses = [c.strip() for c in re.split(r",", besch) if c.strip()]
|
||
terms = []
|
||
for clause in clauses:
|
||
parts = [p.strip() for p in re.split(r"\s+", clause) if p.strip()]
|
||
for p in parts:
|
||
if p.lower() in STOPWORDS: continue
|
||
if re.fullmatch(r"\d+", p): continue
|
||
terms.append(p)
|
||
row_terms_map.append((obj_box, urheber, terms))
|
||
|
||
all_terms = list({t for _,_,terms in row_terms_map for t in terms})
|
||
gnd_results = batch_query_gnd(all_terms)
|
||
wd_results = batch_query_wikidata(all_terms)
|
||
|
||
output_rows = []
|
||
for obj_box, urheber, terms in row_terms_map:
|
||
for term in terms:
|
||
norm_name, norm_id, suggestions = map_to_norm(term, norm_dict, stem_index, lemma_norm_map)
|
||
total_terms += 1
|
||
if norm_name != "KEIN TREFFER":
|
||
total_hits += 1
|
||
out_row = {
|
||
"Box": obj_box,
|
||
"Objekt/Ebene": obj_box,
|
||
"Urheber": urheber,
|
||
"Begriff": term,
|
||
"Norm_Treffer": norm_name,
|
||
"Norm_ID": norm_id,
|
||
"Norm_Vorschlag": ", ".join(suggestions) if suggestions else "",
|
||
"GND_Top1": gnd_results.get(term,""),
|
||
"WD_Top1": wd_results.get(term,"")
|
||
}
|
||
output_rows.append(out_row)
|
||
|
||
out_df = pd.DataFrame(output_rows)
|
||
output_file = OUTPUT_DIR / f"Auswertung_{file_path.stem}{file_path.suffix}"
|
||
version = 1
|
||
while output_file.exists():
|
||
output_file = OUTPUT_DIR / f"Auswertung_{file_path.stem}_({version}){file_path.suffix}"
|
||
version += 1
|
||
|
||
export_results_with_no_hits(out_df, output_file)
|
||
mark_norm_hits(output_file)
|
||
print(f"Auswertung gespeichert: {output_file}")
|
||
|
||
save_cache()
|
||
print(f"Gesamt: {total_terms} Begriffe, {total_hits} Treffer im Normvokabular")
|
||
|
||
# =========================
|
||
# Main
|
||
# =========================
|
||
if __name__ == "__main__":
|
||
process_files()
|
||
print("Fertig") |