191 lines
6.6 KiB
Python
191 lines
6.6 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
VLG_AAT.py Gruppierung, Auflösung "Objektbeschreibung"
|
|
NOCH OHNE AAT-ABGLEICH
|
|
|
|
- Prüft ezodf in aktueller Umgebung
|
|
- Liest ODS aus "Input CSV/"
|
|
- Extrahiert Begriffe aus "Objektbeschreibung"
|
|
- Lemmatisierung (Spacy) + Stopwortfilter
|
|
- Subtokenisierung komplexer Phrasen
|
|
- Zählt Häufigkeiten
|
|
- Ausgabe ODS / CSV-Fallback in "Auswertung Ergebnisse"
|
|
"""
|
|
|
|
import os
|
|
import sys
|
|
import logging
|
|
from collections import Counter
|
|
import pandas as pd
|
|
import spacy
|
|
|
|
# ---------------------------
|
|
# Logging
|
|
# ---------------------------
|
|
logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s')
|
|
|
|
# ---------------------------
|
|
# ezodf prüfen
|
|
# ---------------------------
|
|
try:
|
|
import ezodf
|
|
EZODF_AVAILABLE = True
|
|
logging.info(f"ezodf erkannt")
|
|
except ImportError:
|
|
EZODF_AVAILABLE = False
|
|
logging.error("ezodf konnte nicht importiert werden!")
|
|
logging.error("Möglicherweise nutzen Sie nicht die Python-Umgebung, in der ezodf installiert ist.")
|
|
logging.error(f"Aktuelle Python-Executable: {sys.executable}")
|
|
logging.error("Bitte prüfen Sie Ihre venv oder installieren Sie ezodf in dieser Umgebung:")
|
|
logging.error(" python -m pip install ezodf")
|
|
sys.exit(1)
|
|
|
|
# ---------------------------
|
|
# Spacy laden
|
|
# ---------------------------
|
|
try:
|
|
nlp = spacy.load("de_core_news_sm")
|
|
logging.info("Spacy-Modell geladen.")
|
|
except Exception as e:
|
|
logging.error(f"Spacy-Modell konnte nicht geladen werden: {e}")
|
|
sys.exit(1)
|
|
|
|
# ---------------------------
|
|
# Konfiguration
|
|
# ---------------------------
|
|
INPUT_FOLDER = "Input CSV"
|
|
OUTPUT_FOLDER = "Auswertung Ergebnisse"
|
|
INPUT_FILENAME = None
|
|
TARGET_COLUMN = "Objektbeschreibung"
|
|
STOPWORDS = {"mit", "auf", "von", "und", "der", "die", "das"} # erweiterbar
|
|
MAPPING = { # Projektinterne Sonderfälle
|
|
"exlibris": "exlibris",
|
|
"wappen": "wappen"
|
|
}
|
|
|
|
# ---------------------------
|
|
# Funktionen
|
|
# ---------------------------
|
|
def find_input_file(folder: str, filename_hint: str = None):
|
|
if not os.path.isdir(folder):
|
|
raise FileNotFoundError(f"Input-Ordner '{folder}' existiert nicht.")
|
|
files = [f for f in os.listdir(folder) if f.lower().endswith(".ods")]
|
|
if filename_hint:
|
|
for f in files:
|
|
if f == filename_hint or filename_hint in f:
|
|
return os.path.join(folder, f)
|
|
if not files:
|
|
raise FileNotFoundError(f"Keine .ods-Dateien in '{folder}' gefunden.")
|
|
return os.path.join(folder, files[0])
|
|
|
|
def read_ods_first_sheet(path: str) -> pd.DataFrame:
|
|
"""Lädt ODS, erkennt automatisch Header-Zeile."""
|
|
try:
|
|
df = pd.read_excel(path, engine="odf", header=None)
|
|
logging.info("ODS mit pandas + odfpy geladen.")
|
|
except Exception as e1:
|
|
logging.warning(f"pandas + odfpy konnte ODS nicht lesen ({e1}).")
|
|
if not EZODF_AVAILABLE:
|
|
raise RuntimeError("ezodf nicht installiert und pandas + odfpy fehlgeschlagen.")
|
|
doc = ezodf.opendoc(path)
|
|
sheet = doc.sheets[0]
|
|
data = []
|
|
for row in sheet.rows():
|
|
values = [c.value if hasattr(c, "value") else "" for c in row]
|
|
data.append(values)
|
|
df = pd.DataFrame(data)
|
|
logging.info("ODS mit ezodf geladen.")
|
|
|
|
# Header-Zeile automatisch finden
|
|
header_row_index = None
|
|
for i, row in df.iterrows():
|
|
row_str = row.fillna("").astype(str).str.lower()
|
|
if any("objektbeschreibung" in str(cell) for cell in row_str):
|
|
header_row_index = i
|
|
break
|
|
if header_row_index is None:
|
|
raise KeyError("Keine Header-Zeile mit 'Objektbeschreibung' gefunden.")
|
|
|
|
df.columns = df.iloc[header_row_index]
|
|
df = df.iloc[header_row_index + 1:].reset_index(drop=True)
|
|
return df
|
|
|
|
def tokenize_and_lemmatize(series: pd.Series) -> list:
|
|
"""Tokenisiert, entfernt Stopwords, wendet Mapping + Spacy-Lemmatisierung an."""
|
|
series = series.fillna("").astype(str).str.strip().str.lower()
|
|
all_terms = []
|
|
for text in series:
|
|
if not text:
|
|
continue
|
|
# Komma-Split
|
|
for part in [p.strip() for p in text.split(",") if p.strip()]:
|
|
# Subtokenisierung via Spacy
|
|
doc = nlp(part)
|
|
for token in doc:
|
|
lemma = token.lemma_.lower()
|
|
if lemma in STOPWORDS:
|
|
continue
|
|
lemma = MAPPING.get(lemma, lemma)
|
|
if lemma:
|
|
all_terms.append(lemma)
|
|
return all_terms
|
|
|
|
def write_output(rows: list, outpath: str):
|
|
if EZODF_AVAILABLE:
|
|
if not rows:
|
|
logging.warning("Keine Daten zum Schreiben.")
|
|
return
|
|
keys = list(rows[0].keys())
|
|
doc = ezodf.newdoc(doctype="ods", filename=outpath)
|
|
sheet = ezodf.Sheet("Auswertung", size=(len(rows)+1, len(keys)))
|
|
doc.sheets += sheet
|
|
for ci, k in enumerate(keys):
|
|
sheet[0, ci].set_value(k)
|
|
for ri, row in enumerate(rows, start=1):
|
|
for ci, k in enumerate(keys):
|
|
sheet[ri, ci].set_value(row.get(k, ""))
|
|
doc.save()
|
|
logging.info(f"ODS geschrieben: {outpath}")
|
|
else:
|
|
csv_path = os.path.splitext(outpath)[0] + ".csv"
|
|
df = pd.DataFrame(rows)
|
|
df.to_csv(csv_path, index=False, sep=";", encoding="utf-8")
|
|
logging.info(f"CSV-Fallback geschrieben: {csv_path}")
|
|
|
|
# ---------------------------
|
|
# Hauptfunktion
|
|
# ---------------------------
|
|
def main(input_folder=INPUT_FOLDER, input_filename=INPUT_FILENAME):
|
|
input_path = find_input_file(input_folder, filename_hint=input_filename)
|
|
input_basename = os.path.splitext(os.path.basename(input_path))[0]
|
|
logging.info(f"Verarbeite Datei: {input_path}")
|
|
|
|
df = read_ods_first_sheet(input_path)
|
|
logging.info(f"Geladene Spalten: {list(df.columns)}")
|
|
|
|
if TARGET_COLUMN.lower() not in [str(c).lower() for c in df.columns]:
|
|
raise KeyError(f"Spalte '{TARGET_COLUMN}' nicht gefunden.")
|
|
|
|
terms = tokenize_and_lemmatize(df[TARGET_COLUMN])
|
|
logging.info(f"Gefundene Begriffe: {len(terms)}")
|
|
|
|
counts = Counter(terms)
|
|
sorted_terms = sorted(counts.items(), key=lambda kv: kv[1], reverse=True)
|
|
rows = [{"Begriff": term, "Anzahl": freq} for term, freq in sorted_terms]
|
|
|
|
os.makedirs(OUTPUT_FOLDER, exist_ok=True)
|
|
out_name = f"{input_basename} Auswertung.ods"
|
|
out_path = os.path.join(OUTPUT_FOLDER, out_name)
|
|
write_output(rows, out_path)
|
|
logging.info("Fertig.")
|
|
|
|
if __name__ == "__main__":
|
|
argv = sys.argv[1:]
|
|
folder = INPUT_FOLDER
|
|
fname = INPUT_FILENAME
|
|
if len(argv) >= 1:
|
|
folder = argv[0]
|
|
if len(argv) >= 2:
|
|
fname = argv[1]
|
|
main(input_folder=folder, input_filename=fname)
|